
#ifdef __AVX2__

// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
#if defined(__GNUC__) && __GNUC__ >= 7
# define KORTEST32(a, b) !_kortestz_mask32_u8((a), (b))
# define KAND32(a, b) _kand_mask32((a), (b))
# define KOR32(a, b) _kor_mask32((a), (b))
#else
# define KORTEST32(a, b) ((a) | (b))
# define KAND32(a, b) ((a) & (b))
# define KOR32(a, b) ((a) | (b))
#endif

#pragma pack(16)
static struct {
    /*align16*/ struct { char bytes[16]; } compact[32768];
    uint8_t eqFix[256];
} * HEDLEY_RESTRICT lookups;
#pragma pack()


static HEDLEY_ALWAYS_INLINE __m256i force_align_read_256(const void* p) {
#ifdef _MSC_VER
    // MSVC complains about casting away volatile
    return *(__m256i *)(p);
#else
    return *(volatile __m256i *)(p);
#endif
}

// _mm256_castsi128_si256, but upper is defined to be 0
#if (defined(__clang__) && __clang_major__ >= 5 && (!defined(__APPLE__) || __clang_major__ >= 7)) || (defined(__GNUC__) && __GNUC__ >= 10)
// intrinsic unsupported in GCC 9 and MSVC < 2017
# define zext128_256 _mm256_zextsi128_si256
#else
// technically a cast is incorrect, due to upper 128 bits being undefined, but should usually work fine
// alternative may be `_mm256_set_m128i(_mm_setzero_si128(), v)` but unsupported on GCC < 7, and most compilers generate a VINSERTF128 instruction for it
# ifdef __OPTIMIZE__
#  define zext128_256 _mm256_castsi128_si256
# else
#  define zext128_256(x) _mm256_inserti128_si256(_mm256_setzero_si256(), x, 0)
# endif
#endif


template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
HEDLEY_ALWAYS_INLINE void do_decode_avx2(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
    HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
    HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
    uintptr_t escFirst = _escFirst;
    __m256i yencOffset = escFirst ? _mm256_set_epi8(
        -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,
        -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64
    ) : _mm256_set1_epi8(-42);
    __m256i minMask = _mm256_set1_epi8('.');
    if(_nextMask && isRaw) {
        minMask = _mm256_set_epi8(
            '.','.','.','.','.','.','.','.','.','.','.','.','.','.','.','.',
            '.','.','.','.','.','.','.','.','.','.','.','.','.','.',_nextMask==2?0:'.',_nextMask==1?0:'.'
        );
    }
    intptr_t i;
    for(i = -len; i; i += sizeof(__m256i)*2) {
        __m256i oDataA = _mm256_load_si256((__m256i *)(src+i));
        __m256i oDataB = _mm256_load_si256((__m256i *)(src+i) + 1);

        // search for special chars
        __m256i cmpA = _mm256_cmpeq_epi8(oDataA, _mm256_shuffle_epi8(
            _mm256_set_epi8(
                -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.',
                -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'
            ),
            _mm256_min_epu8(oDataA, minMask)
        ));
        __m256i cmpB = _mm256_cmpeq_epi8(oDataB, _mm256_shuffle_epi8(
            _mm256_set_epi8(
                -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.',
                -1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'
            ),
            _mm256_min_epu8(oDataB, _mm256_set1_epi8('.'))
        ));

        // TODO: can OR the vectors together to save generating a mask, but may not be worth it
        uint64_t mask = (uint32_t)_mm256_movemask_epi8(cmpB); // not the most accurate mask if we have invalid sequences; we fix this up later
        mask = (mask << 32) | (uint32_t)_mm256_movemask_epi8(cmpA);
        __m256i dataA, dataB;
        if(use_isa >= ISA_LEVEL_AVX3)
            dataA = _mm256_add_epi8(oDataA, yencOffset);

        if (mask != 0) {
            __m256i cmpEqA = _mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('='));
            __m256i cmpEqB = _mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('='));
            uint64_t maskEq = (uint32_t)_mm256_movemask_epi8(cmpEqB);
            maskEq = (maskEq << 32) | (uint32_t)_mm256_movemask_epi8(cmpEqA);

            // handle \r\n. sequences
            // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing
            if((isRaw || searchEnd) && LIKELIHOOD(0.45, mask != maskEq)) {
#if 0
                // prefer shuffling data over unaligned loads on Zen (unknown if worth it on Zen2/Excavator)
                // unfortunately not beneficial, probably due to available register pressure; this is left here because it could be beneficial if we figure out how to use fewer registers
                __m256i nextDataA, nextDataB;
                if(searchEnd) {
                    nextDataA = _mm256_inserti128_si256(
                        _mm256_castsi128_si256(_mm256_extracti128_si256(oDataA, 1)),
                        _mm256_castsi256_si128(oDataB),
                        1
                    );
                    nextDataB = _mm256_inserti128_si256(
                        _mm256_castsi128_si256(_mm256_extracti128_si256(oDataB, 1)),
                        _mm_load_si128((__m128i*)(src+i+sizeof(__m256i)*2)),
                        1
                    );
                }
# define SHIFT_DATA_A(offs) (searchEnd ? _mm256_alignr_epi8(nextDataA, oDataA, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs)))
# define SHIFT_DATA_B(offs) (searchEnd ? _mm256_alignr_epi8(nextDataB, oDataB, offs) : _mm256_loadu_si256((__m256i *)(src+i+offs) + 1))
#else
# define SHIFT_DATA_A(offs) _mm256_loadu_si256((__m256i *)(src+i+offs))
# define SHIFT_DATA_B(offs) _mm256_loadu_si256((__m256i *)(src+i+offs) + 1)
#endif
                __m256i tmpData2A = SHIFT_DATA_A(2);
                __m256i tmpData2B = SHIFT_DATA_B(2);
                __m256i match2EqA, match2EqB;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                __mmask32 match2EqMaskA, match2EqMaskB;
                __mmask32 match0CrMaskA, match0CrMaskB;
                __mmask32 match2CrXDtMaskA, match2CrXDtMaskB;
                if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
                    match2EqMaskA = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2A);
                    match2EqMaskB = _mm256_cmpeq_epi8_mask(_mm256_set1_epi8('='), tmpData2B);
                } else
#endif
                if(searchEnd) {
                    match2EqA = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2A);
                    match2EqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), tmpData2B);
                }

                int partialKillDotFound;
                __m256i match2CrXDtA, match2CrXDtB;
                if(isRaw) {
                    // find patterns of \r_.

#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match0CrMaskA = _mm256_cmpeq_epi8_mask(oDataA, _mm256_set1_epi8('\r'));
                        match0CrMaskB = _mm256_cmpeq_epi8_mask(oDataB, _mm256_set1_epi8('\r'));
                        match2CrXDtMaskA = _mm256_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm256_set1_epi8('.'));
                        match2CrXDtMaskB = _mm256_mask_cmpeq_epi8_mask(match0CrMaskB, tmpData2B, _mm256_set1_epi8('.'));
                        partialKillDotFound = KORTEST32(match2CrXDtMaskA, match2CrXDtMaskB);
                    } else
#endif
                    {
                        match2CrXDtA = _mm256_and_si256(
                            _mm256_cmpeq_epi8(oDataA, _mm256_set1_epi8('\r')),
                            _mm256_cmpeq_epi8(tmpData2A, _mm256_set1_epi8('.'))
                        );
                        match2CrXDtB = _mm256_and_si256(
                            _mm256_cmpeq_epi8(oDataB, _mm256_set1_epi8('\r')),
                            _mm256_cmpeq_epi8(tmpData2B, _mm256_set1_epi8('.'))
                        );
                        partialKillDotFound = _mm256_movemask_epi8(_mm256_or_si256(
                            match2CrXDtA, match2CrXDtB
                        ));
                    }
                }

                if(isRaw && LIKELIHOOD(0.002, partialKillDotFound)) {
                    // merge matches for \r\n.
                    __m256i match2NlDotA, match1NlA;
                    __m256i match2NlDotB, match1NlB;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    __mmask32 match1NlMaskA, match1NlMaskB;
                    __mmask32 match2NlDotMaskA, match2NlDotMaskB;
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match1NlMaskA = _mm256_mask_cmpeq_epi8_mask(
                            match0CrMaskA,
                            _mm256_set1_epi8('\n'),
                            SHIFT_DATA_A(1)
                        );
                        match1NlMaskB = _mm256_mask_cmpeq_epi8_mask(
                            match0CrMaskB,
                            _mm256_set1_epi8('\n'),
                            SHIFT_DATA_B(1)
                        );
                        match2NlDotMaskA = KAND32(match2CrXDtMaskA, match1NlMaskA);
                        match2NlDotMaskB = KAND32(match2CrXDtMaskB, match1NlMaskB);
                    } else
#endif
                    {
                        __m256i match1LfA = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('\n'),
                            SHIFT_DATA_A(1)
                        );
                        __m256i match1LfB = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('\n'),
                            SHIFT_DATA_B(1)
                        );
                        // force re-computing these to avoid register spills elsewhere
                        match1NlA = _mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r')));
                        match1NlB = _mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r')));
                        match2NlDotA = _mm256_and_si256(match2CrXDtA, match1NlA);
                        match2NlDotB = _mm256_and_si256(match2CrXDtB, match1NlB);
                    }
                    if(searchEnd) {
                        __m256i tmpData4A;
#if defined(__AVX512VL__) && defined(PLATFORM_AMD64)
                        if(use_isa >= ISA_LEVEL_AVX3)
                            // AVX512 with 32 registers shouldn't have any issue with holding onto oData* in registers
                            tmpData4A = _mm256_alignr_epi32(oDataB, oDataA, 1);
                        else
#endif
                            tmpData4A = SHIFT_DATA_A(4);
                        __m256i tmpData4B = SHIFT_DATA_B(4);
                        // match instances of \r\n.\r\n and \r\n.=y
                        __m256i match3CrA = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('\r'),
                            SHIFT_DATA_A(3)
                        );
                        __m256i match3CrB = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('\r'),
                            SHIFT_DATA_B(3)
                        );
                        __m256i match4LfA = _mm256_cmpeq_epi8(tmpData4A, _mm256_set1_epi8('\n'));
                        __m256i match4LfB = _mm256_cmpeq_epi8(tmpData4B, _mm256_set1_epi8('\n'));
                        __m256i match4EqYA = _mm256_cmpeq_epi16(tmpData4A, _mm256_set1_epi16(0x793d)); // =y
                        __m256i match4EqYB = _mm256_cmpeq_epi16(tmpData4B, _mm256_set1_epi16(0x793d)); // =y

                        int matchEnd;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                        if(use_isa >= ISA_LEVEL_AVX3) {
                            __mmask32 match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
                                match2EqMaskA,
                                _mm256_set1_epi8('y'),
                                SHIFT_DATA_A(3)
                            );
                            __mmask32 match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask(
                                match2EqMaskB,
                                _mm256_set1_epi8('y'),
                                SHIFT_DATA_B(3)
                            );
                            __m256i match34EqYA, match34EqYB;
# ifdef __AVX512VBMI2__
                            if(use_isa >= ISA_LEVEL_VBMI2) {
                                match34EqYA = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskA), match4EqYA, 8);
                                match34EqYB = _mm256_shrdi_epi16(_mm256_movm_epi8(match3EqYMaskB), match4EqYB, 8);
                            } else
# endif
                            {
                                // (match4EqY & 0xff00) | (match3EqY >> 8)
                                match34EqYA = _mm256_mask_blend_epi8(match3EqYMaskA>>1, _mm256_and_si256(match4EqYA, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1));
                                match34EqYB = _mm256_mask_blend_epi8(match3EqYMaskB>>1, _mm256_and_si256(match4EqYB, _mm256_set1_epi16(-0xff)), _mm256_set1_epi8(-1));
                            }
                            // merge \r\n and =y matches for tmpData4
                            __m256i match4EndA = _mm256_ternarylogic_epi32(match34EqYA, match3CrA, match4LfA, 0xF8); // (match3Cr & match4Lf) | match34EqY
                            __m256i match4EndB = _mm256_ternarylogic_epi32(match34EqYB, match3CrB, match4LfB, 0xF8);
                            // merge with \r\n. and combine
                            matchEnd = KORTEST32(
                                KOR32(
                                    _mm256_mask_test_epi8_mask(match2NlDotMaskA, match4EndA, match4EndA),
                                    KAND32(match3EqYMaskA, match1NlMaskA)
                                ),
                                KOR32(
                                    _mm256_mask_test_epi8_mask(match2NlDotMaskB, match4EndB, match4EndB),
                                    KAND32(match3EqYMaskB, match1NlMaskB)
                                )
                            );
                        } else
#endif
                        {
                            __m256i match3EqYA = _mm256_and_si256(match2EqA, _mm256_cmpeq_epi8(
                                _mm256_set1_epi8('y'),
                                SHIFT_DATA_A(3)
                            ));
                            __m256i match3EqYB = _mm256_and_si256(match2EqB, _mm256_cmpeq_epi8(
                                _mm256_set1_epi8('y'),
                                SHIFT_DATA_B(3)
                            ));
                            match4EqYA = _mm256_slli_epi16(match4EqYA, 8); // TODO: also consider using PBLENDVB here with shifted match3EqY instead
                            match4EqYB = _mm256_slli_epi16(match4EqYB, 8);
                            // merge \r\n and =y matches for tmpData4
                            __m256i match4EndA = _mm256_or_si256(
                                _mm256_and_si256(match3CrA, match4LfA),
                                _mm256_or_si256(match4EqYA, _mm256_srli_epi16(match3EqYA, 8)) // _mm256_srli_si256 by 1 also works
                            );
                            __m256i match4EndB = _mm256_or_si256(
                                _mm256_and_si256(match3CrB, match4LfB),
                                _mm256_or_si256(match4EqYB, _mm256_srli_epi16(match3EqYB, 8))
                            );
                            // merge with \r\n.
                            match4EndA = _mm256_and_si256(match4EndA, match2NlDotA);
                            match4EndB = _mm256_and_si256(match4EndB, match2NlDotB);
                            // match \r\n=y
                            __m256i match3EndA = _mm256_and_si256(match3EqYA, match1NlA);
                            __m256i match3EndB = _mm256_and_si256(match3EqYB, match1NlB);
                            // combine match sequences
                            matchEnd = _mm256_movemask_epi8(_mm256_or_si256(
                                _mm256_or_si256(match4EndA, match3EndA),
                                _mm256_or_si256(match4EndB, match3EndB)
                            ));
                        }
                        if(LIKELIHOOD(0.002, matchEnd)) {
                            // terminator found
                            // there's probably faster ways to do this, but reverting to scalar code should be good enough
                            len += (long)i;
                            break;
                        }
                    }
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        mask |= (uint64_t)match2NlDotMaskA << 2;
                        mask |= (uint64_t)match2NlDotMaskB << 34;
                        minMask = _mm256_maskz_mov_epi8(~(match2NlDotMaskB>>30), _mm256_set1_epi8('.'));
                    } else
#endif
                    {
                        mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotA)) << 2;
                        mask |= (uint64_t)((uint32_t)_mm256_movemask_epi8(match2NlDotB)) << 34;
                        match2NlDotB = zext128_256(_mm_srli_si128(_mm256_extracti128_si256(match2NlDotB, 1), 14));
                        minMask = _mm256_subs_epu8(_mm256_set1_epi8('.'), match2NlDotB);
                    }
                }
                else if(searchEnd) {
                    bool partialEndFound;
                    __m256i match3EqYA, match3EqYB;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    __mmask32 match3EqYMaskA, match3EqYMaskB;
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match3EqYMaskA = _mm256_mask_cmpeq_epi8_mask(
                            match2EqMaskA,
                            _mm256_set1_epi8('y'),
                            SHIFT_DATA_A(3)
                        );
                        match3EqYMaskB = _mm256_mask_cmpeq_epi8_mask(
                            match2EqMaskB,
                            _mm256_set1_epi8('y'),
                            SHIFT_DATA_B(3)
                        );
                        partialEndFound = KORTEST32(match3EqYMaskA, match3EqYMaskB);
                    } else
#endif
                    {
                        __m256i match3YA = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('y'),
                            SHIFT_DATA_A(3)
                        );
                        __m256i match3YB = _mm256_cmpeq_epi8(
                            _mm256_set1_epi8('y'),
                            SHIFT_DATA_B(3)
                        );
                        match3EqYA = _mm256_and_si256(match2EqA, match3YA);
                        match3EqYB = _mm256_and_si256(match2EqB, match3YB);
                        partialEndFound = _mm256_movemask_epi8(_mm256_or_si256(match3EqYA, match3EqYB));
                    }
                    if(LIKELIHOOD(0.002, partialEndFound)) {
                        bool endFound;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                        if(use_isa >= ISA_LEVEL_AVX3) {
                            __mmask32 match3LfEqYMaskA = _mm256_mask_cmpeq_epi8_mask(
                                match3EqYMaskA,
                                _mm256_set1_epi8('\n'),
                                SHIFT_DATA_A(1)
                            );
                            __mmask32 match3LfEqYMaskB = _mm256_mask_cmpeq_epi8_mask(
                                match3EqYMaskB,
                                _mm256_set1_epi8('\n'),
                                SHIFT_DATA_B(1)
                            );

                            endFound = KORTEST32(
                                _mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskA, oDataA, _mm256_set1_epi8('\r')),
                                _mm256_mask_cmpeq_epi8_mask(match3LfEqYMaskB, oDataB, _mm256_set1_epi8('\r'))
                            );
                        } else
#endif
                        {
                            __m256i match1LfA = _mm256_cmpeq_epi8(
                                _mm256_set1_epi8('\n'),
                                SHIFT_DATA_A(1)
                            );
                            __m256i match1LfB = _mm256_cmpeq_epi8(
                                _mm256_set1_epi8('\n'),
                                SHIFT_DATA_B(1)
                            );
                            endFound = _mm256_movemask_epi8(_mm256_or_si256(
                                _mm256_and_si256(
                                    match3EqYA,
                                    _mm256_and_si256(match1LfA, _mm256_cmpeq_epi8(force_align_read_256(src+i), _mm256_set1_epi8('\r')))
                                ),
                                _mm256_and_si256(
                                    match3EqYB,
                                    _mm256_and_si256(match1LfB, _mm256_cmpeq_epi8(force_align_read_256(src+i + sizeof(__m256i)), _mm256_set1_epi8('\r')))
                                )
                            ));
                        }
                        if(endFound) {
                            len += (long)i;
                            break;
                        }
                    }
                    if(isRaw) minMask = _mm256_set1_epi8('.');
                }
                else if(isRaw) // no \r_. found
                    minMask = _mm256_set1_epi8('.');
            }
#undef SHIFT_DATA_A
#undef SHIFT_DATA_B

            if(use_isa >= ISA_LEVEL_AVX3)
                dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));

            if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) + escFirst)) != 0)) {
                unsigned tmp = lookups->eqFix[(maskEq&0xff) & ~(uint64_t)escFirst];
                uint64_t maskEq2 = tmp;
                for(int j=8; j<64; j+=8) {
                    tmp = lookups->eqFix[(unsigned)((maskEq>>j)&0xff) & ~(tmp>>7)];
                    maskEq2 |= (uint64_t)tmp<<j;
                }
                maskEq = maskEq2;

                mask &= ~(uint64_t)escFirst;
                escFirst = tmp>>7;
                // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
                maskEq <<= 1;
                mask &= ~maskEq;

                // unescape chars following `=`
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                if(use_isa >= ISA_LEVEL_AVX3) {
                    // GCC < 7 seems to generate rubbish assembly for this
                    dataA = _mm256_mask_add_epi8(
                        dataA,
                        (__mmask32)maskEq,
                        dataA,
                        _mm256_set1_epi8(-64)
                    );
                    dataB = _mm256_mask_add_epi8(
                        dataB,
                        (__mmask32)(maskEq>>32),
                        dataB,
                        _mm256_set1_epi8(-64)
                    );
                } else
#endif
                {
                    // convert maskEq into vector form (i.e. reverse pmovmskb)
#ifdef PLATFORM_AMD64
                    __m256i vMaskEq = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(maskEq));
#else
                    __m256i vMaskEq = _mm256_permute4x64_epi64(_mm256_insert_epi32(
                        _mm256_set_epi32(0,0,0,0, 0,0,0, maskEq & 0xffffffff),
                        maskEq >> 32,
                        1
                    ), 0);
#endif
                    __m256i vMaskEqA = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32(
                        0x03030303, 0x03030303, 0x02020202, 0x02020202,
                        0x01010101, 0x01010101, 0x00000000, 0x00000000
                    ));
                    __m256i vMaskEqB = _mm256_shuffle_epi8(vMaskEq, _mm256_set_epi32(
                        0x07070707, 0x07070707, 0x06060606, 0x06060606,
                        0x05050505, 0x05050505, 0x04040404, 0x04040404
                    ));
                    vMaskEqA = _mm256_cmpeq_epi8(
                        _mm256_and_si256(vMaskEqA, _mm256_set1_epi64x(0x8040201008040201ULL)),
                        _mm256_set1_epi64x(0x8040201008040201ULL)
                    );
                    vMaskEqB = _mm256_cmpeq_epi8(
                        _mm256_and_si256(vMaskEqB, _mm256_set1_epi64x(0x8040201008040201ULL)),
                        _mm256_set1_epi64x(0x8040201008040201ULL)
                    );
                    dataA = _mm256_add_epi8(oDataA, _mm256_blendv_epi8(yencOffset, _mm256_set1_epi8(-42-64), vMaskEqA));
                    dataB = _mm256_add_epi8(oDataB, _mm256_blendv_epi8(_mm256_set1_epi8(-42), _mm256_set1_epi8(-42-64), vMaskEqB));
                }
            } else {
                escFirst = (maskEq >> 63);

#if defined(__AVX512VL__) && defined(__AVX512BW__)
                if(use_isa >= ISA_LEVEL_AVX3) {
                    dataA = _mm256_mask_add_epi8(
                        dataA,
                        (__mmask32)(maskEq << 1),
                        dataA,
                        _mm256_set1_epi8(-64)
                    );
                    dataB = _mm256_mask_add_epi8(
                        dataB,
                        (__mmask32)(maskEq >> 31),
                        dataB,
                        _mm256_set1_epi8(-64)
                    );
                } else
#endif
                {
                    // << 1 byte
                    cmpEqA = _mm256_alignr_epi8(cmpEqA, _mm256_inserti128_si256(
                        _mm256_set1_epi8('='), _mm256_castsi256_si128(cmpEqA), 1
                    ), 15);
                    cmpEqB = _mm256_cmpeq_epi8(_mm256_set1_epi8('='), _mm256_loadu_si256((__m256i *)(src+i-1) + 1));
                    dataA = _mm256_add_epi8(
                        oDataA,
                        _mm256_blendv_epi8(
                            yencOffset,
                            _mm256_set1_epi8(-42-64),
                            cmpEqA
                        )
                    );
                    dataB = _mm256_add_epi8(
                        oDataB,
                        _mm256_blendv_epi8(
                            _mm256_set1_epi8(-42),
                            _mm256_set1_epi8(-42-64),
                            cmpEqB
                        )
                    );
                }
            }
            // subtract 64 from first element if escFirst == 1
#if defined(__AVX512VL__) && defined(__AVX512BW__)
            if(use_isa >= ISA_LEVEL_AVX3) {
                yencOffset = _mm256_mask_add_epi8(_mm256_set1_epi8(-42), (__mmask32)escFirst, _mm256_set1_epi8(-42), _mm256_set1_epi8(-64));
            } else
#endif
            {
                yencOffset = _mm256_xor_si256(_mm256_set1_epi8(-42), zext128_256(
                    _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6)
                ));
            }

            // all that's left is to 'compress' the data (skip over masked chars)
#if defined(__AVX512VBMI2__) && defined(__AVX512VL__)
            if(use_isa >= ISA_LEVEL_VBMI2) {
                _mm256_mask_compressstoreu_epi8(p, KNOT32(mask), dataA);
                p -= popcnt32(mask & 0xffffffff);
                _mm256_mask_compressstoreu_epi8((p + XMM_SIZE*2), KNOT32(mask>>32), dataB);
                p += XMM_SIZE*4 - popcnt32(mask >> 32);
            } else
#endif
            {
                // lookup compress masks and shuffle
                __m256i shuf = _mm256_inserti128_si256(
                    _mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))),
                    *(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)),
                    1
                );
                dataA = _mm256_shuffle_epi8(dataA, shuf);

                _mm_storeu_si128((__m128i*)p, _mm256_castsi256_si128(dataA));
                // increment output position
                p -= popcnt32(mask & 0xffff);

                _mm_storeu_si128((__m128i*)(p + XMM_SIZE), _mm256_extracti128_si256(dataA, 1));
                p -= popcnt32(mask & 0xffff0000);

#ifdef PLATFORM_AMD64
                mask >>= 28;
                shuf = _mm256_inserti128_si256(
                    _mm256_castsi128_si256(_mm_load_si128((__m128i*)((char*)lookups->compact + (mask & 0x7fff0)))),
                    *(__m128i*)((char*)lookups->compact + ((mask >> 16) & 0x7fff0)),
                    1
                );
                dataB = _mm256_shuffle_epi8(dataB, shuf);

                _mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB));
                p -= popcnt32(mask & 0xffff0);

                _mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1));
                p -= popcnt32((unsigned int)(mask >> 20));
#else
                mask >>= 32;
                shuf = _mm256_inserti128_si256(
                    _mm256_castsi128_si256(_mm_load_si128((__m128i*)(lookups->compact + (mask & 0x7fff)))),
                    *(__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0)),
                    1
                );
                dataB = _mm256_shuffle_epi8(dataB, shuf);

                _mm_storeu_si128((__m128i*)(p + XMM_SIZE*2), _mm256_castsi256_si128(dataB));
                p -= popcnt32(mask & 0xffff);

                _mm_storeu_si128((__m128i*)(p + XMM_SIZE*3), _mm256_extracti128_si256(dataB, 1));
                p -= popcnt32(mask & 0xffff0000);
#endif
                p += XMM_SIZE*4;
            }
        } else {
            if(use_isa < ISA_LEVEL_AVX3)
                dataA = _mm256_add_epi8(oDataA, yencOffset);
            dataB = _mm256_add_epi8(oDataB, _mm256_set1_epi8(-42));

            _mm256_storeu_si256((__m256i*)p, dataA);
            _mm256_storeu_si256((__m256i*)p + 1, dataB);
            p += sizeof(__m256i)*2;
            escFirst = 0;
            yencOffset = _mm256_set1_epi8(-42);
        }
    }
    _escFirst = (unsigned char)escFirst;
    if(isRaw) {
        // this would be the trivial solution, but requires the compiler holding onto minMask throughout the loop:
        //_nextMask = ~(uint16_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(minMask, _mm256_set1_epi8('.')));
        // instead, just scan the memory to determine what to set nextMask to
        if(len != 0) { // have to gone through at least one loop cycle
            if(src[i-2] == '\r' && src[i-1] == '\n' && src[i] == '.')
                _nextMask = 1;
            else if(src[i-1] == '\r' && src[i] == '\n' && src[i+1] == '.')
                _nextMask = 2;
            else
                _nextMask = 0;
        }
    } else
        _nextMask = 0;
    _mm256_zeroupper();
}
#endif
