
#ifdef __SSE2__

#if defined(__clang__) && __clang_major__ == 6 && __clang_minor__ == 0
// VBMI2 introduced in clang 6.0, but 128-bit functions misnamed there; fixed in clang 7.0, but we'll handle those on 6.0
# define _mm_mask_compressstoreu_epi8 _mm128_mask_compressstoreu_epi8
# define _mm_shrdi_epi16 _mm128_shrdi_epi16
#endif

// GCC (ver 6-10(dev)) fails to optimize pure C version of mask testing, but has this intrinsic; Clang >= 7 optimizes C version fine
#if defined(__GNUC__) && __GNUC__ >= 7
# define KORTEST16(a, b) !_kortestz_mask16_u8((a), (b))
# define KAND16(a, b) _kand_mask16((a), (b))
# define KOR16(a, b) _kor_mask16((a), (b))
#else
# define KORTEST16(a, b) ((a) | (b))
# define KAND16(a, b) ((a) & (b))
# define KOR16(a, b) ((a) | (b))
#endif

#pragma pack(16)
static struct {
    unsigned char BitsSetTable256inv[256];
    /*align16*/ struct { char bytes[16]; } compact[32768];
    uint8_t eqFix[256];
    /*align8*/ uint64_t eqAdd[256];
    /*align16*/ int8_t unshufMask[32*16];
} * HEDLEY_RESTRICT lookups;
#pragma pack()


static HEDLEY_ALWAYS_INLINE __m128i force_align_read_128(const void* p) {
#ifdef _MSC_VER
    // MSVC complains about casting away volatile
    return *(__m128i *)(p);
#else
    return *(volatile __m128i *)(p);
#endif
}


static void decoder_sse_init() {
    ALIGN_ALLOC(lookups, sizeof(*lookups), 16);
    for(int i=0; i<256; i++) {
        lookups->BitsSetTable256inv[i] = 8 - (
            (i & 1) + ((i>>1) & 1) + ((i>>2) & 1) + ((i>>3) & 1) + ((i>>4) & 1) + ((i>>5) & 1) + ((i>>6) & 1) + ((i>>7) & 1)
        );

#define _X(n, k) ((((n) & (1<<k)) ? 192ULL : 0ULL) << (k*8))
        lookups->eqAdd[i] = _X(i, 0) | _X(i, 1) | _X(i, 2) | _X(i, 3) | _X(i, 4) | _X(i, 5) | _X(i, 6) | _X(i, 7);
#undef _X
    }
    for(int i=0; i<32; i++) {
        for(int j=0; j<16; j++) {
            if(i >= 16) // only used for LZCNT
                lookups->unshufMask[i*16 + j] = ((31-i)>j ? -1 : 0);
            else // only used for BSR
                lookups->unshufMask[i*16 + j] = (i>j ? -1 : 0);
        }
    }
}


// for LZCNT/BSR
#ifdef _MSC_VER
# include <intrin.h>
# include <ammintrin.h>
static HEDLEY_ALWAYS_INLINE unsigned BSR32(unsigned src) {
    unsigned long result;
    _BitScanReverse((unsigned long*)&result, src);
    return result;
}
#elif defined(__GNUC__)
// have seen Clang not like _bit_scan_reverse
# include <x86intrin.h> // for lzcnt
# define BSR32(src) (31^__builtin_clz(src))
#else
# include <x86intrin.h>
# define BSR32 _bit_scan_reverse
#endif

template<enum YEncDecIsaLevel use_isa>
static HEDLEY_ALWAYS_INLINE __m128i sse2_compact_vect(uint32_t mask, __m128i data) {
    while(mask) {
        unsigned bitIndex;
#if defined(__LZCNT__)
        if(use_isa & ISA_FEATURE_LZCNT) {
            // lzcnt is always at least as fast as bsr, so prefer it if it's available
            bitIndex = _lzcnt_u32(mask);
            mask &= 0x7fffffffU>>bitIndex;
        } else
#endif
        {
            bitIndex = BSR32(mask);
            mask ^= 1<<bitIndex;
        }
        __m128i mergeMask = _mm_load_si128((__m128i*)lookups->unshufMask + bitIndex);
        data = _mm_or_si128(
            _mm_and_si128(mergeMask, data),
            _mm_andnot_si128(mergeMask, _mm_srli_si128(data, 1))
        );
    }
    return data;
}

template<bool isRaw, bool searchEnd, enum YEncDecIsaLevel use_isa>
HEDLEY_ALWAYS_INLINE void do_decode_sse(const uint8_t* HEDLEY_RESTRICT src, long& len, unsigned char* HEDLEY_RESTRICT & p, unsigned char& _escFirst, uint16_t& _nextMask) {
    HEDLEY_ASSUME(_escFirst == 0 || _escFirst == 1);
    HEDLEY_ASSUME(_nextMask == 0 || _nextMask == 1 || _nextMask == 2);
    uintptr_t escFirst = _escFirst;
    __m128i yencOffset = escFirst ? _mm_set_epi8(
        -42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42,-42-64
    ) : _mm_set1_epi8(-42);

#if defined(__SSSE3__) && !defined(__tune_atom__) && !defined(__tune_slm__) && !defined(__tune_btver1__)
    const bool _USING_FAST_MATCH = (use_isa >= ISA_LEVEL_SSSE3);
#else
    const bool _USING_FAST_MATCH = false;
#endif
#if defined(__SSE4_1__) && !defined(__tune_slm__) && !defined(__tune_goldmont__) && !defined(__tune_goldmont_plus__) && !defined(__tune_tremont__)
    const bool _USING_BLEND_ADD = (use_isa >= ISA_LEVEL_SSE41);
#else
    const bool _USING_BLEND_ADD = false;
#endif

    __m128i lfCompare = _mm_set1_epi8('\n');
    __m128i minMask = _mm_set1_epi8('.');
    if(_nextMask && isRaw) {
        if(_USING_FAST_MATCH)
            minMask = _mm_insert_epi16(minMask, _nextMask == 1 ? 0x2e00 : 0x002e, 0);
        else
            lfCompare = _mm_insert_epi16(lfCompare, _nextMask == 1 ? 0x0a2e /*".\n"*/ : 0x2e0a /*"\n."*/, 0);
    }
    intptr_t i;
    for(i = -len; i; i += sizeof(__m128i)*2) {
        __m128i oDataA = _mm_load_si128((__m128i *)(src+i));
        __m128i oDataB = _mm_load_si128((__m128i *)(src+i) + 1);

        // search for special chars
        __m128i cmpEqA, cmpEqB, cmpCrA, cmpCrB;
        __m128i cmpA, cmpB;
#if defined(__SSSE3__)
        if(_USING_FAST_MATCH) {
            cmpA = _mm_cmpeq_epi8(oDataA, _mm_shuffle_epi8(
                _mm_set_epi8(-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'),
                _mm_min_epu8(oDataA, minMask)
            ));
            cmpB = _mm_cmpeq_epi8(oDataB, _mm_shuffle_epi8(
                _mm_set_epi8(-1,'=','\r',-1,-1,'\n',-1,-1,-1,-1,-1,-1,-1,-1,-1,'.'),
                _mm_min_epu8(oDataB, _mm_set1_epi8('.'))
            ));
        } else
#endif
        {
            cmpEqA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('='));
            cmpEqB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('='));
            cmpCrA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('\r'));
            cmpCrB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\r'));
            cmpA = _mm_or_si128(
                _mm_or_si128(
                    _mm_cmpeq_epi8(oDataA, lfCompare), cmpCrA
                ),
                cmpEqA
            );
            cmpB = _mm_or_si128(
                _mm_or_si128(
                    _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\n')), cmpCrB
                ),
                cmpEqB
            );
        }

        __m128i dataA, dataB;
        if(!_USING_BLEND_ADD)
            dataA = _mm_add_epi8(oDataA, yencOffset);
        uint32_t mask = (unsigned)_mm_movemask_epi8(cmpA) | ((unsigned)_mm_movemask_epi8(cmpB) << 16); // not the most accurate mask if we have invalid sequences; we fix this up later

        if (LIKELIHOOD(0.42 /* rough guess */, mask != 0)) {
            if(_USING_FAST_MATCH) {
                cmpEqA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('='));
                cmpEqB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('='));
            }

#define LOAD_HALVES(a, b) _mm_castps_si128(_mm_loadh_pi( \
    _mm_castsi128_ps(_mm_loadl_epi64((__m128i*)(a))), \
    (__m64*)(b) \
))

            // a spec compliant encoder should never generate sequences: ==, =\n and =\r, but we'll handle them to be spec compliant
            // the yEnc specification requires any character following = to be unescaped, not skipped over, so we'll deal with that
            // firstly, check for invalid sequences of = (we assume that these are rare, as a spec compliant yEnc encoder should not generate these)
            uint32_t maskEq = (unsigned)_mm_movemask_epi8(cmpEqA) | ((unsigned)_mm_movemask_epi8(cmpEqB) << 16);

            // handle \r\n. sequences
            // RFC3977 requires the first dot on a line to be stripped, due to dot-stuffing
            if((isRaw || searchEnd) && LIKELIHOOD(0.25, mask != maskEq)) {
#if 0
                // for experimentation: prefer shifting data over unaligned loads on CPUs with slow unaligned handling
                // haven't ever seen this be beneficial though
                __m128i nextDataB;
                if(searchEnd && _USING_BLEND_ADD)
                    nextDataB = _mm_cvtsi32_si128(*(uint32_t*)(src+i+sizeof(__m128i)*2));
# define SHIFT_DATA_A(offs) (searchEnd && _USING_BLEND_ADD ? _mm_alignr_epi8(oDataB, oDataA, offs) : _mm_loadu_si128((__m128i *)(src+i+offs)))
# define SHIFT_DATA_B(offs) (searchEnd && _USING_BLEND_ADD ? _mm_alignr_epi8(nextDataB, oDataB, offs) : _mm_loadu_si128((__m128i *)(src+i+offs) + 1))
#else
# define SHIFT_DATA_A(offs) _mm_loadu_si128((__m128i *)(src+i+offs))
# define SHIFT_DATA_B(offs) _mm_loadu_si128((__m128i *)(src+i+offs) + 1)
#endif
                __m128i tmpData2A = SHIFT_DATA_A(2);
                __m128i tmpData2B = SHIFT_DATA_B(2);
                __m128i match2EqA, match2EqB;

#if defined(__AVX512VL__) && defined(__AVX512BW__)
                __mmask16 match2EqMaskA, match2EqMaskB;
                __mmask16 match0CrMaskA, match0CrMaskB;
                __mmask16 match2CrXDtMaskA, match2CrXDtMaskB;
                if(use_isa >= ISA_LEVEL_AVX3 && searchEnd) {
                    match2EqMaskA = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2A);
                    match2EqMaskB = _mm_cmpeq_epi8_mask(_mm_set1_epi8('='), tmpData2B);
                } else
#endif
                if(searchEnd) {
#if !defined(__tune_btver1__)
                    if(use_isa < ISA_LEVEL_SSSE3)
#endif
                        match2EqA = _mm_cmpeq_epi8(_mm_set1_epi8('='), tmpData2A);
                    match2EqB = _mm_cmpeq_epi8(_mm_set1_epi8('='), tmpData2B);
                }
                int partialKillDotFound;
                __m128i match2CrXDtA, match2CrXDtB;
                if(isRaw) {
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match0CrMaskA = _mm_cmpeq_epi8_mask(oDataA, _mm_set1_epi8('\r'));
                        match0CrMaskB = _mm_cmpeq_epi8_mask(oDataB, _mm_set1_epi8('\r'));
                        match2CrXDtMaskA = _mm_mask_cmpeq_epi8_mask(match0CrMaskA, tmpData2A, _mm_set1_epi8('.'));
                        match2CrXDtMaskB = _mm_mask_cmpeq_epi8_mask(match0CrMaskB, tmpData2B, _mm_set1_epi8('.'));
                        partialKillDotFound = KORTEST16(match2CrXDtMaskA, match2CrXDtMaskB);
                    } else
#endif
                    {
                        if(_USING_FAST_MATCH) {
                            cmpCrA = _mm_cmpeq_epi8(oDataA, _mm_set1_epi8('\r'));
                            cmpCrB = _mm_cmpeq_epi8(oDataB, _mm_set1_epi8('\r'));
                        }
                        match2CrXDtA = _mm_and_si128(cmpCrA, _mm_cmpeq_epi8(tmpData2A, _mm_set1_epi8('.')));
                        match2CrXDtB = _mm_and_si128(cmpCrB, _mm_cmpeq_epi8(tmpData2B, _mm_set1_epi8('.')));
                        partialKillDotFound = _mm_movemask_epi8(_mm_or_si128(match2CrXDtA, match2CrXDtB));
                    }
                }

                if(isRaw && LIKELIHOOD(0.001, partialKillDotFound)) {
                    __m128i match2NlDotA, match1NlA;
                    __m128i match2NlDotB, match1NlB;
                    // merge matches for \r\n.
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    __mmask16 match1NlMaskA, match1NlMaskB;
                    __mmask16 match2NlDotMaskA, match2NlDotMaskB;
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match1NlMaskA = _mm_mask_cmpeq_epi8_mask(
                            match0CrMaskA,
                            _mm_set1_epi8('\n'),
                            SHIFT_DATA_A(1)
                        );
                        match1NlMaskB = _mm_mask_cmpeq_epi8_mask(
                            match0CrMaskB,
                            _mm_set1_epi8('\n'),
                            SHIFT_DATA_B(1)
                        );
                        match2NlDotMaskA = KAND16(match2CrXDtMaskA, match1NlMaskA);
                        match2NlDotMaskB = KAND16(match2CrXDtMaskB, match1NlMaskB);
                    } else
#endif
                    {
                        __m128i match1LfA = _mm_cmpeq_epi8(_mm_set1_epi8('\n'), SHIFT_DATA_A(1));
                        __m128i match1LfB = _mm_cmpeq_epi8(_mm_set1_epi8('\n'), SHIFT_DATA_B(1));

                        // always recompute cmpCr to avoid register spills above
                        cmpCrA = _mm_cmpeq_epi8(force_align_read_128(src+i), _mm_set1_epi8('\r'));
                        cmpCrB = _mm_cmpeq_epi8(force_align_read_128(src+i + sizeof(__m128i)), _mm_set1_epi8('\r'));
                        match1NlA = _mm_and_si128(match1LfA, cmpCrA);
                        match1NlB = _mm_and_si128(match1LfB, cmpCrB);
                        match2NlDotA = _mm_and_si128(match2CrXDtA, match1NlA);
                        match2NlDotB = _mm_and_si128(match2CrXDtB, match1NlB);
                    }
                    if(searchEnd) {
                        __m128i tmpData3A = SHIFT_DATA_A(3);
                        __m128i tmpData3B = SHIFT_DATA_B(3);
                        __m128i tmpData4A = SHIFT_DATA_A(4);
                        __m128i tmpData4B = SHIFT_DATA_B(4);
                        // match instances of \r\n.\r\n and \r\n.=y
                        // TODO: consider doing a PALIGNR using match1Nl for match4NlA
                        __m128i match3CrA = _mm_cmpeq_epi8(_mm_set1_epi8('\r'), tmpData3A);
                        __m128i match3CrB = _mm_cmpeq_epi8(_mm_set1_epi8('\r'), tmpData3B);
                        __m128i match4LfA = _mm_cmpeq_epi8(tmpData4A, _mm_set1_epi8('\n'));
                        __m128i match4LfB = _mm_cmpeq_epi8(tmpData4B, _mm_set1_epi8('\n'));
                        __m128i match4EqYA = _mm_cmpeq_epi16(tmpData4A, _mm_set1_epi16(0x793d)); // =y
                        __m128i match4EqYB = _mm_cmpeq_epi16(tmpData4B, _mm_set1_epi16(0x793d)); // =y

                        int matchEnd;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                        if(use_isa >= ISA_LEVEL_AVX3) {
                            __mmask16 match3EqYMaskA = _mm_mask_cmpeq_epi8_mask(
                                match2EqMaskA, _mm_set1_epi8('y'), tmpData3A
                            );
                            __mmask16 match3EqYMaskB = _mm_mask_cmpeq_epi8_mask(
                                match2EqMaskB, _mm_set1_epi8('y'), tmpData3B
                            );
                            __m128i match34EqYA, match34EqYB;
# ifdef __AVX512VBMI2__
                            if(use_isa >= ISA_LEVEL_VBMI2) {
                                match34EqYA = _mm_shrdi_epi16(_mm_movm_epi8(match3EqYMaskA), match4EqYA, 8);
                                match34EqYB = _mm_shrdi_epi16(_mm_movm_epi8(match3EqYMaskB), match4EqYB, 8);
                            } else
# endif
                            {
                                // (match4EqY & 0xff00) | (match3EqY >> 8)
                                match34EqYA = _mm_mask_blend_epi8(match3EqYMaskA>>1, _mm_and_si128(match4EqYA, _mm_set1_epi16(-0xff)), _mm_set1_epi8(-1));
                                match34EqYB = _mm_mask_blend_epi8(match3EqYMaskB>>1, _mm_and_si128(match4EqYB, _mm_set1_epi16(-0xff)), _mm_set1_epi8(-1));
                            }
                            // merge \r\n and =y matches for tmpData4
                            __m128i match4EndA = _mm_ternarylogic_epi32(match34EqYA, match3CrA, match4LfA, 0xF8); // (match3Cr & match4Lf) | match34EqY
                            __m128i match4EndB = _mm_ternarylogic_epi32(match34EqYB, match3CrB, match4LfB, 0xF8);
                            // merge with \r\n. and combine
                            matchEnd = KORTEST16(
                                KOR16(
                                    _mm_mask_test_epi8_mask(match2NlDotMaskA, match4EndA, match4EndA),
                                    KAND16(match3EqYMaskA, match1NlMaskA)
                                ),
                                KOR16(
                                    _mm_mask_test_epi8_mask(match2NlDotMaskB, match4EndB, match4EndB),
                                    KAND16(match3EqYMaskB, match1NlMaskB)
                                )
                            );
                        } else
#endif
                        {
#if defined(__SSSE3__) && !defined(__tune_btver1__)
                            if(use_isa >= ISA_LEVEL_SSSE3)
                                match2EqA = _mm_alignr_epi8(cmpEqB, cmpEqA, 2);
#endif
                            __m128i match3EqYA = _mm_and_si128(match2EqA, _mm_cmpeq_epi8(_mm_set1_epi8('y'), tmpData3A));
                            __m128i match3EqYB = _mm_and_si128(match2EqB, _mm_cmpeq_epi8(_mm_set1_epi8('y'), tmpData3B));
                            match4EqYA = _mm_slli_epi16(match4EqYA, 8); // TODO: also consider using PBLENDVB here with shifted match3EqY instead
                            match4EqYB = _mm_slli_epi16(match4EqYB, 8);
                            // merge \r\n and =y matches for tmpData4
                            __m128i match4EndA = _mm_or_si128(
                                _mm_and_si128(match3CrA, match4LfA),
                                _mm_or_si128(match4EqYA, _mm_srli_epi16(match3EqYA, 8)) // _mm_srli_si128 by 1 also works
                            );
                            __m128i match4EndB = _mm_or_si128(
                                _mm_and_si128(match3CrB, match4LfB),
                                _mm_or_si128(match4EqYB, _mm_srli_epi16(match3EqYB, 8)) // _mm_srli_si128 by 1 also works
                            );
                            // merge with \r\n.
                            match4EndA = _mm_and_si128(match4EndA, match2NlDotA);
                            match4EndB = _mm_and_si128(match4EndB, match2NlDotB);
                            // match \r\n=y
                            __m128i match3EndA = _mm_and_si128(match3EqYA, match1NlA);
                            __m128i match3EndB = _mm_and_si128(match3EqYB, match1NlB);
                            // combine match sequences
                            matchEnd = _mm_movemask_epi8(_mm_or_si128(
                                _mm_or_si128(match4EndA, match3EndA),
                                _mm_or_si128(match4EndB, match3EndB)
                            ));
                        }

                        if(LIKELIHOOD(0.001, matchEnd)) {
                            // terminator found
                            // there's probably faster ways to do this, but reverting to scalar code should be good enough
                            len += (long)i;
                            break;
                        }
                    }
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        mask |= match2NlDotMaskA << 2;
                        mask |= (match2NlDotMaskB << 18) & 0xffffffff;
                        minMask = _mm_maskz_mov_epi8(~(match2NlDotMaskB>>14), _mm_set1_epi8('.'));
                    } else
#endif
                    {
                        mask |= (_mm_movemask_epi8(match2NlDotA) << 2);
                        mask |= (_mm_movemask_epi8(match2NlDotB) << 18) & 0xffffffff;
                        match2NlDotB = _mm_srli_si128(match2NlDotB, 14);
                        if(_USING_FAST_MATCH)
                            minMask = _mm_subs_epu8(_mm_set1_epi8('.'), match2NlDotB);
                        else
                            // this bitiwse trick works because '.'|'\n' == '.'
                            lfCompare = _mm_or_si128(
                                _mm_and_si128(match2NlDotB, _mm_set1_epi8('.')),
                                _mm_set1_epi8('\n')
                            );
                    }
                }
                else if(searchEnd) {
                    bool partialEndFound;
                    __m128i match3EqYA, match3EqYB;
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                    __mmask16 match3EqYMaskA, match3EqYMaskB;
                    if(use_isa >= ISA_LEVEL_AVX3) {
                        match3EqYMaskA = _mm_mask_cmpeq_epi8_mask(
                            match2EqMaskA,
                            _mm_set1_epi8('y'),
                            SHIFT_DATA_A(3)
                        );
                        match3EqYMaskB = _mm_mask_cmpeq_epi8_mask(
                            match2EqMaskB,
                            _mm_set1_epi8('y'),
                            SHIFT_DATA_B(3)
                        );
                        partialEndFound = KORTEST16(match3EqYMaskA, match3EqYMaskB);
                    } else
#endif
                    {
                        __m128i match3YA = _mm_cmpeq_epi8(
                            _mm_set1_epi8('y'),
                            SHIFT_DATA_A(3)
                        );
                        __m128i match3YB = _mm_cmpeq_epi8(
                            _mm_set1_epi8('y'),
                            SHIFT_DATA_B(3)
                        );
#if defined(__SSSE3__) && !defined(__tune_btver1__)
                        if(use_isa >= ISA_LEVEL_SSSE3)
                            match2EqA = _mm_alignr_epi8(cmpEqB, cmpEqA, 2);
#endif
                        match3EqYA = _mm_and_si128(match2EqA, match3YA);
                        match3EqYB = _mm_and_si128(match2EqB, match3YB);
                        partialEndFound = _mm_movemask_epi8(_mm_or_si128(match3EqYA, match3EqYB));
                    }
                    if(LIKELIHOOD(0.001, partialEndFound)) {
                        // if the rare case of '=y' is found, do a more precise check
                        bool endFound;

#if defined(__AVX512VL__) && defined(__AVX512BW__)
                        if(use_isa >= ISA_LEVEL_AVX3) {
                            __mmask16 match3LfEqYMaskA = _mm_mask_cmpeq_epi8_mask(
                                match3EqYMaskA,
                                _mm_set1_epi8('\n'),
                                SHIFT_DATA_A(1)
                            );
                            __mmask16 match3LfEqYMaskB = _mm_mask_cmpeq_epi8_mask(
                                match3EqYMaskB,
                                _mm_set1_epi8('\n'),
                                SHIFT_DATA_B(1)
                            );

                            endFound = KORTEST16(
                                _mm_mask_cmpeq_epi8_mask(match3LfEqYMaskA, oDataA, _mm_set1_epi8('\r')),
                                _mm_mask_cmpeq_epi8_mask(match3LfEqYMaskB, oDataB, _mm_set1_epi8('\r'))
                            );
                        } else
#endif
                        {
                            // always recompute cmpCr to avoid register spills above
                            cmpCrA = _mm_cmpeq_epi8(force_align_read_128(src+i), _mm_set1_epi8('\r'));
                            cmpCrB = _mm_cmpeq_epi8(force_align_read_128(src+i + sizeof(__m128i)), _mm_set1_epi8('\r'));
                            __m128i match1LfA = _mm_cmpeq_epi8(
                                _mm_set1_epi8('\n'),
                                SHIFT_DATA_A(1)
                            );
                            __m128i match1LfB = _mm_cmpeq_epi8(
                                _mm_set1_epi8('\n'),
                                SHIFT_DATA_B(1)
                            );
                            endFound = _mm_movemask_epi8(_mm_or_si128(
                                _mm_and_si128(
                                    match3EqYA,
                                    _mm_and_si128(match1LfA, cmpCrA)
                                ),
                                _mm_and_si128(
                                    match3EqYB,
                                    _mm_and_si128(match1LfB, cmpCrB)
                                )
                            ));
                        }

                        if(endFound) {
                            len += (long)i;
                            break;
                        }
                    }
                    if(isRaw) {
                        if(_USING_FAST_MATCH)
                            minMask = _mm_set1_epi8('.');
                        else
                            lfCompare = _mm_set1_epi8('\n');
                    }
                }
                else if(isRaw) { // no \r_. found
                    if(_USING_FAST_MATCH)
                        minMask = _mm_set1_epi8('.');
                    else
                        lfCompare = _mm_set1_epi8('\n');
                }
            }
#undef SHIFT_DATA_A
#undef SHIFT_DATA_B

            if(!_USING_BLEND_ADD)
                dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42));

            if(LIKELIHOOD(0.0001, (mask & ((maskEq << 1) + escFirst)) != 0)) {
                // resolve invalid sequences of = to deal with cases like '===='
                unsigned tmp = lookups->eqFix[(maskEq&0xff) & ~escFirst];
                uint32_t maskEq2 = tmp;
                for(int j=8; j<32; j+=8) {
                    tmp = lookups->eqFix[((maskEq>>j)&0xff) & ~(tmp>>7)];
                    maskEq2 |= tmp<<j;
                }
                maskEq = maskEq2;

                mask &= ~escFirst;
                escFirst = (maskEq >> 31);
                // next, eliminate anything following a `=` from the special char mask; this eliminates cases of `=\r` so that they aren't removed
                maskEq <<= 1;
                mask &= ~maskEq;

                if(_USING_BLEND_ADD) {
                    dataA = _mm_add_epi8(oDataA, yencOffset);
                    dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42));
                }

                // unescape chars following `=`
#if defined(__AVX512VL__) && defined(__AVX512BW__)
                if(use_isa >= ISA_LEVEL_AVX3) {
                    // GCC < 7 seems to generate rubbish assembly for this
                    dataA = _mm_mask_add_epi8(
                        dataA,
                        (__mmask16)maskEq,
                        dataA,
                        _mm_set1_epi8(-64)
                    );
                    dataB = _mm_mask_add_epi8(
                        dataB,
                        (__mmask16)(maskEq>>16),
                        dataB,
                        _mm_set1_epi8(-64)
                    );
                } else
#endif
                {
                    dataA = _mm_add_epi8(
                        dataA,
                        LOAD_HALVES(
                            lookups->eqAdd + (maskEq&0xff),
                            lookups->eqAdd + ((maskEq>>8)&0xff)
                        )
                    );
                    maskEq >>= 16;
                    dataB = _mm_add_epi8(
                        dataB,
                        LOAD_HALVES(
                            lookups->eqAdd + (maskEq&0xff),
                            lookups->eqAdd + ((maskEq>>8)&0xff)
                        )
                    );

                    yencOffset = _mm_xor_si128(_mm_set1_epi8(-42),
                        _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6)
                    );
                }
            } else {
                // no invalid = sequences found - we can cut out some things from above
                // this code path is a shortened version of above; it's here because it's faster, and what we'll be dealing with most of the time
                escFirst = (maskEq >> 31);

#if defined(__AVX512VL__) && defined(__AVX512BW__)
                if(use_isa >= ISA_LEVEL_AVX3) {
                    dataA = _mm_add_epi8(
                        oDataA,
                        _mm_ternarylogic_epi32(
                            _mm_slli_si128(cmpEqA, 1), yencOffset, _mm_set1_epi8(-42-64), 0xac
                        )
                    );
                    dataB = _mm_add_epi8(
                        oDataB,
                        _mm_ternarylogic_epi32(
                            _mm_alignr_epi8(cmpEqB, cmpEqA, 15), _mm_set1_epi8(-42), _mm_set1_epi8(-42-64), 0xac
                        )
                    );
                } else
#endif
#if defined(__SSE4_1__)
                if(_USING_BLEND_ADD) {
                    /* // the following strategy seems more ideal, however, both GCC and Clang go bonkers over it and spill more registers
                    cmpEqA = _mm_blendv_epi8(_mm_set1_epi8(-42), _mm_set1_epi8(-42-64), cmpEqA);
                    cmpEqB = _mm_blendv_epi8(_mm_set1_epi8(-42), _mm_set1_epi8(-42-64), cmpEqB);
                    dataB = _mm_add_epi8(oDataB, _mm_alignr_epi8(cmpEqB, cmpEqA, 15));
                    dataA = _mm_add_epi8(oDataA, _mm_and_si128(
                        _mm_alignr_epi8(cmpEqA, _mm_set1_epi8(-42), 15),
                        yencOffset
                    ));
                    yencOffset = _mm_alignr_epi8(_mm_set1_epi8(-42), cmpEqB, 15);
                    */

                    dataA = _mm_add_epi8(
                        oDataA,
                        _mm_blendv_epi8(
                            yencOffset, _mm_set1_epi8(-42-64), _mm_slli_si128(cmpEqA, 1)
                        )
                    );
                    dataB = _mm_add_epi8(
                        oDataB,
                        _mm_blendv_epi8(
                            _mm_set1_epi8(-42), _mm_set1_epi8(-42-64), _mm_alignr_epi8(cmpEqB, cmpEqA, 15)
                        )
                    );
                    yencOffset = _mm_xor_si128(_mm_set1_epi8(-42),
                        _mm_slli_epi16(_mm_cvtsi32_si128((int)escFirst), 6)
                    );
                } else
#endif
                {
                    cmpEqA = _mm_and_si128(cmpEqA, _mm_set1_epi8(-64));
                    cmpEqB = _mm_and_si128(cmpEqB, _mm_set1_epi8(-64));
                    yencOffset = _mm_add_epi8(_mm_set1_epi8(-42), _mm_srli_si128(cmpEqB, 15));
#if defined(__SSSE3__) && !defined(__tune_btver1__)
                    if(use_isa >= ISA_LEVEL_SSSE3)
                        cmpEqB = _mm_alignr_epi8(cmpEqB, cmpEqA, 15);
                    else
#endif
                        cmpEqB = _mm_or_si128(
                            _mm_slli_si128(cmpEqB, 1),
                            _mm_srli_si128(cmpEqA, 15)
                        );
                    cmpEqA = _mm_slli_si128(cmpEqA, 1);
                    dataA = _mm_add_epi8(dataA, cmpEqA);
                    dataB = _mm_add_epi8(dataB, cmpEqB);
                }
            }
            // subtract 64 from first element if escFirst == 1
#if defined(__AVX512VL__) && defined(__AVX512BW__)
            if(use_isa >= ISA_LEVEL_AVX3) {
                yencOffset = _mm_mask_add_epi8(_mm_set1_epi8(-42), (__mmask16)escFirst, _mm_set1_epi8(-42), _mm_set1_epi8(-64));
            }
#endif

            // all that's left is to 'compress' the data (skip over masked chars)
#ifdef __SSSE3__
            if(use_isa >= ISA_LEVEL_SSSE3) {
# if defined(__AVX512VBMI2__) && defined(__AVX512VL__) && defined(__POPCNT__)
                if(use_isa >= ISA_LEVEL_VBMI2) {
                    _mm_mask_compressstoreu_epi8(p, KNOT16(mask), dataA);
                    p -= popcnt32(mask & 0xffff);
                    _mm_mask_compressstoreu_epi8(p+XMM_SIZE, KNOT16(mask>>16), dataB);
                    p -= popcnt32(mask>>16);
                    p += XMM_SIZE*2;
                } else
# endif
                {

                    dataA = _mm_shuffle_epi8(dataA, _mm_load_si128((__m128i*)(lookups->compact + (mask&0x7fff))));
                    STOREU_XMM(p, dataA);

                    dataB = _mm_shuffle_epi8(dataB, _mm_load_si128((__m128i*)((char*)lookups->compact + ((mask >> 12) & 0x7fff0))));

# if defined(__POPCNT__) && !defined(__tune_btver1__)
                    if(use_isa & ISA_FEATURE_POPCNT) {
                        p -= popcnt32(mask & 0xffff);
                        STOREU_XMM(p+XMM_SIZE, dataB);
                        p -= popcnt32(mask & 0xffff0000);
                        p += XMM_SIZE*2;
                    } else
# endif
                    {
                        p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff];
                        STOREU_XMM(p, dataB);
                        mask >>= 16;
                        p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff];
                    }
                }
            } else
#endif
            {
                dataA = sse2_compact_vect<use_isa>(mask & 0xffff, dataA);
                STOREU_XMM(p, dataA);
                p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff];
                mask >>= 16;
                dataB = sse2_compact_vect<use_isa>(mask, dataB);
                STOREU_XMM(p, dataB);
                p += lookups->BitsSetTable256inv[mask & 0xff] + lookups->BitsSetTable256inv[(mask >> 8) & 0xff];
            }
#undef LOAD_HALVES
        } else {
            if(_USING_BLEND_ADD)
                dataA = _mm_add_epi8(oDataA, yencOffset);
            dataB = _mm_add_epi8(oDataB, _mm_set1_epi8(-42));

            STOREU_XMM(p, dataA);
            STOREU_XMM(p+XMM_SIZE, dataB);
            p += XMM_SIZE*2;
            escFirst = 0;
            yencOffset = _mm_set1_epi8(-42);
        }
    }
    _escFirst = (unsigned char)escFirst;
    if(isRaw) {
        if(len != 0) { // have to gone through at least one loop cycle
            if(src[i-2] == '\r' && src[i-1] == '\n' && src[i] == '.')
                _nextMask = 1;
            else if(src[i-1] == '\r' && src[i] == '\n' && src[i+1] == '.')
                _nextMask = 2;
            else
                _nextMask = 0;
        }
    } else
        _nextMask = 0;
}
#endif
