diff --git a/include/linux/lz4.h b/include/linux/lz4.h index 394e3d9213b8..31dc0cb5c794 100644 --- a/include/linux/lz4.h +++ b/include/linux/lz4.h @@ -42,7 +42,7 @@ #define __LZ4_H__ #include -#include /* memset, memcpy */ +#include /* memset, memcpy */ /*-************************************************************************ * CONSTANTS @@ -57,23 +57,24 @@ */ #define LZ4_MEMORY_USAGE 14 -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) (\ - (unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE \ - ? 0 \ - : (isize) + ((isize)/255) + 16) +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) \ + ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? \ + 0 : \ + (isize) + ((isize) / 255) + 16) #define LZ4_ACCELERATION_DEFAULT 1 -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_ACCELERATION_MAX 65537 +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE - 2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 16 +#define LZ4HC_MIN_CLEVEL 3 +#define LZ4HC_DEFAULT_CLEVEL 9 +#define LZ4HC_MAX_CLEVEL 16 #define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1< compression run slower on incompressible data */ +static const U32 LZ4_skipTrigger = 6; + +LZ4_stream_t *LZ4_initStream(void *buffer, size_t size); /*-****************************** * Compression functions ********************************/ -static FORCE_INLINE U32 LZ4_hash4( - U32 sequence, - tableType_t const tableType) +static FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) - return ((sequence * 2654435761U) - >> ((MINMATCH * 8) - (LZ4_HASHLOG + 1))); + return ((sequence * 2654435761U) >> + ((MINMATCH * 8) - (LZ4_HASHLOG + 1))); else - return ((sequence * 2654435761U) - >> ((MINMATCH * 8) - LZ4_HASHLOG)); + return ((sequence * 2654435761U) >> + ((MINMATCH * 8) - LZ4_HASHLOG)); } -static FORCE_INLINE U32 LZ4_hash5( - U64 sequence, - tableType_t const tableType) +static FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - const U32 hashLog = (tableType == byU16) - ? LZ4_HASHLOG + 1 - : LZ4_HASHLOG; + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG + 1 : + LZ4_HASHLOG; #if LZ4_LITTLE_ENDIAN static const U64 prime5bytes = 889523592379ULL; @@ -76,245 +75,454 @@ static FORCE_INLINE U32 LZ4_hash5( #endif } -static FORCE_INLINE U32 LZ4_hashPosition( - const void *p, - tableType_t const tableType) +static FORCE_INLINE U32 LZ4_hashPosition(const void *p, + tableType_t const tableType) { #if LZ4_ARCH64 - if (tableType == byU32) + if (tableType != byU16) return LZ4_hash5(LZ4_read_ARCH(p), tableType); #endif return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash( - const BYTE *p, - U32 h, - void *tableBase, - tableType_t const tableType, - const BYTE *srcBase) +static FORCE_INLINE void LZ4_clearHash(U32 h, void *tableBase, + tableType_t const tableType) { switch (tableType) { - case byPtr: - { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ + assert(0); + return; + } + case byPtr: { const BYTE **hashTable = (const BYTE **)tableBase; - - hashTable[h] = p; + hashTable[h] = NULL; return; } - case byU32: - { - U32 *hashTable = (U32 *) tableBase; - - hashTable[h] = (U32)(p - srcBase); + case byU32: { + U32 *hashTable = (U32 *)tableBase; + hashTable[h] = 0; return; } - case byU16: - { - U16 *hashTable = (U16 *) tableBase; - - hashTable[h] = (U16)(p - srcBase); + case byU16: { + U16 *hashTable = (U16 *)tableBase; + hashTable[h] = 0; return; } } } -static FORCE_INLINE void LZ4_putPosition( - const BYTE *p, - void *tableBase, - tableType_t tableType, - const BYTE *srcBase) +static FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void *tableBase, + tableType_t const tableType) +{ + switch (tableType) { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ + assert(0); + return; + } + case byU32: { + U32 *hashTable = (U32 *)tableBase; + hashTable[h] = idx; + return; + } + case byU16: { + U16 *hashTable = (U16 *)tableBase; + assert(idx < 65536); + hashTable[h] = (U16)idx; + return; + } + } +} + +static void LZ4_putPositionOnHash(const BYTE *p, U32 h, void *tableBase, + tableType_t const tableType, + const BYTE *srcBase) +{ + switch (tableType) { + case byPtr: { + const BYTE **hashTable = (const BYTE **)tableBase; + + hashTable[h] = p; + return; + } + case byU32: { + U32 *hashTable = (U32 *)tableBase; + + hashTable[h] = (U32)(p - srcBase); + return; + } + case byU16: { + U16 *hashTable = (U16 *)tableBase; + + hashTable[h] = (U16)(p - srcBase); + return; + } + case clearedTable: { /* fallthrough */ + } + } +} + +static FORCE_INLINE void LZ4_putPosition(const BYTE *p, void *tableBase, + tableType_t tableType, + const BYTE *srcBase) { U32 const h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE *LZ4_getPositionOnHash( - U32 h, - void *tableBase, - tableType_t tableType, - const BYTE *srcBase) +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +static FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void *tableBase, + tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32 *const hashTable = (const U32 *)tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE - 2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16 *const hashTable = (const U16 *)tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE - 1))); + return hashTable[h]; + } + assert(0); + return 0; /* forbidden case */ +} + +static const BYTE *LZ4_getPositionOnHash(U32 h, void *tableBase, + tableType_t tableType, + const BYTE *srcBase) { if (tableType == byPtr) { - const BYTE **hashTable = (const BYTE **) tableBase; + const BYTE **hashTable = (const BYTE **)tableBase; return hashTable[h]; } if (tableType == byU32) { - const U32 * const hashTable = (U32 *) tableBase; + const U32 *const hashTable = (U32 *)tableBase; return hashTable[h] + srcBase; } { /* default, to ensure a return */ - const U16 * const hashTable = (U16 *) tableBase; + const U16 *const hashTable = (U16 *)tableBase; return hashTable[h] + srcBase; } } -static FORCE_INLINE const BYTE *LZ4_getPosition( - const BYTE *p, - void *tableBase, - tableType_t tableType, - const BYTE *srcBase) +static FORCE_INLINE const BYTE *LZ4_getPosition(const BYTE *p, void *tableBase, + tableType_t tableType, + const BYTE *srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } - -/* - * LZ4_compress_generic() : - * inlined, to ensure branches are decided at compilation time - */ -static FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal * const dictPtr, - const char * const source, - char * const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) +static FORCE_INLINE void LZ4_prepareTable(LZ4_stream_t_internal *const cctx, + const int inputSize, + const tableType_t tableType) { - const BYTE *ip = (const BYTE *) source; - const BYTE *base; + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType || + ((tableType == byU16) && + cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) || + ((tableType == byU32) && cctx->currentOffset > 1 * GB) || + tableType == byPtr || inputSize >= 4 * KB) { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", + cctx); + memset(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG( + 4, + "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 * KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time. + * Presumed already validated at this stage: + * - source != NULL + * - inputSize > 0 + */ +static FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal *const cctx, const char *const source, + char *const dest, const int inputSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, const limitedOutput_directive outputDirective, + const tableType_t tableType, const dict_directive dictDirective, + const dictIssue_directive dictIssue, const int acceleration) +{ + int result; + const BYTE *ip = (const BYTE *)source; + + U32 const startIndex = cctx->currentOffset; + const BYTE *base = (const BYTE *)source - startIndex; const BYTE *lowLimit; - const BYTE * const lowRefLimit = ip - dictPtr->dictSize; - const BYTE * const dictionary = dictPtr->dictionary; - const BYTE * const dictEnd = dictionary + dictPtr->dictSize; - const size_t dictDelta = dictEnd - (const BYTE *)source; - const BYTE *anchor = (const BYTE *) source; - const BYTE * const iend = ip + inputSize; - const BYTE * const mflimit = iend - MFLIMIT; - const BYTE * const matchlimit = iend - LASTLITERALS; - BYTE *op = (BYTE *) dest; - BYTE * const olimit = op + maxOutputSize; + const LZ4_stream_t_internal *dictCtx = + (const LZ4_stream_t_internal *)cctx->dictCtx; + const BYTE *const dictionary = dictDirective == usingDictCtx ? + dictCtx->dictionary : + cctx->dictionary; + const U32 dictSize = dictDirective == usingDictCtx ? dictCtx->dictSize : + cctx->dictSize; + const U32 dictDelta = + (dictDirective == usingDictCtx) ? + startIndex - dictCtx->currentOffset : + 0; /* make indexes in dictCtx comparable with index in current context */ + int const maybe_extMem = (dictDirective == usingExtDict) || + (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = + startIndex - + dictSize; /* used when dictDirective == dictSmall */ + const BYTE *const dictEnd = dictionary ? dictionary + dictSize : + dictionary; + const BYTE *anchor = (const BYTE *)source; + const BYTE *const iend = ip + inputSize; + const BYTE *const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE *const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE *dictBase = + (dictionary == NULL) ? + NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE *op = (BYTE *)dest; + BYTE *const olimit = op + maxOutputSize; + + U32 offset = 0; U32 forwardH; - size_t refDelta = 0; - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { - /* Unsupported inputSize, too large (or negative) */ + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", + inputSize, tableType); + assert(ip != NULL); + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; - } - - switch (dict) { - case noDict: - default: - base = (const BYTE *)source; - lowLimit = (const BYTE *)source; - break; - case withPrefix64k: - base = (const BYTE *)source - dictPtr->currentOffset; - lowLimit = (const BYTE *)source - dictPtr->dictSize; - break; - case usingExtDict: - base = (const BYTE *)source - dictPtr->currentOffset; - lowLimit = (const BYTE *)source; - break; - } - - if ((tableType == byU16) - && (inputSize >= LZ4_64Klimit)) { - /* Size too large (not within 64K limit) */ + } /* Impossible to store anything */ + if ((tableType == byU16) && (inputSize >= LZ4_64Klimit)) { return 0; - } + } /* Size too large (not within 64K limit) */ + if (tableType == byPtr) + assert(dictDirective == + noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); - if (inputSize < LZ4_minLength) { - /* Input too small, no compression (all literals) */ - goto _last_literals; + lowLimit = (const BYTE *)source - + (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSize < LZ4_minLength) + goto _last_literals; /* Input too small, no compression (all literals) */ /* First Byte */ - LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ - for ( ; ; ) { + for (;;) { const BYTE *match; BYTE *token; + const BYTE *filledIp; /* Find a match */ - { + if (tableType == byPtr) { const BYTE *forwardIp = ip; - unsigned int step = 1; - unsigned int searchMatchNb = acceleration << LZ4_SKIPTRIGGER; - + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; - ip = forwardIp; forwardIp += step; - step = (searchMatchNb++ >> LZ4_SKIPTRIGGER); + step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); - match = LZ4_getPositionOnHash(h, - dictPtr->hashTable, - tableType, base); + match = LZ4_getPositionOnHash( + h, cctx->hashTable, tableType, base); + forwardH = + LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, + tableType, base); - if (dict == usingExtDict) { - if (match < (const BYTE *)source) { - refDelta = dictDelta; + } while ((match + LZ4_DISTANCE_MAX < ip) || + (LZ4_read32(match) != LZ4_read32(ip))); + + } else { /* byU32, byU16 */ + + const BYTE *forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const cur = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash( + h, cctx->hashTable, tableType); + assert(matchIndex <= cur); + assert(forwardIp - base < + (ptrdiff_t)(2 * GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) + goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash( + h, dictCtx->hashTable, + byU32); + match = dictBase + matchIndex; + matchIndex += + dictDelta; /* make dictCtx index comparable with current context */ lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE *)source; - } } + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG( + 7, + "extDict candidate: matchIndex=%5u < startIndex=%5u", + matchIndex, startIndex); + assert(startIndex - + matchIndex >= + MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE *)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = + LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(cur, h, cctx->hashTable, + tableType); - forwardH = LZ4_hashPosition(forwardIp, - tableType); + DEBUGLOG(7, + "candidate at pos=%u (offset=%u \n", + matchIndex, cur - matchIndex); + if ((dictIssue == dictSmall) && + (matchIndex < prefixIdxLimit)) { + continue; + } /* match outside of valid area */ + assert(matchIndex < cur); + if (((tableType != byU16) || + (LZ4_DISTANCE_MAX < + LZ4_DISTANCE_ABSOLUTE_MAX)) && + (matchIndex + LZ4_DISTANCE_MAX < cur)) { + continue; + } /* too far */ + assert((cur - matchIndex) <= + LZ4_DISTANCE_MAX); /* match now expected within distance */ - LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, - tableType, base); - } while (((dictIssue == dictSmall) - ? (match < lowRefLimit) - : 0) - || ((tableType == byU16) - ? 0 - : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match + refDelta) - != LZ4_read32(ip))); + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) + offset = cur - matchIndex; + break; /* match found */ + } + + } while (1); } /* Catch up */ - while (((ip > anchor) & (match + refDelta > lowLimit)) - && (unlikely(ip[-1] == match[refDelta - 1]))) { + filledIp = ip; + while (((ip > anchor) & (match > lowLimit)) && + (unlikely(ip[-1] == match[-1]))) { ip--; match--; } /* Encode Literals */ { - unsigned const int litLength = (unsigned int)(ip - anchor); - + unsigned const litLength = (unsigned)(ip - anchor); token = op++; - - if ((outputLimited) && - /* Check output buffer overflow */ - (unlikely(op + litLength + - (2 + 1 + LASTLITERALS) + - (litLength / 255) > olimit))) - return 0; - + if ((outputDirective == + limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + + (litLength / 255) > + olimit))) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely( + op + (litLength + 240) / 255 /* litlen */ + + litLength /* literals */ + + 2 /* offset */ + 1 /* token */ + + MFLIMIT - + MINMATCH /* min last literals so last match is <= end - MFLIMIT */ + > olimit))) { + op--; + goto _last_literals; + } if (litLength >= RUN_MASK) { - int len = (int)litLength - RUN_MASK; - + int len = (int)(litLength - RUN_MASK); *token = (RUN_MASK << ML_BITS); - for (; len >= 255; len -= 255) *op++ = 255; *op++ = (BYTE)len; @@ -322,103 +530,222 @@ static FORCE_INLINE int LZ4_compress_generic( *token = (BYTE)(litLength << ML_BITS); /* Copy Literals */ - LZ4_wildCopy(op, anchor, op + litLength); + LZ4_wildCopy8(op, anchor, op + litLength); op += litLength; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor - (const BYTE *)source), + litLength, (int)(ip - (const BYTE *)source)); } _next_match: + /* at this stage, the following variables must be correctly set : + * - ip : at start of LZ operation + * - match : at start of previous pattern occurrence; can be within current prefix, or within extDict + * - offset : if maybe_ext_memSegment==1 (constant) + * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise + * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written + */ + + if ((outputDirective == fillOutput) && + (op + 2 /* offset */ + 1 /* token */ + MFLIMIT - + MINMATCH /* min last literals so last match is <= end - MFLIMIT */ + > olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip - match)); - op += 2; + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, + " with offset=%u (ext if > %i)", + offset, (int)(ip - (const BYTE *)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); + op += 2; + } else { + DEBUGLOG(6, + " with offset=%u (same segment)", + (U32)(ip - match)); + assert(ip - match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); + op += 2; + } /* Encode MatchLength */ { - unsigned int matchCode; - - if ((dict == usingExtDict) - && (lowLimit == dictionary)) { - const BYTE *limit; - - match += refDelta; - limit = ip + (dictEnd - match); + unsigned matchCode; + if ((dictDirective == usingExtDict || + dictDirective == usingDictCtx) && + (lowLimit == + dictionary) /* match within extDict */) { + const BYTE *limit = ip + (dictEnd - match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip + MINMATCH, - match + MINMATCH, limit); - - ip += MINMATCH + matchCode; - + match + MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; if (ip == limit) { - unsigned const int more = LZ4_count(ip, - (const BYTE *)source, + unsigned const more = LZ4_count( + limit, (const BYTE *)source, matchlimit); - matchCode += more; ip += more; } + DEBUGLOG( + 6, + " with matchLength=%u starting in extDict", + matchCode + MINMATCH); } else { matchCode = LZ4_count(ip + MINMATCH, - match + MINMATCH, matchlimit); - ip += MINMATCH + matchCode; + match + MINMATCH, + matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", + matchCode + MINMATCH); } - if (outputLimited && - /* Check output buffer overflow */ - (unlikely(op + - (1 + LASTLITERALS) + - (matchCode >> 8) > olimit))) - return 0; - + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + + (matchCode + 240) / 255 > + olimit))) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = + 15 /* in token */ - + 1 /* to avoid needing a zero byte */ + + ((U32)(olimit - op) - 1 - + LASTLITERALS) * + 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE *ptr; + DEBUGLOG( + 5, + "Clearing %u positions", + (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; + ++ptr) { + U32 const h = + LZ4_hashPosition( + ptr, + tableType); + LZ4_clearHash( + h, + cctx->hashTable, + tableType); + } + } + } else { + assert(outputDirective == + limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4 * 255) { op += 4; LZ4_write32(op, 0xFFFFFFFF); matchCode -= 4 * 255; } - op += matchCode / 255; *op++ = (BYTE)(matchCode % 255); } else *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && + op + 1 + LASTLITERALS > olimit)); anchor = ip; /* Test end of chunk */ - if (ip > mflimit) + if (ip >= mflimitPlusOne) break; /* Fill table */ - LZ4_putPosition(ip - 2, dictPtr->hashTable, tableType, base); + LZ4_putPosition(ip - 2, cctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, dictPtr->hashTable, - tableType, base); - - if (dict == usingExtDict) { - if (match < (const BYTE *)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE *)source; + if (tableType == byPtr) { + match = LZ4_getPosition(ip, cctx->hashTable, tableType, + base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ((match + LZ4_DISTANCE_MAX >= ip) && + (LZ4_read32(match) == LZ4_read32(ip))) { + token = op++; + *token = 0; + goto _next_match; } - } - LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); + } else { /* byU32, byU16 */ - if (((dictIssue == dictSmall) ? (match >= lowRefLimit) : 1) - && (match + MAX_DISTANCE >= ip) - && (LZ4_read32(match + refDelta) == LZ4_read32(ip))) { - token = op++; - *token = 0; - goto _next_match; + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const cur = (U32)(ip - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, + tableType); + assert(matchIndex < cur); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash( + h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = + dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE *) + source; /* required for match length counter */ + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = + dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE *) + source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(cur, h, cctx->hashTable, tableType); + assert(matchIndex < cur); + if (((dictIssue == dictSmall) ? + (matchIndex >= prefixIdxLimit) : + 1) && + (((tableType == byU16) && + (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? + 1 : + (matchIndex + LZ4_DISTANCE_MAX >= cur)) && + (LZ4_read32(match) == LZ4_read32(ip))) { + token = op++; + *token = 0; + if (maybe_extMem) + offset = cur - matchIndex; + DEBUGLOG( + 6, + "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor - (const BYTE *)source), 0, + (int)(ip - (const BYTE *)source)); + goto _next_match; + } } /* Prepare next loop */ @@ -428,398 +755,286 @@ _next_match: _last_literals: /* Encode Last Literals */ { - size_t const lastRun = (size_t)(iend - anchor); - - if ((outputLimited) && - /* Check output buffer overflow */ - ((op - (BYTE *)dest) + lastRun + 1 + - ((lastRun + 255 - RUN_MASK) / 255) > (U32)maxOutputSize)) - return 0; - + size_t lastRun = (size_t)(iend - anchor); + if ((outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) > + olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit - op) - 1 /*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / + 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; for (; accumulator >= 255; accumulator -= 255) *op++ = 255; - *op++ = (BYTE) accumulator; + *op++ = (BYTE)accumulator; } else { *op++ = (BYTE)(lastRun << ML_BITS); } - LZ4_memcpy(op, anchor, lastRun); - + ip = anchor + lastRun; op += lastRun; } - /* End */ - return (int) (((char *)op) - dest); + if (outputDirective == fillOutput) { + *inputConsumed = (int)(((const char *)ip) - source); + } + result = (int)(((char *)op) - dest); + assert(result > 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", + inputSize, result); + return result; } -static int LZ4_compress_fast_extState( - void *state, - const char *source, - char *dest, - int inputSize, - int maxOutputSize, - int acceleration) +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +static FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal *const cctx, const char *const src, + char *const dst, const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, const limitedOutput_directive outputDirective, + const tableType_t tableType, const dict_directive dictDirective, + const dictIssue_directive dictIssue, const int acceleration) { - LZ4_stream_t_internal *ctx = &((LZ4_stream_t *)state)->internal_donotuse; -#if LZ4_ARCH64 - const tableType_t tableType = byU32; -#else - const tableType_t tableType = byPtr; -#endif + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", srcSize, + dstCapacity); - LZ4_resetStream((LZ4_stream_t *)state); + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { + return 0; + } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) + return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert(inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + return LZ4_compress_generic_validated( + cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, tableType, dictDirective, + dictIssue, acceleration); +} + +int LZ4_compress_fast_extState(void *state, const char *source, char *dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + LZ4_stream_t_internal *const ctx = + &LZ4_initStream(state, sizeof(LZ4_stream_t))->internal_donotuse; + assert(ctx != NULL); if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; - - if (maxOutputSize >= LZ4_COMPRESSBOUND(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, - dest, inputSize, 0, - noLimit, byU16, noDict, - noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, - dest, inputSize, 0, - noLimit, tableType, noDict, - noDictIssue, acceleration); + if (acceleration > LZ4_ACCELERATION_MAX) + acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, + inputSize, NULL, 0, + notLimited, byU16, noDict, + noDictIssue, acceleration); + } else { + const tableType_t tableType = + ((sizeof(void *) == 4) && + ((uptrval)source > LZ4_DISTANCE_MAX)) ? + byPtr : + byU32; + return LZ4_compress_generic(ctx, source, dest, + inputSize, NULL, 0, + notLimited, tableType, + noDict, noDictIssue, + acceleration); + } } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, - dest, inputSize, + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic( + ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, - dest, inputSize, + } else { + const tableType_t tableType = + ((sizeof(void *) == 4) && + ((uptrval)source > LZ4_DISTANCE_MAX)) ? + byPtr : + byU32; + return LZ4_compress_generic( + ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } } } int LZ4_compress_fast(const char *source, char *dest, int inputSize, - int maxOutputSize, int acceleration, void *wrkmem) + int maxOutputSize, int acceleration, void *wrkmem) { return LZ4_compress_fast_extState(wrkmem, source, dest, inputSize, - maxOutputSize, acceleration); + maxOutputSize, acceleration); } EXPORT_SYMBOL(LZ4_compress_fast); int LZ4_compress_default(const char *source, char *dest, int inputSize, - int maxOutputSize, void *wrkmem) + int maxOutputSize, void *wrkmem) { - return LZ4_compress_fast(source, dest, inputSize, - maxOutputSize, LZ4_ACCELERATION_DEFAULT, wrkmem); + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, + LZ4_ACCELERATION_DEFAULT, wrkmem); } EXPORT_SYMBOL(LZ4_compress_default); -/*-****************************** - * *_destSize() variant - ********************************/ -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal * const ctx, - const char * const src, - char * const dst, - int * const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) +static int LZ4_compress_destSize_extState(LZ4_stream_t *state, const char *src, + char *dst, int *srcSizePtr, + int targetDstSize) { - const BYTE *ip = (const BYTE *) src; - const BYTE *base = (const BYTE *) src; - const BYTE *lowLimit = (const BYTE *) src; - const BYTE *anchor = ip; - const BYTE * const iend = ip + *srcSizePtr; - const BYTE * const mflimit = iend - MFLIMIT; - const BYTE * const matchlimit = iend - LASTLITERALS; + void *const s = LZ4_initStream(state, sizeof(*state)); + assert(s != NULL); + (void)s; - BYTE *op = (BYTE *) dst; - BYTE * const oend = op + targetDstSize; - BYTE * const oMaxLit = op + targetDstSize - 2 /* offset */ - - 8 /* because 8 + MINMATCH == MFLIMIT */ - 1 /* token */; - BYTE * const oMaxMatch = op + targetDstSize - - (LASTLITERALS + 1 /* token */); - BYTE * const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - /* Init conditions */ - /* Impossible to store anything */ - if (targetDstSize < 1) - return 0; - /* Unsupported input size, too large (or negative) */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) - return 0; - /* Size too large (not within 64K limit) */ - if ((tableType == byU16) && (*srcSizePtr >= LZ4_64Klimit)) - return 0; - /* Input too small, no compression (all literals) */ - if (*srcSizePtr < LZ4_minLength) - goto _last_literals; - - /* First Byte */ - *srcSizePtr = 0; - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE *match; - BYTE *token; - - /* Find a match */ - { - const BYTE *forwardIp = ip; - unsigned int step = 1; - unsigned int searchMatchNb = 1 << LZ4_SKIPTRIGGER; - - do { - U32 h = forwardH; - - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_SKIPTRIGGER); - - if (unlikely(forwardIp > mflimit)) - goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, - tableType, base); - forwardH = LZ4_hashPosition(forwardIp, - tableType); - LZ4_putPositionOnHash(ip, h, - ctx->hashTable, tableType, - base); - - } while (((tableType == byU16) - ? 0 - : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip))); - } - - /* Catch up */ - while ((ip > anchor) - && (match > lowLimit) - && (unlikely(ip[-1] == match[-1]))) { - ip--; - match--; - } - - /* Encode Literal length */ - { - unsigned int litLength = (unsigned int)(ip - anchor); - - token = op++; - if (op + ((litLength + 240) / 255) - + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength >= RUN_MASK) { - unsigned int len = litLength - RUN_MASK; - *token = (RUN_MASK<= 255; len -= 255) - *op++ = 255; - *op++ = (BYTE)len; - } else - *token = (BYTE)(litLength << ML_BITS); - - /* Copy Literals */ - LZ4_wildCopy(op, anchor, op + litLength); - op += litLength; - } - -_next_match: - /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip - match)); op += 2; - - /* Encode MatchLength */ - { - size_t matchLength = LZ4_count(ip + MINMATCH, - match + MINMATCH, matchlimit); - - if (op + ((matchLength + 240)/255) > oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15 - 1) + (oMaxMatch - op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength >= ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { - matchLength -= 255; - *op++ = 255; - } - *op++ = (BYTE)matchLength; - } else - *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) - break; - if (op > oMaxSeq) - break; - - /* Fill table */ - LZ4_putPosition(ip - 2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - - if ((match + MAX_DISTANCE >= ip) - && (LZ4_read32(match) == LZ4_read32(ip))) { - token = op++; *token = 0; - goto _next_match; - } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { - size_t lastRunSize = (size_t)(iend - anchor); - - if (op + 1 /* token */ - + ((lastRunSize + 240) / 255) /* litLength */ - + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend - op) - 1; - lastRunSize -= (lastRunSize + 240) / 255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - - *op++ = RUN_MASK << ML_BITS; - for (; accumulator >= 255; accumulator -= 255) - *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_COMPRESSBOUND(*srcSizePtr)) { - /* compression success is guaranteed */ - return LZ4_compress_fast_extState( - state, src, dst, *srcSizePtr, - targetDstSize, 1); + if (targetDstSize >= + LZ4_compressBound( + *srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, + targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic( - &state->internal_donotuse, - src, dst, srcSizePtr, - targetDstSize, byU16); - else - return LZ4_compress_destSize_generic( - &state->internal_donotuse, - src, dst, srcSizePtr, - targetDstSize, tableType); + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, + src, dst, *srcSizePtr, + srcSizePtr, targetDstSize, + fillOutput, byU16, noDict, + noDictIssue, 1); + } else { + tableType_t const addrMode = + ((sizeof(void *) == 4) && + ((uptrval)src > LZ4_DISTANCE_MAX)) ? + byPtr : + byU32; + return LZ4_compress_generic(&state->internal_donotuse, + src, dst, *srcSizePtr, + srcSizePtr, targetDstSize, + fillOutput, addrMode, + noDict, noDictIssue, 1); + } } } - -int LZ4_compress_destSize( - const char *src, - char *dst, - int *srcSizePtr, - int targetDstSize, - void *wrkmem) +int LZ4_compress_destSize(const char *src, char *dst, int *srcSizePtr, + int targetDstSize, void *wrkmem) { return LZ4_compress_destSize_extState(wrkmem, src, dst, srcSizePtr, - targetDstSize); + targetDstSize); } EXPORT_SYMBOL(LZ4_compress_destSize); /*-****************************** * Streaming functions ********************************/ -void LZ4_resetStream(LZ4_stream_t *LZ4_stream) +static size_t LZ4_stream_t_alignment(void) { - memset(LZ4_stream, 0, sizeof(LZ4_stream_t)); + typedef struct { + char c; + LZ4_stream_t t; + } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); } -int LZ4_loadDict(LZ4_stream_t *LZ4_dict, - const char *dictionary, int dictSize) +static int LZ4_isAligned(const void *ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment - 1)) == 0; +} + +LZ4_stream_t *LZ4_initStream(void *buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { + return NULL; + } + if (size < sizeof(LZ4_stream_t)) { + return NULL; + } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) + return NULL; + memset(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t *)buffer; +} + +void LZ4_resetStream(LZ4_stream_t *LZ4_stream) +{ + memset(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +int LZ4_loadDict(LZ4_stream_t *LZ4_dict, const char *dictionary, int dictSize) { LZ4_stream_t_internal *dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE *p = (const BYTE *)dictionary; - const BYTE * const dictEnd = p + dictSize; + const BYTE *const dictEnd = p + dictSize; const BYTE *base; - if ((dict->initCheck) - || (dict->currentOffset > 1 * GB)) { - /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - } + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, + dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 * KB; if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; return 0; } if ((dictEnd - p) > 64 * KB) p = dictEnd - 64 * KB; - dict->currentOffset += 64 * KB; - base = p - dict->currentOffset; + base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + dict->tableType = (U32)tableType; while (p <= dictEnd - HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); + LZ4_putPosition(p, dict->hashTable, tableType, base); p += 3; } - return dict->dictSize; + return (int)dict->dictSize; } EXPORT_SYMBOL(LZ4_loadDict); -static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict, - const BYTE *src) +static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { - /* address space overflow */ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > + 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 * KB; const BYTE *dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - + DEBUGLOG(4, "LZ4_renormDictT"); for (i = 0; i < LZ4_HASH_SIZE_U32; i++) { if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i] = 0; @@ -835,17 +1050,26 @@ static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict, int LZ4_saveDict(LZ4_stream_t *LZ4_dict, char *safeBuffer, int dictSize) { - LZ4_stream_t_internal * const dict = &LZ4_dict->internal_donotuse; - const BYTE * const previousDictEnd = dict->dictionary + dict->dictSize; + LZ4_stream_t_internal *const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, + safeBuffer); if ((U32)dictSize > 64 * KB) { - /* useless to define a dictionary > 64 * KB */ dictSize = 64 * KB; + } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { + dictSize = (int)dict->dictSize; } - if ((U32)dictSize > dict->dictSize) - dictSize = dict->dictSize; - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + if (safeBuffer == NULL) + assert(dictSize == 0); + if (dictSize > 0) { + const BYTE *const previousDictEnd = + dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + } dict->dictionary = (const BYTE *)safeBuffer; dict->dictSize = (U32)dictSize; @@ -855,82 +1079,121 @@ int LZ4_saveDict(LZ4_stream_t *LZ4_dict, char *safeBuffer, int dictSize) EXPORT_SYMBOL(LZ4_saveDict); int LZ4_compress_fast_continue(LZ4_stream_t *LZ4_stream, const char *source, - char *dest, int inputSize, int maxOutputSize, int acceleration) + char *dest, int inputSize, int maxOutputSize, + int acceleration) { - LZ4_stream_t_internal *streamPtr = &LZ4_stream->internal_donotuse; - const BYTE * const dictEnd = streamPtr->dictionary - + streamPtr->dictSize; + const tableType_t tableType = byU32; + LZ4_stream_t_internal *const streamPtr = &LZ4_stream->internal_donotuse; + const char *dictEnd = streamPtr->dictSize ? + (const char *)streamPtr->dictionary + + streamPtr->dictSize : + NULL; - const BYTE *smallest = (const BYTE *) source; - - if (streamPtr->initCheck) { - /* Uninitialized structure detected */ - return 0; - } - - if ((streamPtr->dictSize > 0) && (smallest > dictEnd)) - smallest = dictEnd; - - LZ4_renormDictT(streamPtr, smallest); + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", + inputSize, streamPtr->dictSize); + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) + acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ((streamPtr->dictSize < + 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && + (inputSize > + 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG( + 5, + "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", + streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE *)source; + dictEnd = source; + } /* Check overlapping input/dictionary space */ { - const BYTE *sourceEnd = (const BYTE *) source + inputSize; - - if ((sourceEnd > streamPtr->dictionary) - && (sourceEnd < dictEnd)) { + const char *const sourceEnd = source + inputSize; + if ((sourceEnd > (const char *)streamPtr->dictionary) && + (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 * KB) streamPtr->dictSize = 64 * KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; + streamPtr->dictionary = + (const BYTE *)dictEnd - streamPtr->dictSize; } } /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE *)source) { - int result; - + if (dictEnd == source) { if ((streamPtr->dictSize < 64 * KB) && - (streamPtr->dictSize < streamPtr->currentOffset)) { - result = LZ4_compress_generic( - streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, + (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic( + streamPtr, source, dest, inputSize, NULL, + maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); - } else { - result = LZ4_compress_generic( - streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, + else + return LZ4_compress_generic( + streamPtr, source, dest, inputSize, NULL, + maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); - } - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; } /* external dictionary mode */ { int result; - - if ((streamPtr->dictSize < 64 * KB) && - (streamPtr->dictSize < streamPtr->currentOffset)) { - result = LZ4_compress_generic( - streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - usingExtDict, dictSmall, acceleration); - } else { - result = LZ4_compress_generic( - streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - usingExtDict, noDictIssue, acceleration); + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 * KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, + sizeof(*streamPtr)); + result = LZ4_compress_generic( + streamPtr, source, dest, inputSize, + NULL, maxOutputSize, limitedOutput, + tableType, usingExtDict, noDictIssue, + acceleration); + } else { + result = LZ4_compress_generic( + streamPtr, source, dest, inputSize, + NULL, maxOutputSize, limitedOutput, + tableType, usingDictCtx, noDictIssue, + acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 * KB) && + (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic( + streamPtr, source, dest, inputSize, + NULL, maxOutputSize, limitedOutput, + tableType, usingExtDict, dictSmall, + acceleration); + } else { + result = LZ4_compress_generic( + streamPtr, source, dest, inputSize, + NULL, maxOutputSize, limitedOutput, + tableType, usingExtDict, noDictIssue, + acceleration); + } } streamPtr->dictionary = (const BYTE *)source; streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; return result; } } diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 71a43e428628..e9eee34de7af 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -44,12 +44,134 @@ * Decompression functions *******************************/ -#define DEBUGLOG(l, ...) {} /* disabled */ +#define LZ4_FAST_DEC_LOOP 1 -#ifndef assert -#define assert(condition) ((void)0) +static const unsigned inc32table[8] = { 0, 1, 2, 1, 0, 4, 4, 4 }; +static const int dec64table[8] = { 0, 0, 0, -1, -4, 1, 2, 3 }; + +#if LZ4_FAST_DEC_LOOP + +static FORCE_INLINE void LZ4_memcpy_using_offset_base(BYTE *dstPtr, + const BYTE *srcPtr, + BYTE *dstEnd, + const size_t offset) +{ + assert(srcPtr + offset == dstPtr); + if (offset < 8) { + LZ4_write32(dstPtr, + 0); /* silence an msan warning when offset==0 */ + dstPtr[0] = srcPtr[0]; + dstPtr[1] = srcPtr[1]; + dstPtr[2] = srcPtr[2]; + dstPtr[3] = srcPtr[3]; + srcPtr += inc32table[offset]; + LZ4_memcpy(dstPtr + 4, srcPtr, 4); + srcPtr -= dec64table[offset]; + dstPtr += 8; + } else { + LZ4_memcpy(dstPtr, srcPtr, 8); + dstPtr += 8; + srcPtr += 8; + } + + LZ4_wildCopy8(dstPtr, srcPtr, dstEnd); +} + +/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd + * this version copies two times 16 bytes (instead of one time 32 bytes) + * because it must be compatible with offsets >= 16. */ +static FORCE_INLINE void LZ4_wildCopy32(void *dstPtr, const void *srcPtr, + void *dstEnd) +{ + BYTE *d = (BYTE *)dstPtr; + const BYTE *s = (const BYTE *)srcPtr; + BYTE *const e = (BYTE *)dstEnd; + + do { + LZ4_memcpy(d, s, 16); + LZ4_memcpy(d + 16, s + 16, 16); + d += 32; + s += 32; + } while (d < e); +} + +/* LZ4_memcpy_using_offset() presumes : + * - dstEnd >= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +static FORCE_INLINE void LZ4_memcpy_using_offset(BYTE *dstPtr, + const BYTE *srcPtr, + BYTE *dstEnd, + const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch (offset) { + case 1: + memset(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); + LZ4_memcpy(&v[4], v, 4); + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} #endif +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { + loop_error = -2, + initial_error = -1, + ok = 0 +} variable_length_error; +static FORCE_INLINE unsigned +read_variable_length(const BYTE **ip, const BYTE *lencheck, int loop_check, + int initial_check, variable_length_error *error) +{ + U32 length = 0; + U32 s; + if (initial_check && + unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && + unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s == 255); + + return length; +} + /* * LZ4_decompress_generic() : * This generic decompression function covers all use cases. @@ -57,419 +179,643 @@ * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ -static FORCE_INLINE int LZ4_decompress_generic( - const char * const src, - char * const dst, - int srcSize, - /* +static FORCE_INLINE int +LZ4_decompress_generic(const char *const src, char *const dst, int srcSize, + /* * If endOnInput == endOnInputSize, * this value is `dstCapacity` */ - int outputSize, - /* endOnOutputSize, endOnInputSize */ - endCondition_directive endOnInput, - /* full, partial */ - earlyEnd_directive partialDecoding, - /* noDict, withPrefix64k, usingExtDict */ - dict_directive dict, - /* always <= dst, == dst when no prefix */ - const BYTE * const lowPrefix, - /* only if dict == usingExtDict */ - const BYTE * const dictStart, - /* note : = 0 if noDict */ - const size_t dictSize - ) + int outputSize, + /* endOnOutputSize, endOnInputSize */ + endCondition_directive endOnInput, + /* full, partial */ + earlyEnd_directive partialDecoding, + /* noDict, withPrefix64k, usingExtDict */ + dict_directive dict, + /* always <= dst, == dst when no prefix */ + const BYTE *const lowPrefix, + /* only if dict == usingExtDict */ + const BYTE *const dictStart, + /* note : = 0 if noDict */ + const size_t dictSize) { - const BYTE *ip = (const BYTE *) src; - const BYTE * const iend = ip + srcSize; - - BYTE *op = (BYTE *) dst; - BYTE * const oend = op + outputSize; - BYTE *cpy; - - const BYTE * const dictEnd = (const BYTE *)dictStart + dictSize; - static const unsigned int inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; - - const int safeDecode = (endOnInput == endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 * KB))); - - /* Set up the "end" pointers for the shortcut. */ - const BYTE *const shortiend = iend - - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; - const BYTE *const shortoend = oend - - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - - DEBUGLOG(5, "%s (srcSize:%i, dstSize:%i)", __func__, - srcSize, outputSize); - - /* Special cases */ - assert(lowPrefix <= op); - assert(src != NULL); - - /* Empty output buffer */ - if ((endOnInput) && (unlikely(outputSize == 0))) - return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; - - if ((!endOnInput) && (unlikely(outputSize == 0))) - return (*ip == 0 ? 1 : -1); - - if ((endOnInput) && unlikely(srcSize == 0)) + if ((src == NULL) || (outputSize < 0)) { return -1; + } + + { + const BYTE *ip = (const BYTE *)src; + const BYTE *const iend = ip + srcSize; + + BYTE *op = (BYTE *)dst; + BYTE *const oend = op + outputSize; + BYTE *cpy; + + const BYTE *const dictEnd = + (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int safeDecode = (endOnInput == endOnInputSize); + const int checkOffset = + ((safeDecode) && (dictSize < (int)(64 * KB))); + + /* Set up the "end" pointers for the shortcut. */ + const BYTE *const shortiend = + iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE *const shortoend = + oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; - /* Main Loop : decode sequences */ - while (1) { - size_t length; const BYTE *match; size_t offset; + unsigned token; + size_t length; - /* get literal length */ - unsigned int const token = *ip++; - length = token>>ML_BITS; + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", + srcSize, outputSize); - /* ip < iend before the increment */ - assert(!endOnInput || ip <= iend); + /* Special cases */ + assert(lowPrefix <= op); + if ((endOnInput) && (unlikely(outputSize == 0))) { + /* Empty output buffer */ + if (partialDecoding) + return 0; + return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; + } + if ((!endOnInput) && (unlikely(outputSize == 0))) { + return (*ip == 0 ? 1 : -1); + } + if ((endOnInput) && unlikely(srcSize == 0)) { + return -1; + } - /* - * A two-stage shortcut for the most common case: - * 1) If the literal length is 0..14, and there is enough - * space, enter the shortcut and copy 16 bytes on behalf - * of the literals (in the fast mode, only 8 bytes can be - * safely copied this way). - * 2) Further if the match length is 4..18, copy 18 bytes - * in a similar manner; but we ensure that there's enough - * space in the output for those 18 bytes earlier, upon - * entering the shortcut (in other words, there is a - * combined check for both stages). - */ - if ((endOnInput ? length != RUN_MASK : length <= 8) - /* - * strictly "less than" on input, to re-enter - * the loop with at least one byte - */ - && likely((endOnInput ? ip < shortiend : 1) & - (op <= shortoend))) { - /* Copy the literals */ - LZ4_memcpy(op, ip, endOnInput ? 16 : 8); - op += length; ip += length; + /* Currently the fast loop shows a regression on qualcomm arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "skip fast decode loop"); + goto safe_decode; + } - /* - * The second stage: - * prepare for match copying, decode full info. - * If it doesn't work out, the info won't be wasted. - */ - length = token & ML_MASK; /* match length */ + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + if (endOnInput) { + assert(ip < iend); + } + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || + ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length( + &ip, iend - RUN_MASK, (int)endOnInput, + (int)endOnInput, &error); + if (error == initial_error) { + goto _output_error; + } + if ((safeDecode) && + unlikely((uptrval)(op) + length < + (uptrval)(op))) { + goto _output_error; + } /* overflow detection */ + if ((safeDecode) && + unlikely((uptrval)(ip) + length < + (uptrval)(ip))) { + goto _output_error; + } /* overflow detection */ + + /* copy literals */ + cpy = op + length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (endOnInput) { /* LZ4_decompress_safe() */ + if ((cpy > oend - 32) || + (ip + length > iend - 32)) { + goto safe_literal_copy; + } + LZ4_wildCopy32(op, ip, cpy); + } else { /* LZ4_decompress_fast() */ + if (cpy > oend - 8) { + goto safe_literal_copy; + } + LZ4_wildCopy8( + op, ip, + cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and only relies on end-of-block properties */ + } + ip += length; + op = cpy; + } else { + cpy = op + length; + if (endOnInput) { /* LZ4_decompress_safe() */ + DEBUGLOG( + 7, + "copy %u bytes in a 16-bytes stripe", + (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > + iend - (16 + + 1 /*max lit + offset + nextToken*/)) { + goto safe_literal_copy; + } + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + LZ4_memcpy(op, ip, 16); + } else { /* LZ4_decompress_fast() */ + /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and relies on end-of-block properties */ + LZ4_memcpy(op, ip, 8); + if (length > 8) { + LZ4_memcpy(op + 8, ip + 8, 8); + } + } + ip += length; + op = cpy; + } + + /* get offset */ offset = LZ4_readLE16(ip); ip += 2; match = op - offset; - assert(match <= op); /* check overflow */ + assert(match <= op); - /* Do not deal with overlapping matches. */ - if ((length != ML_MASK) && - (offset >= 8) && - (dict == withPrefix64k || match >= lowPrefix)) { - /* Copy the match. */ - LZ4_memcpy(op + 0, match + 0, 8); - LZ4_memcpy(op + 8, match + 8, 8); - LZ4_memcpy(op + 16, match + 16, 2); - op += length + MINMATCH; - /* Both stages worked, load the next token. */ + /* get matchlength */ + length = token & ML_MASK; + + if (length == ML_MASK) { + variable_length_error error = ok; + if ((checkOffset) && + (unlikely(match + dictSize < lowPrefix))) { + goto _output_error; + } /* Error : offset outside buffers */ + length += read_variable_length( + &ip, iend - LASTLITERALS + 1, + (int)endOnInput, 0, &error); + if (error != ok) { + goto _output_error; + } + if ((safeDecode) && + unlikely((uptrval)(op) + length < + (uptrval)op)) { + goto _output_error; + } /* overflow detection */ + length += MINMATCH; + if (op + length >= + oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= + oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if ((dict == withPrefix64k) || + (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op + 8, match + 8, + 8); + LZ4_memcpy(op + 16, match + 16, + 2); + op += length; + continue; + } + } + } + + if (checkOffset && + (unlikely(match + dictSize < lowPrefix))) { + goto _output_error; + } /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict == usingExtDict) && (match < lowPrefix)) { + if (unlikely(op + length > + oend - LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG( + 7, + "partialDecoding: dictionary match, close to dstEnd"); + length = min( + length, + (size_t)(oend - op)); + } else { + goto _output_error; /* end-of-block condition violated */ + } + } + + if (length <= (size_t)(lowPrefix - match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, + dictEnd - + (lowPrefix - match), + length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = + (size_t)(lowPrefix - match); + size_t const restSize = + length - copySize; + LZ4_memcpy(op, dictEnd - copySize, + copySize); + op += copySize; + if (restSize > + (size_t)(op - + lowPrefix)) { /* overlap copy */ + BYTE *const endOfMatch = + op + restSize; + const BYTE *copyFrom = + lowPrefix; + while (op < endOfMatch) { + *op++ = *copyFrom++; + } + } else { + LZ4_memcpy(op, lowPrefix, + restSize); + op += restSize; + } + } continue; } - /* - * The second stage didn't work out, but the info - * is ready. Propel it right to the point of match - * copying. - */ - goto _copy_match; + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend - op >= 32)); + if (unlikely(offset < 16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ } +safe_decode: +#endif - /* decode literal length */ - if (length == RUN_MASK) { - unsigned int s; + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ - if (unlikely(endOnInput ? ip >= iend - RUN_MASK : 0)) { - /* overflow detection */ - goto _output_error; - } - do { - s = *ip++; - length += s; - } while (likely(endOnInput - ? ip < iend - RUN_MASK - : 1) & (s == 255)); + assert(!endOnInput || + ip <= iend); /* ip < iend before the increment */ - if ((safeDecode) - && unlikely((uptrval)(op) + - length < (uptrval)(op))) { - /* overflow detection */ - goto _output_error; - } - if ((safeDecode) - && unlikely((uptrval)(ip) + - length < (uptrval)(ip))) { - /* overflow detection */ - goto _output_error; - } - } + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ((endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & + (op <= shortoend))) { + /* Copy the literals */ + LZ4_memcpy(op, ip, endOnInput ? 16 : 8); + op += length; + ip += length; - /* copy literals */ - cpy = op + length; - LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); + ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ - if (((endOnInput) && ((cpy > oend - MFLIMIT) - || (ip + length > iend - (2 + 1 + LASTLITERALS)))) - || ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) { - if (partialDecoding) { - if (cpy > oend) { - /* - * Partial decoding : - * stop in the middle of literal segment - */ - cpy = oend; - length = oend - op; + /* Do not deal with overlapping matches. */ + if ((length != ML_MASK) && (offset >= 8) && + (dict == withPrefix64k || + match >= lowPrefix)) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op + 16, match + 16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; } - if ((endOnInput) - && (ip + length > iend)) { - /* - * Error : - * read attempt beyond - * end of input buffer - */ + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length( + &ip, iend - RUN_MASK, (int)endOnInput, + (int)endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && + unlikely((uptrval)(op) + length < + (uptrval)(op))) { + goto _output_error; + } /* overflow detection */ + if ((safeDecode) && + unlikely((uptrval)(ip) + length < + (uptrval)(ip))) { + goto _output_error; + } /* overflow detection */ + } + + /* copy literals */ + cpy = op + length; +#if LZ4_FAST_DEC_LOOP +safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (((endOnInput) && + ((cpy > oend - MFLIMIT) || + (ip + length > iend - (2 + 1 + LASTLITERALS)))) || + ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + DEBUGLOG( + 7, + "partialDecoding: copying literals, close to input or output end") + DEBUGLOG( + 7, + "partialDecoding: literal length = %u", + (unsigned)length); + DEBUGLOG( + 7, + "partialDecoding: remaining space in dstBuffer : %i", + (int)(oend - op)); + DEBUGLOG( + 7, + "partialDecoding: remaining space in srcBuffer : %i", + (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip + length > iend) { + length = (size_t)(iend - ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op <= oend); + length = (size_t)(oend - op); + } + } else { + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { + goto _output_error; + } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && + ((ip + length != iend) || + (cpy > oend))) { + DEBUGLOG( + 6, + "should have been last run of literals") + DEBUGLOG( + 6, + "ip(%p) + length(%i) = %p != iend (%p)", + ip, (int)length, + ip + length, iend); + DEBUGLOG( + 6, + "or cpy(%p) > oend(%p)", + cpy, oend); + goto _output_error; + } + } + LZ4_memmove( + op, ip, + length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || + (ip >= (iend - 2))) { + break; + } } else { - if ((!endOnInput) - && (cpy != oend)) { - /* - * Error : - * block decoding must - * stop exactly there - */ - goto _output_error; - } - if ((endOnInput) - && ((ip + length != iend) - || (cpy > oend))) { - /* - * Error : - * input must be consumed - */ - goto _output_error; - } + LZ4_wildCopy8( + op, ip, + cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; + op = cpy; } - LZ4_memcpy(op, ip, length); - ip += length; - op += length; + /* get offset */ + offset = LZ4_readLE16(ip); + ip += 2; + match = op - offset; - /* Necessarily EOF, due to parsing restrictions */ - if (!partialDecoding || (cpy == oend)) - break; - } else { - /* may overwrite up to WILDCOPYLENGTH beyond cpy */ - LZ4_wildCopy(op, ip, cpy); - ip += length; - op = cpy; - } - - /* get offset */ - offset = LZ4_readLE16(ip); - ip += 2; - match = op - offset; - - /* get matchlength */ - length = token & ML_MASK; + /* get matchlength */ + length = token & ML_MASK; _copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { - /* Error : offset outside buffers */ - goto _output_error; - } - - /* costs ~1%; silence an msan warning when offset == 0 */ - /* - * note : when partialDecoding, there is no guarantee that - * at least 4 bytes remain available in output buffer - */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - - LZ4_write32(op, (U32)offset); - } - - if (length == ML_MASK) { - unsigned int s; - - do { - s = *ip++; - - if ((endOnInput) && (ip > iend - LASTLITERALS)) + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length( + &ip, iend - LASTLITERALS + 1, + (int)endOnInput, 0, &error); + if (error != ok) goto _output_error; - - length += s; - } while (s == 255); - - if ((safeDecode) - && unlikely( - (uptrval)(op) + length < (uptrval)op)) { - /* overflow detection */ - goto _output_error; + if ((safeDecode) && + unlikely((uptrval)(op) + length < + (uptrval)op)) + goto _output_error; /* overflow detection */ } - } + length += MINMATCH; - length += MINMATCH; +#if LZ4_FAST_DEC_LOOP +safe_match_copy: +#endif + if ((checkOffset) && + (unlikely(match + dictSize < lowPrefix))) + goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict == usingExtDict) && (match < lowPrefix)) { + if (unlikely(op + length > + oend - LASTLITERALS)) { + if (partialDecoding) + length = min( + length, + (size_t)(oend - op)); + else + goto _output_error; /* doesn't respect parsing restriction */ + } - /* match starting within external dictionary */ - if ((dict == usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - LASTLITERALS)) { - /* doesn't respect parsing restriction */ - if (!partialDecoding) - goto _output_error; - length = min(length, (size_t)(oend - op)); - } - - if (length <= (size_t)(lowPrefix - match)) { - /* - * match fits entirely within external - * dictionary : just copy - */ - memmove(op, dictEnd - (lowPrefix - match), - length); - op += length; - } else { - /* - * match stretches into both external - * dictionary and current block - */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - - LZ4_memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { - /* overlap copy */ - BYTE * const endOfMatch = op + restSize; - const BYTE *copyFrom = lowPrefix; - - while (op < endOfMatch) - *op++ = *copyFrom++; + if (length <= (size_t)(lowPrefix - match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, + dictEnd - + (lowPrefix - match), + length); + op += length; } else { - LZ4_memcpy(op, lowPrefix, restSize); - op += restSize; + /* match stretches into both external dictionary and current block */ + size_t const copySize = + (size_t)(lowPrefix - match); + size_t const restSize = + length - copySize; + LZ4_memcpy(op, dictEnd - copySize, + copySize); + op += copySize; + if (restSize > + (size_t)(op - + lowPrefix)) { /* overlap copy */ + BYTE *const endOfMatch = + op + restSize; + const BYTE *copyFrom = + lowPrefix; + while (op < endOfMatch) + *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, + restSize); + op += restSize; + } + } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op <= oend); + if (partialDecoding && + (cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = + min(length, (size_t)(oend - op)); + const BYTE *const matchEnd = match + mlen; + BYTE *const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { + *op++ = *match++; + } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { + break; + } + continue; + } + + if (unlikely(offset < 8)) { + LZ4_write32( + op, + 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op + 4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { + BYTE *const oCopyLimit = + oend - (WILDCOPYLENGTH - 1); + if (cpy > oend - LASTLITERALS) { + goto _output_error; + } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { + *op++ = *match++; + } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { + LZ4_wildCopy8(op + 8, match + 8, cpy); } } - continue; + op = cpy; /* wildcopy correction */ } - /* copy match within block */ - cpy = op + length; - - /* - * partialDecoding : - * may not respect endBlock parsing restrictions - */ - assert(op <= oend); - if (partialDecoding && - (cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { - size_t const mlen = min(length, (size_t)(oend - op)); - const BYTE * const matchEnd = match + mlen; - BYTE * const copyEnd = op + mlen; - - if (matchEnd > op) { - /* overlap copy */ - while (op < copyEnd) - *op++ = *match++; - } else { - LZ4_memcpy(op, match, mlen); - } - op = copyEnd; - if (op == oend) - break; - continue; - } - - if (unlikely(offset < 8)) { - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += inc32table[offset]; - LZ4_memcpy(op + 4, match, 4); - match -= dec64table[offset]; + /* end of decoding */ + if (endOnInput) { + DEBUGLOG(5, "decoded %i bytes", + (int)(((char *)op) - dst)); + return (int)(((char *)op) - + dst); /* Nb of output bytes decoded */ } else { - LZ4_copy8(op, match); - match += 8; + return (int)(((const char *)ip) - + src); /* Nb of input bytes read */ } - op += 8; - - if (unlikely(cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { - BYTE * const oCopyLimit = oend - (WILDCOPYLENGTH - 1); - - if (cpy > oend - LASTLITERALS) { - /* - * Error : last LASTLITERALS bytes - * must be literals (uncompressed) - */ - goto _output_error; - } - - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op < cpy) - *op++ = *match++; - } else { - LZ4_copy8(op, match); - if (length > 16) - LZ4_wildCopy(op + 8, match + 8, cpy); - } - op = cpy; /* wildcopy correction */ - } - - /* end of decoding */ - if (endOnInput) { - /* Nb of output bytes decoded */ - return (int) (((char *)op) - dst); - } else { - /* Nb of input bytes read */ - return (int) (((const char *)ip) - src); - } - - /* Overflow error detected */ + /* Overflow error detected */ _output_error: - return (int) (-(((const char *)ip) - src)) - 1; + return (int)(-(((const char *)ip) - src)) - 1; + } } -int LZ4_decompress_safe(const char *source, char *dest, - int compressedSize, int maxDecompressedSize) +int LZ4_decompress_safe(const char *source, char *dest, int compressedSize, + int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, - compressedSize, maxDecompressedSize, - endOnInputSize, decode_full_block, - noDict, (BYTE *)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, + maxDecompressedSize, endOnInputSize, + decode_full_block, noDict, (BYTE *)dest, + NULL, 0); } -int LZ4_decompress_safe_partial(const char *src, char *dst, - int compressedSize, int targetOutputSize, int dstCapacity) +int LZ4_decompress_safe_partial(const char *src, char *dst, int compressedSize, + int targetOutputSize, int dstCapacity) { dstCapacity = min(targetOutputSize, dstCapacity); return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, - endOnInputSize, partial_decode, - noDict, (BYTE *)dst, NULL, 0); + endOnInputSize, partial_decode, noDict, + (BYTE *)dst, NULL, 0); } int LZ4_decompress_fast(const char *source, char *dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, - withPrefix64k, - (BYTE *)dest - 64 * KB, NULL, 0); + withPrefix64k, (BYTE *)dest - 64 * KB, + NULL, 0); } /* ===== Instantiate a few more decoding cases, used more than once. ===== */ @@ -477,10 +823,9 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize) static int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, int compressedSize, int maxOutputSize) { - return LZ4_decompress_generic(source, dest, - compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, - withPrefix64k, + return LZ4_decompress_generic(source, dest, compressedSize, + maxOutputSize, endOnInputSize, + decode_full_block, withPrefix64k, (BYTE *)dest - 64 * KB, NULL, 0); } @@ -489,10 +834,9 @@ static int LZ4_decompress_safe_withSmallPrefix(const char *source, char *dest, int maxOutputSize, size_t prefixSize) { - return LZ4_decompress_generic(source, dest, - compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, - noDict, + return LZ4_decompress_generic(source, dest, compressedSize, + maxOutputSize, endOnInputSize, + decode_full_block, noDict, (BYTE *)dest - prefixSize, NULL, 0); } @@ -500,19 +844,18 @@ int LZ4_decompress_safe_forceExtDict(const char *source, char *dest, int compressedSize, int maxOutputSize, const void *dictStart, size_t dictSize) { - return LZ4_decompress_generic(source, dest, - compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, - usingExtDict, (BYTE *)dest, - (const BYTE *)dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, + maxOutputSize, endOnInputSize, + decode_full_block, usingExtDict, + (BYTE *)dest, (const BYTE *)dictStart, + dictSize); } static int LZ4_decompress_fast_extDict(const char *source, char *dest, - int originalSize, - const void *dictStart, size_t dictSize) + int originalSize, const void *dictStart, + size_t dictSize) { - return LZ4_decompress_generic(source, dest, - 0, originalSize, + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, usingExtDict, (BYTE *)dest, (const BYTE *)dictStart, dictSize); @@ -523,26 +866,23 @@ static int LZ4_decompress_fast_extDict(const char *source, char *dest, * of the dictionary is passed as prefix, and the second via dictStart + dictSize. * These routines are used only once, in LZ4_decompress_*_continue(). */ -static FORCE_INLINE -int LZ4_decompress_safe_doubleDict(const char *source, char *dest, - int compressedSize, int maxOutputSize, - size_t prefixSize, - const void *dictStart, size_t dictSize) +static FORCE_INLINE int LZ4_decompress_safe_doubleDict( + const char *source, char *dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void *dictStart, size_t dictSize) { - return LZ4_decompress_generic(source, dest, - compressedSize, maxOutputSize, - endOnInputSize, decode_full_block, - usingExtDict, (BYTE *)dest - prefixSize, + return LZ4_decompress_generic(source, dest, compressedSize, + maxOutputSize, endOnInputSize, + decode_full_block, usingExtDict, + (BYTE *)dest - prefixSize, (const BYTE *)dictStart, dictSize); } -static FORCE_INLINE -int LZ4_decompress_fast_doubleDict(const char *source, char *dest, - int originalSize, size_t prefixSize, - const void *dictStart, size_t dictSize) +static FORCE_INLINE int +LZ4_decompress_fast_doubleDict(const char *source, char *dest, int originalSize, + size_t prefixSize, const void *dictStart, + size_t dictSize) { - return LZ4_decompress_generic(source, dest, - 0, originalSize, + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, decode_full_block, usingExtDict, (BYTE *)dest - prefixSize, (const BYTE *)dictStart, dictSize); @@ -551,15 +891,15 @@ int LZ4_decompress_fast_doubleDict(const char *source, char *dest, /* ===== streaming decompression functions ===== */ int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, - const char *dictionary, int dictSize) + const char *dictionary, int dictSize) { LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE *) dictionary + dictSize; + lz4sd->prefixSize = (size_t)dictSize; + lz4sd->prefixEnd = (const BYTE *)dictionary + dictSize; lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; + lz4sd->extDictSize = 0; return 1; } @@ -574,7 +914,8 @@ int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, * and indicate where it stands using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, - const char *source, char *dest, int compressedSize, int maxOutputSize) + const char *source, char *dest, + int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -583,85 +924,86 @@ int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, if (lz4sd->prefixSize == 0) { /* The first call, no dictionary yet. */ assert(lz4sd->extDictSize == 0); - result = LZ4_decompress_safe(source, dest, - compressedSize, maxOutputSize); + result = LZ4_decompress_safe(source, dest, compressedSize, + maxOutputSize); if (result <= 0) return result; - lz4sd->prefixSize = result; + lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE *)dest + result; } else if (lz4sd->prefixEnd == (BYTE *)dest) { /* They're rolling the current segment. */ if (lz4sd->prefixSize >= 64 * KB - 1) - result = LZ4_decompress_safe_withPrefix64k(source, dest, - compressedSize, maxOutputSize); + result = LZ4_decompress_safe_withPrefix64k( + source, dest, compressedSize, maxOutputSize); else if (lz4sd->extDictSize == 0) - result = LZ4_decompress_safe_withSmallPrefix(source, - dest, compressedSize, maxOutputSize, + result = LZ4_decompress_safe_withSmallPrefix( + source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize); else - result = LZ4_decompress_safe_doubleDict(source, dest, - compressedSize, maxOutputSize, - lz4sd->prefixSize, - lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_safe_doubleDict( + source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, + lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; } else { - /* - * The buffer wraps around, or they're - * switching to another buffer. - */ + /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_safe_forceExtDict(source, dest, - compressedSize, maxOutputSize, + result = LZ4_decompress_safe_forceExtDict( + source, dest, compressedSize, maxOutputSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE *)dest + result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE *)dest + result; } return result; } int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, - const char *source, char *dest, int originalSize) + const char *source, char *dest, + int originalSize) { - LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse; + LZ4_streamDecode_t_internal *lz4sd = + &LZ4_streamDecode->internal_donotuse; int result; + assert(originalSize >= 0); if (lz4sd->prefixSize == 0) { assert(lz4sd->extDictSize == 0); result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE *)dest + originalSize; } else if (lz4sd->prefixEnd == (BYTE *)dest) { - if (lz4sd->prefixSize >= 64 * KB - 1 || - lz4sd->extDictSize == 0) - result = LZ4_decompress_fast(source, dest, - originalSize); + if (lz4sd->prefixSize >= 64 * KB - 1 || lz4sd->extDictSize == 0) + result = + LZ4_decompress_fast(source, dest, originalSize); else - result = LZ4_decompress_fast_doubleDict(source, dest, - originalSize, lz4sd->prefixSize, + result = LZ4_decompress_fast_doubleDict( + source, dest, originalSize, lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_fast_extDict(source, dest, - originalSize, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, + lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE *)dest + originalSize; } + return result; } @@ -670,28 +1012,29 @@ int LZ4_decompress_safe_usingDict(const char *source, char *dest, const char *dictStart, int dictSize) { if (dictSize == 0) - return LZ4_decompress_safe(source, dest, - compressedSize, maxOutputSize); - if (dictStart+dictSize == dest) { + return LZ4_decompress_safe(source, dest, compressedSize, + maxOutputSize); + if (dictStart + dictSize == dest) { if (dictSize >= 64 * KB - 1) - return LZ4_decompress_safe_withPrefix64k(source, dest, - compressedSize, maxOutputSize); - return LZ4_decompress_safe_withSmallPrefix(source, dest, - compressedSize, maxOutputSize, dictSize); + return LZ4_decompress_safe_withPrefix64k( + source, dest, compressedSize, maxOutputSize); + return LZ4_decompress_safe_withSmallPrefix( + source, dest, compressedSize, maxOutputSize, dictSize); } - return LZ4_decompress_safe_forceExtDict(source, dest, - compressedSize, maxOutputSize, dictStart, dictSize); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, + maxOutputSize, dictStart, + dictSize); } int LZ4_decompress_fast_usingDict(const char *source, char *dest, - int originalSize, - const char *dictStart, int dictSize) + int originalSize, const char *dictStart, + int dictSize) { if (dictSize == 0 || dictStart + dictSize == dest) return LZ4_decompress_fast(source, dest, originalSize); return LZ4_decompress_fast_extDict(source, dest, originalSize, - dictStart, dictSize); + dictStart, dictSize); } #ifndef STATIC diff --git a/lib/lz4/lz4defs.h b/lib/lz4/lz4defs.h index c91dd96ef629..f7fa464d9958 100644 --- a/lib/lz4/lz4defs.h +++ b/lib/lz4/lz4defs.h @@ -36,7 +36,7 @@ */ #include -#include /* memset, memcpy */ +#include /* memset, memcpy */ #define FORCE_INLINE __always_inline @@ -45,10 +45,10 @@ **************************************/ #include -typedef uint8_t BYTE; +typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; -typedef int32_t S32; +typedef int32_t S32; typedef uint64_t U64; typedef uintptr_t uptrval; @@ -67,22 +67,28 @@ typedef uintptr_t uptrval; #define LZ4_LITTLE_ENDIAN 0 #endif +#define DEBUGLOG(l, ...) \ + { \ + } /* disabled */ + +#ifndef assert +#define assert(condition) ((void)0) +#endif + /*-************************************ * Constants **************************************/ +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#define LZ4_DISTANCE_MAX 65535 #define MINMATCH 4 #define WILDCOPYLENGTH 8 -#define LASTLITERALS 5 -#define MFLIMIT (WILDCOPYLENGTH + MINMATCH) -/* - * ensure it's possible to write 2 x wildcopyLength - * without overflowing output buffer - */ -#define MATCH_SAFEGUARD_DISTANCE ((2 * WILDCOPYLENGTH) - MINMATCH) - -/* Increase this value ==> compression run slower on incompressible data */ -#define LZ4_SKIPTRIGGER 6 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE \ + ((2 * WILDCOPYLENGTH) - \ + MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 #define HASH_UNIT sizeof(size_t) @@ -90,12 +96,16 @@ typedef uintptr_t uptrval; #define MB (1 << 20) #define GB (1U << 30) -#define MAXD_LOG 16 -#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) +#if defined(__x86_64__) +typedef U64 reg_t; /* 64-bits in x32 mode */ +#else +typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + #define STEPSIZE sizeof(size_t) -#define ML_BITS 4 -#define ML_MASK ((1U << ML_BITS) - 1) +#define ML_BITS 4 +#define ML_MASK ((1U << ML_BITS) - 1) #define RUN_BITS (8 - ML_BITS) #define RUN_MASK ((1U << RUN_BITS) - 1) @@ -146,41 +156,24 @@ static FORCE_INLINE void LZ4_writeLE16(void *memPtr, U16 value) * environments. This is needed when decompressing the Linux Kernel, for example. */ #define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +#define LZ4_memmove(dst, src, size) __builtin_memmove(dst, src, size) -static FORCE_INLINE void LZ4_copy8(void *dst, const void *src) -{ -#if LZ4_ARCH64 - U64 a = get_unaligned((const U64 *)src); - - put_unaligned(a, (U64 *)dst); -#else - U32 a = get_unaligned((const U32 *)src); - U32 b = get_unaligned((const U32 *)src + 1); - - put_unaligned(a, (U32 *)dst); - put_unaligned(b, (U32 *)dst + 1); -#endif -} - -/* - * customized variant of memcpy, - * which can overwrite up to 7 bytes beyond dstEnd - */ -static FORCE_INLINE void LZ4_wildCopy(void *dstPtr, - const void *srcPtr, void *dstEnd) +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +static FORCE_INLINE void LZ4_wildCopy8(void *dstPtr, const void *srcPtr, + void *dstEnd) { BYTE *d = (BYTE *)dstPtr; const BYTE *s = (const BYTE *)srcPtr; BYTE *const e = (BYTE *)dstEnd; do { - LZ4_copy8(d, s); + LZ4_memcpy(d, s, 8); d += 8; s += 8; } while (d < e); } -static FORCE_INLINE unsigned int LZ4_NbCommonBytes(register size_t val) +static FORCE_INLINE unsigned int LZ4_NbCommonBytes(reg_t val) { #if LZ4_LITTLE_ENDIAN return __ffs(val) >> 3; @@ -189,56 +182,64 @@ static FORCE_INLINE unsigned int LZ4_NbCommonBytes(register size_t val) #endif } -static FORCE_INLINE unsigned int LZ4_count( - const BYTE *pIn, - const BYTE *pMatch, - const BYTE *pInLimit) +static FORCE_INLINE unsigned int LZ4_count(const BYTE *pIn, const BYTE *pMatch, + const BYTE *pInLimit) { const BYTE *const pStart = pIn; - while (likely(pIn < pInLimit - (STEPSIZE - 1))) { - size_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (likely(pIn < pInLimit - (STEPSIZE - 1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn += STEPSIZE; + pMatch += STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } + } + while (likely(pIn < pInLimit - (STEPSIZE - 1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); if (!diff) { pIn += STEPSIZE; pMatch += STEPSIZE; continue; } - pIn += LZ4_NbCommonBytes(diff); - - return (unsigned int)(pIn - pStart); + return (unsigned)(pIn - pStart); } -#if LZ4_ARCH64 - if ((pIn < (pInLimit - 3)) - && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { + if ((STEPSIZE == 8) && (pIn < (pInLimit - 3)) && + (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn += 4; pMatch += 4; } -#endif - - if ((pIn < (pInLimit - 1)) - && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { + if ((pIn < (pInLimit - 1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn += 2; pMatch += 2; } - if ((pIn < pInLimit) && (*pMatch == *pIn)) pIn++; - - return (unsigned int)(pIn - pStart); + return (unsigned)(pIn - pStart); } -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { + noDict = 0, + withPrefix64k, + usingExtDict, + usingDictCtx +} dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; -#define LZ4_STATIC_ASSERT(c) BUILD_BUG_ON(!(c)) +#define LZ4_STATIC_ASSERT(c) BUILD_BUG_ON(!(c)) #endif