123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861 |
- #ifndef ZSTD_COMPRESS_H
- #define ZSTD_COMPRESS_H
- #include "zstd_internal.h"
- #ifdef ZSTD_MULTITHREAD
- # include "zstdmt_compress.h"
- #endif
- #if defined (__cplusplus)
- extern "C" {
- #endif
- #define kSearchStrength 8
- #define HASH_READ_SIZE 8
- #define ZSTD_DUBT_UNSORTED_MARK 1
- typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
- typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
- typedef struct ZSTD_prefixDict_s {
- const void* dict;
- size_t dictSize;
- ZSTD_dictContentType_e dictContentType;
- } ZSTD_prefixDict;
- typedef struct {
- U32 CTable[HUF_CTABLE_SIZE_U32(255)];
- HUF_repeat repeatMode;
- } ZSTD_hufCTables_t;
- typedef struct {
- FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
- FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
- FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
- FSE_repeat offcode_repeatMode;
- FSE_repeat matchlength_repeatMode;
- FSE_repeat litlength_repeatMode;
- } ZSTD_fseCTables_t;
- typedef struct {
- ZSTD_hufCTables_t huf;
- ZSTD_fseCTables_t fse;
- } ZSTD_entropyCTables_t;
- typedef struct {
- U32 off;
- U32 len;
- } ZSTD_match_t;
- typedef struct {
- int price;
- U32 off;
- U32 mlen;
- U32 litlen;
- U32 rep[ZSTD_REP_NUM];
- } ZSTD_optimal_t;
- typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
- typedef struct {
-
- unsigned* litFreq;
- unsigned* litLengthFreq;
- unsigned* matchLengthFreq;
- unsigned* offCodeFreq;
- ZSTD_match_t* matchTable;
- ZSTD_optimal_t* priceTable;
- U32 litSum;
- U32 litLengthSum;
- U32 matchLengthSum;
- U32 offCodeSum;
- U32 litSumBasePrice;
- U32 litLengthSumBasePrice;
- U32 matchLengthSumBasePrice;
- U32 offCodeSumBasePrice;
- ZSTD_OptPrice_e priceType;
- const ZSTD_entropyCTables_t* symbolCosts;
- } optState_t;
- typedef struct {
- ZSTD_entropyCTables_t entropy;
- U32 rep[ZSTD_REP_NUM];
- } ZSTD_compressedBlockState_t;
- typedef struct {
- BYTE const* nextSrc;
- BYTE const* base;
- BYTE const* dictBase;
- U32 dictLimit;
- U32 lowLimit;
- } ZSTD_window_t;
- typedef struct ZSTD_matchState_t ZSTD_matchState_t;
- struct ZSTD_matchState_t {
- ZSTD_window_t window;
- U32 loadedDictEnd;
- U32 nextToUpdate;
- U32 nextToUpdate3;
- U32 hashLog3;
- U32* hashTable;
- U32* hashTable3;
- U32* chainTable;
- optState_t opt;
- const ZSTD_matchState_t * dictMatchState;
- ZSTD_compressionParameters cParams;
- };
- typedef struct {
- ZSTD_compressedBlockState_t* prevCBlock;
- ZSTD_compressedBlockState_t* nextCBlock;
- ZSTD_matchState_t matchState;
- } ZSTD_blockState_t;
- typedef struct {
- U32 offset;
- U32 checksum;
- } ldmEntry_t;
- typedef struct {
- ZSTD_window_t window;
- ldmEntry_t* hashTable;
- BYTE* bucketOffsets;
- U64 hashPower;
- } ldmState_t;
- typedef struct {
- U32 enableLdm;
- U32 hashLog;
- U32 bucketSizeLog;
- U32 minMatchLength;
- U32 hashRateLog;
- U32 windowLog;
- } ldmParams_t;
- typedef struct {
- U32 offset;
- U32 litLength;
- U32 matchLength;
- } rawSeq;
- typedef struct {
- rawSeq* seq;
- size_t pos;
- size_t size;
- size_t capacity;
- } rawSeqStore_t;
- struct ZSTD_CCtx_params_s {
- ZSTD_format_e format;
- ZSTD_compressionParameters cParams;
- ZSTD_frameParameters fParams;
- int compressionLevel;
- int forceWindow;
- ZSTD_dictAttachPref_e attachDictPref;
-
- int nbWorkers;
- size_t jobSize;
- int overlapLog;
- int rsyncable;
-
- ldmParams_t ldmParams;
-
- ZSTD_customMem customMem;
- };
- struct ZSTD_CCtx_s {
- ZSTD_compressionStage_e stage;
- int cParamsChanged;
- int bmi2;
- ZSTD_CCtx_params requestedParams;
- ZSTD_CCtx_params appliedParams;
- U32 dictID;
- int workSpaceOversizedDuration;
- void* workSpace;
- size_t workSpaceSize;
- size_t blockSize;
- unsigned long long pledgedSrcSizePlusOne;
- unsigned long long consumedSrcSize;
- unsigned long long producedCSize;
- XXH64_state_t xxhState;
- ZSTD_customMem customMem;
- size_t staticSize;
- seqStore_t seqStore;
- ldmState_t ldmState;
- rawSeq* ldmSequences;
- size_t maxNbLdmSequences;
- rawSeqStore_t externSeqStore;
- ZSTD_blockState_t blockState;
- U32* entropyWorkspace;
-
- char* inBuff;
- size_t inBuffSize;
- size_t inToCompress;
- size_t inBuffPos;
- size_t inBuffTarget;
- char* outBuff;
- size_t outBuffSize;
- size_t outBuffContentSize;
- size_t outBuffFlushedSize;
- ZSTD_cStreamStage streamStage;
- U32 frameEnded;
-
- ZSTD_CDict* cdictLocal;
- const ZSTD_CDict* cdict;
- ZSTD_prefixDict prefixDict;
-
- #ifdef ZSTD_MULTITHREAD
- ZSTDMT_CCtx* mtctx;
- #endif
- };
- typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
- typedef enum { ZSTD_noDict = 0, ZSTD_extDict = 1, ZSTD_dictMatchState = 2 } ZSTD_dictMode_e;
- typedef size_t (*ZSTD_blockCompressor) (
- ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
- void const* src, size_t srcSize);
- ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
- MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
- {
- static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
- 8, 9, 10, 11, 12, 13, 14, 15,
- 16, 16, 17, 17, 18, 18, 19, 19,
- 20, 20, 20, 20, 21, 21, 21, 21,
- 22, 22, 22, 22, 22, 22, 22, 22,
- 23, 23, 23, 23, 23, 23, 23, 23,
- 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24 };
- static const U32 LL_deltaCode = 19;
- return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
- }
- MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
- {
- static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
- 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
- 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
- 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
- 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
- 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
- 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
- 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
- static const U32 ML_deltaCode = 36;
- return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
- }
- MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
- {
- #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
- static const BYTE* g_start = NULL;
- if (g_start==NULL) g_start = (const BYTE*)literals;
- { U32 const pos = (U32)((const BYTE*)literals - g_start);
- DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
- pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
- }
- #endif
- assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
-
- assert(seqStorePtr->maxNbLit <= 128 KB);
- assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
- ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
- seqStorePtr->lit += litLength;
-
- if (litLength>0xFFFF) {
- assert(seqStorePtr->longLengthID == 0);
- seqStorePtr->longLengthID = 1;
- seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
- }
- seqStorePtr->sequences[0].litLength = (U16)litLength;
-
- seqStorePtr->sequences[0].offset = offsetCode + 1;
-
- if (mlBase>0xFFFF) {
- assert(seqStorePtr->longLengthID == 0);
- seqStorePtr->longLengthID = 2;
- seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
- }
- seqStorePtr->sequences[0].matchLength = (U16)mlBase;
- seqStorePtr->sequences++;
- }
- static unsigned ZSTD_NbCommonBytes (size_t val)
- {
- if (MEM_isLittleEndian()) {
- if (MEM_64bits()) {
- # if defined(_MSC_VER) && defined(_WIN64)
- unsigned long r = 0;
- _BitScanForward64( &r, (U64)val );
- return (unsigned)(r>>3);
- # elif defined(__GNUC__) && (__GNUC__ >= 4)
- return (__builtin_ctzll((U64)val) >> 3);
- # else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
- 0, 3, 1, 3, 1, 4, 2, 7,
- 0, 2, 3, 6, 1, 5, 3, 5,
- 1, 3, 4, 4, 2, 5, 6, 7,
- 7, 0, 1, 2, 3, 3, 4, 6,
- 2, 6, 5, 5, 3, 4, 5, 6,
- 7, 1, 2, 4, 6, 4, 4, 5,
- 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
- # endif
- } else {
- # if defined(_MSC_VER)
- unsigned long r=0;
- _BitScanForward( &r, (U32)val );
- return (unsigned)(r>>3);
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_ctz((U32)val) >> 3);
- # else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
- 3, 2, 2, 1, 3, 2, 0, 1,
- 3, 3, 1, 2, 2, 2, 2, 0,
- 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
- # endif
- }
- } else {
- if (MEM_64bits()) {
- # if defined(_MSC_VER) && defined(_WIN64)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (unsigned)(r>>3);
- # elif defined(__GNUC__) && (__GNUC__ >= 4)
- return (__builtin_clzll(val) >> 3);
- # else
- unsigned r;
- const unsigned n32 = sizeof(size_t)*4;
- if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
- # endif
- } else {
- # if defined(_MSC_VER)
- unsigned long r = 0;
- _BitScanReverse( &r, (unsigned long)val );
- return (unsigned)(r>>3);
- # elif defined(__GNUC__) && (__GNUC__ >= 3)
- return (__builtin_clz((U32)val) >> 3);
- # else
- unsigned r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
- # endif
- } }
- }
- MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
- {
- const BYTE* const pStart = pIn;
- const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
- if (pIn < pInLoopLimit) {
- { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
- if (diff) return ZSTD_NbCommonBytes(diff); }
- pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
- while (pIn < pInLoopLimit) {
- size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
- if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
- pIn += ZSTD_NbCommonBytes(diff);
- return (size_t)(pIn - pStart);
- } }
- if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
- if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
- if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
- return (size_t)(pIn - pStart);
- }
- MEM_STATIC size_t
- ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
- const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
- {
- const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
- size_t const matchLength = ZSTD_count(ip, match, vEnd);
- if (match + matchLength != mEnd) return matchLength;
- DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
- DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
- DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
- DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
- DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
- return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
- }
- static const U32 prime3bytes = 506832829U;
- static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
- MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); }
- static const U32 prime4bytes = 2654435761U;
- static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
- static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
- static const U64 prime5bytes = 889523592379ULL;
- static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
- static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
- static const U64 prime6bytes = 227718039650203ULL;
- static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
- static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
- static const U64 prime7bytes = 58295818150454627ULL;
- static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
- static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
- static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
- static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
- static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
- MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
- {
- switch(mls)
- {
- default:
- case 4: return ZSTD_hash4Ptr(p, hBits);
- case 5: return ZSTD_hash5Ptr(p, hBits);
- case 6: return ZSTD_hash6Ptr(p, hBits);
- case 7: return ZSTD_hash7Ptr(p, hBits);
- case 8: return ZSTD_hash8Ptr(p, hBits);
- }
- }
- static U64 ZSTD_ipow(U64 base, U64 exponent)
- {
- U64 power = 1;
- while (exponent) {
- if (exponent & 1) power *= base;
- exponent >>= 1;
- base *= base;
- }
- return power;
- }
- #define ZSTD_ROLL_HASH_CHAR_OFFSET 10
- static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
- {
- BYTE const* istart = (BYTE const*)buf;
- size_t pos;
- for (pos = 0; pos < size; ++pos) {
- hash *= prime8bytes;
- hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
- }
- return hash;
- }
- MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
- {
- return ZSTD_rollingHash_append(0, buf, size);
- }
- MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
- {
- return ZSTD_ipow(prime8bytes, length - 1);
- }
- MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
- {
- hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
- hash *= prime8bytes;
- hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
- return hash;
- }
- #define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
- #define ZSTD_CHUNKSIZE_MAX \
- ( ((U32)-1) \
- - ZSTD_CURRENT_MAX)
- MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
- {
- size_t const endT = (size_t)(window->nextSrc - window->base);
- U32 const end = (U32)endT;
- window->lowLimit = end;
- window->dictLimit = end;
- }
- MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
- {
- return window.lowLimit < window.dictLimit;
- }
- MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
- {
- return ZSTD_window_hasExtDict(ms->window) ?
- ZSTD_extDict :
- ms->dictMatchState != NULL ?
- ZSTD_dictMatchState :
- ZSTD_noDict;
- }
- MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
- void const* srcEnd)
- {
- U32 const current = (U32)((BYTE const*)srcEnd - window.base);
- return current > ZSTD_CURRENT_MAX;
- }
- MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
- U32 maxDist, void const* src)
- {
-
- U32 const cycleMask = (1U << cycleLog) - 1;
- U32 const current = (U32)((BYTE const*)src - window->base);
- U32 const newCurrent = (current & cycleMask) + maxDist;
- U32 const correction = current - newCurrent;
- assert((maxDist & cycleMask) == 0);
- assert(current > newCurrent);
-
- assert(correction > 1<<28);
- window->base += correction;
- window->dictBase += correction;
- window->lowLimit -= correction;
- window->dictLimit -= correction;
- DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction,
- window->lowLimit);
- return correction;
- }
- MEM_STATIC void
- ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
- void const* srcEnd,
- U32 maxDist,
- U32* loadedDictEndPtr,
- const ZSTD_matchState_t** dictMatchStatePtr)
- {
- U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base);
- U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
- DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u",
- (unsigned)blockEndIdx, (unsigned)maxDist);
- if (blockEndIdx > maxDist + loadedDictEnd) {
- U32 const newLowLimit = blockEndIdx - maxDist;
- if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
- if (window->dictLimit < window->lowLimit) {
- DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
- (unsigned)window->dictLimit, (unsigned)window->lowLimit);
- window->dictLimit = window->lowLimit;
- }
- if (loadedDictEndPtr)
- *loadedDictEndPtr = 0;
- if (dictMatchStatePtr)
- *dictMatchStatePtr = NULL;
- }
- }
- MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
- void const* src, size_t srcSize)
- {
- BYTE const* const ip = (BYTE const*)src;
- U32 contiguous = 1;
- DEBUGLOG(5, "ZSTD_window_update");
-
- if (src != window->nextSrc) {
-
- size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
- DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);
- window->lowLimit = window->dictLimit;
- assert(distanceFromBase == (size_t)(U32)distanceFromBase);
- window->dictLimit = (U32)distanceFromBase;
- window->dictBase = window->base;
- window->base = ip - distanceFromBase;
-
- if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit;
- contiguous = 0;
- }
- window->nextSrc = ip + srcSize;
-
- if ( (ip+srcSize > window->dictBase + window->lowLimit)
- & (ip < window->dictBase + window->dictLimit)) {
- ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
- U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
- window->lowLimit = lowLimitMax;
- DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
- }
- return contiguous;
- }
- #if (DEBUGLEVEL>=2)
- MEM_STATIC double ZSTD_fWeight(U32 rawStat)
- {
- U32 const fp_accuracy = 8;
- U32 const fp_multiplier = (1 << fp_accuracy);
- U32 const newStat = rawStat + 1;
- U32 const hb = ZSTD_highbit32(newStat);
- U32 const BWeight = hb * fp_multiplier;
- U32 const FWeight = (newStat << fp_accuracy) >> hb;
- U32 const weight = BWeight + FWeight;
- assert(hb + fp_accuracy < 31);
- return (double)weight / fp_multiplier;
- }
- MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
- {
- unsigned u, sum;
- for (u=0, sum=0; u<=max; u++) sum += table[u];
- DEBUGLOG(2, "total nb elts: %u", sum);
- for (u=0; u<=max; u++) {
- DEBUGLOG(2, "%2u: %5u (%.2f)",
- u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) );
- }
- }
- #endif
- #if defined (__cplusplus)
- }
- #endif
- ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
- const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize);
- size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
- const void* dict, size_t dictSize,
- const ZSTD_CDict* cdict,
- ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
- void ZSTD_resetSeqStore(seqStore_t* ssPtr);
- size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
- ZSTD_outBuffer* output,
- ZSTD_inBuffer* input,
- ZSTD_EndDirective const flushMode);
- ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
- size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
- const void* dict, size_t dictSize,
- ZSTD_dictContentType_e dictContentType,
- ZSTD_dictTableLoadMethod_e dtlm,
- const ZSTD_CDict* cdict,
- ZSTD_CCtx_params params,
- unsigned long long pledgedSrcSize);
- size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- ZSTD_CCtx_params params);
- size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity);
- size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq);
- #endif
|