static void CopyStats(SymbolStats source, ref SymbolStats dest) { Array.Copy(source.dists, dest.dists, source.dists.Length); Array.Copy(source.litlens, dest.litlens, source.litlens.Length); Array.Copy(source.ll_symbols, dest.ll_symbols, source.ll_symbols.Length); Array.Copy(source.d_symbols, dest.d_symbols, source.d_symbols.Length); }
static void ZopfliLZ77OptimalFixed(ZopfliBlockState s, byte[] InFile, ulong instart, ulong inend, ZopfliLZ77Store store) { /* Dist to get to here with smallest cost. */ ulong blocksize = inend - instart; ushort[] length_array = new ushort[blocksize + 1]; List <ushort> path = new List <ushort>(); path.Add(0); ZopfliHash h = new ZopfliHash(); float[] costs = new float[blocksize + 1]; SymbolStats stats = new SymbolStats(); s.blockstart = (int)instart; s.blockend = (int)inend; /* Shortest path for fixed tree This one should give the shortest possible * result for fixed tree, no repeated runs are needed since the tree is known. */ LZ77OptimalRun(s, InFile, (int)instart, (int)inend, path, length_array, stats, store, h, costs, true); }
public static void SymbolsExtensionsTest_Initialize(TestContext testContext) { eth = TestHelper.Eth; ethStats = TestHelper.EthStats; trx = TestHelper.Trx; trxStats = TestHelper.TrxStats; }
public async Task SubscribeStatistics(IEnumerable <string> symbols, Action <StatisticsEventArgs> callback, Action <Exception> exception, CancellationToken cancellationToken) { var kucoinSocketClient = new KucoinSocketClient(); CallResult <UpdateSubscription> result = null; try { var kucoinClient = new KucoinClient(); foreach (var symbol in symbols) { result = await kucoinSocketClient.SubscribeToSnapshotUpdatesAsync(symbol, async data => { if (cancellationToken.IsCancellationRequested) { await kucoinSocketClient.Unsubscribe(result.Data).ConfigureAwait(false); return; } try { var symbolStats = new SymbolStats { Symbol = data.Symbol, Exchange = Exchange.Kucoin, CloseTime = data.Timestamp, Volume = data.Volume, LowPrice = data.Low, HighPrice = data.High, LastPrice = data.LastPrice, PriceChange = data.ChangePrice, PriceChangePercent = data.ChangePercentage * 100 }; callback.Invoke(new StatisticsEventArgs { Statistics = new[] { symbolStats } }); } catch (Exception ex) { await kucoinSocketClient.Unsubscribe(result.Data).ConfigureAwait(false); exception.Invoke(ex); return; } }); } } catch (Exception) { if (result != null) { await kucoinSocketClient.Unsubscribe(result.Data).ConfigureAwait(false); } throw; } }
static void ClearStatFreqs(SymbolStats stats) { ulong i; for (i = 0; i < ZOPFLI_NUM_LL; i++) { stats.litlens[i] = 0; } for (i = 0; i < ZOPFLI_NUM_D; i++) { stats.dists[i] = 0; } }
/* * Does a single run for ZopfliLZ77Optimal. For good compression, repeated runs * with updated statistics should be performed. * s: the block state * in: the input data array * instart: where to start * inend: where to stop (not inclusive) * path: pointer to dynamically allocated memory to store the path * pathsize: pointer to the size of the dynamic path array * length_array: array of size (inend - instart) used to store lengths * costmodel: function to use as the cost model for this squeeze run * costcontext: abstract context for the costmodel function * store: place to output the LZ77 data * returns the cost that was, according to the costmodel, needed to get to the end. * This is not the actual cost. */ static double LZ77OptimalRun(ZopfliBlockState s, byte[] InFile, int instart, int inend, List <ushort> path, ushort[] length_array, SymbolStats stats, ZopfliLZ77Store store, ZopfliHash h, float[] costs, bool fixedcosts) { double cost = GetBestLengths(s, InFile, instart, inend, stats, length_array, h, costs, fixedcosts); ulong pathsize = 0; path.Clear(); TraceBackwards((ulong)(inend - instart), length_array, path, ref pathsize); FollowPath(s, InFile, instart, inend, path, pathsize, store, h); Debug.Assert(cost < ZOPFLI_LARGE_FLOAT); return(cost); }
/* * Cost model based on symbol statistics. * type: CostModelFun */ static double GetCostStat(int litlen, int dist, SymbolStats stats) { if (dist == 0) { return(stats.ll_symbols[litlen]); } else { int dbits = Symbols.ZopfliGetDistExtraBits(dist); int lbits = Symbols.ZopfliGetLengthExtraBits(litlen); int lsym = Symbols.ZopfliGetLengthSymbol((ushort)litlen); int dsym = Symbols.ZopfliGetDistSymbol((ushort)dist); return(dbits + lbits + stats.ll_symbols[lsym] + stats.d_symbols[dsym]); } }
/* Adds the bit lengths. */ static void AddWeighedStatFreqs(SymbolStats stats1, double w1, SymbolStats stats2, double w2, SymbolStats result) { ulong i; for (i = 0; i < ZOPFLI_NUM_LL; i++) { result.litlens[i] = (ulong)(stats1.litlens[i] * w1 + stats2.litlens[i] * w2); } for (i = 0; i < ZOPFLI_NUM_D; i++) { result.dists[i] = (ulong)(stats1.dists[i] * w1 + stats2.dists[i] * w2); } result.litlens[256] = 1; /* End symbol. */ }
/* * Finds the minimum possible cost this cost model can return for valid length and * distance symbols. */ static double GetCostModelMinCost(SymbolStats stats) { double mincost; int bestlength = 0; /* length that has lowest cost in the cost model */ int bestdist = 0; /* distance that has lowest cost in the cost model */ int i; /* * Table of distances that have a different distance symbol in the deflate * specification. Each value is the first distance that has a new symbol. Only * different symbols affect the cost model so only these need to be checked. * See RFC 1951 section 3.2.5. Compressed blocks (length and distance codes). */ int[] dsymbols = { 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 }; mincost = ZOPFLI_LARGE_FLOAT; for (i = 3; i < 259; i++) { double c = GetCostStat(i, 1, stats); if (c < mincost) { bestlength = i; mincost = c; } } mincost = ZOPFLI_LARGE_FLOAT; for (i = 0; i < 30; i++) { double c = GetCostStat(3, dsymbols[i], stats); if (c < mincost) { bestdist = dsymbols[i]; mincost = c; } } return(GetCostStat(bestlength, bestdist, stats)); }
/* Appends the symbol statistics from the store. */ static void GetStatistics(ZopfliLZ77Store store, SymbolStats stats) { ulong i; for (i = 0; i < store.size; i++) { if (store.dists[(int)i] == 0) { stats.litlens[store.litlens[(int)i]]++; } else { stats.litlens[Symbols.ZopfliGetLengthSymbol(store.litlens[(int)i])]++; stats.dists[Symbols.ZopfliGetDistSymbol(store.dists[(int)i])]++; } } stats.litlens[256] = 1; /* End symbol. */ stats.CalculateStatistics(); }
static void RandomizeStatFreqs(RanState state, SymbolStats stats) { RandomizeFreqs(state, stats.litlens, ZOPFLI_NUM_LL); RandomizeFreqs(state, stats.dists, ZOPFLI_NUM_D); stats.litlens[256] = 1; /* End symbol. */ }
static void ZopfliLZ77Optimal(ZopfliBlockState s, byte[] InFile, int instart, int inend, int numiterations, ZopfliLZ77Store store) { /* Dist to get to here with smallest cost. */ int blocksize = inend - instart; ushort[] length_array = new ushort[blocksize + 1]; List <ushort> path = new List <ushort>(); ZopfliLZ77Store currentstore = new ZopfliLZ77Store(InFile); ZopfliHash h = new ZopfliHash(); SymbolStats stats = new SymbolStats(); SymbolStats beststats = new SymbolStats(); SymbolStats laststats = new SymbolStats(); int i; float[] costs = new float [blocksize + 1]; double cost; double bestcost = ZOPFLI_LARGE_FLOAT; double lastcost = 0; /* Try randomizing the costs a bit once the size stabilizes. */ RanState ran_state = new RanState(); int lastrandomstep = -1; /* Do regular deflate, then loop multiple shortest path runs, each time using * the statistics of the previous run. */ /* Initial run. */ ZopfliLZ77Greedy(s, InFile, instart, inend, currentstore, h); GetStatistics(currentstore, stats); /* Repeat statistics with each time the cost model from the previous stat * run. */ for (i = 0; i < numiterations; i++) { currentstore.ResetStore(InFile); LZ77OptimalRun(s, InFile, instart, inend, path, length_array, stats, currentstore, h, costs, false); cost = ZopfliCalculateBlockSize(currentstore, 0, currentstore.size, 2); if (Globals.verbose_more > 0 || (Globals.verbose > 0 && cost < bestcost)) { Console.WriteLine("Iteration " + i + ": " + cost + " bit"); } if (cost < bestcost) { /* Copy to the output store. */ ZopfliCopyLZ77Store(currentstore, store); CopyStats(stats, ref beststats); bestcost = cost; } CopyStats(stats, ref laststats); ClearStatFreqs(stats); GetStatistics(currentstore, stats); if (lastrandomstep != -1) { /* This makes it converge slower but better. Do it only once the * randomness kicks in so that if the user does few iterations, it gives a * better result sooner. */ AddWeighedStatFreqs(stats, 1.0, laststats, 0.5, stats); stats.CalculateStatistics(); } if (i > 5 && cost == lastcost) { CopyStats(beststats, ref stats); RandomizeStatFreqs(ran_state, stats); stats.CalculateStatistics(); lastrandomstep = i; } lastcost = cost; } }
/* * Performs the forward pass for "squeeze". Gets the most optimal length to reach * every byte from a previous byte, using cost calculations. * s: the ZopfliBlockState * in: the input data array * instart: where to start * inend: where to stop (not inclusive) * costmodel: function to calculate the cost of some lit/len/dist pair. * costcontext: abstract context for the costmodel function * length_array: output array of size (inend - instart) which will receive the best * length to reach this byte from a previous byte. * returns the cost that was, according to the costmodel, needed to get to the end. */ static double GetBestLengths(ZopfliBlockState s, byte[] InFile, int instart, int inend, SymbolStats stats, ushort[] length_array, ZopfliHash h, float[] costs, bool fixedcosts) { /* Best cost to get here so far. */ ulong blocksize = (ulong)(inend - instart); ulong i, k, kend; ushort leng = 0; //bogus value ushort dist = 0; //bogusvalue ushort[] sublen = new ushort[259]; ulong windowstart = (ulong)(instart > ZopfliHash.ZOPFLI_WINDOW_SIZE ? instart - ZopfliHash.ZOPFLI_WINDOW_SIZE : 0); double result; double mincost = GetCostModelMinCost(stats); double mincostaddcostj; if (instart == inend) { return(0); } h.ZopfliResetHash(); h.ZopfliWarmupHash(InFile, (int)windowstart, inend); for (i = windowstart; i < (ulong)instart; i++) { h.ZopfliUpdateHash(InFile, (int)i, inend); } Array.Fill(costs, (float)Compress.ZOPFLI_LARGE_FLOAT); costs[0] = 0; /* Because it's the start. */ length_array[0] = 0; for (i = (ulong)instart; i < (ulong)inend; i++) { ulong j = i - (ulong)instart; /* Index in the costs array and length_array. */ h.ZopfliUpdateHash(InFile, (int)i, inend); /* If we're in a long repetition of the same character and have more than * ZOPFLI_MAX_MATCH characters before and after our position. */ if (h.same[i & ZopfliHash.ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH * 2 && (int)i > instart + ZOPFLI_MAX_MATCH + 1 && i + ZOPFLI_MAX_MATCH * 2 + 1 < (ulong)inend && h.same[(i - ZOPFLI_MAX_MATCH) & ZopfliHash.ZOPFLI_WINDOW_MASK] > ZOPFLI_MAX_MATCH) { double symbolcost; if (fixedcosts) { symbolcost = GetCostFixed(ZOPFLI_MAX_MATCH, 1); } else { symbolcost = GetCostStat(ZOPFLI_MAX_MATCH, 1, stats); } /* Set the length to reach each one to ZOPFLI_MAX_MATCH, and the cost to * the cost corresponding to that length. Doing this, we skip * ZOPFLI_MAX_MATCH values to avoid calling ZopfliFindLongestMatch. */ for (k = 0; k < ZOPFLI_MAX_MATCH; k++) { costs[j + ZOPFLI_MAX_MATCH] = costs[j] + (float)symbolcost; length_array[j + ZOPFLI_MAX_MATCH] = ZOPFLI_MAX_MATCH; i++; j++; h.ZopfliUpdateHash(InFile, (int)i, inend); } } ZopfliFindLongestMatch(s, h, InFile, (int)i, inend, ZOPFLI_MAX_MATCH, sublen, ref dist, ref leng); /* Literal. */ if (i + 1 <= (ulong)inend) { double newCost; if (fixedcosts) { newCost = GetCostFixed(InFile[i], 0) + costs[j]; } else { newCost = GetCostStat(InFile[i], 0, stats) + costs[j]; } Debug.Assert(newCost >= 0); if (newCost < costs[j + 1]) { costs[j + 1] = (float)newCost; length_array[j + 1] = 1; } } /* Lengths. */ kend = zopfli_min(leng, (ulong)inend - i); mincostaddcostj = mincost + costs[j]; for (k = 3; k <= kend; k++) { double newCost; /* Calling the cost model is expensive, avoid this if we are already at * the minimum possible cost that it can return. */ if (costs[j + k] <= mincostaddcostj) { continue; } if (fixedcosts) { newCost = GetCostFixed((int)k, sublen[k]) + costs[j]; } else { newCost = GetCostStat((int)k, sublen[k], stats) + costs[j]; } Debug.Assert(newCost >= 0); if (newCost < costs[j + k]) { Debug.Assert(k <= ZOPFLI_MAX_MATCH); costs[j + k] = (float)newCost; length_array[j + k] = (ushort)k; } } } Debug.Assert(costs[blocksize] >= 0); result = costs[blocksize]; return(result); }
static Symbol f(Symbol s, SymbolStats ss) { s.SymbolStatistics = ss; return(s); }