public ZlibBaseStream(System.IO.Stream stream, CompressionMode compressionMode, CompressionLevel level, ZlibStreamFlavor flavor, bool leaveOpen) : base() { this._flushMode = FlushType.None; //this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT]; this._stream = stream; this._leaveOpen = leaveOpen; this._compressionMode = compressionMode; this._flavor = flavor; this._level = level; // workitem 7159 if (flavor == ZlibStreamFlavor.GZIP) { this.crc = new Alienlab.Crc.CRC32(); } }
private Int32 FigureCrc32() { if (_crcCalculated == false) { Stream input = null; // get the original stream: if (this._Source == ZipEntrySource.WriteDelegate) { var output = new Alienlab.Crc.CrcCalculatorStream(Stream.Null); // allow the application to write the data this._WriteDelegate(this.FileName, output); _Crc32 = output.Crc; } else if (this._Source == ZipEntrySource.ZipFile) { // nothing to do - the CRC is already set } else { if (this._Source == ZipEntrySource.Stream) { PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.JitStream) { // allow the application to open the stream if (this._sourceStream == null) _sourceStream = this._OpenDelegate(this.FileName); PrepSourceStream(); input = this._sourceStream; } else if (this._Source == ZipEntrySource.ZipOutputStream) { } else { //input = File.OpenRead(LocalFileName); input = File.Open(LocalFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } var crc32 = new Alienlab.Crc.CRC32(); _Crc32 = crc32.GetCrc32(input); if (_sourceStream == null) { #if NETCF input.Close(); #else input.Dispose(); #endif } } _crcCalculated = true; } return _Crc32; }
public void Zlib_ParallelDeflateStream() { var sw = new System.Diagnostics.Stopwatch(); sw.Start(); TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream Start", sw.Elapsed); int sz = 256*1024 + this.rnd.Next(120000); string FileToCompress = System.IO.Path.Combine(TopLevelDir, String.Format("Zlib_ParallelDeflateStream.{0}.txt", sz)); CreateAndFillFileText( FileToCompress, sz); TestContext.WriteLine("{0}: Created file: {1}", sw.Elapsed, FileToCompress ); byte[] original = File.ReadAllBytes(FileToCompress); int crc1 = DoCrc(FileToCompress); TestContext.WriteLine("{0}: Original CRC: {1:X8}", sw.Elapsed, crc1 ); byte[] working = new byte[WORKING_BUFFER_SIZE]; int n = -1; long originalLength; MemoryStream ms1 = new MemoryStream(); { using (FileStream fs1 = File.OpenRead(FileToCompress)) { originalLength = fs1.Length; using (var compressor = new Alienlab.Zlib.ParallelDeflateOutputStream(ms1, true)) { while ((n = fs1.Read(working, 0, working.Length)) != 0) { compressor.Write(working, 0, n); } } } ms1.Seek(0, SeekOrigin.Begin); } TestContext.WriteLine("{0}: Compressed {1} bytes into {2} bytes", sw.Elapsed, originalLength, ms1.Length); var crc = new Alienlab.Crc.CRC32(); int crc2= 0; byte[] decompressedBytes= null; using (MemoryStream ms2 = new MemoryStream()) { using (var decompressor = new DeflateStream(ms1, CompressionMode.Decompress, false)) { while ((n = decompressor.Read(working, 0, working.Length)) != 0) { ms2.Write(working, 0, n); } } TestContext.WriteLine("{0}: Decompressed", sw.Elapsed); TestContext.WriteLine("{0}: Decompressed length: {1}", sw.Elapsed, ms2.Length); ms2.Seek(0, SeekOrigin.Begin); crc2 = crc.GetCrc32(ms2); decompressedBytes = ms2.ToArray(); TestContext.WriteLine("{0}: Decompressed CRC: {1:X8}", sw.Elapsed, crc2 ); } TestContext.WriteLine("{0}: Checking...", sw.Elapsed ); bool check = true; if (originalLength != decompressedBytes.Length) { TestContext.WriteLine("Different lengths."); check = false; } else { for (int i = 0; i < decompressedBytes.Length; i++) { if (original[i] != decompressedBytes[i]) { TestContext.WriteLine("byte {0} differs", i); check = false; break; } } } Assert.IsTrue(check,"Data check failed"); TestContext.WriteLine("{0}: Done...", sw.Elapsed ); }
/// <summary> /// Resets the stream for use with another stream. /// </summary> /// <remarks> /// Because the ParallelDeflateOutputStream is expensive to create, it /// has been designed so that it can be recycled and re-used. You have /// to call Close() on the stream first, then you can call Reset() on /// it, to use it again on another stream. /// </remarks> /// /// <param name="stream"> /// The new output stream for this era. /// </param> /// /// <example> /// <code> /// ParallelDeflateOutputStream deflater = null; /// foreach (var inputFile in listOfFiles) /// { /// string outputFile = inputFile + ".compressed"; /// using (System.IO.Stream input = System.IO.File.OpenRead(inputFile)) /// { /// using (var outStream = System.IO.File.Create(outputFile)) /// { /// if (deflater == null) /// deflater = new ParallelDeflateOutputStream(outStream, /// CompressionLevel.Best, /// CompressionStrategy.Default, /// true); /// deflater.Reset(outStream); /// /// while ((n= input.Read(buffer, 0, buffer.Length)) != 0) /// { /// deflater.Write(buffer, 0, n); /// } /// } /// } /// } /// </code> /// </example> public void Reset(Stream stream) { TraceOutput(TraceBits.Session, "-------------------------------------------------------"); TraceOutput(TraceBits.Session, "Reset {0:X8} firstDone({1})", this.GetHashCode(), _firstWriteDone); if (!_firstWriteDone) return; // reset all status _toWrite.Clear(); _toFill.Clear(); foreach (var workitem in _pool) { _toFill.Enqueue(workitem.index); workitem.ordinal = -1; } _firstWriteDone = false; _totalBytesProcessed = 0L; _runningCrc = new Alienlab.Crc.CRC32(); _isClosed= false; _currentlyFilling = -1; _lastFilled = -1; _lastWritten = -1; _latestCompressed = -1; _outStream = stream; }
private void _InitializePoolOfWorkItems() { _toWrite = new Queue<int>(); _toFill = new Queue<int>(); _pool = new System.Collections.Generic.List<WorkItem>(); int nTasks = BufferPairsPerCore * Environment.ProcessorCount; nTasks = Math.Min(nTasks, _maxBufferPairs); for(int i=0; i < nTasks; i++) { _pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy, i)); _toFill.Enqueue(i); } _newlyCompressedBlob = new AutoResetEvent(false); _runningCrc = new Alienlab.Crc.CRC32(); _currentlyFilling = -1; _lastFilled = -1; _lastWritten = -1; _latestCompressed = -1; }
private void _DeflateOne(Object wi) { // compress one buffer WorkItem workitem = (WorkItem) wi; try { int myItem = workitem.index; Alienlab.Crc.CRC32 crc = new Alienlab.Crc.CRC32(); // calc CRC on the buffer crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable); // deflate it DeflateOneSegment(workitem); // update status workitem.crc = crc.Crc32Result; TraceOutput(TraceBits.Compress, "Compress wi({0}) ord({1}) len({2})", workitem.index, workitem.ordinal, workitem.compressedBytesAvailable ); lock(_latestLock) { if (workitem.ordinal > _latestCompressed) _latestCompressed = workitem.ordinal; } lock (_toWrite) { _toWrite.Enqueue(workitem.index); } _newlyCompressedBlob.Set(); } catch (System.Exception exc1) { lock(_eLock) { // expose the exception to the main thread if (_pendingException!=null) _pendingException = exc1; } } }