private void _InitializeBuffers()
{
_toCompress = new BlockingCollection <int>(Buckets);
_toFill = new BlockingCollection <int>(Buckets);
_toWrite = new BlockingCollection <int>(new ConcurrentQueue <int>());
_pool = new System.Collections.Generic.List <WorkItem>();
for (int i = 0; i < Buckets; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy));
_toFill.Add(i);
// Start one perpetual compressor task per bucket.
Task.Factory.StartNew(_TakeAndCompress);
}
// for diagnostic purposes only
for (int i = 0; i < _pool.Count; i++)
{
_pool[i].index = i;
}
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new Ionic.Zlib.CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
public ZlibBaseStream(System.IO.Stream stream,
CompressionMode compressionMode,
CompressionLevel level,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
this._flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
this._stream = stream;
this._leaveOpen = leaveOpen;
this._compressionMode = compressionMode;
this._flavor = flavor;
this._level = level;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
crc = new CRC32();
}
}
private void _PerpetualWriterMethod(object state)
{
TraceOutput(TraceBits.WriterThread, "_PerpetualWriterMethod START");
try
{
do
{
// wait for the next session
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.WaitOne(begin) PWM");
_sessionReset.WaitOne();
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.WaitOne(done) PWM");
if (_isDisposed)
{
break;
}
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.Reset() PWM");
_sessionReset.Reset();
// repeatedly write buffers as they become ready
WorkItem workitem = null;
Ionic.Zlib.CRC32 c = new Ionic.Zlib.CRC32();
do
{
workitem = _pool[_nextToWrite % _pc];
lock (workitem)
{
if (_noMoreInputForThisSegment)
{
TraceOutput(TraceBits.Write,
"Write drain wi({0}) stat({1}) canuse({2}) cba({3})",
workitem.index,
workitem.status,
(workitem.status == (int)WorkItem.Status.Compressed),
workitem.compressedBytesAvailable);
}
do
{
if (workitem.status == (int)WorkItem.Status.Compressed)
{
TraceOutput(TraceBits.WriteBegin,
"Write begin wi({0}) stat({1}) cba({2})",
workitem.index,
workitem.status,
workitem.compressedBytesAvailable);
workitem.status = (int)WorkItem.Status.Writing;
_outStream.Write(workitem.compressed, 0, workitem.compressedBytesAvailable);
c.Combine(workitem.crc, workitem.inputBytesAvailable);
_totalBytesProcessed += workitem.inputBytesAvailable;
_nextToWrite++;
workitem.inputBytesAvailable = 0;
workitem.status = (int)WorkItem.Status.Done;
TraceOutput(TraceBits.WriteDone,
"Write done wi({0}) stat({1}) cba({2})",
workitem.index,
workitem.status,
workitem.compressedBytesAvailable);
Monitor.Pulse(workitem);
break;
}
else
{
int wcycles = 0;
// I've locked a workitem I cannot use.
// Therefore, wake someone else up, and then release the lock.
while (workitem.status != (int)WorkItem.Status.Compressed)
{
TraceOutput(TraceBits.WriteWait,
"Write waiting wi({0}) stat({1}) nw({2}) nf({3}) nomore({4})",
workitem.index,
workitem.status,
_nextToWrite, _nextToFill,
_noMoreInputForThisSegment);
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
{
break;
}
wcycles++;
// wake up someone else
Monitor.Pulse(workitem);
// release and wait
Monitor.Wait(workitem);
if (workitem.status == (int)WorkItem.Status.Compressed)
{
TraceOutput(TraceBits.WriteWait,
"Write A-OK wi({0}) stat({1}) iba({2}) cba({3}) cyc({4})",
workitem.index,
workitem.status,
workitem.inputBytesAvailable,
workitem.compressedBytesAvailable,
wcycles);
}
}
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
{
break;
}
}
}while (true);
}
if (_noMoreInputForThisSegment)
{
TraceOutput(TraceBits.Write,
"Write nomore nw({0}) nf({1}) break({2})",
_nextToWrite, _nextToFill, (_nextToWrite == _nextToFill));
}
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
{
break;
}
} while (true);
// Finish:
// After writing a series of buffers, closing each one with
// Flush.Sync, we now write the final one as Flush.Finish, and
// then stop.
byte[] buffer = new byte[128];
ZlibCodec compressor = new ZlibCodec();
int rc = compressor.InitializeDeflate(_compressLevel, false);
compressor.InputBuffer = null;
compressor.NextIn = 0;
compressor.AvailableBytesIn = 0;
compressor.OutputBuffer = buffer;
compressor.NextOut = 0;
compressor.AvailableBytesOut = buffer.Length;
rc = compressor.Deflate(FlushType.Finish);
if (rc != ZlibConstants.Z_STREAM_END && rc != ZlibConstants.Z_OK)
{
throw new Exception("deflating: " + compressor.Message);
}
if (buffer.Length - compressor.AvailableBytesOut > 0)
{
TraceOutput(TraceBits.WriteBegin,
"Write begin flush bytes({0})",
buffer.Length - compressor.AvailableBytesOut);
_outStream.Write(buffer, 0, buffer.Length - compressor.AvailableBytesOut);
TraceOutput(TraceBits.WriteBegin,
"Write done flush");
}
compressor.EndDeflate();
_Crc32 = c.Crc32Result;
// signal that writing is complete:
TraceOutput(TraceBits.Synch, "Synch _writingDone.Set() PWM");
_writingDone.Set();
}while (true);
}
catch (System.Exception exc1)
{
lock (_eLock)
{
// expose the exception to the main thread
if (_pendingException != null)
{
_pendingException = exc1;
}
}
}
TraceOutput(TraceBits.WriterThread, "_PerpetualWriterMethod FINIS");
}
private void _PerpetualWriterMethod(object state)
{
TraceOutput(TraceBits.WriterThread, "_PerpetualWriterMethod START");
try
{
do
{
// wait for the next session
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.WaitOne(begin) PWM");
_sessionReset.WaitOne();
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.WaitOne(done) PWM");
if (_isDisposed) break;
TraceOutput(TraceBits.Synch | TraceBits.WriterThread, "Synch _sessionReset.Reset() PWM");
_sessionReset.Reset();
// repeatedly write buffers as they become ready
WorkItem workitem = null;
CRC32 c= new CRC32();
do
{
workitem = _pool[_nextToWrite % _pc];
lock(workitem)
{
if (_noMoreInputForThisSegment)
TraceOutput(TraceBits.Write,
"Write drain wi({0}) stat({1}) canuse({2}) cba({3})",
workitem.index,
workitem.status,
(workitem.status == (int)WorkItem.Status.Compressed),
workitem.compressedBytesAvailable);
do
{
if (workitem.status == (int)WorkItem.Status.Compressed)
{
TraceOutput(TraceBits.WriteBegin,
"Write begin wi({0}) stat({1}) cba({2})",
workitem.index,
workitem.status,
workitem.compressedBytesAvailable);
workitem.status = (int)WorkItem.Status.Writing;
_outStream.Write(workitem.compressed, 0, workitem.compressedBytesAvailable);
c.Combine(workitem.crc, workitem.inputBytesAvailable);
_totalBytesProcessed += workitem.inputBytesAvailable;
_nextToWrite++;
workitem.inputBytesAvailable= 0;
workitem.status = (int)WorkItem.Status.Done;
TraceOutput(TraceBits.WriteDone,
"Write done wi({0}) stat({1}) cba({2})",
workitem.index,
workitem.status,
workitem.compressedBytesAvailable);
Monitor.Pulse(workitem);
break;
}
else
{
int wcycles = 0;
// I've locked a workitem I cannot use.
// Therefore, wake someone else up, and then release the lock.
while (workitem.status != (int)WorkItem.Status.Compressed)
{
TraceOutput(TraceBits.WriteWait,
"Write waiting wi({0}) stat({1}) nw({2}) nf({3}) nomore({4})",
workitem.index,
workitem.status,
_nextToWrite, _nextToFill,
_noMoreInputForThisSegment );
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
break;
wcycles++;
// wake up someone else
Monitor.Pulse(workitem);
// release and wait
Monitor.Wait(workitem);
if (workitem.status == (int)WorkItem.Status.Compressed)
TraceOutput(TraceBits.WriteWait,
"Write A-OK wi({0}) stat({1}) iba({2}) cba({3}) cyc({4})",
workitem.index,
workitem.status,
workitem.inputBytesAvailable,
workitem.compressedBytesAvailable,
wcycles);
}
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
break;
}
}
while (true);
}
if (_noMoreInputForThisSegment)
TraceOutput(TraceBits.Write,
"Write nomore nw({0}) nf({1}) break({2})",
_nextToWrite, _nextToFill, (_nextToWrite == _nextToFill));
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
break;
} while (true);
// Finish:
// After writing a series of buffers, closing each one with
// Flush.Sync, we now write the final one as Flush.Finish, and
// then stop.
byte[] buffer = new byte[128];
ZlibCodec compressor = new ZlibCodec();
int rc = compressor.InitializeDeflate(_compressLevel, false);
compressor.InputBuffer = null;
compressor.NextIn = 0;
compressor.AvailableBytesIn = 0;
compressor.OutputBuffer = buffer;
compressor.NextOut = 0;
compressor.AvailableBytesOut = buffer.Length;
rc = compressor.Deflate(FlushType.Finish);
if (rc != ZlibConstants.Z_STREAM_END && rc != ZlibConstants.Z_OK)
throw new Exception("deflating: " + compressor.Message);
if (buffer.Length - compressor.AvailableBytesOut > 0)
{
TraceOutput(TraceBits.WriteBegin,
"Write begin flush bytes({0})",
buffer.Length - compressor.AvailableBytesOut);
_outStream.Write(buffer, 0, buffer.Length - compressor.AvailableBytesOut);
TraceOutput(TraceBits.WriteBegin,
"Write done flush");
}
compressor.EndDeflate();
_Crc32 = c.Crc32Result;
// signal that writing is complete:
TraceOutput(TraceBits.Synch, "Synch _writingDone.Set() PWM");
_writingDone.Set();
}
while (true);
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
TraceOutput(TraceBits.WriterThread, "_PerpetualWriterMethod FINIS");
}
/// <summary>
/// Resets the stream for use with another stream.
/// </summary>
/// <remarks>
/// Because the ParallelDeflateOutputStream is expensive to create, it
/// has been designed so that it can be recycled and re-used. You have
/// to call Close() on the stream first, then you can call Reset() on
/// it, to use it again on another stream.
/// </remarks>
///
/// <param name="stream">
/// The new output stream for this era.
/// </param>
///
/// <example>
/// <code>
/// ParallelDeflateOutputStream deflater = null;
/// foreach (var inputFile in listOfFiles)
/// {
/// string outputFile = inputFile + ".compressed";
/// using (System.IO.Stream input = System.IO.File.OpenRead(inputFile))
/// {
/// using (var outStream = System.IO.File.Create(outputFile))
/// {
/// if (deflater == null)
/// deflater = new ParallelDeflateOutputStream(outStream,
/// CompressionLevel.Best,
/// CompressionStrategy.Default,
/// true);
/// deflater.Reset(outStream);
///
/// while ((n= input.Read(buffer, 0, buffer.Length)) != 0)
/// {
/// deflater.Write(buffer, 0, n);
/// }
/// }
/// }
/// }
/// </code>
/// </example>
public void Reset(Stream stream)
{
TraceOutput(TraceBits.Session, "-------------------------------------------------------");
TraceOutput(TraceBits.Session, "Reset {0:X8} firstDone({1})", this.GetHashCode(), _firstWriteDone);
if (!_firstWriteDone) return;
// reset all status
_toWrite.Clear();
_toFill.Clear();
foreach (var workitem in _pool)
{
_toFill.Enqueue(workitem.index);
workitem.ordinal = -1;
}
_firstWriteDone = false;
_totalBytesProcessed = 0L;
_runningCrc = new CRC32();
_isClosed= false;
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
_outStream = stream;
}
private void _InitializePoolOfWorkItems()
{
_toWrite = new Queue<int>();
_toFill = new Queue<int>();
_pool = new System.Collections.Generic.List<WorkItem>();
int nTasks = BufferPairsPerCore * Environment.ProcessorCount;
for(int i=0; i < nTasks; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy, i));
_toFill.Enqueue(i);
}
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
private void _DeflateOne(Object wi)
{
// compress one buffer
WorkItem workitem = (WorkItem) wi;
try
{
int myItem = workitem.index;
CRC32 crc = new CRC32();
// calc CRC on the buffer
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
// deflate it
DeflateOneSegment(workitem);
// update status
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress wi({0}) ord({1}) len({2})",
workitem.index,
workitem.ordinal,
workitem.compressedBytesAvailable
);
lock(_latestLock)
{
if (workitem.ordinal > _latestCompressed)
_latestCompressed = workitem.ordinal;
}
lock (_toWrite)
{
_toWrite.Enqueue(workitem.index);
}
_newlyCompressedBlob.Set();
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
}
private void _InitializeBuffers()
{
_toCompress = new BlockingCollection<int>(Buckets);
_toFill = new BlockingCollection<int>(Buckets);
_toWrite = new BlockingCollection<int>(new ConcurrentQueue<int>());
_pool = new System.Collections.Generic.List<WorkItem>();
for(int i=0; i < Buckets; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy));
_toFill.Add(i);
// Start one perpetual compressor task per bucket.
Task.Factory.StartNew( _TakeAndCompress );
}
// for diagnostic purposes only
for(int i=0; i < _pool.Count; i++)
_pool[i].index= i;
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new Ionic.Zlib.CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
private void _TakeAndCompress()
{
var rnd = new System.Random();
while (!_toCompress.IsCompleted)
{
WorkItem workitem = null;
int ix = -1;
try
{
ix = _toCompress.Take();
workitem = _pool[ix];
}
catch (InvalidOperationException)
{
// The collection has been completed.
// Some other thread has called CompleteAdding()
// after this thread passed the
// IsCompleted check.
}
if (workitem == null) continue;
try
{
TraceOutput(TraceBits.Compress,
"Compress lock wi({0}) ord({1})",
workitem.index,
workitem.ordinal);
// compress one buffer
Ionic.Zlib.CRC32 crc = new CRC32();
int ib = workitem.inputBytesAvailable;
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
DeflateOneSegment(workitem);
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress done wi({0}) ord({1}) ib-({2}) cba({3})",
workitem.index,
workitem.ordinal,
ib,
workitem.compressedBytesAvailable
);
lock(_latestLock)
{
if (workitem.ordinal > _latestCompressed)
_latestCompressed = workitem.ordinal;
}
_toWrite.Add(workitem.index);
_newlyCompressedBlob.Set();
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
}
}
// This ctor is private - no validation is done here. This is to allow the use
// of a (specific) negative value for the _lengthLimit, to indicate that there
// is no length set. So we validate the length limit in those ctors that use an
// explicit param, otherwise we don't validate, because it could be our special
// value.
private CrcCalculatorStream(bool leaveOpen, Int64 length, System.IO.Stream stream)
: base()
{
_innerStream = stream;
_Crc32 = new CRC32();
_lengthLimit = length;
_leaveOpen = leaveOpen;
}
private void _DeflateOne(Object wi)
{
WorkItem workitem = (WorkItem) wi;
try
{
// compress one buffer
int myItem = workitem.index;
lock(workitem)
{
if (workitem.status != (int)WorkItem.Status.Filled)
throw new InvalidOperationException();
Ionic.Zlib.CRC32 crc = new CRC32();
// use the workitem:
// calc CRC on the buffer
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
// deflate it
DeflateOneSegment(workitem);
// update status
workitem.status = (int)WorkItem.Status.Compressed;
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress wi({0}) stat({1}) len({2})",
workitem.index,
workitem.status,
workitem.compressedBytesAvailable
);
// release the item
Monitor.Pulse(workitem);
}
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
}
private void _PerpetualWriterMethod(object state)
{
try
{
ZlibCodec zlibCodec;
while (true)
{
_sessionReset.WaitOne();
if (_isDisposed)
{
return;
}
_sessionReset.Reset();
WorkItem workItem = null;
CRC32 cRC = new CRC32();
bool flag;
while (true)
{
workItem = _pool[_nextToWrite % _pc];
lock (workItem)
{
if (_noMoreInputForThisSegment)
{
}
while (true)
{
if (workItem.status == 4)
{
workItem.status = 5;
_outStream.Write(workItem.compressed, 0, workItem.compressedBytesAvailable);
cRC.Combine(workItem.crc, workItem.inputBytesAvailable);
_totalBytesProcessed += workItem.inputBytesAvailable;
_nextToWrite++;
workItem.inputBytesAvailable = 0;
workItem.status = 6;
Monitor.Pulse(workItem);
break;
}
int num = 0;
while (workItem.status != 4 && (!_noMoreInputForThisSegment || _nextToWrite != _nextToFill))
{
num++;
Monitor.Pulse(workItem);
Monitor.Wait(workItem);
if (workItem.status != 4)
{
}
}
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
{
break;
}
flag = true;
}
}
if (_noMoreInputForThisSegment)
{
}
if (_noMoreInputForThisSegment && _nextToWrite == _nextToFill)
{
break;
}
flag = true;
}
byte[] array = new byte[128];
zlibCodec = new ZlibCodec();
int num2 = zlibCodec.InitializeDeflate(_compressLevel, wantRfc1950Header: false);
zlibCodec.InputBuffer = null;
zlibCodec.NextIn = 0;
zlibCodec.AvailableBytesIn = 0;
zlibCodec.OutputBuffer = array;
zlibCodec.NextOut = 0;
zlibCodec.AvailableBytesOut = array.Length;
num2 = zlibCodec.Deflate(FlushType.Finish);
if (num2 != 1 && num2 != 0)
{
break;
}
if (array.Length - zlibCodec.AvailableBytesOut > 0)
{
_outStream.Write(array, 0, array.Length - zlibCodec.AvailableBytesOut);
}
zlibCodec.EndDeflate();
_Crc32 = cRC.Crc32Result;
_writingDone.Set();
flag = true;
}
throw new Exception("deflating: " + zlibCodec.Message);
}
catch (Exception pendingException)
{
lock (_eLock)
{
if (_pendingException != null)
{
_pendingException = pendingException;
}
}
}
}