private IList <SignatureInfo> Process(Stream source, IRdcGenerator rdcGenerator, string fileName,
ISignatureRepository signatureRepository)
{
var result = Enumerable.Range(0, _recursionDepth).Reverse().Select(i => new SignatureInfo(i, fileName)).ToList();
var eof = false;
var eofOutput = false;
// prepare streams
var sigStreams = result.Select(item => signatureRepository.CreateContent(item.Name)).ToList();
var inputBuffer = new RdcBufferPointer
{
Size = 0,
Used = 0,
Data = Marshal.AllocCoTaskMem(InputBufferSize + 16)
};
var rdcBufferPointers = PrepareRdcBufferPointers();
var outputPointers = PrepareOutputPointers(rdcBufferPointers);
try
{
while (!eofOutput)
{
if (inputBuffer.Size == inputBuffer.Used)
{
inputBuffer = GetInputBuffer(source, InputBufferSize, inputBuffer, ref eof);
}
RdcError rdcErrorCode;
var hr = rdcGenerator.Process(
eof,
ref eofOutput,
ref inputBuffer,
(uint)_recursionDepth,
outputPointers,
out rdcErrorCode);
if (hr != 0)
{
throw new RdcException("RdcGenerator failed to process the signature block.", hr, rdcErrorCode);
}
RdcBufferTranslate(outputPointers, rdcBufferPointers);
for (var i = 0; i < _recursionDepth; i++)
{
var resultStream = sigStreams[i];
// Write the signature block to the file.
var bytesWritten = RdcBufferTools.IntPtrCopy(
rdcBufferPointers[i].Data,
resultStream,
(int)rdcBufferPointers[i].Used);
result[i].Length += bytesWritten;
if (rdcBufferPointers[i].Used != bytesWritten)
{
throw new RdcException("Failed to write to the signature file.");
}
rdcBufferPointers[i].Used = 0;
}
RdcBufferTranslate(rdcBufferPointers, outputPointers);
}
}
finally
{
if (inputBuffer.Data != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(inputBuffer.Data);
}
foreach (var item in outputPointers)
{
Marshal.FreeCoTaskMem(item);
}
foreach (var item in rdcBufferPointers)
{
Marshal.FreeCoTaskMem(item.Data);
}
}
result.Reverse();
signatureRepository.Flush(result);
return(result);
}
private IList<SignatureInfo> Process(Stream source, IRdcGenerator rdcGenerator, string fileName,
ISignatureRepository signatureRepository)
{
var result = Enumerable.Range(0, _recursionDepth).Reverse().Select(i => new SignatureInfo(i, fileName)).ToList();
var eof = false;
var eofOutput = false;
// prepare streams
var sigStreams = result.Select(item => signatureRepository.CreateContent(item.Name)).ToList();
var inputBuffer = new RdcBufferPointer
{
Size = 0,
Used = 0,
Data = Marshal.AllocCoTaskMem(InputBufferSize + 16)
};
var rdcBufferPointers = PrepareRdcBufferPointers();
var outputPointers = PrepareOutputPointers(rdcBufferPointers);
try
{
while (!eofOutput)
{
if (inputBuffer.Size == inputBuffer.Used)
inputBuffer = GetInputBuffer(source, InputBufferSize, inputBuffer, ref eof);
RdcError rdcErrorCode;
var hr = rdcGenerator.Process(
eof,
ref eofOutput,
ref inputBuffer,
(uint)_recursionDepth,
outputPointers,
out rdcErrorCode);
if (hr != 0)
throw new RdcException("RdcGenerator failed to process the signature block.", hr, rdcErrorCode);
RdcBufferTranslate(outputPointers, rdcBufferPointers);
for (var i = 0; i < _recursionDepth; i++)
{
var resultStream = sigStreams[i];
// Write the signature block to the file.
var bytesWritten = RdcBufferTools.IntPtrCopy(
rdcBufferPointers[i].Data,
resultStream,
(int)rdcBufferPointers[i].Used);
result[i].Length += bytesWritten;
if (rdcBufferPointers[i].Used != bytesWritten)
{
throw new RdcException("Failed to write to the signature file.");
}
rdcBufferPointers[i].Used = 0;
}
RdcBufferTranslate(rdcBufferPointers, outputPointers);
}
}
finally
{
if (inputBuffer.Data != IntPtr.Zero)
Marshal.FreeCoTaskMem(inputBuffer.Data);
foreach (var item in outputPointers)
Marshal.FreeCoTaskMem(item);
foreach (var item in rdcBufferPointers)
Marshal.FreeCoTaskMem(item.Data);
}
result.Reverse();
signatureRepository.Flush(result);
return result;
}
/// <summary>
/// Generates and computes signatures for a given stream and stores the signature
/// files in the provided working directory.
/// </summary>
/// <param name="source">Source stream to process</param>
/// <returns>A collection of signature objects</returns>
public SignatureCollection GenerateSignatures(Stream source, string BaseName)
{
int hr = 0;
// Compute the appropriate recursion depth for
// this stream if the depth is not provided.
if (recursionDepth < MSRDC.MINIMUM_DEPTH)
{
hr = rdcLibrary.ComputeDefaultRecursionDepth(source.Length, out this.recursionDepth);
if (hr != 0)
{
throw new RdcException("Failed to compute the recursion depth.", hr);
}
if (recursionDepth < MSRDC.MINIMUM_DEPTH)
{
recursionDepth = (int)MSRDC.MINIMUM_DEPTH;
}
}
// Initalize our signature collection. For this
// sample we will create a unique file in the
// working directory for each recursion level.
SignatureCollection signatures = InitializeAndPrepareSignatures(BaseName);
// Array of pointers to generation parameters
IRdcGeneratorParameters[] generatorParameters = new IRdcGeneratorParameters[recursionDepth];
for (int i = 0; i < recursionDepth; i++)
{
hr = rdcLibrary.CreateGeneratorParameters(
GeneratorParametersType.FilterMax,
(uint)i + 1,
out generatorParameters[i]);
if (hr != 0)
{
throw new RdcException("Failed to create the generator parameters.", hr);
}
IRdcGeneratorFilterMaxParameters maxParams = (IRdcGeneratorFilterMaxParameters)generatorParameters[i];
// Set the default properties
maxParams.SetHashWindowSize(i == 0 ? MSRDC.DEFAULT_HASHWINDOWSIZE_1 : MSRDC.DEFAULT_HASHWINDOWSIZE_N);
maxParams.SetHorizonSize(i == 0 ? MSRDC.DEFAULT_HORIZONSIZE_1 : MSRDC.DEFAULT_HORIZONSIZE_N);
}
// Create our RdcGenerator
IRdcGenerator rdcGenerator = null;
hr = rdcLibrary.CreateGenerator((uint)recursionDepth, generatorParameters, out rdcGenerator);
if (hr != 0)
{
throw new RdcException("Failed to create the RdcGenerator.", hr);
}
// Enable similarity
rdcSimGenerator = (IRdcSimilarityGenerator)rdcGenerator;
rdcSimGenerator.EnableSimilarity();
// Create our output buffers
IntPtr[] outputBuffers = new IntPtr[recursionDepth];
RdcBufferPointer[] outputPointer = new RdcBufferPointer[recursionDepth];
IntPtr[] outputPointers = new IntPtr[recursionDepth];
for (int i = 0; i < recursionDepth; i++)
{
outputBuffers[i] = Marshal.AllocCoTaskMem((int)outputBufferSize);
outputPointer[i].size = outputBufferSize;
outputPointer[i].data = outputBuffers[i];
outputPointer[i].used = 0;
// Marshal the managed structure to a native
// pointer and add it to our array.
outputPointers[i] = Marshal.AllocCoTaskMem(Marshal.SizeOf(outputPointer[i]));
Marshal.StructureToPtr(outputPointer[i], outputPointers[i], false);
}
// Create and allocate memory for our input buffer.
IntPtr inputBuffer = Marshal.AllocCoTaskMem((int)inputBufferSize + 16);
RdcBufferPointer inputPointer = new RdcBufferPointer();
inputPointer.size = 0;
inputPointer.used = 0;
inputPointer.data = inputBuffer;
long totalBytesRead = 0;
bool eof = false;
bool eofOutput = false;
while (hr == 0 && !eofOutput)
{
if (inputPointer.size == inputPointer.used && !eof)
{
if (eof)
{
inputPointer.size = 0;
inputPointer.used = 0;
}
else
{
// When the input buffer is completely empty
// refill it.
int bytesRead = 0;
try
{
bytesRead = IntPtrCopy(source, inputBuffer, 0, (int)inputBufferSize);
}
catch (Exception ex)
{
// TODO: Cleanup
throw new RdcException("Failed to read from the source stream.", ex);
}
totalBytesRead += bytesRead;
inputPointer.size = (uint)bytesRead;
inputPointer.used = 0;
if (bytesRead < inputBufferSize)
{
eof = true;
}
}
}
RdcError rdcErrorCode = RdcError.NoError;
//Force garbage collection.
GC.Collect();
// Wait for all finalizers to complete before continuing.
// Without this call to GC.WaitForPendingFinalizers,
// the worker loop below might execute at the same time
// as the finalizers.
// With this call, the worker loop executes only after
// all finalizers have been called.
GC.WaitForPendingFinalizers();
hr = rdcGenerator.Process(
eof,
ref eofOutput,
ref inputPointer,
(uint)recursionDepth,
outputPointers,
out rdcErrorCode);
if (hr != 0 || rdcErrorCode != RdcError.NoError)
{
throw new RdcException("RdcGenerator failed to process the signature block.", rdcErrorCode);
}
RdcBufferTranslate(outputPointers, outputPointer);
for (int i = 0; i < recursionDepth; i++)
{
int bytesWritten = 0;
// Write the signature block to the file.
bytesWritten = IntPtrCopy(
outputBuffers[i],
signatures[i].InnerStream,
0,
(int)outputPointer[i].used);
signatures[i].InnerStream.Flush();
signatures[i].Length += bytesWritten;
if (outputPointer[i].used != bytesWritten)
{
throw new RdcException("Failed to write to the signature file.");
}
outputPointer[i].used = 0;
}
RdcBufferTranslate(outputPointer, outputPointers);
}
Marshal.ReleaseComObject(rdcGenerator);
rdcGenerator = null;
if (inputBuffer != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(inputBuffer);
}
// To make it easier on the consuming application,
// reverse the order of the signatures in our collection.
signatures.Reverse();
return(signatures);
}
