public void Synchronize(string remoteFile, string seedFile, string outputFile)
{
this.remoteFile = remoteFile;
this.seedFile = seedFile;
this.outputFile = outputFile;
// Initialize our managed RDC wrapper
RdcServices rdcServices = new RdcServices();
SignatureCollection seedSignatures = null;
// Get the RDC version of the server so we
// can make sure this client is supported.
Client.RdcProxy.RdcService rdcWebService = new Client.RdcProxy.RdcService();
Client.RdcProxy.RdcVersion rdcVersion = rdcWebService.GetRdcVersion();
//rdcServices.CheckVersion(rdcVersion);
// Open the local seed file stream
using (FileStream stream = File.OpenRead(seedFile))
{
rdcServices.WorkingDirectory = workingDir;
rdcServices.RecursionDepth = recursionDepth;
// Generate the signature files
seedSignatures = rdcServices.GenerateSignatures(stream, Path.GetFileName(seedFile));
if (seedSignatures.Count < 1)
{
throw new RdcException("Failed to generate the signatures.");
}
}
// Now make the call the the Rdc web service
// and retrieve the copy of the signature
// manifest.
Client.RdcProxy.SignatureManifest signatureManifest = rdcWebService.GetSignatureManifest(remoteFile, recursionDepth);
SignatureCollection sourceSignatures = new SignatureCollection();
/*
* Realistically, the soure file length should be checked
* against a predetermined minimum limit. So if the source
* file length is less than 1MB, just download the source
* file instead of generating signaures and needs.
*/
//if signatureManifest.FileLength < MINIMUM_SIZE)
// DownloadSourceFile(...);
ulong TargetDataWritten = 0;
ulong TotalSourceData = 0;
ulong TotalSeedData = 0;
ulong TotalSigData = 0;
// Now that we have the signature manifiest, let's go ahead
// transfer local copies of the signature files to our
// working signature directory.
int Depth = 0;
foreach (Client.RdcProxy.SignatureInfo sig in signatureManifest.Signatures)
{
Console.WriteLine(string.Format("\n----------\nProcessing: {0}\n", sig.Name + ".sig"));
if (sig.Length > 0)
{
GC.WaitForPendingFinalizers();
// Create the signature stream
Microsoft.RDC.SignatureInfo sigInfo = new Microsoft.RDC.SignatureInfo(sig.Name, -1, workingDir, true);
sourceSignatures.Add(sigInfo); // hang on to them to keep them alive and for clean up
if (Depth == 0) // always transfer the complete first remote signature
{
Console.WriteLine(string.Format("Transfering: {0}\n", sig.Name + ".sig"));
for (int i = 0; i < sig.Length; i += blockSize)
{
int readBytes = Math.Min((int)(sig.Length - i), blockSize);
byte[] data = rdcWebService.TransferDataBlock(Path.Combine(sig.Path, sig.Name) + ".sig", i, readBytes);
sigInfo.InnerStream.Write(data, 0, data.Length);
TotalSigData += (ulong)data.Length;
}
}
// select source and target stream
FileStream SourceStream;
FileStream TargetStream;
string RemoteSourcePath;
// if there are other signatures after this one, they become the source and target
if (Depth < seedSignatures.Count - 1)
{
SourceStream = seedSignatures[Depth + 1].InnerStream;
TargetStream = File.Create(Path.Combine(workingDir, signatureManifest.Signatures[Depth + 1].Name) + ".sig", blockSize);
RemoteSourcePath = Path.Combine(signatureManifest.Signatures[Depth + 1].Path, signatureManifest.Signatures[Depth + 1].Name) + ".sig";
Console.WriteLine(string.Format("Creating: {0}\n----------\n\n", signatureManifest.Signatures[Depth + 1].Name + ".sig"));
}
else // create the final target file
{
SourceStream = File.OpenRead(seedFile);
TargetStream = File.Create(outputFile, blockSize);
RemoteSourcePath = remoteFile;
Console.WriteLine(string.Format("Creating: {0}\n----------\n\n", Path.GetFileName(outputFile)));
}
// reset signature streams for reading
seedSignatures[Depth].InnerStream.Position = 0;
sigInfo.InnerStream.Position = 0;
// Compare the signatures and get
// the needs array that we will use to create the
// target output file.
ArrayList needsList = rdcServices.CreateNeedsList(seedSignatures[Depth], sigInfo);
foreach (RdcNeed need in needsList)
{
switch (need.blockType)
{
case RdcNeedType.Source:
// Copy this block from the remote server.
TotalSourceData += need.blockLength;
byte[] data = rdcWebService.TransferDataBlock(
RemoteSourcePath,
(int)need.fileOffset,
(int)need.blockLength);
TargetStream.Write(data, 0, (int)need.blockLength);
break;
case RdcNeedType.Seed:
TotalSeedData += need.blockLength;
byte[] seedData = new Byte[need.blockLength];
SourceStream.Seek((int)need.fileOffset, SeekOrigin.Begin);
SourceStream.Read(seedData, 0, (int)need.blockLength);
TargetStream.Write(seedData, 0, (int)need.blockLength);
break;
default:
break;
}
Console.WriteLine(string.Format("NEED: length:{0,12}\toffset:{1,12}\tsource:{2,12}\tblock type:{3,12}", need.blockLength, need.fileOffset, TargetDataWritten, need.blockType.ToString()));
TargetDataWritten += need.blockLength;
}
// Close our IO file streams.
if (Depth == seedSignatures.Count - 1)
{
SourceStream.Close(); // only non-signature sources
}
TargetStream.Close();
}
Depth++;
}
Console.WriteLine(string.Format("\nFrom source:{0,12:N0}\tFrom seed:{1,12:N0}\tTotal:{2,12:N0}", TotalSourceData, TotalSeedData, TotalSourceData + TotalSeedData));
Console.WriteLine(string.Format("\nTransfer: {0:N0} bytes from source, file size: {1:N0}, RDC Savings: {2:0.00}%\n",
TotalSourceData + TotalSigData,
TotalSourceData + TotalSeedData,
(1.0 - (double)(TotalSourceData + TotalSigData) / (double)(TotalSourceData + TotalSeedData)) * 100.0));
// release all signature resources
rdcServices.PurgeSignatureStore(seedSignatures);
rdcServices.PurgeSignatureStore(sourceSignatures);
rdcWebService.Finialize(signatureManifest);
rdcServices.Dispose();
}
/// <summary>
/// Builds a comparator and perform the RDC comparison logic against the provided signatures to generate a needs list.
/// </summary>
/// <param name="seedSignature">Seed signature to compare</param>
/// <param name="sourceSignature">Source signature to compare</param>
/// <returns>RDC Needs list</returns>
public ArrayList CreateNeedsList(SignatureInfo seedSignature, SignatureInfo sourceSignature)
{
int hr = 0;
IRdcFileReader fileReader = (IRdcFileReader) new RdcFileReader(seedSignature.InnerStream);
IRdcComparator comparator;
GC.Collect();
GC.WaitForPendingFinalizers();
hr = rdcLibrary.CreateComparator(fileReader, 1000000, out comparator);
// Create and allocate memory for our input buffer.
IntPtr inputBuffer = Marshal.AllocCoTaskMem((int)inputBufferSize + 16);
RdcBufferPointer inputPointer = new RdcBufferPointer();
inputPointer.size = 0;
inputPointer.used = 0;
inputPointer.data = inputBuffer;
ArrayList needsList = new ArrayList();
IntPtr outputBuffer = Marshal.AllocCoTaskMem(Marshal.SizeOf(typeof(RdcNeed)) * 256);
RdcNeedPointer outputPointer = new RdcNeedPointer();
long totalBytesRead = 0;
bool eof = false;
bool eofOutput = false;
while (hr == 0 && !eofOutput)
{
if (inputPointer.size == inputPointer.used && !eof)
{
// Fill our input buffer with the signature
// data from the source.
// When the input buffer is completely empty
// refill it.
int bytesRead = 0;
try
{
bytesRead = IntPtrCopy(sourceSignature.InnerStream, inputBuffer, 0, (int)inputBufferSize);
}
catch (Exception ex)
{
// TODO: Cleanup
throw new RdcException("Failed to read from the source stream.", ex);
}
totalBytesRead += bytesRead;
inputPointer.size = (uint)bytesRead;
inputPointer.used = 0;
if (bytesRead < inputBufferSize)
{
eof = true;
}
}
// Initialize our output needs array
outputPointer.size = 256;
outputPointer.used = 0;
outputPointer.data = outputBuffer;
RdcError error = RdcError.NoError;
// Perform the signature comparison.
// This function may not produce needs output every time.
// Also, it may not use all available input each time either.
// You may call it with any number of input bytes
// and output needs array entries. Obviously, it is more
// efficient to give it reasonably sized buffers for each.
// This sample waits until Process() consumes an entire input
// buffer before reading more data from the source signatures file.
// Continue calling this function until it sets "eofOutput" to true.
hr = comparator.Process(
eof,
ref eofOutput,
ref inputPointer,
ref outputPointer,
out error);
if (hr != 0)
{
throw new RdcException("Failed to process the signature block!");
}
if (error != RdcError.NoError)
{
throw new RdcException("Failed!");
}
// Convert the stream to a Needs array.
RdcNeed[] needs = GetRdcNeedList(outputPointer);
foreach (RdcNeed need in needs)
{
// assign the needs to our arraylist.
needsList.Add(need);
}
}
if (hr != 0)
{
throw new RdcException("Failed!");
}
// Free our resources
if (outputBuffer != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(outputBuffer);
}
if (inputBuffer != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(inputBuffer);
}
return(needsList);
}
