public ChecksummerStream(ClientDeduplicatorStream inputStream)
{
this.outputStream = inputStream;
this.currentPos = 0;
this.length = 0;
hasher = MD5CryptoServiceProvider.Create();
this.BlockMetadata = new List <IFileBlockMetadata>();
}
public ChecksummerStream_MHash(ClientDeduplicatorStream inputStream){
this.inputStream = inputStream;
this.currentPos = 0;
this.length = 0;
md5 = MD5Native.Create();
this.BlockMetadata = new List<IFileBlockMetadata>();
}
public TigerTreeHasherStream(ClientDeduplicatorStream inputStream, HashAlgorithm hashAlgorithm, int maxBlockSize)
{
this.inputStream = inputStream;
this.currentPos = 0;
this.length = 0;
Block_Size = maxBlockSize;
hashAlgo = hashAlgorithm;
hashAlgo.Initialize();
LeafCollection = new List <HashHolder>();
this.BlockMetadata = new List <IFileBlockMetadata>();
data = new byte[Block_Size * 2 + 2]; // holds 2 data nodes + leaf marks
}
internal void Init()
{
Logger.Append(Severity.INFO, "Creating data pipeline with mode = " + this.Mode + ", flags = '" + this.Flags.ToString() + "'");
#if DEBUG
if (ConfigManager.GetValue("BENCHMARK") != null)
{
this.OutputStream = new DummyStream();
}
#endif
if (this.Flags.HasFlag(DataProcessingFlags.CChecksum))
{
counterStream = new NullSinkStream(new ChunkHasherStream(this.OutputStream), this.Mode);
}
else
{
counterStream = new NullSinkStream(this.OutputStream, this.Mode);
}
finalStream = counterStream;
// top-of-chain streams
firstStream = finalStream;
if (this.Flags.HasFlag(DataProcessingFlags.CEncrypt) || this.Flags.HasFlag(DataProcessingFlags.SEncrypt))
{
if (this.Mode == PipelineMode.Read)
{
throw new NotImplementedException("PipeLine read mode with decryption not yet implemented");
}
else
{
Console.WriteLine("Pipeline.init() : this.CryptoKey=" + this.CryptoKey);
EncryptorStream encStream = new EncryptorStream(firstStream, true, this.CryptoKey, this.IV);
this.EncryptionMetaData = encStream.EncryptionMetadata;
// TODO !! take encryptionMetadata and add it to index
firstStream = encStream;
}
}
if (this.Flags.HasFlag(DataProcessingFlags.CCompress) || this.Flags.HasFlag(DataProcessingFlags.SCompress))
{
if (this.Mode == PipelineMode.Read)
{
firstStream = new LZ4Decompressor(firstStream);
//firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Decompress);
}
else
{
//firstStream = new QuickLZCompressionStream(firstStream);
//firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress);
firstStream = new LZ4CompressorStream(firstStream);
}
}
if (this.Flags.HasFlag(DataProcessingFlags.CDedup) || this.Flags.HasFlag(DataProcessingFlags.SDedup))
{
cdds = new ClientDeduplicatorStream(firstStream, this.StorageNode, ddb /*DedupIndex.Instance(0, true)*/);
/*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution
* firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds);
* }
* catch(Exception e){*/
firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds);
//firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds);
}
}
internal DataPipeline_(PipelineMode mode, Session s, Backup b, DataProcessingFlags flags)
{
backup = b;
storageSession = s;
BinaryFormatter formatter = new BinaryFormatter();
BChunkHeader header = new BChunkHeader();
header.DataFlags = flags;
header.Version = Utilities.PlatForm.Instance().NodeVersion;
//header.TaskId = taskId;
header.TaskId = b.TaskId;
// end-of-chain stream
sessionStream = new NetworkStream(storageSession.DataSocket);
this.Flags = flags;
#if DEBUG
if (ConfigManager.GetValue("BENCHMARK") != null)
{
sessionStream = new DummyStream();
}
#endif
if (flags.HasFlag(DataProcessingFlags.CChecksum))
{
finalStream = new NullSinkStream(new ChunkHasherStream(sessionStream), mode);
}
else
{
finalStream = new NullSinkStream(sessionStream, mode); // dummy dest stream
}
//firstStream = new NullSinkStream(); // test and benchmarking
// top-of-chain streams
firstStream = finalStream;
if (flags.HasFlag(DataProcessingFlags.CEncrypt))
{
EncryptorStream encStream = new EncryptorStream(firstStream, true, null);
header.EncryptionMetaData = encStream.EncryptionMetadata;
// TODO !! take encryptionMetadata and add it to index
firstStream = encStream;
}
if (flags.HasFlag(DataProcessingFlags.CCompress))
{
//firstStream = new CompressorStream(firstStream, CompressorAlgorithm.Lzo, 1024);
firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress);
}
if (flags.HasFlag(DataProcessingFlags.CDedup))
{
cdds = new ClientDeduplicatorStream(firstStream, s.ClientId);
/*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution
* firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds);
* }
* catch(Exception e){*/
firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds);
//firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds);
/*}*/
// Pre-Initialize dedup index (if needed)
DedupIndex.Instance().Initialize();
}
MemoryStream headerStream = new MemoryStream();
formatter.Serialize(headerStream, header);
headerData = headerStream.ToArray();
Logger.Append(Severity.INFO, "Created data pipeline with flags " + flags.ToString());
//privilegesManager = new Utilities.PrivilegesManager();
}