public ClientDeduplicatorStream(IDataProcessorStream inputStream, uint storageNode, DedupIndex ddb)
{
this.inputStream = inputStream;
this.currentPos = 0;
this.length = 0;
this.DedupedCount = 0;
storageNodeId = storageNode;
this.BlockMetadata = new List <IFileBlockMetadata>();
dedupedBlocks = new List <long>();
this.ddb = ddb;
}
public static DedupIndex Instance(int bsId, bool forWriting)
{
if (currentBackupSet == 0)
{
currentBackupSet = bsId;
}
else if (bsId != currentBackupSet) // refuse to have multiple dedub db loaded simultaneously
{
throw new Exception("A dedup DB (#" + currentBackupSet + " is already loaded.");
}
if (_instance == null)
{
_instance = new DedupIndex();
long nbItems = _instance.Initialize(forWriting);
Logger.Append(Severity.INFO, "Deduplication indexes DB ready (" + nbItems + " items)");
}
return(_instance);
}
internal DataPipeline(PipelineMode mode, DataProcessingFlags flags, int bsId, DedupIndex ddidx) : this(mode, flags)
{
this.bsId = bsId;
ddb = ddidx;
}
public void Dispose()
{
currentBackupSet = 0;
index = null;
_instance = null;
}
internal DataPipeline_(PipelineMode mode, Session s, Backup b, DataProcessingFlags flags)
{
backup = b;
storageSession = s;
BinaryFormatter formatter = new BinaryFormatter();
BChunkHeader header = new BChunkHeader();
header.DataFlags = flags;
header.Version = Utilities.PlatForm.Instance().NodeVersion;
//header.TaskId = taskId;
header.TaskId = b.TaskId;
// end-of-chain stream
sessionStream = new NetworkStream(storageSession.DataSocket);
this.Flags = flags;
#if DEBUG
if (ConfigManager.GetValue("BENCHMARK") != null)
{
sessionStream = new DummyStream();
}
#endif
if (flags.HasFlag(DataProcessingFlags.CChecksum))
{
finalStream = new NullSinkStream(new ChunkHasherStream(sessionStream), mode);
}
else
{
finalStream = new NullSinkStream(sessionStream, mode); // dummy dest stream
}
//firstStream = new NullSinkStream(); // test and benchmarking
// top-of-chain streams
firstStream = finalStream;
if (flags.HasFlag(DataProcessingFlags.CEncrypt))
{
EncryptorStream encStream = new EncryptorStream(firstStream, true, null);
header.EncryptionMetaData = encStream.EncryptionMetadata;
// TODO !! take encryptionMetadata and add it to index
firstStream = encStream;
}
if (flags.HasFlag(DataProcessingFlags.CCompress))
{
//firstStream = new CompressorStream(firstStream, CompressorAlgorithm.Lzo, 1024);
firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress);
}
if (flags.HasFlag(DataProcessingFlags.CDedup))
{
cdds = new ClientDeduplicatorStream(firstStream, s.ClientId);
/*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution
* firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds);
* }
* catch(Exception e){*/
firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds);
//firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds);
/*}*/
// Pre-Initialize dedup index (if needed)
DedupIndex.Instance().Initialize();
}
MemoryStream headerStream = new MemoryStream();
formatter.Serialize(headerStream, header);
headerData = headerStream.ToArray();
Logger.Append(Severity.INFO, "Created data pipeline with flags " + flags.ToString());
//privilegesManager = new Utilities.PrivilegesManager();
}