public ClientDeduplicatorStream(IDataProcessorStream inputStream, uint storageNode, DedupIndex ddb) { this.inputStream = inputStream; this.currentPos = 0; this.length = 0; this.DedupedCount = 0; storageNodeId = storageNode; this.BlockMetadata = new List <IFileBlockMetadata>(); dedupedBlocks = new List <long>(); this.ddb = ddb; }
public static DedupIndex Instance(int bsId, bool forWriting) { if (currentBackupSet == 0) { currentBackupSet = bsId; } else if (bsId != currentBackupSet) // refuse to have multiple dedub db loaded simultaneously { throw new Exception("A dedup DB (#" + currentBackupSet + " is already loaded."); } if (_instance == null) { _instance = new DedupIndex(); long nbItems = _instance.Initialize(forWriting); Logger.Append(Severity.INFO, "Deduplication indexes DB ready (" + nbItems + " items)"); } return(_instance); }
internal DataPipeline(PipelineMode mode, DataProcessingFlags flags, int bsId, DedupIndex ddidx) : this(mode, flags) { this.bsId = bsId; ddb = ddidx; }
public void Dispose() { currentBackupSet = 0; index = null; _instance = null; }
internal DataPipeline_(PipelineMode mode, Session s, Backup b, DataProcessingFlags flags) { backup = b; storageSession = s; BinaryFormatter formatter = new BinaryFormatter(); BChunkHeader header = new BChunkHeader(); header.DataFlags = flags; header.Version = Utilities.PlatForm.Instance().NodeVersion; //header.TaskId = taskId; header.TaskId = b.TaskId; // end-of-chain stream sessionStream = new NetworkStream(storageSession.DataSocket); this.Flags = flags; #if DEBUG if (ConfigManager.GetValue("BENCHMARK") != null) { sessionStream = new DummyStream(); } #endif if (flags.HasFlag(DataProcessingFlags.CChecksum)) { finalStream = new NullSinkStream(new ChunkHasherStream(sessionStream), mode); } else { finalStream = new NullSinkStream(sessionStream, mode); // dummy dest stream } //firstStream = new NullSinkStream(); // test and benchmarking // top-of-chain streams firstStream = finalStream; if (flags.HasFlag(DataProcessingFlags.CEncrypt)) { EncryptorStream encStream = new EncryptorStream(firstStream, true, null); header.EncryptionMetaData = encStream.EncryptionMetadata; // TODO !! take encryptionMetadata and add it to index firstStream = encStream; } if (flags.HasFlag(DataProcessingFlags.CCompress)) { //firstStream = new CompressorStream(firstStream, CompressorAlgorithm.Lzo, 1024); firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress); } if (flags.HasFlag(DataProcessingFlags.CDedup)) { cdds = new ClientDeduplicatorStream(firstStream, s.ClientId); /*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution * firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds); * } * catch(Exception e){*/ firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds); //firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds); /*}*/ // Pre-Initialize dedup index (if needed) DedupIndex.Instance().Initialize(); } MemoryStream headerStream = new MemoryStream(); formatter.Serialize(headerStream, header); headerData = headerStream.ToArray(); Logger.Append(Severity.INFO, "Created data pipeline with flags " + flags.ToString()); //privilegesManager = new Utilities.PrivilegesManager(); }