示例#1
0
        internal void Init()
        {
            Logger.Append(Severity.INFO, "Creating data pipeline with mode = " + this.Mode + ", flags = '" + this.Flags.ToString() + "'");
#if DEBUG
            if (ConfigManager.GetValue("BENCHMARK") != null)
            {
                this.OutputStream = new DummyStream();
            }
#endif
            if (this.Flags.HasFlag(DataProcessingFlags.CChecksum))
            {
                counterStream = new NullSinkStream(new ChunkHasherStream(this.OutputStream), this.Mode);
            }
            else
            {
                counterStream = new NullSinkStream(this.OutputStream, this.Mode);
            }
            finalStream = counterStream;


            // top-of-chain streams
            firstStream = finalStream;
            if (this.Flags.HasFlag(DataProcessingFlags.CEncrypt) || this.Flags.HasFlag(DataProcessingFlags.SEncrypt))
            {
                if (this.Mode == PipelineMode.Read)
                {
                    throw new NotImplementedException("PipeLine read mode with decryption not yet implemented");
                }
                else
                {
                    Console.WriteLine("Pipeline.init() : this.CryptoKey=" + this.CryptoKey);
                    EncryptorStream encStream = new EncryptorStream(firstStream, true, this.CryptoKey, this.IV);
                    this.EncryptionMetaData = encStream.EncryptionMetadata;
                    // TODO !! take encryptionMetadata and add it to index
                    firstStream = encStream;
                }
            }
            if (this.Flags.HasFlag(DataProcessingFlags.CCompress) || this.Flags.HasFlag(DataProcessingFlags.SCompress))
            {
                if (this.Mode == PipelineMode.Read)
                {
                    firstStream = new LZ4Decompressor(firstStream);
                    //firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Decompress);
                }
                else
                {
                    //firstStream = new QuickLZCompressionStream(firstStream);
                    //firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress);
                    firstStream = new LZ4CompressorStream(firstStream);
                }
            }
            if (this.Flags.HasFlag(DataProcessingFlags.CDedup) || this.Flags.HasFlag(DataProcessingFlags.SDedup))
            {
                cdds = new ClientDeduplicatorStream(firstStream, this.StorageNode, ddb /*DedupIndex.Instance(0, true)*/);

                /*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution
                 *      firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds);
                 * }
                 * catch(Exception e){*/
                firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds);
                //firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds);
            }
        }
示例#2
0
        internal DataPipeline_(PipelineMode mode, Session s, Backup b, DataProcessingFlags flags)
        {
            backup = b;

            storageSession = s;
            BinaryFormatter formatter = new BinaryFormatter();
            BChunkHeader    header    = new BChunkHeader();

            header.DataFlags = flags;

            header.Version = Utilities.PlatForm.Instance().NodeVersion;
            //header.TaskId = taskId;
            header.TaskId = b.TaskId;

            // end-of-chain stream
            sessionStream = new NetworkStream(storageSession.DataSocket);

            this.Flags = flags;
#if DEBUG
            if (ConfigManager.GetValue("BENCHMARK") != null)
            {
                sessionStream = new DummyStream();
            }
#endif
            if (flags.HasFlag(DataProcessingFlags.CChecksum))
            {
                finalStream = new NullSinkStream(new ChunkHasherStream(sessionStream), mode);
            }
            else
            {
                finalStream = new NullSinkStream(sessionStream, mode);                // dummy dest stream
            }
            //firstStream = new NullSinkStream(); // test and benchmarking

            // top-of-chain streams
            firstStream = finalStream;
            if (flags.HasFlag(DataProcessingFlags.CEncrypt))
            {
                EncryptorStream encStream = new EncryptorStream(firstStream, true, null);
                header.EncryptionMetaData = encStream.EncryptionMetadata;

                // TODO !! take encryptionMetadata and add it to index
                firstStream = encStream;
            }
            if (flags.HasFlag(DataProcessingFlags.CCompress))
            {
                //firstStream = new CompressorStream(firstStream, CompressorAlgorithm.Lzo, 1024);
                firstStream = new GZCompressorStream(firstStream, System.IO.Compression.CompressionMode.Compress);
            }
            if (flags.HasFlag(DataProcessingFlags.CDedup))
            {
                cdds = new ClientDeduplicatorStream(firstStream, s.ClientId);

                /*try{ // TODO ! remove cksum provider selection from here, find a more elegant solution
                 *      firstStream = new ChecksummerStream_MHash((ClientDeduplicatorStream)cdds);
                 * }
                 * catch(Exception e){*/
                firstStream = new ChecksummerStream((ClientDeduplicatorStream)cdds);
                //firstStream = new TigerTreeHasherStream((ClientDeduplicatorStream)cdds);

                /*}*/
                // Pre-Initialize dedup index (if needed)
                DedupIndex.Instance().Initialize();
            }
            MemoryStream headerStream = new MemoryStream();
            formatter.Serialize(headerStream, header);
            headerData = headerStream.ToArray();
            Logger.Append(Severity.INFO, "Created data pipeline with flags " + flags.ToString());
            //privilegesManager = new Utilities.PrivilegesManager();
        }