Ejemplo n.º 1
0
 private long GetTransmissionLengthInternal(FileObjectStoreData storeData)
 {
     if (storeData.BlobID.HasValue)
         return BlobDatabase.Get<Blobsize>(x => x.BlobID == storeData.BlobID.Value).Length;
     return GetFileForDataID(storeData.Lookup).Length;
 }
Ejemplo n.º 2
0
 protected void WriteRecordStream(FileObjectStoreData storeData, System.IO.Stream outputStream)
 {
     System.IO.Stream dataStream;
     if (storeData.Mode == StorageMode.Delta)
     {
         Stream baseStream;
         FileInfo tempBaseFile;
         dataStream = OpenDeltaCodecStream(OpenLegacyStream(storeData), out baseStream, out tempBaseFile);
         ChunkedChecksum.ApplyDelta(baseStream, dataStream, outputStream);
         dataStream.Dispose();
         baseStream.Dispose();
         tempBaseFile.Delete();
     }
     else
     {
         if (storeData.Mode == StorageMode.Legacy)
             dataStream = new LZHAMLegacyStream(OpenLegacyStream(storeData), true);
         else if (storeData.Mode == StorageMode.Flat)
             dataStream = OpenCodecStream(OpenLegacyStream(storeData));
         else
             throw new Exception();
         Printer.InteractivePrinter printer = null;
         long total = 0;
         byte[] dataBlob = new byte[16 * 1024 * 1024];
         while (true)
         {
             var res = dataStream.Read(dataBlob, 0, dataBlob.Length);
             if (res == 0)
                 break;
             if (printer != null)
                 printer.Update(total);
             outputStream.Write(dataBlob, 0, res);
             total += res;
         }
         if (printer != null)
             printer.End(total);
         dataStream.Dispose();
     }
 }
Ejemplo n.º 3
0
        public override bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency)
        {
            StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
            dependency = null;
            lock (trans)
            {
                trans.m_PendingCount++;
                Printer.PrintDiagnostics("Importing data for {0}", directName);
                FileObjectStoreData data = new FileObjectStoreData()
                {
                    Lookup = directName,
                };
                byte[] buffer = trans.ScratchBuffer;
                string filename = null;
                Stream outputStream;
                MemoryStream backingStore = null;
                if (dataStream.Length > 16 * 1024)
                {
                    lock (this)
                    {
                        List<string> ignored;
                        if (HasDataDirect(directName, out ignored))
                            throw new Exception();
                        do
                        {
                            filename = Path.GetRandomFileName();
                        } while (TempFiles.Contains(filename));
                        TempFiles.Add(filename);
                    }
                    trans.Cleanup.Add(filename);
                    string fn = Path.Combine(TempFolder.FullName, filename);
                    outputStream = new FileInfo(fn).OpenWrite();
                }
                else
                {
                    backingStore = new MemoryStream();
                    outputStream = backingStore;
                }
                using (var fileOutput = outputStream)
                {
                    bool readData = true;
                    byte[] sig = new byte[8];
                    dataStream.Read(sig, 0, 4);
                    if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'k')
                    {
                        data.Mode = StorageMode.Flat;
                        fileOutput.Write(sig, 0, 4);
                        dataStream.Read(sig, 0, 4);
                        fileOutput.Write(sig, 0, 4);
                        if ((BitConverter.ToUInt32(sig, 0) & 0x8000) != 0)
                            data.HasSignatureData = true;
                        dataStream.Read(sig, 0, 8);
                        fileOutput.Write(sig, 0, 8);
                        data.FileSize = BitConverter.ToInt64(sig, 0);
                    }
                    else if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'x')
                    {
                        data.Mode = StorageMode.Delta;
                        fileOutput.Write(sig, 0, 4);

                        dataStream.Read(sig, 0, 4);
                        fileOutput.Write(sig, 0, 4);

                        dataStream.Read(sig, 0, 8);
                        fileOutput.Write(sig, 0, 8);
                        data.FileSize = BitConverter.ToInt64(sig, 0);

                        dataStream.Read(sig, 0, 8);
                        fileOutput.Write(sig, 0, 8);

                        dataStream.Read(sig, 0, 4);
                        fileOutput.Write(sig, 0, 4);

                        int baseLookupLength = BitConverter.ToInt32(sig, 0);
                        byte[] baseLookupData = new byte[baseLookupLength];
                        dataStream.Read(baseLookupData, 0, baseLookupData.Length);
                        fileOutput.Write(baseLookupData, 0, baseLookupData.Length);
                        string baseLookup = ASCIIEncoding.ASCII.GetString(baseLookupData);

                        dependency = baseLookup;

                        data.Mode = StorageMode.Delta;
                        data.DeltaBase = baseLookup;
                    }
                    else
                    {
                        if (true)
                        {
                            data.Mode = StorageMode.Flat;
                            Printer.PrintDiagnostics(" - Importing legacy record...");
                            fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
                            dataStream.Read(sig, 0, 8);
                            data.FileSize = BitConverter.ToInt64(sig, 0);

                            string importTemp;
                            lock (this)
                            {
                                do
                                {
                                    importTemp = Path.GetRandomFileName();
                                } while (TempFiles.Contains(importTemp));
                                TempFiles.Add(importTemp);
                            }
                            trans.Cleanup.Add(importTemp);
                            importTemp = Path.Combine(TempFolder.FullName, importTemp);
                            FileInfo importTempInfo = new FileInfo(importTemp);
                            using (var fs = importTempInfo.Create())
                            using (LZHAMLegacyStream legacy = new LZHAMLegacyStream(dataStream, false, data.FileSize))
                            {
                                while (true)
                                {
                                    var read = legacy.Read(buffer, 0, buffer.Length);
                                    if (read == 0)
                                        break;
                                    fs.Write(buffer, 0, read);
                                }
                            }
                            using (var fileInput = importTempInfo.OpenRead())
                            {
                                int signature = (int)CompressionMode.LZHAM;
                                bool computeSignature = data.FileSize > 1024 * 64;
                                if (computeSignature)
                                    signature |= 0x8000;
                                fileOutput.Write(BitConverter.GetBytes(signature), 0, 4);
                                fileOutput.Write(BitConverter.GetBytes(data.FileSize), 0, 8);
                                if (computeSignature)
                                {
                                    Printer.PrintDiagnostics(" - Computing signature");
                                    var checksum = ChunkedChecksum.Compute(1024, fileInput);
                                    fileInput.Position = 0;
                                    ChunkedChecksum.Write(fileOutput, checksum);
                                }
                                Printer.PrintDiagnostics(" - Compressing data");
                                long resultSize = 0;
                                LZHAMWriter.CompressToStream(data.FileSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput);
                            }
                        }
                        else
                        {
                            fileOutput.Write(sig, 0, 4);
                            data.Mode = StorageMode.Legacy;
                            dataStream.Read(sig, 0, 8);
                            fileOutput.Write(sig, 0, 8);
                            data.FileSize = BitConverter.ToInt64(sig, 0);
                        }
                    }
                    while (readData)
                    {
                        var read = dataStream.Read(buffer, 0, buffer.Length);
                        if (read == 0)
                            break;
                        fileOutput.Write(buffer, 0, read);
                    }
                }
                trans.m_PendingBytes += data.FileSize;
                byte[] payload = null;
                if (backingStore != null)
                    payload = backingStore.ToArray();
                trans.PendingTransactions.Add(
                    new StandardObjectStoreTransaction.PendingTransaction()
                    {
                        Data = data,
                        Filename = filename,
                        Payload = payload
                    }
                );
                return true;
            }
        }
Ejemplo n.º 4
0
 private ChunkedChecksum LoadSignature(FileObjectStoreData storeData)
 {
     if (storeData.Mode == StorageMode.Legacy)
         throw new Exception();
     else if (storeData.Mode == StorageMode.Flat)
     {
         using (var stream = OpenLegacyStream(storeData))
         {
             return LoadSignatureFromStream(stream);
         }
     }
     throw new Exception();
 }
Ejemplo n.º 5
0
        public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry)
        {
            StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
            lock (trans)
            {
                if (trans.Inputs.Contains(GetLookup(newRecord)))
                    return true;
                string filename;
                lock (this)
                {
                    if (HasData(newRecord))
                        return true;
                    do
                    {
                        filename = Path.GetRandomFileName();
                    } while (TempFiles.Contains(filename));
                    TempFiles.Add(filename);
                }
                trans.Inputs.Add(GetLookup(newRecord));
                trans.m_PendingCount++;
                trans.m_PendingBytes += newRecord.Size;
                Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName);
                trans.Cleanup.Add(filename);
                long resultSize;
                string fn = Path.Combine(TempFolder.FullName, filename);
                if (priorRecord != null)
                {
                    // try to delta encode it
                    string priorLookup = GetLookup(priorRecord);
                    var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup);
                    if (priorData != null)
                    {
                        if (priorData.Mode == StorageMode.Delta)
                            priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase);
                        if (priorData.HasSignatureData)
                        {
                            try
                            {
                                var signature = LoadSignature(priorData);
                                long deltaSize;
                                List<ChunkedChecksum.FileBlock> blocks;
                                Printer.PrintDiagnostics(" - Trying delta encoding");
                                Printer.InteractivePrinter printer = null;
                                if (newRecord.Size > 16 * 1024 * 1024)
                                    printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
                                using (var fileInput = fileEntry.Info.OpenRead())
                                {
                                    blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
                                }
                                if (printer != null)
                                {
                                    printer.End(newRecord.Size);
                                    printer = null;
                                }
                                // dont encode as delta unless we get a 50% saving
                                if (deltaSize < fileEntry.Length / 2)
                                {
                                    FileObjectStoreData data = new FileObjectStoreData()
                                    {
                                        FileSize = newRecord.Size,
                                        HasSignatureData = false,
                                        Lookup = GetLookup(newRecord),
                                        Mode = StorageMode.Delta,
                                        DeltaBase = priorData.Lookup,
                                        Offset = 0
                                    };
                                    trans.Cleanup.Add(filename + ".delta");
                                    Printer.PrintDiagnostics(" - Delta encoding");
                                    using (var fileInput = fileEntry.Info.OpenRead())
                                    using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite())
                                    {
                                        ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks);
                                    }
                                    Printer.PrintDiagnostics(" - Compressing data");
                                    deltaSize = new FileInfo(fn + ".delta").Length;
                                    using (var fileInput = new FileInfo(fn + ".delta").OpenRead())
                                    using (var fileOutput = new FileInfo(fn).OpenWrite())
                                    {
                                        fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4);
                                        CompressionMode cmode = DefaultCompression;
                                        if (cmode != CompressionMode.None && deltaSize < 16 * 1024)
                                            cmode = CompressionMode.LZ4;
                                        if (cmode != CompressionMode.None && deltaSize < 1024)
                                            cmode = CompressionMode.None;
                                        int sig = (int)cmode;
                                        fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
                                        fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
                                        fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8);
                                        fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4);
                                        byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup);
                                        fileOutput.Write(lookupBytes, 0, lookupBytes.Length);
                                        if (deltaSize > 16 * 1024 * 1024)
                                        {
                                            printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
                                            {
                                                return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize));
                                            },
                                            (obj) =>
                                            {
                                                return (float)((long)obj / (double)deltaSize) * 100.0f;
                                            },
                                            (obj, lol) => { return string.Empty; }, 40);
                                        }
                                        if (cmode == CompressionMode.LZHAM)
                                            LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else if (cmode == CompressionMode.LZ4)
                                            LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else if (cmode == CompressionMode.LZ4HC)
                                            LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else
                                        {
                                            resultSize = deltaSize;
                                            fileInput.CopyTo(fileOutput);
                                        }
                                        if (printer != null)
                                            printer.End(newRecord.Size);
                                    }
                                    Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize));
                                    trans.PendingTransactions.Add(
                                        new StandardObjectStoreTransaction.PendingTransaction()
                                        {
                                            Record = newRecord,
                                            Data = data,
                                            Filename = filename
                                        }
                                    );
                                    return true;
                                }
                            }
                            catch
                            {

                            }
                        }
                    }
                }
                bool computeSignature = newRecord.Size > 1024 * 64;
                FileObjectStoreData storeData = new FileObjectStoreData()
                {
                    FileSize = newRecord.Size,
                    HasSignatureData = computeSignature,
                    Lookup = GetLookup(newRecord),
                    Mode = StorageMode.Flat,
                    Offset = 0
                };
                using (var fileInput = fileEntry.Info.OpenRead())
                using (var fileOutput = new FileInfo(fn).OpenWrite())
                {
                    fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
                    CompressionMode cmode = DefaultCompression;
                    if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024)
                        cmode = CompressionMode.LZ4;
                    if (cmode != CompressionMode.None && newRecord.Size < 1024)
                        cmode = CompressionMode.None;
                    int sig = (int)cmode;
                    if (computeSignature)
                        sig |= 0x8000;
                    fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
                    fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
                    Printer.InteractivePrinter printer = null;
                    if (newRecord.Size > 16 * 1024 * 1024)
                        printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
                    if (computeSignature)
                    {
                        Printer.PrintDiagnostics(" - Computing signature");
                        var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
                        fileInput.Position = 0;
                        ChunkedChecksum.Write(fileOutput, checksum);
                    }
                    Printer.PrintDiagnostics(" - Compressing data");
                    if (printer != null)
                    {
                        printer.End(newRecord.Size);
                        printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
                        {
                            return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size));
                        },
                        (obj) =>
                        {
                            return (float)((long)obj / (double)newRecord.Size) * 100.0f;
                        },
                        (obj, lol) => { return string.Empty; }, 40);
                    }
                    if (cmode == CompressionMode.LZHAM)
                        LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else if (cmode == CompressionMode.LZ4)
                        LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else if (cmode == CompressionMode.LZ4HC)
                        LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else
                    {
                        resultSize = newRecord.Size;
                        fileInput.CopyTo(fileOutput);
                    }
                    if (printer != null)
                        printer.End(newRecord.Size);
                }
                Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : "");
                trans.PendingTransactions.Add(
                    new StandardObjectStoreTransaction.PendingTransaction()
                    {
                        Record = newRecord,
                        Data = storeData,
                        Filename = filename
                    }
                );
                return true;
            }
        }
Ejemplo n.º 6
0
 private void ImportRecordFromFlatStore(Record x)
 {
     if (x.HasData)
     {
         if (!CheckFileForDataIDExists(x.DataIdentifier))
             return;
         var recordData = new FileObjectStoreData();
         recordData.FileSize = x.Size;
         recordData.HasSignatureData = false;
         recordData.Lookup = x.DataIdentifier;
         recordData.Mode = StorageMode.Legacy;
         recordData.Offset = 0;
         recordData.PackFileID = null;
         try
         {
             ObjectDatabase.InsertSafe(recordData);
         }
         catch (SQLite.SQLiteException e)
         {
             if (e.Result != SQLite.SQLite3.Result.Constraint)
                 throw;
         }
     }
 }
Ejemplo n.º 7
0
 private Stream OpenLegacyStream(FileObjectStoreData storeData)
 {
     if (storeData.BlobID.HasValue)
         return new MemoryStream(BlobDatabase.Get<Blobject>(storeData.BlobID.Value).Data);
     FileInfo info = GetFileForDataID(storeData.Lookup);
     if (info == null)
         return null;
     return info.OpenRead();
 }