private long GetTransmissionLengthInternal(FileObjectStoreData storeData) { if (storeData.BlobID.HasValue) return BlobDatabase.Get<Blobsize>(x => x.BlobID == storeData.BlobID.Value).Length; return GetFileForDataID(storeData.Lookup).Length; }
protected void WriteRecordStream(FileObjectStoreData storeData, System.IO.Stream outputStream) { System.IO.Stream dataStream; if (storeData.Mode == StorageMode.Delta) { Stream baseStream; FileInfo tempBaseFile; dataStream = OpenDeltaCodecStream(OpenLegacyStream(storeData), out baseStream, out tempBaseFile); ChunkedChecksum.ApplyDelta(baseStream, dataStream, outputStream); dataStream.Dispose(); baseStream.Dispose(); tempBaseFile.Delete(); } else { if (storeData.Mode == StorageMode.Legacy) dataStream = new LZHAMLegacyStream(OpenLegacyStream(storeData), true); else if (storeData.Mode == StorageMode.Flat) dataStream = OpenCodecStream(OpenLegacyStream(storeData)); else throw new Exception(); Printer.InteractivePrinter printer = null; long total = 0; byte[] dataBlob = new byte[16 * 1024 * 1024]; while (true) { var res = dataStream.Read(dataBlob, 0, dataBlob.Length); if (res == 0) break; if (printer != null) printer.Update(total); outputStream.Write(dataBlob, 0, res); total += res; } if (printer != null) printer.End(total); dataStream.Dispose(); } }
public override bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; dependency = null; lock (trans) { trans.m_PendingCount++; Printer.PrintDiagnostics("Importing data for {0}", directName); FileObjectStoreData data = new FileObjectStoreData() { Lookup = directName, }; byte[] buffer = trans.ScratchBuffer; string filename = null; Stream outputStream; MemoryStream backingStore = null; if (dataStream.Length > 16 * 1024) { lock (this) { List<string> ignored; if (HasDataDirect(directName, out ignored)) throw new Exception(); do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Cleanup.Add(filename); string fn = Path.Combine(TempFolder.FullName, filename); outputStream = new FileInfo(fn).OpenWrite(); } else { backingStore = new MemoryStream(); outputStream = backingStore; } using (var fileOutput = outputStream) { bool readData = true; byte[] sig = new byte[8]; dataStream.Read(sig, 0, 4); if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'k') { data.Mode = StorageMode.Flat; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); if ((BitConverter.ToUInt32(sig, 0) & 0x8000) != 0) data.HasSignatureData = true; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } else if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'x') { data.Mode = StorageMode.Delta; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); int baseLookupLength = BitConverter.ToInt32(sig, 0); byte[] baseLookupData = new byte[baseLookupLength]; dataStream.Read(baseLookupData, 0, baseLookupData.Length); fileOutput.Write(baseLookupData, 0, baseLookupData.Length); string baseLookup = ASCIIEncoding.ASCII.GetString(baseLookupData); dependency = baseLookup; data.Mode = StorageMode.Delta; data.DeltaBase = baseLookup; } else { if (true) { data.Mode = StorageMode.Flat; Printer.PrintDiagnostics(" - Importing legacy record..."); fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); dataStream.Read(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); string importTemp; lock (this) { do { importTemp = Path.GetRandomFileName(); } while (TempFiles.Contains(importTemp)); TempFiles.Add(importTemp); } trans.Cleanup.Add(importTemp); importTemp = Path.Combine(TempFolder.FullName, importTemp); FileInfo importTempInfo = new FileInfo(importTemp); using (var fs = importTempInfo.Create()) using (LZHAMLegacyStream legacy = new LZHAMLegacyStream(dataStream, false, data.FileSize)) { while (true) { var read = legacy.Read(buffer, 0, buffer.Length); if (read == 0) break; fs.Write(buffer, 0, read); } } using (var fileInput = importTempInfo.OpenRead()) { int signature = (int)CompressionMode.LZHAM; bool computeSignature = data.FileSize > 1024 * 64; if (computeSignature) signature |= 0x8000; fileOutput.Write(BitConverter.GetBytes(signature), 0, 4); fileOutput.Write(BitConverter.GetBytes(data.FileSize), 0, 8); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); long resultSize = 0; LZHAMWriter.CompressToStream(data.FileSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput); } } else { fileOutput.Write(sig, 0, 4); data.Mode = StorageMode.Legacy; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } } while (readData) { var read = dataStream.Read(buffer, 0, buffer.Length); if (read == 0) break; fileOutput.Write(buffer, 0, read); } } trans.m_PendingBytes += data.FileSize; byte[] payload = null; if (backingStore != null) payload = backingStore.ToArray(); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Data = data, Filename = filename, Payload = payload } ); return true; } }
private ChunkedChecksum LoadSignature(FileObjectStoreData storeData) { if (storeData.Mode == StorageMode.Legacy) throw new Exception(); else if (storeData.Mode == StorageMode.Flat) { using (var stream = OpenLegacyStream(storeData)) { return LoadSignatureFromStream(stream); } } throw new Exception(); }
public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; lock (trans) { if (trans.Inputs.Contains(GetLookup(newRecord))) return true; string filename; lock (this) { if (HasData(newRecord)) return true; do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Inputs.Add(GetLookup(newRecord)); trans.m_PendingCount++; trans.m_PendingBytes += newRecord.Size; Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName); trans.Cleanup.Add(filename); long resultSize; string fn = Path.Combine(TempFolder.FullName, filename); if (priorRecord != null) { // try to delta encode it string priorLookup = GetLookup(priorRecord); var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup); if (priorData != null) { if (priorData.Mode == StorageMode.Delta) priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase); if (priorData.HasSignatureData) { try { var signature = LoadSignature(priorData); long deltaSize; List<ChunkedChecksum.FileBlock> blocks; Printer.PrintDiagnostics(" - Trying delta encoding"); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); using (var fileInput = fileEntry.Info.OpenRead()) { blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); } if (printer != null) { printer.End(newRecord.Size); printer = null; } // dont encode as delta unless we get a 50% saving if (deltaSize < fileEntry.Length / 2) { FileObjectStoreData data = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = false, Lookup = GetLookup(newRecord), Mode = StorageMode.Delta, DeltaBase = priorData.Lookup, Offset = 0 }; trans.Cleanup.Add(filename + ".delta"); Printer.PrintDiagnostics(" - Delta encoding"); using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite()) { ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks); } Printer.PrintDiagnostics(" - Compressing data"); deltaSize = new FileInfo(fn + ".delta").Length; using (var fileInput = new FileInfo(fn + ".delta").OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && deltaSize < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && deltaSize < 1024) cmode = CompressionMode.None; int sig = (int)cmode; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8); fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4); byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup); fileOutput.Write(lookupBytes, 0, lookupBytes.Length); if (deltaSize > 16 * 1024 * 1024) { printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize)); }, (obj) => { return (float)((long)obj / (double)deltaSize) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = deltaSize; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize)); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = data, Filename = filename } ); return true; } } catch { } } } } bool computeSignature = newRecord.Size > 1024 * 64; FileObjectStoreData storeData = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = computeSignature, Lookup = GetLookup(newRecord), Mode = StorageMode.Flat, Offset = 0 }; using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && newRecord.Size < 1024) cmode = CompressionMode.None; int sig = (int)cmode; if (computeSignature) sig |= 0x8000; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); if (printer != null) { printer.End(newRecord.Size); printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size)); }, (obj) => { return (float)((long)obj / (double)newRecord.Size) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = newRecord.Size; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : ""); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = storeData, Filename = filename } ); return true; } }
private void ImportRecordFromFlatStore(Record x) { if (x.HasData) { if (!CheckFileForDataIDExists(x.DataIdentifier)) return; var recordData = new FileObjectStoreData(); recordData.FileSize = x.Size; recordData.HasSignatureData = false; recordData.Lookup = x.DataIdentifier; recordData.Mode = StorageMode.Legacy; recordData.Offset = 0; recordData.PackFileID = null; try { ObjectDatabase.InsertSafe(recordData); } catch (SQLite.SQLiteException e) { if (e.Result != SQLite.SQLite3.Result.Constraint) throw; } } }
private Stream OpenLegacyStream(FileObjectStoreData storeData) { if (storeData.BlobID.HasValue) return new MemoryStream(BlobDatabase.Get<Blobject>(storeData.BlobID.Value).Data); FileInfo info = GetFileForDataID(storeData.Lookup); if (info == null) return null; return info.OpenRead(); }