public override bool EndStorageTransaction(ObjectStoreTransaction transaction) { lock (this) { return CompleteTransaction(transaction as StandardObjectStoreTransaction, false); } }
public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; lock (trans) { if (trans.Inputs.Contains(GetLookup(newRecord))) return true; string filename; lock (this) { if (HasData(newRecord)) return true; do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Inputs.Add(GetLookup(newRecord)); trans.m_PendingCount++; trans.m_PendingBytes += newRecord.Size; Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName); trans.Cleanup.Add(filename); long resultSize; string fn = Path.Combine(TempFolder.FullName, filename); if (priorRecord != null) { // try to delta encode it string priorLookup = GetLookup(priorRecord); var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup); if (priorData != null) { if (priorData.Mode == StorageMode.Delta) priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase); if (priorData.HasSignatureData) { try { var signature = LoadSignature(priorData); long deltaSize; List<ChunkedChecksum.FileBlock> blocks; Printer.PrintDiagnostics(" - Trying delta encoding"); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); using (var fileInput = fileEntry.Info.OpenRead()) { blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); } if (printer != null) { printer.End(newRecord.Size); printer = null; } // dont encode as delta unless we get a 50% saving if (deltaSize < fileEntry.Length / 2) { FileObjectStoreData data = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = false, Lookup = GetLookup(newRecord), Mode = StorageMode.Delta, DeltaBase = priorData.Lookup, Offset = 0 }; trans.Cleanup.Add(filename + ".delta"); Printer.PrintDiagnostics(" - Delta encoding"); using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite()) { ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks); } Printer.PrintDiagnostics(" - Compressing data"); deltaSize = new FileInfo(fn + ".delta").Length; using (var fileInput = new FileInfo(fn + ".delta").OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && deltaSize < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && deltaSize < 1024) cmode = CompressionMode.None; int sig = (int)cmode; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8); fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4); byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup); fileOutput.Write(lookupBytes, 0, lookupBytes.Length); if (deltaSize > 16 * 1024 * 1024) { printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize)); }, (obj) => { return (float)((long)obj / (double)deltaSize) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = deltaSize; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize)); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = data, Filename = filename } ); return true; } } catch { } } } } bool computeSignature = newRecord.Size > 1024 * 64; FileObjectStoreData storeData = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = computeSignature, Lookup = GetLookup(newRecord), Mode = StorageMode.Flat, Offset = 0 }; using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && newRecord.Size < 1024) cmode = CompressionMode.None; int sig = (int)cmode; if (computeSignature) sig |= 0x8000; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); if (printer != null) { printer.End(newRecord.Size); printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size)); }, (obj) => { return (float)((long)obj / (double)newRecord.Size) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = newRecord.Size; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : ""); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = storeData, Filename = filename } ); return true; } }
public override bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; dependency = null; lock (trans) { trans.m_PendingCount++; Printer.PrintDiagnostics("Importing data for {0}", directName); FileObjectStoreData data = new FileObjectStoreData() { Lookup = directName, }; byte[] buffer = trans.ScratchBuffer; string filename = null; Stream outputStream; MemoryStream backingStore = null; if (dataStream.Length > 16 * 1024) { lock (this) { List<string> ignored; if (HasDataDirect(directName, out ignored)) throw new Exception(); do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Cleanup.Add(filename); string fn = Path.Combine(TempFolder.FullName, filename); outputStream = new FileInfo(fn).OpenWrite(); } else { backingStore = new MemoryStream(); outputStream = backingStore; } using (var fileOutput = outputStream) { bool readData = true; byte[] sig = new byte[8]; dataStream.Read(sig, 0, 4); if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'k') { data.Mode = StorageMode.Flat; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); if ((BitConverter.ToUInt32(sig, 0) & 0x8000) != 0) data.HasSignatureData = true; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } else if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'x') { data.Mode = StorageMode.Delta; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); int baseLookupLength = BitConverter.ToInt32(sig, 0); byte[] baseLookupData = new byte[baseLookupLength]; dataStream.Read(baseLookupData, 0, baseLookupData.Length); fileOutput.Write(baseLookupData, 0, baseLookupData.Length); string baseLookup = ASCIIEncoding.ASCII.GetString(baseLookupData); dependency = baseLookup; data.Mode = StorageMode.Delta; data.DeltaBase = baseLookup; } else { if (true) { data.Mode = StorageMode.Flat; Printer.PrintDiagnostics(" - Importing legacy record..."); fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); dataStream.Read(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); string importTemp; lock (this) { do { importTemp = Path.GetRandomFileName(); } while (TempFiles.Contains(importTemp)); TempFiles.Add(importTemp); } trans.Cleanup.Add(importTemp); importTemp = Path.Combine(TempFolder.FullName, importTemp); FileInfo importTempInfo = new FileInfo(importTemp); using (var fs = importTempInfo.Create()) using (LZHAMLegacyStream legacy = new LZHAMLegacyStream(dataStream, false, data.FileSize)) { while (true) { var read = legacy.Read(buffer, 0, buffer.Length); if (read == 0) break; fs.Write(buffer, 0, read); } } using (var fileInput = importTempInfo.OpenRead()) { int signature = (int)CompressionMode.LZHAM; bool computeSignature = data.FileSize > 1024 * 64; if (computeSignature) signature |= 0x8000; fileOutput.Write(BitConverter.GetBytes(signature), 0, 4); fileOutput.Write(BitConverter.GetBytes(data.FileSize), 0, 8); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); long resultSize = 0; LZHAMWriter.CompressToStream(data.FileSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput); } } else { fileOutput.Write(sig, 0, 4); data.Mode = StorageMode.Legacy; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } } while (readData) { var read = dataStream.Read(buffer, 0, buffer.Length); if (read == 0) break; fileOutput.Write(buffer, 0, read); } } trans.m_PendingBytes += data.FileSize; byte[] payload = null; if (backingStore != null) payload = backingStore.ToArray(); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Data = data, Filename = filename, Payload = payload } ); return true; } }
public abstract bool FlushStorageTransaction(ObjectStoreTransaction transaction);
public abstract bool EndStorageTransaction(ObjectStoreTransaction transaction);
public abstract bool RecordData(ObjectStoreTransaction transaction, Objects.Record newRecord, Objects.Record priorRecord, Entry fileEntry);
public abstract bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency);
public abstract string CreateDataStream(ObjectStoreTransaction transaction, System.IO.Stream stream);