public AlterationVM(Alteration alteration, Area area, Version version) { _alteration = alteration; _area = area; _version = version; if (_alteration.PriorRecord.HasValue) _priorRecord = _area.GetRecord(_alteration.PriorRecord.Value); if (_alteration.NewRecord.HasValue) _newRecord = _area.GetRecord(_alteration.NewRecord.Value); DiffWithPreviousCommand = new DelegateCommand(DiffWithPrevious, CanDiffWithPrevious); DiffWithCurrentCommand = new DelegateCommand(DiffWithCurrent, CanDiffWithCurrent); LogCommand = new DelegateCommand(Log); SaveVersionAsCommand = new DelegateCommand(SaveVersionAs, CanSaveVersionAs); }
public static void CheckRecord(SharedNetworkInfo sharedInfo, Record record) { if (record == null) return; sharedInfo.RemoteRecordMap[record.Id] = record; if (!sharedInfo.UnknownRecordSet.Contains(record.Id)) { var localRecord = sharedInfo.Workspace.LocateRecord(record); if (localRecord == null) { Printer.PrintDiagnostics("Received version dependent on missing record {0}", record.UniqueIdentifier); sharedInfo.UnknownRecords.Add(record.Id); sharedInfo.UnknownRecordSet.Add(record.Id); } else { localRecord.CanonicalName = record.CanonicalName; sharedInfo.LocalRecordMap[record.Id] = localRecord; } } }
public override void WriteRecordStream(Record record, System.IO.Stream outputStream) { string lookup = GetLookup(record); var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup); WriteRecordStream(storeData, outputStream); }
private Stream GetDataStream(Record record) { string lookup = GetLookup(record); var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup); return OpenLegacyStream(storeData); throw new Exception(); }
public override bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null) { if (!record.HasData) { return true; } long dataSize = GetTransmissionLength(record); if (dataSize == -1) return false; using (System.IO.Stream dataStream = GetDataStream(record)) { if (dataStream == null) return false; if (beginTransmission != null) beginTransmission(); sender(BitConverter.GetBytes(dataSize), 8, false); while (true) { var readCount = dataStream.Read(scratchBuffer, 0, scratchBuffer.Length); if (readCount == 0) break; sender(scratchBuffer, readCount, false); } } return true; }
public override long GetTransmissionLength(Record record) { if (!record.HasData) return 0; string lookup = GetLookup(record); var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup); if (storeData == null) return -1; return GetTransmissionLengthInternal(storeData); }
public abstract long GetTransmissionLength(Record record);
private Record DeserializeRecord(BinaryReader br) { Record rec = new Record(); rec.Id = br.ReadInt64(); long parent = br.ReadInt64(); if (parent != -1) rec.Parent = parent; rec.Size = br.ReadInt64(); rec.Attributes = (Attributes)br.ReadUInt32(); rec.Fingerprint = br.ReadString(); rec.CanonicalNameId = br.ReadInt64(); rec.ModificationTime = new DateTime(br.ReadInt64()); rec.CanonicalName = br.ReadString(); return rec; }
private void ImportRecordFromFlatStore(Record x) { if (x.HasData) { if (!CheckFileForDataIDExists(x.DataIdentifier)) return; var recordData = new FileObjectStoreData(); recordData.FileSize = x.Size; recordData.HasSignatureData = false; recordData.Lookup = x.DataIdentifier; recordData.Mode = StorageMode.Legacy; recordData.Offset = 0; recordData.PackFileID = null; try { ObjectDatabase.InsertSafe(recordData); } catch (SQLite.SQLiteException e) { if (e.Result != SQLite.SQLite3.Result.Constraint) throw; } } }
internal override RecordInfo GetInfo(Record x) { if (!x.HasData) { return new RecordInfo() { AllocatedSize = 0, DeltaCompressed = false, ID = -1, }; } string lookup = GetLookup(x); var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup); if (storeData == null) return null; return new RecordInfo() { AllocatedSize = GetTransmissionLengthInternal(storeData), DeltaCompressed = storeData.Mode == StorageMode.Delta, ID = storeData.ID }; }
private string GetLookup(Record record) { return record.Fingerprint + "-" + record.Size.ToString(); }
private List <Record> Consolidate(List <Record> baseList, List <Alteration> alterations, List <Record> deletions) { System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); sw.Start(); Dictionary <long, Record> records = new Dictionary <long, Record>(); foreach (var x in baseList) { records[x.Id] = x; } #if FULL_CONSOLIDATE_DEBUG Printer.PrintMessage("Initial Snapshot Contains: "); foreach (var x in baseList) { Printer.PrintMessage("Initial: ({2}) {0} - {1}", x.CanonicalName, x.UniqueIdentifier, x.Id); } #endif List <long> pending = new List <long>(); CacheRecords(alterations.Select(x => x.NewRecord).Where(x => x.HasValue).Select(x => x.Value)); foreach (var x in alterations.Select(x => x).Reverse()) { Objects.Record rec = null; switch (x.Type) { case AlterationType.Add: case AlterationType.Copy: { var record = GetCachedRecord(x.NewRecord.Value); #if FULL_CONSOLIDATE_DEBUG Printer.PrintMessage("Add: ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id); if (records.ContainsKey(record.Id)) { Printer.PrintMessage(" (error, already present)"); } #endif records[record.Id] = record; break; } case AlterationType.Move: { var record = GetCachedRecord(x.NewRecord.Value); if (deletions != null) { rec = Get <Objects.Record>(x.PriorRecord); rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName; deletions.Add(rec); } #if FULL_CONSOLIDATE_DEBUG rec = Get <Objects.Record>(x.PriorRecord); rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName; Printer.PrintMessage("Move (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id); if (!records.ContainsKey(rec.Id)) { Printer.PrintMessage(" (error, not present)"); } Printer.PrintMessage("Move (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id); if (records.ContainsKey(record.Id)) { Printer.PrintMessage(" (error, already present)"); } #endif if (!records.Remove(x.PriorRecord.Value)) { throw new Exception("Consistency constraint invalid!"); } records[record.Id] = record; break; } case AlterationType.Update: { var record = GetCachedRecord(x.NewRecord.Value); #if FULL_CONSOLIDATE_DEBUG rec = Get <Objects.Record>(x.PriorRecord); rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName; Printer.PrintMessage("Update (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id); if (!records.ContainsKey(rec.Id)) { Printer.PrintMessage(" (error, not present)"); } Printer.PrintMessage("Update (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id); if (records.ContainsKey(record.Id)) { Printer.PrintMessage(" (error, already present)"); } #endif if (!records.Remove(x.PriorRecord.Value)) { throw new Exception(); } records[record.Id] = record; break; } case AlterationType.Discard: break; case AlterationType.Delete: if (deletions != null) { rec = Get <Objects.Record>(x.PriorRecord); rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName; deletions.Add(rec); } #if FULL_CONSOLIDATE_DEBUG rec = Get <Objects.Record>(x.PriorRecord); rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName; Printer.PrintMessage("Delete: ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id); if (!records.ContainsKey(rec.Id)) { Printer.PrintMessage(" (error, not present)"); } #endif if (!records.Remove(x.PriorRecord.Value)) { throw new Exception("Consistency constraint invalid!"); } break; default: throw new Exception(); } } var result = records.Select(x => x.Value).ToList(); #if DEBUG HashSet <string> namecheck = new HashSet <string>(); foreach (var x in result) { if (namecheck.Contains(x.CanonicalName)) { throw new Exception("Inconsistency in internal state!"); } namecheck.Add(x.CanonicalName); } #endif return(result); }
public virtual bool HasData(Record recordInfo) { List<string> ignored; return HasData(recordInfo, out ignored); }
internal abstract RecordInfo GetInfo(Record x);
public abstract void WriteRecordStream(Record rec, System.IO.Stream outputStream);
public bool DataEquals(Record other) { return other.Size == Size && other.Fingerprint == Fingerprint; }
public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; lock (trans) { if (trans.Inputs.Contains(GetLookup(newRecord))) return true; string filename; lock (this) { if (HasData(newRecord)) return true; do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Inputs.Add(GetLookup(newRecord)); trans.m_PendingCount++; trans.m_PendingBytes += newRecord.Size; Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName); trans.Cleanup.Add(filename); long resultSize; string fn = Path.Combine(TempFolder.FullName, filename); if (priorRecord != null) { // try to delta encode it string priorLookup = GetLookup(priorRecord); var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup); if (priorData != null) { if (priorData.Mode == StorageMode.Delta) priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase); if (priorData.HasSignatureData) { try { var signature = LoadSignature(priorData); long deltaSize; List<ChunkedChecksum.FileBlock> blocks; Printer.PrintDiagnostics(" - Trying delta encoding"); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); using (var fileInput = fileEntry.Info.OpenRead()) { blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); } if (printer != null) { printer.End(newRecord.Size); printer = null; } // dont encode as delta unless we get a 50% saving if (deltaSize < fileEntry.Length / 2) { FileObjectStoreData data = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = false, Lookup = GetLookup(newRecord), Mode = StorageMode.Delta, DeltaBase = priorData.Lookup, Offset = 0 }; trans.Cleanup.Add(filename + ".delta"); Printer.PrintDiagnostics(" - Delta encoding"); using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite()) { ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks); } Printer.PrintDiagnostics(" - Compressing data"); deltaSize = new FileInfo(fn + ".delta").Length; using (var fileInput = new FileInfo(fn + ".delta").OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && deltaSize < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && deltaSize < 1024) cmode = CompressionMode.None; int sig = (int)cmode; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8); fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4); byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup); fileOutput.Write(lookupBytes, 0, lookupBytes.Length); if (deltaSize > 16 * 1024 * 1024) { printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize)); }, (obj) => { return (float)((long)obj / (double)deltaSize) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = deltaSize; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize)); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = data, Filename = filename } ); return true; } } catch { } } } } bool computeSignature = newRecord.Size > 1024 * 64; FileObjectStoreData storeData = new FileObjectStoreData() { FileSize = newRecord.Size, HasSignatureData = computeSignature, Lookup = GetLookup(newRecord), Mode = StorageMode.Flat, Offset = 0 }; using (var fileInput = fileEntry.Info.OpenRead()) using (var fileOutput = new FileInfo(fn).OpenWrite()) { fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); CompressionMode cmode = DefaultCompression; if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024) cmode = CompressionMode.LZ4; if (cmode != CompressionMode.None && newRecord.Size < 1024) cmode = CompressionMode.None; int sig = (int)cmode; if (computeSignature) sig |= 0x8000; fileOutput.Write(BitConverter.GetBytes(sig), 0, 4); fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8); Printer.InteractivePrinter printer = null; if (newRecord.Size > 16 * 1024 * 1024) printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); }); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); }); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); if (printer != null) { printer.End(newRecord.Size); printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) => { return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size)); }, (obj) => { return (float)((long)obj / (double)newRecord.Size) * 100.0f; }, (obj, lol) => { return string.Empty; }, 40); } if (cmode == CompressionMode.LZHAM) LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4) LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else if (cmode == CompressionMode.LZ4HC) LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); }); else { resultSize = newRecord.Size; fileInput.CopyTo(fileOutput); } if (printer != null) printer.End(newRecord.Size); } Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : ""); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Record = newRecord, Data = storeData, Filename = filename } ); return true; } }
public override bool HasData(Record recordInfo, out List<string> requestedData) { requestedData = null; if (!recordInfo.HasData) return true; return HasDataDirect(GetLookup(recordInfo), out requestedData); }
private void SerializeRecord(Record x, BinaryWriter bw) { bw.Write(x.Id); bw.Write(x.Parent.HasValue ? x.Parent.Value : -1L); bw.Write(x.Size); bw.Write((uint)x.Attributes); bw.Write(x.Fingerprint); bw.Write(x.CanonicalNameId); bw.Write(x.ModificationTime.Ticks); bw.Write(x.CanonicalName); }
public abstract bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null);