public AlterationVM(Alteration alteration, Area area, Version version)
{
_alteration = alteration;
_area = area;
_version = version;
if (_alteration.PriorRecord.HasValue)
_priorRecord = _area.GetRecord(_alteration.PriorRecord.Value);
if (_alteration.NewRecord.HasValue)
_newRecord = _area.GetRecord(_alteration.NewRecord.Value);
DiffWithPreviousCommand = new DelegateCommand(DiffWithPrevious, CanDiffWithPrevious);
DiffWithCurrentCommand = new DelegateCommand(DiffWithCurrent, CanDiffWithCurrent);
LogCommand = new DelegateCommand(Log);
SaveVersionAsCommand = new DelegateCommand(SaveVersionAs, CanSaveVersionAs);
}
public static void CheckRecord(SharedNetworkInfo sharedInfo, Record record)
{
if (record == null)
return;
sharedInfo.RemoteRecordMap[record.Id] = record;
if (!sharedInfo.UnknownRecordSet.Contains(record.Id))
{
var localRecord = sharedInfo.Workspace.LocateRecord(record);
if (localRecord == null)
{
Printer.PrintDiagnostics("Received version dependent on missing record {0}", record.UniqueIdentifier);
sharedInfo.UnknownRecords.Add(record.Id);
sharedInfo.UnknownRecordSet.Add(record.Id);
}
else
{
localRecord.CanonicalName = record.CanonicalName;
sharedInfo.LocalRecordMap[record.Id] = localRecord;
}
}
}
public override void WriteRecordStream(Record record, System.IO.Stream outputStream)
{
string lookup = GetLookup(record);
var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
WriteRecordStream(storeData, outputStream);
}
private Stream GetDataStream(Record record)
{
string lookup = GetLookup(record);
var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
return OpenLegacyStream(storeData);
throw new Exception();
}
public override bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null)
{
if (!record.HasData)
{
return true;
}
long dataSize = GetTransmissionLength(record);
if (dataSize == -1)
return false;
using (System.IO.Stream dataStream = GetDataStream(record))
{
if (dataStream == null)
return false;
if (beginTransmission != null)
beginTransmission();
sender(BitConverter.GetBytes(dataSize), 8, false);
while (true)
{
var readCount = dataStream.Read(scratchBuffer, 0, scratchBuffer.Length);
if (readCount == 0)
break;
sender(scratchBuffer, readCount, false);
}
}
return true;
}
public override long GetTransmissionLength(Record record)
{
if (!record.HasData)
return 0;
string lookup = GetLookup(record);
var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
if (storeData == null)
return -1;
return GetTransmissionLengthInternal(storeData);
}
public abstract long GetTransmissionLength(Record record);
private Record DeserializeRecord(BinaryReader br)
{
Record rec = new Record();
rec.Id = br.ReadInt64();
long parent = br.ReadInt64();
if (parent != -1)
rec.Parent = parent;
rec.Size = br.ReadInt64();
rec.Attributes = (Attributes)br.ReadUInt32();
rec.Fingerprint = br.ReadString();
rec.CanonicalNameId = br.ReadInt64();
rec.ModificationTime = new DateTime(br.ReadInt64());
rec.CanonicalName = br.ReadString();
return rec;
}
private void ImportRecordFromFlatStore(Record x)
{
if (x.HasData)
{
if (!CheckFileForDataIDExists(x.DataIdentifier))
return;
var recordData = new FileObjectStoreData();
recordData.FileSize = x.Size;
recordData.HasSignatureData = false;
recordData.Lookup = x.DataIdentifier;
recordData.Mode = StorageMode.Legacy;
recordData.Offset = 0;
recordData.PackFileID = null;
try
{
ObjectDatabase.InsertSafe(recordData);
}
catch (SQLite.SQLiteException e)
{
if (e.Result != SQLite.SQLite3.Result.Constraint)
throw;
}
}
}
internal override RecordInfo GetInfo(Record x)
{
if (!x.HasData)
{
return new RecordInfo()
{
AllocatedSize = 0,
DeltaCompressed = false,
ID = -1,
};
}
string lookup = GetLookup(x);
var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
if (storeData == null)
return null;
return new RecordInfo()
{
AllocatedSize = GetTransmissionLengthInternal(storeData),
DeltaCompressed = storeData.Mode == StorageMode.Delta,
ID = storeData.ID
};
}
private string GetLookup(Record record)
{
return record.Fingerprint + "-" + record.Size.ToString();
}
private List <Record> Consolidate(List <Record> baseList, List <Alteration> alterations, List <Record> deletions)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
Dictionary <long, Record> records = new Dictionary <long, Record>();
foreach (var x in baseList)
{
records[x.Id] = x;
}
#if FULL_CONSOLIDATE_DEBUG
Printer.PrintMessage("Initial Snapshot Contains: ");
foreach (var x in baseList)
{
Printer.PrintMessage("Initial: ({2}) {0} - {1}", x.CanonicalName, x.UniqueIdentifier, x.Id);
}
#endif
List <long> pending = new List <long>();
CacheRecords(alterations.Select(x => x.NewRecord).Where(x => x.HasValue).Select(x => x.Value));
foreach (var x in alterations.Select(x => x).Reverse())
{
Objects.Record rec = null;
switch (x.Type)
{
case AlterationType.Add:
case AlterationType.Copy:
{
var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
Printer.PrintMessage("Add: ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
records[record.Id] = record;
break;
}
case AlterationType.Move:
{
var record = GetCachedRecord(x.NewRecord.Value);
if (deletions != null)
{
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
deletions.Add(rec);
}
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Move (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
Printer.PrintMessage("Move (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception("Consistency constraint invalid!");
}
records[record.Id] = record;
break;
}
case AlterationType.Update:
{
var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Update (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
Printer.PrintMessage("Update (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception();
}
records[record.Id] = record;
break;
}
case AlterationType.Discard:
break;
case AlterationType.Delete:
if (deletions != null)
{
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
deletions.Add(rec);
}
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Delete: ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception("Consistency constraint invalid!");
}
break;
default:
throw new Exception();
}
}
var result = records.Select(x => x.Value).ToList();
#if DEBUG
HashSet <string> namecheck = new HashSet <string>();
foreach (var x in result)
{
if (namecheck.Contains(x.CanonicalName))
{
throw new Exception("Inconsistency in internal state!");
}
namecheck.Add(x.CanonicalName);
}
#endif
return(result);
}
public virtual bool HasData(Record recordInfo)
{
List<string> ignored;
return HasData(recordInfo, out ignored);
}
internal abstract RecordInfo GetInfo(Record x);
public abstract void WriteRecordStream(Record rec, System.IO.Stream outputStream);
public bool DataEquals(Record other)
{
return other.Size == Size && other.Fingerprint == Fingerprint;
}
public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry)
{
StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
lock (trans)
{
if (trans.Inputs.Contains(GetLookup(newRecord)))
return true;
string filename;
lock (this)
{
if (HasData(newRecord))
return true;
do
{
filename = Path.GetRandomFileName();
} while (TempFiles.Contains(filename));
TempFiles.Add(filename);
}
trans.Inputs.Add(GetLookup(newRecord));
trans.m_PendingCount++;
trans.m_PendingBytes += newRecord.Size;
Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName);
trans.Cleanup.Add(filename);
long resultSize;
string fn = Path.Combine(TempFolder.FullName, filename);
if (priorRecord != null)
{
// try to delta encode it
string priorLookup = GetLookup(priorRecord);
var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup);
if (priorData != null)
{
if (priorData.Mode == StorageMode.Delta)
priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase);
if (priorData.HasSignatureData)
{
try
{
var signature = LoadSignature(priorData);
long deltaSize;
List<ChunkedChecksum.FileBlock> blocks;
Printer.PrintDiagnostics(" - Trying delta encoding");
Printer.InteractivePrinter printer = null;
if (newRecord.Size > 16 * 1024 * 1024)
printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
using (var fileInput = fileEntry.Info.OpenRead())
{
blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
}
if (printer != null)
{
printer.End(newRecord.Size);
printer = null;
}
// dont encode as delta unless we get a 50% saving
if (deltaSize < fileEntry.Length / 2)
{
FileObjectStoreData data = new FileObjectStoreData()
{
FileSize = newRecord.Size,
HasSignatureData = false,
Lookup = GetLookup(newRecord),
Mode = StorageMode.Delta,
DeltaBase = priorData.Lookup,
Offset = 0
};
trans.Cleanup.Add(filename + ".delta");
Printer.PrintDiagnostics(" - Delta encoding");
using (var fileInput = fileEntry.Info.OpenRead())
using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite())
{
ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks);
}
Printer.PrintDiagnostics(" - Compressing data");
deltaSize = new FileInfo(fn + ".delta").Length;
using (var fileInput = new FileInfo(fn + ".delta").OpenRead())
using (var fileOutput = new FileInfo(fn).OpenWrite())
{
fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4);
CompressionMode cmode = DefaultCompression;
if (cmode != CompressionMode.None && deltaSize < 16 * 1024)
cmode = CompressionMode.LZ4;
if (cmode != CompressionMode.None && deltaSize < 1024)
cmode = CompressionMode.None;
int sig = (int)cmode;
fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8);
fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4);
byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup);
fileOutput.Write(lookupBytes, 0, lookupBytes.Length);
if (deltaSize > 16 * 1024 * 1024)
{
printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
{
return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize));
},
(obj) =>
{
return (float)((long)obj / (double)deltaSize) * 100.0f;
},
(obj, lol) => { return string.Empty; }, 40);
}
if (cmode == CompressionMode.LZHAM)
LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4)
LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4HC)
LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else
{
resultSize = deltaSize;
fileInput.CopyTo(fileOutput);
}
if (printer != null)
printer.End(newRecord.Size);
}
Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize));
trans.PendingTransactions.Add(
new StandardObjectStoreTransaction.PendingTransaction()
{
Record = newRecord,
Data = data,
Filename = filename
}
);
return true;
}
}
catch
{
}
}
}
}
bool computeSignature = newRecord.Size > 1024 * 64;
FileObjectStoreData storeData = new FileObjectStoreData()
{
FileSize = newRecord.Size,
HasSignatureData = computeSignature,
Lookup = GetLookup(newRecord),
Mode = StorageMode.Flat,
Offset = 0
};
using (var fileInput = fileEntry.Info.OpenRead())
using (var fileOutput = new FileInfo(fn).OpenWrite())
{
fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
CompressionMode cmode = DefaultCompression;
if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024)
cmode = CompressionMode.LZ4;
if (cmode != CompressionMode.None && newRecord.Size < 1024)
cmode = CompressionMode.None;
int sig = (int)cmode;
if (computeSignature)
sig |= 0x8000;
fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
Printer.InteractivePrinter printer = null;
if (newRecord.Size > 16 * 1024 * 1024)
printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
if (computeSignature)
{
Printer.PrintDiagnostics(" - Computing signature");
var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
fileInput.Position = 0;
ChunkedChecksum.Write(fileOutput, checksum);
}
Printer.PrintDiagnostics(" - Compressing data");
if (printer != null)
{
printer.End(newRecord.Size);
printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
{
return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size));
},
(obj) =>
{
return (float)((long)obj / (double)newRecord.Size) * 100.0f;
},
(obj, lol) => { return string.Empty; }, 40);
}
if (cmode == CompressionMode.LZHAM)
LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4)
LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4HC)
LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else
{
resultSize = newRecord.Size;
fileInput.CopyTo(fileOutput);
}
if (printer != null)
printer.End(newRecord.Size);
}
Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : "");
trans.PendingTransactions.Add(
new StandardObjectStoreTransaction.PendingTransaction()
{
Record = newRecord,
Data = storeData,
Filename = filename
}
);
return true;
}
}
public override bool HasData(Record recordInfo, out List<string> requestedData)
{
requestedData = null;
if (!recordInfo.HasData)
return true;
return HasDataDirect(GetLookup(recordInfo), out requestedData);
}
private void SerializeRecord(Record x, BinaryWriter bw)
{
bw.Write(x.Id);
bw.Write(x.Parent.HasValue ? x.Parent.Value : -1L);
bw.Write(x.Size);
bw.Write((uint)x.Attributes);
bw.Write(x.Fingerprint);
bw.Write(x.CanonicalNameId);
bw.Write(x.ModificationTime.Ticks);
bw.Write(x.CanonicalName);
}
public abstract bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null);