private long GetTransmissionLengthInternal(FileObjectStoreData storeData)
{
if (storeData.BlobID.HasValue)
return BlobDatabase.Get<Blobsize>(x => x.BlobID == storeData.BlobID.Value).Length;
return GetFileForDataID(storeData.Lookup).Length;
}
protected void WriteRecordStream(FileObjectStoreData storeData, System.IO.Stream outputStream)
{
System.IO.Stream dataStream;
if (storeData.Mode == StorageMode.Delta)
{
Stream baseStream;
FileInfo tempBaseFile;
dataStream = OpenDeltaCodecStream(OpenLegacyStream(storeData), out baseStream, out tempBaseFile);
ChunkedChecksum.ApplyDelta(baseStream, dataStream, outputStream);
dataStream.Dispose();
baseStream.Dispose();
tempBaseFile.Delete();
}
else
{
if (storeData.Mode == StorageMode.Legacy)
dataStream = new LZHAMLegacyStream(OpenLegacyStream(storeData), true);
else if (storeData.Mode == StorageMode.Flat)
dataStream = OpenCodecStream(OpenLegacyStream(storeData));
else
throw new Exception();
Printer.InteractivePrinter printer = null;
long total = 0;
byte[] dataBlob = new byte[16 * 1024 * 1024];
while (true)
{
var res = dataStream.Read(dataBlob, 0, dataBlob.Length);
if (res == 0)
break;
if (printer != null)
printer.Update(total);
outputStream.Write(dataBlob, 0, res);
total += res;
}
if (printer != null)
printer.End(total);
dataStream.Dispose();
}
}
public override bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency)
{
StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
dependency = null;
lock (trans)
{
trans.m_PendingCount++;
Printer.PrintDiagnostics("Importing data for {0}", directName);
FileObjectStoreData data = new FileObjectStoreData()
{
Lookup = directName,
};
byte[] buffer = trans.ScratchBuffer;
string filename = null;
Stream outputStream;
MemoryStream backingStore = null;
if (dataStream.Length > 16 * 1024)
{
lock (this)
{
List<string> ignored;
if (HasDataDirect(directName, out ignored))
throw new Exception();
do
{
filename = Path.GetRandomFileName();
} while (TempFiles.Contains(filename));
TempFiles.Add(filename);
}
trans.Cleanup.Add(filename);
string fn = Path.Combine(TempFolder.FullName, filename);
outputStream = new FileInfo(fn).OpenWrite();
}
else
{
backingStore = new MemoryStream();
outputStream = backingStore;
}
using (var fileOutput = outputStream)
{
bool readData = true;
byte[] sig = new byte[8];
dataStream.Read(sig, 0, 4);
if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'k')
{
data.Mode = StorageMode.Flat;
fileOutput.Write(sig, 0, 4);
dataStream.Read(sig, 0, 4);
fileOutput.Write(sig, 0, 4);
if ((BitConverter.ToUInt32(sig, 0) & 0x8000) != 0)
data.HasSignatureData = true;
dataStream.Read(sig, 0, 8);
fileOutput.Write(sig, 0, 8);
data.FileSize = BitConverter.ToInt64(sig, 0);
}
else if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'x')
{
data.Mode = StorageMode.Delta;
fileOutput.Write(sig, 0, 4);
dataStream.Read(sig, 0, 4);
fileOutput.Write(sig, 0, 4);
dataStream.Read(sig, 0, 8);
fileOutput.Write(sig, 0, 8);
data.FileSize = BitConverter.ToInt64(sig, 0);
dataStream.Read(sig, 0, 8);
fileOutput.Write(sig, 0, 8);
dataStream.Read(sig, 0, 4);
fileOutput.Write(sig, 0, 4);
int baseLookupLength = BitConverter.ToInt32(sig, 0);
byte[] baseLookupData = new byte[baseLookupLength];
dataStream.Read(baseLookupData, 0, baseLookupData.Length);
fileOutput.Write(baseLookupData, 0, baseLookupData.Length);
string baseLookup = ASCIIEncoding.ASCII.GetString(baseLookupData);
dependency = baseLookup;
data.Mode = StorageMode.Delta;
data.DeltaBase = baseLookup;
}
else
{
if (true)
{
data.Mode = StorageMode.Flat;
Printer.PrintDiagnostics(" - Importing legacy record...");
fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
dataStream.Read(sig, 0, 8);
data.FileSize = BitConverter.ToInt64(sig, 0);
string importTemp;
lock (this)
{
do
{
importTemp = Path.GetRandomFileName();
} while (TempFiles.Contains(importTemp));
TempFiles.Add(importTemp);
}
trans.Cleanup.Add(importTemp);
importTemp = Path.Combine(TempFolder.FullName, importTemp);
FileInfo importTempInfo = new FileInfo(importTemp);
using (var fs = importTempInfo.Create())
using (LZHAMLegacyStream legacy = new LZHAMLegacyStream(dataStream, false, data.FileSize))
{
while (true)
{
var read = legacy.Read(buffer, 0, buffer.Length);
if (read == 0)
break;
fs.Write(buffer, 0, read);
}
}
using (var fileInput = importTempInfo.OpenRead())
{
int signature = (int)CompressionMode.LZHAM;
bool computeSignature = data.FileSize > 1024 * 64;
if (computeSignature)
signature |= 0x8000;
fileOutput.Write(BitConverter.GetBytes(signature), 0, 4);
fileOutput.Write(BitConverter.GetBytes(data.FileSize), 0, 8);
if (computeSignature)
{
Printer.PrintDiagnostics(" - Computing signature");
var checksum = ChunkedChecksum.Compute(1024, fileInput);
fileInput.Position = 0;
ChunkedChecksum.Write(fileOutput, checksum);
}
Printer.PrintDiagnostics(" - Compressing data");
long resultSize = 0;
LZHAMWriter.CompressToStream(data.FileSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput);
}
}
else
{
fileOutput.Write(sig, 0, 4);
data.Mode = StorageMode.Legacy;
dataStream.Read(sig, 0, 8);
fileOutput.Write(sig, 0, 8);
data.FileSize = BitConverter.ToInt64(sig, 0);
}
}
while (readData)
{
var read = dataStream.Read(buffer, 0, buffer.Length);
if (read == 0)
break;
fileOutput.Write(buffer, 0, read);
}
}
trans.m_PendingBytes += data.FileSize;
byte[] payload = null;
if (backingStore != null)
payload = backingStore.ToArray();
trans.PendingTransactions.Add(
new StandardObjectStoreTransaction.PendingTransaction()
{
Data = data,
Filename = filename,
Payload = payload
}
);
return true;
}
}
private ChunkedChecksum LoadSignature(FileObjectStoreData storeData)
{
if (storeData.Mode == StorageMode.Legacy)
throw new Exception();
else if (storeData.Mode == StorageMode.Flat)
{
using (var stream = OpenLegacyStream(storeData))
{
return LoadSignatureFromStream(stream);
}
}
throw new Exception();
}
public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry)
{
StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
lock (trans)
{
if (trans.Inputs.Contains(GetLookup(newRecord)))
return true;
string filename;
lock (this)
{
if (HasData(newRecord))
return true;
do
{
filename = Path.GetRandomFileName();
} while (TempFiles.Contains(filename));
TempFiles.Add(filename);
}
trans.Inputs.Add(GetLookup(newRecord));
trans.m_PendingCount++;
trans.m_PendingBytes += newRecord.Size;
Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName);
trans.Cleanup.Add(filename);
long resultSize;
string fn = Path.Combine(TempFolder.FullName, filename);
if (priorRecord != null)
{
// try to delta encode it
string priorLookup = GetLookup(priorRecord);
var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup);
if (priorData != null)
{
if (priorData.Mode == StorageMode.Delta)
priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase);
if (priorData.HasSignatureData)
{
try
{
var signature = LoadSignature(priorData);
long deltaSize;
List<ChunkedChecksum.FileBlock> blocks;
Printer.PrintDiagnostics(" - Trying delta encoding");
Printer.InteractivePrinter printer = null;
if (newRecord.Size > 16 * 1024 * 1024)
printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
using (var fileInput = fileEntry.Info.OpenRead())
{
blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
}
if (printer != null)
{
printer.End(newRecord.Size);
printer = null;
}
// dont encode as delta unless we get a 50% saving
if (deltaSize < fileEntry.Length / 2)
{
FileObjectStoreData data = new FileObjectStoreData()
{
FileSize = newRecord.Size,
HasSignatureData = false,
Lookup = GetLookup(newRecord),
Mode = StorageMode.Delta,
DeltaBase = priorData.Lookup,
Offset = 0
};
trans.Cleanup.Add(filename + ".delta");
Printer.PrintDiagnostics(" - Delta encoding");
using (var fileInput = fileEntry.Info.OpenRead())
using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite())
{
ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks);
}
Printer.PrintDiagnostics(" - Compressing data");
deltaSize = new FileInfo(fn + ".delta").Length;
using (var fileInput = new FileInfo(fn + ".delta").OpenRead())
using (var fileOutput = new FileInfo(fn).OpenWrite())
{
fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4);
CompressionMode cmode = DefaultCompression;
if (cmode != CompressionMode.None && deltaSize < 16 * 1024)
cmode = CompressionMode.LZ4;
if (cmode != CompressionMode.None && deltaSize < 1024)
cmode = CompressionMode.None;
int sig = (int)cmode;
fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8);
fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4);
byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup);
fileOutput.Write(lookupBytes, 0, lookupBytes.Length);
if (deltaSize > 16 * 1024 * 1024)
{
printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
{
return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize));
},
(obj) =>
{
return (float)((long)obj / (double)deltaSize) * 100.0f;
},
(obj, lol) => { return string.Empty; }, 40);
}
if (cmode == CompressionMode.LZHAM)
LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4)
LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4HC)
LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else
{
resultSize = deltaSize;
fileInput.CopyTo(fileOutput);
}
if (printer != null)
printer.End(newRecord.Size);
}
Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize));
trans.PendingTransactions.Add(
new StandardObjectStoreTransaction.PendingTransaction()
{
Record = newRecord,
Data = data,
Filename = filename
}
);
return true;
}
}
catch
{
}
}
}
}
bool computeSignature = newRecord.Size > 1024 * 64;
FileObjectStoreData storeData = new FileObjectStoreData()
{
FileSize = newRecord.Size,
HasSignatureData = computeSignature,
Lookup = GetLookup(newRecord),
Mode = StorageMode.Flat,
Offset = 0
};
using (var fileInput = fileEntry.Info.OpenRead())
using (var fileOutput = new FileInfo(fn).OpenWrite())
{
fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
CompressionMode cmode = DefaultCompression;
if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024)
cmode = CompressionMode.LZ4;
if (cmode != CompressionMode.None && newRecord.Size < 1024)
cmode = CompressionMode.None;
int sig = (int)cmode;
if (computeSignature)
sig |= 0x8000;
fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
Printer.InteractivePrinter printer = null;
if (newRecord.Size > 16 * 1024 * 1024)
printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
if (computeSignature)
{
Printer.PrintDiagnostics(" - Computing signature");
var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
fileInput.Position = 0;
ChunkedChecksum.Write(fileOutput, checksum);
}
Printer.PrintDiagnostics(" - Compressing data");
if (printer != null)
{
printer.End(newRecord.Size);
printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
{
return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size));
},
(obj) =>
{
return (float)((long)obj / (double)newRecord.Size) * 100.0f;
},
(obj, lol) => { return string.Empty; }, 40);
}
if (cmode == CompressionMode.LZHAM)
LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4)
LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else if (cmode == CompressionMode.LZ4HC)
LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
else
{
resultSize = newRecord.Size;
fileInput.CopyTo(fileOutput);
}
if (printer != null)
printer.End(newRecord.Size);
}
Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : "");
trans.PendingTransactions.Add(
new StandardObjectStoreTransaction.PendingTransaction()
{
Record = newRecord,
Data = storeData,
Filename = filename
}
);
return true;
}
}
private void ImportRecordFromFlatStore(Record x)
{
if (x.HasData)
{
if (!CheckFileForDataIDExists(x.DataIdentifier))
return;
var recordData = new FileObjectStoreData();
recordData.FileSize = x.Size;
recordData.HasSignatureData = false;
recordData.Lookup = x.DataIdentifier;
recordData.Mode = StorageMode.Legacy;
recordData.Offset = 0;
recordData.PackFileID = null;
try
{
ObjectDatabase.InsertSafe(recordData);
}
catch (SQLite.SQLiteException e)
{
if (e.Result != SQLite.SQLite3.Result.Constraint)
throw;
}
}
}
private Stream OpenLegacyStream(FileObjectStoreData storeData)
{
if (storeData.BlobID.HasValue)
return new MemoryStream(BlobDatabase.Get<Blobject>(storeData.BlobID.Value).Data);
FileInfo info = GetFileForDataID(storeData.Lookup);
if (info == null)
return null;
return info.OpenRead();
}