protected void WriteRecordStream(FileObjectStoreData storeData, System.IO.Stream outputStream) { System.IO.Stream dataStream; if (storeData.Mode == StorageMode.Delta) { Stream baseStream; FileInfo tempBaseFile; dataStream = OpenDeltaCodecStream(OpenLegacyStream(storeData), out baseStream, out tempBaseFile); ChunkedChecksum.ApplyDelta(baseStream, dataStream, outputStream); dataStream.Dispose(); baseStream.Dispose(); tempBaseFile.Delete(); } else { if (storeData.Mode == StorageMode.Legacy) dataStream = new LZHAMLegacyStream(OpenLegacyStream(storeData), true); else if (storeData.Mode == StorageMode.Flat) dataStream = OpenCodecStream(OpenLegacyStream(storeData)); else throw new Exception(); Printer.InteractivePrinter printer = null; long total = 0; byte[] dataBlob = new byte[16 * 1024 * 1024]; while (true) { var res = dataStream.Read(dataBlob, 0, dataBlob.Length); if (res == 0) break; if (printer != null) printer.Update(total); outputStream.Write(dataBlob, 0, res); total += res; } if (printer != null) printer.End(total); dataStream.Dispose(); } }
public override bool ReceiveRecordData(ObjectStoreTransaction transaction, string directName, System.IO.Stream dataStream, out string dependency) { StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction; dependency = null; lock (trans) { trans.m_PendingCount++; Printer.PrintDiagnostics("Importing data for {0}", directName); FileObjectStoreData data = new FileObjectStoreData() { Lookup = directName, }; byte[] buffer = trans.ScratchBuffer; string filename = null; Stream outputStream; MemoryStream backingStore = null; if (dataStream.Length > 16 * 1024) { lock (this) { List<string> ignored; if (HasDataDirect(directName, out ignored)) throw new Exception(); do { filename = Path.GetRandomFileName(); } while (TempFiles.Contains(filename)); TempFiles.Add(filename); } trans.Cleanup.Add(filename); string fn = Path.Combine(TempFolder.FullName, filename); outputStream = new FileInfo(fn).OpenWrite(); } else { backingStore = new MemoryStream(); outputStream = backingStore; } using (var fileOutput = outputStream) { bool readData = true; byte[] sig = new byte[8]; dataStream.Read(sig, 0, 4); if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'k') { data.Mode = StorageMode.Flat; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); if ((BitConverter.ToUInt32(sig, 0) & 0x8000) != 0) data.HasSignatureData = true; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } else if (sig[0] == 'd' && sig[1] == 'b' && sig[2] == 'l' && sig[3] == 'x') { data.Mode = StorageMode.Delta; fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); dataStream.Read(sig, 0, 4); fileOutput.Write(sig, 0, 4); int baseLookupLength = BitConverter.ToInt32(sig, 0); byte[] baseLookupData = new byte[baseLookupLength]; dataStream.Read(baseLookupData, 0, baseLookupData.Length); fileOutput.Write(baseLookupData, 0, baseLookupData.Length); string baseLookup = ASCIIEncoding.ASCII.GetString(baseLookupData); dependency = baseLookup; data.Mode = StorageMode.Delta; data.DeltaBase = baseLookup; } else { if (true) { data.Mode = StorageMode.Flat; Printer.PrintDiagnostics(" - Importing legacy record..."); fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4); dataStream.Read(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); string importTemp; lock (this) { do { importTemp = Path.GetRandomFileName(); } while (TempFiles.Contains(importTemp)); TempFiles.Add(importTemp); } trans.Cleanup.Add(importTemp); importTemp = Path.Combine(TempFolder.FullName, importTemp); FileInfo importTempInfo = new FileInfo(importTemp); using (var fs = importTempInfo.Create()) using (LZHAMLegacyStream legacy = new LZHAMLegacyStream(dataStream, false, data.FileSize)) { while (true) { var read = legacy.Read(buffer, 0, buffer.Length); if (read == 0) break; fs.Write(buffer, 0, read); } } using (var fileInput = importTempInfo.OpenRead()) { int signature = (int)CompressionMode.LZHAM; bool computeSignature = data.FileSize > 1024 * 64; if (computeSignature) signature |= 0x8000; fileOutput.Write(BitConverter.GetBytes(signature), 0, 4); fileOutput.Write(BitConverter.GetBytes(data.FileSize), 0, 8); if (computeSignature) { Printer.PrintDiagnostics(" - Computing signature"); var checksum = ChunkedChecksum.Compute(1024, fileInput); fileInput.Position = 0; ChunkedChecksum.Write(fileOutput, checksum); } Printer.PrintDiagnostics(" - Compressing data"); long resultSize = 0; LZHAMWriter.CompressToStream(data.FileSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput); } } else { fileOutput.Write(sig, 0, 4); data.Mode = StorageMode.Legacy; dataStream.Read(sig, 0, 8); fileOutput.Write(sig, 0, 8); data.FileSize = BitConverter.ToInt64(sig, 0); } } while (readData) { var read = dataStream.Read(buffer, 0, buffer.Length); if (read == 0) break; fileOutput.Write(buffer, 0, read); } } trans.m_PendingBytes += data.FileSize; byte[] payload = null; if (backingStore != null) payload = backingStore.ToArray(); trans.PendingTransactions.Add( new StandardObjectStoreTransaction.PendingTransaction() { Data = data, Filename = filename, Payload = payload } ); return true; } }