internal static List<string> RequestRecordDataUnmapped(SharedNetworkInfo sharedInfo, List<string> missingRecordData) { HashSet<string> successes = new HashSet<string>(); int index = 0; List<string> returnedRecords = new List<string>(); HashSet<string> recordDataIdentifiers = new HashSet<string>(); List<string> dependentData = new List<string>(); if (sharedInfo.Client) Printer.PrintMessage("Requesting #b#{0}## records from remote...", missingRecordData.Count); while (index < missingRecordData.Count) { RequestRecordDataUnmapped rrd = new RequestRecordDataUnmapped(); List<string> recordsInPack = new List<string>(); while (recordsInPack.Count < 1024 * 32 && index < missingRecordData.Count) { var data = missingRecordData[index++]; if (!recordDataIdentifiers.Contains(data)) { recordDataIdentifiers.Add(data); recordsInPack.Add(data); } } if (recordsInPack.Count > 0) { rrd.RecordDataKeys = recordsInPack.ToArray(); ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.RequestRecordUnmapped }, ProtoBuf.PrefixStyle.Fixed32); Utilities.SendEncrypted<RequestRecordDataUnmapped>(sharedInfo, rrd); RecordStatus status = new RecordStatus(); var receiverStream = new Versionr.Utilities.ChunkedReceiverStream(() => { ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.DataReceived }, ProtoBuf.PrefixStyle.Fixed32); var pack = Utilities.ReceiveEncrypted<DataPayload>(sharedInfo); status.Bytes += pack.Data.Length; if (pack.EndOfStream) { return new Tuple<byte[], bool>(pack.Data, true); } else { return new Tuple<byte[], bool>(pack.Data, false); } }); status.Stopwatch = new System.Diagnostics.Stopwatch(); status.Requested = recordsInPack.Count; Printer.InteractivePrinter printer = null; if (sharedInfo.Client) printer = Printer.CreateProgressBarPrinter("Importing data", string.Empty, (obj) => { RecordStatus stat = (RecordStatus)obj; return string.Format("{0}/sec", Versionr.Utilities.Misc.FormatSizeFriendly((long)(stat.Bytes / stat.Stopwatch.Elapsed.TotalSeconds))); }, (obj) => { RecordStatus stat = (RecordStatus)obj; return (100.0f * stat.Processed) / (float)stat.Requested; }, (pct, obj) => { RecordStatus stat = (RecordStatus)obj; return string.Format("{0}/{1}", stat.Processed, stat.Requested); }, 60); status.Stopwatch.Start(); var transaction = sharedInfo.Workspace.ObjectStore.BeginStorageTransaction(); try { while (!receiverStream.EndOfStream) { byte[] blob = new byte[8]; int successFlag; int requestIndex; long recordSize; if (printer != null) printer.Update(status); status.Processed++; if (receiverStream.Read(blob, 0, 8) != 8) continue; requestIndex = BitConverter.ToInt32(blob, 0); successFlag = BitConverter.ToInt32(blob, 4); if (successFlag == 0) { Printer.PrintDiagnostics("Record {0} not located on remote.", recordsInPack[requestIndex]); continue; } receiverStream.Read(blob, 0, 8); recordSize = BitConverter.ToInt64(blob, 0); Printer.PrintDiagnostics("Unpacking record {0}, payload size: {1}", recordsInPack[requestIndex], recordSize); returnedRecords.Add(recordsInPack[requestIndex]); string dependencies; sharedInfo.Workspace.ImportRecordData(transaction, recordsInPack[requestIndex], new Versionr.Utilities.RestrictedStream(receiverStream, recordSize), out dependencies); if (dependencies != null) dependentData.Add(dependencies); if (transaction.PendingRecords > 1024 || transaction.PendingRecordBytes > 256 * 1024 * 1024) sharedInfo.Workspace.ObjectStore.FlushStorageTransaction(transaction); } sharedInfo.Workspace.ObjectStore.EndStorageTransaction(transaction); if (printer != null) printer.End(status); } catch { sharedInfo.Workspace.ObjectStore.AbortStorageTransaction(transaction); throw; } ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.Acknowledge }, ProtoBuf.PrefixStyle.Fixed32); } } List<string> filteredDeps = new List<string>(); foreach (var x in dependentData) { if (!recordDataIdentifiers.Contains(x) && !sharedInfo.Workspace.HasObjectDataDirect(x)) { recordDataIdentifiers.Add(x); filteredDeps.Add(x); } } if (filteredDeps.Count > 0) RequestRecordDataUnmapped(sharedInfo, filteredDeps); return returnedRecords; }
internal static void RequestRecordData(SharedNetworkInfo sharedInfo) { List<string> dependentData = new List<string>(); var records = sharedInfo.UnknownRecords; int index = 0; HashSet<string> recordDataIdentifiers = new HashSet<string>(); while (index < records.Count) { RequestRecordData rrd = new RequestRecordData(); List<long> recordsInPack = new List<long>(); while (recordsInPack.Count < 1024 * 32 && index < records.Count) { long recordIndex = records[index++]; var rec = sharedInfo.RemoteRecordMap[recordIndex]; if (rec.IsDirectory) continue; if (sharedInfo.Workspace.HasObjectData(rec) || !sharedInfo.Workspace.Included(rec.CanonicalName)) continue; if (recordDataIdentifiers.Contains(rec.DataIdentifier)) continue; recordDataIdentifiers.Add(rec.DataIdentifier); recordsInPack.Add(recordIndex); } if (recordsInPack.Count > 0) { Printer.PrintDiagnostics("Requesting {0} records.", recordsInPack.Count); rrd.Records = recordsInPack.ToArray(); ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.RequestRecord }, ProtoBuf.PrefixStyle.Fixed32); Utilities.SendEncrypted<RequestRecordData>(sharedInfo, rrd); RecordStatus status = new RecordStatus(); var receiverStream = new Versionr.Utilities.ChunkedReceiverStream(() => { ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.DataReceived }, ProtoBuf.PrefixStyle.Fixed32); var pack = Utilities.ReceiveEncrypted<DataPayload>(sharedInfo); status.Bytes += pack.Data.Length; if (pack.EndOfStream) { return new Tuple<byte[], bool>(pack.Data, true); } else { return new Tuple<byte[], bool>(pack.Data, false); } }); status.Stopwatch = new System.Diagnostics.Stopwatch(); status.Requested = recordsInPack.Count; Printer.InteractivePrinter printer = null; if (sharedInfo.Client) printer = Printer.CreateProgressBarPrinter("Importing data", string.Empty, (obj) => { RecordStatus stat = (RecordStatus)obj; return string.Format("{0}/sec", Versionr.Utilities.Misc.FormatSizeFriendly((long)(stat.Bytes / stat.Stopwatch.Elapsed.TotalSeconds))); }, (obj) => { RecordStatus stat = (RecordStatus)obj; return (100.0f * stat.Processed) / (float)stat.Requested; }, (pct, obj) => { RecordStatus stat = (RecordStatus)obj; return string.Format("{0}/{1}", stat.Processed, stat.Requested); }, 60); status.Stopwatch.Start(); var transaction = sharedInfo.Workspace.ObjectStore.BeginStorageTransaction(); try { while (!receiverStream.EndOfStream) { byte[] blob = new byte[16]; long recordIndex; long recordSize; if (receiverStream.Read(blob, 0, 8) != 8) continue; if (printer != null) printer.Update(status); status.Processed++; recordIndex = BitConverter.ToInt64(blob, 0); if (recordIndex < 0) { continue; } receiverStream.Read(blob, 8, 8); recordSize = BitConverter.ToInt64(blob, 8); Printer.PrintDiagnostics("Unpacking remote record {0}, payload size: {1}", recordIndex, recordSize); var rec = sharedInfo.RemoteRecordMap[recordIndex]; string dependencies = null; sharedInfo.Workspace.ImportRecordData(transaction, rec.DataIdentifier, new Versionr.Utilities.RestrictedStream(receiverStream, recordSize), out dependencies); if (dependencies != null) dependentData.Add(dependencies); if (transaction.PendingRecords > 8192 || transaction.PendingRecordBytes > 512 * 1024 * 1024) sharedInfo.Workspace.ObjectStore.FlushStorageTransaction(transaction); } sharedInfo.Workspace.ObjectStore.EndStorageTransaction(transaction); } catch { sharedInfo.Workspace.ObjectStore.AbortStorageTransaction(transaction); throw; } if (printer != null) printer.End(status); ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.Acknowledge }, ProtoBuf.PrefixStyle.Fixed32); } } List<string> filteredDeps = new List<string>(); foreach (var x in dependentData) { if (!recordDataIdentifiers.Contains(x) && !sharedInfo.Workspace.HasObjectDataDirect(x)) { recordDataIdentifiers.Add(x); filteredDeps.Add(x); } } if (filteredDeps.Count > 0) RequestRecordDataUnmapped(sharedInfo, filteredDeps); }