internal static List<string> RequestRecordDataUnmapped(SharedNetworkInfo sharedInfo, List<string> missingRecordData)
{
HashSet<string> successes = new HashSet<string>();
int index = 0;
List<string> returnedRecords = new List<string>();
HashSet<string> recordDataIdentifiers = new HashSet<string>();
List<string> dependentData = new List<string>();
if (sharedInfo.Client)
Printer.PrintMessage("Requesting #b#{0}## records from remote...", missingRecordData.Count);
while (index < missingRecordData.Count)
{
RequestRecordDataUnmapped rrd = new RequestRecordDataUnmapped();
List<string> recordsInPack = new List<string>();
while (recordsInPack.Count < 1024 * 32 && index < missingRecordData.Count)
{
var data = missingRecordData[index++];
if (!recordDataIdentifiers.Contains(data))
{
recordDataIdentifiers.Add(data);
recordsInPack.Add(data);
}
}
if (recordsInPack.Count > 0)
{
rrd.RecordDataKeys = recordsInPack.ToArray();
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.RequestRecordUnmapped }, ProtoBuf.PrefixStyle.Fixed32);
Utilities.SendEncrypted<RequestRecordDataUnmapped>(sharedInfo, rrd);
RecordStatus status = new RecordStatus();
var receiverStream = new Versionr.Utilities.ChunkedReceiverStream(() =>
{
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.DataReceived }, ProtoBuf.PrefixStyle.Fixed32);
var pack = Utilities.ReceiveEncrypted<DataPayload>(sharedInfo);
status.Bytes += pack.Data.Length;
if (pack.EndOfStream)
{
return new Tuple<byte[], bool>(pack.Data, true);
}
else
{
return new Tuple<byte[], bool>(pack.Data, false);
}
});
status.Stopwatch = new System.Diagnostics.Stopwatch();
status.Requested = recordsInPack.Count;
Printer.InteractivePrinter printer = null;
if (sharedInfo.Client)
printer = Printer.CreateProgressBarPrinter("Importing data", string.Empty,
(obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return string.Format("{0}/sec", Versionr.Utilities.Misc.FormatSizeFriendly((long)(stat.Bytes / stat.Stopwatch.Elapsed.TotalSeconds)));
},
(obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return (100.0f * stat.Processed) / (float)stat.Requested;
},
(pct, obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return string.Format("{0}/{1}", stat.Processed, stat.Requested);
},
60);
status.Stopwatch.Start();
var transaction = sharedInfo.Workspace.ObjectStore.BeginStorageTransaction();
try
{
while (!receiverStream.EndOfStream)
{
byte[] blob = new byte[8];
int successFlag;
int requestIndex;
long recordSize;
if (printer != null)
printer.Update(status);
status.Processed++;
if (receiverStream.Read(blob, 0, 8) != 8)
continue;
requestIndex = BitConverter.ToInt32(blob, 0);
successFlag = BitConverter.ToInt32(blob, 4);
if (successFlag == 0)
{
Printer.PrintDiagnostics("Record {0} not located on remote.", recordsInPack[requestIndex]);
continue;
}
receiverStream.Read(blob, 0, 8);
recordSize = BitConverter.ToInt64(blob, 0);
Printer.PrintDiagnostics("Unpacking record {0}, payload size: {1}", recordsInPack[requestIndex], recordSize);
returnedRecords.Add(recordsInPack[requestIndex]);
string dependencies;
sharedInfo.Workspace.ImportRecordData(transaction, recordsInPack[requestIndex], new Versionr.Utilities.RestrictedStream(receiverStream, recordSize), out dependencies);
if (dependencies != null)
dependentData.Add(dependencies);
if (transaction.PendingRecords > 1024 || transaction.PendingRecordBytes > 256 * 1024 * 1024)
sharedInfo.Workspace.ObjectStore.FlushStorageTransaction(transaction);
}
sharedInfo.Workspace.ObjectStore.EndStorageTransaction(transaction);
if (printer != null)
printer.End(status);
}
catch
{
sharedInfo.Workspace.ObjectStore.AbortStorageTransaction(transaction);
throw;
}
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.Acknowledge }, ProtoBuf.PrefixStyle.Fixed32);
}
}
List<string> filteredDeps = new List<string>();
foreach (var x in dependentData)
{
if (!recordDataIdentifiers.Contains(x) && !sharedInfo.Workspace.HasObjectDataDirect(x))
{
recordDataIdentifiers.Add(x);
filteredDeps.Add(x);
}
}
if (filteredDeps.Count > 0)
RequestRecordDataUnmapped(sharedInfo, filteredDeps);
return returnedRecords;
}
internal static void RequestRecordData(SharedNetworkInfo sharedInfo)
{
List<string> dependentData = new List<string>();
var records = sharedInfo.UnknownRecords;
int index = 0;
HashSet<string> recordDataIdentifiers = new HashSet<string>();
while (index < records.Count)
{
RequestRecordData rrd = new RequestRecordData();
List<long> recordsInPack = new List<long>();
while (recordsInPack.Count < 1024 * 32 && index < records.Count)
{
long recordIndex = records[index++];
var rec = sharedInfo.RemoteRecordMap[recordIndex];
if (rec.IsDirectory)
continue;
if (sharedInfo.Workspace.HasObjectData(rec) || !sharedInfo.Workspace.Included(rec.CanonicalName))
continue;
if (recordDataIdentifiers.Contains(rec.DataIdentifier))
continue;
recordDataIdentifiers.Add(rec.DataIdentifier);
recordsInPack.Add(recordIndex);
}
if (recordsInPack.Count > 0)
{
Printer.PrintDiagnostics("Requesting {0} records.", recordsInPack.Count);
rrd.Records = recordsInPack.ToArray();
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.RequestRecord }, ProtoBuf.PrefixStyle.Fixed32);
Utilities.SendEncrypted<RequestRecordData>(sharedInfo, rrd);
RecordStatus status = new RecordStatus();
var receiverStream = new Versionr.Utilities.ChunkedReceiverStream(() =>
{
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.DataReceived }, ProtoBuf.PrefixStyle.Fixed32);
var pack = Utilities.ReceiveEncrypted<DataPayload>(sharedInfo);
status.Bytes += pack.Data.Length;
if (pack.EndOfStream)
{
return new Tuple<byte[], bool>(pack.Data, true);
}
else
{
return new Tuple<byte[], bool>(pack.Data, false);
}
});
status.Stopwatch = new System.Diagnostics.Stopwatch();
status.Requested = recordsInPack.Count;
Printer.InteractivePrinter printer = null;
if (sharedInfo.Client)
printer = Printer.CreateProgressBarPrinter("Importing data", string.Empty,
(obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return string.Format("{0}/sec", Versionr.Utilities.Misc.FormatSizeFriendly((long)(stat.Bytes / stat.Stopwatch.Elapsed.TotalSeconds)));
},
(obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return (100.0f * stat.Processed) / (float)stat.Requested;
},
(pct, obj) =>
{
RecordStatus stat = (RecordStatus)obj;
return string.Format("{0}/{1}", stat.Processed, stat.Requested);
},
60);
status.Stopwatch.Start();
var transaction = sharedInfo.Workspace.ObjectStore.BeginStorageTransaction();
try
{
while (!receiverStream.EndOfStream)
{
byte[] blob = new byte[16];
long recordIndex;
long recordSize;
if (receiverStream.Read(blob, 0, 8) != 8)
continue;
if (printer != null)
printer.Update(status);
status.Processed++;
recordIndex = BitConverter.ToInt64(blob, 0);
if (recordIndex < 0)
{
continue;
}
receiverStream.Read(blob, 8, 8);
recordSize = BitConverter.ToInt64(blob, 8);
Printer.PrintDiagnostics("Unpacking remote record {0}, payload size: {1}", recordIndex, recordSize);
var rec = sharedInfo.RemoteRecordMap[recordIndex];
string dependencies = null;
sharedInfo.Workspace.ImportRecordData(transaction, rec.DataIdentifier, new Versionr.Utilities.RestrictedStream(receiverStream, recordSize), out dependencies);
if (dependencies != null)
dependentData.Add(dependencies);
if (transaction.PendingRecords > 8192 || transaction.PendingRecordBytes > 512 * 1024 * 1024)
sharedInfo.Workspace.ObjectStore.FlushStorageTransaction(transaction);
}
sharedInfo.Workspace.ObjectStore.EndStorageTransaction(transaction);
}
catch
{
sharedInfo.Workspace.ObjectStore.AbortStorageTransaction(transaction);
throw;
}
if (printer != null)
printer.End(status);
ProtoBuf.Serializer.SerializeWithLengthPrefix<NetCommand>(sharedInfo.Stream, new NetCommand() { Type = NetCommandType.Acknowledge }, ProtoBuf.PrefixStyle.Fixed32);
}
}
List<string> filteredDeps = new List<string>();
foreach (var x in dependentData)
{
if (!recordDataIdentifiers.Contains(x) && !sharedInfo.Workspace.HasObjectDataDirect(x))
{
recordDataIdentifiers.Add(x);
filteredDeps.Add(x);
}
}
if (filteredDeps.Count > 0)
RequestRecordDataUnmapped(sharedInfo, filteredDeps);
}
