private void ProcessData(DataWriterExtensionLogicBase dataWriter, DataReaderProgressRecord progressRecord, bool applyStatus)
{
var buffers = progressRecord.DatasetToRecordMap.Values.Select(dataRecord =>
{
double[] data;
var elementType = dataRecord.Dataset.GetType().GetElementType();
if (applyStatus || elementType != typeof(double))
{
data = BufferUtilities.ApplyDatasetStatus2(dataRecord.Dataset, dataRecord.Status);
}
else
{
data = (double[])dataRecord.Dataset;
}
return((IBuffer)BufferUtilities.CreateSimpleBuffer(data));
}).ToList();
var period = progressRecord.End - progressRecord.Begin;
dataWriter.Write(progressRecord.Begin, period, buffers);
// clean up
buffers = null;
GC.Collect();
}
private bool CreateFiles(DataWriterExtensionLogicBase dataWriter, ZipSettings zipSettings)
{
DateTime dateTime;
TimeSpan dataStoragePeriod;
ulong currentRowCount;
ulong lastRow;
// START progress
ulong currentDataset;
ulong datasetCount;
ulong currentSegment;
ulong segmentCount;
datasetCount = (ulong)zipSettings.CampaignInfo.Value.SelectMany(variableInfo => variableInfo.Value).Count();
currentSegment = 0;
segmentCount = (ulong)Math.Ceiling(zipSettings.Block / (double)zipSettings.SegmentLength);
// END progress
currentRowCount = zipSettings.SegmentLength;
lastRow = zipSettings.Start + zipSettings.Block;
IList <IDataStorage> dataStorageSet;
// for each segment
for (ulong currentStart = zipSettings.Start; currentStart < lastRow; currentStart += currentRowCount)
{
currentDataset = 0;
dataStorageSet = new List <IDataStorage>();
if (currentStart + currentRowCount > zipSettings.Start + zipSettings.Block)
{
currentRowCount = zipSettings.Start + zipSettings.Block - currentStart;
}
// load data
foreach (var variableInfo in zipSettings.CampaignInfo.Value)
{
variableInfo.Value.ForEach(datasetName =>
{
dataStorageSet.Add(this.LoadDataset(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }/{ datasetName }", currentStart, 1, currentRowCount, 1));
this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Loading dataset segment { currentSegment * datasetCount + currentDataset + 1 } / { segmentCount * datasetCount } ..."));
currentDataset++;
});
}
this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Writing data of segment { currentSegment + 1 } / { segmentCount } ..."));
dateTime = zipSettings.DateTimeBegin.AddSeconds((currentStart - zipSettings.Start) / zipSettings.SampleRate);
dataStoragePeriod = TimeSpan.FromSeconds(currentRowCount / zipSettings.SampleRate);
dataWriter.Write(dateTime, dataStoragePeriod, dataStorageSet);
// clean up
dataStorageSet = null;
GC.Collect();
if (_cancellationToken.IsCancellationRequested)
{
return(false);
}
currentSegment++;
}
return(true);
}