private void ProcessData(DataWriterExtensionLogicBase dataWriter, DataReaderProgressRecord progressRecord, bool applyStatus)
{
var buffers = progressRecord.DatasetToRecordMap.Values.Select(dataRecord =>
{
double[] data;
var elementType = dataRecord.Dataset.GetType().GetElementType();
if (applyStatus || elementType != typeof(double))
{
data = BufferUtilities.ApplyDatasetStatus2(dataRecord.Dataset, dataRecord.Status);
}
else
{
data = (double[])dataRecord.Dataset;
}
return((IBuffer)BufferUtilities.CreateSimpleBuffer(data));
}).ToList();
var period = progressRecord.End - progressRecord.Begin;
dataWriter.Write(progressRecord.Begin, period, buffers);
// clean up
buffers = null;
GC.Collect();
}
private void CreateFiles(ClaimsPrincipal user,
DataWriterExtensionLogicBase dataWriter,
ExportParameters exportParameters,
SparseProjectInfo sparseProject,
CancellationToken cancellationToken)
{
var datasets = sparseProject.Channels.SelectMany(channel => channel.Datasets);
var registrationToDatasetsMap = new Dictionary <DataReaderRegistration, List <DatasetInfo> >();
foreach (var dataset in datasets)
{
if (!registrationToDatasetsMap.ContainsKey(dataset.Registration))
{
registrationToDatasetsMap[dataset.Registration] = new List <DatasetInfo>();
}
registrationToDatasetsMap[dataset.Registration].Add(dataset);
}
var progressHandler = (EventHandler <double>)((sender, e) =>
{
this.OnProgress(new ProgressUpdatedEventArgs(e, $"Loading data ..."));
});
foreach (var entry in registrationToDatasetsMap)
{
if (entry.Value.Any())
{
var registration = entry.Key;
var dataReader = _databaseManager.GetDataReader(user, registration);
dataReader.Progress.ProgressChanged += progressHandler;
try
{
var isAggregation = dataReader.Registration.Equals(_databaseManager.State.AggregationRegistration);
foreach (var progressRecord in dataReader.Read(entry.Value, exportParameters.Begin, exportParameters.End, _blockSizeLimit, cancellationToken))
{
this.ProcessData(dataWriter, progressRecord, applyStatus: !isAggregation);
}
}
finally
{
dataReader.Progress.ProgressChanged -= progressHandler;
}
}
}
cancellationToken.ThrowIfCancellationRequested();
}
private bool CreateFiles(DataWriterExtensionLogicBase dataWriter, ZipSettings zipSettings)
{
DateTime dateTime;
TimeSpan dataStoragePeriod;
ulong currentRowCount;
ulong lastRow;
// START progress
ulong currentDataset;
ulong datasetCount;
ulong currentSegment;
ulong segmentCount;
datasetCount = (ulong)zipSettings.CampaignInfo.Value.SelectMany(variableInfo => variableInfo.Value).Count();
currentSegment = 0;
segmentCount = (ulong)Math.Ceiling(zipSettings.Block / (double)zipSettings.SegmentLength);
// END progress
currentRowCount = zipSettings.SegmentLength;
lastRow = zipSettings.Start + zipSettings.Block;
IList <IDataStorage> dataStorageSet;
// for each segment
for (ulong currentStart = zipSettings.Start; currentStart < lastRow; currentStart += currentRowCount)
{
currentDataset = 0;
dataStorageSet = new List <IDataStorage>();
if (currentStart + currentRowCount > zipSettings.Start + zipSettings.Block)
{
currentRowCount = zipSettings.Start + zipSettings.Block - currentStart;
}
// load data
foreach (var variableInfo in zipSettings.CampaignInfo.Value)
{
variableInfo.Value.ForEach(datasetName =>
{
dataStorageSet.Add(this.LoadDataset(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }/{ datasetName }", currentStart, 1, currentRowCount, 1));
this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Loading dataset segment { currentSegment * datasetCount + currentDataset + 1 } / { segmentCount * datasetCount } ..."));
currentDataset++;
});
}
this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Writing data of segment { currentSegment + 1 } / { segmentCount } ..."));
dateTime = zipSettings.DateTimeBegin.AddSeconds((currentStart - zipSettings.Start) / zipSettings.SampleRate);
dataStoragePeriod = TimeSpan.FromSeconds(currentRowCount / zipSettings.SampleRate);
dataWriter.Write(dateTime, dataStoragePeriod, dataStorageSet);
// clean up
dataStorageSet = null;
GC.Collect();
if (_cancellationToken.IsCancellationRequested)
{
return(false);
}
currentSegment++;
}
return(true);
}
