private void ProcessData(DataWriterExtensionLogicBase dataWriter, DataReaderProgressRecord progressRecord, bool applyStatus) { var buffers = progressRecord.DatasetToRecordMap.Values.Select(dataRecord => { double[] data; var elementType = dataRecord.Dataset.GetType().GetElementType(); if (applyStatus || elementType != typeof(double)) { data = BufferUtilities.ApplyDatasetStatus2(dataRecord.Dataset, dataRecord.Status); } else { data = (double[])dataRecord.Dataset; } return((IBuffer)BufferUtilities.CreateSimpleBuffer(data)); }).ToList(); var period = progressRecord.End - progressRecord.Begin; dataWriter.Write(progressRecord.Begin, period, buffers); // clean up buffers = null; GC.Collect(); }
private bool CreateFiles(DataWriterExtensionLogicBase dataWriter, ZipSettings zipSettings) { DateTime dateTime; TimeSpan dataStoragePeriod; ulong currentRowCount; ulong lastRow; // START progress ulong currentDataset; ulong datasetCount; ulong currentSegment; ulong segmentCount; datasetCount = (ulong)zipSettings.CampaignInfo.Value.SelectMany(variableInfo => variableInfo.Value).Count(); currentSegment = 0; segmentCount = (ulong)Math.Ceiling(zipSettings.Block / (double)zipSettings.SegmentLength); // END progress currentRowCount = zipSettings.SegmentLength; lastRow = zipSettings.Start + zipSettings.Block; IList <IDataStorage> dataStorageSet; // for each segment for (ulong currentStart = zipSettings.Start; currentStart < lastRow; currentStart += currentRowCount) { currentDataset = 0; dataStorageSet = new List <IDataStorage>(); if (currentStart + currentRowCount > zipSettings.Start + zipSettings.Block) { currentRowCount = zipSettings.Start + zipSettings.Block - currentStart; } // load data foreach (var variableInfo in zipSettings.CampaignInfo.Value) { variableInfo.Value.ForEach(datasetName => { dataStorageSet.Add(this.LoadDataset(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }/{ datasetName }", currentStart, 1, currentRowCount, 1)); this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Loading dataset segment { currentSegment * datasetCount + currentDataset + 1 } / { segmentCount * datasetCount } ...")); currentDataset++; }); } this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Writing data of segment { currentSegment + 1 } / { segmentCount } ...")); dateTime = zipSettings.DateTimeBegin.AddSeconds((currentStart - zipSettings.Start) / zipSettings.SampleRate); dataStoragePeriod = TimeSpan.FromSeconds(currentRowCount / zipSettings.SampleRate); dataWriter.Write(dateTime, dataStoragePeriod, dataStorageSet); // clean up dataStorageSet = null; GC.Collect(); if (_cancellationToken.IsCancellationRequested) { return(false); } currentSegment++; } return(true); }