private void ProcessData(DataWriterExtensionLogicBase dataWriter, DataReaderProgressRecord progressRecord, bool applyStatus) { var buffers = progressRecord.DatasetToRecordMap.Values.Select(dataRecord => { double[] data; var elementType = dataRecord.Dataset.GetType().GetElementType(); if (applyStatus || elementType != typeof(double)) { data = BufferUtilities.ApplyDatasetStatus2(dataRecord.Dataset, dataRecord.Status); } else { data = (double[])dataRecord.Dataset; } return((IBuffer)BufferUtilities.CreateSimpleBuffer(data)); }).ToList(); var period = progressRecord.End - progressRecord.Begin; dataWriter.Write(progressRecord.Begin, period, buffers); // clean up buffers = null; GC.Collect(); }
private void CreateFiles(ClaimsPrincipal user, DataWriterExtensionLogicBase dataWriter, ExportParameters exportParameters, SparseProjectInfo sparseProject, CancellationToken cancellationToken) { var datasets = sparseProject.Channels.SelectMany(channel => channel.Datasets); var registrationToDatasetsMap = new Dictionary <DataReaderRegistration, List <DatasetInfo> >(); foreach (var dataset in datasets) { if (!registrationToDatasetsMap.ContainsKey(dataset.Registration)) { registrationToDatasetsMap[dataset.Registration] = new List <DatasetInfo>(); } registrationToDatasetsMap[dataset.Registration].Add(dataset); } var progressHandler = (EventHandler <double>)((sender, e) => { this.OnProgress(new ProgressUpdatedEventArgs(e, $"Loading data ...")); }); foreach (var entry in registrationToDatasetsMap) { if (entry.Value.Any()) { var registration = entry.Key; var dataReader = _databaseManager.GetDataReader(user, registration); dataReader.Progress.ProgressChanged += progressHandler; try { var isAggregation = dataReader.Registration.Equals(_databaseManager.State.AggregationRegistration); foreach (var progressRecord in dataReader.Read(entry.Value, exportParameters.Begin, exportParameters.End, _blockSizeLimit, cancellationToken)) { this.ProcessData(dataWriter, progressRecord, applyStatus: !isAggregation); } } finally { dataReader.Progress.ProgressChanged -= progressHandler; } } } cancellationToken.ThrowIfCancellationRequested(); }
private bool CreateFiles(DataWriterExtensionLogicBase dataWriter, ZipSettings zipSettings) { DateTime dateTime; TimeSpan dataStoragePeriod; ulong currentRowCount; ulong lastRow; // START progress ulong currentDataset; ulong datasetCount; ulong currentSegment; ulong segmentCount; datasetCount = (ulong)zipSettings.CampaignInfo.Value.SelectMany(variableInfo => variableInfo.Value).Count(); currentSegment = 0; segmentCount = (ulong)Math.Ceiling(zipSettings.Block / (double)zipSettings.SegmentLength); // END progress currentRowCount = zipSettings.SegmentLength; lastRow = zipSettings.Start + zipSettings.Block; IList <IDataStorage> dataStorageSet; // for each segment for (ulong currentStart = zipSettings.Start; currentStart < lastRow; currentStart += currentRowCount) { currentDataset = 0; dataStorageSet = new List <IDataStorage>(); if (currentStart + currentRowCount > zipSettings.Start + zipSettings.Block) { currentRowCount = zipSettings.Start + zipSettings.Block - currentStart; } // load data foreach (var variableInfo in zipSettings.CampaignInfo.Value) { variableInfo.Value.ForEach(datasetName => { dataStorageSet.Add(this.LoadDataset(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }/{ datasetName }", currentStart, 1, currentRowCount, 1)); this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Loading dataset segment { currentSegment * datasetCount + currentDataset + 1 } / { segmentCount * datasetCount } ...")); currentDataset++; }); } this.OnProgressUpdated(new ProgressUpdatedEventArgs((currentSegment * (double)datasetCount + currentDataset) / (segmentCount * datasetCount) * 100, $"Writing data of segment { currentSegment + 1 } / { segmentCount } ...")); dateTime = zipSettings.DateTimeBegin.AddSeconds((currentStart - zipSettings.Start) / zipSettings.SampleRate); dataStoragePeriod = TimeSpan.FromSeconds(currentRowCount / zipSettings.SampleRate); dataWriter.Write(dateTime, dataStoragePeriod, dataStorageSet); // clean up dataStorageSet = null; GC.Collect(); if (_cancellationToken.IsCancellationRequested) { return(false); } currentSegment++; } return(true); }