/// <summary> /// Initializes a new instance of the <see cref="DataInformation"/> class. /// </summary> /// <param name="dre"> /// The data reader engine /// </param> public DataInformation(IDataReaderEngine dre) { if (dre == null) { throw new ArgumentException("No DataReaderEngine specified."); } dre.Reset(); ISet<string> keySet = new HashSet<string>(); ISet<string> groupSet = new HashSet<string>(); while (dre.MoveNextKeyable()) { IKeyable currentKey = dre.CurrentKey; if (currentKey.Series) { keySet.Add(currentKey.ShortCode); } else { groupSet.Add(currentKey.ShortCode); } while (dre.MoveNextObservation()) { this._observations++; } } this._keys = keySet.Count; this._groups = groupSet.Count; }
/// <summary> /// Initializes a new instance of the <see cref="AbstractWrapperedDataReaderEngine"/> class. /// </summary> /// <param name="dataReaderEngine">The data reader engine.</param> public AbstractWrapperedDataReaderEngine(IDataReaderEngine dataReaderEngine) { if (dataReaderEngine == null) { throw new ArgumentNullException("dataReaderEngine"); } this._dataReaderEngine = dataReaderEngine; }
/// <summary> /// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor /// </summary> /// <param name="dataReader"> /// The IDataReaderEngine to read te SDMX-ML from /// </param> public override void ReadData(IDataReaderEngine dataReader) { this.DataSetStore.BeginDataSetImport(); bool isTimeSeries = KeyFamily.TimeDimension != null; while (dataReader.MoveNextKeyable()) { // In DatasetAttributes ci sono gli attributi a livello di dataset foreach (var key in dataReader.DatasetAttributes) { this.DataSetStore.AddToStore(key.Concept, key.Code); } // In CurrentKey.Key ci sono le dimensioni foreach (var key in dataReader.CurrentKey.Key) { this.DataSetStore.AddToStore(key.Concept, key.Code); } // In CurrentKey.Attributes ci sono gli attributi a livello di serie foreach (var key in dataReader.CurrentKey.Attributes) { this.DataSetStore.AddToStore(key.Concept, key.Code); } while (dataReader.MoveNextObservation()) { if (isTimeSeries) { this.DataSetStore.AddToStore(DimensionObject.TimeDimensionFixedId, dataReader.CurrentObservation.ObsTime); } if (dataReader.CurrentObservation.CrossSection) { this.DataSetStore.AddToStore(dataReader.CurrentObservation.CrossSectionalValue.Concept, dataReader.CurrentObservation.CrossSectionalValue.Code); } // In CurrentObservation.Attributes ci sono gli attributi a livello di osservazione foreach (var key in dataReader.CurrentObservation.Attributes) { this.DataSetStore.AddToStore(key.Concept, key.Code); } this.DataSetStore.AddToStore(PrimaryMeasure.FixedId, dataReader.CurrentObservation.ObservationValue); this.DataSetStore.AddRow(); } } this.DataSetStore.Commit(); }
/// <summary> /// Gets all reported dates. /// </summary> /// <param name="dataReaderEngine">The data reader engine</param> /// <returns> /// The reported dates /// </returns> public IDictionary<TimeFormat, IList<string>> GetAllReportedDates(IDataReaderEngine dataReaderEngine) { dataReaderEngine.Reset(); var processedDates = new HashSet<string>(StringComparer.Ordinal); var timeFormatToSortedMap = new Dictionary<TimeFormatEnumType, IDictionary<DateTime, string>>(); try { while (dataReaderEngine.MoveNextKeyable()) { var obs = dataReaderEngine.CurrentObservation; var obsTime = obs.ObsTime; // Check we have not already processed this date if (processedDates.Contains(obsTime)) { var obsTimeFormat = obs.ObsTimeFormat; IDictionary<DateTime, string> sortedMap; // Get the correct sorted map (or create it if it does not yet exist) if (!timeFormatToSortedMap.TryGetValue(obsTimeFormat, out sortedMap)) { sortedMap = new SortedDictionary<DateTime, string>(); timeFormatToSortedMap.Add(obsTimeFormat, sortedMap); } Debug.Assert(obs.ObsAsTimeDate.HasValue, "Observation time was null. Ported from Java where null is valid key for HashMap."); sortedMap.Add(obs.ObsAsTimeDate.Value, obsTime); processedDates.Add(obsTime); } } // Create the response map. TODO split method. var responseMap = new Dictionary<TimeFormat, IList<string>>(); foreach (var keyValuePair in timeFormatToSortedMap) { var sortedDateList = new List<string>(); responseMap.Add(TimeFormat.GetFromEnum(keyValuePair.Key), sortedDateList); var sortedMap = keyValuePair.Value; sortedDateList.AddRange(sortedMap.Values); } return responseMap; } finally { dataReaderEngine.Reset(); } }
/// <summary> /// Gets the fixed concepts. /// </summary> /// <param name="dre">The data reader engine</param> /// <param name="includeObs">The include observation</param> /// <param name="includeAttributes">The include attributes</param> /// <returns> /// The list of key values /// </returns> public IList<IKeyValue> GetFixedConcepts(IDataReaderEngine dre, bool includeObs, bool includeAttributes) { dre.Reset(); var conceptMap = new Dictionary<string, string>(StringComparer.Ordinal); var skipConcepts = new HashSet<string>(StringComparer.Ordinal); while (dre.MoveNextKeyable()) { var key = dre.CurrentKey; if (includeAttributes) { ProcessKeyValues(key.Attributes, conceptMap, skipConcepts); } ProcessKeyValues(key.Key, conceptMap, skipConcepts); if (includeObs) { while (dre.MoveNextObservation()) { var obs = dre.CurrentObservation; if (includeAttributes) { ProcessKeyValues(obs.Attributes, conceptMap, skipConcepts); } if (obs.CrossSection) { ProcessKeyValue(obs.CrossSectionalValue, conceptMap, skipConcepts); } } } } var fixedKeyValues = new List<IKeyValue>(conceptMap.Select(pair => new KeyValueImpl(pair.Value, pair.Key))); return fixedKeyValues; }
/// <summary> /// Returns a list of dimension - value pairs where there is only a single value in the data for the dimension. For /// example if the entire /// dataset had FREQ=A then one of the returned KeyValue pairs would be FREQ,A. If FREQ=A and Q this would not be /// returned. /// <p/> /// <b>Note : an initial call to DataReaderEngine.reset will be made</b> /// </summary> /// <param name="dre"> /// The data reader engine /// </param> /// <param name="includeObs"> /// The include observation /// </param> /// <param name="includeAttributes"> /// If true will also report the attributes that have only one value in the entire dataset /// </param> /// <returns> /// The list of dimension /// </returns> public IList<IKeyValue> GetFixedConcepts(IDataReaderEngine dre, bool includeObs, bool includeAttributes) { return this._fixedConceptEngine.GetFixedConcepts(dre, includeObs, includeAttributes); }
/// <summary> /// Returns DataInformation about the data, this processes the entire dataset to give an overview of what is in the /// dataset. /// </summary> /// <param name="dre"> /// The data reader engine /// </param> /// <returns> /// The DataInformation /// </returns> public DataInformation GetDataInformation(IDataReaderEngine dre) { return new DataInformation(dre); }
/// <summary> /// Returns an ordered list of all the unique dates for each time format in the dataset. /// <p/> /// This list is ordered with the earliest date first. /// <p/> /// This method will call /// <code> /// reset() /// </code> /// on the dataReaderEngine before and after the /// information has been gathered /// </summary> /// <param name="dataReaderEngine"> /// The data reader engine /// </param> /// <returns> /// The dictionary of time format to the list of dates that are contained for the time format /// </returns> public IDictionary<TimeFormat, IList<string>> GetAllReportedDates(IDataReaderEngine dataReaderEngine) { return this._reportedDateEngine.GetAllReportedDates(dataReaderEngine); }
/// <summary> /// Build constraint. /// </summary> /// <param name="dataReaderEngine"> /// The data reader engine. /// </param> /// <param name="attachment"> /// The attachment. /// </param> /// <param name="dataSourceAttachment"> /// The data source attachment. /// </param> /// <param name="indexAttributes"> /// The index attributes. /// </param> /// <param name="indexDataset"> /// The index dataset. /// </param> /// <param name="indexReportingPeriod"> /// The index reporting period. /// </param> /// <param name="indexTimeSeries"> /// The index time series. /// </param> /// <param name="definingDataPresent"> /// The defining data present. /// </param> /// <param name="refParameters"> /// The ref parameters. /// </param> /// <returns> /// The <see cref="IContentConstraintObject"/>. /// </returns> public virtual IContentConstraintObject BuildConstraint( IDataReaderEngine dataReaderEngine, IStructureReference attachment, IDataSource dataSourceAttachment, bool indexAttributes, bool indexDataset, bool indexReportingPeriod, bool indexTimeSeries, bool definingDataPresent, IMaintainableRefObject refParameters) { dataReaderEngine.Reset(); // TODO Should it check if there is more then one dataset in the datasource if (!dataReaderEngine.MoveNextDataset()) { throw new SdmxSemmanticException( "Can not index time series for registered datasource, the data retrieved from the datasource does not contain a dataset"); } if (!indexAttributes && !indexDataset && !indexReportingPeriod && !indexTimeSeries) { return null; } IDatasetHeader header = dataReaderEngine.CurrentDatasetHeader; if (indexTimeSeries && !header.Timeseries) { throw new SdmxSemmanticException( "Can not index time series for registered datasource, the data retrieved from the datasource is not time series"); } // Create mutable Maintainable IContentConstraintMutableObject mutableBean = new ContentConstraintMutableCore(); mutableBean.AgencyId = refParameters.AgencyId; mutableBean.Id = refParameters.MaintainableId; mutableBean.Version = refParameters.Version; mutableBean.AddName("en", "Generated Constraint"); mutableBean.AddDescription("en", "Constraint built from dataset"); mutableBean.IsDefiningActualDataPresent = true; mutableBean.ConstraintAttachment = BuildAttachement(attachment); IConstraintDataKeySetMutableObject dataKeySet = null; IDictionary<string, ISet<string>> cubeRegionMap = new Dictionary<string, ISet<string>>(); IDictionary<string, ISet<string>> attributeMap = new Dictionary<string, ISet<string>>(); DateTime? reportFrom = null; DateTime? reportTo = null; ISet<string> processedDates = new HashSet<string>(); while (dataReaderEngine.MoveNextKeyable()) { IKeyable key = dataReaderEngine.CurrentKey; if (key.Series) { // TODO Check if Cross Sectional and put out exception if it is // 1. If indexing the time series store the time Series Key on the Constraint if (indexTimeSeries) { if (dataKeySet == null) { dataKeySet = new ConstraintDataKeySetMutableCore(); mutableBean.IncludedSeriesKeys = dataKeySet; } IConstrainedDataKeyMutableObject dataKey = new ConstrainedDataKeyMutableCore(); dataKey.KeyValues.AddAll(key.Key); dataKeySet.AddConstrainedDataKey(dataKey); } // 2. If indexing the dataset, store the individual code values if (indexDataset) { StoreKeyValuesOnMap(key.Key, cubeRegionMap); } // 3. If indexing attributes, store the individual code values for each attribute if (indexAttributes) { StoreKeyValuesOnMap(key.Attributes, attributeMap); } } if (indexAttributes || indexReportingPeriod) { while (dataReaderEngine.MoveNextObservation()) { IObservation obs = dataReaderEngine.CurrentObservation; // If indexing the dates, determine the data start and end dates from the obs dates // To save time, do not process the same date twice if (indexReportingPeriod) { if (!processedDates.Contains(obs.ObsTime)) { DateTime obsDate = obs.ObsAsTimeDate.GetValueOrDefault(); if (reportFrom == null || (reportFrom.Value.Ticks / 10000) > (obsDate.Ticks / 10000)) { reportFrom = obsDate; } if (reportTo == null || (reportTo.Value.Ticks / 10000) < (obsDate.Ticks / 10000)) { reportTo = obsDate; } processedDates.Add(obs.ObsTime); } } if (indexAttributes) { StoreKeyValuesOnMap(obs.Attributes, attributeMap); } } } } if (indexAttributes || indexDataset) { ICubeRegionMutableObject cubeRegionMutableBean = new CubeRegionMutableCore(); mutableBean.IncludedCubeRegion = cubeRegionMutableBean; if (indexAttributes) { CreateKeyValues(attributeMap, cubeRegionMutableBean.AttributeValues); } if (indexDataset) { CreateKeyValues(cubeRegionMap, cubeRegionMutableBean.KeyValues); } } if (indexReportingPeriod && reportFrom != null && reportTo.HasValue) { IReferencePeriodMutableObject refPeriodMutable = new ReferencePeriodMutableCore(); refPeriodMutable.EndTime = reportTo.Value; refPeriodMutable.StartTime = reportFrom.Value; mutableBean.ReferencePeriod = refPeriodMutable; } return mutableBean.ImmutableInstance; }
/// <summary> /// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor /// </summary> /// <param name="dataReader"> /// The XMLReader to read te SDMX-ML from /// </param> public abstract void ReadData(IDataReaderEngine dataReader);