/// <summary>
/// Initializes a new instance of the <see cref="DataInformation"/> class.
/// </summary>
/// <param name="dre">
/// The data reader engine
/// </param>
public DataInformation(IDataReaderEngine dre)
{
if (dre == null)
{
throw new ArgumentException("No DataReaderEngine specified.");
}
dre.Reset();
ISet<string> keySet = new HashSet<string>();
ISet<string> groupSet = new HashSet<string>();
while (dre.MoveNextKeyable())
{
IKeyable currentKey = dre.CurrentKey;
if (currentKey.Series)
{
keySet.Add(currentKey.ShortCode);
}
else
{
groupSet.Add(currentKey.ShortCode);
}
while (dre.MoveNextObservation())
{
this._observations++;
}
}
this._keys = keySet.Count;
this._groups = groupSet.Count;
}
/// <summary>
/// Initializes a new instance of the <see cref="AbstractWrapperedDataReaderEngine"/> class.
/// </summary>
/// <param name="dataReaderEngine">The data reader engine.</param>
public AbstractWrapperedDataReaderEngine(IDataReaderEngine dataReaderEngine)
{
if (dataReaderEngine == null)
{
throw new ArgumentNullException("dataReaderEngine");
}
this._dataReaderEngine = dataReaderEngine;
}
/// <summary>
/// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor
/// </summary>
/// <param name="dataReader">
/// The IDataReaderEngine to read te SDMX-ML from
/// </param>
public override void ReadData(IDataReaderEngine dataReader)
{
this.DataSetStore.BeginDataSetImport();
bool isTimeSeries = KeyFamily.TimeDimension != null;
while (dataReader.MoveNextKeyable())
{
// In DatasetAttributes ci sono gli attributi a livello di dataset
foreach (var key in dataReader.DatasetAttributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
// In CurrentKey.Key ci sono le dimensioni
foreach (var key in dataReader.CurrentKey.Key)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
// In CurrentKey.Attributes ci sono gli attributi a livello di serie
foreach (var key in dataReader.CurrentKey.Attributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
while (dataReader.MoveNextObservation())
{
if (isTimeSeries)
{
this.DataSetStore.AddToStore(DimensionObject.TimeDimensionFixedId,
dataReader.CurrentObservation.ObsTime);
}
if (dataReader.CurrentObservation.CrossSection)
{
this.DataSetStore.AddToStore(dataReader.CurrentObservation.CrossSectionalValue.Concept,
dataReader.CurrentObservation.CrossSectionalValue.Code);
}
// In CurrentObservation.Attributes ci sono gli attributi a livello di osservazione
foreach (var key in dataReader.CurrentObservation.Attributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
this.DataSetStore.AddToStore(PrimaryMeasure.FixedId, dataReader.CurrentObservation.ObservationValue);
this.DataSetStore.AddRow();
}
}
this.DataSetStore.Commit();
}
/// <summary>
/// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor
/// </summary>
/// <param name="dataReader">
/// The IDataReaderEngine to read te SDMX-ML from
/// </param>
public override void ReadData(IDataReaderEngine dataReader)
{
this.DataSetStore.BeginDataSetImport();
bool isTimeSeries = KeyFamily.TimeDimension != null;
while (dataReader.MoveNextKeyable())
{
// In DatasetAttributes ci sono gli attributi a livello di dataset
foreach (var key in dataReader.DatasetAttributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
// In CurrentKey.Key ci sono le dimensioni
foreach (var key in dataReader.CurrentKey.Key)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
// In CurrentKey.Attributes ci sono gli attributi a livello di serie
foreach (var key in dataReader.CurrentKey.Attributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
while (dataReader.MoveNextObservation())
{
if (isTimeSeries)
{
this.DataSetStore.AddToStore(DimensionObject.TimeDimensionFixedId,
dataReader.CurrentObservation.ObsTime);
}
if (dataReader.CurrentObservation.CrossSection)
{
this.DataSetStore.AddToStore(dataReader.CurrentObservation.CrossSectionalValue.Concept,
dataReader.CurrentObservation.CrossSectionalValue.Code);
}
// In CurrentObservation.Attributes ci sono gli attributi a livello di osservazione
foreach (var key in dataReader.CurrentObservation.Attributes)
{
this.DataSetStore.AddToStore(key.Concept, key.Code);
}
this.DataSetStore.AddToStore(PrimaryMeasure.FixedId, dataReader.CurrentObservation.ObservationValue);
this.DataSetStore.AddRow();
}
}
this.DataSetStore.Commit();
}
/// <summary>
/// Gets all reported dates.
/// </summary>
/// <param name="dataReaderEngine">The data reader engine</param>
/// <returns>
/// The reported dates
/// </returns>
public IDictionary<TimeFormat, IList<string>> GetAllReportedDates(IDataReaderEngine dataReaderEngine)
{
dataReaderEngine.Reset();
var processedDates = new HashSet<string>(StringComparer.Ordinal);
var timeFormatToSortedMap = new Dictionary<TimeFormatEnumType, IDictionary<DateTime, string>>();
try
{
while (dataReaderEngine.MoveNextKeyable())
{
var obs = dataReaderEngine.CurrentObservation;
var obsTime = obs.ObsTime;
// Check we have not already processed this date
if (processedDates.Contains(obsTime))
{
var obsTimeFormat = obs.ObsTimeFormat;
IDictionary<DateTime, string> sortedMap;
// Get the correct sorted map (or create it if it does not yet exist)
if (!timeFormatToSortedMap.TryGetValue(obsTimeFormat, out sortedMap))
{
sortedMap = new SortedDictionary<DateTime, string>();
timeFormatToSortedMap.Add(obsTimeFormat, sortedMap);
}
Debug.Assert(obs.ObsAsTimeDate.HasValue, "Observation time was null. Ported from Java where null is valid key for HashMap.");
sortedMap.Add(obs.ObsAsTimeDate.Value, obsTime);
processedDates.Add(obsTime);
}
}
// Create the response map. TODO split method.
var responseMap = new Dictionary<TimeFormat, IList<string>>();
foreach (var keyValuePair in timeFormatToSortedMap)
{
var sortedDateList = new List<string>();
responseMap.Add(TimeFormat.GetFromEnum(keyValuePair.Key), sortedDateList);
var sortedMap = keyValuePair.Value;
sortedDateList.AddRange(sortedMap.Values);
}
return responseMap;
}
finally
{
dataReaderEngine.Reset();
}
}
/// <summary>
/// Gets the fixed concepts.
/// </summary>
/// <param name="dre">The data reader engine</param>
/// <param name="includeObs">The include observation</param>
/// <param name="includeAttributes">The include attributes</param>
/// <returns>
/// The list of key values
/// </returns>
public IList<IKeyValue> GetFixedConcepts(IDataReaderEngine dre, bool includeObs, bool includeAttributes)
{
dre.Reset();
var conceptMap = new Dictionary<string, string>(StringComparer.Ordinal);
var skipConcepts = new HashSet<string>(StringComparer.Ordinal);
while (dre.MoveNextKeyable())
{
var key = dre.CurrentKey;
if (includeAttributes)
{
ProcessKeyValues(key.Attributes, conceptMap, skipConcepts);
}
ProcessKeyValues(key.Key, conceptMap, skipConcepts);
if (includeObs)
{
while (dre.MoveNextObservation())
{
var obs = dre.CurrentObservation;
if (includeAttributes)
{
ProcessKeyValues(obs.Attributes, conceptMap, skipConcepts);
}
if (obs.CrossSection)
{
ProcessKeyValue(obs.CrossSectionalValue, conceptMap, skipConcepts);
}
}
}
}
var fixedKeyValues = new List<IKeyValue>(conceptMap.Select(pair => new KeyValueImpl(pair.Value, pair.Key)));
return fixedKeyValues;
}
/// <summary>
/// Returns a list of dimension - value pairs where there is only a single value in the data for the dimension. For
/// example if the entire
/// dataset had FREQ=A then one of the returned KeyValue pairs would be FREQ,A. If FREQ=A and Q this would not be
/// returned.
/// <p/>
/// <b>Note : an initial call to DataReaderEngine.reset will be made</b>
/// </summary>
/// <param name="dre">
/// The data reader engine
/// </param>
/// <param name="includeObs">
/// The include observation
/// </param>
/// <param name="includeAttributes">
/// If true will also report the attributes that have only one value in the entire dataset
/// </param>
/// <returns>
/// The list of dimension
/// </returns>
public IList<IKeyValue> GetFixedConcepts(IDataReaderEngine dre, bool includeObs, bool includeAttributes)
{
return this._fixedConceptEngine.GetFixedConcepts(dre, includeObs, includeAttributes);
}
/// <summary>
/// Returns DataInformation about the data, this processes the entire dataset to give an overview of what is in the
/// dataset.
/// </summary>
/// <param name="dre">
/// The data reader engine
/// </param>
/// <returns>
/// The DataInformation
/// </returns>
public DataInformation GetDataInformation(IDataReaderEngine dre)
{
return new DataInformation(dre);
}
/// <summary>
/// Returns an ordered list of all the unique dates for each time format in the dataset.
/// <p/>
/// This list is ordered with the earliest date first.
/// <p/>
/// This method will call
/// <code>
/// reset()
/// </code>
/// on the dataReaderEngine before and after the
/// information has been gathered
/// </summary>
/// <param name="dataReaderEngine">
/// The data reader engine
/// </param>
/// <returns>
/// The dictionary of time format to the list of dates that are contained for the time format
/// </returns>
public IDictionary<TimeFormat, IList<string>> GetAllReportedDates(IDataReaderEngine dataReaderEngine)
{
return this._reportedDateEngine.GetAllReportedDates(dataReaderEngine);
}
/// <summary>
/// Build constraint.
/// </summary>
/// <param name="dataReaderEngine">
/// The data reader engine.
/// </param>
/// <param name="attachment">
/// The attachment.
/// </param>
/// <param name="dataSourceAttachment">
/// The data source attachment.
/// </param>
/// <param name="indexAttributes">
/// The index attributes.
/// </param>
/// <param name="indexDataset">
/// The index dataset.
/// </param>
/// <param name="indexReportingPeriod">
/// The index reporting period.
/// </param>
/// <param name="indexTimeSeries">
/// The index time series.
/// </param>
/// <param name="definingDataPresent">
/// The defining data present.
/// </param>
/// <param name="refParameters">
/// The ref parameters.
/// </param>
/// <returns>
/// The <see cref="IContentConstraintObject"/>.
/// </returns>
public virtual IContentConstraintObject BuildConstraint(
IDataReaderEngine dataReaderEngine,
IStructureReference attachment,
IDataSource dataSourceAttachment,
bool indexAttributes,
bool indexDataset,
bool indexReportingPeriod,
bool indexTimeSeries,
bool definingDataPresent,
IMaintainableRefObject refParameters)
{
dataReaderEngine.Reset();
// TODO Should it check if there is more then one dataset in the datasource
if (!dataReaderEngine.MoveNextDataset())
{
throw new SdmxSemmanticException(
"Can not index time series for registered datasource, the data retrieved from the datasource does not contain a dataset");
}
if (!indexAttributes && !indexDataset && !indexReportingPeriod && !indexTimeSeries)
{
return null;
}
IDatasetHeader header = dataReaderEngine.CurrentDatasetHeader;
if (indexTimeSeries && !header.Timeseries)
{
throw new SdmxSemmanticException(
"Can not index time series for registered datasource, the data retrieved from the datasource is not time series");
}
// Create mutable Maintainable
IContentConstraintMutableObject mutableBean = new ContentConstraintMutableCore();
mutableBean.AgencyId = refParameters.AgencyId;
mutableBean.Id = refParameters.MaintainableId;
mutableBean.Version = refParameters.Version;
mutableBean.AddName("en", "Generated Constraint");
mutableBean.AddDescription("en", "Constraint built from dataset");
mutableBean.IsDefiningActualDataPresent = true;
mutableBean.ConstraintAttachment = BuildAttachement(attachment);
IConstraintDataKeySetMutableObject dataKeySet = null;
IDictionary<string, ISet<string>> cubeRegionMap = new Dictionary<string, ISet<string>>();
IDictionary<string, ISet<string>> attributeMap = new Dictionary<string, ISet<string>>();
DateTime? reportFrom = null;
DateTime? reportTo = null;
ISet<string> processedDates = new HashSet<string>();
while (dataReaderEngine.MoveNextKeyable())
{
IKeyable key = dataReaderEngine.CurrentKey;
if (key.Series)
{
// TODO Check if Cross Sectional and put out exception if it is
// 1. If indexing the time series store the time Series Key on the Constraint
if (indexTimeSeries)
{
if (dataKeySet == null)
{
dataKeySet = new ConstraintDataKeySetMutableCore();
mutableBean.IncludedSeriesKeys = dataKeySet;
}
IConstrainedDataKeyMutableObject dataKey = new ConstrainedDataKeyMutableCore();
dataKey.KeyValues.AddAll(key.Key);
dataKeySet.AddConstrainedDataKey(dataKey);
}
// 2. If indexing the dataset, store the individual code values
if (indexDataset)
{
StoreKeyValuesOnMap(key.Key, cubeRegionMap);
}
// 3. If indexing attributes, store the individual code values for each attribute
if (indexAttributes)
{
StoreKeyValuesOnMap(key.Attributes, attributeMap);
}
}
if (indexAttributes || indexReportingPeriod)
{
while (dataReaderEngine.MoveNextObservation())
{
IObservation obs = dataReaderEngine.CurrentObservation;
// If indexing the dates, determine the data start and end dates from the obs dates
// To save time, do not process the same date twice
if (indexReportingPeriod)
{
if (!processedDates.Contains(obs.ObsTime))
{
DateTime obsDate = obs.ObsAsTimeDate.GetValueOrDefault();
if (reportFrom == null || (reportFrom.Value.Ticks / 10000) > (obsDate.Ticks / 10000))
{
reportFrom = obsDate;
}
if (reportTo == null || (reportTo.Value.Ticks / 10000) < (obsDate.Ticks / 10000))
{
reportTo = obsDate;
}
processedDates.Add(obs.ObsTime);
}
}
if (indexAttributes)
{
StoreKeyValuesOnMap(obs.Attributes, attributeMap);
}
}
}
}
if (indexAttributes || indexDataset)
{
ICubeRegionMutableObject cubeRegionMutableBean = new CubeRegionMutableCore();
mutableBean.IncludedCubeRegion = cubeRegionMutableBean;
if (indexAttributes)
{
CreateKeyValues(attributeMap, cubeRegionMutableBean.AttributeValues);
}
if (indexDataset)
{
CreateKeyValues(cubeRegionMap, cubeRegionMutableBean.KeyValues);
}
}
if (indexReportingPeriod && reportFrom != null && reportTo.HasValue)
{
IReferencePeriodMutableObject refPeriodMutable = new ReferencePeriodMutableCore();
refPeriodMutable.EndTime = reportTo.Value;
refPeriodMutable.StartTime = reportFrom.Value;
mutableBean.ReferencePeriod = refPeriodMutable;
}
return mutableBean.ImmutableInstance;
}
/// <summary> /// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor /// </summary> /// <param name="dataReader"> /// The XMLReader to read te SDMX-ML from /// </param> public abstract void ReadData(IDataReaderEngine dataReader);
/// <summary> /// Read SDMX-ML DataSet files into the <see cref="IDataSetStore"/> given at the constructor /// </summary> /// <param name="dataReader"> /// The XMLReader to read te SDMX-ML from /// </param> public abstract void ReadData(IDataReaderEngine dataReader);
