/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfoTabular"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the sdmx query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
/// <param name="complexWriter">
/// The tabular Writer.
/// </param>
public DataRetrievalInfoComplex(
MappingSetEntity mappingSet,
IComplexDataQuery query,
ConnectionStringSettings connectionStringSettings,
IDataWriterEngine complexWriter)
: base(mappingSet, query, connectionStringSettings, complexWriter)
{
this.Limit = query.DefaultLimit.HasValue ? query.DefaultLimit.Value : 0;
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfoTabular"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the sdmx query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
/// <param name="tabularWriter">
/// The tabular Writer.
/// </param>
public DataRetrievalInfoTabular(
MappingSetEntity mappingSet,
IDataQuery query,
ConnectionStringSettings connectionStringSettings,
ITabularWriter tabularWriter)
: base(mappingSet, query, connectionStringSettings)
{
this._tabularWriter = tabularWriter;
}
/// <summary>
/// This method checks whether a mapping set is complete, in order to be used for the Data Retriever.
/// For example,one of the checks is whether all dimensions and mandatory attributes are mapped.
/// </summary>
/// <param name="mappingSet">
/// The mapping set to be checked
/// </param>
/// <returns>
/// True if the mapping is complete
/// </returns>
public static bool IsMappingSetComplete(MappingSetEntity mappingSet)
{
var componentMappings = new Dictionary<ComponentEntity, MappingEntity>();
foreach (MappingEntity mapping in mappingSet.Mappings)
{
foreach (ComponentEntity component in mapping.Components)
{
componentMappings.Add(component, mapping);
}
}
return IsMappingSetComplete(mappingSet.Dataflow.Dsd, componentMappings);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfoSeries" /> class.
/// </summary>
/// <param name="mappingSet">The mapping set of the dataflow found in the sdmx query</param>
/// <param name="query">The current SDMX Query object</param>
/// <param name="connectionStringSettings">The Mapping Store connection string settings</param>
/// <param name="seriesWriter">The series Writer.</param>
/// <param name="sdmxSchemaVersion">The SDMX schema version.</param>
public DataRetrievalInfoSeries(MappingSetEntity mappingSet, IDataQuery query, ConnectionStringSettings connectionStringSettings, IRDFDataWriterEngine seriesWriter)
: base(mappingSet, query, connectionStringSettings)
{
this._seriesWriter = seriesWriter;
this._seriesObsComponents = new List<IComponentMapping>();
this._dataSetAttributes = new List<IComponentMapping>();
this._groups = new Dictionary<GroupEntity, GroupInformation>();
this._useDataSetSqlQuery = mappingSet.Dataflow.Dsd.Groups.Count > 0;
this.BuildSeriesMappings();
// add dimension mappings to groups
this.BuildTimeSeriesGroupMappings();
}
/// <summary>
/// This method generates the FROM part of the query.
/// </summary>
/// <param name="mappingSet">
/// The <see cref="MappingSetEntity"/> that contains the mappings for the Dataflow of the query
/// </param>
/// <param name="log">
/// The log
/// </param>
/// <returns>
/// A string containing the FROM part
/// </returns>
protected static string GenerateFrom(MappingSetEntity mappingSet)
{
Logger.Info(Resources.DataRetriever_GenerateFrom_Begin_GenerateFrom____);
var from = new StringBuilder(" from ");
from.AppendFormat(CultureInfo.InvariantCulture, "( {0} ) virtualDataset ", mappingSet.DataSet.Query);
// log for easy debug
Logger.Info(
string.Format(
CultureInfo.InvariantCulture, Resources.DataRetriever_GenerateFrom_Generated_FROM___0_, from));
Logger.Info(Resources.DataRetriever_GenerateFrom_End_GenerateFrom____);
return from.ToString();
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfoXS"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the sdmx query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
/// <param name="xsWriter">
/// The ICrossSectionalWriter Writer.
/// </param>
public DataRetrievalInfoXS(
MappingSetEntity mappingSet,
IDataQuery query,
ConnectionStringSettings connectionStringSettings,
ICrossSectionalWriterEngine xsWriter)
: base(mappingSet, query, connectionStringSettings)
{
this._xsWriter = xsWriter;
if (this.MeasureComponent == null)
{
foreach (var crossSectionalMeasure in this.MappingSet.Dataflow.Dsd.CrossSectionalMeasures)
{
this._xsMeasureCodeToConcept.Add(
crossSectionalMeasure.CrossSectionalMeasureCode, crossSectionalMeasure.Id);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfo"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the SDMX query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
public DataRetrievalInfo(MappingSetEntity mappingSet, IComplexDataQuery query, ConnectionStringSettings connectionStringSettings)
{
if (mappingSet == null)
{
throw new ArgumentNullException("mappingSet");
}
if (connectionStringSettings == null)
{
throw new ArgumentNullException("connectionStringSettings");
}
this._schemaVersion = SdmxSchemaEnumType.VersionTwoPointOne;
this._mappingSet = mappingSet;
this._complexQuery = query;
this._connectionStringSettings = connectionStringSettings;
this._limit = query.DefaultLimit.HasValue ? query.DefaultLimit.Value : 0;
this._isTimePeriodAtObservation = DimensionObject.TimeDimensionFixedId.Equals(query.DimensionAtObservation);
if (!this._isTimePeriodAtObservation)
{
this._dimensionAtObservation = query.DimensionAtObservation;
}
this.BuildMappings();
this._buildEffectiveDimensionAtObservation = this.BuildEffectiveDimensionAtObservation();
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfo" /> class.
/// </summary>
/// <param name="mappingSet">The mapping set of the dataflow found in the SDMX query</param>
/// <param name="query">The current SDMX Query object</param>
/// <param name="connectionStringSettings">The Mapping Store connection string settings</param>
/// <param name="schemaVersion">The schema version.</param>
/// <exception cref="System.ArgumentNullException">
/// mappingSet
/// or
/// connectionStringSettings
/// </exception>
public DataRetrievalInfo(MappingSetEntity mappingSet, IDataQuery query, ConnectionStringSettings connectionStringSettings, SdmxSchemaEnumType schemaVersion)
{
if (mappingSet == null)
{
throw new ArgumentNullException("mappingSet");
}
if (connectionStringSettings == null)
{
throw new ArgumentNullException("connectionStringSettings");
}
this._mappingSet = mappingSet;
this._query = query;
this._connectionStringSettings = connectionStringSettings;
this._schemaVersion = schemaVersion;
if (schemaVersion == SdmxSchemaEnumType.VersionTwoPointOne)
{
this._isTimePeriodAtObservation = DimensionObject.TimeDimensionFixedId.Equals(query.DimensionAtObservation);
if (!this._isTimePeriodAtObservation)
{
this._dimensionAtObservation = query.DimensionAtObservation;
}
this._limit = 0; // REST does not support default limit.
}
else
{
this._limit = query.FirstNObservations.HasValue ? query.FirstNObservations.Value : 0;
this._isTimePeriodAtObservation = true;
}
this.BuildMappings();
this._buildEffectiveDimensionAtObservation = this.BuildEffectiveDimensionAtObservation();
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfo"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the sdmx query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
public DataRetrievalInfo(MappingSetEntity mappingSet, IDataQuery query, ConnectionStringSettings connectionStringSettings)
: this(mappingSet, query, connectionStringSettings, SdmxSchemaEnumType.VersionTwo)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DataRetrievalInfo"/> class.
/// </summary>
/// <param name="mappingSet">
/// The mapping set of the dataflow found in the sdmx query
/// </param>
/// <param name="query">
/// The current SDMX Query object
/// </param>
/// <param name="connectionStringSettings">
/// The Mapping Store connection string settings
/// </param>
public DataRetrievalInfo(MappingSetEntity mappingSet, IComplexDataQuery query, ConnectionStringSettings connectionStringSettings)
{
if (mappingSet == null)
{
throw new ArgumentNullException("mappingSet");
}
if (connectionStringSettings == null)
{
throw new ArgumentNullException("connectionStringSettings");
}
this._mappingSet = mappingSet;
this._complexQuery = query;
this._connectionStringSettings = connectionStringSettings;
this._limit = query.FirstNObservations.HasValue ? query.FirstNObservations.Value : 0;
//this._dimensionAtObservation = query.DimensionAtObservation();
this._isTimePeriodAtObservation = DimensionObject.TimeDimensionFixedId.Equals(query.DimensionAtObservation);
if (!this._isTimePeriodAtObservation)
{
this._dimensionAtObservation = query.DimensionAtObservation;
}
this.BuildMappings();
}
/// <summary>
/// SRA-349 Filter data to be returned
/// </summary>
/// <param name="query">
/// The IDataQuery containing the SDMX query
/// </param>
/// <returns>
/// The filtered mapping set entity for the dataflow in <paramref name="query"/>
/// </returns>
private MappingSetEntity FilterMappingSet(IDataQuery query, MappingSetEntity mappingSet)
{
return mappingSet;
if (query.DataQueryDetail.EnumType == DataQueryDetailEnumType.DataOnly || query.DataQueryDetail.EnumType == DataQueryDetailEnumType.SeriesKeysOnly)
{
mappingSet.Dataflow.Dsd.Attributes.Clear();
mappingSet.Dataflow.Dsd.Groups.Clear();
}
bool bReiterate = false;
//Request details of the data to be returned (Full, DataOnly, SeriesKeyOnly, NoData) SRA-349
switch (query.DataQueryDetail.EnumType)
{
case DataQueryDetailEnumType.DataOnly: //No Data attributes.
//Remove all attributes mappings from the MappingSetEntity
foreach (MappingEntity mapEntity in mappingSet.Mappings)
{
bReiterate = true;
while (bReiterate)
{
bReiterate = false;
foreach (ComponentEntity comp in mapEntity.Components)
{
if (IsAttributeComponent(comp))
{
bReiterate = true;
mapEntity.Components.Remove(comp);
break;
}
}
}
}
break;
case DataQueryDetailEnumType.SeriesKeysOnly: //Only dimensions, only the Series element
//Remove all attributes & observation mapping from the MappingSetEntity
foreach (MappingEntity mapEntity in mappingSet.Mappings)
{
bReiterate = true;
while (bReiterate)
{
bReiterate = false;
foreach (ComponentEntity comp in mapEntity.Components)
{
if (IsObservationComponent(comp) || IsAttributeComponent(comp))
{
bReiterate = true;
mapEntity.Components.Remove(comp);
break;
}
}
}
}
break;
case DataQueryDetailEnumType.NoData: //Groups, Series, Annotations, Attributes, Dimensions but no observations.
//Remove all observation mapping from the MappingSetEntity
foreach (MappingEntity mapEntity in mappingSet.Mappings)
{
bReiterate = true;
while (bReiterate)
{
bReiterate = false;
foreach (ComponentEntity comp in mapEntity.Components)
{
if (IsObservationComponent(comp))
{
bReiterate = true;
mapEntity.Components.Remove(comp);
break;
}
}
}
}
break;
default:
break;
}
return mappingSet;
}
/// <summary>
/// This method retrieves a list of <see cref="MappingEntity"/> objects
/// by the mapping set system identifier
/// </summary>
/// <param name="mappingStoreDb">
/// The <see cref="Database"/> instance for Mapping Store database
/// </param>
/// <param name="mappingSet">
/// The mapping set
/// </param>
private static void GetMappingsByMappingSetSysId(Database mappingStoreDb, MappingSetEntity mappingSet)
{
long sysId = mappingSet.SysId;
var dictionary = new Dictionary<long, MappingEntity>();
var componentMap = new Dictionary<long, ComponentEntity>();
bool transcoded = false;
MappingEntity timeDimensionMapping = null;
AddToComponentMap(mappingSet.Dataflow.Dsd.Dimensions, componentMap);
if (mappingSet.Dataflow.Dsd.TimeDimension != null)
{
componentMap.Add(mappingSet.Dataflow.Dsd.TimeDimension.SysId, mappingSet.Dataflow.Dsd.TimeDimension);
}
componentMap.Add(mappingSet.Dataflow.Dsd.PrimaryMeasure.SysId, mappingSet.Dataflow.Dsd.PrimaryMeasure);
AddToComponentMap(mappingSet.Dataflow.Dsd.Attributes, componentMap);
AddToComponentMap(mappingSet.Dataflow.Dsd.CrossSectionalMeasures, componentMap);
string paramId = mappingStoreDb.BuildParameterName(ParameterNameConstants.IdParameter);
var sqlCommand = new StringBuilder();
// select
sqlCommand.Append("SELECT CMAP.MAP_ID, CMAP.TYPE, CMAP.CONSTANT, CMAPCOMP.COMP_ID, TR.TR_ID, TR.EXPRESSION ");
// from1
sqlCommand.Append("FROM COMPONENT_MAPPING CMAP INNER JOIN MAPPING_SET MSET ON MSET.MAP_SET_ID = CMAP.MAP_SET_ID INNER JOIN COM_COL_MAPPING_COMPONENT CMAPCOMP ON CMAPCOMP.MAP_ID = CMAP.MAP_ID ");
sqlCommand.Append(" LEFT OUTER JOIN TRANSCODING TR ON TR.MAP_ID = CMAP.MAP_ID ");
// where
sqlCommand.AppendFormat(" WHERE MSET.MAP_SET_ID = {0} ", paramId);
using (DbCommand command = mappingStoreDb.GetSqlStringCommand(sqlCommand.ToString()))
{
mappingStoreDb.AddInParameter(command, ParameterNameConstants.IdParameter, DbType.Int64, sysId);
using (IDataReader dataReader = mappingStoreDb.ExecuteReader(command))
{
int mapID = dataReader.GetOrdinal("MAP_ID");
int type = dataReader.GetOrdinal("TYPE");
int constant = dataReader.GetOrdinal("CONSTANT");
int compIDIdx = dataReader.GetOrdinal("COMP_ID");
int tridIdx = dataReader.GetOrdinal("TR_ID");
int exprIdx = dataReader.GetOrdinal("EXPRESSION");
while (dataReader.Read())
{
// get existing or create new mapping object
long mapId = DataReaderHelper.GetInt64(dataReader, mapID);
MappingEntity mapping;
if (!dictionary.TryGetValue(mapId, out mapping))
{
// create new mapping and append it to the mapping set
mapping = new MappingEntity(mapId) { MappingType = DataReaderHelper.GetString(dataReader, type), Constant = DataReaderHelper.GetString(dataReader, constant) };
long trid = DataReaderHelper.GetInt64(dataReader, tridIdx);
if (trid > 0)
{
mapping.Transcoding = new TranscodingEntity(trid) { Expression = DataReaderHelper.GetString(dataReader, exprIdx) };
transcoded = true;
}
dictionary.Add(mapId, mapping);
mappingSet.Mappings.Add(mapping);
}
long componentSysId = DataReaderHelper.GetInt64(dataReader, compIDIdx);
ComponentEntity component;
if (componentMap.TryGetValue(componentSysId, out component))
{
mapping.Components.Add(component);
if (component.Equals(mappingSet.Dataflow.Dsd.TimeDimension) && mapping.Transcoding != null)
{
timeDimensionMapping = mapping;
}
}
}
}
}
// second select
sqlCommand = new StringBuilder();
sqlCommand.Append(
"SELECT CMAP.MAP_ID, CMAP.TYPE, CMAP.CONSTANT, DSCOL.COL_ID, DSCOL.NAME, DSCOL.DS_ID ");
sqlCommand.Append(
"FROM COMPONENT_MAPPING CMAP, MAPPING_SET MSET, COM_COL_MAPPING_COLUMN CMAPCOL, DATASET_COLUMN DSCOL ");
sqlCommand.Append("WHERE MSET.MAP_SET_ID = CMAP.MAP_SET_ID ");
sqlCommand.Append("AND CMAP.MAP_ID = CMAPCOL.MAP_ID ");
sqlCommand.Append("AND CMAPCOL.COL_ID = DSCOL.COL_ID ");
sqlCommand.AppendFormat("AND MSET.MAP_SET_ID = {0} ", paramId);
using (DbCommand command = mappingStoreDb.GetSqlStringCommand(sqlCommand.ToString()))
{
mappingStoreDb.AddInParameter(command, ParameterNameConstants.IdParameter, DbType.Int64, sysId);
using (IDataReader dataReader = mappingStoreDb.ExecuteReader(command))
{
int mapID = dataReader.GetOrdinal("MAP_ID");
int type = dataReader.GetOrdinal("TYPE");
int constant = dataReader.GetOrdinal("CONSTANT");
int colIDIdx = dataReader.GetOrdinal("COL_ID");
int nameIdx = dataReader.GetOrdinal("NAME");
////int descIdx = dataReader.GetOrdinal("DESCRIPTION");
while (dataReader.Read())
{
// get existing or create new mapping object
long mapId = DataReaderHelper.GetInt64(dataReader, mapID);
MappingEntity mapping;
if (!dictionary.TryGetValue(mapId, out mapping))
{
// create new mapping and append it to the mapping set
mapping = new MappingEntity(mapId) { MappingType = DataReaderHelper.GetString(dataReader, type), Constant = DataReaderHelper.GetString(dataReader, constant) };
dictionary.Add(mapId, mapping);
mappingSet.Mappings.Add(mapping);
}
var column = new DataSetColumnEntity(DataReaderHelper.GetInt64(dataReader, colIDIdx)) { Name = DataReaderHelper.GetString(dataReader, nameIdx) };
// dsColumn.SysId = DataReaderHelper.GetInt64(dataReader, "DS_ID");
////column.Description = DataReaderHelper.GetString(dataReader, descIdx);
mapping.Columns.Add(column);
}
}
}
if (transcoded)
{
GetTranscodingRulesByMapSetId(mappingStoreDb, sysId, timeDimensionMapping, dictionary);
RetrieveTimeDimensionTranscoding(mappingStoreDb, timeDimensionMapping);
}
}
/// <summary>
/// This method queries the mapping store for a mapping set for a specific dataflow.
/// If such a mapping set exists, it is returned as a <see cref="MappingSetEntity"/>.
/// Else null is returned.
/// </summary>
/// <param name="connectionStringSettings">
/// The connection string parameters used to connect to the Mapping Store Database
/// </param>
/// <param name="maintainableRef">
/// The reference of the DataFlow
/// </param>
/// <param name="allowedDataflows">
/// The collection of dataflow that can be returned. It can be null in which case all dataflows are returned, If is empty or the requested dataflow is not in the allowed collection then null is returned.
/// </param>
/// <exception cref="ArgumentNullException">
/// connectionStringSettings or dataflowId is null
/// </exception>
/// <returns>
/// The populated <see cref="MappingSetEntity"/> or null if the dataflow is not found or it is not in the allowed list
/// </returns>
public static MappingSetEntity GetMappingSet(
ConnectionStringSettings connectionStringSettings,
IMaintainableRefObject maintainableRef,
IList<IMaintainableRefObject> allowedDataflows)
{
if (connectionStringSettings == null)
{
throw new ArgumentNullException("connectionStringSettings");
}
if (maintainableRef == null)
{
throw new ArgumentNullException("maintainableRef");
}
if (!maintainableRef.HasMaintainableId())
{
throw new ArgumentException(ErrorMessages.DataflowIdIsNull, "maintainableRef");
}
if (!SecurityHelper.Contains(allowedDataflows, maintainableRef))
{
return null;
}
MappingSetEntity ret = null;
long datasetId = 0;
var mappingStoreDb = new Database(connectionStringSettings);
using (DbCommand command = BuildSqlCommand(maintainableRef, mappingStoreDb, allowedDataflows))
{
// populate MappingSet
using (IDataReader dataReader = mappingStoreDb.ExecuteReader(command))
{
// we expect only 1 record here
if (dataReader.Read())
{
ret = new MappingSetEntity(DataReaderHelper.GetInt64(dataReader, "MAP_SET_ID"));
ret.Id = DataReaderHelper.GetString(dataReader, "ID");
ret.Description = DataReaderHelper.GetString(dataReader, "DESCRIPTION");
ret.Dataflow = new DataflowEntity(DataReaderHelper.GetInt64(dataReader, "DF_ID"));
ret.Dataflow.Id = DataReaderHelper.GetString(dataReader, "DFID");
ret.Dataflow.Version = DataReaderHelper.GetString(dataReader, "DFVER");
ret.Dataflow.Agency = DataReaderHelper.GetString(dataReader, "DFAG");
ret.Dataflow.MappingSet = ret;
datasetId = DataReaderHelper.GetInt64(dataReader, "DS_ID");
}
}
}
if (ret == null)
{
// throw new Exception(String.Format(CultureInfo.InvariantCulture,"There was no mapping for dataflow:{0}", dataflowId));
// MAt-395
return null;
}
// populate the dataset field
ret.DataSet = GetDatasetById(mappingStoreDb, datasetId);
// populate the dataflow field
// ret.Dataflow = GetDataFlowByMappingSetSysId(connectionStringSettings, ret.SysId);
ret.Dataflow.Dsd = GetDsdByDataFlowSysId(mappingStoreDb, ret.Dataflow.SysId);
// populate the mapping field
GetMappingsByMappingSetSysId(mappingStoreDb, ret);
return ret;
}
