public IDataStructureMutableObject buildDataStructure(string id, string agency, string version)
{
IDataStructureMutableObject dsd = new DataStructureMutableCore();
dsd.Id = id;
dsd.AgencyId = agency;
dsd.Version = version;
return dsd;
}
public void TestDimensionAtObservation()
{
IDataStructureMutableObject dsd = new DataStructureMutableCore() { Id = "TEST", AgencyId = "TEST_AGENCY", Version = "1.0" };
dsd.AddName("en", "Test name");
IDimensionMutableObject dimension = new DimensionMutableCore();
dimension.ConceptRef = new StructureReferenceImpl("TEST_AGENCY", "TEST_CONCEPTS", "1.0", SdmxStructureEnumType.Concept, "TEST_DIM");
dimension.Representation = new RepresentationMutableCore { Representation = new StructureReferenceImpl("TEST_AGENCY", "CL_TEST", "2.0", SdmxStructureEnumType.CodeList) };
dsd.AddDimension(dimension);
dsd.AddPrimaryMeasure(new StructureReferenceImpl("TEST_AGENCY", "TEST_CONCEPTS", "1.0", SdmxStructureEnumType.Concept, "OBS_VALUE"));
var immutableDsd = dsd.ImmutableInstance;
var dataflowMutable = new DataflowMutableCore { Id = "TEST_DF", AgencyId = "TEST_AGENCY", Version = "1.2" };
dataflowMutable.AddName("en", "Test");
dataflowMutable.DataStructureRef = immutableDsd.AsReference;
var dataflow = dataflowMutable.ImmutableInstance;
IDataQuery query = new DataQueryImpl(immutableDsd, null, DataQueryDetail.GetFromEnum(DataQueryDetailEnumType.Full),null,null, null, dataflow, "AllDimensions", new HashSet<IDataQuerySelection>(), null, null);
Assert.AreEqual("AllDimensions", query.DimensionAtObservation);
}
public void TestDataWriterEngineDimensionAtObs(DataEnumType format, string dimAtObs)
{
string outfile = string.Format("{0}-{1}.xml", format, dimAtObs);
DataType fromEnum = DataType.GetFromEnum(format);
using (Stream writer = File.Create(outfile))
{
IConceptSchemeMutableObject conceptScheme = new ConceptSchemeMutableCore { Id = "CONCEPTS_TEST", AgencyId = "TEST" };
conceptScheme.AddName("en", "Dummy concept scheme build for this tests");
////conceptScheme.Annotations.Add(new AnnotationMutableCore() { Id = "ANNOTABLETEST", Title = "Test", Type = "ATYPE" });
conceptScheme.FinalStructure = TertiaryBool.GetFromEnum(TertiaryBoolEnumType.True);
IConceptSchemeObject parent = conceptScheme.ImmutableInstance;
IConceptObject freqConcept = CreateConcept(parent, "FREQ", "Frequency");
IConceptObject adjustmentConcept = CreateConcept(parent, "ADJUSTMENT", "The Adjustment");
IConceptObject activityConcpet = CreateConcept(parent, "STS_ACTIVITY", "Name of activity ");
IConceptObject timeDimensionConcpet = CreateConcept(parent, "TIME_PERIOD", "Name of Time Period");
IConceptObject decimalsConcept = CreateConcept(parent, "DECIMALS", "Name of concept");
IConceptObject obsConcept = CreateConcept(parent, "OBS_VALUE", "Name of observation value");
ICodelistObject freqCl = CreateCodelist("CL_FREQ", "Freq codelist", "Q", "A", "M");
ICodelistObject adjCl = CreateCodelist("CL_ADJUSTMENT", "Adjustment codelist", "N", "S", "W");
ICodelistObject actCl = CreateCodelist("CL_ACTIVITY", "Activity codelist", "A", "B", "C");
ICodelistObject deciCl = CreateCodelist("CL_DECIMALS", "DECIMALS codelist", "1", "2", "0");
IDataStructureMutableObject mutable = new DataStructureMutableCore { Id = "TEST_DSD", AgencyId = "TEST" };
mutable.AddName("en", "FOO BAR");
mutable.AddDimension(
new DimensionMutableCore { ConceptRef = freqConcept.AsReference, FrequencyDimension = true, Representation = new RepresentationMutableCore { Representation = freqCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = adjustmentConcept.AsReference, Representation = new RepresentationMutableCore { Representation = adjCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = activityConcpet.AsReference, Representation = new RepresentationMutableCore { Representation = actCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = timeDimensionConcpet.AsReference, TimeDimension = true });
IList<string> dimList = new List<string> { freqConcept.Id, adjustmentConcept.Id, adjustmentConcept.Id };
var attributeMutableCore = new AttributeMutableCore
{
ConceptRef = decimalsConcept.AsReference,
Representation = new RepresentationMutableCore { Representation = deciCl.AsReference },
AttachmentLevel = AttributeAttachmentLevel.DimensionGroup,
AssignmentStatus = AttributeAssignmentStatus.Mandatory.ToString()
};
attributeMutableCore.DimensionReferences.AddAll(dimList);
mutable.AddAttribute(attributeMutableCore);
mutable.PrimaryMeasure = new PrimaryMeasureMutableCore { ConceptRef = obsConcept.AsReference };
var manager = new DataWriterManager();
IDataWriterEngine dataWriter = manager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer);
IDataStructureObject dataStructureObject = mutable.ImmutableInstance;
IList<IDatasetStructureReference> structures = new List<IDatasetStructureReference> { new DatasetStructureReferenceCore(null, dataStructureObject.AsReference, null, null, dimAtObs) };
IList<IParty> receiver = new List<IParty> { new PartyCore(new List<ITextTypeWrapper>(), "ZZ9", new List<IContact>(), null) };
var sender = new PartyCore(new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "TEST SENDER", null) }, "ZZ1", null, null);
IHeader header = new HeaderImpl(
null,
structures,
null,
DatasetAction.GetFromEnum(DatasetActionEnumType.Information),
"TEST_DATAFLOW",
"DATASET_ID",
null,
DateTime.Now,
DateTime.Now,
null,
null,
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "test header name", null) },
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "source 1", null) },
receiver,
sender,
true);
dataWriter.WriteHeader(header);
dataWriter.StartDataset(null, dataStructureObject, new DatasetHeaderCore(null, DatasetAction.GetFromEnum(DatasetActionEnumType.Information), structures.First()));
var sw = new Stopwatch();
sw.Start();
var startTime = new DateTime(2005, 1, 1);
var series = (from f in freqCl.Items
from ad in adjCl.Items
from ac in actCl.Items
from t in Enumerable.Range(0, 100)
select new { Freq = f.Id, Adj = ad.Id, Activity = ac.Id, Time = BuildPeriodResolver(f.Id, startTime)(t) }).OrderBy(
arg => BuildOrderBy(arg.Freq, arg.Adj, arg.Activity, arg.Time, dimAtObs)).ToArray();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
sw.Reset();
sw.Start();
string lastKey = null;
int i = 0;
foreach (var key in series)
{
var currentKey = new List<string>();
string crossValue = null;
if (!string.Equals(freqConcept.Id, dimAtObs))
{
currentKey.Add(key.Freq);
}
else
{
crossValue = key.Freq;
}
if (!string.Equals(adjustmentConcept.Id, dimAtObs))
{
currentKey.Add(key.Adj);
}
else
{
crossValue = key.Adj;
}
if (!string.Equals(activityConcpet.Id, dimAtObs))
{
currentKey.Add(key.Activity);
}
else
{
crossValue = key.Activity;
}
if (!string.Equals(timeDimensionConcpet.Id, dimAtObs))
{
currentKey.Add(key.Time);
}
else
{
crossValue = key.Time;
}
var currentKeyValues = string.Join(",", currentKey);
if (lastKey == null || !lastKey.Equals(currentKeyValues))
{
lastKey = currentKeyValues;
i = 0;
dataWriter.StartSeries();
if (!string.Equals(freqConcept.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(freqConcept.Id, key.Freq);
}
if (!string.Equals(adjustmentConcept.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(adjustmentConcept.Id, key.Adj);
}
if (!string.Equals(activityConcpet.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(activityConcpet.Id, key.Activity);
}
if (!string.Equals(timeDimensionConcpet.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(timeDimensionConcpet.Id, key.Time);
}
}
dataWriter.WriteObservation(crossValue, i.ToString(CultureInfo.InvariantCulture));
i++;
}
dataWriter.Close();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
}
public void TestDataWriterEngineAllDimensions(DataEnumType format)
{
string outfile = string.Format("{0}-alldim.xml", format);
DataType fromEnum = DataType.GetFromEnum(format);
using (Stream writer = File.Create(outfile))
{
IConceptSchemeMutableObject conceptScheme = new ConceptSchemeMutableCore { Id = "CONCEPTS_TEST", AgencyId = "TEST" };
conceptScheme.AddName("en", "Dummy concept scheme build for this tests");
////conceptScheme.Annotations.Add(new AnnotationMutableCore() { Id = "ANNOTABLETEST", Title = "Test", Type = "ATYPE" });
conceptScheme.FinalStructure = TertiaryBool.GetFromEnum(TertiaryBoolEnumType.True);
IConceptSchemeObject parent = conceptScheme.ImmutableInstance;
IConceptObject freqConcept = CreateConcept(parent, "FREQ", "Frequency");
IConceptObject adjustmentConcept = CreateConcept(parent, "ADJUSTMENT", "The Adjustment");
IConceptObject activityConcpet = CreateConcept(parent, "STS_ACTIVITY", "Name of activity ");
IConceptObject timeDimensionConcpet = CreateConcept(parent, "TIME_PERIOD", "Name of Time Period");
IConceptObject decimalsConcept = CreateConcept(parent, "DECIMALS", "Name of concept");
IConceptObject obsConcept = CreateConcept(parent, "OBS_VALUE", "Name of observation value");
ICodelistObject freqCl = CreateCodelist("CL_FREQ", "Freq codelist", "Q", "A", "M");
ICodelistObject adjCl = CreateCodelist("CL_ADJUSTMENT", "Adjustment codelist", "N", "S", "W");
ICodelistObject actCl = CreateCodelist("CL_ACTIVITY", "Activity codelist", "A", "B", "C");
ICodelistObject deciCl = CreateCodelist("CL_DECIMALS", "DECIMALS codelist", "1", "2", "0");
IDataStructureMutableObject mutable = new DataStructureMutableCore { Id = "TEST_DSD", AgencyId = "TEST" };
mutable.AddName("en", "FOO BAR");
mutable.AddDimension(
new DimensionMutableCore { ConceptRef = freqConcept.AsReference, FrequencyDimension = true, Representation = new RepresentationMutableCore { Representation = freqCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = adjustmentConcept.AsReference, Representation = new RepresentationMutableCore { Representation = adjCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = activityConcpet.AsReference, Representation = new RepresentationMutableCore { Representation = actCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = timeDimensionConcpet.AsReference, TimeDimension = true });
IList<string> dimList = new List<string> { freqConcept.Id, adjustmentConcept.Id, adjustmentConcept.Id };
var attributeMutableCore = new AttributeMutableCore
{
ConceptRef = decimalsConcept.AsReference,
Representation = new RepresentationMutableCore { Representation = deciCl.AsReference },
AttachmentLevel = AttributeAttachmentLevel.DimensionGroup,
AssignmentStatus = AttributeAssignmentStatus.Mandatory.ToString()
};
attributeMutableCore.DimensionReferences.AddAll(dimList);
mutable.AddAttribute(attributeMutableCore);
mutable.PrimaryMeasure = new PrimaryMeasureMutableCore { ConceptRef = obsConcept.AsReference };
var manager = new DataWriterManager();
IDataWriterEngine dataWriter = manager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer);
IDataStructureObject dataStructureObject = mutable.ImmutableInstance;
IList<IDatasetStructureReference> structures = new List<IDatasetStructureReference>
{
new DatasetStructureReferenceCore(
null,
dataStructureObject.AsReference,
null,
null,
DatasetStructureReference.AllDimensions)
};
IList<IParty> receiver = new List<IParty> { new PartyCore(new List<ITextTypeWrapper>(), "ZZ9", new List<IContact>(), null) };
var sender = new PartyCore(new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "TEST SENDER", null) }, "ZZ1", null, null);
IHeader header = new HeaderImpl(
null,
structures,
null,
DatasetAction.GetFromEnum(DatasetActionEnumType.Information),
"TEST_DATAFLOW",
"DATASET_ID",
null,
DateTime.Now,
DateTime.Now,
null,
null,
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "test header name", null) },
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "source 1", null) },
receiver,
sender,
true);
dataWriter.WriteHeader(header);
dataWriter.StartDataset(null, dataStructureObject, new DatasetHeaderCore(null, DatasetAction.GetFromEnum(DatasetActionEnumType.Information), structures.First()));
var sw = new Stopwatch();
sw.Start();
var series = from f in freqCl.Items from ad in adjCl.Items from ac in actCl.Items select new { f, ad, ac };
sw.Stop();
Trace.WriteLine(sw.Elapsed);
sw.Reset();
var startTime = new DateTime(2005, 1, 1);
sw.Start();
foreach (var key in series)
{
dataWriter.StartSeries();
dataWriter.WriteSeriesKeyValue(freqConcept.Id, key.f.Id);
dataWriter.WriteSeriesKeyValue(adjustmentConcept.Id, key.ad.Id);
dataWriter.WriteSeriesKeyValue(activityConcpet.Id, key.ac.Id);
Func<int, string> getPeriod = null;
switch (key.f.Id)
{
case "Q":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(3 * i);
return DateUtil.FormatDate(months, TimeFormatEnumType.QuarterOfYear);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "1");
break;
case "A":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(12 * i);
return DateUtil.FormatDate(months, TimeFormatEnumType.Year);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "0");
break;
case "M":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(i + 1);
return DateUtil.FormatDate(months, TimeFormatEnumType.Month);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "2");
break;
default:
Assert.Fail("Test bug. Check CL_FREQ codes");
break;
}
for (int i = 0; i < 100; i++)
{
string period = getPeriod(i);
dataWriter.WriteObservation(DimensionObject.TimeDimensionFixedId, period, i.ToString(CultureInfo.InvariantCulture));
}
}
dataWriter.Close();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
}
/// <summary>
/// Builds the data structure object.
/// </summary>
/// <param name="dimension">The dimension.</param>
/// <returns>
/// The <see cref="IDataStructureObject"/>
/// </returns>
private static IDataStructureObject BuildDataStructureObject(IDimensionMutableObject dimension)
{
IDataStructureMutableObject dsd = new DataStructureMutableCore() { Id = "TEST_DSD", AgencyId = "TEST", Version = "1.0" };
dsd.AddName("en", "TEST_DSD");
dsd.AddPrimaryMeasure(new StructureReferenceImpl("TEST_AGENCY", "TEST_CONCEPTS", "1.0", SdmxStructureEnumType.Concept, "OBS_VALUE"));
dsd.AddDimension(dimension);
var immutable = dsd.ImmutableInstance;
return immutable;
}
/// <summary>
/// Builds the DSD.
/// </summary>
/// <returns>
/// The <see cref="IDataStructureObject" />.
/// </returns>
private static IDataStructureObject BuildDsd()
{
IDataStructureMutableObject dsdMutableObject = new DataStructureMutableCore { AgencyId = "TEST", Id = "TEST_DSD", Version = "1.0" };
dsdMutableObject.AddName("en", "Test data");
// FREQ="Q" ADJUSTMENT="N" STS_ACTIVITY="A"
dsdMutableObject.AddDimension(
new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "FREQ"),
new StructureReferenceImpl("SDMX", "CL_FREQ", "1.0", SdmxStructureEnumType.CodeList));
dsdMutableObject.AddDimension(
new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "ADJUSTMENT"),
new StructureReferenceImpl("SDMX", "CL_ADJUSTMENT", "1.0", SdmxStructureEnumType.CodeList));
dsdMutableObject.AddDimension(
new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "STS_ACTIVITY"),
new StructureReferenceImpl("STS", "CL_STS_ACTIVITY", "1.0", SdmxStructureEnumType.CodeList));
dsdMutableObject.AddDimension(
new DimensionMutableCore { ConceptRef = new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "TIME_PERIOD"), TimeDimension = true });
dsdMutableObject.AddPrimaryMeasure(new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "OBS_VALUE"));
var attributeMutableObject = dsdMutableObject.AddAttribute(
new StructureReferenceImpl("TEST", "TEST_CS", "1.0", SdmxStructureEnumType.Concept, "DECIMALS"),
new StructureReferenceImpl("STS", "CL_DECIMALS", "1.0", SdmxStructureEnumType.CodeList));
attributeMutableObject.AttachmentLevel = AttributeAttachmentLevel.DimensionGroup;
attributeMutableObject.DimensionReferences.AddAll(new[] { "FREQ", "ADJUSTMENT", "STS_ACTIVITY" });
attributeMutableObject.AssignmentStatus = "Mandatory";
return dsdMutableObject.ImmutableInstance;
}
/// <summary>
/// Builds the DSD.
/// </summary>
/// <returns>The DSD.</returns>
private static IDataStructureObject BuildDsd(IMaintainableRefObject dsdRef, int dimensions)
{
IDataStructureMutableObject dsdMutableObject = new DataStructureMutableCore
{
AgencyId = dsdRef.AgencyId,
Id = dsdRef.MaintainableId,
Version = dsdRef.HasVersion() ? dsdRef.Version : "1.0"
};
dsdMutableObject.AddName("en", "Test data");
// FREQ="Q" ADJUSTMENT="N" STS_ACTIVITY="A"
dsdMutableObject.AddDimension(
new StructureReferenceImpl(agencyId: dsdRef.AgencyId, maintainableId: "TEST_CS", version: "1.0", targetStructureEnum: SdmxStructureEnumType.Concept, identfiableIds: "FREQ"),
new StructureReferenceImpl("SDMX", "CL_FREQ", "1.0", SdmxStructureEnumType.CodeList));
var groupMutableCore = new GroupMutableCore() { Id = "Sibling" };
dsdMutableObject.AddGroup(groupMutableCore);
for (int i = 0; i < dimensions; i++)
{
string id = "DIM" + i.ToString(CultureInfo.InvariantCulture);
var dim = dsdMutableObject.AddDimension(
new StructureReferenceImpl(
agencyId: dsdRef.AgencyId,
maintainableId: "TEST_CS",
version: "1.0",
targetStructureEnum: SdmxStructureEnumType.Concept,
identfiableIds: id), codelistRef: new StructureReferenceImpl(dsdRef.AgencyId, "CL_" + id, "1.0", SdmxStructureEnumType.CodeList));
groupMutableCore.DimensionRef.Add(id);
}
dsdMutableObject.AddDimension(
new DimensionMutableCore
{
ConceptRef =
new StructureReferenceImpl(
agencyId: dsdRef.AgencyId,
maintainableId: "TEST_CS",
version: "1.0",
targetStructureEnum: SdmxStructureEnumType.Concept,
identfiableIds: "TIME_PERIOD"),
TimeDimension = true
});
dsdMutableObject.AddPrimaryMeasure(
new StructureReferenceImpl(agencyId: dsdRef.AgencyId, maintainableId: "TEST_CS", version: "1.0", targetStructureEnum: SdmxStructureEnumType.Concept, identfiableIds: "OBS_VALUE"));
var obsStatus =
dsdMutableObject.AddAttribute(
new StructureReferenceImpl(agencyId: dsdRef.AgencyId, maintainableId: "TEST_CS", version: "1.0", targetStructureEnum: SdmxStructureEnumType.Concept, identfiableIds: "OBS_STATUS"),
new StructureReferenceImpl(dsdRef.AgencyId, "CL_OBS_STATUS", "1.0", SdmxStructureEnumType.CodeList));
obsStatus.AttachmentLevel = AttributeAttachmentLevel.Observation;
obsStatus.AssignmentStatus = "Mandatory";
var obsConf =
dsdMutableObject.AddAttribute(
new StructureReferenceImpl(agencyId: dsdRef.AgencyId, maintainableId: "TEST_CS", version: "1.0", targetStructureEnum: SdmxStructureEnumType.Concept, identfiableIds: "OBS_CONF"),
new StructureReferenceImpl(dsdRef.AgencyId, "CL_OBS_CONF", "1.0", SdmxStructureEnumType.CodeList));
obsConf.AttachmentLevel = AttributeAttachmentLevel.Observation;
obsConf.AssignmentStatus = "Conditional";
AddCodedAttribute(dsdMutableObject, "UNIT_MULT");
AddCodedAttribute(dsdMutableObject, "UNIT");
AddUnCodedAttribute(dsdMutableObject, "TITLE_COMPL");
return dsdMutableObject.ImmutableInstance;
}
/// <summary>
/// Builds the data structure.
/// </summary>
/// <returns>
/// The <see cref="IDataStructureObject" />.
/// </returns>
public IDataStructureObject BuildDataStructure()
{
IDataStructureMutableObject dsd = new DataStructureMutableCore();
dsd.AgencyId = "SDMXSOURCE";
dsd.Id = "WDI";
dsd.AddName("en", "World Development Indicators");
dsd.AddDimension(CreateConceptReference("COUNTRY"), CreateCodelistReference("CL_COUNTRY"));
dsd.AddDimension(CreateConceptReference("INDICATOR"), CreateCodelistReference("CL_INDICATOR"));
IDimensionMutableObject timeDim = dsd.AddDimension(CreateConceptReference("TIME"), null);
timeDim.TimeDimension = true;
dsd.AddPrimaryMeasure(CreateConceptReference("OBS_VALUE"));
return dsd.ImmutableInstance;
}
