public void TestDataWriterEngineDimensionAtObs(DataEnumType format, string dimAtObs)
{
string outfile = string.Format("{0}-{1}.xml", format, dimAtObs);
DataType fromEnum = DataType.GetFromEnum(format);
using (Stream writer = File.Create(outfile))
{
IConceptSchemeMutableObject conceptScheme = new ConceptSchemeMutableCore { Id = "CONCEPTS_TEST", AgencyId = "TEST" };
conceptScheme.AddName("en", "Dummy concept scheme build for this tests");
////conceptScheme.Annotations.Add(new AnnotationMutableCore() { Id = "ANNOTABLETEST", Title = "Test", Type = "ATYPE" });
conceptScheme.FinalStructure = TertiaryBool.GetFromEnum(TertiaryBoolEnumType.True);
IConceptSchemeObject parent = conceptScheme.ImmutableInstance;
IConceptObject freqConcept = CreateConcept(parent, "FREQ", "Frequency");
IConceptObject adjustmentConcept = CreateConcept(parent, "ADJUSTMENT", "The Adjustment");
IConceptObject activityConcpet = CreateConcept(parent, "STS_ACTIVITY", "Name of activity ");
IConceptObject timeDimensionConcpet = CreateConcept(parent, "TIME_PERIOD", "Name of Time Period");
IConceptObject decimalsConcept = CreateConcept(parent, "DECIMALS", "Name of concept");
IConceptObject obsConcept = CreateConcept(parent, "OBS_VALUE", "Name of observation value");
ICodelistObject freqCl = CreateCodelist("CL_FREQ", "Freq codelist", "Q", "A", "M");
ICodelistObject adjCl = CreateCodelist("CL_ADJUSTMENT", "Adjustment codelist", "N", "S", "W");
ICodelistObject actCl = CreateCodelist("CL_ACTIVITY", "Activity codelist", "A", "B", "C");
ICodelistObject deciCl = CreateCodelist("CL_DECIMALS", "DECIMALS codelist", "1", "2", "0");
IDataStructureMutableObject mutable = new DataStructureMutableCore { Id = "TEST_DSD", AgencyId = "TEST" };
mutable.AddName("en", "FOO BAR");
mutable.AddDimension(
new DimensionMutableCore { ConceptRef = freqConcept.AsReference, FrequencyDimension = true, Representation = new RepresentationMutableCore { Representation = freqCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = adjustmentConcept.AsReference, Representation = new RepresentationMutableCore { Representation = adjCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = activityConcpet.AsReference, Representation = new RepresentationMutableCore { Representation = actCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = timeDimensionConcpet.AsReference, TimeDimension = true });
IList<string> dimList = new List<string> { freqConcept.Id, adjustmentConcept.Id, adjustmentConcept.Id };
var attributeMutableCore = new AttributeMutableCore
{
ConceptRef = decimalsConcept.AsReference,
Representation = new RepresentationMutableCore { Representation = deciCl.AsReference },
AttachmentLevel = AttributeAttachmentLevel.DimensionGroup,
AssignmentStatus = AttributeAssignmentStatus.Mandatory.ToString()
};
attributeMutableCore.DimensionReferences.AddAll(dimList);
mutable.AddAttribute(attributeMutableCore);
mutable.PrimaryMeasure = new PrimaryMeasureMutableCore { ConceptRef = obsConcept.AsReference };
var manager = new DataWriterManager();
IDataWriterEngine dataWriter = manager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer);
IDataStructureObject dataStructureObject = mutable.ImmutableInstance;
IList<IDatasetStructureReference> structures = new List<IDatasetStructureReference> { new DatasetStructureReferenceCore(null, dataStructureObject.AsReference, null, null, dimAtObs) };
IList<IParty> receiver = new List<IParty> { new PartyCore(new List<ITextTypeWrapper>(), "ZZ9", new List<IContact>(), null) };
var sender = new PartyCore(new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "TEST SENDER", null) }, "ZZ1", null, null);
IHeader header = new HeaderImpl(
null,
structures,
null,
DatasetAction.GetFromEnum(DatasetActionEnumType.Information),
"TEST_DATAFLOW",
"DATASET_ID",
null,
DateTime.Now,
DateTime.Now,
null,
null,
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "test header name", null) },
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "source 1", null) },
receiver,
sender,
true);
dataWriter.WriteHeader(header);
dataWriter.StartDataset(null, dataStructureObject, new DatasetHeaderCore(null, DatasetAction.GetFromEnum(DatasetActionEnumType.Information), structures.First()));
var sw = new Stopwatch();
sw.Start();
var startTime = new DateTime(2005, 1, 1);
var series = (from f in freqCl.Items
from ad in adjCl.Items
from ac in actCl.Items
from t in Enumerable.Range(0, 100)
select new { Freq = f.Id, Adj = ad.Id, Activity = ac.Id, Time = BuildPeriodResolver(f.Id, startTime)(t) }).OrderBy(
arg => BuildOrderBy(arg.Freq, arg.Adj, arg.Activity, arg.Time, dimAtObs)).ToArray();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
sw.Reset();
sw.Start();
string lastKey = null;
int i = 0;
foreach (var key in series)
{
var currentKey = new List<string>();
string crossValue = null;
if (!string.Equals(freqConcept.Id, dimAtObs))
{
currentKey.Add(key.Freq);
}
else
{
crossValue = key.Freq;
}
if (!string.Equals(adjustmentConcept.Id, dimAtObs))
{
currentKey.Add(key.Adj);
}
else
{
crossValue = key.Adj;
}
if (!string.Equals(activityConcpet.Id, dimAtObs))
{
currentKey.Add(key.Activity);
}
else
{
crossValue = key.Activity;
}
if (!string.Equals(timeDimensionConcpet.Id, dimAtObs))
{
currentKey.Add(key.Time);
}
else
{
crossValue = key.Time;
}
var currentKeyValues = string.Join(",", currentKey);
if (lastKey == null || !lastKey.Equals(currentKeyValues))
{
lastKey = currentKeyValues;
i = 0;
dataWriter.StartSeries();
if (!string.Equals(freqConcept.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(freqConcept.Id, key.Freq);
}
if (!string.Equals(adjustmentConcept.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(adjustmentConcept.Id, key.Adj);
}
if (!string.Equals(activityConcpet.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(activityConcpet.Id, key.Activity);
}
if (!string.Equals(timeDimensionConcpet.Id, dimAtObs))
{
dataWriter.WriteSeriesKeyValue(timeDimensionConcpet.Id, key.Time);
}
}
dataWriter.WriteObservation(crossValue, i.ToString(CultureInfo.InvariantCulture));
i++;
}
dataWriter.Close();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
}
public void TestDataWriterEngineMultiDataset(DataEnumType format)
{
string outfile = string.Format("{0}-multi.xml", format);
DataType fromEnum = DataType.GetFromEnum(format);
var objects = GetSdmxObjects();
IDataStructureObject dataStructureObject = objects.DataStructures.First();
var dataWriterManager = new DataWriterManager();
var startTime = new DateTime(2012, 1, 1);
using (Stream writer = File.Create(outfile))
{
using (IDataWriterEngine dataWriter = dataWriterManager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer))
{
var header = new HeaderImpl("IDTEST", "ZZ9");
dataWriter.WriteHeader(header);
IDatasetHeader dataSetHeader = new DatasetHeaderCore(
"UPDATE",
DatasetAction.GetFromEnum(DatasetActionEnumType.Replace),
new DatasetStructureReferenceCore(dataStructureObject.AsReference));
dataWriter.StartDataset(null, dataStructureObject, dataSetHeader);
dataWriter.StartSeries();
dataWriter.WriteSeriesKeyValue("FREQ", "M");
dataWriter.WriteSeriesKeyValue("REF_AREA", "DE");
dataWriter.WriteSeriesKeyValue("ADJUSTMENT", "N");
dataWriter.WriteSeriesKeyValue("STS_INDICATOR", "PROD");
dataWriter.WriteSeriesKeyValue("STS_ACTIVITY", "NS0030");
dataWriter.WriteSeriesKeyValue("STS_INSTITUTION", "1");
dataWriter.WriteSeriesKeyValue("STS_BASE_YEAR", "2000");
var getPeriod1 = GetPeriodFunc("M", startTime);
dataWriter.WriteAttributeValue("TIME_FORMAT", GetTimeFormat("M"));
for (int i = 0; i < 5; i++)
{
string period = getPeriod1(i);
dataWriter.WriteObservation(period, i.ToString(CultureInfo.InvariantCulture));
dataWriter.WriteAttributeValue("OBS_STATUS", "A");
}
IDatasetHeader dataSetHeaderDelete = new DatasetHeaderCore(
"DELETE",
DatasetAction.GetFromEnum(DatasetActionEnumType.Delete),
new DatasetStructureReferenceCore(dataStructureObject.AsReference));
dataWriter.StartDataset(null, dataStructureObject, dataSetHeaderDelete);
dataWriter.StartSeries();
dataWriter.WriteSeriesKeyValue("FREQ", "M");
dataWriter.WriteSeriesKeyValue("REF_AREA", "DE");
dataWriter.WriteSeriesKeyValue("ADJUSTMENT", "N");
dataWriter.WriteSeriesKeyValue("STS_INDICATOR", "PROD");
dataWriter.WriteSeriesKeyValue("STS_ACTIVITY", "NS0030");
dataWriter.WriteSeriesKeyValue("STS_INSTITUTION", "1");
dataWriter.WriteSeriesKeyValue("STS_BASE_YEAR", "2000");
var getPeriod = GetPeriodFunc("M", new DateTime(2006, 1, 1));
for (int i = 0; i < 5; i++)
{
string period = getPeriod(i);
dataWriter.WriteObservation(period, "NaN");
}
dataWriter.Close();
}
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
}
public void TestDataWriterEngineAllDimensions(DataEnumType format)
{
string outfile = string.Format("{0}-alldim.xml", format);
DataType fromEnum = DataType.GetFromEnum(format);
using (Stream writer = File.Create(outfile))
{
IConceptSchemeMutableObject conceptScheme = new ConceptSchemeMutableCore { Id = "CONCEPTS_TEST", AgencyId = "TEST" };
conceptScheme.AddName("en", "Dummy concept scheme build for this tests");
////conceptScheme.Annotations.Add(new AnnotationMutableCore() { Id = "ANNOTABLETEST", Title = "Test", Type = "ATYPE" });
conceptScheme.FinalStructure = TertiaryBool.GetFromEnum(TertiaryBoolEnumType.True);
IConceptSchemeObject parent = conceptScheme.ImmutableInstance;
IConceptObject freqConcept = CreateConcept(parent, "FREQ", "Frequency");
IConceptObject adjustmentConcept = CreateConcept(parent, "ADJUSTMENT", "The Adjustment");
IConceptObject activityConcpet = CreateConcept(parent, "STS_ACTIVITY", "Name of activity ");
IConceptObject timeDimensionConcpet = CreateConcept(parent, "TIME_PERIOD", "Name of Time Period");
IConceptObject decimalsConcept = CreateConcept(parent, "DECIMALS", "Name of concept");
IConceptObject obsConcept = CreateConcept(parent, "OBS_VALUE", "Name of observation value");
ICodelistObject freqCl = CreateCodelist("CL_FREQ", "Freq codelist", "Q", "A", "M");
ICodelistObject adjCl = CreateCodelist("CL_ADJUSTMENT", "Adjustment codelist", "N", "S", "W");
ICodelistObject actCl = CreateCodelist("CL_ACTIVITY", "Activity codelist", "A", "B", "C");
ICodelistObject deciCl = CreateCodelist("CL_DECIMALS", "DECIMALS codelist", "1", "2", "0");
IDataStructureMutableObject mutable = new DataStructureMutableCore { Id = "TEST_DSD", AgencyId = "TEST" };
mutable.AddName("en", "FOO BAR");
mutable.AddDimension(
new DimensionMutableCore { ConceptRef = freqConcept.AsReference, FrequencyDimension = true, Representation = new RepresentationMutableCore { Representation = freqCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = adjustmentConcept.AsReference, Representation = new RepresentationMutableCore { Representation = adjCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = activityConcpet.AsReference, Representation = new RepresentationMutableCore { Representation = actCl.AsReference } });
mutable.AddDimension(new DimensionMutableCore { ConceptRef = timeDimensionConcpet.AsReference, TimeDimension = true });
IList<string> dimList = new List<string> { freqConcept.Id, adjustmentConcept.Id, adjustmentConcept.Id };
var attributeMutableCore = new AttributeMutableCore
{
ConceptRef = decimalsConcept.AsReference,
Representation = new RepresentationMutableCore { Representation = deciCl.AsReference },
AttachmentLevel = AttributeAttachmentLevel.DimensionGroup,
AssignmentStatus = AttributeAssignmentStatus.Mandatory.ToString()
};
attributeMutableCore.DimensionReferences.AddAll(dimList);
mutable.AddAttribute(attributeMutableCore);
mutable.PrimaryMeasure = new PrimaryMeasureMutableCore { ConceptRef = obsConcept.AsReference };
var manager = new DataWriterManager();
IDataWriterEngine dataWriter = manager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer);
IDataStructureObject dataStructureObject = mutable.ImmutableInstance;
IList<IDatasetStructureReference> structures = new List<IDatasetStructureReference>
{
new DatasetStructureReferenceCore(
null,
dataStructureObject.AsReference,
null,
null,
DatasetStructureReference.AllDimensions)
};
IList<IParty> receiver = new List<IParty> { new PartyCore(new List<ITextTypeWrapper>(), "ZZ9", new List<IContact>(), null) };
var sender = new PartyCore(new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "TEST SENDER", null) }, "ZZ1", null, null);
IHeader header = new HeaderImpl(
null,
structures,
null,
DatasetAction.GetFromEnum(DatasetActionEnumType.Information),
"TEST_DATAFLOW",
"DATASET_ID",
null,
DateTime.Now,
DateTime.Now,
null,
null,
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "test header name", null) },
new List<ITextTypeWrapper> { new TextTypeWrapperImpl("en", "source 1", null) },
receiver,
sender,
true);
dataWriter.WriteHeader(header);
dataWriter.StartDataset(null, dataStructureObject, new DatasetHeaderCore(null, DatasetAction.GetFromEnum(DatasetActionEnumType.Information), structures.First()));
var sw = new Stopwatch();
sw.Start();
var series = from f in freqCl.Items from ad in adjCl.Items from ac in actCl.Items select new { f, ad, ac };
sw.Stop();
Trace.WriteLine(sw.Elapsed);
sw.Reset();
var startTime = new DateTime(2005, 1, 1);
sw.Start();
foreach (var key in series)
{
dataWriter.StartSeries();
dataWriter.WriteSeriesKeyValue(freqConcept.Id, key.f.Id);
dataWriter.WriteSeriesKeyValue(adjustmentConcept.Id, key.ad.Id);
dataWriter.WriteSeriesKeyValue(activityConcpet.Id, key.ac.Id);
Func<int, string> getPeriod = null;
switch (key.f.Id)
{
case "Q":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(3 * i);
return DateUtil.FormatDate(months, TimeFormatEnumType.QuarterOfYear);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "1");
break;
case "A":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(12 * i);
return DateUtil.FormatDate(months, TimeFormatEnumType.Year);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "0");
break;
case "M":
getPeriod = i =>
{
DateTime months = startTime.AddMonths(i + 1);
return DateUtil.FormatDate(months, TimeFormatEnumType.Month);
};
dataWriter.WriteAttributeValue(decimalsConcept.Id, "2");
break;
default:
Assert.Fail("Test bug. Check CL_FREQ codes");
break;
}
for (int i = 0; i < 100; i++)
{
string period = getPeriod(i);
dataWriter.WriteObservation(DimensionObject.TimeDimensionFixedId, period, i.ToString(CultureInfo.InvariantCulture));
}
}
dataWriter.Close();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
}
public void TestDataWriterEngine(DataEnumType format)
{
string outfile = string.Format("{0}.xml", format);
DataType fromEnum = DataType.GetFromEnum(format);
var objects = GetSdmxObjects();
IDataStructureObject dataStructureObject = objects.DataStructures.First();
dynamic series = BuildSeries(objects);
var dataWriterManager = new DataWriterManager();
var startTime = new DateTime(2005, 1, 1);
using (Stream writer = File.Create(outfile))
{
Stopwatch sw;
using (IDataWriterEngine dataWriter = dataWriterManager.GetDataWriterEngine(new SdmxDataFormatCore(fromEnum), writer))
{
var header = GetHeader(dataStructureObject);
dataWriter.WriteHeader(header);
dataWriter.StartDataset(null, dataStructureObject, null);
sw = new Stopwatch();
sw.Start();
sw.Stop();
Trace.WriteLine(sw.Elapsed);
sw.Reset();
sw.Start();
foreach (var key in series)
{
dataWriter.StartSeries();
dataWriter.WriteSeriesKeyValue("FREQ", key.f.Id);
dataWriter.WriteSeriesKeyValue("REF_AREA", "DE");
dataWriter.WriteSeriesKeyValue("ADJUSTMENT", key.ad.Id);
dataWriter.WriteSeriesKeyValue("STS_INDICATOR", "PROD");
dataWriter.WriteSeriesKeyValue("STS_ACTIVITY", key.ac.Id);
dataWriter.WriteSeriesKeyValue("STS_INSTITUTION", "1");
dataWriter.WriteSeriesKeyValue("STS_BASE_YEAR", "2000");
var getPeriod = GetPeriodFunc(key.f.Id, startTime);
dataWriter.WriteAttributeValue("TIME_FORMAT", GetTimeFormat(key.f.Id));
for (int i = 0; i < 10; i++)
{
string period = getPeriod(i);
dataWriter.WriteObservation(period, i.ToString(CultureInfo.InvariantCulture));
dataWriter.WriteAttributeValue("OBS_STATUS", "A");
}
}
dataWriter.Close();
}
sw.Stop();
Trace.WriteLine(sw.Elapsed);
}
if (fromEnum.BaseDataFormat.EnumType == BaseDataFormatEnumType.Generic)
{
var fileReadableDataLocation = new FileReadableDataLocation(outfile);
XMLParser.ValidateXml(fileReadableDataLocation, fromEnum.SchemaVersion);
}
var dataReaderManager = new DataReaderManager();
int dataSetCount = 0;
using (var location = new FileReadableDataLocation(outfile))
using (var reader = dataReaderManager.GetDataReaderEngine(location, dataStructureObject, null))
{
while (reader.MoveNextDataset())
{
Assert.IsEmpty(reader.DatasetAttributes);
dataSetCount++;
int seriesCount = 0;
while (reader.MoveNextKeyable())
{
var keyValues = reader.CurrentKey;
if (keyValues.Series)
{
dynamic expectedSeries = series[seriesCount];
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("FREQ")).Code, expectedSeries.f.Id);
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("ADJUSTMENT")).Code, expectedSeries.ad.Id);
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("STS_ACTIVITY")).Code, expectedSeries.ac.Id);
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("REF_AREA")).Code, "DE");
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("STS_INDICATOR")).Code, "PROD");
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("STS_INSTITUTION")).Code, "1");
Assert.AreEqual(keyValues.Key.First(k => k.Concept.Equals("STS_BASE_YEAR")).Code, "2000");
if (fromEnum.SchemaVersion != SdmxSchemaEnumType.Edi)
{
Assert.AreEqual(keyValues.Attributes.First(k => k.Concept.Equals("TIME_FORMAT")).Code, GetTimeFormat(expectedSeries.f.Id));
}
seriesCount++;
var getPeriod = GetPeriodFunc(expectedSeries.f.Id, startTime);
int obsCount = 0;
while (reader.MoveNextObservation())
{
var currentObservation = reader.CurrentObservation;
Assert.AreEqual(obsCount.ToString(CultureInfo.InvariantCulture), currentObservation.ObservationValue);
Assert.AreEqual(getPeriod(obsCount), currentObservation.ObsTime);
obsCount++;
}
Assert.AreEqual(10, obsCount);
}
}
Assert.AreEqual(series.Length, seriesCount);
}
Assert.AreEqual(1, dataSetCount);
}
}