void Insert(Stream dataStream, Stream dsdStream, Func <KeyFamily, XmlWriter, DataWriter> createWriter)
{
var doc = XDocument.Load(dsdStream);
doc.Descendants().Where(i => i.Name.LocalName == "TextFormat" && i.Parent.Name.LocalName == "TimeDimension").Single().Remove();
var mngr = new XmlNamespaceManager(new NameTable());
mngr.AddNamespace("structure", "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure");
mngr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instanc");
mngr.AddNamespace("xsd", "http://www.w3.org/2001/XMLSchema");
XElement xElementParse = ParseElement("<structure:Concept id='TIME_FORMAT'><structure:Name xml:lang='en'>Time format</structure:Name></structure:Concept>", mngr);
doc.Descendants().Where(i => i.Name.LocalName == "ConceptScheme").Single().Add(xElementParse);
using (var xReader = doc.CreateReader())
{
var structure = StructureMessage.Read(xReader);
var keyFamily = structure.KeyFamilies[0];
var data1 = new Dictionary <string, string>();
var data2 = new Dictionary <string, string>();
var settings = new XmlWriterSettings()
{
Indent = true
};
var dataDoc = new XDocument();
using (var reader = DataReader.Create(dataStream, keyFamily))
using (var writer = createWriter(keyFamily, dataDoc.CreateWriter()))
//using (var writer = new CompactDataWriter(Utility.GetPath("lib\\aei_ps_alt.sdmx\\aei_ps_alt.sdmx2.xml"), keyFamily, settings, "data", "urn:sdmx:org.sdmx.infomodel.keyfamily.KeyFamily=EUROSTAT:aei_ps_alt_DSD:compact"))
{
writer.WriteHeader(GetHeader());
while (reader.Read())
{
data1.Add(GetData(reader), null);
writer.WriteRecord(reader);
}
}
using (var reader = DataReader.Create(dataDoc.CreateReader(), keyFamily))
{
while (reader.Read())
{
data2.Add(GetData(reader), null);
}
}
Assert.IsTrue(CompareData(data1, data2));
}
}
public void Invalid_structure_missing_codelist()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var doc = XDocument.Load(dsdPath);
var series = doc.Descendants().Where(i => i.Name.LocalName == "CodeList" && i.Attribute("id").Value == "CL_FREQ").First();
series.Remove();
var errorList = new List <ValidationMessage>();
var message = StructureMessage.Read(doc.CreateReader(), v => errorList.Add(v));
Assert.AreEqual(1, errorList.Count);
}
public void Invalid_structure_duplicate_concept()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var doc = XDocument.Load(dsdPath);
var series = doc.Descendants().Where(i => i.Name.LocalName == "Concept" && i.Attribute("id").Value == "TIME").First();
var copy = new XElement(series);
series.AddAfterSelf(copy);
var errorList = new List <ValidationMessage>();
var message = StructureMessage.Read(doc.CreateReader(), v => errorList.Add(v));
Assert.AreEqual(1, errorList.Count);
}
void Insert(Stream dataStream, Stream dsdStream)
{
var doc = XDocument.Load(dsdStream);
doc.Descendants().Where(i => i.Name.LocalName == "TextFormat" && i.Parent.Name.LocalName == "TimeDimension").Single().Remove();
var mngr = new XmlNamespaceManager(new NameTable());
mngr.AddNamespace("structure", "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure");
mngr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instanc");
mngr.AddNamespace("xsd", "http://www.w3.org/2001/XMLSchema");
XElement xElementParse = ParseElement("<structure:Concept id='TIME_FORMAT'><structure:Name xml:lang='en'>Time format</structure:Name></structure:Concept>", mngr);
doc.Descendants().Where(i => i.Name.LocalName == "ConceptScheme").Single().Add(xElementParse);
using (var xReader = doc.CreateReader())
{
var structure = StructureMessage.Read(xReader);
var keyFamily = structure.KeyFamilies[0];
using (var reader = DataReader.Create(dataStream, keyFamily))
{
reader.Map("TIME_PERIOD", "TIME_PERIOD", i => ((TimePeriod)i).Year);
var table = ((IDataReader)reader).GetSchemaTable();
CreateTable(table);
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(_connectionString))
{
bulkCopy.BatchSize = 10000;
bulkCopy.DestinationTableName = table.TableName;
bulkCopy.WriteToServer(reader);
}
int count = 0;
ExecuteReader("select count(*) from dbo." + table.TableName, r => count = (int)r[0]);
Assert.AreNotEqual(0, count);
}
}
}
public void LoadWriteLoad()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var message = StructureMessage.Load(dsdPath);
var doc = new XDocument();
using (var writer = doc.CreateWriter())
message.Write(writer);
// Console.Write(doc);
using (var reader = doc.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader, w => Console.WriteLine(w), e => Console.WriteLine(e)));
StructureMessage message2 = null;
using (var reader = doc.CreateReader())
message2 = StructureMessage.Read(reader);
}
public void Invalid_data_file()
{
string dsdPath = @"C:\Temp\ert_bil_eur_d.sdmx\ert_bil_eur_d.dsd.xml";
string dataPath = @"C:\Temp\ert_bil_eur_d.sdmx\ert_bil_eur_d.sdmx.xml";
var dsdDoc = XDocument.Load(dsdPath);
var concept = dsdDoc.Descendants().Where(i => i.Name.LocalName == "Concept" && i.Attribute("id").Value == "TIME_PERIOD").First();
var copy = new XElement(concept);
copy.SetAttributeValue("id", "TIME_FORMAT");
concept.AddAfterSelf(copy);
// Remove TextFormat="String"
var textFormat = dsdDoc.Descendants().Where(i => i.Name.LocalName == "TextFormat" && i.Attribute("textType").Value == "String").First();
textFormat.Remove();
StructureMessage dsd = null;
using (var reader = dsdDoc.CreateReader())
{
dsd = StructureMessage.Read(reader, v => Console.WriteLine(v.Message));
}
var keyFamily = dsd.KeyFamilies[0];
//using (var customReader = new CustomXmlReader(dataPath))
using (var reader = DataReader.Create(dataPath, keyFamily))
{
var header = reader.ReadHeader();
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (!reader.IsValid)
{
foreach (var error in reader.Errors)
{
bool errorCorrected = false;
if (error is ParseError)
{
var parseError = (ParseError)error;
if (parseError.Name == "TIME_PERIOD")
{
string year = parseError.Value.Substring(0, 4);
string month = parseError.Value.Substring(4, 2);
string day = parseError.Value.Substring(6, 2);
// recover time value
var timePeriodValue = new DateTime(int.Parse(year), int.Parse(month), int.Parse(day));
errorCorrected = true;
//Console.WriteLine("FREQ={0},otp={1},unit={2},currency={3},TIME_PERIOD={4},OBS_VALUE={5},TIME_FORMAT={6},OBS_STATUS={7}",
// reader["FREQ"],
// reader["otp"],
// reader["unit"],
// reader["currency"],
// timePeriodValue,
// reader["OBS_VALUE"],
// reader["TIME_FORMAT"],
// reader["OBS_STATUS"]);
}
else if (parseError.Name == "OBS_STATUS")
{
if (parseError.Value == "iz")
{
// do something to correct the error
errorCorrected = true;
}
}
}
else if (error is MandatoryComponentMissing)
{
var missingError = (MandatoryComponentMissing)error;
if (missingError.Name == "OBS_VALUE")
{
// do something to correct the error
errorCorrected = true;
}
}
if (!errorCorrected)
{
Console.WriteLine(error.Message);
Assert.Fail();
}
}
}
else
{
// WriteRecord(reader);
}
}
}
}
public void Fix_dsd()
{
string dsdPath = @"C:\Temp\ei_bsin_m.sdmx\ei_bsin_m.dsd.xml";
string dataPath = @"C:\Temp\ei_bsin_m.sdmx\ei_bsin_m.sdmx.xml";
// load the DSD in XDocument to fix programatically
var dsdDoc = XDocument.Load(dsdPath);
// add TIME_FORMAT concept
// find TIME_PERIOD node and add TIME_FORMAT
var concept = dsdDoc.Descendants().Where(i => i.Name.LocalName == "Concept" && i.Attribute("id").Value == "TIME_PERIOD").First();
var copy = new XElement(concept);
copy.SetAttributeValue("id", "TIME_FORMAT");
concept.AddAfterSelf(copy);
// Remove TextFormat="String"
var textFormat = dsdDoc.Descendants().Where(i => i.Name.LocalName == "TextFormat" && i.Attribute("textType").Value == "String").First();
textFormat.Remove();
StructureMessage dsd = null;
using (var reader = dsdDoc.CreateReader())
{
// add Console.WriteLine action to avoid throwing excpetions
dsd = StructureMessage.Read(reader, v => Console.WriteLine(v.Message));
}
var keyFamily = dsd.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
{
var header = reader.ReadHeader();
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (reader.IsValid)
{
//Console.WriteLine("FREQ={0},indic={1},s_adj={2},unit={3},geo={4},TIME_PERIOD={5},OBS_VALUE={6},TIME_FORMAT={7},OBS_STATUS={8}",
// reader["FREQ"],
// reader["indic"],
// reader["s_adj"],
// reader["unit"],
// reader["geo"],
// reader["TIME_PERIOD"],
// reader["OBS_VALUE"],
// reader["TIME_FORMAT"],
// reader["OBS_STATUS"]);
}
else
{
WriteErrors(reader);
Assert.Fail();
}
}
}
}
