void test_read_write_read(string dataFile, Func <KeyFamily, XmlWriter, DataWriter> createWriter)
{
var dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var structure = StructureMessage.Load(dsdPath);
var keyFamily = structure.KeyFamilies[0];
string dataPath = Utility.GetPath(dataFile);
var data1 = new Dictionary <string, string>();
var data2 = new Dictionary <string, string>();
var doc = new XDocument();
using (var reader = DataReader.Create(dataPath, keyFamily))
using (var writer = createWriter(keyFamily, doc.CreateWriter()))
{
writer.WriteHeader(GetHeader());
while (reader.Read())
{
data1.Add(GetData(reader), null);
writer.WriteRecord(reader);
}
}
using (var reader = DataReader.Create(doc.CreateReader(), keyFamily))
{
while (reader.Read())
{
data2.Add(GetData(reader), null);
}
}
Assert.IsTrue(CompareData(data1, data2));
}
public void WriteCompact_FromDB()
{
var dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var structure = StructureMessage.Load(dsdPath);
var keyFamily = structure.KeyFamilies[0];
string targetNameSpace = "urn:sdmx:org.sdmx.infomodel.keyfamily.KeyFamily=BIS:EXT_DEBT:compact";
string dataPath = Utility.GetPath("lib\\CompactSample22.xml");
using (var reader = GetReader())
{
var settings = new XmlWriterSettings()
{
Indent = true
};
using (var writer = new CompactDataWriter(dataPath, keyFamily, settings, "bisc", targetNameSpace))
{
writer.WriteHeader(GetHeader());
writer.Write(reader);
}
}
string compactSchema = Utility.GetPath("lib\\BIS_JOINT_DEBT_Compact.xsd");
Assert.IsTrue(Utility.IsValidMessage(XDocument.Load(dataPath), Utility.GetComapctSchema(dsdPath, targetNameSpace)));
}
void TestRead_NoHeader(string dataPath, int count)
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
var doc = XDocument.Load(dataPath);
doc.Descendants().Where(i => i.Name.LocalName == "Header").Single().Remove();
int counter = 0;
using (var reader = DataReader.Create(doc.CreateReader(), keyFamily))
{
var header = reader.ReadHeader();
Assert.IsNull(header);
while (reader.Read())
{
Assert.AreEqual(17, reader.Count());
//foreach (var item in reader)
// Console.Write("{0}={1},", item.Key, item.Value);
//Console.WriteLine();
counter++;
}
}
Assert.AreEqual(count, counter);
}
public void BulkCopy()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
{
reader.Map("TIME", "TIME", i => i.ToString());
var table = ((IDataReader)reader).GetSchemaTable();
CreateTable(table);
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(_connectionString))
{
bulkCopy.DestinationTableName = table.TableName;
bulkCopy.WriteToServer(reader);
}
int count = 0;
ExecuteReader("select count(*) from dbo." + table.TableName, r => count = (int)r[0]);
Assert.AreNotEqual(0, count);
}
}
public void read_generic_duplicate_obs_tag()
{
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var doc = XDocument.Load(dataPath);
var series = doc.Descendants().Where(i => i.Name.LocalName == "Value" && i.Attribute("concept").Value == "OBS_STATUS").First();
var copy = new XElement(series);
copy.SetAttributeValue("concept", "FREQ");
copy.SetAttributeValue("value", "M");
series.AddAfterSelf(copy);
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var reader = DataReader.Create(doc.CreateReader(), keyFamily))
{
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (!reader.IsValid)
{
Assert.AreEqual(1, reader.Errors.Count);
Assert.IsTrue(reader.Errors[0] is ValidationError);
//Debug.WriteLine(reader.Errors[0].Message);
counter++;
}
}
}
Assert.AreEqual(1, counter);
}
public void BIS()
{
string dsdPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\WEBSTATS_IBLR_DATAFLOW-1361479877306.xml";
string dataPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\BISWEB-WEBSTATS_IBLR_DATAFLOW_formatted.xml";
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
{
var header = reader.ReadHeader();
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (!reader.IsValid)
{
WriteErrors(reader);
Assert.Fail();
}
else
{
// WriteRecord(reader);
}
}
}
}
public void read_from_database_into_file_stop_writing_but_continue_validating()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
IDataReader reader = null;
bool valid = true;
using (reader)
using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns"))
{
writer.WriteHeader(null);
while (reader.Read())
{
Dictionary <string, string> record = null;
var errors = writer.ValidateRecord(reader, out record);
if (errors.Count > 0)
{
valid = false;
// display errors
}
if (valid)
{
writer.WriteRecord(record);
}
}
}
}
public void test_quest_b()
{
string dataPath = "C:\\Work\\Edu_meter_sid_2014\\UIS_ED_A_2014_AT_import222.xml";
var dsd = StructureMessage.Load(@"C:\Work\Edu_meter_sid_2014\UOE.xml");
var keyFamily = dsd.KeyFamilies[0];
var validator = new DataValidator(keyFamily);
int counter = 0;
using (var reader = DataReader.Create(dataPath, keyFamily))
{
reader.Map("OBS_STATUS", "OBS_STATUS", i => i == null ? i : ((string)i).ToUpper());
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (validator.Validate(reader).Count() > 0)
{
counter++;
}
}
}
Assert.AreEqual(0, counter);
}
void TestRead(XDocument doc, int count)
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var r = doc.CreateReader())
using (var reader = DataReader.Create(r, keyFamily))
{
var header = reader.ReadHeader();
Assert.IsNotNull(header);
while (reader.Read())
{
Assert.AreEqual(17, reader.Count());
//foreach (var item in reader)
// Console.Write("{0}={1},", item.Key, item.Value);
//Console.WriteLine();
counter++;
}
}
Assert.AreEqual(count, counter);
}
public void Read3()
{
string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml");
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var reader = new MessageGroupReader(dataPath, keyFamily))
{
var header = reader.ReadHeader();
Assert.IsNotNull(header);
while (reader.Read())
{
Assert.AreEqual(17, reader.Count());
//foreach (var item in reader)
// Console.Write("{0}={1},", item.Key, item.Value);
//Console.WriteLine();
counter++;
}
}
Assert.AreEqual(13, counter);
}
public void WebService()
{
string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml");
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var reader = new MessageGroupReader(dataPath, keyFamily))
{
var header = reader.ReadHeader();
Assert.IsNotNull(header);
while (reader.Read())
{
Assert.AreEqual(17, reader.Count());
WriteRecord(reader);
counter++;
}
}
Assert.AreEqual(13, counter);
}
public void LoadTest()
{
for (int i = 0; i < 300; i++)
{
var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml");
}
}
//[Test]
//public void ValidateDataQuery_Success()
//{
// var keyFamily = GetKeyFamily();
// var dataQuery = new DataQuery();
// var and = new AndCriterion();
// and.Add(new DimensionCriterion() { Name = "JD_TYPE", Value = "P" });
// and.Add(new AttributeCriterion() { Name = "OBS_CONF", Value = "C" });
// and.Add(new TimePeriodCriterion()
// {
// StartTime = new YearValue(1999),
// EndTime = new YearValue(2002)
// });
// dataQuery.Criterion = and;
// Assert.IsTrue(keyFamily.ValidateDataQuery(dataQuery));
//}
private KeyFamily GetKeyFamily()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var message = StructureMessage.Load(dsdPath);
return(message.KeyFamilies[0]);
}
void test_duplicate_tag(XDocument doc)
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var reader = DataReader.Create(doc.CreateReader(), keyFamily))
{
reader.ThrowExceptionIfNotValid = false;
while (reader.Read())
{
if (!reader.IsValid)
{
Assert.AreEqual(1, reader.Errors.Count);
Assert.IsTrue(reader.Errors[0] is ValidationError);
//Debug.WriteLine(reader.Errors[0].Message);
counter++;
}
}
}
Assert.AreEqual(12, counter);
}
public void bop_its_tot()
{
var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml");
message.Save(@"c:\temp\bop_its_tot.dsd2.xml");
message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd2.xml");
message.Save(@"c:\temp\bop_its_tot.dsd3.xml");
}
public void WBDSD_HierarchicalCodeList()
{
// load the DSD
string dsdPath = Utility.GetPath("lib\\DSD_WB.xml");
var dsd = StructureMessage.Load(dsdPath);
// create hierarchical code list and add it to the DSD
var hlist = new HierarchicalCodeList("MDG_Regions", "MDG");
hlist.Name["en"] = "MDG Regions";
dsd.HierarchicalCodeLists.Add(hlist);
// get REF_AREA code list from the DSD and add it to the hierarchical code list
var refAreaCodeList = dsd.CodeLists.Where(codeList => codeList.Id == "CL_REF_AREA_MDG").Single();
hlist.AddCodeList(refAreaCodeList, "REF_AREA_MDG");
// get parent country code
var MDG_DEVELOPED = refAreaCodeList.Where(code => code.Id == "MDG_DEVELOPED").Single();
// child country codes
string[] ids = new[] { "ALB", "AND", "AUS", "AUT", "BEL", "BIH", "BMU", "BGR", "HRV", "CAN", "CZE", "DNK", "EST", "FRO", "FIN", "FRA", "DEU", "GRC", "HUN", "ISL", "IRL", "ITA", "JPN", "LVA", "LIE", "LTU", "LUX", "MKD", "MLT", "MCO", "MNE", "NLD", "NZL", "NOR", "POL", "PRT", "ROU", "SVK", "SMR", "SVN", "SRB", "ESP", "SWE", "CHE", "GBR", "USA" };
var countryCodes = (from c in refAreaCodeList
join cid in ids on c.Id.ToString() equals cid
select c).ToList();
// create hirarchy with parent code and child code and add it to
// the hierarchical code list
var hierarchy = new Hierarchy("Developed_Countries", new CodeRef(MDG_DEVELOPED, countryCodes));
hierarchy.Name["en"] = "Developed Countries";
hlist.Add(hierarchy);
// create another hierarchy and add it to the hierarchical code list
var MDG_NAFR = refAreaCodeList.Where(code => code.Id == "MDG_NAFR").Single();
ids = new[] { "DZA", "EGY", "LBY", "MAR", "TUN" };
countryCodes = (from c in refAreaCodeList
join cid in ids on c.Id.ToString() equals cid
select c).ToList();
hierarchy = new Hierarchy("MDG_NAFR", new CodeRef(MDG_NAFR, () => countryCodes));
hierarchy.Name["en"] = "MDG Northern Africa Countries";
hlist.Add(hierarchy);
// save the DSD
dsd.Save(Utility.GetPath("lib\\DSD_WB2.xml"));
string messageText = dsd.ToString();
var doc = XDocument.Parse(messageText);
using (var reader = doc.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader));
}
public void convert_format_from_file_to_file()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns"))
{
writer.Write(reader);
}
}
public void StructureSampleTest()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var message = StructureMessage.Load(dsdPath);
//message.Save(@"c:\temp\StructureSample2.xml");
var doc = new XDocument();
using (var writer = doc.CreateWriter())
message.Write(writer);
using (var reader = doc.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader));
}
public void read_from_database_into_file_quit_if_notValid()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
IDataReader reader = null;
using (reader)
using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns"))
{
writer.WriteHeader(null);
writer.Write(reader);
}
}
public void read_file_into_db_quit_if_file_not_valid()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
{
reader.Map("TIME", "TIME", i => i.ToString());
using (var bulkCopy = new SqlBulkCopy(_connectionString))
{
bulkCopy.DestinationTableName = "DestinationTableName";
bulkCopy.WriteToServer(reader);
}
}
}
public void HeirarchicalCodeListSample()
{
string dsdPath = Utility.GetPath("lib\\HeirarchicalCodeListSample.xml");
var sample = XDocument.Load(dsdPath);
using (var reader = sample.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader));
StructureMessageMap map = new StructureMessageMap();
StructureMessage message = StructureMessage.Load(dsdPath);
message.Save(Utility.GetPath("lib\\HeirarchicalCodeListSample2.xml"));
var doc = XDocument.Load(Utility.GetPath("lib\\HeirarchicalCodeListSample2.xml"));
using (var reader = doc.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader));
}
public void LoadWriteLoad()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var message = StructureMessage.Load(dsdPath);
var doc = new XDocument();
using (var writer = doc.CreateWriter())
message.Write(writer);
// Console.Write(doc);
using (var reader = doc.CreateReader())
Assert.IsTrue(MessageValidator.ValidateXml(reader, w => Console.WriteLine(w), e => Console.WriteLine(e)));
StructureMessage message2 = null;
using (var reader = doc.CreateReader())
message2 = StructureMessage.Read(reader);
}
public void SpeedTest()
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
const int smallLoopCount = 1;
const int bigLoopCount = 1000;
var list = new List <decimal>();
var list2 = new List <TimeSpan>();
for (int i = 0; i < bigLoopCount; i++)
{
var stopWatch = new System.Diagnostics.Stopwatch();
stopWatch.Start();
for (int j = 0; j < smallLoopCount; j++)
{
using (var reader = XmlReader.Create(dsdPath))
{
while (reader.Read())
{
}
}
}
stopWatch.Stop();
long ticks = stopWatch.ElapsedTicks;
stopWatch.Restart();
for (int k = 0; k < smallLoopCount; k++)
{
var message = StructureMessage.Load(dsdPath);
}
stopWatch.Stop();
Console.Write((decimal)stopWatch.ElapsedTicks / ticks);
list.Add((decimal)stopWatch.ElapsedTicks / ticks);
list2.Add(stopWatch.Elapsed);
Console.WriteLine(" : {0}", stopWatch.Elapsed);
}
Console.WriteLine("Average: {0}", list.Average());
Console.WriteLine("Average Time: {0}", TimeSpan.FromTicks((long)list2.Select(i => i.Ticks).Average()));
}
public void WriteGeneric_FromDB()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
var keyFamily = structure.KeyFamilies[0];
string path = Utility.GetPath("lib\\GenericSample22.xml");
using (var reader = GetReader())
{
var settings = new XmlWriterSettings()
{
Indent = true
};
using (var writer = new GenericDataWriter(path, keyFamily, settings))
{
writer.WriteHeader(GetHeader());
writer.Write(reader);
}
}
Assert.IsTrue(MessageValidator.ValidateXml(path, s => Debug.WriteLine(s), s => Debug.WriteLine(s)));
}
void TestRead_SkipHeader(string dataPath, int count)
{
string dsdPath = Utility.GetPath("lib\\StructureSample.xml");
var dsd = StructureMessage.Load(dsdPath);
var keyFamily = dsd.KeyFamilies[0];
int counter = 0;
using (var reader = DataReader.Create(dataPath, keyFamily))
{
while (reader.Read())
{
Assert.AreEqual(17, reader.Count());
//foreach (var item in reader)
// Console.Write("{0}={1},", item.Key, item.Value);
//Console.WriteLine();
counter++;
}
}
Assert.AreEqual(count, counter);
}
public void read_file_into_db_insert_only_valid()
{
var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml"));
string dataPath = Utility.GetPath("lib\\GenericSample.xml");
var keyFamily = structure.KeyFamilies[0];
using (var reader = DataReader.Create(dataPath, keyFamily))
{
reader.Map("TIME", "TIME", i => i.ToString());
reader.ThrowExceptionIfNotValid = false;
var table = ((IDataReader)reader).GetSchemaTable();
CreateTable(table);
while (reader.Read())
{
if (reader.IsValid)
{
// insert
}
}
}
}
