void test_read_write_read(string dataFile, Func <KeyFamily, XmlWriter, DataWriter> createWriter) { var dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var structure = StructureMessage.Load(dsdPath); var keyFamily = structure.KeyFamilies[0]; string dataPath = Utility.GetPath(dataFile); var data1 = new Dictionary <string, string>(); var data2 = new Dictionary <string, string>(); var doc = new XDocument(); using (var reader = DataReader.Create(dataPath, keyFamily)) using (var writer = createWriter(keyFamily, doc.CreateWriter())) { writer.WriteHeader(GetHeader()); while (reader.Read()) { data1.Add(GetData(reader), null); writer.WriteRecord(reader); } } using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { while (reader.Read()) { data2.Add(GetData(reader), null); } } Assert.IsTrue(CompareData(data1, data2)); }
public void WriteCompact_FromDB() { var dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var structure = StructureMessage.Load(dsdPath); var keyFamily = structure.KeyFamilies[0]; string targetNameSpace = "urn:sdmx:org.sdmx.infomodel.keyfamily.KeyFamily=BIS:EXT_DEBT:compact"; string dataPath = Utility.GetPath("lib\\CompactSample22.xml"); using (var reader = GetReader()) { var settings = new XmlWriterSettings() { Indent = true }; using (var writer = new CompactDataWriter(dataPath, keyFamily, settings, "bisc", targetNameSpace)) { writer.WriteHeader(GetHeader()); writer.Write(reader); } } string compactSchema = Utility.GetPath("lib\\BIS_JOINT_DEBT_Compact.xsd"); Assert.IsTrue(Utility.IsValidMessage(XDocument.Load(dataPath), Utility.GetComapctSchema(dsdPath, targetNameSpace))); }
void TestRead_NoHeader(string dataPath, int count) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; var doc = XDocument.Load(dataPath); doc.Descendants().Where(i => i.Name.LocalName == "Header").Single().Remove(); int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { var header = reader.ReadHeader(); Assert.IsNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(count, counter); }
public void BulkCopy() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("TIME", "TIME", i => i.ToString()); var table = ((IDataReader)reader).GetSchemaTable(); CreateTable(table); using (SqlBulkCopy bulkCopy = new SqlBulkCopy(_connectionString)) { bulkCopy.DestinationTableName = table.TableName; bulkCopy.WriteToServer(reader); } int count = 0; ExecuteReader("select count(*) from dbo." + table.TableName, r => count = (int)r[0]); Assert.AreNotEqual(0, count); } }
public void read_generic_duplicate_obs_tag() { string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var doc = XDocument.Load(dataPath); var series = doc.Descendants().Where(i => i.Name.LocalName == "Value" && i.Attribute("concept").Value == "OBS_STATUS").First(); var copy = new XElement(series); copy.SetAttributeValue("concept", "FREQ"); copy.SetAttributeValue("value", "M"); series.AddAfterSelf(copy); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { Assert.AreEqual(1, reader.Errors.Count); Assert.IsTrue(reader.Errors[0] is ValidationError); //Debug.WriteLine(reader.Errors[0].Message); counter++; } } } Assert.AreEqual(1, counter); }
public void BIS() { string dsdPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\WEBSTATS_IBLR_DATAFLOW-1361479877306.xml"; string dataPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\BISWEB-WEBSTATS_IBLR_DATAFLOW_formatted.xml"; var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { var header = reader.ReadHeader(); reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { WriteErrors(reader); Assert.Fail(); } else { // WriteRecord(reader); } } } }
public void read_from_database_into_file_stop_writing_but_continue_validating() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; IDataReader reader = null; bool valid = true; using (reader) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.WriteHeader(null); while (reader.Read()) { Dictionary <string, string> record = null; var errors = writer.ValidateRecord(reader, out record); if (errors.Count > 0) { valid = false; // display errors } if (valid) { writer.WriteRecord(record); } } } }
public void test_quest_b() { string dataPath = "C:\\Work\\Edu_meter_sid_2014\\UIS_ED_A_2014_AT_import222.xml"; var dsd = StructureMessage.Load(@"C:\Work\Edu_meter_sid_2014\UOE.xml"); var keyFamily = dsd.KeyFamilies[0]; var validator = new DataValidator(keyFamily); int counter = 0; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("OBS_STATUS", "OBS_STATUS", i => i == null ? i : ((string)i).ToUpper()); reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (validator.Validate(reader).Count() > 0) { counter++; } } } Assert.AreEqual(0, counter); }
void TestRead(XDocument doc, int count) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var r = doc.CreateReader()) using (var reader = DataReader.Create(r, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(count, counter); }
public void Read3() { string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml"); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = new MessageGroupReader(dataPath, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(13, counter); }
public void WebService() { string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml"); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = new MessageGroupReader(dataPath, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); WriteRecord(reader); counter++; } } Assert.AreEqual(13, counter); }
public void LoadTest() { for (int i = 0; i < 300; i++) { var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml"); } }
//[Test] //public void ValidateDataQuery_Success() //{ // var keyFamily = GetKeyFamily(); // var dataQuery = new DataQuery(); // var and = new AndCriterion(); // and.Add(new DimensionCriterion() { Name = "JD_TYPE", Value = "P" }); // and.Add(new AttributeCriterion() { Name = "OBS_CONF", Value = "C" }); // and.Add(new TimePeriodCriterion() // { // StartTime = new YearValue(1999), // EndTime = new YearValue(2002) // }); // dataQuery.Criterion = and; // Assert.IsTrue(keyFamily.ValidateDataQuery(dataQuery)); //} private KeyFamily GetKeyFamily() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var message = StructureMessage.Load(dsdPath); return(message.KeyFamilies[0]); }
void test_duplicate_tag(XDocument doc) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { Assert.AreEqual(1, reader.Errors.Count); Assert.IsTrue(reader.Errors[0] is ValidationError); //Debug.WriteLine(reader.Errors[0].Message); counter++; } } } Assert.AreEqual(12, counter); }
public void bop_its_tot() { var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml"); message.Save(@"c:\temp\bop_its_tot.dsd2.xml"); message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd2.xml"); message.Save(@"c:\temp\bop_its_tot.dsd3.xml"); }
public void WBDSD_HierarchicalCodeList() { // load the DSD string dsdPath = Utility.GetPath("lib\\DSD_WB.xml"); var dsd = StructureMessage.Load(dsdPath); // create hierarchical code list and add it to the DSD var hlist = new HierarchicalCodeList("MDG_Regions", "MDG"); hlist.Name["en"] = "MDG Regions"; dsd.HierarchicalCodeLists.Add(hlist); // get REF_AREA code list from the DSD and add it to the hierarchical code list var refAreaCodeList = dsd.CodeLists.Where(codeList => codeList.Id == "CL_REF_AREA_MDG").Single(); hlist.AddCodeList(refAreaCodeList, "REF_AREA_MDG"); // get parent country code var MDG_DEVELOPED = refAreaCodeList.Where(code => code.Id == "MDG_DEVELOPED").Single(); // child country codes string[] ids = new[] { "ALB", "AND", "AUS", "AUT", "BEL", "BIH", "BMU", "BGR", "HRV", "CAN", "CZE", "DNK", "EST", "FRO", "FIN", "FRA", "DEU", "GRC", "HUN", "ISL", "IRL", "ITA", "JPN", "LVA", "LIE", "LTU", "LUX", "MKD", "MLT", "MCO", "MNE", "NLD", "NZL", "NOR", "POL", "PRT", "ROU", "SVK", "SMR", "SVN", "SRB", "ESP", "SWE", "CHE", "GBR", "USA" }; var countryCodes = (from c in refAreaCodeList join cid in ids on c.Id.ToString() equals cid select c).ToList(); // create hirarchy with parent code and child code and add it to // the hierarchical code list var hierarchy = new Hierarchy("Developed_Countries", new CodeRef(MDG_DEVELOPED, countryCodes)); hierarchy.Name["en"] = "Developed Countries"; hlist.Add(hierarchy); // create another hierarchy and add it to the hierarchical code list var MDG_NAFR = refAreaCodeList.Where(code => code.Id == "MDG_NAFR").Single(); ids = new[] { "DZA", "EGY", "LBY", "MAR", "TUN" }; countryCodes = (from c in refAreaCodeList join cid in ids on c.Id.ToString() equals cid select c).ToList(); hierarchy = new Hierarchy("MDG_NAFR", new CodeRef(MDG_NAFR, () => countryCodes)); hierarchy.Name["en"] = "MDG Northern Africa Countries"; hlist.Add(hierarchy); // save the DSD dsd.Save(Utility.GetPath("lib\\DSD_WB2.xml")); string messageText = dsd.ToString(); var doc = XDocument.Parse(messageText); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); }
public void convert_format_from_file_to_file() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.Write(reader); } }
public void StructureSampleTest() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var message = StructureMessage.Load(dsdPath); //message.Save(@"c:\temp\StructureSample2.xml"); var doc = new XDocument(); using (var writer = doc.CreateWriter()) message.Write(writer); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); }
public void read_from_database_into_file_quit_if_notValid() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; IDataReader reader = null; using (reader) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.WriteHeader(null); writer.Write(reader); } }
public void read_file_into_db_quit_if_file_not_valid() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("TIME", "TIME", i => i.ToString()); using (var bulkCopy = new SqlBulkCopy(_connectionString)) { bulkCopy.DestinationTableName = "DestinationTableName"; bulkCopy.WriteToServer(reader); } } }
public void HeirarchicalCodeListSample() { string dsdPath = Utility.GetPath("lib\\HeirarchicalCodeListSample.xml"); var sample = XDocument.Load(dsdPath); using (var reader = sample.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); StructureMessageMap map = new StructureMessageMap(); StructureMessage message = StructureMessage.Load(dsdPath); message.Save(Utility.GetPath("lib\\HeirarchicalCodeListSample2.xml")); var doc = XDocument.Load(Utility.GetPath("lib\\HeirarchicalCodeListSample2.xml")); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); }
public void LoadWriteLoad() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var message = StructureMessage.Load(dsdPath); var doc = new XDocument(); using (var writer = doc.CreateWriter()) message.Write(writer); // Console.Write(doc); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader, w => Console.WriteLine(w), e => Console.WriteLine(e))); StructureMessage message2 = null; using (var reader = doc.CreateReader()) message2 = StructureMessage.Read(reader); }
public void SpeedTest() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); const int smallLoopCount = 1; const int bigLoopCount = 1000; var list = new List <decimal>(); var list2 = new List <TimeSpan>(); for (int i = 0; i < bigLoopCount; i++) { var stopWatch = new System.Diagnostics.Stopwatch(); stopWatch.Start(); for (int j = 0; j < smallLoopCount; j++) { using (var reader = XmlReader.Create(dsdPath)) { while (reader.Read()) { } } } stopWatch.Stop(); long ticks = stopWatch.ElapsedTicks; stopWatch.Restart(); for (int k = 0; k < smallLoopCount; k++) { var message = StructureMessage.Load(dsdPath); } stopWatch.Stop(); Console.Write((decimal)stopWatch.ElapsedTicks / ticks); list.Add((decimal)stopWatch.ElapsedTicks / ticks); list2.Add(stopWatch.Elapsed); Console.WriteLine(" : {0}", stopWatch.Elapsed); } Console.WriteLine("Average: {0}", list.Average()); Console.WriteLine("Average Time: {0}", TimeSpan.FromTicks((long)list2.Select(i => i.Ticks).Average())); }
public void WriteGeneric_FromDB() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); var keyFamily = structure.KeyFamilies[0]; string path = Utility.GetPath("lib\\GenericSample22.xml"); using (var reader = GetReader()) { var settings = new XmlWriterSettings() { Indent = true }; using (var writer = new GenericDataWriter(path, keyFamily, settings)) { writer.WriteHeader(GetHeader()); writer.Write(reader); } } Assert.IsTrue(MessageValidator.ValidateXml(path, s => Debug.WriteLine(s), s => Debug.WriteLine(s))); }
void TestRead_SkipHeader(string dataPath, int count) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = DataReader.Create(dataPath, keyFamily)) { while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(count, counter); }
public void read_file_into_db_insert_only_valid() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("TIME", "TIME", i => i.ToString()); reader.ThrowExceptionIfNotValid = false; var table = ((IDataReader)reader).GetSchemaTable(); CreateTable(table); while (reader.Read()) { if (reader.IsValid) { // insert } } } }