public void LoadTest() { for (int i = 0; i < 300; i++) { var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml"); } }
public void BIS() { string dsdPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\WEBSTATS_IBLR_DATAFLOW-1361479877306.xml"; string dataPath = @"C:\Temp\WEBSTATS_IBLR_DATAFLOW-1361479813975\BISWEB-WEBSTATS_IBLR_DATAFLOW_formatted.xml"; var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { var header = reader.ReadHeader(); reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { WriteErrors(reader); Assert.Fail(); } else { // WriteRecord(reader); } } } }
public void WebService() { string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml"); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = new MessageGroupReader(dataPath, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); WriteRecord(reader); counter++; } } Assert.AreEqual(13, counter); }
public void read_generic_duplicate_obs_tag() { string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var doc = XDocument.Load(dataPath); var series = doc.Descendants().Where(i => i.Name.LocalName == "Value" && i.Attribute("concept").Value == "OBS_STATUS").First(); var copy = new XElement(series); copy.SetAttributeValue("concept", "FREQ"); copy.SetAttributeValue("value", "M"); series.AddAfterSelf(copy); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { Assert.AreEqual(1, reader.Errors.Count); Assert.IsTrue(reader.Errors[0] is ValidationError); //Debug.WriteLine(reader.Errors[0].Message); counter++; } } } Assert.AreEqual(1, counter); }
void TestRead_NoHeader(string dataPath, int count) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; var doc = XDocument.Load(dataPath); doc.Descendants().Where(i => i.Name.LocalName == "Header").Single().Remove(); int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { var header = reader.ReadHeader(); Assert.IsNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(count, counter); }
public void Read3() { string dataPath = Utility.GetPath("lib\\MessageGroupSample3.xml"); string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = new MessageGroupReader(dataPath, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(13, counter); }
//[Test] //public void ValidateDataQuery_Success() //{ // var keyFamily = GetKeyFamily(); // var dataQuery = new DataQuery(); // var and = new AndCriterion(); // and.Add(new DimensionCriterion() { Name = "JD_TYPE", Value = "P" }); // and.Add(new AttributeCriterion() { Name = "OBS_CONF", Value = "C" }); // and.Add(new TimePeriodCriterion() // { // StartTime = new YearValue(1999), // EndTime = new YearValue(2002) // }); // dataQuery.Criterion = and; // Assert.IsTrue(keyFamily.ValidateDataQuery(dataQuery)); //} private KeyFamily GetKeyFamily() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var message = StructureMessage.Load(dsdPath); return(message.KeyFamilies[0]); }
public void WriteCompact_FromDB() { var dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var structure = StructureMessage.Load(dsdPath); var keyFamily = structure.KeyFamilies[0]; string targetNameSpace = "urn:sdmx:org.sdmx.infomodel.keyfamily.KeyFamily=BIS:EXT_DEBT:compact"; string dataPath = Utility.GetPath("lib\\CompactSample22.xml"); using (var reader = GetReader()) { var settings = new XmlWriterSettings() { Indent = true }; using (var writer = new CompactDataWriter(dataPath, keyFamily, settings, "bisc", targetNameSpace)) { writer.WriteHeader(GetHeader()); writer.Write(reader); } } string compactSchema = Utility.GetPath("lib\\BIS_JOINT_DEBT_Compact.xsd"); Assert.IsTrue(Utility.IsValidMessage(XDocument.Load(dataPath), Utility.GetComapctSchema(dsdPath, targetNameSpace))); }
void test_read_write_read(string dataFile, Func <KeyFamily, XmlWriter, DataWriter> createWriter) { var dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var structure = StructureMessage.Load(dsdPath); var keyFamily = structure.KeyFamilies[0]; string dataPath = Utility.GetPath(dataFile); var data1 = new Dictionary <string, string>(); var data2 = new Dictionary <string, string>(); var doc = new XDocument(); using (var reader = DataReader.Create(dataPath, keyFamily)) using (var writer = createWriter(keyFamily, doc.CreateWriter())) { writer.WriteHeader(GetHeader()); while (reader.Read()) { data1.Add(GetData(reader), null); writer.WriteRecord(reader); } } using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { while (reader.Read()) { data2.Add(GetData(reader), null); } } Assert.IsTrue(CompareData(data1, data2)); }
public void BulkCopy() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("TIME", "TIME", i => i.ToString()); var table = ((IDataReader)reader).GetSchemaTable(); CreateTable(table); using (SqlBulkCopy bulkCopy = new SqlBulkCopy(_connectionString)) { bulkCopy.DestinationTableName = table.TableName; bulkCopy.WriteToServer(reader); } int count = 0; ExecuteReader("select count(*) from dbo." + table.TableName, r => count = (int)r[0]); Assert.AreNotEqual(0, count); } }
void test_duplicate_tag(XDocument doc) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var reader = DataReader.Create(doc.CreateReader(), keyFamily)) { reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (!reader.IsValid) { Assert.AreEqual(1, reader.Errors.Count); Assert.IsTrue(reader.Errors[0] is ValidationError); //Debug.WriteLine(reader.Errors[0].Message); counter++; } } } Assert.AreEqual(12, counter); }
public void test_quest_b() { string dataPath = "C:\\Work\\Edu_meter_sid_2014\\UIS_ED_A_2014_AT_import222.xml"; var dsd = StructureMessage.Load(@"C:\Work\Edu_meter_sid_2014\UOE.xml"); var keyFamily = dsd.KeyFamilies[0]; var validator = new DataValidator(keyFamily); int counter = 0; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("OBS_STATUS", "OBS_STATUS", i => i == null ? i : ((string)i).ToUpper()); reader.ThrowExceptionIfNotValid = false; while (reader.Read()) { if (validator.Validate(reader).Count() > 0) { counter++; } } } Assert.AreEqual(0, counter); }
public KeyFamilyMap(StructureMessage message) { AttributesOrder("id", "agencyID", "version", "uri", "isFinal", "isExternalReference", "validFrom", "validTo"); ElementsOrder("Name", "Description", "Components", "Annotations"); var components = MapContainer("Components", false); components.MapCollection(o => o.Dimensions).ToElement("Dimension", false) .Set(v => _keyFamily.AddDimension(v)) .ClassMap(() => new DimensionMap(message)); components.Map(o => o.TimeDimension).ToElement("TimeDimension", false) .Set(v => _keyFamily.TimeDimension = v) .ClassMap(() => new TimeDimensionMap(message)); components.MapCollection(o => o.Groups).ToElement("Group", false) .ClassMap(() => new GroupDescriptorMap(_keyFamily)); components.Map(o => o.PrimaryMeasure).ToElement("PrimaryMeasure", true) .Set(v => _keyFamily.PrimaryMeasure = v) .ClassMap(() => new PrimaryMeasureMap(message)); components.MapCollection(o => o.CrossSectionalMeasures).ToElement("CrossSectionalMeasure", false) .Set(v => _keyFamily.AddMeasure(v)) .ClassMap(() => new CrossSectionalMeasureMap(message)); components.MapCollection(o => o.Attributes).ToElement("Attribute", false) .Set(v => _keyFamily.AddAttribute(v)) .ClassMap(() => new AttributeMap(message)); }
void TestRead(XDocument doc, int count) { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var dsd = StructureMessage.Load(dsdPath); var keyFamily = dsd.KeyFamilies[0]; int counter = 0; using (var r = doc.CreateReader()) using (var reader = DataReader.Create(r, keyFamily)) { var header = reader.ReadHeader(); Assert.IsNotNull(header); while (reader.Read()) { Assert.AreEqual(17, reader.Count()); //foreach (var item in reader) // Console.Write("{0}={1},", item.Key, item.Value); //Console.WriteLine(); counter++; } } Assert.AreEqual(count, counter); }
internal PrimaryMeasureMap(StructureMessage message) : base(message) { AttributesOrder("conceptRef", "codelist"); ElementsOrder("TextFormat", "Annotations"); }
public void read_from_database_into_file_stop_writing_but_continue_validating() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; IDataReader reader = null; bool valid = true; using (reader) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.WriteHeader(null); while (reader.Read()) { Dictionary <string, string> record = null; var errors = writer.ValidateRecord(reader, out record); if (errors.Count > 0) { valid = false; // display errors } if (valid) { writer.WriteRecord(record); } } } }
public void bop_its_tot() { var message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd.xml"); message.Save(@"c:\temp\bop_its_tot.dsd2.xml"); message = StructureMessage.Load(@"c:\temp\bop_its_tot.dsd2.xml"); message.Save(@"c:\temp\bop_its_tot.dsd3.xml"); }
public AttributeMap(StructureMessage message) : base(message) { AttributesOrder("conceptRef", "codelist", "attachmentLevel", "assignmentStatus", "isTimeFormat", "crossSectionalAttachmentLevel", "isEntityAttribute", "isNonObservationalTimeAttribute", "isCountAttribute", "isFrequencyAttribute", "isIdentityAttribute"); ElementsOrder("TextFormat", "AttachmentGroup", "AttachmentMeasure", "Annotations"); Map(o => o.AttachementLevel).ToAttribute("attachmentLevel", true) .Set(v => _attribute.AttachementLevel = v) .Converter(new EnumConverter <AttachmentLevel>()); Map(o => o.AssignmentStatus).ToAttribute("assignmentStatus", true) .Set(v => _attribute.AssignmentStatus = v) .Converter(new EnumConverter <AssignmentStatus>()); Map(o => o.IsTimeFormat).ToAttribute("isTimeFormat", false, "false") .Set(v => _attribute.IsTimeFormat = v) .Converter(new BooleanConverter()); Map(o => o.IsEntityAttribute).ToAttribute("isEntityAttribute", false, "false") .Set(v => _attribute.IsEntityAttribute = v) .Converter(new BooleanConverter()); Map(o => o.IsNonObservationalTimeAttribute).ToAttribute("isNonObservationalTimeAttribute", false, "false") .Set(v => _attribute.IsNonObservationalTimeAttribute = v) .Converter(new BooleanConverter()); Map(o => o.IsCountAttribute).ToAttribute("isCountAttribute", false, "false") .Set(v => _attribute.IsCountAttribute = v) .Converter(new BooleanConverter()); Map(o => o.IsFrequencyAttribute).ToAttribute("isFrequencyAttribute", false, "false") .Set(v => _attribute.IsFrequencyAttribute = v) .Converter(new BooleanConverter()); Map(o => o.IsIdentityAttribute).ToAttribute("isIdentityAttribute", false, "false") .Set(v => _attribute.IsIdentityAttribute = v) .Converter(new BooleanConverter()); MapCollection(o => o.AttachmentGroups).ToSimpleElement("AttachmentGroup", false) .Set(v => _attribute.AttachmentGroups.Add(v)) .Converter(new IDConverter()); MapCollection(o => o.AttachmentMeasures).ToSimpleElement("AttachmentMeasure", false) .Set(v => _attribute.AttachmentMeasures.Add(v)) .Converter(new IDConverter()); }
public TimeDimensionMap(StructureMessage message) : base(message) { AttributesOrder("conceptRef", "codelist", "crossSectionalAttachmentLevel"); ElementsOrder("TextFormat", "Annotations"); }
public void WBDSD_HierarchicalCodeList() { // load the DSD string dsdPath = Utility.GetPath("lib\\DSD_WB.xml"); var dsd = StructureMessage.Load(dsdPath); // create hierarchical code list and add it to the DSD var hlist = new HierarchicalCodeList("MDG_Regions", "MDG"); hlist.Name["en"] = "MDG Regions"; dsd.HierarchicalCodeLists.Add(hlist); // get REF_AREA code list from the DSD and add it to the hierarchical code list var refAreaCodeList = dsd.CodeLists.Where(codeList => codeList.Id == "CL_REF_AREA_MDG").Single(); hlist.AddCodeList(refAreaCodeList, "REF_AREA_MDG"); // get parent country code var MDG_DEVELOPED = refAreaCodeList.Where(code => code.Id == "MDG_DEVELOPED").Single(); // child country codes string[] ids = new[] { "ALB", "AND", "AUS", "AUT", "BEL", "BIH", "BMU", "BGR", "HRV", "CAN", "CZE", "DNK", "EST", "FRO", "FIN", "FRA", "DEU", "GRC", "HUN", "ISL", "IRL", "ITA", "JPN", "LVA", "LIE", "LTU", "LUX", "MKD", "MLT", "MCO", "MNE", "NLD", "NZL", "NOR", "POL", "PRT", "ROU", "SVK", "SMR", "SVN", "SRB", "ESP", "SWE", "CHE", "GBR", "USA" }; var countryCodes = (from c in refAreaCodeList join cid in ids on c.Id.ToString() equals cid select c).ToList(); // create hirarchy with parent code and child code and add it to // the hierarchical code list var hierarchy = new Hierarchy("Developed_Countries", new CodeRef(MDG_DEVELOPED, countryCodes)); hierarchy.Name["en"] = "Developed Countries"; hlist.Add(hierarchy); // create another hierarchy and add it to the hierarchical code list var MDG_NAFR = refAreaCodeList.Where(code => code.Id == "MDG_NAFR").Single(); ids = new[] { "DZA", "EGY", "LBY", "MAR", "TUN" }; countryCodes = (from c in refAreaCodeList join cid in ids on c.Id.ToString() equals cid select c).ToList(); hierarchy = new Hierarchy("MDG_NAFR", new CodeRef(MDG_NAFR, () => countryCodes)); hierarchy.Name["en"] = "MDG Northern Africa Countries"; hlist.Add(hierarchy); // save the DSD dsd.Save(Utility.GetPath("lib\\DSD_WB2.xml")); string messageText = dsd.ToString(); var doc = XDocument.Parse(messageText); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); }
void Insert(Stream dataStream, Stream dsdStream, Func <KeyFamily, XmlWriter, DataWriter> createWriter) { var doc = XDocument.Load(dsdStream); doc.Descendants().Where(i => i.Name.LocalName == "TextFormat" && i.Parent.Name.LocalName == "TimeDimension").Single().Remove(); var mngr = new XmlNamespaceManager(new NameTable()); mngr.AddNamespace("structure", "http://www.SDMX.org/resources/SDMXML/schemas/v2_0/structure"); mngr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instanc"); mngr.AddNamespace("xsd", "http://www.w3.org/2001/XMLSchema"); XElement xElementParse = ParseElement("<structure:Concept id='TIME_FORMAT'><structure:Name xml:lang='en'>Time format</structure:Name></structure:Concept>", mngr); doc.Descendants().Where(i => i.Name.LocalName == "ConceptScheme").Single().Add(xElementParse); using (var xReader = doc.CreateReader()) { var structure = StructureMessage.Read(xReader); var keyFamily = structure.KeyFamilies[0]; var data1 = new Dictionary <string, string>(); var data2 = new Dictionary <string, string>(); var settings = new XmlWriterSettings() { Indent = true }; var dataDoc = new XDocument(); using (var reader = DataReader.Create(dataStream, keyFamily)) using (var writer = createWriter(keyFamily, dataDoc.CreateWriter())) //using (var writer = new CompactDataWriter(Utility.GetPath("lib\\aei_ps_alt.sdmx\\aei_ps_alt.sdmx2.xml"), keyFamily, settings, "data", "urn:sdmx:org.sdmx.infomodel.keyfamily.KeyFamily=EUROSTAT:aei_ps_alt_DSD:compact")) { writer.WriteHeader(GetHeader()); while (reader.Read()) { data1.Add(GetData(reader), null); writer.WriteRecord(reader); } } using (var reader = DataReader.Create(dataDoc.CreateReader(), keyFamily)) { while (reader.Read()) { data2.Add(GetData(reader), null); } } Assert.IsTrue(CompareData(data1, data2)); } }
public void convert_format_from_file_to_file() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.Write(reader); } }
public TimeDimensionMap(StructureMessage message) : base(message) { AttributesOrder("conceptRef", "codelist", "crossSectionalAttachmentLevel"); ElementsOrder("TextFormat", "Annotations"); Map(o => o.CrossSectionalAttachmentLevel).ToAttributeGroup("crossSectionalAttachmentLevel", CrossSectionalAttachmentLevel.None) .Set(v => _timeDimension.CrossSectionalAttachmentLevel = v) .GroupTypeMap(new CrossSectionalAttachmentLevelMap()); }
internal MeasureMap(StructureMessage message) { Map(o => TempConceptRef.Create(o.Concept)).ToAttributeGroup("conceptRef") .Set(v => _measure = Create(message.GetConcept(v.ID, v.AgencyID, v.Version))) .GroupTypeMap(new TempConceptRefMap()); Map(o => TempCodelistRef.Create(o.CodeList)).ToAttributeGroup("codelist") .Set(v => _measure.CodeList = message.GetCodeList(v.ID, v.AgencyID, v.Version)) .GroupTypeMap(new TempCodelistRefMap()); Map(o => o.TextFormat).ToElement("TextFormat", false) .Set(v => _measure.TextFormat = v) .ClassMap(() => new TextFormatMap()); }
public void Invalid_structure_missing_codelist() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var doc = XDocument.Load(dsdPath); var series = doc.Descendants().Where(i => i.Name.LocalName == "CodeList" && i.Attribute("id").Value == "CL_FREQ").First(); series.Remove(); var errorList = new List <ValidationMessage>(); var message = StructureMessage.Read(doc.CreateReader(), v => errorList.Add(v)); Assert.AreEqual(1, errorList.Count); }
public void StructureSampleTest() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var message = StructureMessage.Load(dsdPath); //message.Save(@"c:\temp\StructureSample2.xml"); var doc = new XDocument(); using (var writer = doc.CreateWriter()) message.Write(writer); using (var reader = doc.CreateReader()) Assert.IsTrue(MessageValidator.ValidateXml(reader)); }
public ComponentMap(StructureMessage message) { Map(o => ConceptRef.Create(o.Concept)).ToAttributeGroup("conceptRef") .Set(v => _component = Create(GetConcept(message, v))) .GroupTypeMap(new ConceptRefMap()); Map(o => TempCodelistRef.Create(o.CodeList)).ToAttributeGroup("codelist") .Set(v => _component.CodeList = GetCodeList(message, v)) .GroupTypeMap(new TempCodelistRefMap()); Map(o => GetTextFormat(o)).ToElement("TextFormat", false) .Set(v => SetTextFormat(v)) .ClassMap(() => new TextFormatMap()); }
public void read_from_database_into_file_quit_if_notValid() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; IDataReader reader = null; using (reader) using (var writer = new CompactDataWriter("path", keyFamily, "uis", "ns")) { writer.WriteHeader(null); writer.Write(reader); } }
public void Invalid_structure_duplicate_concept() { string dsdPath = Utility.GetPath("lib\\StructureSample.xml"); var doc = XDocument.Load(dsdPath); var series = doc.Descendants().Where(i => i.Name.LocalName == "Concept" && i.Attribute("id").Value == "TIME").First(); var copy = new XElement(series); series.AddAfterSelf(copy); var errorList = new List <ValidationMessage>(); var message = StructureMessage.Read(doc.CreateReader(), v => errorList.Add(v)); Assert.AreEqual(1, errorList.Count); }
public void CreateCodeList() { var codelist = new CodeList(new InternationalString("en", "Countries"), "CL_COUNTRY", "UIS"); var code = new Code("CAN"); code.Description["en"] = "Canada"; codelist.Add(code); code = new Code("USA"); code.Description["en"] = "United States of America"; codelist.Add(code); var message = new StructureMessage(); message.Header = new Header("MSD_HDR", new Party("UIS")) { Prepared = DateTime.Now }; message.CodeLists.Add(codelist); message.Save("CL_COUNTRY.xml"); }
public void read_file_into_db_quit_if_file_not_valid() { var structure = StructureMessage.Load(Utility.GetPath("lib\\StructureSample.xml")); string dataPath = Utility.GetPath("lib\\GenericSample.xml"); var keyFamily = structure.KeyFamilies[0]; using (var reader = DataReader.Create(dataPath, keyFamily)) { reader.Map("TIME", "TIME", i => i.ToString()); using (var bulkCopy = new SqlBulkCopy(_connectionString)) { bulkCopy.DestinationTableName = "DestinationTableName"; bulkCopy.WriteToServer(reader); } } }