public void go(String[] args) { int passed = 0; int failed = 0; int total = 0; Console.WriteLine("Testing Saxon " + processor.ProductVersion); testSuiteDir = args[0]; if (testSuiteDir.EndsWith("/")) { testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1); } Hashtable exceptions = new Hashtable(); if (args.Length > 1) { testPattern = (args[1]); } for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-w")) { //showWarnings = true; } } try { schemaAwareProcessor = new Processor(true); } catch (Exception err) { Console.WriteLine("Cannot load Saxon-SA: continuing without it"); } processor.SetProperty("http://saxon.sf.net/feature/preferJaxpParser", "true"); if (schemaAwareProcessor != null) { schemaAwareProcessor.SetProperty("http://saxon.sf.net/feature/preferJaxpParser", "true"); } fileComparer = new FileComparer(processor, testSuiteDir); String testURI = "http://www.w3.org/2005/05/xslt20-test-catalog"; QName testCaseNT = new QName(testURI, "testcase"); QName nameNT = new QName(testURI, "name"); QName inputNT = new QName(testURI, "input"); QName outputNT = new QName(testURI, "output"); QName stylesheetNT = new QName(testURI, "stylesheet"); QName schemaNT = new QName(testURI, "schema"); QName initialModeNT = new QName(testURI, "initial-mode"); QName entryNamedTemplateNT = new QName(testURI, "entry-named-template"); QName sourceDocumentNT = new QName(testURI, "source-document"); QName stylesheetParametersNT = new QName(testURI, "stylesheet-parameters"); QName paramNT = new QName(testURI, "param"); QName resultDocumentNT = new QName(testURI, "result-document"); QName errorNT = new QName(testURI, "error"); QName validationNT = new QName(testURI, "validation"); QName discretionaryItemsNT = new QName(testURI, "discretionary-items"); QName discretionaryFeatureNT = new QName(testURI, "discretionary-feature"); QName discretionaryChoiceNT = new QName(testURI, "discretionary-choice"); QName initialContextNodeNT = new QName(testURI, "initial-context-node"); QName fileAtt = new QName("", "file"); QName errorIdAtt = new QName("", "error-id"); QName typeAtt = new QName("", "type"); QName nameAtt = new QName("", "name"); QName behaviorAtt = new QName("", "behavior"); QName qnameAtt = new QName("", "qname"); QName modeAtt = new QName("", "mode"); QName validatesAtt = new QName("", "validates"); QName roleAtt = new QName("", "role"); DocumentBuilder builder = processor.NewDocumentBuilder(); XdmNode exceptionsDoc = builder.Build(new Uri(testSuiteDir + '/' + getResultDirectoryName() + "/exceptions.xml")); IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, testCaseNT); while (exceptionTestCases.MoveNext()) { XdmNode n = (XdmNode)exceptionTestCases.Current; String nameAttVal = n.GetAttributeValue(nameAtt); char[] seps = { ' ', '\n', '\t' }; String[] parts = nameAttVal.Split(seps); foreach (string p in parts) { if (!exceptions.ContainsKey(p)) { exceptions.Add(p, "Kilroy"); } } } XdmNode catalog = builder.Build(new Uri(testSuiteDir + "/catalog.xml")); results = new StreamWriter(testSuiteDir + "/SaxonResults.net/results" + processor.ProductVersion + ".xml"); results.WriteLine("<test-suite-result>"); results.WriteLine(" <implementation name='Saxon-SA' version='" + processor.ProductVersion + "' anonymous-result-column='false'>"); results.WriteLine(" <organization name='http://www.saxonica.com/' anonymous='false'/>"); results.WriteLine(" <submitter name='Michael Kay' email='*****@*****.**'/>"); outputDiscretionaryItems(); results.WriteLine(" </implementation>"); total = 0; IEnumerator testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT); while (testCases.MoveNext()) { total++; } testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT); while (testCases.MoveNext()) { bool useAssociated = false; XdmNode testCase = (XdmNode)testCases.Current; String testName = getChildElement(testCase, nameNT).StringValue; if (testPattern != null && !testName.StartsWith(testPattern)) { continue; } if (exceptions.ContainsKey(testName)) { continue; } if (isExcluded(testName)) { continue; } Console.WriteLine("Test " + testName); XdmNode testInput = getChildElement(testCase, inputNT); XdmNode stylesheet = getChildElement(testInput, stylesheetNT); String absXSLName = null; if (stylesheet == null) { useAssociated = true; } else { absXSLName = testSuiteDir + "/TestInputs/" + stylesheet.GetAttributeValue(fileAtt); } XdmNode sourceDocument = getChildElement(testInput, sourceDocumentNT); String absXMLName = null; if (sourceDocument != null) { absXMLName = testSuiteDir + "/TestInputs/" + sourceDocument.GetAttributeValue(fileAtt); } bool schemaAware = false; bool recoverRecoverable = true; bool backwardsCompatibility = true; bool supportsDOE = true; bool recoverSESU0007 = false; XdmNode discretionaryItems = getChildElement(testCase, discretionaryItemsNT); if (discretionaryItems != null) { IEnumerator features = discretionaryItems.EnumerateAxis(XdmAxis.Child, discretionaryFeatureNT); while (features.MoveNext()) { XdmNode feature = (XdmNode)features.Current; String featureName = feature.GetAttributeValue(nameAtt); if ("schema_aware".Equals(featureName) || "Saxon-PE".Equals(featureName)) // TODO: test Saxon-PE properly { schemaAware = "on".Equals(feature.GetAttributeValue(behaviorAtt)); } else if ("XML_1.1".Equals(featureName)) { xml11 = "on".Equals(feature.GetAttributeValue(behaviorAtt)); } else if ("backwards_compatibility".Equals(featureName)) { backwardsCompatibility = "on".Equals(feature.GetAttributeValue(behaviorAtt)); } else if ("disabling_output_escaping".Equals(featureName)) { supportsDOE = "on".Equals(feature.GetAttributeValue(behaviorAtt)); } } IEnumerator choices = discretionaryItems.EnumerateAxis( XdmAxis.Child, discretionaryChoiceNT); while (choices.MoveNext()) { XdmNode choice = (XdmNode)choices.Current; String featureName = choice.GetAttributeValue(nameAtt); if ("error".Equals(choice.GetAttributeValue(behaviorAtt))) { recoverRecoverable = false; } else if ("SESU0007".Equals(featureName)) { recoverSESU0007 = "recovery".Equals(choice.GetAttributeValue(behaviorAtt)); } } } if (!backwardsCompatibility) { // Saxon cannot run with BC switched off results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires backwards-compatibility=off'/>"); continue; } if (!supportsDOE) { // Saxon cannot run with DOE switched off results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires disable-output-escaping=off'/>"); continue; } if (recoverSESU0007) { // Saxon cannot recover from error SESU0007 results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires recovery from error SESU0007'/>"); continue; } XdmNode initialMode = getChildElement(testInput, initialModeNT); QName initialModeName = null; if (initialMode != null) { String ini = initialMode.GetAttributeValue(qnameAtt); initialModeName = makeQName(ini, initialMode); } XdmNode initialTemplate = getChildElement(testInput, entryNamedTemplateNT); QName initialTemplateName = null; if (initialTemplate != null) { String ini = initialTemplate.GetAttributeValue(qnameAtt); initialTemplateName = makeQName(ini, initialTemplate); } XdmNode initialContextNode = getChildElement(testInput, initialContextNodeNT); String initialContextPath = null; if (initialContextNode != null) { initialContextPath = initialContextNode.StringValue; } XdmNode validation = getChildElement(testInput, validationNT); String validationMode = null; if (validation != null) { validationMode = validation.GetAttributeValue(modeAtt); } Hashtable paramTable = null; XdmNode paramList = getChildElement(testInput, stylesheetParametersNT); if (paramList != null) { paramTable = new Hashtable(5); IEnumerator paramIter = paramList.EnumerateAxis(XdmAxis.Child, paramNT); while (paramIter.MoveNext()) { XdmNode param = (XdmNode)paramIter.Current; QName name = makeQName(param.GetAttributeValue(qnameAtt), param); String value = param.StringValue; paramTable.Add(name, value); } } IEnumerator schemas = testInput.EnumerateAxis(XdmAxis.Child, schemaNT); while (schemas.MoveNext()) { XdmNode schema = (XdmNode)schemas.Current; if (schema == null) { break; } schemaAware = true; String role = schema.GetAttributeValue(roleAtt); if (("source-validator".Equals(role) || "source-reference".Equals(role)) /* && schema.GetAttributeValue(validatesAtt) != null */) { validationMode = "strict"; // TODO: control which source documents are validated... } } XdmNode testOutput = getChildElement(testCase, outputNT); XdmNode resultDocument = getChildElement(testOutput, resultDocumentNT); // TODO: handle alternative result documents String refFileName = null; String outFileName; String comparator = "xml"; if (resultDocument != null) { String relativePath = resultDocument.GetAttributeValue(fileAtt); int slash = relativePath.IndexOf('/'); if (slash > 0) { String relativeDir = relativePath.Substring(0, slash); String fullDir = testSuiteDir + '/' + getResultDirectoryName() + "/" + relativeDir; if (!Directory.Exists(fullDir)) { Directory.CreateDirectory(fullDir); } } refFileName = testSuiteDir + "/ExpectedTestResults/" + relativePath; outFileName = testSuiteDir + '/' + getResultDirectoryName() + "/" + relativePath; comparator = resultDocument.GetAttributeValue(typeAtt); } else { outFileName = testSuiteDir + '/' + getResultDirectoryName() + "/temp.out"; } XdmNode error = getChildElement(testOutput, errorNT); String expectedError = null; if (error != null) { expectedError = error.GetAttributeValue(errorIdAtt); } bool success; Exception xsltOutcome = runXSLT(testName, absXMLName, absXSLName, initialModeName, initialTemplateName, outFileName, paramTable, initialContextPath, useAssociated, schemaAware, validationMode, recoverRecoverable); if (xsltOutcome == null) { success = true; if (expectedError != null && resultDocument == null) { Console.WriteLine("Test failed. Expected error " + expectedError + ", got success"); feedback.Feedback(passed, failed++, total); success = false; results.WriteLine(" <testcase name='" + testName + "' result='differ' comment='Expected error " + expectedError + ", got success'/>"); } else { feedback.Feedback(passed++, failed, total); } } else { String errorCode = null; if (xsltOutcome is StaticError) { errorCode = ((StaticError)xsltOutcome).ErrorCode.LocalName; } else if (xsltOutcome is DynamicError) { errorCode = ((DynamicError)xsltOutcome).ErrorCode.LocalName; } if (expectedError != null && errorCode != null && errorCode.Equals(expectedError)) { feedback.Feedback(passed++, failed, total); Console.WriteLine("Test succeeded (" + expectedError + ')'); results.WriteLine(" <testcase name='" + testName + "' result='full' comment='Error " + expectedError + " as expected'/>"); } else if (expectedError != null) { feedback.Feedback(passed++, failed, total); Console.WriteLine("Test succeeded (??) (expected " + expectedError + ", got " + errorCode + ')'); results.WriteLine(" <testcase name='" + testName + "' result='different-error' comment='Expected " + expectedError + " got " + errorCode + "'/>"); } else { feedback.Feedback(passed, failed++, total); Console.WriteLine("Test failed. Expected success, got " + errorCode); results.WriteLine(" <testcase name='" + testName + "' result='differ' comment='Expected success, got " + errorCode + "'/>"); results.WriteLine(" <!--" + xsltOutcome.Message + "-->"); } success = false; continue; } if (success) { String outcome = fileComparer.compare(outFileName, refFileName, comparator); if (outcome == "OK") { results.WriteLine(" <testcase name='" + testName + "' result='full'/>"); } else if (outcome.StartsWith("#")) { results.WriteLine(" <testcase name='" + testName + "' result='full' + comments='" + outcome.Substring(1) + "/>"); } else { results.WriteLine(" <testcase name='" + testName + "' result='differ' comments='" + outcome + "'/>"); } } } results.WriteLine("</test-suite-result>"); results.Close(); //} }
/** * Run the tests * @param args command line arguments * @throws SAXException * @throws ParserConfigurationException * @throws XPathException * @throws IOException * @throws URISyntaxException */ public void go(String[] args) { if (args.Length == 0 || args[0] == "-?") { Console.WriteLine("SchemaTestSuiteDriver testDir [-w] [-onwards] -c:contributor? -s:setName? -g:groupName?"); } Processor processor = new Processor(true); Console.WriteLine("Testing Saxon " + processor.ProductVersion); testSuiteDir = args[0]; if (testSuiteDir.EndsWith("/")) { testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1); } String testSetPattern = null; // TODO use a regex String testGroupPattern = null; String contributor = null; Hashtable exceptions = new Hashtable(); for (int i = 1; i < args.Length; i++) { if (args[i] == ("-w")) { //showWarnings = true; } else if (args[i] == ("-onwards")) { onwards = true; } else if (args[i].StartsWith("-c:")) { contributor = args[i].Substring(3); } else if (args[i].StartsWith("-s:")) { testSetPattern = args[i].Substring(3); } else if (args[i].StartsWith("-g:")) { testGroupPattern = args[i].Substring(3); } else if (args[i] == "-?") { Console.WriteLine("Usage: SchemaTestSuiteDriver testDir [-w] [-s:testSetPattern] [-g:testGroupPattern]"); } } int total = 39700; int passed = 0; int failed = 0; try { xlinkHref = new QName("xlink", "http://www.w3.org/1999/xlink", "href"); QName testCaseNT = new QName("", "", "testcase"); QName commentNT = new QName("", "", "comment"); QName testSetRefNT = new QName(testNS, "testSetRef"); QName testGroupNT = new QName(testNS, "testGroup"); QName testSetNT = new QName(testNS, "testSet"); QName schemaTestNT = new QName(testNS, "schemaTest"); QName instanceTestNT = new QName(testNS, "instanceTest"); QName schemaDocumentNT = new QName(testNS, "schemaDocument"); QName instanceDocumentNT = new QName(testNS, "instanceDocument"); QName expectedNT = new QName(testNS, "expected"); QName currentNT = new QName(testNS, "current"); QName validityAtt = new QName("", "", "validity"); QName nameAtt = new QName("", "", "name"); QName contributorAtt = new QName("", "", "contributor"); QName setAtt = new QName("", "", "set"); QName groupAtt = new QName("", "", "group"); QName statusAtt = new QName("", "", "status"); QName bugzillaAtt = new QName("", "", "bugzilla"); QName targetNamespaceAtt = new QName("", "", "targetNamespace"); QName schemaVersion = new QName("saxon", "http://saxon.sf.net/", "schemaVersion"); DocumentBuilder builder = processor.NewDocumentBuilder(); builder.BaseUri = new Uri(testSuiteDir + "/"); XdmNode catalog = builder.Build( new FileStream(testSuiteDir + "/suite.xml", FileMode.Open, FileAccess.Read, FileShare.Read)); results = new StreamWriter(testSuiteDir + "/saxon/SaxonResults" + processor.ProductVersion + "n.xml"); results.Write("<testSuiteResults xmlns='" + testNS + "' xmlns:saxon='http://saxon.sf.net/' " + "suite='TS_2006' " + "processor='Saxon-SA (Java) 8.8++' submitDate='2007-01-05' publicationPermission='public'>\n"); XdmNode exceptionsDoc = builder.Build( new FileStream(testSuiteDir + "/saxon/exceptions.xml", FileMode.Open, FileAccess.Read, FileShare.Read)); IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, new QName("", "testcase")); while (exceptionTestCases.MoveNext()) { XdmNode testCase = (XdmNode)exceptionTestCases.Current; String set = testCase.GetAttributeValue(setAtt); String group = testCase.GetAttributeValue(groupAtt); String comment = getChildElement(testCase, commentNT).StringValue; exceptions[set + "#" + group] = comment; } IEnumerator testSets = catalog.EnumerateAxis(XdmAxis.Descendant, testSetRefNT); while (testSets.MoveNext()) { XdmNode testSetRef = (XdmNode)testSets.Current; XdmNode testSetDoc = getLinkedDocument(testSetRef, processor, false); XdmNode testSetElement = getChildElement(testSetDoc, testSetNT); if (testSetElement == null) { feedback.Message("test set doc has no TestSet child: " + testSetDoc.BaseUri, true); continue; } String testSetName = testSetElement.GetAttributeValue(nameAtt); if (testSetPattern != null && !testSetName.StartsWith(testSetPattern)) { continue; } if (contributor != null && contributor != testSetElement.GetAttributeValue(contributorAtt)) { continue; } bool needs11 = (testSetElement.GetAttributeValue(schemaVersion) == "1.1"); IEnumerator testGroups = testSetElement.EnumerateAxis(XdmAxis.Child, testGroupNT); while (testGroups.MoveNext()) { XdmNode testGroup = (XdmNode)testGroups.Current; String testGroupName = testGroup.GetAttributeValue(nameAtt); String exception = (String)exceptions[testSetName + "#" + testGroupName]; if (testGroupPattern != null && !testGroupName.StartsWith(testGroupPattern)) { continue; } Console.WriteLine("TEST SET " + testSetName + " GROUP " + testGroupName, false); if (onwards) { testGroupPattern = null; testSetPattern = null; } Processor testConfig = new Processor(true); if (needs11) { testConfig.SetProperty("http://saxon.sf.net/feature/xsd-version", "1.1"); } SchemaManager schemaManager = testConfig.SchemaManager; //testConfig.setHostLanguage(Configuration.XML_SCHEMA); testConfig.SetProperty("http://saxon.sf.net/feature/validation-warnings", "true"); IEnumerator schemaTests = testGroup.EnumerateAxis(XdmAxis.Child, schemaTestNT); bool schemaQueried = false; String bugzillaRef = null; while (schemaTests.MoveNext()) { XdmNode schemaTest = (XdmNode)schemaTests.Current; if (schemaTest == null) { break; } bugzillaRef = null; String testName = schemaTest.GetAttributeValue(nameAtt); if (exception != null) { results.Write("<testResult set='" + testSetName + "' group='" + testGroupName + "' test='" + testName + "' validity='notKnown' saxon:outcome='notRun' saxon:comment='" + exception + "'/>\n"); continue; } bool queried = false; XdmNode statusElement = getChildElement(schemaTest, currentNT); if (statusElement != null) { String status = statusElement.GetAttributeValue(statusAtt); queried = ("queried" == status); bugzillaRef = statusElement.GetAttributeValue(bugzillaAtt); } if (queried) { schemaQueried = true; } Console.WriteLine("TEST SCHEMA " + testName + (queried ? " (queried)" : "")); bool success = true; IEnumerator schemata = schemaTest.EnumerateAxis(XdmAxis.Child, schemaDocumentNT); while (schemata.MoveNext()) { XdmNode schemaDocumentRef = (XdmNode)schemata.Current; if (schemaDocumentRef == null) { break; } Console.WriteLine("Loading schema at " + schemaDocumentRef.GetAttributeValue(xlinkHref)); XdmNode schemaDoc = getLinkedDocument(schemaDocumentRef, testConfig, false); IEnumerator schemaDocKids = schemaDoc.EnumerateAxis(XdmAxis.Child); XdmNode schemaElement = null; while (schemaDocKids.MoveNext()) { schemaElement = (XdmNode)schemaDocKids.Current; if (schemaElement.NodeKind == XmlNodeType.Element) { break; } } String targetNamespace = schemaElement.GetAttributeValue(targetNamespaceAtt); //if (targetNamespace != null && schemaManager. isSchemaAvailable(targetNamespace)) { // do nothing // TODO: this is the only way I can get MS additional test addB132 to work. // It's not ideal: addSchemaSource() ought to be a no-op if the schema components // are already loaded, but in fact recompiling the imported schema document on its // own is losing the substitution group membership that was defined in the // importing document. //} else { IList <StaticError> errorList = new List <StaticError>(); schemaManager.ErrorList = errorList; try { schemaManager.Compile(schemaDoc); } catch (Exception e) { if (errorList.Count == 0) { feedback.Message("In " + testName + ", exception thrown but no errors in ErrorList\n", true); results.Write("<!--" + e.Message + "-->"); success = false; } } for (int i = 0; i < errorList.Count; i++) { if (errorList[i] is StaticError) { StaticError err = (StaticError)errorList[i]; if (!err.IsWarning) { success = false; break; } } else { feedback.Message("In " + testName + " wrong kind of error!" + errorList[i].GetType() + "\n", true); } } } XdmNode expected = getChildElement(schemaTest, expectedNT); bool expectedSuccess = expected == null || expected.GetAttributeValue(validityAtt) == "valid"; if (success == expectedSuccess) { passed++; } else { failed++; } feedback.Feedback(passed, failed, total); results.Write("<testResult set='" + testSetName + "' group='" + testGroupName + "' test='" + testName + "' validity='" + (success ? "valid" : "invalid") + (queried ? "' saxon:queried='true' saxon:bugzilla='" + bugzillaRef : "") + "' saxon:outcome='" + (success == expectedSuccess ? "same" : "different") + "'/>\n"); } IEnumerator instanceTests = testGroup.EnumerateAxis(XdmAxis.Child, instanceTestNT); while (instanceTests.MoveNext()) { XdmNode instanceTest = (XdmNode)instanceTests.Current; String testName = instanceTest.GetAttributeValue(nameAtt); if (exception != null) { results.Write("<testResult set='" + testSetName + "' group='" + testGroupName + "' test='" + testName + "' validity='notKnown' saxon:outcome='notRun' saxon:comment='" + exception + "'/>\n"); continue; } bool queried = false; XdmNode statusElement = getChildElement(instanceTest, currentNT); if (statusElement != null) { String status = statusElement.GetAttributeValue(statusAtt); queried = ("queried" == status); String instanceBug = statusElement.GetAttributeValue(bugzillaAtt); if (instanceBug != null) { bugzillaRef = instanceBug; } } queried |= schemaQueried; Console.WriteLine("TEST INSTANCE " + testName + (queried ? " (queried)" : "")); XdmNode instanceDocument = getChildElement(instanceTest, instanceDocumentNT); bool success = true; try { XdmNode instanceDoc = getLinkedDocument(instanceDocument, testConfig, true); } catch (Exception) { success = false; } XdmNode expected = getChildElement(instanceTest, expectedNT); bool expectedSuccess = expected == null || expected.GetAttributeValue(validityAtt) == "valid"; if (success == expectedSuccess) { passed++; } else { failed++; } feedback.Feedback(passed, failed, total); results.Write("<testResult set='" + testSetName + "' group='" + testGroupName + "' test='" + testName + "' validity='" + (success ? "valid" : "invalid") + (queried ? "' saxon:queried='true' saxon:bugzilla='" + bugzillaRef : "") + "' saxon:outcome='" + (success == expectedSuccess ? "same" : "different") + "'/>\n"); } } } results.Write("</testSuiteResults>"); results.Close(); } catch (Exception e) { feedback.Message("Test failed: " + e.Message, true); } }