/** * Run a test case * * * @param testCase the test case element in the catalog * @param xpc the XPath compiler to be used for compiling XPath expressions against the catalog * @ */ protected override void runTestCase(XdmNode testCase, XPathCompiler xpc) { bool run = true; bool xpDependency = false; string hostLang; string langVersion; Spec specOpt = Spec.NULL; XPathCompiler xpath = driverProc.NewXPathCompiler(); string testCaseName = testCase.GetAttributeValue(new QName("name")); string testSetName = testCase.Parent.GetAttributeValue(new QName("name")); bool needSerializedResult = ((XdmAtomicValue)xpc.EvaluateSingle( "exists(./result//assert-serialization-error) or exists(./result//serialization-matches)", testCase)).GetBooleanValue(); bool needResultValue = true; if (needSerializedResult) { needResultValue = ((XdmAtomicValue)xpc.EvaluateSingle( "exists(./result//*[not(self::serialization-matches or self::assert-serialization-error or self::any-of or self::all-of)])", testCase)).GetBooleanValue(); } XdmNode alternativeResult = null; XdmNode optimization = null; hostLang = ((SpecAttr)spec.GetAttr()).sname; langVersion = ((SpecAttr)spec.GetAttr()).version; Environment env = getEnvironment(testCase, xpc); if (env == null) { notrun++; return; } env.xpathCompiler.BackwardsCompatible = false; env.processor.XmlVersion = (decimal)1.0; foreach (XdmItem dependency in xpc.Evaluate("/*/dependency, ./dependency", testCase)) { string type = ((XdmNode)dependency).GetAttributeValue(new QName("type")); if (type == null) { // throw new IllegalStateException("dependency/@type is missing"); //TODO } string value = ((XdmNode)dependency).GetAttributeValue(new QName("value")); if (value == null) { //throw new IllegalStateException("dependency/@value is missing"); //TODO } if (type.Equals("spec")) { bool applicable = false; if (!value.Contains(((SpecAttr)spec.GetAttr()).sname)) { applicable = false; } else if (value.Contains(((SpecAttr)spec.GetAttr()).svname)) { applicable = true; } else if (((SpecAttr)spec.GetAttr()).svname.Equals("XQ30") && value.Contains("XQ10+")) { applicable = true; } else if (((SpecAttr)spec.GetAttr()).svname.Equals("XP30") && value.Contains("XP20+")) { applicable = true; } if (!applicable) { writeTestcaseElement(testCaseName, "n/a", "not" + ((SpecAttr)spec.GetAttr()).svname, spec); notrun++; return; } } if (langVersion.Equals("3.0")) { /* EnvironmentVariableResolver resolver = new EnvironmentVariableResolver() { * public Set<string> getAvailableEnvironmentVariables() { * Set<string> strings = new HashSet<string>(); * strings.add("QTTEST"); * strings.add("QTTEST2"); * strings.add("QTTESTEMPTY"); * return strings; * } * * public string getEnvironmentVariable(string name) { * if (name.Equals("QTTEST")) { * return "42"; * } else if (name.Equals("QTTEST2")) { * return "other"; * } else if (name.Equals("QTTESTEMPTY")) { * return ""; * } else { * return null; * } * } * }; */ //TODO //env.processor.SetProperty(JFeatureKeys.ENVIRONMENT_VARIABLE_RESOLVER, resolver); } if (type.Equals("feature") && value.Equals("xpath-1.0-compatibility")) { hostLang = "XP"; langVersion = "3.0"; xpDependency = true; specOpt = Spec.XP30; } if (type.Equals("feature") && value.Equals("namespace-axis")) { hostLang = "XP"; langVersion = "3.0"; xpDependency = true; specOpt = Spec.XP30; } if (!dependencyIsSatisfied((XdmNode)dependency, env)) { println("*** Dependency not satisfied: " + ((XdmNode)dependency).GetAttributeValue(new QName("type"))); writeTestcaseElement(testCaseName, "n/a", "Dependency not satisfied", spec); run = false; notrun++; return; } } XdmNode exceptionElement; try{ exceptionElement = exceptionsMap[testCaseName]; } catch (Exception) { exceptionElement = null; } if (exceptionElement != null) { XdmItem config = xpath.EvaluateSingle("configuration", exceptionElement); string runAtt = exceptionElement.GetAttributeValue(new QName("run")); string reasonMsg = xpath.EvaluateSingle("reason", exceptionElement).ToString(); string reportAtt = exceptionElement.GetAttributeValue(new QName("report")); if (config != null) { XdmItem paramValue = xpath.EvaluateSingle("param[@name='not-unfolded' and @value='yes']/@name", config); if (unfolded && paramValue != null) { writeTestcaseElement(testCaseName, "notRun", reasonMsg, spec); notrun++; return; } } if ("false".Equals(runAtt)) { writeTestcaseElement(testCaseName, reportAtt, reasonMsg, spec); notrun++; return; } alternativeResult = (XdmNode)xpc.EvaluateSingle("result", exceptionElement); optimization = (XdmNode)xpc.EvaluateSingle("optimization", exceptionElement); } if (run && (specOpt == Spec.NULL || specOpt == spec)) { TestOutcome outcome = new TestOutcome(this); string exp = null; try { exp = xpc.Evaluate("if (test/@file) then unparsed-text(resolve-uri(test/@file, base-uri(.))) else string(test)", testCase).ToString(); } catch (Exception err) { println("*** Failed to read query: " + err.Message); outcome.SetException((DynamicError)err); } //noinspection ThrowableResultOfMethodCallIgnored if (outcome.GetException() == null) { if (hostLang.Equals("XP") || hostLang.Equals("XT")) { XPathCompiler testXpc = env.xpathCompiler; testXpc.XPathLanguageVersion = langVersion; testXpc.DeclareNamespace("fn", "http://www.w3.org/2005/xpath-functions"); testXpc.DeclareNamespace("xs", "http://www.w3.org/2001/XMLSchema"); testXpc.DeclareNamespace("map", "http://www.w3.org/2005/xpath-functions/map"); //testXpc.DeclareNamespace("math", NamespaceConstant.MATH); //testXpc.DeclareNamespace("Dictionary", NamespaceConstant.Dictionary_FUNCTIONS); try { XPathSelector selector = testXpc.Compile(exp).Load(); foreach (QName varName in env.params1.Keys) { selector.SetVariable(varName, env.params1[varName]); } if (env.contextItem != null) { selector.ContextItem = env.contextItem; } selector.InputXmlResolver = new TestUriResolver(env); if (env.unparsedTextResolver != null) { //selector.getUnderlyingXPathContext().setUnparsedTextURIResolver(env.unparsedTextResolver); //TODO } XdmValue result = selector.Evaluate(); outcome.SetPrincipalResult(result); } catch (Exception err) { println(err.Message); outcome.SetException(err); } } else if (hostLang.Equals("XQ")) { XQueryCompiler testXqc = env.xqueryCompiler; testXqc.XQueryLanguageVersion = langVersion; testXqc.DeclareNamespace("fn", "http://www.w3.org/2005/xpath-functions"); testXqc.DeclareNamespace("xs", "http://www.w3.org/2001/XMLSchema"); //testXqc.DeclareNamespace("math", NamespaceConstant.MATH); testXqc.DeclareNamespace("map", "http://www.w3.org/2005/xpath-functions/map"); // ErrorCollector errorCollector = new ErrorCollector(); testXqc.ErrorList = new ArrayList(); string decVars = env.paramDecimalDeclarations.ToString(); if (decVars.Length != 0) { int x = (exp.IndexOf("(:%DECL%:)")); if (x < 0) { exp = decVars + exp; } else { exp = exp.Substring(0, x) + decVars + exp.Substring(x + 13); } } string vars = env.paramDeclarations.ToString(); if (vars.Length != 0) { int x = (exp.IndexOf("(:%VARDECL%:)")); if (x < 0) { exp = vars + exp; } else { exp = exp.Substring(0, x) + vars + exp.Substring(x + 13); } } // ModuleResolver mr = new ModuleResolver(xpc); // mr.setTestCase(testCase); // testXqc.QueryResolver = mr;// .setModuleURIResolver(mr); //TODO try { XQueryExecutable q = testXqc.Compile(exp); if (optimization != null) { // Test whether required optimizations have been performed XdmDestination expDest = new XdmDestination(); JConfiguration config = driverProc.Implementation; //ExpressionPresenter presenter = new ExpressionPresenter(config, expDest.getReceiver(config)); //q.getUnderlyingCompiledQuery().explain(presenter); //presenter.close(); XdmNode explanation = expDest.XdmNode; XdmItem optResult = xpc.EvaluateSingle(optimization.GetAttributeValue(new QName("assert")), explanation); if (((XdmAtomicValue)optResult).GetBooleanValue()) { println("Optimization result OK"); } else { println("Failed optimization test"); Serializer ser = new Serializer(); ser.SetOutputStream((Stream)System.Console.OpenStandardError()); driverProc.WriteXdmValue(explanation, ser); writeTestcaseElement(testCaseName, "fail", "Failed optimization assertions", spec); failures++; return; } } XQueryEvaluator selector = q.Load(); foreach (QName varName in env.params1.Keys) { selector.SetExternalVariable(varName, env.params1[varName]); } if (env.contextItem != null) { selector.ContextItem = env.contextItem; } //selector.InputXmlResolver = .SetURIResolver(new TestURIResolver(env)); //TODO if (env.unparsedTextResolver != null) { //selector.Implementation.setURIResolver(env.unparsedTextResolver);// TODO } if (needSerializedResult) { StringWriter sw = new StringWriter(); Serializer serializer = new Serializer(); //env.processor.NewSerializer(sw); //TODO serializer.SetOutputWriter(sw); selector.Run(serializer); outcome.SetPrincipalSerializedResult(sw.ToString()); } if (needResultValue) { XdmValue result = selector.Evaluate(); outcome.SetPrincipalResult(result); } } catch (Exception err) { println("in TestSet " + testSetName); println(err.Message); outcome.SetException(err); outcome.SetErrorsReported((IList)testXqc.ErrorList); } } else { writeTestcaseElement(testCaseName, "notRun", "No processor found", spec); notrun++; return; } } if (env.resetAction != null) { env.resetAction.reset(env); } XdmNode assertion; if (alternativeResult != null) { assertion = (XdmNode)xpc.EvaluateSingle("*[1]", alternativeResult); } else { assertion = (XdmNode)xpc.EvaluateSingle("result/*[1]", testCase); } if (assertion == null) { println("*** No assertions found for test case " + testCaseName); writeTestcaseElement(testCaseName, "disputed", "No assertions in test case", spec); feedback.Feedback(successes, failures++, total); return; } XPathCompiler assertXpc = env.processor.NewXPathCompiler(); assertXpc.XPathLanguageVersion = "3.0"; assertXpc.DeclareNamespace("fn", "http://www.w3.org/2005/xpath-functions"); assertXpc.DeclareNamespace("xs", "http://www.w3.org/2001/XMLSchema"); assertXpc.DeclareNamespace("math", "http://www.w3.org/2005/xpath-functions/math"); assertXpc.DeclareNamespace("MAP_FUNCTIONS", "http://www.w3.org/2005/xpath-functions/map"); assertXpc.DeclareVariable(new QName("result")); bool b = outcome.TestAssertion(assertion, outcome.GetPrincipalResultDoc(), assertXpc, xpc, debug); if (b) { //println("OK"); writeTestcaseElement(testCaseName, "pass", null, spec); feedback.Feedback(successes++, failures, total); } else { if (outcome.IsException()) { XdmItem expectedError = xpc.EvaluateSingle("result//error/@code", testCase); if (expectedError == null) { // if (debug) { // outcome.getException().printStackTrace(System.out); // } writeTestcaseElement(testCaseName, "fail", "Expected success, got " + outcome.GetErrorCode().LocalName, spec); println("*** fail, result " + outcome.GetException() + " Expected success."); feedback.Feedback(successes, failures++, total); } else { writeTestcaseElement(testCaseName, "wrongError", "Expected error:" + expectedError.ToString() + ", got " + outcome.GetErrorCode().LocalName, spec); println("*** fail, result " + outcome.GetErrorCode().LocalName + " Expected error:" + expectedError.ToString()); wrongErrorResults++; feedback.Feedback(successes++, failures, total); } } else { writeTestcaseElement(testCaseName, "fail", "Wrong results, got " + truncate(outcome.Serialize(assertXpc.Processor, outcome.GetPrincipalResultDoc())), spec); feedback.Feedback(successes, failures++, total); if (debug) { try { println("Result:"); driverProc.WriteXdmValue(outcome.GetPrincipalResult(), driverSerializer); println("<======="); } catch (Exception err) { } //println(outcome.getResult()); } else { println("*** fail (use -debug to show actual result)"); //failures++; } } } } }
public void go(String[] args) { Console.WriteLine("Testing Saxon " + processor.ProductVersion); testSuiteDir = args[0]; Hashtable exceptions = new Hashtable(); if (args.Length > 1) { testPattern = (args[1]); // TODO: allow a regex } for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-w")) { //showWarnings = true; } else if (args[i].Equals("-debug")) { debug = true; } } XPathCompiler xpc = processor.NewXPathCompiler(); xpc.DeclareNamespace("t", testURI); xpc.DeclareVariable(new QName("", "param")); findSourcePath = xpc.Compile("//t:test-suite/t:sources/t:source[@ID=$param]"); findCollection = xpc.Compile("//t:test-suite/t:sources/t:collection[@ID=$param]"); xpc = processor.NewXPathCompiler(); xpc.DeclareNamespace("t", testURI); xpc.DeclareVariable(new QName("", "testcase")); xpc.DeclareVariable(new QName("", "moduleuri")); findModule = xpc.Compile("for $m in $testcase/t:module[@namespace=$moduleuri] " + "return concat('file:///" + testSuiteDir + "', root($testcase)/t:test-suite/t:sources/t:module[@ID=string($m)]/@FileName, '.xq')"); xpc = processor.NewXPathCompiler(); xpc.DeclareNamespace("saxon", "http://saxon.sf.net/"); xpc.DeclareVariable(new QName("", "actual")); xpc.DeclareVariable(new QName("", "gold")); xpc.DeclareVariable(new QName("", "debug")); compareDocuments = xpc.Compile("saxon:deep-equal($actual, $gold, (), if ($debug) then 'JNCPS?!' else 'JNCPS')"); QName testCaseNT = new QName(testURI, "test-case"); QName nameNT = new QName(testURI, "name"); QName queryNT = new QName(testURI, "query"); QName inputNT = new QName(testURI, "input"); QName inputFileNT = new QName(testURI, "input-file"); QName inputUriNT = new QName(testURI, "input-URI"); QName defaultCollectionNT = new QName(testURI, "defaultCollection"); QName outputFileNT = new QName(testURI, "output-file"); QName expectedErrorNT = new QName(testURI, "expected-error"); QName schemaNT = new QName(testURI, "schema"); QName contextItemNT = new QName(testURI, "contextItem"); QName inputQueryNT = new QName(testURI, "input-query"); QName sourceDocumentNT = new QName(testURI, "source-document"); QName errorNT = new QName(testURI, "error"); QName validationNT = new QName(testURI, "validation"); QName discretionaryItemsNT = new QName(testURI, "discretionary-items"); QName discretionaryFeatureNT = new QName(testURI, "discretionary-feature"); QName discretionaryChoiceNT = new QName(testURI, "discretionary-choice"); QName initialContextNodeNT = new QName(testURI, "initial-context-node"); QName fileAtt = new QName("", "file"); QName filePathAtt = new QName("", "FilePath"); QName fileNameAtt = new QName("", "FileName"); QName errorIdAtt = new QName("", "error-id"); QName compareAtt = new QName("", "compare"); QName nameAtt = new QName("", "name"); QName behaviorAtt = new QName("", "behavior"); QName qnameAtt = new QName("", "qname"); QName modeAtt = new QName("", "mode"); QName validatesAtt = new QName("", "validates"); QName variableAtt = new QName("", "variable"); QName roleAtt = new QName("", "role"); DocumentBuilder builder = processor.NewDocumentBuilder(); XdmNode exceptionsDoc = builder.Build(new Uri(testSuiteDir + "SaxonDriver/exceptions.xml")); // The exceptions.xml file contains details of tests that aren't to be run, for example // because they have known bugs or require special configuration IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, new QName("", "exception")); while (exceptionTestCases.MoveNext()) { XdmNode n = (XdmNode)exceptionTestCases.Current; String nameAttVal = n.StringValue; char[] seps = { ' ', '\n', '\t' }; String[] parts = nameAttVal.Split(seps); foreach (string p in parts) { if (!exceptions.ContainsKey(p)) { exceptions.Add(p, "Exception"); } } } // Hash table containing all source documents. The key is the document name in the // catalog, the value is the corresponding document node Hashtable sourceDocs = new Hashtable(50); // Load the catalog XdmNode catalog = builder.Build(new Uri(testSuiteDir + "/XQTScatalog.xml")); // Add all Static Typing test cases to the exceptions list xpc = processor.NewXPathCompiler(); xpc.DeclareNamespace("t", testURI); XPathSelector st = xpc.Compile("//t:test-group[@name='StaticTyping']//t:test-case").Load(); st.ContextItem = catalog; IEnumerator ste = st.GetEnumerator(); while (ste.MoveNext()) { XdmNode testCase = (XdmNode)ste.Current; exceptions.Add(testCase.GetAttributeValue(nameAtt), "StaticTypingException"); } // Create the results file results = new StreamWriter(testSuiteDir + "/SaxonDriver/results" + processor.ProductVersion + "n.xml"); results.WriteLine("<test-suite-result xmlns='http://www.w3.org/2005/02/query-test-XQTSResult'>"); // Pre-load all the schemas SchemaManager mgr = processor.SchemaManager; IEnumerator se = catalog.EnumerateAxis(XdmAxis.Descendant, schemaNT); while (se.MoveNext()) { XdmNode schemaNode = (XdmNode)se.Current; Console.WriteLine("Loading schema " + schemaNode.GetAttributeValue(fileNameAtt)); Uri location = new Uri(testSuiteDir + schemaNode.GetAttributeValue(fileNameAtt)); mgr.Compile(location); } // Process the test cases in turn IEnumerator testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT); while (testCases.MoveNext()) { XdmNode testCase = (XdmNode)testCases.Current; String testName = testCase.GetAttributeValue(nameAtt); if (testPattern != null && !testName.StartsWith(testPattern)) { continue; } if (exceptions.ContainsKey(testName)) { continue; } Console.WriteLine("Test " + testName); // Compile the query String errorCode = null; String filePath = testCase.GetAttributeValue(filePathAtt); XdmNode query = getChildElement(testCase, queryNT); String queryName = query.GetAttributeValue(nameAtt); String queryPath = testSuiteDir + "/Queries/XQuery/" + filePath + queryName + ".xq"; XQueryCompiler compiler = processor.NewXQueryCompiler(); compiler.BaseUri = new Uri(queryPath).ToString(); compiler.QueryResolver = new XqtsModuleResolver(testCase, findModule); ArrayList errors = new ArrayList(); compiler.ErrorList = errors; XQueryEvaluator xqe = null; FileStream stream = null; try { stream = new FileStream(queryPath, FileMode.Open, FileAccess.Read, FileShare.Read); xqe = compiler.Compile(stream).Load(); } catch (Exception e) { if (errors.Count > 0 && ((StaticError)errors[0]).ErrorCode != null) { errorCode = ((StaticError)errors[0]).ErrorCode.LocalName; } else if (e is StaticError && ((StaticError)e).ErrorCode != null) { Console.WriteLine(e.Message); errorCode = ((StaticError)e).ErrorCode.LocalName; } else { Console.WriteLine(e.Message); errorCode = "ErrorXXX"; } } finally { if (stream != null) { stream.Close(); } } // if the query compiled successfully, try to run it String outputPath = null; if (errorCode == null && xqe != null) { // Supply any input documents IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, inputFileNT); while (en.MoveNext()) { XdmNode file = (XdmNode)en.Current; String var = file.GetAttributeValue(variableAtt); if (var != null) { String sourceName = file.StringValue; XdmNode sourceDoc; if (sourceDocs.ContainsKey(sourceName)) { sourceDoc = (XdmNode)sourceDocs[sourceName]; } else { sourceDoc = buildSource(catalog, builder, sourceName); sourceDocs.Add(sourceName, sourceDoc); } xqe.SetExternalVariable(new QName("", var), sourceDoc); } } // Supply any input URIs IEnumerator eu = testCase.EnumerateAxis(XdmAxis.Child, inputUriNT); while (eu.MoveNext()) { XdmNode file = (XdmNode)eu.Current; String var = file.GetAttributeValue(variableAtt); if (var != null) { String sourceName = file.StringValue; if (sourceName.StartsWith("collection")) { // Supply a collection URI. // This seems to be the only way to distinguish a document URI // from a collection URI. String uri = "collection:" + sourceName; XPathSelector xpe = findCollection.Load(); xpe.SetVariable(new QName("", "param"), new XdmAtomicValue(sourceName)); xpe.ContextItem = catalog; XdmNode collectionNode = (XdmNode)xpe.EvaluateSingle(); if (collectionNode == null) { Console.WriteLine("*** Collection " + sourceName + " not found"); } processor.RegisterCollection(new Uri(uri), getCollection(collectionNode)); xqe.SetExternalVariable(new QName("", var), new XdmAtomicValue(uri)); } else { // Supply a document URI. // We exploit the fact that the short name of the document is // always the same as the file name in these tests String uri = "file:///" + testSuiteDir + "TestSources/" + sourceName + ".xml"; xqe.SetExternalVariable(new QName("", var), new XdmAtomicValue(uri)); } } } // Supply the default collection if required XdmNode defaultCollection = getChildElement(testCase, defaultCollectionNT); if (defaultCollection != null) { String sourceName = defaultCollection.StringValue; XPathSelector xpe = findCollection.Load(); xpe.SetVariable(new QName("", "param"), new XdmAtomicValue(sourceName)); xpe.ContextItem = catalog; XdmNode collectionNode = (XdmNode)xpe.EvaluateSingle(); if (collectionNode == null) { Console.WriteLine("*** Collection " + sourceName + " not found"); } processor.RegisterCollection(null, getCollection(collectionNode)); } // Supply any external variables defined as the result of a separate query IEnumerator ev = testCase.EnumerateAxis(XdmAxis.Child, inputQueryNT); while (ev.MoveNext()) { XdmNode inputQuery = (XdmNode)ev.Current; String fileName = inputQuery.GetAttributeValue(nameAtt); String subQueryPath = testSuiteDir + "/Queries/XQuery/" + filePath + fileName + ".xq"; XQueryCompiler subCompiler = processor.NewXQueryCompiler(); compiler.BaseUri = new Uri(subQueryPath).ToString(); FileStream subStream = new FileStream(subQueryPath, FileMode.Open, FileAccess.Read, FileShare.Read); XdmValue value = subCompiler.Compile(subStream).Load().Evaluate(); String var = inputQuery.GetAttributeValue(variableAtt); xqe.SetExternalVariable(new QName("", var), value); } // Supply the context item if required IEnumerator ci = testCase.EnumerateAxis(XdmAxis.Child, contextItemNT); while (ci.MoveNext()) { XdmNode file = (XdmNode)ci.Current; String sourceName = file.StringValue; if (!sourceDocs.ContainsKey(sourceName)) { XdmNode doc = buildSource(catalog, builder, sourceName); sourceDocs.Add(sourceName, doc); } XdmNode sourceDoc = (XdmNode)sourceDocs[sourceName]; xqe.ContextItem = sourceDoc; } // Create a serializer for the output outputPath = testSuiteDir + "SaxonDriver/results.net/" + filePath + queryName + ".out"; Serializer sr = new Serializer(); try { sr.SetOutputFile(outputPath); sr.SetOutputProperty(new QName("", "method"), "xml"); sr.SetOutputProperty(new QName("", "omit-xml-declaration"), "yes"); sr.SetOutputProperty(new QName("", "indent"), "no"); } catch (DynamicError) { // probably means that no output directory exists, which is probably because // an error is expected outputPath = testSuiteDir + "SaxonDriver/results.net/" + queryName + ".out"; sr.SetOutputFile(outputPath); } // Finally, run the query try { xqe.Run(sr); } catch (DynamicError e) { Console.WriteLine(e.Message); QName code = e.ErrorCode; if (code != null && code.LocalName != null) { errorCode = code.LocalName; } else { errorCode = "ErrYYYYY"; } } catch (Exception e2) { Console.WriteLine("Unexpected exception: " + e2.Message); Console.WriteLine(e2.StackTrace); errorCode = "CRASH!!!"; } } // Compare actual results with expected results if (errorCode != null) { // query returned an error at compile time or run-time, check this was expected string expectedError = ""; bool matched = false; IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, expectedErrorNT); while (en.MoveNext()) { XdmNode error = (XdmNode)en.Current; String expectedErrorCode = error.StringValue; expectedError += (expectedErrorCode + " "); if (expectedErrorCode.Equals(errorCode)) { matched = true; Console.WriteLine("Error " + errorCode + " as expected"); results.WriteLine("<test-case name='" + testName + "' result='pass'/>"); break; } } if (!matched) { if (expectedError.Equals("")) { Console.WriteLine("Error " + errorCode + ", expected success"); results.WriteLine("<test-case name='" + testName + "' result='fail' comment='error " + errorCode + ", expected success'/>"); } else { Console.WriteLine("Error " + errorCode + ", expected " + expectedError); results.WriteLine("<test-case name='" + testName + "' result='pass' comment='error " + errorCode + ", expected " + expectedError + "'/>"); } } } else { // query returned no error bool matched = false; IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, outputFileNT); while (en.MoveNext()) { XdmNode outputFile = (XdmNode)en.Current; String fileName = testSuiteDir + "ExpectedTestResults/" + filePath + outputFile.StringValue; String comparator = outputFile.GetAttributeValue(compareAtt); if (comparator.Equals("Inspect")) { matched = true; results.WriteLine("<test-case name='" + testName + "' result='inspect'/>"); break; } else { matched = compare(outputPath, fileName, comparator); if (matched) { results.WriteLine("<test-case name='" + testName + "' result='pass'/>"); break; } } } if (!matched) { string expectedError = ""; IEnumerator ee = testCase.EnumerateAxis(XdmAxis.Child, expectedErrorNT); while (ee.MoveNext()) { XdmNode error = (XdmNode)ee.Current; String expectedErrorCode = error.StringValue; expectedError += (expectedErrorCode + " "); } if (expectedError.Equals("")) { Console.WriteLine("Results differ from expected results"); results.WriteLine("<test-case name='" + testName + "' result='fail'/>"); } else { Console.WriteLine("Error " + expectedError + "expected but not reported"); results.WriteLine("<test-case name='" + testName + "' result='fail' comment='expected error " + expectedError + "not reported'/>"); } } } } results.WriteLine("</test-suite-result>"); results.Close(); }