Ejemplo n.º 1
0
    public void go(String[] args)
    {
        int passed = 0;
        int failed = 0;
        int total  = 0;

        Console.WriteLine("Testing Saxon " + processor.ProductVersion);
        testSuiteDir = args[0];
        if (testSuiteDir.EndsWith("/"))
        {
            testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1);
        }
        Hashtable exceptions = new Hashtable();

        if (args.Length > 1)
        {
            testPattern = (args[1]);
        }

        for (int i = 0; i < args.Length; i++)
        {
            if (args[i].Equals("-w"))
            {
                //showWarnings = true;
            }
        }

        try
        {
            schemaAwareProcessor = new Processor(true);
        }
        catch (Exception err)
        {
            Console.WriteLine("Cannot load Saxon-SA: continuing without it");
        }

        processor.SetProperty("http://saxon.sf.net/feature/preferJaxpParser", "true");

        if (schemaAwareProcessor != null)
        {
            schemaAwareProcessor.SetProperty("http://saxon.sf.net/feature/preferJaxpParser", "true");
        }
        fileComparer = new FileComparer(processor, testSuiteDir);

        String testURI = "http://www.w3.org/2005/05/xslt20-test-catalog";

        QName testCaseNT             = new QName(testURI, "testcase");
        QName nameNT                 = new QName(testURI, "name");
        QName inputNT                = new QName(testURI, "input");
        QName outputNT               = new QName(testURI, "output");
        QName stylesheetNT           = new QName(testURI, "stylesheet");
        QName schemaNT               = new QName(testURI, "schema");
        QName initialModeNT          = new QName(testURI, "initial-mode");
        QName entryNamedTemplateNT   = new QName(testURI, "entry-named-template");
        QName sourceDocumentNT       = new QName(testURI, "source-document");
        QName stylesheetParametersNT = new QName(testURI, "stylesheet-parameters");
        QName paramNT                = new QName(testURI, "param");
        QName resultDocumentNT       = new QName(testURI, "result-document");
        QName errorNT                = new QName(testURI, "error");
        QName validationNT           = new QName(testURI, "validation");
        QName discretionaryItemsNT   = new QName(testURI, "discretionary-items");
        QName discretionaryFeatureNT = new QName(testURI, "discretionary-feature");
        QName discretionaryChoiceNT  = new QName(testURI, "discretionary-choice");
        QName initialContextNodeNT   = new QName(testURI, "initial-context-node");


        QName fileAtt      = new QName("", "file");
        QName errorIdAtt   = new QName("", "error-id");
        QName typeAtt      = new QName("", "type");
        QName nameAtt      = new QName("", "name");
        QName behaviorAtt  = new QName("", "behavior");
        QName qnameAtt     = new QName("", "qname");
        QName modeAtt      = new QName("", "mode");
        QName validatesAtt = new QName("", "validates");
        QName roleAtt      = new QName("", "role");

        DocumentBuilder builder       = processor.NewDocumentBuilder();
        XdmNode         exceptionsDoc = builder.Build(new Uri(testSuiteDir + '/' + getResultDirectoryName() + "/exceptions.xml"));

        IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, testCaseNT);

        while (exceptionTestCases.MoveNext())
        {
            XdmNode  n          = (XdmNode)exceptionTestCases.Current;
            String   nameAttVal = n.GetAttributeValue(nameAtt);
            char[]   seps       = { ' ', '\n', '\t' };
            String[] parts      = nameAttVal.Split(seps);
            foreach (string p in parts)
            {
                if (!exceptions.ContainsKey(p))
                {
                    exceptions.Add(p, "Kilroy");
                }
            }
        }

        XdmNode catalog = builder.Build(new Uri(testSuiteDir + "/catalog.xml"));

        results = new StreamWriter(testSuiteDir + "/SaxonResults.net/results"
                                   + processor.ProductVersion + ".xml");

        results.WriteLine("<test-suite-result>");
        results.WriteLine(" <implementation name='Saxon-SA' version='" + processor.ProductVersion +
                          "' anonymous-result-column='false'>");
        results.WriteLine("  <organization name='http://www.saxonica.com/' anonymous='false'/>");
        results.WriteLine("  <submitter name='Michael Kay' email='*****@*****.**'/>");
        outputDiscretionaryItems();
        results.WriteLine(" </implementation>");

        total = 0;
        IEnumerator testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT);

        while (testCases.MoveNext())
        {
            total++;
        }

        testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT);
        while (testCases.MoveNext())
        {
            bool    useAssociated = false;
            XdmNode testCase      = (XdmNode)testCases.Current;

            String testName = getChildElement(testCase, nameNT).StringValue;
            if (testPattern != null && !testName.StartsWith(testPattern))
            {
                continue;
            }
            if (exceptions.ContainsKey(testName))
            {
                continue;
            }
            if (isExcluded(testName))
            {
                continue;
            }
            Console.WriteLine("Test " + testName);
            XdmNode testInput = getChildElement(testCase, inputNT);

            XdmNode stylesheet = getChildElement(testInput, stylesheetNT);
            String  absXSLName = null;
            if (stylesheet == null)
            {
                useAssociated = true;
            }
            else
            {
                absXSLName = testSuiteDir + "/TestInputs/" + stylesheet.GetAttributeValue(fileAtt);
            }

            XdmNode sourceDocument = getChildElement(testInput, sourceDocumentNT);
            String  absXMLName     = null;
            if (sourceDocument != null)
            {
                absXMLName = testSuiteDir + "/TestInputs/" + sourceDocument.GetAttributeValue(fileAtt);
            }

            bool    schemaAware            = false;
            bool    recoverRecoverable     = true;
            bool    backwardsCompatibility = true;
            bool    supportsDOE            = true;
            bool    recoverSESU0007        = false;
            XdmNode discretionaryItems     = getChildElement(testCase, discretionaryItemsNT);
            if (discretionaryItems != null)
            {
                IEnumerator features = discretionaryItems.EnumerateAxis(XdmAxis.Child, discretionaryFeatureNT);
                while (features.MoveNext())
                {
                    XdmNode feature     = (XdmNode)features.Current;
                    String  featureName = feature.GetAttributeValue(nameAtt);
                    if ("schema_aware".Equals(featureName) || "Saxon-PE".Equals(featureName))   // TODO: test Saxon-PE properly
                    {
                        schemaAware = "on".Equals(feature.GetAttributeValue(behaviorAtt));
                    }
                    else if ("XML_1.1".Equals(featureName))
                    {
                        xml11 = "on".Equals(feature.GetAttributeValue(behaviorAtt));
                    }
                    else if ("backwards_compatibility".Equals(featureName))
                    {
                        backwardsCompatibility = "on".Equals(feature.GetAttributeValue(behaviorAtt));
                    }
                    else if ("disabling_output_escaping".Equals(featureName))
                    {
                        supportsDOE = "on".Equals(feature.GetAttributeValue(behaviorAtt));
                    }
                }
                IEnumerator choices = discretionaryItems.EnumerateAxis(
                    XdmAxis.Child, discretionaryChoiceNT);
                while (choices.MoveNext())
                {
                    XdmNode choice      = (XdmNode)choices.Current;
                    String  featureName = choice.GetAttributeValue(nameAtt);
                    if ("error".Equals(choice.GetAttributeValue(behaviorAtt)))
                    {
                        recoverRecoverable = false;
                    }
                    else if ("SESU0007".Equals(featureName))
                    {
                        recoverSESU0007 = "recovery".Equals(choice.GetAttributeValue(behaviorAtt));
                    }
                }
            }

            if (!backwardsCompatibility)
            {
                // Saxon cannot run with BC switched off
                results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires backwards-compatibility=off'/>");
                continue;
            }

            if (!supportsDOE)
            {
                // Saxon cannot run with DOE switched off
                results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires disable-output-escaping=off'/>");
                continue;
            }

            if (recoverSESU0007)
            {
                // Saxon cannot recover from error SESU0007
                results.WriteLine(" <testcase name='" + testName + "' result='not run' comment='requires recovery from error SESU0007'/>");
                continue;
            }

            XdmNode initialMode     = getChildElement(testInput, initialModeNT);
            QName   initialModeName = null;
            if (initialMode != null)
            {
                String ini = initialMode.GetAttributeValue(qnameAtt);
                initialModeName = makeQName(ini, initialMode);
            }

            XdmNode initialTemplate     = getChildElement(testInput, entryNamedTemplateNT);
            QName   initialTemplateName = null;
            if (initialTemplate != null)
            {
                String ini = initialTemplate.GetAttributeValue(qnameAtt);
                initialTemplateName = makeQName(ini, initialTemplate);
            }

            XdmNode initialContextNode = getChildElement(testInput, initialContextNodeNT);
            String  initialContextPath = null;
            if (initialContextNode != null)
            {
                initialContextPath = initialContextNode.StringValue;
            }

            XdmNode validation     = getChildElement(testInput, validationNT);
            String  validationMode = null;
            if (validation != null)
            {
                validationMode = validation.GetAttributeValue(modeAtt);
            }

            Hashtable paramTable = null;
            XdmNode   paramList  = getChildElement(testInput, stylesheetParametersNT);
            if (paramList != null)
            {
                paramTable = new Hashtable(5);
                IEnumerator paramIter = paramList.EnumerateAxis(XdmAxis.Child, paramNT);
                while (paramIter.MoveNext())
                {
                    XdmNode param = (XdmNode)paramIter.Current;
                    QName   name  = makeQName(param.GetAttributeValue(qnameAtt), param);
                    String  value = param.StringValue;
                    paramTable.Add(name, value);
                }
            }

            IEnumerator schemas = testInput.EnumerateAxis(XdmAxis.Child, schemaNT);
            while (schemas.MoveNext())
            {
                XdmNode schema = (XdmNode)schemas.Current;
                if (schema == null)
                {
                    break;
                }
                schemaAware = true;
                String role = schema.GetAttributeValue(roleAtt);
                if (("source-validator".Equals(role) || "source-reference".Equals(role))
                    /* && schema.GetAttributeValue(validatesAtt) != null */)
                {
                    validationMode = "strict";
                    // TODO: control which source documents are validated...
                }
            }
            XdmNode testOutput     = getChildElement(testCase, outputNT);
            XdmNode resultDocument = getChildElement(testOutput, resultDocumentNT);
            // TODO: handle alternative result documents
            String refFileName = null;
            String outFileName;
            String comparator = "xml";
            if (resultDocument != null)
            {
                String relativePath = resultDocument.GetAttributeValue(fileAtt);
                int    slash        = relativePath.IndexOf('/');
                if (slash > 0)
                {
                    String relativeDir = relativePath.Substring(0, slash);
                    String fullDir     = testSuiteDir + '/' + getResultDirectoryName() + "/" + relativeDir;
                    if (!Directory.Exists(fullDir))
                    {
                        Directory.CreateDirectory(fullDir);
                    }
                }
                refFileName = testSuiteDir + "/ExpectedTestResults/" + relativePath;
                outFileName = testSuiteDir + '/' + getResultDirectoryName() + "/" + relativePath;
                comparator  = resultDocument.GetAttributeValue(typeAtt);
            }
            else
            {
                outFileName = testSuiteDir + '/' + getResultDirectoryName() + "/temp.out";
            }
            XdmNode error         = getChildElement(testOutput, errorNT);
            String  expectedError = null;
            if (error != null)
            {
                expectedError = error.GetAttributeValue(errorIdAtt);
            }
            bool      success;
            Exception xsltOutcome =
                runXSLT(testName, absXMLName, absXSLName, initialModeName, initialTemplateName,
                        outFileName, paramTable, initialContextPath,
                        useAssociated, schemaAware, validationMode, recoverRecoverable);
            if (xsltOutcome == null)
            {
                success = true;
                if (expectedError != null && resultDocument == null)
                {
                    Console.WriteLine("Test failed. Expected error " + expectedError + ", got success");
                    feedback.Feedback(passed, failed++, total);
                    success = false;
                    results.WriteLine(" <testcase name='" + testName +
                                      "' result='differ' comment='Expected error " +
                                      expectedError + ", got success'/>");
                }
                else
                {
                    feedback.Feedback(passed++, failed, total);
                }
            }
            else
            {
                String errorCode = null;
                if (xsltOutcome is StaticError)
                {
                    errorCode = ((StaticError)xsltOutcome).ErrorCode.LocalName;
                }
                else if (xsltOutcome is DynamicError)
                {
                    errorCode = ((DynamicError)xsltOutcome).ErrorCode.LocalName;
                }
                if (expectedError != null && errorCode != null && errorCode.Equals(expectedError))
                {
                    feedback.Feedback(passed++, failed, total);
                    Console.WriteLine("Test succeeded (" + expectedError + ')');
                    results.WriteLine(" <testcase name='" + testName +
                                      "' result='full' comment='Error " +
                                      expectedError + " as expected'/>");
                }
                else if (expectedError != null)
                {
                    feedback.Feedback(passed++, failed, total);
                    Console.WriteLine("Test succeeded (??) (expected " + expectedError + ", got " + errorCode + ')');
                    results.WriteLine(" <testcase name='" + testName +
                                      "' result='different-error' comment='Expected " +
                                      expectedError + " got " + errorCode + "'/>");
                }
                else
                {
                    feedback.Feedback(passed, failed++, total);
                    Console.WriteLine("Test failed. Expected success, got " + errorCode);
                    results.WriteLine(" <testcase name='" + testName +
                                      "' result='differ' comment='Expected success, got " +
                                      errorCode + "'/>");
                    results.WriteLine(" <!--" + xsltOutcome.Message + "-->");
                }
                success = false;
                continue;
            }


            if (success)
            {
                String outcome = fileComparer.compare(outFileName, refFileName, comparator);
                if (outcome == "OK")
                {
                    results.WriteLine(" <testcase name='" + testName + "' result='full'/>");
                }
                else if (outcome.StartsWith("#"))
                {
                    results.WriteLine(" <testcase name='" + testName + "' result='full' + comments='" + outcome.Substring(1) + "/>");
                }
                else
                {
                    results.WriteLine(" <testcase name='" + testName + "' result='differ' comments='" + outcome + "'/>");
                }
            }
        }

        results.WriteLine("</test-suite-result>");
        results.Close();

        //}
    }
Ejemplo n.º 2
0
        /**
         * Run the tests
         * @param args command line arguments
         * @throws SAXException
         * @throws ParserConfigurationException
         * @throws XPathException
         * @throws IOException
         * @throws URISyntaxException
         */

        public void go(String[] args)
        {
            if (args.Length == 0 || args[0] == "-?")
            {
                Console.WriteLine("SchemaTestSuiteDriver testDir [-w] [-onwards] -c:contributor? -s:setName? -g:groupName?");
            }
            Processor processor = new Processor(true);

            Console.WriteLine("Testing Saxon " + processor.ProductVersion);

            testSuiteDir = args[0];
            if (testSuiteDir.EndsWith("/"))
            {
                testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1);
            }
            String    testSetPattern   = null;  // TODO use a regex
            String    testGroupPattern = null;
            String    contributor      = null;
            Hashtable exceptions       = new Hashtable();

            for (int i = 1; i < args.Length; i++)
            {
                if (args[i] == ("-w"))
                {
                    //showWarnings = true;
                }
                else if (args[i] == ("-onwards"))
                {
                    onwards = true;
                }
                else if (args[i].StartsWith("-c:"))
                {
                    contributor = args[i].Substring(3);
                }
                else if (args[i].StartsWith("-s:"))
                {
                    testSetPattern = args[i].Substring(3);
                }
                else if (args[i].StartsWith("-g:"))
                {
                    testGroupPattern = args[i].Substring(3);
                }
                else if (args[i] == "-?")
                {
                    Console.WriteLine("Usage: SchemaTestSuiteDriver testDir [-w] [-s:testSetPattern] [-g:testGroupPattern]");
                }
            }

            int total  = 39700;
            int passed = 0;
            int failed = 0;

            try {
                xlinkHref = new QName("xlink", "http://www.w3.org/1999/xlink", "href");

                QName testCaseNT = new QName("", "", "testcase");
                QName commentNT  = new QName("", "", "comment");

                QName testSetRefNT       = new QName(testNS, "testSetRef");
                QName testGroupNT        = new QName(testNS, "testGroup");
                QName testSetNT          = new QName(testNS, "testSet");
                QName schemaTestNT       = new QName(testNS, "schemaTest");
                QName instanceTestNT     = new QName(testNS, "instanceTest");
                QName schemaDocumentNT   = new QName(testNS, "schemaDocument");
                QName instanceDocumentNT = new QName(testNS, "instanceDocument");
                QName expectedNT         = new QName(testNS, "expected");
                QName currentNT          = new QName(testNS, "current");

                QName validityAtt        = new QName("", "", "validity");
                QName nameAtt            = new QName("", "", "name");
                QName contributorAtt     = new QName("", "", "contributor");
                QName setAtt             = new QName("", "", "set");
                QName groupAtt           = new QName("", "", "group");
                QName statusAtt          = new QName("", "", "status");
                QName bugzillaAtt        = new QName("", "", "bugzilla");
                QName targetNamespaceAtt = new QName("", "", "targetNamespace");
                QName schemaVersion      = new QName("saxon", "http://saxon.sf.net/", "schemaVersion");


                DocumentBuilder builder = processor.NewDocumentBuilder();
                builder.BaseUri = new Uri(testSuiteDir + "/");
                XdmNode catalog = builder.Build(
                    new FileStream(testSuiteDir + "/suite.xml", FileMode.Open, FileAccess.Read, FileShare.Read));

                results = new StreamWriter(testSuiteDir + "/saxon/SaxonResults"
                                           + processor.ProductVersion + "n.xml");

                results.Write("<testSuiteResults xmlns='" + testNS + "' xmlns:saxon='http://saxon.sf.net/' " +
                              "suite='TS_2006' " +
                              "processor='Saxon-SA (Java) 8.8++' submitDate='2007-01-05' publicationPermission='public'>\n");

                XdmNode exceptionsDoc = builder.Build(
                    new FileStream(testSuiteDir + "/saxon/exceptions.xml",
                                   FileMode.Open, FileAccess.Read, FileShare.Read));


                IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, new QName("", "testcase"));
                while (exceptionTestCases.MoveNext())
                {
                    XdmNode testCase = (XdmNode)exceptionTestCases.Current;
                    String  set      = testCase.GetAttributeValue(setAtt);
                    String  group    = testCase.GetAttributeValue(groupAtt);
                    String  comment  = getChildElement(testCase, commentNT).StringValue;
                    exceptions[set + "#" + group] = comment;
                }

                IEnumerator testSets = catalog.EnumerateAxis(XdmAxis.Descendant, testSetRefNT);
                while (testSets.MoveNext())
                {
                    XdmNode testSetRef     = (XdmNode)testSets.Current;
                    XdmNode testSetDoc     = getLinkedDocument(testSetRef, processor, false);
                    XdmNode testSetElement = getChildElement(testSetDoc, testSetNT);

                    if (testSetElement == null)
                    {
                        feedback.Message("test set doc has no TestSet child: " + testSetDoc.BaseUri, true);
                        continue;
                    }

                    String testSetName = testSetElement.GetAttributeValue(nameAtt);
                    if (testSetPattern != null && !testSetName.StartsWith(testSetPattern))
                    {
                        continue;
                    }
                    if (contributor != null && contributor != testSetElement.GetAttributeValue(contributorAtt))
                    {
                        continue;
                    }

                    bool needs11 = (testSetElement.GetAttributeValue(schemaVersion) == "1.1");

                    IEnumerator testGroups = testSetElement.EnumerateAxis(XdmAxis.Child, testGroupNT);
                    while (testGroups.MoveNext())
                    {
                        XdmNode testGroup = (XdmNode)testGroups.Current;

                        String testGroupName = testGroup.GetAttributeValue(nameAtt);
                        String exception     = (String)exceptions[testSetName + "#" + testGroupName];

                        if (testGroupPattern != null && !testGroupName.StartsWith(testGroupPattern))
                        {
                            continue;
                        }
                        Console.WriteLine("TEST SET " + testSetName + " GROUP " + testGroupName, false);
                        if (onwards)
                        {
                            testGroupPattern = null;
                            testSetPattern   = null;
                        }
                        Processor testConfig = new Processor(true);
                        if (needs11)
                        {
                            testConfig.SetProperty("http://saxon.sf.net/feature/xsd-version", "1.1");
                        }
                        SchemaManager schemaManager = testConfig.SchemaManager;


                        //testConfig.setHostLanguage(Configuration.XML_SCHEMA);
                        testConfig.SetProperty("http://saxon.sf.net/feature/validation-warnings", "true");
                        IEnumerator schemaTests   = testGroup.EnumerateAxis(XdmAxis.Child, schemaTestNT);
                        bool        schemaQueried = false;
                        String      bugzillaRef   = null;

                        while (schemaTests.MoveNext())
                        {
                            XdmNode schemaTest = (XdmNode)schemaTests.Current;
                            if (schemaTest == null)
                            {
                                break;
                            }
                            bugzillaRef = null;
                            String testName = schemaTest.GetAttributeValue(nameAtt);
                            if (exception != null)
                            {
                                results.Write("<testResult set='" + testSetName +
                                              "' group='" + testGroupName +
                                              "' test='" + testName +
                                              "' validity='notKnown' saxon:outcome='notRun' saxon:comment='" + exception +
                                              "'/>\n");
                                continue;
                            }
                            bool    queried       = false;
                            XdmNode statusElement = getChildElement(schemaTest, currentNT);
                            if (statusElement != null)
                            {
                                String status = statusElement.GetAttributeValue(statusAtt);
                                queried     = ("queried" == status);
                                bugzillaRef = statusElement.GetAttributeValue(bugzillaAtt);
                            }
                            if (queried)
                            {
                                schemaQueried = true;
                            }
                            Console.WriteLine("TEST SCHEMA " + testName + (queried ? " (queried)" : ""));
                            bool        success  = true;
                            IEnumerator schemata = schemaTest.EnumerateAxis(XdmAxis.Child, schemaDocumentNT);
                            while (schemata.MoveNext())
                            {
                                XdmNode schemaDocumentRef = (XdmNode)schemata.Current;
                                if (schemaDocumentRef == null)
                                {
                                    break;
                                }
                                Console.WriteLine("Loading schema at " + schemaDocumentRef.GetAttributeValue(xlinkHref));
                                XdmNode     schemaDoc     = getLinkedDocument(schemaDocumentRef, testConfig, false);
                                IEnumerator schemaDocKids = schemaDoc.EnumerateAxis(XdmAxis.Child);
                                XdmNode     schemaElement = null;
                                while (schemaDocKids.MoveNext())
                                {
                                    schemaElement = (XdmNode)schemaDocKids.Current;
                                    if (schemaElement.NodeKind == XmlNodeType.Element)
                                    {
                                        break;
                                    }
                                }
                                String targetNamespace = schemaElement.GetAttributeValue(targetNamespaceAtt);
                                //if (targetNamespace != null && schemaManager. isSchemaAvailable(targetNamespace)) {
                                // do nothing
                                // TODO: this is the only way I can get MS additional test addB132 to work.
                                // It's not ideal: addSchemaSource() ought to be a no-op if the schema components
                                // are already loaded, but in fact recompiling the imported schema document on its
                                // own is losing the substitution group membership that was defined in the
                                // importing document.
                                //} else {
                                IList <StaticError> errorList = new List <StaticError>();
                                schemaManager.ErrorList = errorList;
                                try
                                {
                                    schemaManager.Compile(schemaDoc);
                                }
                                catch (Exception e)
                                {
                                    if (errorList.Count == 0)
                                    {
                                        feedback.Message("In " + testName + ", exception thrown but no errors in ErrorList\n", true);
                                        results.Write("<!--" + e.Message + "-->");
                                        success = false;
                                    }
                                }
                                for (int i = 0; i < errorList.Count; i++)
                                {
                                    if (errorList[i] is StaticError)
                                    {
                                        StaticError err = (StaticError)errorList[i];
                                        if (!err.IsWarning)
                                        {
                                            success = false;
                                            break;
                                        }
                                    }
                                    else
                                    {
                                        feedback.Message("In " + testName + " wrong kind of error!" + errorList[i].GetType() + "\n", true);
                                    }
                                }
                            }
                            XdmNode expected        = getChildElement(schemaTest, expectedNT);
                            bool    expectedSuccess = expected == null ||
                                                      expected.GetAttributeValue(validityAtt) == "valid";
                            if (success == expectedSuccess)
                            {
                                passed++;
                            }
                            else
                            {
                                failed++;
                            }
                            feedback.Feedback(passed, failed, total);
                            results.Write("<testResult set='" + testSetName +
                                          "' group='" + testGroupName +
                                          "' test='" + testName +
                                          "' validity='" + (success ? "valid" : "invalid") +
                                          (queried ? "' saxon:queried='true' saxon:bugzilla='" + bugzillaRef : "") +
                                          "' saxon:outcome='" + (success == expectedSuccess ? "same" : "different") +
                                          "'/>\n");
                        }
                        IEnumerator instanceTests = testGroup.EnumerateAxis(XdmAxis.Child, instanceTestNT);
                        while (instanceTests.MoveNext())
                        {
                            XdmNode instanceTest = (XdmNode)instanceTests.Current;
                            String  testName     = instanceTest.GetAttributeValue(nameAtt);

                            if (exception != null)
                            {
                                results.Write("<testResult set='" + testSetName +
                                              "' group='" + testGroupName +
                                              "' test='" + testName +
                                              "' validity='notKnown' saxon:outcome='notRun' saxon:comment='" + exception +
                                              "'/>\n");
                                continue;
                            }

                            bool    queried       = false;
                            XdmNode statusElement = getChildElement(instanceTest, currentNT);
                            if (statusElement != null)
                            {
                                String status = statusElement.GetAttributeValue(statusAtt);
                                queried = ("queried" == status);
                                String instanceBug = statusElement.GetAttributeValue(bugzillaAtt);
                                if (instanceBug != null)
                                {
                                    bugzillaRef = instanceBug;
                                }
                            }
                            queried |= schemaQueried;

                            Console.WriteLine("TEST INSTANCE " + testName + (queried ? " (queried)" : ""));

                            XdmNode instanceDocument = getChildElement(instanceTest, instanceDocumentNT);

                            bool success = true;
                            try
                            {
                                XdmNode instanceDoc = getLinkedDocument(instanceDocument, testConfig, true);
                            }
                            catch (Exception)
                            {
                                success = false;
                            }

                            XdmNode expected        = getChildElement(instanceTest, expectedNT);
                            bool    expectedSuccess = expected == null ||
                                                      expected.GetAttributeValue(validityAtt) == "valid";
                            if (success == expectedSuccess)
                            {
                                passed++;
                            }
                            else
                            {
                                failed++;
                            }
                            feedback.Feedback(passed, failed, total);
                            results.Write("<testResult set='" + testSetName +
                                          "' group='" + testGroupName +
                                          "' test='" + testName +
                                          "' validity='" + (success ? "valid" : "invalid") +
                                          (queried ? "' saxon:queried='true' saxon:bugzilla='" + bugzillaRef : "") +
                                          "' saxon:outcome='" + (success == expectedSuccess ? "same" : "different") +
                                          "'/>\n");
                        }
                    }
                }

                results.Write("</testSuiteResults>");
                results.Close();
            } catch (Exception e) {
                feedback.Message("Test failed: " + e.Message, true);
            }
        }