private void PrintXPath(string sel, Hashtable nsTable)
{
XPathSelector xsel = new XPathSelector(sel, nsTable);
xsel.Matches(e);
// Console.WriteLine(xsel.XPath);
}
public override bool TestAssertion(string assertion)
{
bool DocOK = true;
bool FileOK = true;
if (resultDocument != null)
{
XPathCompiler xPathCompiler = processor.NewXPathCompiler();
XPathExecutable exec = xPathCompiler.Compile(assertion);
XPathSelector selector = exec.Load();
selector.ContextItem = resultDocument;
DocOK = selector.EffectiveBooleanValue();
}
if (resultFile != null)
{
DocumentBuilder builder = processor.NewDocumentBuilder();
XdmNode resultDoc = builder.Build(new Uri(resultFile));
XPathCompiler xPathCompiler = processor.NewXPathCompiler();
XPathExecutable exec = xPathCompiler.Compile(assertion);
XPathSelector selector = exec.Load();
selector.ContextItem = resultDoc;
FileOK = selector.EffectiveBooleanValue();
}
return(DocOK && FileOK);
}
public static ISelector GetSelector(ExtractByAttribute extractBy)
{
var value = extractBy.Value;
ISelector selector = null;
switch (extractBy.Type)
{
case ExtractType.Css:
selector = new CssSelector(value);
break;
case ExtractType.Regex:
selector = new RegexSelector(value);
break;
case ExtractType.XPath:
selector = new XPathSelector(value);
break;
case ExtractType.JsonPath:
selector = new JsonPathSelector(value);
break;
}
return(selector);
}
/**
* Show how to transform a tree starting at a node other than the root.
*/
public static void ExampleSaxonToSaxonNonRoot(String sourceUri, String xsltUri)
{
// Create a Processor instance.
Processor processor = new Processor();
// Load the source document
XdmNode input = processor.NewDocumentBuilder().Build(new Uri(sourceUri));
// Navigate to the first grandchild
XPathSelector eval = processor.NewXPathCompiler().Compile("/*/*[1]").Load();
eval.ContextItem = input;
input = (XdmNode)eval.EvaluateSingle();
// Create an XSLT compiler
XsltCompiler compiler = processor.NewXsltCompiler();
// Compile the stylesheet
XsltTransformer transformer = compiler.Compile(new Uri(xsltUri)).Load();
// Run the transformation
transformer.InitialContextNode = input;
XdmDestination result = new XdmDestination();
transformer.Run(result);
// Serialize the result so we can see that it worked
Console.WriteLine(result.XdmNode.OuterXml);
}
public bool ValidateSchematron(string filePath)
{
if (string.IsNullOrEmpty(schematronPath))
{
return(true);
}
if (string.IsNullOrEmpty(phase1))
{
string schSkeletonContent;
using (Stream schSkeletonStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("LantanaGroup.XmlHarvester.iso-sch-conformance1-5.xsl"))
{
using (StreamReader schSkeletonReader = new StreamReader(schSkeletonStream))
{
schSkeletonContent = schSkeletonReader.ReadToEnd();
}
}
phase1 = Transform(schSkeletonContent, schematronPath, true);
}
string phase2 = Transform(phase1, filePath);
XdmNode phase2Results = builder.Build(new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(phase2))));
XPathCompiler xpathCompiler = processor.NewXPathCompiler();
XPathExecutable xpathExec = xpathCompiler.Compile("//failed-assert");
XPathSelector xpathSelector = xpathExec.Load();
xpathSelector.ContextItem = phase2Results;
var xpathResults = xpathSelector.Evaluate();
return(xpathResults.Count == 0);
}
public void PageProcess(PagePathogen pagePathogen)
{
var selector = new XPathSelector(pagePathogen.PageSource);
//小说
Novel novel = new Novel();
//文章信息
novel.Articles = new List <Article>();
var nameEle = selector.SelectSingleNode("//*[@id='info']/h1");
if (nameEle != null)
{
//小说名称
novel.Name = nameEle.InnerText;
}
var authorEle = selector.SelectSingleNode("//*[@id='info']/p[1]");
if (authorEle != null)
{
string pStr = authorEle.InnerText;
//作者
novel.Author = pStr.Split('：')[1];
}
//获取对应文章信息
GetArticles(selector, novel, pagePathogen.Url, "//*[@id='list']/dl/dd/a");
//传递抓取数据信息
pagePathogen.AddResult("novel", novel);
}
public static Dictionary <int, string> XPath(string xmlData, string xPath)
{
Processor processor = new Processor();
XPathCompiler compiler = processor.NewXPathCompiler();
XDocument xDoc = XDocument.Parse(xmlData);
// Detect namespaces in the document so the user doesn't have to specify them.
var xmlNameSpaceList = ((IEnumerable)xDoc.XPathEvaluate(@"//namespace::*[not(. = ../../namespace::*)]")).Cast <XAttribute>();
foreach (var nsNode in xmlNameSpaceList)
{
compiler.DeclareNamespace(nsNode.Name.LocalName, nsNode.Value);
}
DocumentBuilder documentBuilder = processor.NewDocumentBuilder();
documentBuilder.IsLineNumbering = true;
documentBuilder.BaseUri = new Uri("file://");
XdmNode document = documentBuilder.Build(new StringReader(xmlData));
XPathSelector selector = compiler.Compile(xPath).Load();
selector.ContextItem = document;
XdmValue values = selector.Evaluate();
Dictionary <int, string> lines = new Dictionary <int, string>();
foreach (XdmNode value in values)
{
lines.Add(value.LineNumber - 1, value.ToString());
}
return(lines);
}
/**
* Construct source object. This method allows subclassing e.g. to build a DOM or XOM source.
* @param xml
* @return
* @throws XPathException
*/
protected XdmNode buildSource(XdmNode catalog, DocumentBuilder builder, String sourceName)
{
// Find the source element in the catalog
XPathSelector xps = findSourcePath.Load();
xps.SetVariable(new QName("", "param"), new XdmAtomicValue(sourceName));
xps.ContextItem = catalog;
XdmNode source = (XdmNode)xps.EvaluateSingle();
// decide whether schema validation is needed
bool validate = source.GetAttributeValue(new QName("", "schema")) != null;
if (validate)
{
builder.SchemaValidationMode = SchemaValidationMode.Strict;
}
else
{
builder.SchemaValidationMode = SchemaValidationMode.None;
}
// build the document tree from the source file
string filename = testSuiteDir + "/" + source.GetAttributeValue(new QName("", "FileName"));
if (source == null)
{
throw new ArgumentException("Source " + sourceName + " not found in catalog");
}
return(builder.Build(new Uri(filename)));
}
/// <summary>
///
/// </summary>
/// <param name="selector"></param>
/// <param name="novel"></param>
/// <param name="uri"></param>
/// <param name="xpath"></param>
/// <returns></returns>
private void GetArticles(XPathSelector selector, Novel novel, string uri, string xpath)
{
var aEles = selector.SelectNodes(xpath);
if (aEles != null)
{
foreach (var ele in aEles)
{
string articleUri = uri + ele.GetAttributeValue("href", "");
string[] s = articleUri.Split('/');
string uN = s[s.Length - 1];
long htmlFileName = long.Parse(uN.Substring(0, uN.IndexOf('.')));
string articleTitle = ele.InnerText;
var article = new Article()
{
Title = articleTitle,
Url = articleUri,
Novel = novel,
Seq = htmlFileName
};
//回调
Config.CallBack?.Invoke(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "--文章标题：" + articleTitle + " 文章地址：" + articleUri);
novel.Articles.Add(article);
}
}
}
public void EqualsPredicate_Constructor_SetsXpathSelector()
{
var expectedXPathSelector = new XPathSelector("!$");
var fields = new TestPredicateFields();
var predicate = new EqualsPredicate<TestPredicateFields>(fields, xpath: expectedXPathSelector);
Assert.AreEqual(expectedXPathSelector, predicate.XPathSelector);
}
public void XPathSelector_Constructor_WithNamespace_SetsSelector()
{
const string expectedSelector = "//Test";
var selector = new XPathSelector(expectedSelector, null);
Assert.AreEqual(expectedSelector, selector.Selector);
}
public void Constructor_SetsSelector()
{
const string expectedSelector = "//Test";
var selector = new XPathSelector(expectedSelector);
Assert.AreEqual(expectedSelector, selector.Selector);
}
public void Constructor_SetsXpathSelector()
{
var expectedXPathSelector = new XPathSelector("!$");
var fields = new TestPredicateFields();
var predicate = new StartsWithPredicate <TestPredicateFields>(fields, false, null, expectedXPathSelector);
Assert.AreEqual(expectedXPathSelector, predicate.Selector);
}
public void XPathSelector_Constructor_WithNamespace_SetsNamespaceDictionary()
{
var namespaces = new Dictionary <string, string> {
{ "isbn", "http://xmlnamespaces.org/isbn" }
};
var selector = new XPathSelector("//isbn:book", namespaces);
Assert.AreEqual(namespaces, selector.Namespaces);
}
public Uri[] GetModules(String moduleUri, Uri baseUri, String[] locationHints)
{
XPathSelector xps = findModule.Load();
xps.SetVariable(new QName("", "testcase"), testCase);
xps.SetVariable(new QName("", "moduleuri"), new XdmAtomicValue(moduleUri));
XdmAtomicValue s = (XdmAtomicValue)xps.EvaluateSingle();
return(new Uri[] { new Uri((String)s.Value) });
}
private bool assertDeepEq(XdmNode assertion, SingleResultDoc result, XPathCompiler assertXpc)
{
if (IsException())
{
return(false);
}
else
{
XPathSelector s = assertXpc.Compile("deep-equal($result , (" + assertion.StringValue + "))").Load();
s.SetVariable(new QName("result"), result.value);
return(((XdmAtomicValue)s.Evaluate()).GetBooleanValue());
}
}
private bool AssertType(XdmNode assertion, SingleResultDoc result, XPathCompiler assertXpc)
{
if (IsException())
{
return(false);
}
else
{
XPathSelector s = assertXpc.Compile("$result instance of " + assertion.StringValue).Load();
s.SetVariable(new QName("result"), result.value);
return(((XdmAtomicValue)s.EvaluateSingle()).GetBooleanValue());
}
}
public Uri[] GetModules(String moduleUri, Uri baseUri, String[] locationHints)
{
XPathSelector xps = findModule.Load();
xps.SetVariable(new QName("", "testcase"), testCase);
xps.SetVariable(new QName("", "moduleuri"), new XdmAtomicValue(moduleUri));
ArrayList uris = new ArrayList();
foreach (XdmItem i in xps)
{
uris.Add(new Uri((String)((XdmAtomicValue)i).Value));
}
return((Uri[])uris.ToArray(typeof(Uri)));
}
private bool assertEq(XdmNode assertion, SingleResultDoc result, XPathCompiler assertXpc)
{
if (IsException())
{
return(false);
}
else
{
XPathSelector s = assertXpc.Compile("$result eq " + assertion.StringValue).Load();
s.SetVariable(new QName("result"), result.value);
XdmAtomicValue item = (XdmAtomicValue)s.EvaluateSingle();
return(item != null && item.GetBooleanValue());
}
}
public override void HtmlParser(OnCompleteEventArgs e)
{
try
{
lock (_lock)
{
//解析器
var selector = new XPathSelector(e.Page);
//最热小说
var dtHotNovels = selector.SelectNodes("//*[@id='hotcontent']/div/div/div[1]/a");
//最近更新
var latestNovels = selector.SelectNodes("//*[@id='newscontent']/div[1]/ul/li/span[2]/a");
//点击榜
var mostClickNovels = selector.SelectNodes("//*[@id='newscontent']/div[2]/ul/li/span[2]/a");
IList <string> urls = new List <string>();
foreach (var aEle in dtHotNovels)
{
urls.Add("http://" + e.Host + aEle.GetAttributeValue("href", ""));
}
foreach (var aEle in latestNovels)
{
urls.Add("http://" + e.Host + aEle.GetAttributeValue("href", ""));
}
foreach (var aEle in mostClickNovels)
{
urls.Add("http://" + e.Host + aEle.GetAttributeValue("href", ""));
}
var spiderManager = ContainerManager.Resolve <SpiderManager>();
//启动小说爬虫
spiderManager.RunTask("NovelSpider", new SpiderConfig()
{
Uris = urls
});
}
}
catch (Exception exception)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().FullName,
ActionName = exception.TargetSite.Name,
Msg = e.Uri + "---" + exception.StackTrace
});
}
}
private ProcessResult ProcessResult(XmlNodeList nodes, XPathSelector selector)
{
var pr = new ProcessResult();
switch (selector.Type)
{
case XPathTypeEnum.InnerXml:
{
foreach (XmlNode node in nodes)
{
pr.Matches.Add(node.InnerXml);
}
break;
}
case XPathTypeEnum.InnerText:
{
foreach (XmlNode node in nodes)
{
pr.Matches.Add(node.InnerText);
}
break;
}
case XPathTypeEnum.Attr:
{
foreach (XmlNode node in nodes)
{
if (!string.IsNullOrEmpty(selector.AttrName))
{
var attr = node.Attributes[selector.AttrName].Value;
pr.Matches.Add(attr);
}
}
break;
}
case XPathTypeEnum.OuterXml:
{
foreach (XmlNode node in nodes)
{
pr.Matches.Add(node.OuterXml);
}
break;
}
}
return(pr);
}
private bool assertPermutation(XdmNode assertion, SingleResultDoc result, XPathCompiler assertXpc)
{
// TODO: extend this to handle nodes (if required)
if (IsException())
{
return(false);
}
else
{
return(true);
try {
int expectedItems = 0;
HashSet <string> expected = new HashSet <string>();
XPathSelector s = assertXpc.Compile("(" + assertion.StringValue + ")").Load();
s.SetVariable(new QName("result"), result.value); // not used, but we declared it
JCodepointCollator collator = JCodepointCollator.getInstance();
//JXPathContext context = new JXPathContextMajor(stringValue.EMPTY_string, assertXpc.getUnderlyingStaticContext().getConfiguration());
foreach (XdmItem item in s)
{
expectedItems++;
XdmValue value = (XdmValue)item.Simplify;
// value.Simplify.
/* Object comparable = value.isNaN() ?
* AtomicSortComparer.COLLATION_KEY_NaN :
* value.getXPathComparable(false, collator, context);
* expected.Add(comparable);*/
}
int actualItems = 0;
foreach (XdmItem item in GetPrincipalResult())
{
actualItems++;
XdmValue value = (XdmValue)item.Simplify;
/*Object comparable = value.isNaN() ?
* AtomicSortComparer.COLLATION_KEY_NaN :
* value.getXPathComparable(false, collator, context); */
// if (!expected.Contains(comparable)) {
return(false);
// }
}
return(actualItems == expectedItems);
} catch (DynamicError) {
return(false);
}
}
}
public override void HtmlParser(OnCompleteEventArgs e)
{
try
{
lock (_lock)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
IList <Img> imgs = new List <Img>();
//初始化解析器
var selector = new XPathSelector(e.Page);
var aEles = selector.SelectNodes("//*[@id='main']/div/div[1]/a");
foreach (var aEle in aEles)
{
string url = aEle.GetAttributeValue("href", "");
imgs.Add(new Img()
{
Url = url
});
}
_imageService.AddImages(imgs);
stopWatch.Stop();
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = e.Uri + "请求消耗:" + e.Duration + "---" + "数据解析消耗:" + stopWatch.ElapsedMilliseconds,
ClassName = "",
ActionName = "",
Duration = e.Duration + stopWatch.ElapsedMilliseconds,
LogLevel = (int)LCore.Logger.LogLevel.Info
});
}
}
catch (Exception exception)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().Name,
ActionName = exception.TargetSite.Name,
Msg = e.Uri + "---" + exception.Message
});
}
}
/// <summary>
/// Compare two files using the XML comparator.
/// </summary>
/// <param name="actual">Filename of results obtained in this test run</param>
/// <param name="gold">Filename of reference results (expected results)</param>
/// <returns>true if the results are the same</returns>
private bool compareXML(String actual, String gold)
{
try {
XdmNode doc1 = processor.NewDocumentBuilder().Build(new Uri(actual));
XdmNode doc2 = processor.NewDocumentBuilder().Build(new Uri(gold));
XPathSelector t = compareDocuments.Load();
t.SetVariable(new QName("", "actual"), doc1);
t.SetVariable(new QName("", "gold"), doc2);
t.SetVariable(new QName("", "debug"), new XdmAtomicValue(debug));
XdmAtomicValue result = (XdmAtomicValue)t.EvaluateSingle();
return((bool)result.Value);
} catch (Exception e) {
Console.WriteLine(e.StackTrace);
Console.WriteLine("***" + e.Message);
return(false);
}
}
/// <summary>
/// Compare XML fragments
/// </summary>
/// <param name="actual">Actual results (the results, not the filename)</param>
/// <param name="gold">Reference results (the results, not the filename)</param>
/// <returns>true if equivalent</returns>
private bool compareFragments(String actual, String gold)
{
try {
String a = expandSpecialChars("<d>" + actual + "</d>");
String g = expandSpecialChars("<d>" + actual + "</d>");
XdmNode doc1 = processor.NewDocumentBuilder().Build(
new XmlTextReader(new StringReader(a)));
XdmNode doc2 = processor.NewDocumentBuilder().Build(
new XmlTextReader(new StringReader(g)));
XPathSelector t = compareDocuments.Load();
t.SetVariable(new QName("", "actual"), doc1);
t.SetVariable(new QName("", "gold"), doc2);
t.SetVariable(new QName("", "debug"), new XdmAtomicValue(debug));
XdmAtomicValue result = (XdmAtomicValue)t.EvaluateSingle();
return((bool)result.Value);
} catch (Exception e) {
Console.WriteLine(e.StackTrace);
Console.WriteLine("***" + e.Message);
return(false);
}
}
/// <summary>
/// 页面解析
/// </summary>
/// <param name="pagePathogen"></param>
public void PageProcess(PagePathogen pagePathogen)
{
try
{
//添加请求地址
pagePathogen.AddResult("requestUrl", pagePathogen.Url);
var selector = new XPathSelector(pagePathogen.PageSource);
var node = selector.SelectSingleNode("//*[@id='content']");
if (node != null)
{
pagePathogen.AddResult("article", node.InnerHtml);
}
else
{
//记录爬取日志
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
Msg = pagePathogen.Url + "---未解析到数据！",
ClassName = "",
ActionName = "",
Duration = 0,
LogLevel = (int)LCore.Logger.LogLevel.Warn
});
}
}
catch (Exception e)
{
//记录错误信息
_loggerService.WriteLog(new Log()
{
DateTime = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"),
LogLevel = (int)LCore.Logger.LogLevel.Error,
ClassName = this.GetType().FullName,
ActionName = e.TargetSite.Name,
Msg = pagePathogen.Url + "---" + e.Message + "---" + e.StackTrace
});
}
}
/// <summary>
///
/// </summary>
/// <param name="selector"></param>
/// <param name="novel"></param>
/// <param name="uri"></param>
/// <param name="xpath"></param>
/// <returns></returns>
private void GetArticles(XPathSelector selector, Novel novel, string uri, string xpath)
{
var aEles = selector.SelectNodes(xpath);
if (aEles != null)
{
foreach (var ele in aEles)
{
string articleUri = uri + ele.GetAttributeValue("href", "");
string[] s = articleUri.Split('/');
string uN = s[s.Length - 1];
long htmlFileName = long.Parse(uN.Substring(0, uN.IndexOf('.')));
string articleTitle = ele.InnerText;
var article = new Article()
{
Title = articleTitle,
Url = articleUri,
Novel = novel,
Seq = htmlFileName
};
novel.Articles.Add(article);
}
}
}
protected string runXSLT(String testName, String xml, String xsl, QName initialMode,
QName initialTemplate, String outfile, Hashtable paramTable, String initialContextPath,
bool useAssociated, bool schemaAware,
String validationMode, bool recoverRecoverable)
{
Serializer sr = new Serializer();
sr.SetOutputFile(outfile);
Processor f;
//if (noCacheTests.contains(testName) || testName.startsWith("schemaas20") ||
// testName.startsWith("striptype20") || testName.startsWith("notation20")) {
// create a custom Processor to avoid schema caching
//} else {
if (schemaAware)
{
f = schemaAwareProcessor;
}
else if (xml11)
{
f = processor;
// Use an Xml 1.1 processor
}
else
{
f = processor;
}
//}
XdmNode source = null;
IList errors = new ArrayList();
XsltCompiler compiler = f.NewXsltCompiler();
compiler.ErrorList = errors;
XsltExecutable sheet = null;
XsltTransformer inst;
if (useAssociated)
{
try {
source = buildSource(f.NewDocumentBuilder(), xml, validationMode);
} catch (Exception e) {
Console.WriteLine("Failed to build source document: " + e.Message);
return("ErrorBBB");
}
try {
sheet = compiler.CompileAssociatedStylesheet(source);
} catch (Exception e) {
Console.WriteLine("Failed to compile stylesheet: " + e.Message);
if (errors.Count > 0)
{
QName code = ((StaticError)errors[0]).ErrorCode;
return(code == null ? "ErrorXXX" : code.LocalName);
}
else
{
return("ErrorXXX");
}
}
}
else
{
Stream stream = new FileStream(xsl, FileMode.Open, FileAccess.Read);
compiler.BaseUri = new Uri(xsl);
try {
sheet = compiler.Compile(stream);
} catch (Exception e) {
if (errors.Count > 0)
{
return(((StaticError)errors[0]).ErrorCode.LocalName);
}
else
{
Console.WriteLine(e.Message);
return("ErrorXXX");
}
} finally {
stream.Close();
}
}
if (source == null && xml != null)
{
try {
source = buildSource(f.NewDocumentBuilder(), xml, validationMode);
} catch (Exception e) {
Console.WriteLine("Failed to build source document: " + e.Message);
return("ErrorCCC");
}
}
if (initialContextPath != null)
{
XPathCompiler xc = f.NewXPathCompiler();
XPathExecutable exp = xc.Compile(initialContextPath);
XPathSelector xpe = exp.Load();
xpe.ContextItem = source;
XdmNode node = (XdmNode)xpe.EvaluateSingle();
source = node;
}
inst = sheet.Load();
if (initialMode != null)
{
inst.InitialMode = initialMode;
}
if (initialTemplate != null)
{
try {
inst.InitialTemplate = initialTemplate;
} catch (DynamicError e) {
QName code = e.ErrorCode;
if (code != null)
{
return(code.LocalName);
}
else
{
return("ErrYYYYY");
}
}
}
if (paramTable != null)
{
foreach (DictionaryEntry de in paramTable)
{
inst.SetParameter((QName)de.Key, new XdmAtomicValue(de.Value.ToString()));
}
}
inst.InitialContextNode = source;
//inst.setURIResolver(factory.getURIResolver());
//inst.setErrorListener(errorListener);
//((Controller)inst).setRecoveryPolicy(recoverRecoverable ? Configuration.RECOVER_SILENTLY : Configuration.DO_NOT_RECOVER);
// To avoid test results being dependent on the date and time (and timezone), set a fixed
// date and time for the run
//((Controller)inst).setCurrentDateTime(new DateTimeValue("2005-01-01T12:49:30.5+01:00"));
try {
inst.Run(sr);
} catch (DynamicError e) {
Console.WriteLine(e.Message);
QName code = e.ErrorCode;
if (code != null)
{
return(code.LocalName);
}
else
{
return("ErrYYYYY");
}
}
return(null); // indicating success
}
/// <summary>
/// Get the name from the "<div class="caption">" tag
/// We have to use this method as we can't use "selector" inside "selector" - "startindex" out of range exception
/// </summary>
/// <param name="text">Text with html tags</param>
/// <returns></returns>
private string NameResolver(string text)
{
var htmlAdapter = new XPathSelector();
return(htmlAdapter.Select(text, "//div[contains(@class, 'caption')]").InnerText);
}
public void go(String[] args)
{
Console.WriteLine("Testing Saxon " + processor.ProductVersion);
testSuiteDir = args[0];
if (testSuiteDir.EndsWith("/"))
{
testSuiteDir = testSuiteDir.Substring(0, testSuiteDir.Length - 1);
}
saxonResultsDir = args[1];
if (saxonResultsDir.EndsWith("/"))
{
saxonResultsDir = saxonResultsDir.Substring(0, testSuiteDir.Length - 1);
}
Hashtable exceptions = new Hashtable();
if (args.Length > 1)
{
testPattern = (args[2]); // TODO: allow a regex
}
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-w"))
{
//showWarnings = true;
}
else if (args[i].Equals("-debug"))
{
debug = true;
}
}
fileComparer = new FileComparer(processor, testSuiteDir);
XPathCompiler xpc = processor.NewXPathCompiler();
xpc.DeclareNamespace("t", testURI);
xpc.DeclareVariable(new QName("", "param"));
findSourcePath = xpc.Compile("//t:test-suite/t:sources/t:source[@ID=$param]");
findCollection = xpc.Compile("//t:test-suite/t:sources/t:collection[@ID=$param]");
xpc = processor.NewXPathCompiler();
xpc.DeclareNamespace("t", testURI);
xpc.DeclareVariable(new QName("", "testcase"));
xpc.DeclareVariable(new QName("", "moduleuri"));
findModule = xpc.Compile("for $m in $testcase/t:module[@namespace=$moduleuri] " +
"return concat('file:///" + testSuiteDir +
"/', root($testcase)/t:test-suite/t:sources/t:module[@ID=string($m)]/@FileName, '.xq')");
//xpc = processor.NewXPathCompiler();
//xpc.DeclareNamespace("saxon", "http://saxon.sf.net/");
//xpc.DeclareVariable(new QName("", "actual"));
//xpc.DeclareVariable(new QName("", "gold"));
//xpc.DeclareVariable(new QName("", "debug"));
//compareDocuments = xpc.Compile("saxon:deep-equal($actual, $gold, (), if ($debug) then 'JNCPS?!' else 'JNCPS')");
QName testCaseNT = new QName(testURI, "test-case");
QName nameNT = new QName(testURI, "name");
QName queryNT = new QName(testURI, "query");
QName inputNT = new QName(testURI, "input");
QName inputFileNT = new QName(testURI, "input-file");
QName inputUriNT = new QName(testURI, "input-URI");
QName defaultCollectionNT = new QName(testURI, "defaultCollection");
QName outputFileNT = new QName(testURI, "output-file");
QName expectedErrorNT = new QName(testURI, "expected-error");
QName schemaNT = new QName(testURI, "schema");
QName contextItemNT = new QName(testURI, "contextItem");
QName inputQueryNT = new QName(testURI, "input-query");
QName sourceDocumentNT = new QName(testURI, "source-document");
QName errorNT = new QName(testURI, "error");
QName validationNT = new QName(testURI, "validation");
QName discretionaryItemsNT = new QName(testURI, "discretionary-items");
QName discretionaryFeatureNT = new QName(testURI, "discretionary-feature");
QName discretionaryChoiceNT = new QName(testURI, "discretionary-choice");
QName initialContextNodeNT = new QName(testURI, "initial-context-node");
QName fileAtt = new QName("", "file");
QName filePathAtt = new QName("", "FilePath");
QName fileNameAtt = new QName("", "FileName");
QName errorIdAtt = new QName("", "error-id");
QName compareAtt = new QName("", "compare");
QName nameAtt = new QName("", "name");
QName behaviorAtt = new QName("", "behavior");
QName qnameAtt = new QName("", "qname");
QName modeAtt = new QName("", "mode");
QName validatesAtt = new QName("", "validates");
QName variableAtt = new QName("", "variable");
QName roleAtt = new QName("", "role");
DocumentBuilder builder = processor.NewDocumentBuilder();
XdmNode exceptionsDoc = builder.Build(new Uri(saxonResultsDir + "/exceptions.xml"));
// The exceptions.xml file contains details of tests that aren't to be run, for example
// because they have known bugs or require special configuration
IEnumerator exceptionTestCases = exceptionsDoc.EnumerateAxis(XdmAxis.Descendant, new QName("", "exception"));
while (exceptionTestCases.MoveNext())
{
XdmNode n = (XdmNode)exceptionTestCases.Current;
String nameAttVal = n.StringValue;
char[] seps = { ' ', '\n', '\t' };
String[] parts = nameAttVal.Split(seps);
foreach (string p in parts)
{
if (!exceptions.ContainsKey(p))
{
exceptions.Add(p, "Exception");
}
}
}
// Hash table containing all source documents. The key is the document name in the
// catalog, the value is the corresponding document node
Hashtable sourceDocs = new Hashtable(50);
// Load the catalog
XdmNode catalog = builder.Build(new Uri(testSuiteDir + "/XQTScatalog.xml"));
// Add all Static Typing test cases to the exceptions list
xpc = processor.NewXPathCompiler();
xpc.DeclareNamespace("t", testURI);
XPathSelector st = xpc.Compile("//t:test-group[@name='StaticTyping']//t:test-case").Load();
st.ContextItem = catalog;
IEnumerator ste = st.GetEnumerator();
while (ste.MoveNext())
{
XdmNode testCase = (XdmNode)ste.Current;
exceptions.Add(testCase.GetAttributeValue(nameAtt), "StaticTypingException");
}
// Create the results file
results = new StreamWriter(saxonResultsDir + "/results"
+ processor.ProductVersion + "n.xml");
results.WriteLine("<test-suite-result xmlns='http://www.w3.org/2005/02/query-test-XQTSResult'>");
// Pre-load all the schemas
SchemaManager mgr = processor.SchemaManager;
IEnumerator se = catalog.EnumerateAxis(XdmAxis.Descendant, schemaNT);
while (se.MoveNext())
{
XdmNode schemaNode = (XdmNode)se.Current;
Console.WriteLine("Loading schema " + schemaNode.GetAttributeValue(fileNameAtt));
Uri location = new Uri(testSuiteDir + "/" + schemaNode.GetAttributeValue(fileNameAtt));
mgr.Compile(location);
}
total = 0;
IEnumerator testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT);
while (testCases.MoveNext())
{
total++;
}
// Process the test cases in turn
testCases = catalog.EnumerateAxis(XdmAxis.Descendant, testCaseNT);
while (testCases.MoveNext())
{
XdmNode testCase = (XdmNode)testCases.Current;
String testName = testCase.GetAttributeValue(nameAtt);
if (testPattern != null && !testName.StartsWith(testPattern))
{
continue;
}
if (exceptions.ContainsKey(testName))
{
continue;
}
Console.WriteLine("Test " + testName);
// Compile the query
String errorCode = null;
String filePath = testCase.GetAttributeValue(filePathAtt);
XdmNode query = getChildElement(testCase, queryNT);
String queryName = query.GetAttributeValue(nameAtt);
String queryPath = testSuiteDir + "/Queries/XQuery/" + filePath + queryName + ".xq";
XQueryCompiler compiler = processor.NewXQueryCompiler();
compiler.BaseUri = new Uri(queryPath).ToString();
compiler.QueryResolver = new XqtsModuleResolver(testCase, findModule);
ArrayList errors = new ArrayList();
compiler.ErrorList = errors;
XQueryEvaluator xqe = null;
FileStream stream = null;
try
{
stream = new FileStream(queryPath, FileMode.Open, FileAccess.Read, FileShare.Read);
xqe = compiler.Compile(stream).Load();
}
catch (Exception e)
{
if (errors.Count > 0 && ((StaticError)errors[0]).ErrorCode != null)
{
errorCode = ((StaticError)errors[0]).ErrorCode.LocalName;
}
else if (e is StaticError && ((StaticError)e).ErrorCode != null)
{
Console.WriteLine(e.Message);
errorCode = ((StaticError)e).ErrorCode.LocalName;
}
else
{
Console.WriteLine(e.Message);
errorCode = "ErrorXXX";
}
}
finally
{
if (stream != null)
{
stream.Close();
}
}
// if the query compiled successfully, try to run it
String outputPath = null;
if (errorCode == null && xqe != null)
{
// Supply any input documents
IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, inputFileNT);
while (en.MoveNext())
{
XdmNode file = (XdmNode)en.Current;
String var = file.GetAttributeValue(variableAtt);
if (var != null)
{
String sourceName = file.StringValue;
XdmNode sourceDoc;
if (sourceDocs.ContainsKey(sourceName))
{
sourceDoc = (XdmNode)sourceDocs[sourceName];
}
else
{
sourceDoc = buildSource(catalog, builder, sourceName);
sourceDocs.Add(sourceName, sourceDoc);
}
xqe.SetExternalVariable(new QName("", var), sourceDoc);
}
}
// Supply any input URIs
IEnumerator eu = testCase.EnumerateAxis(XdmAxis.Child, inputUriNT);
while (eu.MoveNext())
{
XdmNode file = (XdmNode)eu.Current;
String var = file.GetAttributeValue(variableAtt);
if (var != null)
{
String sourceName = file.StringValue;
if (sourceName.StartsWith("collection"))
{
// Supply a collection URI.
// This seems to be the only way to distinguish a document URI
// from a collection URI.
String uri = "collection:" + sourceName;
XPathSelector xpe = findCollection.Load();
xpe.SetVariable(new QName("", "param"), new XdmAtomicValue(sourceName));
xpe.ContextItem = catalog;
XdmNode collectionNode = (XdmNode)xpe.EvaluateSingle();
if (collectionNode == null)
{
Console.WriteLine("*** Collection " + sourceName + " not found");
}
processor.RegisterCollection(new Uri(uri), getCollection(collectionNode));
xqe.SetExternalVariable(new QName("", var), new XdmAtomicValue(uri));
}
else
{
// Supply a document URI.
// We exploit the fact that the short name of the document is
// always the same as the file name in these tests
String uri = "file:///" + testSuiteDir + "/TestSources/" + sourceName + ".xml";
xqe.SetExternalVariable(new QName("", var), new XdmAtomicValue(uri));
}
}
}
// Supply the default collection if required
XdmNode defaultCollection = getChildElement(testCase, defaultCollectionNT);
if (defaultCollection != null)
{
String sourceName = defaultCollection.StringValue;
XPathSelector xpe = findCollection.Load();
xpe.SetVariable(new QName("", "param"), new XdmAtomicValue(sourceName));
xpe.ContextItem = catalog;
XdmNode collectionNode = (XdmNode)xpe.EvaluateSingle();
if (collectionNode == null)
{
Console.WriteLine("*** Collection " + sourceName + " not found");
}
processor.RegisterCollection(null, getCollection(collectionNode));
}
// Supply any external variables defined as the result of a separate query
IEnumerator ev = testCase.EnumerateAxis(XdmAxis.Child, inputQueryNT);
while (ev.MoveNext())
{
XdmNode inputQuery = (XdmNode)ev.Current;
String fileName = inputQuery.GetAttributeValue(nameAtt);
String subQueryPath = testSuiteDir + "/Queries/XQuery/" + filePath + fileName + ".xq";
XQueryCompiler subCompiler = processor.NewXQueryCompiler();
compiler.BaseUri = new Uri(subQueryPath).ToString();
FileStream subStream = new FileStream(subQueryPath, FileMode.Open, FileAccess.Read, FileShare.Read);
XdmValue value = subCompiler.Compile(subStream).Load().Evaluate();
String var = inputQuery.GetAttributeValue(variableAtt);
xqe.SetExternalVariable(new QName("", var), value);
}
// Supply the context item if required
IEnumerator ci = testCase.EnumerateAxis(XdmAxis.Child, contextItemNT);
while (ci.MoveNext())
{
XdmNode file = (XdmNode)ci.Current;
String sourceName = file.StringValue;
if (!sourceDocs.ContainsKey(sourceName))
{
XdmNode doc = buildSource(catalog, builder, sourceName);
sourceDocs.Add(sourceName, doc);
}
XdmNode sourceDoc = (XdmNode)sourceDocs[sourceName];
xqe.ContextItem = sourceDoc;
}
// Create a serializer for the output
outputPath = saxonResultsDir + "/results.net/" + filePath + queryName + ".out";
Serializer sr = new Serializer();
try
{
sr.SetOutputFile(outputPath);
sr.SetOutputProperty(new QName("", "method"), "xml");
sr.SetOutputProperty(new QName("", "omit-xml-declaration"), "yes");
sr.SetOutputProperty(new QName("", "indent"), "no");
}
catch (DynamicError)
{
// probably means that no output directory exists, which is probably because
// an error is expected
outputPath = saxonResultsDir + "/results.net/" + queryName + ".out";
sr.SetOutputFile(outputPath);
}
// Finally, run the query
try
{
xqe.Run(sr);
}
catch (DynamicError e)
{
Console.WriteLine(e.Message);
QName code = e.ErrorCode;
if (code != null && code.LocalName != null)
{
errorCode = code.LocalName;
}
else
{
errorCode = "ErrYYYYY";
}
}
catch (Exception e2)
{
Console.WriteLine("Unexpected exception: " + e2.Message);
Console.WriteLine(e2.StackTrace);
errorCode = "CRASH!!!";
}
}
// Compare actual results with expected results
if (errorCode != null)
{
// query returned an error at compile time or run-time, check this was expected
string expectedError = "";
bool matched = false;
IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, expectedErrorNT);
while (en.MoveNext())
{
XdmNode error = (XdmNode)en.Current;
String expectedErrorCode = error.StringValue;
expectedError += (expectedErrorCode + " ");
if (expectedErrorCode.Equals(errorCode))
{
matched = true;
feedback.Feedback(passed++, failed, total);
Console.WriteLine("Error " + errorCode + " as expected");
results.WriteLine("<test-case name='" + testName + "' result='pass'/>");
break;
}
}
if (!matched)
{
if (expectedError.Equals(""))
{
feedback.Feedback(passed, failed++, total);
Console.WriteLine("Error " + errorCode + ", expected success");
results.WriteLine("<test-case name='" + testName + "' result='fail' comment='error " + errorCode + ", expected success'/>");
}
else
{
feedback.Feedback(passed++, failed, total);
Console.WriteLine("Error " + errorCode + ", expected " + expectedError);
results.WriteLine("<test-case name='" + testName + "' result='pass' comment='error " + errorCode + ", expected " + expectedError + "'/>");
}
}
}
else
{
// query returned no error
bool matched = false;
String diag = "";
IEnumerator en = testCase.EnumerateAxis(XdmAxis.Child, outputFileNT);
while (en.MoveNext())
{
XdmNode outputFile = (XdmNode)en.Current;
String fileName = testSuiteDir + "/ExpectedTestResults/" + filePath + outputFile.StringValue;
String comparator = outputFile.GetAttributeValue(compareAtt);
if (comparator.Equals("Inspect"))
{
matched = true;
feedback.Feedback(passed++, failed, total);
results.WriteLine("<test-case name='" + testName + "' result='inspect'/>");
break;
}
else
{
String comparison = fileComparer.compare(outputPath, fileName, comparator);
matched = (comparison == "OK" || comparison.StartsWith("#"));
if (matched)
{
feedback.Feedback(passed++, failed, total);
results.WriteLine("<test-case name='" + testName + "' result='pass'/>");
diag = diag + ("<!-- " + comparison + " -->\n");
break;
}
}
}
if (!matched)
{
string expectedError = "";
IEnumerator ee = testCase.EnumerateAxis(XdmAxis.Child, expectedErrorNT);
while (ee.MoveNext())
{
XdmNode error = (XdmNode)ee.Current;
String expectedErrorCode = error.StringValue;
expectedError += (expectedErrorCode + " ");
}
if (expectedError.Equals(""))
{
feedback.Feedback(passed, failed++, total);
Console.WriteLine("Results differ from expected results");
results.WriteLine("<test-case name='" + testName + "' result='fail'/>");
}
else
{
feedback.Feedback(passed, failed++, total);
Console.WriteLine("Error " + expectedError + "expected but not reported");
results.WriteLine("<test-case name='" + testName + "' result='fail' comment='expected error " + expectedError + "not reported'/>");
}
}
}
}
results.WriteLine("</test-suite-result>");
results.Close();
}
