private static void Links(PagesSource <Page> pages)
{
var source = pages
.Select(p => PageResult.Create(p.info, p.links().Select(l => l.title).ToEnumerable()));
Write(source);
}
private static void CategoryInfo(PagesSource <Page> pages)
{
var source = pages.Select(p => new { p.info.title, p.categoryinfo.pages, p.categoryinfo.hidden })
.ToEnumerable();
Write(source);
}
private static void Categories(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
new
{
p.info,
categories =
p.categories()
.Where(c => c.show == categoriesshow.not_hidden)
.Select(c => new { c.title, c.sortkeyprefix })
.ToEnumerable()
.Take(1)
}
);
foreach (var page in source.ToEnumerable().Take(10))
{
Console.WriteLine(page.info.title);
foreach (var item in page.categories.Take(10))
{
Console.WriteLine(" " + item);
}
}
}
private static void IwLinks(PagesSource <Page> pages)
{
var source = pages
.Select(p => PageResult.Create(p.info, p.iwlinks().ToEnumerable()))
.ToEnumerable()
.Where(p => p.Data.Any());
Write(source);
}
private static void Revisions(PagesSource <Page> pages)
{
var source =
pages.Select(
p =>
PageResult.Create(p.info, p.revisions().OrderBy(r => r).Select(r => r.user).ToEnumerable().First()))
.ToEnumerable();
Write(source);
}
private static void LangLinks(PagesSource <Page> pages)
{
var source = pages
.Select(
p => PageResult.Create(
p.info,
p.langlinks().Where(l => l.url).ToEnumerable()))
.ToEnumerable();
Write(source);
}
private static void Images(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
PageResult.Create(
p.info,
p.images().ToEnumerable())
);
Write(source);
}
private static void ImageInfo(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
PageResult.Create(
p.info,
p.imageinfo()
.Select(i => new { i.timestamp, i.comment })
.ToEnumerable())
);
Write(source);
}
private static void ExtLinks(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
PageResult.Create(
p.info,
p.extlinks()
.Where(l => l.query == "toolserver.org")
.Select(l => l.value)
.ToEnumerable())
);
Write(source);
}
private static void Templates(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
PageResult.Create(
p.info,
p.templates()
.Where(t => t.ns == new[] { Namespace.Project, Namespace.User, Namespace.Article })
.Select(t => t.title)
.ToEnumerable()))
.ToEnumerable()
.Where(p => p.Data.Any());
Write(source);
}
private static void PageResultProps(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
PageResult.Create(
p.info,
p.categories()
.Where(c => c.show == categoriesshow.not_hidden)
.OrderByDescending(c => c)
.Select(c => new { c.title, c.sortkeyprefix })
.ToEnumerable())
);
Write(source);
}
private static void DuplicateFiles(PagesSource <Page> pages)
{
var source = pages
.Select(
p =>
new
{
p.info.title,
duplicatefiles = p.duplicatefiles().Select(d => d.name)
.ToEnumerable(),
}
);
foreach (var page in source.ToEnumerable().Where(i => i.duplicatefiles.Any()).Take(5))
{
Console.WriteLine(page.title);
foreach (var item in page.duplicatefiles.Take(10))
{
Console.WriteLine(" " + item);
}
}
}
private IEnumerable <LeechedResourceBase> leechPages(PagesSource <Page> pagesSource, WikipediaItem parent)
{
IEnumerator <infoResult> pagesEnumerator = null;
try
{
pagesEnumerator = pagesSource.Select(p => p.info).ToEnumerable().GetEnumerator();
}
catch (Exception ex)
{
Debug.WriteLine("Exception encountered enumerating the pages of \"{0}\": {1}", parent.Title, ex);
}
if (pagesEnumerator != null)
{
infoResult page = null;
do
{
try
{
if (!pagesEnumerator.MoveNext())
{
page = null;
}
else
{
page = pagesEnumerator.Current;
}
}
catch (Exception ex)
{
Debug.WriteLine("Exception encountered leeching the childs of \"{0}\": {1}", parent.Title, ex);
}
if (page != null)
{
WikipediaItem pageItem = new WikipediaItem(page, parent);
yield return(pageItem);
if (pageItem.Links != null &&
pageItem.Links.Count > 0)
{
foreach (string link in pageItem.Links)
{
IEnumerator <LeechedResourceBase> childPagesEnumerator = null;
try
{
var childPages = _wiki.Query.allpages()
.Where(c => c.from == link &&
c.to == link)
.Pages;
childPagesEnumerator = leechPages(childPages, pageItem).GetEnumerator();
}
catch (Exception ex)
{
Debug.WriteLine("Exception encountered enumerating the link \"{2}\" of \"{0}\": {1}", pageItem.Title, ex, link);
}
if (childPagesEnumerator != null)
{
LeechedResourceBase childPage = null;
do
{
try
{
if (!childPagesEnumerator.MoveNext())
{
childPage = null;
}
else
{
childPage = childPagesEnumerator.Current;
}
}
catch (Exception ex)
{
Debug.WriteLine("Exception encountered leeching the link \"{2}\" of \"{0}\": {1}", pageItem.Title, ex, link);
}
if (childPage != null)
{
yield return(childPage);
}
} while (childPage != null);
}
}
}
}
} while (page != null);
}
}
private WikipediaResourceInformation extractResourceInformation(WikipediaItem resource, PagesSource <Page> page)
{
WikipediaResourceInformation ret;
try
{
ret = page.Select(p => new WikipediaResourceInformation
{
Content = resource.Title + Environment.NewLine
+ p.revisions().Select(r => r.value).ToEnumerable().FirstOrDefault(),
Categories = p.categories().Where(c => c.show == categoriesshow.not_hidden)
.Select(c => new WikipediaCategory {
Name = c.title
})
.ToList().Cast <ResourceCategory>().ToList(),
Links = resource.Depth < _maximumDepth
? p.links().Where(l => l.ns == Namespace.Article).Select(l => l.title).ToList()
: new List <string>()
})
.ToEnumerable().FirstOrDefault();
if (ret != null && (ret.Categories == null || ret.Categories.Count == 0))
{
ret.Categories = new List <ResourceCategory> {
new WikipediaCategory {
Name = resource.ParentItem.Title
}
};
}
}
catch (Exception ex)
{
Debug.WriteLine("Error occurred extracting the Resource Information: {0}{1} * Resource Title: {2}",
ex,
Environment.NewLine,
resource.Title);
ret = null;
}
return(ret);
}
