public Dictionary <Int64, PageIndex> ReadPageIndices()
{
string coreFilename = Path.Combine(this.inputFolder, $"{this.wikiName}{XowaParser.CoreSuffix}");
string sql = "SELECT page_id, page_title, page_is_redirect, page_redirect_id FROM page";
Dictionary <Int64, PageIndex> pageIndices = new Dictionary <Int64, PageIndex>();
System.Console.WriteLine("Reading page indices...");
XowaParser.ExecuteSql(coreFilename, sql, (reader) =>
{
Int64 pageId = (Int64)reader["page_id"];
string title = (string)reader["page_title"];
bool isRedirect = (Int64)reader["page_is_redirect"] == 1;
Int64 redirectId = (Int64)reader["page_redirect_id"];
if (isRedirect)
{
PageIndex idx;
if (pageIndices.TryGetValue(redirectId, out idx))
{
idx.SecondaryTitles.Add(title);
}
else
{
// New page with secondary title but no primary title
pageIndices.Add(redirectId, new PageIndex
{
Id = redirectId,
PrimaryTitle = "<Unset>",
SecondaryTitles = new List <string>
{
title
}
});
}
}
else
{
PageIndex idx;
if (pageIndices.TryGetValue(pageId, out idx))
{
// A redirect added the main title for us.
idx.PrimaryTitle = title;
}
else
{
// New page with primary title and no secondary titles
pageIndices.Add(pageId, new PageIndex
{
Id = pageId,
PrimaryTitle = title,
SecondaryTitles = new List <string>()
});
}
}
});
System.Console.WriteLine($"Read {pageIndices.Count} unique page indices.");
return(pageIndices);
}
public WikiPageList ReadHtmlDbFile(string filename, Dictionary <Int64, PageIndex> pageIndices)
{
string sql = "SELECT page_id, body FROM html";
WikiPageList pages = new WikiPageList();
System.Console.WriteLine($"Reading HTML files from {Path.GetFileName(filename)}...");
XowaParser.ExecuteSql(filename, sql, (reader) =>
{
Int64 pageId = (Int64)reader["page_id"];
byte[] body = (byte[])reader["body"];
WikiPage page = new WikiPage()
{
Id = pageId
};
// Decode body (GZip'd data)
using (MemoryStream stream = new MemoryStream(body))
using (GZipInputStream gzipStream = new GZipInputStream(stream))
using (StreamReader streamReader = new StreamReader(gzipStream))
{
page.Content = streamReader.ReadToEnd();
}
// Associated and save
PageIndex idx = null;
if (pageIndices.TryGetValue(pageId, out idx))
{
page.PrimaryTitle = idx.PrimaryTitle;
page.SecondaryTitles = idx.SecondaryTitles;
pages.ValidPages.Add(page);
}
else
{
page.PrimaryTitle = "<Unknown>";
page.SecondaryTitles = new List <string>();
pages.InvalidPages.Add(page);
}
});
System.Console.WriteLine($"Read {pages.ValidPages.Count} total valid pages and {pages.InvalidPages.Count} invalid pages.");
return(pages);
}
/// <summary>
/// TODO: Deduplicate
/// </summary>
public CompressedWikiPageList ReadCompressedHtmlDbFile(string filename, Dictionary <Int64, PageIndex> pageIndices)
{
string sql = "SELECT page_id, body FROM html";
CompressedWikiPageList pages = new CompressedWikiPageList();
System.Console.WriteLine($"Reading HTML files from {Path.GetFileName(filename)}...");
XowaParser.ExecuteSql(filename, sql, (reader) =>
{
Int64 pageId = (Int64)reader["page_id"];
byte[] body = (byte[])reader["body"];
CompressedWikiPage page = new CompressedWikiPage()
{
Id = pageId,
Content = body
};
// Associated and save
PageIndex idx = null;
if (pageIndices.TryGetValue(pageId, out idx))
{
page.PrimaryTitle = idx.PrimaryTitle;
page.SecondaryTitles = idx.SecondaryTitles;
pages.ValidPages.Add(page);
}
else
{
page.PrimaryTitle = "<Unknown>";
page.SecondaryTitles = new List <string>();
pages.InvalidPages.Add(page);
}
});
System.Console.WriteLine($"Read {pages.ValidPages.Count} total valid pages and {pages.InvalidPages.Count} invalid pages.");
return(pages);
}