//Create a table with post-soviet ranges to push to PostGreqsql later
static public void DecodePageNames(String f1, String f2)
{
StreamReader sr = new StreamReader(f1, Encoding.UTF8);
StreamWriter sw = new StreamWriter(f2, false);
Dictionary <String, String> mapping = new Dictionary <string, string>();
String str = "";
while ((str = sr.ReadLine()) != null)
{
if (str == "")
{
continue;
}
String[] items = str.Split('\t');
if (!mapping.ContainsKey(items[1]))
{
String import = QuotedPrintable.DecodeQuotedPrintable(items[1], "UTF-8");
import = QuotedPrintable.DecodeQuotedPrintable(import, "UTF-8");
mapping.Add(items[1], QuotedPrintable.DecodeQuotedPrintable(import, "UTF-8"));
}
items[1] = mapping[items[1]];
str = "";
foreach (String s in items)
{
str += s + "\t";
}
str = str.Substring(0, str.Length - 1);
sw.WriteLine(str);
}
sr.Close();
sw.Close();
}
static public List <List <String> > ReturnWikiCategory(SortedDictionary <String, Int64> keys)
{
List <List <String> > buf = new List <List <String> >();
var document = new HtmlDocument();
var client = new WebClient();
bool over = true;
String baseString = "https://ru.wikipedia.org";
//foreach (Char c in Common.alpha)
String tail = "/w/index.php?title=Служебная:Все_страницы&from=%28hed%29+P.E.&namespace=14";
do
{
var stream = client.OpenRead(baseString + tail);
var reader = new StreamReader(stream, Encoding.GetEncoding("UTF-8"));
var html = reader.ReadToEnd();
document.LoadHtml(html);
String tags = "//li";
HtmlNodeCollection nodes = document.DocumentNode.SelectNodes(tags);
foreach (HtmlNode node in nodes)
{
if (!node.InnerText.Contains("Категория:"))
{
break;
}
String value = node.InnerText.Substring("Категория:".Length).ToLower();
foreach (String key in keys.Keys)
{
if (value.Contains(key.ToLower()))
{
List <String> pair = new List <string>();
pair.Add(key);
pair.Add(node.InnerText.Substring("Категория:".Length));
pair.Add("14");
buf.Add(pair);
}
}
}
tags = "//div[@class='mw-allpages-nav']";
nodes = document.DocumentNode.SelectNodes(tags);
foreach (HtmlNode node in nodes)
{
String val = node.InnerHtml;
if (node.InnerText.Contains("|"))
{
val = val.Substring(val.IndexOf("|"));
}
if (node.InnerText.Contains("Следующая страница"))
{
int firstpos = val.IndexOf("\"");
int lastpos = val.Substring(firstpos + 1).IndexOf("\"");
tail = val.Substring(firstpos + 1, lastpos);
tail = QuotedPrintable.DecodeQuotedPrintable(tail, "Привет");
tail = tail.Replace("amp;", "");
goto jump;
}
}
over = false;
jump :;
} while (over);
return(buf);
}