public static HashSet <String> DataTableToHashSet(DataTable dt)
{
HashSet <String> hs = new HashSet <string>();
foreach (DataRow dr in dt.Rows)
{
hs.Add(QuotedPrintable.EncodeQuotedPrintable(dr[0].ToString().Trim()));
}
return(hs);
}
public static void PrintCategogy(StreamWriter sw, String page, String source, Boolean pageFlag)
{
Boolean first = pageFlag;
foreach (String key in Common.keys.Keys)
{
if (source.Contains(":" + key) && first)
{
sw.WriteLine(page + "\t" + QuotedPrintable.EncodeQuotedPrintable(page) + "\t" + key);
first = false;
Common.keysPower[key] = 1;
continue;
}
if (source.Contains(":" + key))
{
Common.keysPower[key]++;
}
}
}
static public void ParseXMLwikiForEdits(String file)
{
XmlReader XMLreader = XmlReader.Create(new StreamReader(Common.wikixmlFile, Encoding.UTF8));
PostGrePlugIn.openConnection();
Console.OutputEncoding = Encoding.Unicode;
StreamWriter editsStream = new StreamWriter(Common.editsFile, false, Encoding.UTF8);
StreamWriter editorsStream = new StreamWriter(Common.editorsFile, false, Encoding.UTF8);
editsStream.Close();
editorsStream.Close();
List <String> editsStore = new List <string>();
Dictionary <String, List <Object> > editorsStore = new Dictionary <string, List <Object> >();
Int64 editsCounter = 0;
Int64 editorsCounter = 0;
Boolean flagReadTitle = false;
Boolean flagReadBody = false;
Boolean flagReadPage = false;
String pageName = "";
String oldPage = "";
Boolean pageFlag = false;
Boolean readRevisionMode = false;
String editorID = "";
String comment = "";
String pageID = "";
try
{
while (XMLreader.Read())
{
if (XMLreader.Name == "title" && (XMLreader.NodeType == XmlNodeType.Element))
{
flagReadTitle = true;
continue;
}
if (XMLreader.Name == "title" && (XMLreader.NodeType == XmlNodeType.EndElement))
{
flagReadTitle = false;
continue;
}
if (XMLreader.Name == "page" && (XMLreader.NodeType == XmlNodeType.Element))
{
flagReadPage = true;
continue;
}
if (XMLreader.Name == "page" && (XMLreader.NodeType == XmlNodeType.EndElement))
{
flagReadPage = false;
readRevisionMode = false;
continue;
}
if (XMLreader.Name == "text" && (XMLreader.NodeType == XmlNodeType.Element))
{
flagReadBody = true;
continue;
}
if (XMLreader.Name == "text" && (XMLreader.NodeType == XmlNodeType.EndElement))
{
flagReadBody = false;
continue;
}
if (XMLreader.NodeType == XmlNodeType.Text && flagReadTitle)
{
pageName = XMLreader.Value;
if (Common.interestPages.Contains(QuotedPrintable.EncodeQuotedPrintable(pageName)))
{
readRevisionMode = true;
}
while (XMLreader.Name != "id")
{
XMLreader.Read();
}
XMLreader.Read();
pageID = XMLreader.Value;
flagReadTitle = false;
}
if (XMLreader.Name == "revision" && XMLreader.NodeType == XmlNodeType.Element)
{
String timeString = "";
DateTime day = new DateTime();
String ip = "NA";
String country = "NA";
Byte troll = 0;
Int32 typetroll = -1;
String name = "NA";
Int32 nameID = -1;
Double distance = -1;
Byte inGroup = 0;
if (readRevisionMode)
{
inGroup = 1;
}
while (!(XMLreader.NodeType == XmlNodeType.EndElement && XMLreader.Name == "revision"))
{
XMLreader.Read();
if (XMLreader.NodeType == XmlNodeType.Element && XMLreader.Name == "timestamp")
{
XMLreader.Read();
timeString = XMLreader.Value;
day = DateTime.Parse(timeString);
XMLreader.Read();
}
if (XMLreader.NodeType == XmlNodeType.Element && XMLreader.Name == "contributor")
{
while (XMLreader.Name != "ip" && XMLreader.Name != "username")
{
XMLreader.Read();
}
if (XMLreader.Name == "ip")
{
XMLreader.Read();
ip = editorID = XMLreader.Value;
country = PostGrePlugIn.DataTableToList(PostGrePlugIn.getTablePostGre(PostGrePlugIn.ReturnIpQuery("countryipranges", ip, "country")))[0];
PostGrePlugIn.GetTrollResults(PostGrePlugIn.getTablePostGre(PostGrePlugIn.ReturnIpQuery("troll_bases", ip, " c1,c2,c3,c4, distance ")), out troll, out distance, out typetroll);
}
else
{
XMLreader.Read();
name = XMLreader.Value.Replace(@"\", "_slash_");
while (XMLreader.Name != "id")
{
XMLreader.Read();
}
XMLreader.Read();
nameID = Int32.Parse(XMLreader.Value);
editorID = nameID.ToString();
}
while (XMLreader.Name != "contributor")
{
XMLreader.Read();
}
XMLreader.Read();
XMLreader.Read();
if (XMLreader.Name == "comment")
{
XMLreader.Read();
comment = XMLreader.Value.Replace('\n', ' ');
comment = comment.Replace(@"\", " ");
comment = comment.Replace('\t', ' ');
}
else
{
comment = "";
}
break;
}
}
if (!editorsStore.ContainsKey(editorID))
{
List <object> list = new List <object>();
list.Add(1);
list.Add(1 * inGroup);
name = name.Replace(@"\", "_slash_");
list.Add(ip + "\t" + name + "\t" + nameID + "\t" + country + "\t" + troll.ToString() + "\t" + typetroll.ToString() + "\t" + distance.ToString());
editorsStore.Add(editorID, list);
}
else
{
editorsStore[editorID][0] = Int32.Parse(editorsStore[editorID][0].ToString()) + 1;
editorsStore[editorID][1] = Int32.Parse(editorsStore[editorID][0].ToString()) * inGroup + 1;
}
if (distance > 0)
{
;
}
editsStore.Add(pageName + "\t" + pageID + "\t" + inGroup.ToString() + "\t" + CleanExactTime(timeString) + "\t" + day.ToString("yyyy-MM-dd") + "\t" + ip + "\t" + name + "\t" + nameID + "\t" + country + "\t" + troll.ToString() + "\t" + typetroll.ToString() + "\t" + distance.ToString() + "\t" + comment);
if (editsStore.Count > 10000)
{
editorsStore = ReleaseFile(Common.editorsFile, editorsStore);
}
if (editsStore.Count > 10000)
{
editsStore = ReleaseFile(Common.editsFile, editsStore);
}
}
}
}
catch (XmlException ex)
{
Console.WriteLine("Ended abnormally..." + ex);
}
eend :;
}