private async void StartProcess(WikiDump dump, string path)
{
await Task.Delay(500);
stopWatch.Start();
dispatcherTimer.Start();
// Dump download
ActivateProgressStep(1);
DumpDownload dumpDownload = new DumpDownload(this);
await dumpDownload.Start(dump, path);
// Dump decompression
ActivateProgressStep(2);
DumpDecompress dumpDecompress = new DumpDecompress(this);
await dumpDecompress.Start(dump, path);
// Reading dump
ActivateProgressStep(3);
DumpRead dumpRead = new DumpRead(this);
await dumpRead.Start(dump, path);
// Creating WG files
ActivateProgressStep(4);
FileCreator fileCreator = new FileCreator(this);
await fileCreator.PrepareMaps(dump, path);
// Finishing up (clean up)
ActivateProgressStep(5);
await fileCreator.CreateGraphFiles();
ActivateProgressStep(6);
stopWatch.Stop();
}
private void FetchAllDumpsCompleted(object sender, DownloadStringCompletedEventArgs e)
{
if (e.Error != null)
{
return;
}
int inProgressCount = 0;
JObject json = JObject.Parse(e.Result);
IList <JToken> results = json["wikis"].Children().ToList();
foreach (JToken result in results)
{
WikiDump dump = new WikiDump();
dump.Name = ((JProperty)result).Name;
long fileSize = 0;
foreach (string job in neededJobs)
{
if ((string)result.Children().ToList()[0]["jobs"][job]["status"] == "done")
{
dump.LastUpdated = DateTime.Parse((string)result.Children().ToList()[0]["jobs"][job]["updated"]);
dump.Date = ((string)result.Children().ToList()[0]["jobs"][job]["files"].First.Children().ToList()[0]["url"]).Substring(dump.Name.Length + 2, 8);
dump.Files.Add((string)result.Children().ToList()[0]["jobs"][job]["files"].First.Children().ToList()[0]["url"]);
fileSize += (long)result.Children().ToList()[0]["jobs"][job]["files"].First.Children().ToList()[0]["size"];
}
else
{
dump.Name = ((JProperty)result).Name + " (in progress)";
dump.IsReady = false;
inProgressCount++;
}
}
dump.Size = (double)fileSize / 1000000;
dump_list.Items.Add(dump);
}
fetching_status.Content = "Fetched " + results.Count + " items.";
loading_icon.Source = new BitmapImage(new Uri(@"/Assets/ok.png", UriKind.Relative));
this.fetchAllDumpsLoading = false;
AnimateLoader(loading_icon, this.fetchAllDumpsLoading);
if (inProgressCount > 0)
{
MessageBox.Show("Some dumps are being created right now. Try fetching them again in a few hours or choose created one.", "Dumps status");
}
}
public async Task Start(WikiDump dump, string path)
{
try {
var tasks = new List <Task>();
totalToRecive = dump.Size * 1000000;
foreach (string file in dump.Files)
{
using (WebClient wc = new WebClient()) {
progress.Add(file, 0);
tasks.Add(DownloadFile(wc, file, path));
}
}
await Task.WhenAll(tasks.ToArray());
window.UpdateProgress(1, 1);
} catch (WebException ex) {
window.ErrorProgress(1, "Error occured when trying to download dump from server:\n" + ex.Message, "Dump download error");
}
}
public async Task Start(WikiDump dump, string path)
{
try {
var tasks = new List <Task>();
window.UpdateProgress(2, 0, "Decompressed " + filesDecompressed + " out of " + window.numberOfFiles + " files");
foreach (string file in dump.Files)
{
tasks.Add(DecompressGZip(new FileInfo(path + file.Replace('/', '\\'))));
}
await Task.WhenAll(tasks.ToArray()).ContinueWith((action) => {
// Deleting compressed gz files
foreach (string file in dump.Files)
{
File.Delete(path + file.Replace('/', '\\'));
}
});
} catch (Exception ex) {
window.ErrorProgress(2, "Error occured when trying to decompress downloaded dump:\n" + ex.Message, "Dump decompressing error");
}
}
public async Task Start(WikiDump dump, string path)
{
try {
List <FileInfo> files = new List <FileInfo>();
window.UpdateProgress(3, 1, "Processed " + filesCreated + " out of " + numberOfFiles + " files");
// Page and category titles
files.Add(new FileInfo(path + Array.Find(dump.Files.ToArray(), (el) => el.Contains("page.sql")).Replace('/', '\\').Replace(".gz", "")));
await CreateTitlesMaps(files[0]);
await sortMapnumeric(files[0].FullName.Substring(0, files[0].FullName.Length - 4), files[0].DirectoryName);
await sortMapnumeric(files[0].FullName.Substring(0, files[0].FullName.Length - 8) + "category", files[0].DirectoryName);
// Titles sorting (for search use)
//await sortTitles(files[0].FullName.Substring(0, files[0].FullName.Length - 4), files[0].DirectoryName);
//await sortTitles(files[0].FullName.Substring(0, files[0].FullName.Length - 8) + "category", files[0].DirectoryName);
// Page to page links
files.Add(new FileInfo(path + Array.Find(dump.Files.ToArray(), (el) => el.Contains("pagelinks.sql")).Replace('/', '\\').Replace(".gz", "")));
await CreatePageLinksMap(files[1], DUMP_TYPE.PAGELINKS);
await sortMapnumeric(files[1].FullName.Substring(0, files[1].FullName.Length - 4), files[1].DirectoryName);
// Category links
files.Add(new FileInfo(path + Array.Find(dump.Files.ToArray(), (el) => el.Contains("categorylinks.sql")).Replace('/', '\\').Replace(".gz", "")));
await CreateCategoryLinksMap(files[2], DUMP_TYPE.CATEGORYLINKS);
await sortMapnumeric(files[2].FullName.Substring(0, files[2].FullName.Length - 4) + "frompage", files[2].DirectoryName);
await sortMapnumeric(files[2].FullName.Substring(0, files[2].FullName.Length - 4) + "fromcategory", files[2].DirectoryName);
// Deleting sql files
foreach (FileInfo file in files)
{
File.Delete(file.FullName);
}
} catch (Exception ex) {
window.ErrorProgress(3, "Error occured when trying to read dump:\n" + ex.Message, "Dump reading error");
}
}
public async Task PrepareMaps(WikiDump dump, string path)
{
// Dump name variables:
// dump.Name => "simplewiki"
// dump.Date => "20191101"
this.pathToMaps = path + "\\" + dump.Name + "\\" + dump.Date + "\\";
this.pageTitlesMap = dump.Name + "-" + dump.Date + "-" + "page.map";
this.categoryTitlesMap = dump.Name + "-" + dump.Date + "-" + "category.map";
this.pageLinksMap = dump.Name + "-" + dump.Date + "-" + "pagelinks.map";
this.catFromCatMap = dump.Name + "-" + dump.Date + "-" + "categorylinksfromcategory.map";
this.catFromPageMap = dump.Name + "-" + dump.Date + "-" + "categorylinksfrompage.map";
string extension = ".wg";
this.outOffsetFileName = pathToMaps + dump.Name + extension + "m";
this.outTitleFileName = pathToMaps + dump.Name + extension + "t";
this.outGraphFileName = pathToMaps + dump.Name + extension + "g";
this.outInfoFileName = pathToMaps + dump.Name + extension + "i";
this.outSortedFileName = pathToMaps + dump.Name + extension + "s";
// Progress update:
window.UpdateProgress(4, 1, "Generating reverse maps: Page Links");
// ZLICZANIE WSZYSTKICH STRON
using (FileStream fs = File.Open(pathToMaps + pageTitlesMap, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs)) {
string line;
while ((line = sr.ReadLine()) != null)
{
this.amountOfPages += 1;
}
}
// TWORZENIE ODWROTNYCH ODWZOROWAŃ
await Task.Run(() => createReverseMap(pageLinksMap));
window.UpdateProgress(4, 1, "Generating reverse maps: Category From Page");
await Task.Run(() => createReverseMap(catFromPageMap));
//await createReverseMap(catFromPageMap);
window.UpdateProgress(4, 1, "Generating reverse maps: Category From Category");
await Task.Run(() => createReverseMap(catFromCatMap));
//await createReverseMap(catFromCatMap);
window.UpdateProgress(4, 1, "Sorting Titles and mapping Wiki IDs");
await Task.Delay(5000);
Dictionary <int, string> sortedTitles = new Dictionary <int, string>();
// Przechodzi po tytułach wszytstkich artykułów, tworzy odwzorowanie ID artykułu na jego miejsce w kolejności
using (FileStream fs = File.Open(pathToMaps + this.pageTitlesMap, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs)) {
string line;
while ((line = sr.ReadLine()) != null)
{
GraphObject g = new GraphObject();
g.id = System.Convert.ToInt32(line.Split('\t')[0]);
g.title = line.Split('\t')[1];
g.order = this.currentAmount;
pageDict[g.id] = g.order;
sortedTitles[g.order] = g.title;
this.currentAmount += 1;
}
}
// odwzorowanie kategorii
using (FileStream fs = File.Open(pathToMaps + this.categoryTitlesMap, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs)) {
string line;
while ((line = sr.ReadLine()) != null)
{
GraphObject g = new GraphObject();
g.id = System.Convert.ToInt32(line.Split('\t')[0]);
g.title = line.Split('\t')[1];
g.order = this.currentAmount;
sortedTitles[g.order] = g.title;
catDict[g.id] = g.order;
this.currentAmount += 1;
}
}
// Sortowanie i zapis tytułów
// PLIK .wgs
BinaryWriter bwSortedTitles = createNewBinaryFile(this.outSortedFileName);
List <KeyValuePair <int, string> > sortedTitlesList = sortedTitles.ToList();
// Zwalnianie pamieci
sortedTitles.Clear();
sortedTitlesList.Sort(
delegate(KeyValuePair <int, string> pair1,
KeyValuePair <int, string> pair2) {
return(pair1.Value.CompareTo(pair2.Value));
}
);
foreach (var v in sortedTitlesList)
{
bwSortedTitles.Write(Encoding.UTF8.GetBytes(v.Value));
bwSortedTitles.Write(Encoding.UTF8.GetBytes(";"));
bwSortedTitles.Write(Encoding.UTF8.GetBytes(v.Key.ToString()));
bwSortedTitles.Write(Encoding.UTF8.GetBytes("\n"));
}
bwSortedTitles.Close();
}
