private async void AnalyzeAppStore(object sender, RoutedEventArgs e)
{
AppStoreHTMLCrawler crawler = new AppStoreHTMLCrawler();
AppStoreHTMLAnalyzer analyzer = new AppStoreHTMLAnalyzer();
await InitializeAppStoreFilesReadersAndWriters(crawler, analyzer);
StopwatchAppStore.Start();
for (int i = AppStoreSearchWordsCount; i < SearchWords.Count; i++)
{
AppStoreSearchWordsTextBlock.Text = (++AppStoreSearchWordsCount).ToString();
List <string> appParts = new List <string>();
try
{
appParts = await crawler.FindSearchResultForOneSearchWord(SearchWords[i]);
}
catch (Exception) { }
await AnalyzeHTMLStringsAppStore(appParts, analyzer);
//write counts to passedSearchWords file
AppStorePassedSearchWordsWriter.WriteLine();
AppStorePassedSearchWordsWriter.Write(AppStoreSearchWordsCount + " " +
AppStoreDownloadedCount + " " +
AppStoreAnalyzedCount + " " +
AppStoreErrorsCount
);
}
StopwatchAppStore.Stop();
}
//methods:
private async Task InitializeAppStoreFilesReadersAndWriters(AppStoreHTMLCrawler crawler, AppStoreHTMLAnalyzer analyzer)
{
StorageFile AppStoreLogExcelFile;
StorageFile AppStoreLogHumansFile;
StorageFile AppStoreFoundURLsFile;
StorageFile AppStorePassedSearchWordsFile;
try
{
AppStoreLogExcelFile = await AppStoreWorkingFolder.GetFileAsync("LogExcel.txt");
}
catch (Exception)
{
AppStoreLogExcelFile = await AppStoreWorkingFolder.CreateFileAsync("LogExcel.txt");
}
try
{
AppStoreLogHumansFile = await AppStoreWorkingFolder.GetFileAsync("LogHumans.txt");
}
catch (Exception)
{
AppStoreLogHumansFile = await AppStoreWorkingFolder.CreateFileAsync("LogHumans.txt");
}
bool foundURLs = false;
try
{
//If we found?
AppStoreFoundURLsFile = await AppStoreWorkingFolder.GetFileAsync("AppStoreFoundURLs.txt");
foundURLs = true;
}
catch (Exception)
{
//If we don't
AppStoreFoundURLsFile = await AppStoreWorkingFolder.CreateFileAsync("AppStoreFoundURLs.txt");
}
bool foundPassedSearchWords = false;
try
{
//If we found:
AppStorePassedSearchWordsFile = await AppStoreWorkingFolder.GetFileAsync("AppStorePassedSearchWords.txt");
foundPassedSearchWords = true;
}
catch (Exception)
{
//if we don't
AppStorePassedSearchWordsFile = await AppStoreWorkingFolder.CreateFileAsync("AppStorePassedSearchWords.txt");
}
//basic write streams:
var logExcelStream = await AppStoreLogExcelFile.OpenStreamForWriteAsync();
var logHumansStream = await AppStoreLogHumansFile.OpenStreamForWriteAsync();
var foundURLsStream = await AppStoreFoundURLsFile.OpenStreamForWriteAsync();
var passedSearchWordsStream = await AppStorePassedSearchWordsFile.OpenStreamForWriteAsync();
//basic read streams:
var foundURLsReadStream = await AppStoreFoundURLsFile.OpenStreamForReadAsync();
var passedSearchWordsReadStream = await AppStorePassedSearchWordsFile.OpenStreamForReadAsync();
//better write streams:
var AppStoreLogExcelStream = logExcelStream.AsOutputStream();
var AppStoreLogHumansStream = logHumansStream.AsOutputStream();
var AppStoreFoundURLsStream = foundURLsStream.AsOutputStream();
var AppStorePassedSearchWordsStream = passedSearchWordsStream.AsOutputStream();
//better read streams:
var AppStoreFoundURLsReadStream = foundURLsReadStream.AsInputStream();
var AppStorePassedSearchWordsReadStream = passedSearchWordsReadStream.AsInputStream();
//writers:
AppStoreLogExcelWriter = new StreamWriter(AppStoreLogExcelStream.AsStreamForWrite());
AppStoreLogHumanWriter = new StreamWriter(AppStoreLogHumansStream.AsStreamForWrite());
AppStoreFoundURLsWriter = new StreamWriter(AppStoreFoundURLsStream.AsStreamForWrite());
AppStorePassedSearchWordsWriter = new StreamWriter(AppStorePassedSearchWordsStream.AsStreamForWrite());
//readers:
AppStoreFoundURLsReader = new StreamReader(AppStoreFoundURLsReadStream.AsStreamForRead());
AppStorePassedSearchWordsReader = new StreamReader(AppStorePassedSearchWordsReadStream.AsStreamForRead());
AppStoreLogExcelWriter.AutoFlush = true;
AppStoreLogHumanWriter.AutoFlush = true;
AppStoreFoundURLsWriter.AutoFlush = true;
AppStorePassedSearchWordsWriter.AutoFlush = true;
if (foundURLs)
{
//goes into analyzer
await analyzer.LoadURLmap(AppStoreFoundURLsReader);
}
if (foundPassedSearchWords)
{
Tuple <int, int, int, int> counts = await crawler.SkipPassedSearchWords(AppStorePassedSearchWordsReader); //And also loads counts
AppStoreSearchWordsCount = counts.Item1;
AppStoreDownloadedCount = counts.Item2;
AppStoreAnalyzedCount = counts.Item3;
AppStoreErrorsCount = counts.Item4;
}
}
