private async void AnalyzeAppStore(object sender, RoutedEventArgs e) { AppStoreHTMLCrawler crawler = new AppStoreHTMLCrawler(); AppStoreHTMLAnalyzer analyzer = new AppStoreHTMLAnalyzer(); await InitializeAppStoreFilesReadersAndWriters(crawler, analyzer); StopwatchAppStore.Start(); for (int i = AppStoreSearchWordsCount; i < SearchWords.Count; i++) { AppStoreSearchWordsTextBlock.Text = (++AppStoreSearchWordsCount).ToString(); List <string> appParts = new List <string>(); try { appParts = await crawler.FindSearchResultForOneSearchWord(SearchWords[i]); } catch (Exception) { } await AnalyzeHTMLStringsAppStore(appParts, analyzer); //write counts to passedSearchWords file AppStorePassedSearchWordsWriter.WriteLine(); AppStorePassedSearchWordsWriter.Write(AppStoreSearchWordsCount + " " + AppStoreDownloadedCount + " " + AppStoreAnalyzedCount + " " + AppStoreErrorsCount ); } StopwatchAppStore.Stop(); }
private async Task AnalyzeHTMLStringsAppStore(List <string> appParts, AppStoreHTMLAnalyzer analyzer) { foreach (string app in appParts) { try { var info = analyzer.AnalyzeHTMLCodeFromString(app); if (info == null) { AppStoreErrorsTextBlock.Text = (++AppStoreErrorsCount).ToString(); } else { await AppStoreLogHumanWriter.WriteLineAsync(info.Log()); await AppStoreLogExcelWriter.WriteLineAsync(info.Log2()); AppStoreAnalyzedTextBlock.Text = (++AppStoreAnalyzedCount).ToString(); } } catch (IOException) { AppStoreCopyCountTextBlock.Text = (++AppStoreCopyCount).ToString(); } catch (Exception) { AppStoreErrorsTextBlock.Text = (++AppStoreErrorsCount).ToString(); } AppStoreUpdateTime(); } AppStoreUpdateTime(); }
//methods: private async Task InitializeAppStoreFilesReadersAndWriters(AppStoreHTMLCrawler crawler, AppStoreHTMLAnalyzer analyzer) { StorageFile AppStoreLogExcelFile; StorageFile AppStoreLogHumansFile; StorageFile AppStoreFoundURLsFile; StorageFile AppStorePassedSearchWordsFile; try { AppStoreLogExcelFile = await AppStoreWorkingFolder.GetFileAsync("LogExcel.txt"); } catch (Exception) { AppStoreLogExcelFile = await AppStoreWorkingFolder.CreateFileAsync("LogExcel.txt"); } try { AppStoreLogHumansFile = await AppStoreWorkingFolder.GetFileAsync("LogHumans.txt"); } catch (Exception) { AppStoreLogHumansFile = await AppStoreWorkingFolder.CreateFileAsync("LogHumans.txt"); } bool foundURLs = false; try { //If we found? AppStoreFoundURLsFile = await AppStoreWorkingFolder.GetFileAsync("AppStoreFoundURLs.txt"); foundURLs = true; } catch (Exception) { //If we don't AppStoreFoundURLsFile = await AppStoreWorkingFolder.CreateFileAsync("AppStoreFoundURLs.txt"); } bool foundPassedSearchWords = false; try { //If we found: AppStorePassedSearchWordsFile = await AppStoreWorkingFolder.GetFileAsync("AppStorePassedSearchWords.txt"); foundPassedSearchWords = true; } catch (Exception) { //if we don't AppStorePassedSearchWordsFile = await AppStoreWorkingFolder.CreateFileAsync("AppStorePassedSearchWords.txt"); } //basic write streams: var logExcelStream = await AppStoreLogExcelFile.OpenStreamForWriteAsync(); var logHumansStream = await AppStoreLogHumansFile.OpenStreamForWriteAsync(); var foundURLsStream = await AppStoreFoundURLsFile.OpenStreamForWriteAsync(); var passedSearchWordsStream = await AppStorePassedSearchWordsFile.OpenStreamForWriteAsync(); //basic read streams: var foundURLsReadStream = await AppStoreFoundURLsFile.OpenStreamForReadAsync(); var passedSearchWordsReadStream = await AppStorePassedSearchWordsFile.OpenStreamForReadAsync(); //better write streams: var AppStoreLogExcelStream = logExcelStream.AsOutputStream(); var AppStoreLogHumansStream = logHumansStream.AsOutputStream(); var AppStoreFoundURLsStream = foundURLsStream.AsOutputStream(); var AppStorePassedSearchWordsStream = passedSearchWordsStream.AsOutputStream(); //better read streams: var AppStoreFoundURLsReadStream = foundURLsReadStream.AsInputStream(); var AppStorePassedSearchWordsReadStream = passedSearchWordsReadStream.AsInputStream(); //writers: AppStoreLogExcelWriter = new StreamWriter(AppStoreLogExcelStream.AsStreamForWrite()); AppStoreLogHumanWriter = new StreamWriter(AppStoreLogHumansStream.AsStreamForWrite()); AppStoreFoundURLsWriter = new StreamWriter(AppStoreFoundURLsStream.AsStreamForWrite()); AppStorePassedSearchWordsWriter = new StreamWriter(AppStorePassedSearchWordsStream.AsStreamForWrite()); //readers: AppStoreFoundURLsReader = new StreamReader(AppStoreFoundURLsReadStream.AsStreamForRead()); AppStorePassedSearchWordsReader = new StreamReader(AppStorePassedSearchWordsReadStream.AsStreamForRead()); AppStoreLogExcelWriter.AutoFlush = true; AppStoreLogHumanWriter.AutoFlush = true; AppStoreFoundURLsWriter.AutoFlush = true; AppStorePassedSearchWordsWriter.AutoFlush = true; if (foundURLs) { //goes into analyzer await analyzer.LoadURLmap(AppStoreFoundURLsReader); } if (foundPassedSearchWords) { Tuple <int, int, int, int> counts = await crawler.SkipPassedSearchWords(AppStorePassedSearchWordsReader); //And also loads counts AppStoreSearchWordsCount = counts.Item1; AppStoreDownloadedCount = counts.Item2; AppStoreAnalyzedCount = counts.Item3; AppStoreErrorsCount = counts.Item4; } }