public static async Task <SummonerQueue> ReadFromFile(string filename) { string indexName = filename + ".index"; var queue = new SummonerQueue(); if (File.Exists(indexName)) { queue._index = int.Parse(File.ReadAllText(indexName)); } using (var reader = new StreamReader(File.OpenRead(filename))) { while (!reader.EndOfStream) { queue._list.Add(long.Parse(await reader.ReadLineAsync())); } } return(queue); }
public async Task Crawl(CancellationToken token) { SummonerQueue summonerQueue; if (File.Exists(SUMMONERS_QUEUE_FILE)) { _logger.Info("Read existing queue"); summonerQueue = await SummonerQueue.ReadFromFile(SUMMONERS_QUEUE_FILE); } else { _logger.Info("Create new queue"); summonerQueue = new SummonerQueue(); summonerQueue.Add(SEED_ACCOUNT_ID); } long accountId; while ((accountId = summonerQueue.Get()) != 0) { _logger.InfoFormat("{0} summoners to crawl. Current : {1}", summonerQueue.Count, summonerQueue.Index); _logger.InfoFormat("Query information about summoner {0}", accountId); if (summonerQueue.Index % 20 == 0) { await summonerQueue.SaveToFile(SUMMONERS_QUEUE_FILE); _logger.Info("Summoner queue successfully saved"); } MatchlistDto accountMatches; try { accountMatches = await _api.GetMatchsForAccount(accountId, DATE_RUNE_REFORGED); } catch (Exception ex) { _logger.WarnFormat("Error while querying about summoner {0} : {1}", accountId, ex); continue; } if (accountMatches?.matches == null) { continue; } //We browse all of his matchs foreach (var match in accountMatches.matches) { if (token.IsCancellationRequested) { await summonerQueue.SaveToFile(SUMMONERS_QUEUE_FILE); _logger.Info("Summoner queue successfully saved"); return; } MatchDto dbMatch = (await _coll.FindAsync(m => m.gameId == match.gameId)).FirstOrDefault(); if (dbMatch == null) { //Query match info try { var matchInfo = await _api.GetMatch(match.gameId); foreach (var identity in matchInfo.participantIdentities) { summonerQueue.Add(identity.player.accountId); } await _coll.InsertOneAsync(matchInfo); } catch (Exception ex) { _logger.WarnFormat("Error while querying about match {0} : {1}", match.gameId, ex); } } else { _logger.InfoFormat("Data for game {0} already exists", match.gameId); } } } }