public static async Task <SummonerQueue> ReadFromFile(string filename)
{
string indexName = filename + ".index";
var queue = new SummonerQueue();
if (File.Exists(indexName))
{
queue._index = int.Parse(File.ReadAllText(indexName));
}
using (var reader = new StreamReader(File.OpenRead(filename)))
{
while (!reader.EndOfStream)
{
queue._list.Add(long.Parse(await reader.ReadLineAsync()));
}
}
return(queue);
}
public async Task Crawl(CancellationToken token)
{
SummonerQueue summonerQueue;
if (File.Exists(SUMMONERS_QUEUE_FILE))
{
_logger.Info("Read existing queue");
summonerQueue = await SummonerQueue.ReadFromFile(SUMMONERS_QUEUE_FILE);
}
else
{
_logger.Info("Create new queue");
summonerQueue = new SummonerQueue();
summonerQueue.Add(SEED_ACCOUNT_ID);
}
long accountId;
while ((accountId = summonerQueue.Get()) != 0)
{
_logger.InfoFormat("{0} summoners to crawl. Current : {1}", summonerQueue.Count, summonerQueue.Index);
_logger.InfoFormat("Query information about summoner {0}", accountId);
if (summonerQueue.Index % 20 == 0)
{
await summonerQueue.SaveToFile(SUMMONERS_QUEUE_FILE);
_logger.Info("Summoner queue successfully saved");
}
MatchlistDto accountMatches;
try
{
accountMatches = await _api.GetMatchsForAccount(accountId, DATE_RUNE_REFORGED);
}
catch (Exception ex)
{
_logger.WarnFormat("Error while querying about summoner {0} : {1}", accountId, ex);
continue;
}
if (accountMatches?.matches == null)
{
continue;
}
//We browse all of his matchs
foreach (var match in accountMatches.matches)
{
if (token.IsCancellationRequested)
{
await summonerQueue.SaveToFile(SUMMONERS_QUEUE_FILE);
_logger.Info("Summoner queue successfully saved");
return;
}
MatchDto dbMatch = (await _coll.FindAsync(m => m.gameId == match.gameId)).FirstOrDefault();
if (dbMatch == null)
{
//Query match info
try
{
var matchInfo = await _api.GetMatch(match.gameId);
foreach (var identity in matchInfo.participantIdentities)
{
summonerQueue.Add(identity.player.accountId);
}
await _coll.InsertOneAsync(matchInfo);
}
catch (Exception ex)
{
_logger.WarnFormat("Error while querying about match {0} : {1}", match.gameId, ex);
}
}
else
{
_logger.InfoFormat("Data for game {0} already exists", match.gameId);
}
}
}
}
