public async Task runChecks() { if (isFirstRun) { lastCheckTime = DateTime.MinValue; } else { lastCheckTime = actualTime; } tempTime = DateTime.Now; token.ThrowIfCancellationRequested(); List <Task <List <StellarObject> > > tasks = new List <Task <List <StellarObject> > >(); List <StellarObject> newData = new List <StellarObject>(); isRunning = true; if (updateDelegate != null) { this.updateDelegate(); } DateTime oecStart = DateTime.Now; Logger.WriteLine("Begin running plugins."); DateTime start = DateTime.Now; updateCount = 0; total = 0; foreach (IPlugin plugin in plugins) { Console.WriteLine("Running plugin: {0}", plugin.GetName()); Logger.WriteLine("Running plugin: {0}", plugin.GetName()); tasks.Add(Task <List <StellarObject> > .Run(() => plugin.Run(lastCheckTime.ToString("yyyy-MM-dd")), token)); } foreach (Task <List <StellarObject> > task in tasks) { Console.WriteLine("waiting"); token.ThrowIfCancellationRequested(); List <StellarObject> planets = await task; newData.AddRange(planets); } Console.WriteLine(newData.Count); updatesFound = newData.Count; updatesLeft = newData.Count; Console.WriteLine("Finished running plugins in: {0} seconds", (DateTime.Now - start).TotalSeconds); finished = 0; commitQueue = new ConcurrentQueue <StellarObject>(); updateList = newData; Task[] workers = new Task[Workers]; Console.WriteLine("Starting workers"); for (int i = 0; i < workers.Length; i++) { workers[i] = updateWorker(); } int limit = 0; while (finished != Workers) { while (!commitQueue.IsEmpty) { StellarObject update; while (!commitQueue.TryDequeue(out update) && finished != Workers) { token.ThrowIfCancellationRequested(); } try { total++; StringBuilder output = new StringBuilder(); XmlWriterSettings ws = new XmlWriterSettings(); ws.Indent = true; ws.OmitXmlDeclaration = true; using (XmlWriter xw = XmlWriter.Create(output, ws)) { update.Write(xw); xw.Flush(); } String newContent = output.ToString(); output.Clear(); await rm.BeginCommitAndPush("systems/" + update.names[0].MeasurementValue + ".xml", newContent, update.Source, update.isNew); updatesLeft = updateList.Count; updateCount++; limit++; if (updateDelegate != null) { this.updateDelegate(); } Console.WriteLine("Successfully commited data for: {0} ", update.names[0].MeasurementValue); Logger.WriteLine("Successfully commited data for: {0} ", update.names[0].MeasurementValue); token.ThrowIfCancellationRequested(); //60 pushes / min? if (limit >= 35) { Logger.WriteLine("GitHub push limit reached awaiting 60s."); await Task.Delay(60000, token); limit = 0; } } catch (ForbiddenException fex) { Console.WriteLine("Triggered abuse mechanism? Sleeping for 60s. " + fex.Message); Logger.WriteWarning("Triggered abuse mechanism? Sleeping for 60s. " + fex.Message); if (errorCount >= 5) { errorCount = 0; lastRunCondition = "Error: Abuse Mechanism"; return; } await Task.Delay(60000, token); commitQueue.Enqueue(update); limit = 0; errorCount++; } catch (OperationCanceledException ex) { Console.WriteLine("Bot cancelled"); return; } catch (ApiValidationException ex) { Console.WriteLine("Failed to update planet: " + update.names[0].MeasurementValue + ". " + ex.Message + ex.GetType()); Console.WriteLine(ex.HttpResponse.Body); Logger.WriteError("Failed to update planet: " + update.names[0].MeasurementValue + ". " + ex.Message + ex.GetType()); Logger.WriteError(ex.HttpResponse.Body.ToString()); } catch (Exception ex) { Console.WriteLine("Failed to update planet: " + update.names[0].MeasurementValue + ". " + ex.Message + ex.GetType()); Console.WriteLine(ex.StackTrace); Logger.WriteError("Failed to update planet: " + update.names[0].MeasurementValue + ". " + ex.Message + ex.GetType()); Logger.WriteError(ex.StackTrace); } } } Console.WriteLine("Finished running!"); lastRunCondition = "Finished"; Logger.WriteLine("Finished running OECBot in {0} seconds was {1}first run.", (DateTime.Now - oecStart).TotalSeconds, isFirstRun ? "" : "not "); Logger.WriteLine("Successfully updated {0} systems out of {1} system updates found.", updateCount, total); isRunning = false; if (updateDelegate != null) { this.updateDelegate(); } if (isFirstRun) { isFirstRun = false; } lastCheckTime = tempTime; actualTime = lastCheckTime; if (finishDelegate != null) { this.finishDelegate(); } }