public long UpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <string> ReportProgress) { DateTime lastrecordtime = SQLiteConnectionSystem.GetLastEDSMRecordTimeUTC(); long updates = 0; double fetchmult = 1; while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition { // where we do a set, the time moves to just before now, // and we then do another set with minimum amount of hours if (PendingClose()) { return(updates); } if (updates == 0) { LogLine("Checking for updated EDSM systems (may take a few moments)."); } EDSMClass edsm = new EDSMClass(); double hourstofetch = 3; if (lastrecordtime < ED21date.AddHours(-48)) { hourstofetch = 48; } else if (lastrecordtime < ED23date.AddHours(-12)) { hourstofetch = 12; } else if (lastrecordtime < ED30date.AddHours(-6)) { hourstofetch = 6; } DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch * fetchmult); if (enddate > DateTime.UtcNow) { enddate = DateTime.UtcNow; } LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}"); System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}"); string json = null; BaseUtils.ResponseData response; try { Stopwatch sw = new Stopwatch(); response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000); fetchmult = Math.Max(0.1, Math.Min(Math.Min(fetchmult * 1.1, 1.0), 5000.0 / sw.ElapsedMilliseconds)); } catch (WebException ex) { ReportProgress($"EDSM request failed"); if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse) { string status = ((HttpWebResponse)ex.Response).StatusDescription; LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run"); } else { LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run"); } return(updates); } catch (Exception ex) { ReportProgress($"EDSM request failed"); LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run"); return(updates); } if (response.Error) { if ((int)response.StatusCode == 429) { LogLine($"EDSM rate limit hit - waiting 2 minutes"); for (int sec = 0; sec < 120; sec++) { if (!PendingClose()) { System.Threading.Thread.Sleep(1000); } } } else { LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run"); return(updates); } } json = response.Body; if (json == null) { ReportProgress("EDSM request failed"); LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run"); return(updates); } // debug File.WriteAllText(@"c:\code\json.txt", json); DateTime prevrectime = lastrecordtime; System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length); long updated = 0; try { ReportProgress($"EDSM star database update from UTC " + lastrecordtime.ToUniversalTime().ToString()); updated = SystemsDB.ParseEDSMJSONString(json, grididallow, ref lastrecordtime, PendingClose, ReportProgress, ""); System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}"); System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime()); // if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely) if (lastrecordtime <= prevrectime) { lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck } } catch (Exception e) { System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString()); ReportProgress("EDSM request failed"); LogLine("Processing EDSM systems download failed, will try next time program is run"); return(updates); } updates += updated; SQLiteConnectionSystem.SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception int delay = 10; // Anthor's normal delay int ratelimitlimit; int ratelimitremain; int ratelimitreset; if (response.Headers != null && response.Headers["X-Rate-Limit-Limit"] != null && response.Headers["X-Rate-Limit-Remaining"] != null && response.Headers["X-Rate-Limit-Reset"] != null && Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) && Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) && Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset)) { if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later.. { delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota } else { delay = 0; } System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay); } for (int sec = 0; sec < delay; sec++) { if (!PendingClose()) { System.Threading.Thread.Sleep(1000); } } } return(updates); }