public long UpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <string> ReportProgress)
{
DateTime lastrecordtime = SystemsDatabase.Instance.GetLastEDSMRecordTimeUTC();
DateTime maximumupdatetimewindow = DateTime.UtcNow.AddDays(-ForcedFullDownloadDays); // limit download to this amount of days
if (lastrecordtime < maximumupdatetimewindow)
{
lastrecordtime = maximumupdatetimewindow; // this stops crazy situations where somehow we have a very old date but the full sync did not take care of it
}
long updates = 0;
double fetchmult = 1;
DateTime minimumfetchspan = DateTime.UtcNow.AddHours(-UpdateFetchHours / 2); // we don't bother fetching if last record time is beyond this point
while (lastrecordtime < minimumfetchspan) // stop at X mins before now, so we don't get in a condition
{ // where we do a set, the time moves to just before now,
// and we then do another set with minimum amount of hours
if (PendingClose())
{
return(updates);
}
if (updates == 0)
{
LogLine("Checking for updated EDSM systems (may take a few moments).");
}
EDSMClass edsm = new EDSMClass();
double hourstofetch = UpdateFetchHours; //EDSM new server feb 2021, more capable,
DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch * fetchmult);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()} {hourstofetch}");
string json = null;
BaseUtils.ResponseData response;
try
{
Stopwatch sw = new Stopwatch();
response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000);
fetchmult = Math.Max(0.1, Math.Min(Math.Min(fetchmult * 1.1, 1.0), 5000.0 / sw.ElapsedMilliseconds));
}
catch (WebException ex)
{
ReportProgress($"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
return(updates);
}
catch (Exception ex)
{
ReportProgress($"EDSM request failed");
LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
return(updates);
}
if (response.Error)
{
if ((int)response.StatusCode == 429)
{
LogLine($"EDSM rate limit hit - waiting 2 minutes");
for (int sec = 0; sec < 120; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
else
{
LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run");
return(updates);
}
}
json = response.Body;
if (json == null)
{
ReportProgress("EDSM request failed");
LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
return(updates);
}
// debug File.WriteAllText(@"c:\code\json.txt", json);
DateTime prevrectime = lastrecordtime;
System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length);
long updated = 0;
try
{
ReportProgress($"EDSM star database update from UTC " + lastrecordtime.ToUniversalTime().ToString());
updated = SystemsDB.ParseEDSMJSONString(json, grididallow, ref lastrecordtime, PendingClose, ReportProgress, "");
System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime());
// if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely)
if (lastrecordtime <= prevrectime)
{
lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString());
ReportProgress("EDSM request failed");
LogLine("Processing EDSM systems download failed, will try next time program is run");
return(updates);
}
updates += updated;
SystemsDatabase.Instance.SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception
int delay = 10; // Anthor's normal delay
int ratelimitlimit;
int ratelimitremain;
int ratelimitreset;
if (response.Headers != null &&
response.Headers["X-Rate-Limit-Limit"] != null &&
response.Headers["X-Rate-Limit-Remaining"] != null &&
response.Headers["X-Rate-Limit-Reset"] != null &&
Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset))
{
if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later..
{
delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota
}
else
{
delay = 0;
}
System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay);
}
for (int sec = 0; sec < delay; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
return(updates);
}
