// Partial update sync, do the 12 hour get and store until we come up to UTC time now approx.
public static long PerformEDSMUpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
DateTime lastrecordtime = GetLastEDSMRecordTimeUTC(); // this is in UTC, as it comes out of the EDSM records
if (DateTime.UtcNow.Subtract(lastrecordtime).TotalHours <= 1) // If we have partial synced for 1 hour, don't waste our time
{
System.Diagnostics.Debug.WriteLine("EDSM No partial sync, last record less than 1 hour old");
return(updates);
}
// Go For SYNC
LogLine("Checking for updated EDSM systems (may take a few moments).");
EDSMClass edsm = new EDSMClass();
while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition
{ // where we do a set, the time moves to just before now,
// and we then do another set with minimum amount of hours
if (PendingClose())
{
return(updates);
}
int hourstofetch = 6;
if (lastrecordtime < ED21date.AddHours(-48))
{
hourstofetch = 48;
}
else if (lastrecordtime < ED23date.AddHours(-12))
{
hourstofetch = 12;
}
DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
ReportProgress(-1, "Requesting systems from EDSM");
System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
string json = null;
BaseUtils.ResponseData response;
try
{
response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000);
}
catch (WebException ex)
{
ReportProgress(-1, $"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
return(updates);
}
catch (Exception ex)
{
ReportProgress(-1, $"EDSM request failed");
LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
return(updates);
}
if (response.Error)
{
if ((int)response.StatusCode == 429)
{
LogLine($"EDSM rate limit hit - waiting 2 minutes");
for (int sec = 0; sec < 120; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
else
{
LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run");
return(updates);
}
}
json = response.Body;
if (json == null)
{
ReportProgress(-1, "EDSM request failed");
LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
return(updates);
}
// debug File.WriteAllText(@"c:\code\json.txt", json);
DateTime prevrectime = lastrecordtime;
System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length);
long updated = 0;
try
{
updated = ParseEDSMUpdateSystemsString(json, grididallow, ref lastrecordtime, false, PendingClose, ReportProgress, false);
System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime());
// if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely)
if (lastrecordtime <= prevrectime)
{
lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString());
ReportProgress(-1, "EDSM request failed");
LogLine("Processing EDSM systems download failed, will try next time program is run");
return(updates);
}
updates += updated;
SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception
int delay = 10; // Anthor's normal delay
int ratelimitlimit;
int ratelimitremain;
int ratelimitreset;
if (response.Headers != null &&
response.Headers["X-Rate-Limit-Limit"] != null &&
response.Headers["X-Rate-Limit-Remaining"] != null &&
response.Headers["X-Rate-Limit-Reset"] != null &&
Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset))
{
if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later..
{
delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota
}
else
{
delay = 0;
}
System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay);
}
for (int sec = 0; sec < delay; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
return(updates);
}