private static bool ShouldSendEvent(HistoryEntry he)
{
if (lastDiscardFetch < DateTime.UtcNow.AddMinutes(-30))
{
EDSMClass edsm = new EDSMClass();
discardEvents = new HashSet <string>(edsm.GetJournalEventsToDiscard());
lastDiscardFetch = DateTime.UtcNow;
}
string eventtype = he.EntryType.ToString();
if (he.EdsmSync ||
he.MultiPlayer ||
discardEvents.Contains(eventtype) ||
alwaysDiscard.Contains(eventtype))
{
return(false);
}
return(true);
}
public static void UpdateDiscardList()
{
if (lastDiscardFetch < DateTime.UtcNow.AddMinutes(-120)) // check if we need a new discard list
{
try
{
EDSMClass edsm = new EDSMClass();
var newdiscardEvents = new HashSet <string>(edsm.GetJournalEventsToDiscard());
lock (alwaysDiscard) // use this as a perm proxy to lock discardEvents
{
discardEvents = newdiscardEvents;
}
lastDiscardFetch = DateTime.UtcNow;
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine($"Unable to retrieve events to be discarded: {ex.ToString()}");
}
}
}
public static void SendTravelLog(HistoryEntry he) // (verified with EDSM 29/9/2016, seen UTC time being sent, and same UTC time on ESDM).
{
EDSMClass edsm = new EDSMClass();
var cmdr = he.Commander;
edsm.commanderName = cmdr.EdsmName ?? cmdr.Name;
edsm.apiKey = cmdr.APIKey;
if (!edsm.IsApiKeySet)
{
return;
}
string errmsg;
int errno;
bool firstdiscover;
int edsmid;
Task taskEDSM = Task.Factory.StartNew(() =>
{ // LOCAL time, there is a UTC converter inside this call
if (edsm.SendTravelLog(he.System.name, he.EventTimeUTC, he.System.HasCoordinate, he.System.x, he.System.y, he.System.z, out errmsg, out errno, out firstdiscover, out edsmid))
{
if (edsmid != 0 && he.System.id_edsm <= 0)
{
he.System.id_edsm = edsmid;
JournalEntry.UpdateEDSMIDPosJump(he.Journalid, he.System, false, -1);
}
if (firstdiscover)
{
he.SetFirstDiscover();
}
he.SetEdsmSync();
}
});
}
// returns null if EDSM says not there, else if returns list of bodies and a flag indicating if from cache.
// all this is done in a lock inside a task - the only way to sequence the code and prevent multiple lookups in an await structure
// so we must pass back all the info we can to tell the caller what happened.
// optional call back is allowed in lock - may not be used.
// Verified Nov 20
public static Tuple <List <JournalScan>, bool> GetBodiesList(ISystem sys, bool edsmweblookup = true, Action <List <JournalScan>, ISystem> onnewbodies = null)
{
try
{
lock (bodylock) // only one request at a time going, this is to prevent multiple requests for the same body
{
// System.Threading.Thread.Sleep(2000); //debug - delay to show its happening
// System.Diagnostics.Debug.WriteLine("EDSM Cache check " + sys.EDSMID + " " + sys.SystemAddress + " " + sys.Name);
if (DictEDSMBodies != null && sys.EDSMID > 0 && DictEDSMBodies.ContainsKey(sys.EDSMID)) // Cache EDSM bidies during run of EDD.
{
System.Diagnostics.Debug.WriteLine("Found sys.EDSMID " + sys.EDSMID);
return(new Tuple <List <JournalScan>, bool>(DictEDSMBodies[sys.EDSMID], true));
}
else if (DictEDSMBodiesByID64 != null && sys.SystemAddress != null && sys.SystemAddress > 0 && DictEDSMBodiesByID64.ContainsKey(sys.SystemAddress.Value))
{
System.Diagnostics.Debug.WriteLine("Found sys.EDSMID64 " + sys.SystemAddress.Value);
return(new Tuple <List <JournalScan>, bool>(DictEDSMBodiesByID64[sys.SystemAddress.Value], true));
}
if (!edsmweblookup) // must be set for a web lookup
{
return(null);
}
System.Diagnostics.Debug.WriteLine("EDSM Web lookup");
List <JournalScan> bodies = new List <JournalScan>();
EDSMClass edsm = new EDSMClass();
JObject jo = null;
if (sys.EDSMID > 0)
{
jo = edsm.GetBodies(sys.EDSMID); // Colonia
}
else if (sys.SystemAddress != null && sys.SystemAddress > 0)
{
jo = edsm.GetBodiesByID64(sys.SystemAddress.Value);
}
else if (sys.Name != null)
{
jo = edsm.GetBodies(sys.Name);
}
if (jo != null && jo["bodies"] != null)
{
foreach (JObject edsmbody in jo["bodies"])
{
try
{
JObject jbody = EDSMClass.ConvertFromEDSMBodies(edsmbody);
JournalScan js = new JournalScan(jbody);//TBD, sys.EDSMID);
bodies.Add(js);
}
catch (Exception ex)
{
BaseUtils.HttpCom.WriteLog($"Exception Loop: {ex.Message}", "");
BaseUtils.HttpCom.WriteLog($"ETrace: {ex.StackTrace}", "");
Trace.WriteLine($"Exception Loop: {ex.Message}");
Trace.WriteLine($"ETrace: {ex.StackTrace}");
}
}
if (sys.EDSMID > 0)
{
DictEDSMBodies[sys.EDSMID] = bodies;
}
if (sys.SystemAddress != null && sys.SystemAddress > 0)
{
DictEDSMBodiesByID64[sys.SystemAddress.Value] = bodies;
}
System.Diagnostics.Debug.WriteLine("EDSM Web Lookup complete " + sys.Name + " " + bodies.Count);
onnewbodies?.Invoke(bodies, sys); // inside the lock, with new bodies, allow it to do processsing.
return(new Tuple <List <JournalScan>, bool>(bodies, false));
}
if (sys.EDSMID > 0)
{
DictEDSMBodies[sys.EDSMID] = null;
}
if (sys.SystemAddress != null && sys.SystemAddress > 0)
{
DictEDSMBodiesByID64[sys.SystemAddress.Value] = null;
}
}
}
catch (Exception ex)
{
Trace.WriteLine($"Exception: {ex.Message}");
Trace.WriteLine($"ETrace: {ex.StackTrace}");
return(null);
}
return(null);
}
static private bool SendToEDSM(List <HistoryEntry> hl, EDCommander cmdr, out string errmsg)
{
EDSMClass edsm = new EDSMClass(cmdr); // Ensure we use the commanders EDSM credentials.
errmsg = null;
List <JObject> entries = new List <JObject>();
foreach (HistoryEntry he in hl)
{
JournalEntry je = he.journalEntry;
if (je == null)
{
je = JournalEntry.Get(he.Journalid);
}
JObject json = je.GetJson();
RemoveCommonKeys(json);
if (je.EventTypeID == JournalTypeEnum.FSDJump && json["FuelUsed"].Empty())
{
json["_convertedNetlog"] = true;
}
if (json["StarPosFromEDSM"].Bool(false)) // Remove star pos from EDSM
{
json.Remove("StarPos");
}
json.Remove("StarPosFromEDSM");
json["_systemName"] = he.System.Name;
json["_systemCoordinates"] = new JArray(he.System.X, he.System.Y, he.System.Z);
if (he.System.SystemAddress != null)
{
json["_systemAddress"] = he.System.SystemAddress;
}
if (he.IsDocked)
{
json["_stationName"] = he.StationName;
if (he.MarketID != null)
{
json["_stationMarketId"] = he.MarketID;
}
}
json["_shipId"] = he.ShipId;
entries.Add(json);
}
List <JObject> results = edsm.SendJournalEvents(entries, out errmsg);
if (results == null)
{
return(false);
}
else
{
using (var cn = new SQLiteConnectionUser(utc: true))
{
using (var txn = cn.BeginTransaction())
{
for (int i = 0; i < hl.Count && i < results.Count; i++)
{
HistoryEntry he = hl[i];
JObject result = results[i];
int msgnr = result["msgnum"].Int();
int systemId = result["systemId"].Int();
bool systemCreated = result["systemCreated"].Bool();
if ((msgnr >= 100 && msgnr < 200) || msgnr == 500)
{
if (he.EntryType == JournalTypeEnum.FSDJump || he.EntryType == JournalTypeEnum.Location)
{
if (systemId != 0)
{
he.System.EDSMID = systemId;
JournalEntry.UpdateEDSMIDPosJump(he.Journalid, he.System, false, 0, cn, txn);
}
if (systemCreated)
{
he.SetFirstDiscover(true);
}
}
he.SetEdsmSync(cn, txn);
if (msgnr == 500)
{
System.Diagnostics.Trace.WriteLine($"Warning submitting event {he.Journalid} \"{he.EventSummary}\": {msgnr} {result["msg"].Str()}");
}
}
else
{
System.Diagnostics.Trace.WriteLine($"Error submitting event {he.Journalid} \"{he.EventSummary}\": {msgnr} {result["msg"].Str()}");
}
}
txn.Commit();
}
}
return(true);
}
}
public static List <JournalScan> GetBodiesList(long edsmid, bool edsmweblookup = true, long?id64 = null, string sysname = null) // get this edsmid, optionally lookup web protected against bad json
{
try
{
if (DictEDSMBodies != null && edsmid > 0 && DictEDSMBodies.ContainsKey(edsmid)) // Cache EDSM bidies during run of EDD.
{
// System.Diagnostics.Debug.WriteLine(".. found EDSM Lookup bodies from cache " + edsmid);
return(DictEDSMBodies[edsmid]);
}
else if (DictEDSMBodiesByID64 != null && id64 != null && id64 > 0 && DictEDSMBodiesByID64.ContainsKey(id64.Value))
{
return(DictEDSMBodiesByID64[id64.Value]);
}
if (!edsmweblookup) // must be set for a web lookup
{
return(null);
}
List <JournalScan> bodies = new List <JournalScan>();
EDSMClass edsm = new EDSMClass();
//System.Diagnostics.Debug.WriteLine("EDSM Web Lookup bodies " + edsmid);
JObject jo = null;
if (edsmid > 0)
{
jo = edsm.GetBodies(edsmid); // Colonia
}
else if (id64 != null && id64 > 0)
{
jo = edsm.GetBodiesByID64(id64.Value);
}
else if (sysname != null)
{
jo = edsm.GetBodies(sysname);
}
if (jo != null && jo["bodies"] != null)
{
foreach (JObject edsmbody in jo["bodies"])
{
try
{
JObject jbody = EDSMClass.ConvertFromEDSMBodies(edsmbody);
JournalScan js = new JournalScan(jbody, edsmid);
bodies.Add(js);
}
catch (Exception ex)
{
BaseUtils.HttpCom.WriteLog($"Exception Loop: {ex.Message}", "");
BaseUtils.HttpCom.WriteLog($"ETrace: {ex.StackTrace}", "");
Trace.WriteLine($"Exception Loop: {ex.Message}");
Trace.WriteLine($"ETrace: {ex.StackTrace}");
}
}
if (edsmid > 0)
{
DictEDSMBodies[edsmid] = bodies;
}
if (id64 != null && id64 > 0)
{
DictEDSMBodiesByID64[id64.Value] = bodies;
}
return(bodies);
}
if (edsmid > 0)
{
DictEDSMBodies[edsmid] = null;
}
if (id64 != null && id64 > 0)
{
DictEDSMBodiesByID64[id64.Value] = null;
}
}
catch (Exception ex)
{
Trace.WriteLine($"Exception: {ex.Message}");
Trace.WriteLine($"ETrace: {ex.StackTrace}");
return(null);
}
return(null);
}
static private bool SendToEDSM(List <HistoryEntry> hl, EDCommander cmdr, out string errmsg, out string firstdiscovers)
{
EDSMClass edsm = new EDSMClass(cmdr); // Ensure we use the commanders EDSM credentials.
errmsg = null;
firstdiscovers = "";
List <JObject> entries = new List <JObject>();
foreach (HistoryEntry he in hl)
{
JournalEntry je = he.journalEntry;
if (je == null)
{
je = JournalEntry.Get(he.Journalid);
}
JObject json = je.GetJson();
if (json == null)
{
continue;
}
RemoveCommonKeys(json);
if (je.EventTypeID == JournalTypeEnum.FSDJump && json["FuelUsed"].Empty())
{
json["_convertedNetlog"] = true;
}
if (json["StarPosFromEDSM"].Bool(false)) // Remove star pos from EDSM
{
json.Remove("StarPos");
}
json.Remove("StarPosFromEDSM");
json["_systemName"] = he.System.Name;
json["_systemCoordinates"] = new JArray(he.System.X, he.System.Y, he.System.Z);
if (he.System.SystemAddress != null)
{
json["_systemAddress"] = he.System.SystemAddress;
}
if (he.IsDocked)
{
json["_stationName"] = he.WhereAmI;
if (he.MarketID != null)
{
json["_stationMarketId"] = he.MarketID;
}
}
json["_shipId"] = he.ShipId;
entries.Add(json);
}
List <JObject> results = edsm.SendJournalEvents(entries, out errmsg);
if (results == null)
{
return(false);
}
else
{
firstdiscovers = UserDatabase.Instance.ExecuteWithDatabase <string>(cn =>
{
string firsts = "";
using (var txn = cn.Connection.BeginTransaction())
{
for (int i = 0; i < hl.Count && i < results.Count; i++)
{
HistoryEntry he = hl[i];
JObject result = results[i];
int msgnr = result["msgnum"].Int();
int systemId = result["systemId"].Int();
if ((msgnr >= 100 && msgnr < 200) || msgnr == 500)
{
if (he.EntryType == JournalTypeEnum.FSDJump || he.EntryType == JournalTypeEnum.Location)
{
if (systemId != 0)
{
he.System.EDSMID = systemId;
JournalEntry.UpdateEDSMIDPosJump(he.Journalid, he.System, false, 0, cn.Connection, txn);
}
}
if (he.EntryType == JournalTypeEnum.FSDJump) // only on FSD, confirmed with Anthor. 25/4/2018
{
bool systemCreated = result["systemCreated"].Bool();
if (systemCreated)
{
System.Diagnostics.Debug.WriteLine("** EDSM indicates first entry for " + he.System.Name);
(he.journalEntry as JournalFSDJump).UpdateFirstDiscover(true, cn.Connection, txn);
firsts = firsts.AppendPrePad(he.System.Name, ";");
}
}
he.journalEntry.SetEdsmSync(cn.Connection, txn);
if (msgnr == 500)
{
System.Diagnostics.Trace.WriteLine($"Warning submitting event {he.Journalid} \"{he.EventSummary}\": {msgnr} {result["msg"].Str()}");
}
}
else
{
System.Diagnostics.Trace.WriteLine($"Error submitting event {he.Journalid} \"{he.EventSummary}\": {msgnr} {result["msg"].Str()}");
}
}
txn.Commit();
return(firsts);
}
});
return(true);
}
}
// Called by Perform, by Sync, by above.
public static bool SendEDSMEvents(Action <string> log, IEnumerable <HistoryEntry> helist, bool manual = false)
{
if (lastDiscardFetch < DateTime.UtcNow.AddMinutes(-120)) // check if we need a new discard list
{
try
{
EDSMClass edsm = new EDSMClass();
discardEvents = new HashSet <string>(edsm.GetJournalEventsToDiscard());
lastDiscardFetch = DateTime.UtcNow;
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine($"Unable to retrieve events to be discarded: {ex.ToString()}");
}
}
System.Diagnostics.Debug.WriteLine("Send " + helist.Count());
int eventCount = 0;
bool hasbeta = false;
DateTime betatime = DateTime.MinValue;
foreach (HistoryEntry he in helist) // push list of events to historylist queue..
{
if (!he.EdsmSync) // if we have not sent it..
{
string eventtype = he.EntryType.ToString();
if (he.Commander.Name.StartsWith("[BETA]", StringComparison.InvariantCultureIgnoreCase) || he.IsBetaMessage)
{
hasbeta = true;
betatime = he.EventTimeUTC;
he.journalEntry.SetEdsmSync(); // crappy slow but unusual, but lets mark them as sent..
}
else if (!(he.MultiPlayer || discardEvents.Contains(eventtype) || alwaysDiscard.Contains(eventtype)))
{
historylist.Enqueue(new HistoryQueueEntry {
HistoryEntry = he, Logger = log, ManualSync = manual
});
eventCount++;
}
}
}
if (hasbeta && eventCount == 0)
{
log?.Invoke($"Cannot send Beta logs to EDSM - most recent timestamp: {betatime.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'")}");
}
if (manual) // if in manual mode, we want to tell the user the end, so push an end marker
{
string logline = (eventCount > 0) ? $"Sending {eventCount} journal event(s) to EDSM" : "No new events to send to EDSM";
log?.Invoke(logline);
if (eventCount > 0) // push end of sync event.
{
historylist.Enqueue(new HistoryQueueEntry {
HistoryEntry = null, Logger = log, ManualSync = manual
});
}
}
if (eventCount > 0)
{
historyevent.Set();
// Start the sync thread if it's not already running
if (Interlocked.CompareExchange(ref running, 1, 0) == 0)
{
Exit = false;
exitevent.Reset();
ThreadEDSMSync = new System.Threading.Thread(new System.Threading.ThreadStart(SyncThread));
ThreadEDSMSync.Name = "EDSM Journal Sync";
ThreadEDSMSync.IsBackground = true;
ThreadEDSMSync.Start();
}
}
return(true);
}
// Partial update sync, do the 12 hour get and store until we come up to UTC time now approx.
public static long PerformEDSMUpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
DateTime lastrecordtime = GetLastEDSMRecordTimeUTC(); // this is in UTC, as it comes out of the EDSM records
if (DateTime.UtcNow.Subtract(lastrecordtime).TotalHours <= 1) // If we have partial synced for 1 hour, don't waste our time
{
System.Diagnostics.Debug.WriteLine("EDSM No partial sync, last record less than 1 hour old");
return(updates);
}
// Go For SYNC
LogLine("Checking for updated EDSM systems (may take a few moments).");
EDSMClass edsm = new EDSMClass();
while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition
{ // where we do a set, the time moves to just before now,
// and we then do another set with minimum amount of hours
if (PendingClose())
{
return(updates);
}
int hourstofetch = 6;
if (lastrecordtime < ED21date.AddHours(-48))
{
hourstofetch = 48;
}
else if (lastrecordtime < ED23date.AddHours(-12))
{
hourstofetch = 12;
}
DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
ReportProgress(-1, "Requesting systems from EDSM");
System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
string json = null;
BaseUtils.ResponseData response;
try
{
response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000);
}
catch (WebException ex)
{
ReportProgress(-1, $"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
return(updates);
}
catch (Exception ex)
{
ReportProgress(-1, $"EDSM request failed");
LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
return(updates);
}
if (response.Error)
{
if ((int)response.StatusCode == 429)
{
LogLine($"EDSM rate limit hit - waiting 2 minutes");
for (int sec = 0; sec < 120; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
else
{
LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run");
return(updates);
}
}
json = response.Body;
if (json == null)
{
ReportProgress(-1, "EDSM request failed");
LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
return(updates);
}
// debug File.WriteAllText(@"c:\code\json.txt", json);
DateTime prevrectime = lastrecordtime;
System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length);
long updated = 0;
try
{
updated = ParseEDSMUpdateSystemsString(json, grididallow, ref lastrecordtime, false, PendingClose, ReportProgress, false);
System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime());
// if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely)
if (lastrecordtime <= prevrectime)
{
lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString());
ReportProgress(-1, "EDSM request failed");
LogLine("Processing EDSM systems download failed, will try next time program is run");
return(updates);
}
updates += updated;
SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception
int delay = 10; // Anthor's normal delay
int ratelimitlimit;
int ratelimitremain;
int ratelimitreset;
if (response.Headers != null &&
response.Headers["X-Rate-Limit-Limit"] != null &&
response.Headers["X-Rate-Limit-Remaining"] != null &&
response.Headers["X-Rate-Limit-Reset"] != null &&
Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset))
{
if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later..
{
delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota
}
else
{
delay = 0;
}
System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay);
}
for (int sec = 0; sec < delay; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
return(updates);
}
// Partial update sync, do the 12 hour get and store until we come up to UTC time now approx.
public static long PerformEDSMUpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
DateTime lastrecordtime = GetLastEDSMRecordTimeUTC(); // this is in UTC, as it comes out of the EDSM records
if (DateTime.UtcNow.Subtract(lastrecordtime).TotalHours <= 1) // If we have partial synced for 1 hour, don't waste our time
{
System.Diagnostics.Debug.WriteLine("EDSM No partial sync, last record less than 1 hour old");
return(updates);
}
// Go For SYNC
LogLine("Checking for updated EDSM systems (may take a few moments).");
EDSMClass edsm = new EDSMClass();
while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition
{ // where we do a set, the time moves to just before now,
// and we then do another set with minimum amount of hours
if (PendingClose())
{
return(updates);
}
int hourstofetch = 6;
if (lastrecordtime < ED21date.AddHours(-48))
{
hourstofetch = 48;
}
else if (lastrecordtime < ED23date.AddHours(-12))
{
hourstofetch = 12;
}
DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
ReportProgress(-1, "Requesting systems from EDSM");
System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
string json = null;
try
{
json = edsm.RequestSystems(lastrecordtime, enddate, timeout: 20000);
}
catch (WebException ex)
{
ReportProgress(-1, $"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
return(updates);
}
catch (Exception ex)
{
ReportProgress(-1, $"EDSM request failed");
LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
return(updates);
}
if (json == null)
{
ReportProgress(-1, "EDSM request failed");
LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
return(updates);
}
// debug File.WriteAllText(@"c:\code\json.txt", json);
DateTime prevrectime = lastrecordtime;
System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length);
long updated = 0;
try
{
updated = ParseEDSMUpdateSystemsString(json, grididallow, ref lastrecordtime, false, PendingClose, ReportProgress, false);
System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime());
// if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely)
if (lastrecordtime <= prevrectime)
{
lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString());
}
updates += updated;
SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception
}
return(updates);
}
// Called from EDDiscoveryController, in DoPerform, on back thread, to determine what sync to do..
public static long PerformEDSMFullSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
EDSMClass edsm = new EDSMClass();
LogLine("Get hidden systems from EDSM and remove from database");
RemoveHiddenSystems();
if (PendingClose())
{
return(updates);
}
LogLine("Download systems file from EDSM.");
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
LogLine("Resyncing all downloaded EDSM systems with System Database." + Environment.NewLine + "This will take a while.");
// string s = edsmsystems; bool success = true; // debug, comment out next two lines
bool newfile;
bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) =>
{
SQLiteConnectionSystem.CreateTempSystemsTable();
DateTime maxdate = new DateTime(2000, 1, 1);
try
{
using (var reader = new StreamReader(s))
updates = ParseEDSMUpdateSystemsStream(reader, grididallow, ref maxdate, true, PendingClose, ReportProgress, useCache: false, useTempSystems: true);
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.3 Exception " + e.ToString());
}
if (!PendingClose()) // abort, without saving time, to make it do it again
{
LogLine("Replacing old systems table with new systems table and re-indexing - please wait");
ReportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait");
SQLiteConnectionSystem.ReplaceSystemsTable();
SetLastEDSMRecordTimeUTC(maxdate); // record the last record seen in time
ReportProgress(-1, "");
LogLine("System Database updated with EDSM data, " + updates + " systems updated.");
GC.Collect();
}
else
{
success = false;
throw new OperationCanceledException();
}
});
if (!success)
{
LogLine("Failed to download EDSM system file from server, will check next time");
}
return(updates);
}
public static void PerformSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError, SystemsSyncState state) // big check.. done in a thread.
{
reportProgress(-1, "");
state.performhistoryrefresh = false;
state.syncwasfirstrun = SystemClassDB.IsSystemsTableEmpty(); // remember if DB is empty
// Force a full sync if newest data is more than 14 days old
bool outoforder = SQLiteConnectionSystem.GetSettingBool("EDSMSystemsOutOfOrder", true);
DateTime lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast();
if (DateTime.UtcNow.Subtract(lastmod).TotalDays >= 14)
{
state.performedsmsync = true;
}
bool edsmoreddbsync = state.performedsmsync || state.performeddbsync; // remember if we are syncing
state.syncwaseddboredsm = edsmoreddbsync;
if (state.performedsmsync || state.performeddbsync)
{
if (state.performedsmsync && !cancelRequested())
{
// Download new systems
try
{
state.performhistoryrefresh |= PerformEDSMFullSync(cancelRequested, reportProgress, logLine, logError);
state.performedsmsync = false;
}
catch (Exception ex)
{
logError("GetAllEDSMSystems exception:" + ex.Message);
}
}
if (!cancelRequested())
{
logLine("Indexing systems table");
SQLiteConnectionSystem.CreateSystemsTableIndexes();
try
{
EliteDangerousCore.EDDB.SystemClassEDDB.PerformEDDBFullSync(cancelRequested, reportProgress, logLine, logError);
state.performeddbsync = false;
}
catch (Exception ex)
{
logError("GetEDDBUpdate exception: " + ex.Message);
}
state.performhistoryrefresh = true;
}
}
if (!cancelRequested())
{
logLine("Indexing systems table");
SQLiteConnectionSystem.CreateSystemsTableIndexes();
lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast();
if (DateTime.UtcNow.Subtract(lastmod).TotalHours >= 1)
{
logLine("Checking for new EDSM systems (may take a few moments).");
EDSMClass edsm = new EDSMClass();
long updates = edsm.GetNewSystems(cancelRequested, reportProgress, logLine);
logLine($"EDSM updated {updates:N0} systems.");
state.performhistoryrefresh |= (updates > 0);
}
}
reportProgress(-1, "");
}
public static bool PerformEDSMFullSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError)
{
string rwsystime = SQLiteConnectionSystem.GetSettingString("EDSMLastSystems", "2000-01-01 00:00:00"); // Latest time from RW file.
DateTime edsmdate;
if (!DateTime.TryParse(rwsystime, CultureInfo.InvariantCulture, DateTimeStyles.None, out edsmdate))
{
edsmdate = new DateTime(2000, 1, 1);
}
long updates = 0;
// Delete all old systems
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", "2010-01-01 00:00:00");
SQLiteConnectionSystem.PutSettingString("EDDBSystemsTime", "0");
EDSMClass edsm = new EDSMClass();
logLine("Get hidden systems from EDSM and remove from database");
RemoveHiddenSystems();
if (cancelRequested())
{
return(false);
}
logLine("Download systems file from EDSM.");
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
logLine("Resyncing all downloaded EDSM systems with local database." + Environment.NewLine + "This will take a while.");
bool newfile;
bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) =>
{
SQLiteConnectionSystem.CreateTempSystemsTable();
string rwsysfiletime = "2014-01-01 00:00:00";
bool outoforder = false;
using (var reader = new StreamReader(s))
updates = ParseEDSMUpdateSystemsStream(reader, ref rwsysfiletime, ref outoforder, true, cancelRequested, reportProgress, useCache: false, useTempSystems: true);
if (!cancelRequested()) // abort, without saving time, to make it do it again
{
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", rwsysfiletime);
logLine("Replacing old systems table with new systems table and re-indexing - please wait");
reportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait");
SQLiteConnectionSystem.ReplaceSystemsTable();
SQLiteConnectionSystem.PutSettingBool("EDSMSystemsOutOfOrder", outoforder);
reportProgress(-1, "");
}
else
{
throw new OperationCanceledException();
}
});
if (!success)
{
logLine("Failed to download EDSM system file from server, will check next time");
return(false);
}
// Stop if requested
if (cancelRequested())
{
return(false);
}
logLine("Local database updated with EDSM data, " + updates + " systems updated.");
GC.Collect();
return(updates > 0);
}
private void Sync(EDSMClass edsm, HistoryList hl)
{
try
{
logout("EDSM sync begin");
List <HistoryEntry> hlfsdunsyncedlist = hl.FilterByNotEDSMSyncedAndFSD; // first entry is oldest
if (_syncTo && hlfsdunsyncedlist.Count > 0) // send systems to edsm (verified with dates, 29/9/2016, utc throughout)
{
DateTime logstarttime = DateTime.MinValue;
DateTime logendtime = DateTime.MinValue;
logout("EDSM: Sending " + hlfsdunsyncedlist.Count.ToString() + " flightlog entries");
List <HistoryEntry> edsmsystemlog = null;
int edsmsystemssent = 0;
foreach (var he in hlfsdunsyncedlist)
{
if (Exit)
{
running = false;
return;
}
if (edsmsystemlog == null || he.EventTimeUTC >= logendtime.AddDays(-1))
{
edsm.GetLogs(he.EventTimeUTC.AddDays(-1), null, out edsmsystemlog, out logstarttime, out logendtime); // always returns a log, time is in UTC as per HistoryEntry and JournalEntry
}
if (logendtime < logstarttime)
{
running = false;
return;
}
if (edsmsystemlog == null)
{
running = false;
return;
}
HistoryEntry ps2 = (from c in edsmsystemlog where c.System.name == he.System.name && c.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks select c).FirstOrDefault();
if (ps2 != null) // it did, just make sure EDSM sync flag is set..
{
he.SetEdsmSync();
}
else
{
string errmsg; // (verified with EDSM 29/9/2016)
bool firstdiscover;
int edsmid;
if (edsm.SendTravelLog(he.System.name, he.EventTimeUTC, he.System.HasCoordinate && !he.IsStarPosFromEDSM, he.System.x, he.System.y, he.System.z, out errmsg, out firstdiscover, out edsmid))
{
if (edsmid != 0 && he.System.id_edsm <= 0)
{
he.System.id_edsm = edsmid;
JournalEntry.UpdateEDSMIDPosJump(he.Journalid, he.System, false, -1);
}
if (firstdiscover)
{
he.SetFirstDiscover();
}
he.SetEdsmSync();
edsmsystemssent++;
}
if (errmsg.Length > 0)
{
logout(errmsg);
break;
}
}
}
logout(string.Format("EDSM Systems sent {0}", edsmsystemssent));
}
// TBD Comments to edsm?
if (_syncFrom) // Verified ok with time 29/9/2016
{
var json = edsm.GetComments(new DateTime(2011, 1, 1));
if (json != null)
{
JObject msg = JObject.Parse(json);
int msgnr = msg["msgnum"].Value <int>();
JArray comments = (JArray)msg["comments"];
if (comments != null)
{
int commentsadded = 0;
foreach (JObject jo in comments)
{
string name = jo["system"].Value <string>();
string note = jo["comment"].Value <string>();
string utctime = jo["lastUpdate"].Value <string>();
int edsmid = 0;
if (!Int32.TryParse(jo["systemId"].Str("0"), out edsmid))
{
edsmid = 0;
}
DateTime localtime = DateTime.ParseExact(utctime, "yyyy-MM-dd HH:mm:ss",
CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime();
SystemNoteClass curnote = SystemNoteClass.GetNoteOnSystem(name, edsmid);
if (curnote != null) // curnote uses local time to store
{
if (localtime.Ticks > curnote.Time.Ticks) // if newer, add on (verified with EDSM 29/9/2016)
{
curnote.UpdateNote(curnote.Note + ". EDSM: " + note, true, localtime, edsmid, true);
commentsadded++;
}
}
else
{
SystemNoteClass.MakeSystemNote(note, localtime, name, 0, edsmid, true); // new one! its an FSD one as well
commentsadded++;
}
}
logout(string.Format("EDSM Comments downloaded/updated {0}", commentsadded));
}
}
}
logout("EDSM sync Done");
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message);
logout("EDSM sync Exception " + ex.Message);
}
}
private void FetcherThreadProc()
{
Trace.WriteLine($"EDSM Thread logs start");
bool jupdate = false;
DateTime lastCommentFetch = DateTime.MinValue;
int waittime = 2000; // Max 1 request every 2 seconds, with a backoff if the rate limit is hit
if (EDSMRequestBackoffTime > DateTime.UtcNow)
{
waittime = (int)Math.Min(EDSMMaxLogAgeMinutes * 60000, Math.Min(BackoffInterval.TotalSeconds * 1000, EDSMRequestBackoffTime.Subtract(DateTime.UtcNow).TotalSeconds * 1000));
}
while (!ExitRequested.WaitOne(waittime))
{
EDSMClass edsm = new EDSMClass(Commander);
if (edsm.ValidCredentials && DateTime.UtcNow > lastCommentFetch.AddHours(1))
{
edsm.GetComments(l => Trace.WriteLine(l));
lastCommentFetch = DateTime.UtcNow;
}
DateTime logstarttime = DateTime.MinValue; // return what we got..
DateTime logendtime = DateTime.MinValue;
List <JournalFSDJump> edsmlogs = null;
int res = -1; //return code
if (edsm.ValidCredentials && Commander.SyncFromEdsm && DateTime.UtcNow > EDSMRequestBackoffTime)
{
if (DateTime.UtcNow.Subtract(LastEventTime).TotalMinutes >= EDSMMaxLogAgeMinutes)
{
System.Diagnostics.Debug.WriteLine($"Retrieving EDSM logs starting {LastEventTime}");
res = edsm.GetLogs(LastEventTime, null, out edsmlogs, out logstarttime, out logendtime);
}
else if (FirstEventTime > GammaStart)
{
System.Diagnostics.Debug.WriteLine($"Retrieving EDSM logs ending {FirstEventTime}");
res = edsm.GetLogs(null, FirstEventTime, out edsmlogs, out logstarttime, out logendtime);
}
}
if (ExitRequested.WaitOne(0))
{
return;
}
if (res == 429) // Rate Limit Exceeded
{
Trace.WriteLine($"EDSM Log request rate limit hit - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (logstarttime > LastEventTime && logendtime < FirstEventTime)
{
Trace.WriteLine($"Bad start and/or end times returned by EDSM - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (res == 100 && edsmlogs != null)
{
if (edsmlogs.Count > 0) // if anything to process..
{
//Trace.WriteLine($"Retrieving EDSM logs count {edsmlogs.Count}");
BackoffInterval = TimeSpan.FromSeconds(60);
if (logendtime > DateTime.UtcNow)
{
logendtime = DateTime.UtcNow;
}
if (logstarttime < DateTime.MinValue.AddDays(1))
{
logstarttime = DateTime.MinValue.AddDays(1);
}
// Get all of the local entries now that we have the entries from EDSM
// Moved here to avoid the race that could have been causing duplicate entries
// EDSM only returns FSD entries, so only look for them. Tested 27/4/2018 after the HE optimisations
List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr, logstarttime.AddDays(-1), logendtime.AddDays(1)).
OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).
Select(je => HistoryEntry.FromJournalEntry(je, null, out jupdate)).ToList(); // using HE just because of the FillEDSM func
HistoryList hl = new HistoryList(hlfsdlist); // just so we can access the FillEDSM func
List <JournalFSDJump> toadd = new List <JournalFSDJump>();
int previdx = -1;
foreach (JournalFSDJump jfsd in edsmlogs) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == jfsd.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(jfsd.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.EDSMID <= 0) // if we don't have a valid EDMSID..
{
vi.v.System.EDSMID = 0;
hl.FillEDSM(vi.v);
}
if (vi.v.System.EDSMID <= 0 || vi.v.System.EDSMID == jfsd.EdsmID)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0) // its not a duplicate, add to db
{
toadd.Add(jfsd);
}
else
{ // it is a duplicate, check if the first discovery flag is set right
JournalFSDJump existingfsd = hlfsdlist[index].journalEntry as JournalFSDJump;
if (existingfsd != null && existingfsd.EDSMFirstDiscover != jfsd.EDSMFirstDiscover) // if we have a FSD one, and first discover is different
{
existingfsd.UpdateFirstDiscover(jfsd.EDSMFirstDiscover);
}
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
Trace.WriteLine($"Adding EDSM logs count {toadd.Count}");
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDCommander.CurrentCmdrID;
tlu.Add(); // Add to Database
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
foreach (JournalFSDJump jfsd in toadd)
{
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", jfsd.EventTimeUTC, jfsd.StarSystem));
jfsd.SetTLUCommander(tlu.id, jfsd.CommanderId); // update its TLU id to the TLU made above
jfsd.Add(jfsd.CreateFSDJournalEntryJson(), cn);
}
}
LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}");
if (logendtime > LastEventTime || logstarttime <= GammaStart)
{
if (OnDownloadedSystems != null)
{
OnDownloadedSystems();
}
}
}
}
if (logstarttime < FirstEventTime)
{
FirstEventTime = logstarttime;
}
if (logendtime > LastEventTime)
{
LastEventTime = logendtime;
}
}
}
}
private void FetcherThreadProc()
{
Trace.WriteLine($"EDSM Thread logs start");
DateTime lastCommentFetch = DateTime.MinValue;
int waittime = 1000; // initial waittime, will be reestimated later
DateTime curtime = DateTime.UtcNow;
KeyName(out string latestdatekeyname, out string oldestdatekeyname);
while (!ExitRequested.WaitOne(waittime))
{
if (ExitRequested.WaitOne(0))
{
return;
}
EDSMClass edsm = new EDSMClass(Commander);
// logic checked 21/12/2018 RJP
if (edsm.ValidCredentials && Commander.SyncFromEdsm)
{
if (DateTime.UtcNow > lastCommentFetch.AddHours(1))
{
edsm.GetComments(l => Trace.WriteLine(l));
lastCommentFetch = DateTime.UtcNow;
}
DateTime latestentry = EliteDangerousCore.DB.UserDatabase.Instance.GetSettingDate(latestdatekeyname, GammaStart); // lastest entry
DateTime oldestentry = EliteDangerousCore.DB.UserDatabase.Instance.GetSettingDate(oldestdatekeyname, DateTime.UtcNow); // oldest entry
DateTime logstarttime = DateTime.MinValue; // return what we got..
DateTime logendtime = DateTime.MinValue;
List <JournalFSDJump> edsmlogs = null;
BaseUtils.ResponseData response = default(BaseUtils.ResponseData);
int res = -1;
if (DateTime.UtcNow.Subtract(latestentry).TotalMinutes >= EDSMMaxLogAgeMinutes) // is latest entry old?
{
DateTime askfor = DateTime.UtcNow;
System.Diagnostics.Debug.WriteLine("Fetch latest since Curtime > lastestentry + gap " + askfor.ToStringZulu());
res = edsm.GetLogs(null, askfor, out edsmlogs, out logstarttime, out logendtime, out response);
//res = 100; logstarttime = askfor.AddDays(-7); logendtime = askfor; // debug it
}
else if (oldestentry > GammaStart) // if oldest entry younger than gamma?
{
System.Diagnostics.Debug.WriteLine("Go back in time to gamma ");
res = edsm.GetLogs(null, oldestentry, out edsmlogs, out logstarttime, out logendtime, out response);
//res = 100; logstarttime = oldestentry.AddDays(-7); logendtime = oldestentry; // debug it
}
if (res == 100) // hunky dory - note if Anthor faults, we just retry again and again
{
System.Diagnostics.Debug.WriteLine("Data stored from " + oldestentry.ToStringZulu() + " -> " + latestentry.ToStringZulu());
System.Diagnostics.Debug.WriteLine("Process logs from " + logstarttime.ToStringZulu() + " => " + logendtime.ToStringZulu());
if (edsmlogs != null && edsmlogs.Count > 0) // if anything to process..
{
Process(edsmlogs, logstarttime, logendtime);
}
if (logendtime > latestentry)
{
EliteDangerousCore.DB.UserDatabase.Instance.PutSettingDate(latestdatekeyname, logendtime);
}
if (logstarttime < oldestentry)
{
EliteDangerousCore.DB.UserDatabase.Instance.PutSettingDate(oldestdatekeyname, logstarttime);
}
}
else if (res != -1)
{
System.Diagnostics.Debug.WriteLine("EDSM Log request rejected with " + res);
}
if (response.Headers != null &&
response.Headers["X-Rate-Limit-Limit"] != null &&
response.Headers["X-Rate-Limit-Remaining"] != null &&
response.Headers["X-Rate-Limit-Reset"] != null &&
Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out int ratelimitlimit) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out int ratelimitremain) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out int ratelimitreset))
{
if (ratelimitremain < ratelimitlimit * 2 / 4) // lets keep at least X remaining for other purposes later..
{
waittime = 1000 * ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota
}
else
{
waittime = 1000; // 1 second so we don't thrash
}
System.Diagnostics.Debug.WriteLine("EDSM Log Delay Parameters {0} {1} {2} => {3}ms", ratelimitlimit, ratelimitremain, ratelimitreset, waittime);
}
}
}
}
private void FetcherThreadProc()
{
Trace.WriteLine($"EDSM Thread logs start");
bool jupdate = false;
int waittime = 2000; // Max 1 request every 2 seconds, with a backoff if the rate limit is hit
if (EDSMRequestBackoffTime > DateTime.UtcNow)
{
waittime = (int)Math.Min(EDSMMaxLogAgeMinutes * 60000, Math.Min(BackoffInterval.TotalSeconds * 1000, EDSMRequestBackoffTime.Subtract(DateTime.UtcNow).TotalSeconds * 1000));
}
// get them as of now.. since we are searching back in time it should be okay. On a refresh we would start again!
List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr).OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).Select(je => HistoryEntry.FromJournalEntry(je, null, false, out jupdate)).ToList();
while (!ExitRequested.WaitOne(waittime))
{
EDSMClass edsm = new EDSMClass {
apiKey = Commander.APIKey, commanderName = Commander.EdsmName
};
List <HistoryEntry> edsmlogs = null;
DateTime logstarttime = DateTime.MinValue;
DateTime logendtime = DateTime.MinValue;
int res = -1;
if (edsm.IsApiKeySet && Commander.SyncFromEdsm && DateTime.UtcNow > EDSMRequestBackoffTime)
{
if (DateTime.UtcNow.Subtract(LastEventTime).TotalMinutes >= EDSMMaxLogAgeMinutes)
{
//Trace.WriteLine($"Retrieving EDSM logs starting {LastEventTime}");
res = edsm.GetLogs(LastEventTime, null, out edsmlogs, out logstarttime, out logendtime);
}
else if (FirstEventTime > GammaStart)
{
//Trace.WriteLine($"Retrieving EDSM logs ending {FirstEventTime}");
res = edsm.GetLogs(null, FirstEventTime, out edsmlogs, out logstarttime, out logendtime);
}
}
if (ExitRequested.WaitOne(0))
{
return;
}
if (res == 429) // Rate Limit Exceeded
{
Trace.WriteLine($"EDSM Log request rate limit hit - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (logstarttime > LastEventTime && logendtime < FirstEventTime)
{
Trace.WriteLine($"Bad start and/or end times returned by EDSM - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (res == 100 && edsmlogs != null)
{
if (edsmlogs.Count > 0) // if anything to process..
{
//Trace.WriteLine($"Retrieving EDSM logs count {edsmlogs.Count}");
BackoffInterval = TimeSpan.FromSeconds(60);
if (logendtime > DateTime.UtcNow)
{
logendtime = DateTime.UtcNow;
}
HistoryList hl = new HistoryList(hlfsdlist);
List <DateTime> hlfsdtimes = hlfsdlist.Select(he => he.EventTimeUTC).ToList();
List <HistoryEntry> toadd = new List <HistoryEntry>();
int previdx = -1;
foreach (HistoryEntry he in edsmlogs) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(he.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.id_edsm <= 0)
{
vi.v.System.id_edsm = 0;
hl.FillEDSM(vi.v);
}
if (vi.v.System.id_edsm <= 0 || vi.v.System.id_edsm == he.System.id_edsm)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0)
{
toadd.Add(he);
}
else
{
HistoryEntry lhe = hlfsdlist[index];
if (he.IsEDSMFirstDiscover && !lhe.IsEDSMFirstDiscover)
{
lhe.SetFirstDiscover();
}
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
Trace.WriteLine($"Adding EDSM logs count {toadd.Count}");
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDCommander.CurrentCmdrID;
tlu.Add(); // Add to Database
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
foreach (HistoryEntry he in toadd)
{
JObject jo = JournalEntry.CreateFSDJournalEntryJson(he.EventTimeUTC,
he.System.name, he.System.x, he.System.y, he.System.z,
EliteConfigInstance.InstanceConfig.DefaultMapColour);
JournalEntry je =
JournalEntry.CreateFSDJournalEntry(tlu.id, tlu.CommanderId.Value,
(int)SyncFlags.EDSM, jo);
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", je.EventTimeUTC, he.System.name));
je.Add(jo, cn);
}
}
LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}");
if (logendtime > LastEventTime || logstarttime <= GammaStart)
{
if (OnDownloadedSystems != null)
{
OnDownloadedSystems();
}
}
}
}
if (logstarttime < FirstEventTime)
{
FirstEventTime = logstarttime;
}
if (logendtime > LastEventTime)
{
LastEventTime = logendtime;
}
}
}
}