private void Sync(EDSMClass edsm) { try { mainForm.LogLine("EDSM sync begin"); List <HistoryEntry> hlfsdunsyncedlist = mainForm.history.FilterByNotEDSMSyncedAndFSD; // first entry is oldest if (_syncTo && hlfsdunsyncedlist.Count > 0) // send systems to edsm (verified with dates, 29/9/2016, utc throughout) { DateTime logstarttime = DateTime.MinValue; DateTime logendtime = DateTime.MinValue; mainForm.LogLine("EDSM: Sending " + hlfsdunsyncedlist.Count.ToString() + " flightlog entries"); List <HistoryEntry> edsmsystemlog = null; int edsmsystemssent = 0; foreach (var he in hlfsdunsyncedlist) { if (Exit) { running = false; return; } if (edsmsystemlog == null || he.EventTimeUTC >= logendtime.AddDays(-1)) { edsm.GetLogs(he.EventTimeUTC.AddDays(-1), null, out edsmsystemlog, out logstarttime, out logendtime); // always returns a log, time is in UTC as per HistoryEntry and JournalEntry } if (logendtime < logstarttime) { running = false; return; } if (edsmsystemlog == null) { running = false; return; } HistoryEntry ps2 = (from c in edsmsystemlog where c.System.name == he.System.name && c.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks select c).FirstOrDefault(); if (ps2 != null) // it did, just make sure EDSM sync flag is set.. { he.SetEdsmSync(); } else { string errmsg; // (verified with EDSM 29/9/2016) bool firstdiscover; int edsmid; if (edsm.SendTravelLog(he.System.name, he.EventTimeUTC, he.System.HasCoordinate && !he.IsStarPosFromEDSM, he.System.x, he.System.y, he.System.z, out errmsg, out firstdiscover, out edsmid)) { if (edsmid != 0 && he.System.id_edsm <= 0) { he.System.id_edsm = edsmid; EDDiscovery.EliteDangerous.JournalEntry.UpdateEDSMIDPosJump(he.Journalid, he.System, false, -1); } if (firstdiscover) { he.SetFirstDiscover(); } he.SetEdsmSync(); edsmsystemssent++; } if (errmsg.Length > 0) { mainForm.LogLine(errmsg); break; } } } mainForm.LogLine(string.Format("EDSM Systems sent {0}", edsmsystemssent)); } // TBD Comments to edsm? if (_syncFrom) // Verified ok with time 29/9/2016 { var json = edsm.GetComments(new DateTime(2011, 1, 1)); if (json != null) { JObject msg = JObject.Parse(json); int msgnr = msg["msgnum"].Value <int>(); JArray comments = (JArray)msg["comments"]; if (comments != null) { int commentsadded = 0; foreach (JObject jo in comments) { string name = jo["system"].Value <string>(); string note = jo["comment"].Value <string>(); string utctime = jo["lastUpdate"].Value <string>(); int edsmid = 0; if (!Int32.TryParse(JSONHelper.GetStringDef(jo["systemId"], "0"), out edsmid)) { edsmid = 0; } DateTime localtime = DateTime.ParseExact(utctime, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime(); SystemNoteClass curnote = SystemNoteClass.GetNoteOnSystem(name, edsmid); if (curnote != null) { // curnote uses local time to store if (localtime.Ticks > curnote.Time.Ticks) // if newer, add on (verified with EDSM 29/9/2016) { curnote.Note += ". EDSM: " + note; curnote.Time = localtime; curnote.EdsmId = edsmid; curnote.Update(); commentsadded++; } } else { curnote = new SystemNoteClass(); curnote.Note = note; curnote.Time = localtime; curnote.Name = name; curnote.Journalid = 0; curnote.EdsmId = edsmid; curnote.Add(); commentsadded++; } } mainForm.LogLine(string.Format("EDSM Comments downloaded/updated {0}", commentsadded)); } } } mainForm.LogLine("EDSM sync Done"); } catch (Exception ex) { System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message); mainForm.LogLineHighlight("EDSM sync Exception " + ex.Message); } }
private void FetcherThreadProc() { bool jupdate = false; LastEventTime = DateTime.UtcNow; FirstEventTime = LastEventTime; int waittime = 2000; // Max 1 request every 2 seconds, with a backoff if the rate limit is hit if (EDSMRequestBackoffTime > DateTime.UtcNow) { waittime = (int)Math.Min(EDSMMaxLogAgeMinutes * 60000, Math.Min(BackoffInterval.TotalSeconds * 1000, EDSMRequestBackoffTime.Subtract(DateTime.UtcNow).TotalSeconds * 1000)); } while (!ExitRequested.WaitOne(waittime)) { EDSMClass edsm = new EDSMClass { apiKey = Commander.APIKey, commanderName = Commander.EdsmName }; List <HistoryEntry> edsmlogs = null; DateTime logstarttime = DateTime.MinValue; DateTime logendtime = DateTime.MinValue; int res = -1; if (edsm.IsApiKeySet && Commander.SyncFromEdsm && DateTime.UtcNow > EDSMRequestBackoffTime) { if (DateTime.UtcNow.Subtract(LastEventTime).TotalMinutes >= EDSMMaxLogAgeMinutes) { Trace.WriteLine($"Retrieving logs starting {LastEventTime}"); res = edsm.GetLogs(LastEventTime, null, out edsmlogs, out logstarttime, out logendtime); } else if (FirstEventTime > GammaStart) { Trace.WriteLine($"Retrieving logs ending {FirstEventTime}"); res = edsm.GetLogs(null, FirstEventTime, out edsmlogs, out logstarttime, out logendtime); } } if (ExitRequested.WaitOne(0)) { return; } if (res == 429) // Rate Limit Exceeded { Trace.WriteLine($"EDSM Log request rate limit hit - backing off for {BackoffInterval.TotalSeconds}s"); EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval; BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60); } else if (logstarttime > LastEventTime && logendtime < FirstEventTime) { Trace.WriteLine($"Bad start and/or end times returned by EDSM - backing off for {BackoffInterval.TotalSeconds}s"); EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval; BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60); } else if (res == 100 && edsmlogs != null) { BackoffInterval = TimeSpan.FromSeconds(60); if (logendtime > DateTime.UtcNow) { logendtime = DateTime.UtcNow; } List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr).OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).Select(je => HistoryEntry.FromJournalEntry(je, null, false, out jupdate)).ToList(); HistoryList hl = new HistoryList(hlfsdlist); List <DateTime> hlfsdtimes = hlfsdlist.Select(he => he.EventTimeUTC).ToList(); List <HistoryEntry> toadd = new List <HistoryEntry>(); int previdx = -1; foreach (HistoryEntry he in edsmlogs) // find out list of ones not present { int index = hlfsdlist.FindIndex(x => x.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks); if (index < 0) { // Look for any entries where DST may have thrown off the time foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase))) { if (vi.i > previdx) { double hdiff = vi.v.EventTimeUTC.Subtract(he.EventTimeUTC).TotalHours; if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff)) { if (vi.v.System.id_edsm <= 0) { vi.v.System.id_edsm = 0; hl.FillEDSM(vi.v); } if (vi.v.System.id_edsm <= 0 || vi.v.System.id_edsm == he.System.id_edsm) { index = vi.i; break; } } } } } if (index < 0) { toadd.Add(he); } else { HistoryEntry lhe = hlfsdlist[index]; if (he.IsEDSMFirstDiscover && !lhe.IsEDSMFirstDiscover) { lhe.SetFirstDiscover(); } previdx = index; } } if (toadd.Count > 0) // if we have any, we can add { TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it tlu.type = 2; // EDSM tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); tlu.Size = 0; tlu.Path = "EDSM"; tlu.CommanderId = EDCommander.CurrentCmdrID; tlu.Add(); // Add to Database using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true)) { foreach (HistoryEntry he in toadd) { JObject jo = EDDiscovery.EliteDangerous.JournalEntry.CreateFSDJournalEntryJson(he.EventTimeUTC, he.System.name, he.System.x, he.System.y, he.System.z, EDDConfig.Instance.DefaultMapColour); EDDiscovery.EliteDangerous.JournalEntry je = EDDiscovery.EliteDangerous.JournalEntry.CreateFSDJournalEntry(tlu.id, tlu.CommanderId.Value, (int)EDDiscovery.EliteDangerous.SyncFlags.EDSM, jo); System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", je.EventTimeUTC, he.System.name)); je.Add(jo, cn); } } LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}"); if (logendtime > LastEventTime || logstarttime <= GammaStart) { if (OnDownloadedSystems != null) { OnDownloadedSystems(); } } } if (logstarttime < FirstEventTime) { FirstEventTime = logstarttime; } if (logendtime > LastEventTime) { LastEventTime = logendtime; } } } }