static public long ParseEDDBUpdateSystems(string filename, Action <string> logline) { StreamReader sr = new StreamReader(filename); // read directly from file.. if (sr == null) { return(0); } string line; int updated = 0; int inserted = 0; while (!sr.EndOfStream) { using (SQLiteTxnLockED <SQLiteConnectionSystem> tl = new SQLiteTxnLockED <SQLiteConnectionSystem>()) { tl.OpenWriter(); using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: EDDbAccessMode.Writer)) // open the db { DbCommand selectCmd = null; DbCommand insertCmd = null; DbCommand updateCmd = null; DbCommand updateSysCmd = null; using (DbTransaction txn = cn.BeginTransaction()) { try { selectCmd = cn.CreateCommand("SELECT EddbId, Population, EddbUpdatedAt FROM EddbSystems WHERE EdsmId = @EdsmId LIMIT 1", txn); // 1 return matching ID selectCmd.AddParameter("@Edsmid", DbType.Int64); insertCmd = cn.CreateCommand("INSERT INTO EddbSystems (EdsmId, EddbId, Name, Faction, Population, GovernmentId, AllegianceId, State, Security, PrimaryEconomyId, NeedsPermit, EddbUpdatedAt) " + "VALUES (@EdsmId, @EddbId, @Name, @Faction, @Population, @GovernmentId, @AllegianceId, @State, @Security, @PrimaryEconomyid, @NeedsPermit, @EddbUpdatedAt)", txn); insertCmd.AddParameter("@EdsmId", DbType.Int64); insertCmd.AddParameter("@EddbId", DbType.Int64); insertCmd.AddParameter("@Name", DbType.String); insertCmd.AddParameter("@Faction", DbType.String); insertCmd.AddParameter("@Population", DbType.Int64); insertCmd.AddParameter("@GovernmentId", DbType.Int64); insertCmd.AddParameter("@AllegianceId", DbType.Int64); insertCmd.AddParameter("@State", DbType.Int64); insertCmd.AddParameter("@Security", DbType.Int64); insertCmd.AddParameter("@PrimaryEconomyId", DbType.Int64); insertCmd.AddParameter("@NeedsPermit", DbType.Int64); insertCmd.AddParameter("@EddbUpdatedAt", DbType.Int64); updateCmd = cn.CreateCommand("UPDATE EddbSystems SET EddbId=@EddbId, Name=@Name, Faction=@Faction, Population=@Population, GovernmentId=@GovernmentId, AllegianceId=@AllegianceId, State=@State, Security=@Security, PrimaryEconomyId=@PrimaryEconomyId, NeedsPermit=@NeedsPermit, EddbUpdatedAt=@EddbUpdatedAt WHERE EdsmId=@EdsmId", txn); updateCmd.AddParameter("@EdsmId", DbType.Int64); updateCmd.AddParameter("@EddbId", DbType.Int64); updateCmd.AddParameter("@Name", DbType.String); updateCmd.AddParameter("@Faction", DbType.String); updateCmd.AddParameter("@Population", DbType.Int64); updateCmd.AddParameter("@GovernmentId", DbType.Int64); updateCmd.AddParameter("@AllegianceId", DbType.Int64); updateCmd.AddParameter("@State", DbType.Int64); updateCmd.AddParameter("@Security", DbType.Int64); updateCmd.AddParameter("@PrimaryEconomyId", DbType.Int64); updateCmd.AddParameter("@NeedsPermit", DbType.Int64); updateCmd.AddParameter("@EddbUpdatedAt", DbType.Int64); updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET EddbId=@EddbId WHERE EdsmId=@EdsmId"); updateSysCmd.AddParameter("@EdsmId", DbType.Int64); updateSysCmd.AddParameter("@EddbId", DbType.Int64); int c = 0; int hasinfo = 0; int lasttc = Environment.TickCount; while (!SQLiteConnectionSystem.IsReadWaiting) { line = sr.ReadLine(); if (line == null) // End of stream { break; } { JObject jo = JObject.Parse(line); ISystem system = SystemClassDB.FromJson(jo, SystemInfoSource.EDDB); if (system.HasEDDBInformation) // screen out for speed any EDDB data with empty interesting fields { hasinfo++; selectCmd.Parameters["@EdsmId"].Value = system.id_edsm; // EDDB carries EDSM ID, so find entry in dB //DEBUGif ( c > 30000 ) Console.WriteLine("EDDB ID " + system.id_eddb + " EDSM ID " + system.id_edsm + " " + system.name + " Late info system"); long updated_at = 0; long population = 0; long eddbid = 0; using (DbDataReader reader1 = selectCmd.ExecuteReader()) // if found (if not, we ignore EDDB system) { if (reader1.Read()) // its there.. check its got the right stuff in it. { eddbid = (long)reader1["EddbId"]; updated_at = (long)reader1["EddbUpdatedAt"]; population = (long)reader1["Population"]; } } updateSysCmd.Parameters["@EdsmId"].Value = system.id_edsm; updateSysCmd.Parameters["@EddbId"].Value = system.id_eddb; updateSysCmd.ExecuteNonQuery(); if (eddbid != 0) { if (updated_at != system.eddb_updated_at || population != system.population) { updateCmd.Parameters["@EddbId"].Value = system.id_eddb; updateCmd.Parameters["@Name"].Value = system.name; updateCmd.Parameters["@Faction"].Value = system.faction; updateCmd.Parameters["@Population"].Value = system.population; updateCmd.Parameters["@GovernmentId"].Value = system.government; updateCmd.Parameters["@AllegianceId"].Value = system.allegiance; updateCmd.Parameters["@State"].Value = system.state; updateCmd.Parameters["@Security"].Value = system.security; updateCmd.Parameters["@PrimaryEconomyId"].Value = system.primary_economy; updateCmd.Parameters["@NeedsPermit"].Value = system.needs_permit; updateCmd.Parameters["@EddbUpdatedAt"].Value = system.eddb_updated_at; updateCmd.Parameters["@EdsmId"].Value = system.id_edsm; updateCmd.ExecuteNonQuery(); updated++; } } else { insertCmd.Parameters["@EdsmId"].Value = system.id_edsm; insertCmd.Parameters["@EddbId"].Value = system.id_eddb; insertCmd.Parameters["@Name"].Value = system.name; insertCmd.Parameters["@Faction"].Value = system.faction; insertCmd.Parameters["@Population"].Value = system.population; insertCmd.Parameters["@GovernmentId"].Value = system.government; insertCmd.Parameters["@AllegianceId"].Value = system.allegiance; insertCmd.Parameters["@State"].Value = system.state; insertCmd.Parameters["@Security"].Value = system.security; insertCmd.Parameters["@PrimaryEconomyId"].Value = system.primary_economy; insertCmd.Parameters["@NeedsPermit"].Value = system.needs_permit; insertCmd.Parameters["@EddbUpdatedAt"].Value = system.eddb_updated_at; insertCmd.ExecuteNonQuery(); inserted++; } } else { //Console.WriteLine("EDDB ID " + system.id_eddb + " EDSM ID " + system.id_edsm + " " + system.name + " No info reject"); } if (++c % 10000 == 0) { Console.WriteLine("EDDB Count " + c + " Delta " + (Environment.TickCount - lasttc) + " info " + hasinfo + " update " + updated + " new " + inserted); lasttc = Environment.TickCount; } } } txn.Commit(); } catch { ExtendedControls.MessageBoxTheme.Show("There is a problem using the EDDB systems file." + Environment.NewLine + "Please perform a manual EDDB sync (see Admin menu) next time you run the program ", "EDDB Sync Error"); break; } finally { if (selectCmd != null) { selectCmd.Dispose(); } if (updateCmd != null) { updateCmd.Dispose(); } if (insertCmd != null) { insertCmd.Dispose(); } } } } } } return(updated + inserted); }
private static long DoParseEDSMUpdateSystemsReader(JsonTextReader jr, ref string date, ref bool outoforder, Func <bool> cancelRequested, Action <int, string> reportProgress, bool useCache = true, bool useTempSystems = false) { DateTime maxdate; if (!DateTime.TryParse(date, CultureInfo.InvariantCulture, DateTimeStyles.None, out maxdate)) { maxdate = new DateTime(2010, 1, 1); } Dictionary <long, SystemClassBase> systemsByEdsmId = useCache ? GetEdsmSystemsLite() : new Dictionary <long, SystemClassBase>(); int count = 0; int updatecount = 0; int insertcount = 0; Random rnd = new Random(); int[] histogramsystems = new int[50000]; string sysnamesTableName = useTempSystems ? "SystemNames_temp" : "SystemNames"; string edsmsysTableName = useTempSystems ? "EdsmSystems_temp" : "EdsmSystems"; Stopwatch sw = Stopwatch.StartNew(); const int BlockSize = 10000; while (!cancelRequested()) { bool jr_eof = false; List <EDSMDumpSystem> objs = new List <EDSMDumpSystem>(BlockSize); while (!cancelRequested()) { if (jr.Read()) { if (jr.TokenType == JsonToken.StartObject) { objs.Add(EDSMDumpSystem.Deserialize(jr)); if (objs.Count >= BlockSize) { break; } } } else { jr_eof = true; break; } } IEnumerator <EDSMDumpSystem> jo_enum = objs.GetEnumerator(); bool jo_enum_finished = false; while (!jo_enum_finished && !cancelRequested()) { int blkcount = 0; int oldinsertcnt = insertcount; int oldupdatecnt = updatecount; int oldcount = count; using (SQLiteTxnLockED <SQLiteConnectionSystem> tl = new SQLiteTxnLockED <SQLiteConnectionSystem>()) { tl.OpenWriter(); using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: EDDbAccessMode.Writer)) { using (DbTransaction txn = cn.BeginTransaction()) { DbCommand updateNameCmd = null; DbCommand updateSysCmd = null; DbCommand insertNameCmd = null; DbCommand insertSysCmd = null; DbCommand selectSysCmd = null; DbCommand selectNameCmd = null; try { updateNameCmd = cn.CreateCommand("UPDATE SystemNames SET Name=@Name WHERE EdsmId=@EdsmId", txn); updateNameCmd.AddParameter("@Name", DbType.String); updateNameCmd.AddParameter("@EdsmId", DbType.Int64); updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET X=@X, Y=@Y, Z=@Z, UpdateTimestamp=@UpdateTimestamp, VersionTimestamp=@VersionTimestamp, GridId=@GridId, RandomId=@RandomId WHERE EdsmId=@EdsmId", txn); updateSysCmd.AddParameter("@X", DbType.Int64); updateSysCmd.AddParameter("@Y", DbType.Int64); updateSysCmd.AddParameter("@Z", DbType.Int64); updateSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64); updateSysCmd.AddParameter("@VersionTimestamp", DbType.Int64); updateSysCmd.AddParameter("@GridId", DbType.Int64); updateSysCmd.AddParameter("@RandomId", DbType.Int64); updateSysCmd.AddParameter("@EdsmId", DbType.Int64); insertNameCmd = cn.CreateCommand("INSERT INTO " + sysnamesTableName + " (Name, EdsmId) VALUES (@Name, @EdsmId)", txn); insertNameCmd.AddParameter("@Name", DbType.String); insertNameCmd.AddParameter("@EdsmId", DbType.Int64); insertSysCmd = cn.CreateCommand("INSERT INTO " + edsmsysTableName + " (EdsmId, X, Y, Z, CreateTimestamp, UpdateTimestamp, VersionTimestamp, GridId, RandomId) VALUES (@EdsmId, @X, @Y, @Z, @CreateTimestamp, @UpdateTimestamp, @VersionTimestamp, @GridId, @RandomId)", txn); insertSysCmd.AddParameter("@EdsmId", DbType.Int64); insertSysCmd.AddParameter("@X", DbType.Int64); insertSysCmd.AddParameter("@Y", DbType.Int64); insertSysCmd.AddParameter("@Z", DbType.Int64); insertSysCmd.AddParameter("@CreateTimestamp", DbType.Int64); insertSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64); insertSysCmd.AddParameter("@VersionTimestamp", DbType.Int64); insertSysCmd.AddParameter("@GridId", DbType.Int64); insertSysCmd.AddParameter("@RandomId", DbType.Int64); selectSysCmd = cn.CreateCommand("SELECT Id, X, Y, Z, GridId, RandomId FROM EdsmSystems WHERE EdsmId=@EdsmId"); selectSysCmd.AddParameter("@EdsmId", DbType.Int64); selectNameCmd = cn.CreateCommand("SELECT Name FROM SystemNames WHERE EdsmId = @EdsmId"); selectNameCmd.AddParameter("@EdsmId", DbType.Int64); while (!cancelRequested()) { if (!jo_enum.MoveNext()) { reportProgress(-1, $"Syncing EDSM systems: {count:N0} processed, {insertcount:N0} new systems, {updatecount:N0} updated systems"); txn.Commit(); if (jr_eof) { date = maxdate.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); Console.WriteLine($"Import took {sw.ElapsedMilliseconds}ms"); for (int id = 0; id < histogramsystems.Length; id++) { if (histogramsystems[id] != 0) { Console.WriteLine("Id " + id + " count " + histogramsystems[id]); } } return(updatecount + insertcount); } jo_enum_finished = true; break; } else if (SQLiteConnectionSystem.IsReadWaiting) { if (blkcount < objs.Count * 3 / 4) // Let the reader barge in if we've processed less than 3/4 of the items { // Reset the counts, roll back the transaction, and let the reader through... insertcount = oldinsertcnt; updatecount = oldupdatecnt; count = oldcount; jo_enum.Reset(); txn.Rollback(); break; } } EDSMDumpSystem jo = jo_enum.Current; EDSMDumpSystemCoords coords = jo.coords; if (coords != null) { double x = coords.x; double y = coords.y; double z = coords.z; long edsmid = jo.id; string name = jo.name; int gridid = GridId.Id(x, z); int randomid = rnd.Next(0, 99); DateTime updatedate = jo.date; histogramsystems[gridid]++; if (updatedate > maxdate) { maxdate = updatedate; } else if (updatedate < maxdate - TimeSpan.FromHours(1)) { outoforder = true; } SystemClassBase dbsys = null; if (!useTempSystems) { if (useCache && systemsByEdsmId.ContainsKey(edsmid)) { dbsys = systemsByEdsmId[edsmid]; } if (!useCache) { selectSysCmd.Parameters["@EdsmId"].Value = edsmid; using (DbDataReader reader = selectSysCmd.ExecuteReader()) { if (reader.Read()) { dbsys = new SystemClassBase { id = (long)reader["id"], id_edsm = edsmid }; if (System.DBNull.Value == reader["x"]) { dbsys.x = double.NaN; dbsys.y = double.NaN; dbsys.z = double.NaN; } else { dbsys.x = ((double)(long)reader["X"]) / SystemClassDB.XYZScalar; dbsys.y = ((double)(long)reader["Y"]) / SystemClassDB.XYZScalar; dbsys.z = ((double)(long)reader["Z"]) / SystemClassDB.XYZScalar; } dbsys.id_edsm = edsmid; dbsys.gridid = reader["GridId"] == DBNull.Value ? 0 : (int)((long)reader["GridId"]); dbsys.randomid = reader["RandomId"] == DBNull.Value ? 0 : (int)((long)reader["RandomId"]); } } if (dbsys != null) { selectNameCmd.Parameters["@EdsmId"].Value = edsmid; using (DbDataReader reader = selectNameCmd.ExecuteReader()) { if (reader.Read()) { dbsys.name = (string)reader["Name"]; } } } } } if (dbsys != null) { // see if EDSM data changed.. if (!dbsys.name.Equals(name)) { updateNameCmd.Parameters["@Name"].Value = name; updateNameCmd.Parameters["@EdsmId"].Value = edsmid; updateNameCmd.ExecuteNonQuery(); } if (Math.Abs(dbsys.x - x) > 0.01 || Math.Abs(dbsys.y - y) > 0.01 || Math.Abs(dbsys.z - z) > 0.01 || dbsys.gridid != gridid) // position changed { updateSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; updateSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; updateSysCmd.Parameters["@GridId"].Value = gridid; updateSysCmd.Parameters["@RandomId"].Value = randomid; updateSysCmd.Parameters["@EdsmId"].Value = edsmid; updateSysCmd.ExecuteNonQuery(); updatecount++; } } else // not in database.. { insertNameCmd.Parameters["@Name"].Value = name; insertNameCmd.Parameters["@EdsmId"].Value = edsmid; insertNameCmd.ExecuteNonQuery(); insertSysCmd.Parameters["@EdsmId"].Value = edsmid; insertSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@CreateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@GridId"].Value = gridid; insertSysCmd.Parameters["@RandomId"].Value = randomid; insertSysCmd.ExecuteNonQuery(); insertcount++; } } count++; blkcount++; } } finally { if (updateNameCmd != null) { updateNameCmd.Dispose(); } if (updateSysCmd != null) { updateSysCmd.Dispose(); } if (insertNameCmd != null) { insertNameCmd.Dispose(); } if (insertSysCmd != null) { insertSysCmd.Dispose(); } if (selectSysCmd != null) { selectSysCmd.Dispose(); } } } } } } } if (cancelRequested()) { throw new OperationCanceledException(); } return(updatecount + insertcount); }
private void NamedStars() // background thread.. run after Update. Thread never deletes, only adds to its own structures { try // just in case someone tears us down.. { int lylimit = (int)(_starlimitly / _lastcamera.LastZoom); lylimit = Math.Max(lylimit, 1); int sqlylimit = lylimit * lylimit; // in squared distance limit from viewpoint StarGrid.TransFormInfo ti = new StarGrid.TransFormInfo(_resmat, _znear, _glControl.Width, _glControl.Height, sqlylimit, _lastcamera.LastCameraPos); SortedDictionary <float, StarGrid.InViewInfo> inviewlist = new SortedDictionary <float, StarGrid.InViewInfo>(new DuplicateKeyComparer <float>()); // who's in view, sorted by distance //Stopwatch sw1 = new Stopwatch();sw1.Start(); Tools.LogToFile(String.Format("starnamesest Estimate at {0} len {1}", ti.campos, sqlylimit)); _stargrids.GetSystemsInView(ref inviewlist, 2000.0F, ti); // consider all grids under 2k from current pos. //Tools.LogToFile(String.Format("starnamesest Took {0} in view {1}", sw1.ElapsedMilliseconds, inviewlist.Count)); float textscalingw = Math.Min(_starnamemaxly, Math.Max(_starnamesizely / _lastcamera.LastZoom, _starnameminly)); // per char float starsize = Math.Min(Math.Max(_lastcamera.LastZoom / 10F, 1.0F), 20F); // Normal stars are at 1F. //Console.WriteLine("Per char {0} h {1} sc {2} ", textscalingw, textscalingh, starsize); foreach (StarNames s in _starnamesbackground.Values) // all items not processed { s.updatedinview = false; // only items remaining will clear this } _needrepaint = false; // assume nothing changes int painted = 0; //string res = ""; // used to view whats added/removed/draw.. foreach (StarGrid.InViewInfo inview in inviewlist.Values) // for all in viewport, sorted by distance from camera position { using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem()) { StarNames sys = null; bool draw = false; if (_starnamesbackground.ContainsKey(inview.position)) // if already there.. { sys = _starnamesbackground[inview.position]; sys.updatedinview = true; draw = (_discson && sys.paintstar == null && sys.newstar == null) || (_nameson && ((sys.nametexture == null && sys.newnametexture == null))); painted++; } else if (painted < maxstars) { ISystem sc = _formmap.FindSystem(inview.position, cn); // with the open connection, find this star.. if (sc != null) // if can't be resolved, ignore { sys = new StarNames(sc, inview.position); // we keep position in here using same floats as inview so it will match _starnamesbackground.Add(inview.position, sys); // add to our database _starnamestoforeground.Add(sys); // send to foreground for adding draw = true; painted++; //Tools.LogToFile(String.Format("starnamesest: push {0}", sys.Pos)); //res += "N"; } } else { break; // no point doing any more.. got our fill of items } if (draw) { _needrepaint = true; // changed a item.. needs a repaint if (_nameson) { float width = textscalingw * sys.name.Length; Bitmap map = DatasetBuilder.DrawString(sys.name, _namecolour, _starfont); sys.newnametexture = TexturedQuadData.FromBitmap(map, new PointData(sys.pos.X, sys.pos.Y, sys.pos.Z), _lastcamera.Rotation, width, textscalingw * 4.0F, _startextoffset + width / 2, 0); sys.rotation = _lastcamera.Rotation; // remember when we were when we draw it sys.zoom = _lastcamera.LastZoom; } if (_discson) { sys.newstar = new PointData(sys.pos.X, sys.pos.Y, sys.pos.Z, starsize, inview.AsColor); } } } } foreach (StarNames s in _starnamesbackground.Values) // only items above will remain. { //if (s.inview != s.updatedinview) res += (s.updatedinview) ? "+" : "-"; _needrepaint = _needrepaint || (s.inview != s.updatedinview); // set if we change our mind on any of the items s.inview = s.updatedinview; // copy flag over, causes foreground to start removing them } //if ( _needrepaint) Console.WriteLine(String.Format("starnamesest in view {0} limit {1} repaint {2} {3}", inviewlist.Count, lylimit, _needrepaint, res)); //Tools.LogToFile(String.Format("starnamesest added all delta {0} topaint {1}", sw1.ElapsedMilliseconds, painted)); } catch (Exception ex) { System.Diagnostics.Trace.WriteLine("Exception watcher: " + ex.Message); System.Diagnostics.Trace.WriteLine("Trace: " + ex.StackTrace); } _formmap.BeginInvoke((System.Windows.Forms.MethodInvoker) delegate // kick the UI thread to process. { _formmap.ChangeNamedStars(); }); }
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, out bool journalupdate, SQLiteConnectionSystem conn = null, EDCommander cmdr = null) { ISystem isys = prev == null ? new SystemClass("Unknown") : prev.System; int indexno = prev == null ? 1 : prev.Indexno + 1; int mapcolour = 0; journalupdate = false; bool starposfromedsm = false; bool firstdiscover = false; if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump) { JournalLocOrJump jl = je as JournalLocOrJump; ISystem newsys; if (jl != null && jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it { newsys = new SystemClass(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z) { EDSMID = jl.EdsmID < 0 ? 0 : jl.EdsmID, // pass across the EDSMID for the lazy load process. Faction = jl.Faction, Government = jl.EDGovernment, PrimaryEconomy = jl.EDEconomy, Security = jl.EDSecurity, Population = jl.Population ?? 0, State = jl.EDState, Allegiance = jl.EDAllegiance, UpdateDate = jl.EventTimeUTC, status = SystemStatusEnum.EDDiscovery, SystemAddress = jl.SystemAddress, }; // If it was a new system, pass the coords back to the StartJump if (prev != null && prev.journalEntry is JournalStartJump) { prev.System = newsys; // give the previous startjump our system.. } } else { // NOTE Rob: 09-JAN-2018 I've removed the Jumpstart looking up a system by name since they were using up lots of lookup time during history reading. // This is used for pre 2.2 systems without co-ords, which now should be limited. // JumpStart still gets the system when the FSD loc is processed, see above. // Jumpstart was also screwing about with the EDSM ID fill in which was broken. This is now working again. // Default one newsys = new SystemClass(jl.StarSystem); newsys.EDSMID = je.EdsmID; ISystem s = SystemCache.FindSystem(newsys, conn); // has no co-ord, did we find it? if (s != null) // found a system.. { if (jl != null && jl.HasCoordinate) // if journal Loc, and journal has a star position, use that instead of EDSM.. { s.X = Math.Round(jl.StarPos.X * 32.0) / 32.0; s.Y = Math.Round(jl.StarPos.Y * 32.0) / 32.0; s.Z = Math.Round(jl.StarPos.Z * 32.0) / 32.0; } //Debug.WriteLine("HistoryList found system {0} {1}", s.id_edsm, s.name); newsys = s; if (jl != null && je.EdsmID <= 0 && newsys.EDSMID > 0) // only update on a JL.. { journalupdate = true; Debug.WriteLine("HE EDSM ID update requested {0} {1}", newsys.EDSMID, newsys.Name); } } else { newsys.EDSMID = -1; // mark as checked but not found } } JournalFSDJump jfsd = je as JournalFSDJump; if (jfsd != null) { if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord { jfsd.JumpDist = isys.Distance(newsys); // fill it out here if (jfsd.JumpDist > 0) { journalupdate = true; Debug.WriteLine("Je Jump distance update(3) requested {0} {1} {2}", newsys.EDSMID, newsys.Name, jfsd.JumpDist); } } mapcolour = jfsd.MapColor; } isys = newsys; starposfromedsm = (jl != null && jl.HasCoordinate) ? jl.StarPosFromEDSM : newsys.HasCoordinate; firstdiscover = jl == null ? false : jl.EDSMFirstDiscover; } string summary, info, detailed; je.FillInformation(out summary, out info, out detailed); HistoryEntry he = new HistoryEntry { Indexno = indexno, EntryType = je.EventTypeID, Journalid = je.Id, journalEntry = je, System = isys, EventTimeUTC = je.EventTimeUTC, MapColour = mapcolour, EdsmSync = je.SyncedEDSM, EDDNSync = je.SyncedEDDN, EGOSync = je.SyncedEGO, StartMarker = je.StartMarker, StopMarker = je.StopMarker, EventSummary = summary, EventDescription = info, EventDetailedInfo = detailed, IsStarPosFromEDSM = starposfromedsm, IsEDSMFirstDiscover = firstdiscover, Commander = cmdr ?? EDCommander.GetCommander(je.CommanderId) }; // WORK out docked/landed state if (prev != null) { if (prev.docked.HasValue) // copy docked.. { he.docked = prev.docked; } if (prev.landed.HasValue) { he.landed = prev.landed; } if (prev.hyperspace.HasValue) { he.hyperspace = prev.hyperspace; } if (prev.marketId != null) { he.marketId = prev.marketId; } he.stationName = prev.stationName; he.shiptype = prev.shiptype; he.shipid = prev.shipid; he.whereami = prev.whereami; he.onCrewWithCaptain = prev.onCrewWithCaptain; he.gamemode = prev.gamemode; he.group = prev.group; } if (je.EventTypeID == JournalTypeEnum.Location) { JournalLocation jl = je as JournalLocation; he.docked = jl.Docked; he.landed = jl.Latitude.HasValue; he.whereami = jl.Docked ? jl.StationName : jl.Body; he.hyperspace = false; } else if (je.EventTypeID == JournalTypeEnum.Docked) { JournalDocked jl = je as JournalDocked; he.docked = true; he.whereami = jl.StationName; he.stationName = jl.StationName; he.marketId = jl.MarketID; } else if (je.EventTypeID == JournalTypeEnum.Undocked) { he.docked = false; he.stationName = null; he.marketId = null; } else if (je.EventTypeID == JournalTypeEnum.Touchdown) { he.landed = true; } else if (je.EventTypeID == JournalTypeEnum.Liftoff) { he.landed = !(je as JournalLiftoff).PlayerControlled; } else if (je.EventTypeID == JournalTypeEnum.SupercruiseEntry) { he.whereami = (je as JournalSupercruiseEntry).StarSystem; he.hyperspace = true; } else if (je.EventTypeID == JournalTypeEnum.SupercruiseExit) { he.whereami = (je as JournalSupercruiseExit).Body; he.hyperspace = false; } else if (je.EventTypeID == JournalTypeEnum.FSDJump) { he.whereami = (je as JournalFSDJump).StarSystem; he.hyperspace = true; } else if (je.EventTypeID == JournalTypeEnum.StartJump) { he.hyperspace = true; // some of these are just to make sure, as FSDJump will also set it } else if (je.EventTypeID == JournalTypeEnum.LoadGame) { JournalLoadGame jl = je as JournalLoadGame; he.onCrewWithCaptain = null; // can't be in a crew at this point he.gamemode = jl.GameMode; // set game mode he.group = jl.Group; // and group, may be empty he.landed = jl.StartLanded; he.hyperspace = false; if (jl.Ship.IndexOf("buggy", StringComparison.InvariantCultureIgnoreCase) == -1) // load game with buggy, can't tell what ship we get back into, so ignore { he.shiptype = (je as JournalLoadGame).Ship; he.shipid = (je as JournalLoadGame).ShipId; } } else if (je.EventTypeID == JournalTypeEnum.ShipyardBuy) // BUY does not have ship id, but the new entry will that is written later - journals 8.34 { he.shiptype = (je as JournalShipyardBuy).ShipType; } else if (je.EventTypeID == JournalTypeEnum.ShipyardNew) { he.shiptype = (je as JournalShipyardNew).ShipType; he.shipid = (je as JournalShipyardNew).ShipId; } else if (je.EventTypeID == JournalTypeEnum.ShipyardSwap) { he.shiptype = (je as JournalShipyardSwap).ShipType; he.shipid = (je as JournalShipyardSwap).ShipId; } else if (je.EventTypeID == JournalTypeEnum.JoinACrew) { he.onCrewWithCaptain = (je as JournalJoinACrew).Captain; } else if (je.EventTypeID == JournalTypeEnum.QuitACrew) { he.onCrewWithCaptain = null; } if (prev != null && prev.travelling) // if we are travelling.. { he.travelled_distance = prev.travelled_distance; he.travelled_missingjump = prev.travelled_missingjump; he.travelled_jumps = prev.travelled_jumps; if (he.IsFSDJump && !he.MultiPlayer) // if jump, and not multiplayer.. { double dist = ((JournalFSDJump)je).JumpDist; if (dist <= 0) { he.travelled_missingjump++; } else { he.travelled_distance += dist; he.travelled_jumps++; } } he.travelled_seconds = prev.travelled_seconds; TimeSpan diff = he.EventTimeUTC.Subtract(prev.EventTimeUTC); if (he.EntryType != JournalTypeEnum.LoadGame && diff < new TimeSpan(2, 0, 0)) // time between last entry and load game is not real time { he.travelled_seconds += diff; } if (he.StopMarker || he.StartMarker) { //Debug.WriteLine("Travelling stop at " + he.Indexno); he.travelling = false; he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelled " + he.travelled_distance.ToStringInvariant("0.0") + " LY" + ", " + he.travelled_jumps + " jumps" + ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") + ", time " + he.travelled_seconds; he.travelled_distance = 0; he.travelled_seconds = new TimeSpan(0); } else { he.travelling = true; if (he.IsFSDJump) { he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelling" + " distance " + he.travelled_distance.ToString("0.0") + " LY" + ", " + he.travelled_jumps + " jumps" + ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") + ", time " + he.travelled_seconds; } } } if (he.StartMarker) { //Debug.WriteLine("Travelling start at " + he.Indexno); he.travelling = true; } return(he); }
// Protected against bad JSON public int GetLogs(DateTime?starttimeutc, DateTime?endtimeutc, out List <JournalFSDJump> log, out DateTime logstarttime, out DateTime logendtime) { log = new List <JournalFSDJump>(); logstarttime = DateTime.MaxValue; logendtime = DateTime.MinValue; if (!ValidCredentials) { return(0); } string query = "get-logs?showId=1&apiKey=" + apiKey + "&commanderName=" + HttpUtility.UrlEncode(commanderName); if (starttimeutc != null) { query += "&startDateTime=" + HttpUtility.UrlEncode(starttimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); } if (endtimeutc != null) { query += "&endDateTime=" + HttpUtility.UrlEncode(endtimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); } var response = RequestGet("api-logs-v1/" + query, handleException: true); if (response.Error) { return(0); } var json = response.Body; if (json == null) { return(0); } try { JObject msg = JObject.Parse(json); int msgnr = msg["msgnum"].Int(0); JArray logs = (JArray)msg["logs"]; if (logs != null) { string startdatestr = msg["startDateTime"].Value <string>(); string enddatestr = msg["endDateTime"].Value <string>(); if (startdatestr == null || !DateTime.TryParseExact(startdatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logstarttime)) { logstarttime = DateTime.MaxValue; } if (enddatestr == null || !DateTime.TryParseExact(enddatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logendtime)) { logendtime = DateTime.MinValue; } using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem()) { foreach (JObject jo in logs) { string name = jo["system"].Value <string>(); string ts = jo["date"].Value <string>(); long id = jo["systemId"].Value <long>(); bool firstdiscover = jo["firstDiscover"].Value <bool>(); DateTime etutc = DateTime.ParseExact(ts, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); // UTC time ISystem sc = SystemClassDB.GetSystem(id, cn, SystemClassDB.SystemIDType.EdsmId, name: name); if (sc == null) { if (DateTime.UtcNow.Subtract(etutc).TotalHours < 6) // Avoid running into the rate limit { sc = GetSystemsByName(name)?.FirstOrDefault(s => s.EDSMID == id); } if (sc == null) { sc = new SystemClass(name) { EDSMID = id }; } } JournalFSDJump fsd = new JournalFSDJump(etutc, sc, EliteConfigInstance.InstanceConfig.DefaultMapColour, firstdiscover, (int)SyncFlags.EDSM); log.Add(fsd); } } } return(msgnr); } catch (Exception e) { System.Diagnostics.Debug.WriteLine("Failed due to " + e.ToString()); return(499); // BAD JSON } }
// Called from EDDiscoveryController, in DoPerform, on back thread, to determine what sync to do.. public static long PerformEDSMFullSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight) { long updates = 0; EDSMClass edsm = new EDSMClass(); LogLine("Get hidden systems from EDSM and remove from database"); RemoveHiddenSystems(); if (PendingClose()) { return(updates); } LogLine("Download systems file from EDSM."); string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json"); LogLine("Resyncing all downloaded EDSM systems with System Database." + Environment.NewLine + "This will take a while."); // string s = edsmsystems; bool success = true; // debug, comment out next two lines // with the new downloader, the file is stored, then processed. If the user cancels, we will use the old file again. Once complete, the file is removed ReportProgress(-1, "Downloading star database from EDSM"); bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDSMFullSystemsURL, edsmsystems, false, out bool newfile, (n, s) => { SQLiteConnectionSystem.CreateTempSystemsTable(); DateTime maxdate = new DateTime(2000, 1, 1); try { using (var reader = new StreamReader(s)) updates = ParseEDSMUpdateSystemsStream(reader, grididallow, ref maxdate, true, PendingClose, ReportProgress, useCache: false, useTempSystems: true); } catch (Exception e) { System.Diagnostics.Debug.WriteLine("SysClassEDSM.3 Exception " + e.ToString()); } if (!PendingClose()) // abort, without saving time, to make it do it again { LogLine("Replacing old systems table with new systems table and re-indexing - please wait"); ReportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait"); SQLiteConnectionSystem.ReplaceSystemsTable(); SetLastEDSMRecordTimeUTC(maxdate); // record the last record seen in time ReportProgress(-1, ""); BaseUtils.FileHelpers.DeleteFileNoError(edsmsystems); // remove file - don't hold in storage LogLine("System Database updated with EDSM data, " + updates + " systems updated."); GC.Collect(); } else { success = false; ReportProgress(-1, "Operation Cancelled"); throw new OperationCanceledException(); } }, cancelRequested: PendingClose); if (!success) { ReportProgress(-1, "EDSM Failed to download correctly"); LogLine("Failed to download EDSM system file from server, will check next time"); } return(updates); }
// Time storers static public void ForceEDSMFullUpdate() { SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", "2010-01-01 00:00:00"); }
public int GetLogs(DateTime?starttimeutc, DateTime?endtimeutc, out List <HistoryEntry> log, out DateTime logstarttime, out DateTime logendtime) { log = new List <HistoryEntry>(); logstarttime = DateTime.MaxValue; logendtime = DateTime.MinValue; if (!IsApiKeySet) { return(0); } string query = "get-logs?showId=1&apiKey=" + apiKey + "&commanderName=" + HttpUtility.UrlEncode(commanderName); if (starttimeutc != null) { query += "&startDateTime=" + HttpUtility.UrlEncode(starttimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); } if (endtimeutc != null) { query += "&endDateTime=" + HttpUtility.UrlEncode(endtimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); } var response = RequestGet("api-logs-v1/" + query, handleException: true); if (response.Error) { return(0); } var json = response.Body; if (json == null) { return(0); } JObject msg = JObject.Parse(json); int msgnr = msg["msgnum"].Int(0); JArray logs = (JArray)msg["logs"]; if (logs != null) { string startdatestr = msg["startDateTime"].Value <string>(); string enddatestr = msg["endDateTime"].Value <string>(); if (startdatestr == null || !DateTime.TryParseExact(startdatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logstarttime)) { logstarttime = DateTime.MaxValue; } if (enddatestr == null || !DateTime.TryParseExact(enddatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logendtime)) { logendtime = DateTime.MinValue; } using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem()) { foreach (JObject jo in logs) { string name = jo["system"].Value <string>(); string ts = jo["date"].Value <string>(); long id = jo["systemId"].Value <long>(); bool firstdiscover = jo["firstDiscover"].Value <bool>(); DateTime etutc = DateTime.ParseExact(ts, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); // UTC time ISystem sc = SystemClassDB.GetSystem(id, cn, SystemClassDB.SystemIDType.EdsmId); if (sc == null) { sc = GetSystemsByName(name).FirstOrDefault(s => s.id_edsm == id); if (sc == null) { sc = new SystemClass(name) { id_edsm = id }; } } HistoryEntry he = HistoryEntry.MakeVSEntry(sc, etutc, EliteConfigInstance.InstanceConfig.DefaultMapColour, "", "", firstdiscover: firstdiscover); // FSD jump entry log.Add(he); } } } return(msgnr); }
public static HistoryList LoadHistory(EDJournalClass journalmonitor, Func <bool> cancelRequested, Action <int, string> reportProgress, string NetLogPath = null, bool ForceNetLogReload = false, bool ForceJournalReload = false, bool CheckEdsm = false, int CurrentCommander = Int32.MinValue, bool Keepuievents = true) { HistoryList hist = new HistoryList(); EDCommander cmdr = null; if (CurrentCommander >= 0) { cmdr = EDCommander.GetCommander(CurrentCommander); journalmonitor.ParseJournalFiles(() => cancelRequested(), (p, s) => reportProgress(p, s), forceReload: ForceJournalReload); // Parse files stop monitor.. if (NetLogPath != null) { string errstr = null; NetLogClass.ParseFiles(NetLogPath, out errstr, EliteConfigInstance.InstanceConfig.DefaultMapColour, () => cancelRequested(), (p, s) => reportProgress(p, s), ForceNetLogReload, currentcmdrid: CurrentCommander); } } reportProgress(-1, "Resolving systems"); List <JournalEntry> jlist = JournalEntry.GetAll(CurrentCommander).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList(); List <Tuple <JournalEntry, HistoryEntry> > jlistUpdated = new List <Tuple <JournalEntry, HistoryEntry> >(); using (SQLiteConnectionSystem conn = new SQLiteConnectionSystem()) { HistoryEntry prev = null; JournalEntry jprev = null; foreach (JournalEntry je in jlist) { if (MergeEntries(jprev, je)) // if we merge.. we may have updated info, so reprint. { jprev.FillInformation(out prev.EventSummary, out prev.EventDescription, out prev.EventDetailedInfo); // need to keep this up to date.. continue; } if (je.IsUIEvent && !Keepuievents) // filter out any UI events { System.Diagnostics.Debug.WriteLine("**** Filter out " + je.EventTypeStr + " on " + je.EventTimeLocal.ToString()); continue; } bool journalupdate = false; HistoryEntry he = HistoryEntry.FromJournalEntry(je, prev, CheckEdsm, out journalupdate, conn, cmdr); prev = he; jprev = je; hist.historylist.Add(he); if (journalupdate) { jlistUpdated.Add(new Tuple <JournalEntry, HistoryEntry>(je, he)); } } } if (jlistUpdated.Count > 0) { reportProgress(-1, "Updating journal entries"); using (SQLiteConnectionUser conn = new SQLiteConnectionUser(utc: true)) { using (DbTransaction txn = conn.BeginTransaction()) { foreach (Tuple <JournalEntry, HistoryEntry> jehe in jlistUpdated) { JournalEntry je = jehe.Item1; HistoryEntry he = jehe.Item2; JournalFSDJump jfsd = je as JournalFSDJump; if (jfsd != null) { JournalEntry.UpdateEDSMIDPosJump(jfsd.Id, he.System, !jfsd.HasCoordinate && he.System.HasCoordinate, jfsd.JumpDist, conn, txn); } } txn.Commit(); } } } // now database has been updated due to initial fill, now fill in stuff which needs the user database hist.CommanderId = CurrentCommander; hist.ProcessUserHistoryListEntries(h => h.ToList()); // here, we update the DBs in HistoryEntry and any global DBs in historylist hist.SendEDSMStatusInfo(hist.GetLast, true); return(hist); }
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, bool checkedsm, out bool journalupdate, SQLiteConnectionSystem conn = null, EDCommander cmdr = null) { ISystem isys = prev == null ? new SystemClassDB("Unknown") : prev.System; int indexno = prev == null ? 1 : prev.Indexno + 1; int mapcolour = 0; journalupdate = false; bool starposfromedsm = false; bool firstdiscover = false; if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump) { JournalLocOrJump jl = je as JournalLocOrJump; JournalFSDJump jfsd = je as JournalFSDJump; ISystem newsys; if (jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it { newsys = new SystemClassDB(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z); newsys.id_edsm = jl.EdsmID < 0 ? 0 : jl.EdsmID; // pass across the EDSMID for the lazy load process. if (jfsd != null && jfsd.JumpDist <= 0 && isys.HasCoordinate) // if we don't have a jump distance (pre 2.2) but the last sys does have pos, we can compute distance and update entry { jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here journalupdate = true; } } else { // Default one newsys = new SystemClassDB(jl.StarSystem); newsys.id_edsm = jl.EdsmID; if (checkedsm) // see if we can find the right system { SystemClassDB s = SystemClassDB.FindEDSM(newsys, conn); // has no co-ord, did we find it? if (s != null) // yes, use, and update the journal with the esdmid, and also the position if we have a co-ord { // so next time we don't have to do this again.. if (jl.HasCoordinate) { s.x = Math.Round(jl.StarPos.X * 32.0) / 32.0; s.y = Math.Round(jl.StarPos.Y * 32.0) / 32.0; s.z = Math.Round(jl.StarPos.Z * 32.0) / 32.0; } newsys = s; if (jfsd != null && jfsd.JumpDist <= 0 && newsys.HasCoordinate && isys.HasCoordinate) // if we don't have a jump distance (pre 2.2) but the last sys does, we can compute { jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here. EDSM systems always have co-ords, but we should check anyway journalupdate = true; } if (jl.EdsmID <= 0 && newsys.id_edsm > 0) { journalupdate = true; } } } } if (jfsd != null) { if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord { jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here journalupdate = true; } mapcolour = jfsd.MapColor; } isys = newsys; starposfromedsm = jl.HasCoordinate ? jl.StarPosFromEDSM : newsys.HasCoordinate; firstdiscover = jl.EDSMFirstDiscover; } string summary, info, detailed; je.FillInformation(out summary, out info, out detailed); HistoryEntry he = new HistoryEntry { Indexno = indexno, EntryType = je.EventTypeID, Journalid = je.Id, journalEntry = je, System = isys, EventTimeUTC = je.EventTimeUTC, MapColour = mapcolour, EdsmSync = je.SyncedEDSM, EDDNSync = je.SyncedEDDN, EGOSync = je.SyncedEGO, StartMarker = je.StartMarker, StopMarker = je.StopMarker, EventSummary = summary, EventDescription = info, EventDetailedInfo = detailed, IsStarPosFromEDSM = starposfromedsm, IsEDSMFirstDiscover = firstdiscover, Commander = cmdr ?? EDCommander.GetCommander(je.CommanderId) }; // WORK out docked/landed state if (prev != null) { if (prev.docked.HasValue) // copy docked.. { he.docked = prev.docked; } if (prev.landed.HasValue) { he.landed = prev.landed; } he.shiptype = prev.shiptype; he.shipid = prev.shipid; he.whereami = prev.whereami; he.onCrewWithCaptain = prev.onCrewWithCaptain; he.gamemode = prev.gamemode; he.group = prev.group; } if (je.EventTypeID == JournalTypeEnum.Location) { JournalLocation jl = je as JournalLocation; he.docked = jl.Docked; he.whereami = jl.Docked ? jl.StationName : jl.Body; } else if (je.EventTypeID == JournalTypeEnum.Docked) { JournalDocked jl = je as JournalDocked; he.docked = true; he.whereami = jl.StationName; } else if (je.EventTypeID == JournalTypeEnum.Undocked) { he.docked = false; } else if (je.EventTypeID == JournalTypeEnum.Touchdown) { he.landed = true; } else if (je.EventTypeID == JournalTypeEnum.Liftoff) { he.landed = false; } else if (je.EventTypeID == JournalTypeEnum.SupercruiseEntry) { he.whereami = (je as JournalSupercruiseEntry).StarSystem; } else if (je.EventTypeID == JournalTypeEnum.SupercruiseExit) { he.whereami = (je as JournalSupercruiseExit).Body; } else if (je.EventTypeID == JournalTypeEnum.FSDJump) { he.whereami = (je as JournalFSDJump).StarSystem; } else if (je.EventTypeID == JournalTypeEnum.LoadGame) { JournalLoadGame jl = je as JournalLoadGame; he.onCrewWithCaptain = null; // can't be in a crew at this point he.gamemode = jl.GameMode; // set game mode he.group = jl.Group; // and group, may be empty he.landed = jl.StartLanded; if (jl.Ship.IndexOf("buggy", StringComparison.InvariantCultureIgnoreCase) == -1) // load game with buggy, can't tell what ship we get back into, so ignore { he.shiptype = (je as JournalLoadGame).Ship; he.shipid = (je as JournalLoadGame).ShipId; } } else if (je.EventTypeID == JournalTypeEnum.ShipyardBuy) // BUY does not have ship id, but the new entry will that is written later - journals 8.34 { he.shiptype = (je as JournalShipyardBuy).ShipType; } else if (je.EventTypeID == JournalTypeEnum.ShipyardNew) { he.shiptype = (je as JournalShipyardNew).ShipType; he.shipid = (je as JournalShipyardNew).ShipId; } else if (je.EventTypeID == JournalTypeEnum.ShipyardSwap) { he.shiptype = (je as JournalShipyardSwap).ShipType; he.shipid = (je as JournalShipyardSwap).ShipId; } else if (je.EventTypeID == JournalTypeEnum.JoinACrew) { he.onCrewWithCaptain = (je as JournalJoinACrew).Captain; } else if (je.EventTypeID == JournalTypeEnum.QuitACrew) { he.onCrewWithCaptain = null; } if (prev != null && prev.travelling) // if we are travelling.. { he.travelled_distance = prev.travelled_distance; he.travelled_missingjump = prev.travelled_missingjump; he.travelled_jumps = prev.travelled_jumps; if (he.IsFSDJump && !he.MultiPlayer) // if jump, and not multiplayer.. { double dist = ((JournalFSDJump)je).JumpDist; if (dist <= 0) { he.travelled_missingjump++; } else { he.travelled_distance += dist; he.travelled_jumps++; } } he.travelled_seconds = prev.travelled_seconds; TimeSpan diff = he.EventTimeUTC.Subtract(prev.EventTimeUTC); if (he.EntryType != JournalTypeEnum.LoadGame && diff < new TimeSpan(2, 0, 0)) // time between last entry and load game is not real time { he.travelled_seconds += diff; } if (he.StopMarker || he.StartMarker) { //Debug.WriteLine("Travelling stop at " + he.Indexno); he.travelling = false; he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelled " + he.travelled_distance.ToStringInvariant("0.0") + " LY" + ", " + he.travelled_jumps + " jumps" + ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") + ", time " + he.travelled_seconds; he.travelled_distance = 0; he.travelled_seconds = new TimeSpan(0); } else { he.travelling = true; if (he.IsFSDJump) { he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelling" + " distance " + he.travelled_distance.ToString("0.0") + " LY" + ", " + he.travelled_jumps + " jumps" + ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") + ", time " + he.travelled_seconds; } } } if (he.StartMarker) { //Debug.WriteLine("Travelling start at " + he.Indexno); he.travelling = true; } return(he); }
internal long GetNewSystems(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine) { string lstsyst; DateTime lstsystdate; // First system in EDSM is from 2015-05-01 00:39:40 DateTime gammadate = new DateTime(2015, 5, 1, 0, 0, 0, DateTimeKind.Utc); bool outoforder = SQLiteConnectionSystem.GetSettingBool("EDSMSystemsOutOfOrder", true); if (SystemClassDB.IsSystemsTableEmpty()) { lstsystdate = gammadate; } else { // Get the most recent modify time returned from EDSM DateTime lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast(); lstsystdate = lastmod - TimeSpan.FromSeconds(1); if (lstsystdate < gammadate) { lstsystdate = gammadate; } } lstsyst = lstsystdate.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); Console.WriteLine("EDSM Check date: " + lstsyst); long updates = 0; while (lstsystdate < DateTime.UtcNow) { if (cancelRequested()) { return(updates); } DateTime enddate = lstsystdate + TimeSpan.FromHours(12); if (enddate > DateTime.UtcNow) { enddate = DateTime.UtcNow; } logLine($"Downloading systems from {lstsystdate.ToLocalTime().ToString()} to {enddate.ToLocalTime().ToString()}"); reportProgress(-1, "Requesting systems from EDSM"); string json = null; try { json = RequestSystems(lstsystdate, enddate); } catch (WebException ex) { reportProgress(-1, $"EDSM request failed"); if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse) { string status = ((HttpWebResponse)ex.Response).StatusDescription; logLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run"); } else { logLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run"); } break; } catch (Exception ex) { reportProgress(-1, $"EDSM request failed"); logLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run"); break; } if (json == null) { reportProgress(-1, "EDSM request failed"); logLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run"); break; } updates += SystemClassEDSM.ParseEDSMUpdateSystemsString(json, ref lstsyst, ref outoforder, false, cancelRequested, reportProgress, false); lstsystdate += TimeSpan.FromHours(12); } logLine($"System download complete"); return(updates); }
public long UpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <string> ReportProgress) { DateTime lastrecordtime = SQLiteConnectionSystem.GetLastEDSMRecordTimeUTC(); long updates = 0; double fetchmult = 1; while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition { // where we do a set, the time moves to just before now, // and we then do another set with minimum amount of hours if (PendingClose()) { return(updates); } if (updates == 0) { LogLine("Checking for updated EDSM systems (may take a few moments)."); } EDSMClass edsm = new EDSMClass(); double hourstofetch = 3; if (lastrecordtime < ED21date.AddHours(-48)) { hourstofetch = 48; } else if (lastrecordtime < ED23date.AddHours(-12)) { hourstofetch = 12; } else if (lastrecordtime < ED30date.AddHours(-6)) { hourstofetch = 6; } DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch * fetchmult); if (enddate > DateTime.UtcNow) { enddate = DateTime.UtcNow; } LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}"); System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}"); string json = null; BaseUtils.ResponseData response; try { Stopwatch sw = new Stopwatch(); response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000); fetchmult = Math.Max(0.1, Math.Min(Math.Min(fetchmult * 1.1, 1.0), 5000.0 / sw.ElapsedMilliseconds)); } catch (WebException ex) { ReportProgress($"EDSM request failed"); if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse) { string status = ((HttpWebResponse)ex.Response).StatusDescription; LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run"); } else { LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run"); } return(updates); } catch (Exception ex) { ReportProgress($"EDSM request failed"); LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run"); return(updates); } if (response.Error) { if ((int)response.StatusCode == 429) { LogLine($"EDSM rate limit hit - waiting 2 minutes"); for (int sec = 0; sec < 120; sec++) { if (!PendingClose()) { System.Threading.Thread.Sleep(1000); } } } else { LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run"); return(updates); } } json = response.Body; if (json == null) { ReportProgress("EDSM request failed"); LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run"); return(updates); } // debug File.WriteAllText(@"c:\code\json.txt", json); DateTime prevrectime = lastrecordtime; System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length); long updated = 0; try { ReportProgress($"EDSM star database update from UTC " + lastrecordtime.ToUniversalTime().ToString()); updated = SystemsDB.ParseEDSMJSONString(json, grididallow, ref lastrecordtime, PendingClose, ReportProgress, ""); System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}"); System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime()); // if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely) if (lastrecordtime <= prevrectime) { lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck } } catch (Exception e) { System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString()); ReportProgress("EDSM request failed"); LogLine("Processing EDSM systems download failed, will try next time program is run"); return(updates); } updates += updated; SQLiteConnectionSystem.SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception int delay = 10; // Anthor's normal delay int ratelimitlimit; int ratelimitremain; int ratelimitreset; if (response.Headers != null && response.Headers["X-Rate-Limit-Limit"] != null && response.Headers["X-Rate-Limit-Remaining"] != null && response.Headers["X-Rate-Limit-Reset"] != null && Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) && Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) && Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset)) { if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later.. { delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota } else { delay = 0; } System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay); } for (int sec = 0; sec < delay; sec++) { if (!PendingClose()) { System.Threading.Thread.Sleep(1000); } } } return(updates); }
private void DoPerformSync() // in Background worker { InvokeAsyncOnUiThread.Invoke(() => OnSyncStarting?.Invoke()); // tell listeners sync is starting resyncEDSMEDDBRequestedFlag = 1; // sync is happening, stop any async requests.. // check for 102, and if not performing a full sync, upgrade it SQLiteConnectionSystem.UpgradeSystemTableFrom102TypeDB(() => PendingClose, ReportSyncProgress, syncstate.perform_edsm_fullsync); Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Perform EDSM/EDDB sync"); try { bool[] grids = new bool[GridId.MaxGridID]; foreach (int i in GridId.FromString(EDDConfig.Instance.EDSMGridIDs)) { grids[i] = true; } syncstate.ClearCounters(); if (syncstate.perform_edsm_fullsync || syncstate.perform_eddb_edsmalias_sync) { if (syncstate.perform_edsm_fullsync && !PendingClose) { // Download new systems try { string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json"); ReportSyncProgress("Performing full download of EDSM Database from server"); bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDSMFullSystemsURL, edsmsystems, false, out bool newfile); syncstate.perform_edsm_fullsync = false; if (success) { syncstate.edsm_fullsync_count = SQLiteConnectionSystem.UpgradeSystemTableFromFile(edsmsystems, grids, () => PendingClose, ReportSyncProgress); if (syncstate.edsm_fullsync_count < 0) // this should always update something, the table is replaced. If its not, its been cancelled { return; } BaseUtils.FileHelpers.DeleteFileNoError(edsmsystems); // remove file - don't hold in storage } } catch (Exception ex) { LogLineHighlight("GetAllEDSMSystems exception:" + ex.Message); } } if (!PendingClose) { try { EDSMClass edsm = new EDSMClass(); string jsonhidden = edsm.GetHiddenSystems(); if (jsonhidden != null) { SystemsDB.ParseAliasString(jsonhidden); string eddbsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "eddbsystems.json"); bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDDBSystemsURL, eddbsystems, false, out bool newfile); syncstate.perform_eddb_edsmalias_sync = false; if (success) { syncstate.eddb_sync_count = SystemsDB.ParseEDDBJSONFile(eddbsystems, () => PendingClose); if (syncstate.eddb_sync_count < 0) // on a cancel or error { return; } SQLiteConnectionSystem.SetLastEDDBDownloadTime(); BaseUtils.FileHelpers.DeleteFileNoError(eddbsystems); // remove file - don't hold in storage } } } catch (Exception ex) { LogLineHighlight("GetEDDBUpdate exception: " + ex.Message); } } } if (!PendingClose) { syncstate.edsm_updatesync_count = UpdateSync(grids, () => PendingClose, ReportSyncProgress); } } catch (OperationCanceledException) { // Swallow Operation Cancelled exceptions } catch (Exception ex) { LogLineHighlight("Check Systems exception: " + ex.Message + Environment.NewLine + "Trace: " + ex.StackTrace); } InvokeAsyncOnUiThread(() => PerformSyncCompletedonUI()); }
public static void PerformSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError, SystemsSyncState state) // big check.. done in a thread. { reportProgress(-1, ""); state.performhistoryrefresh = false; state.syncwasfirstrun = SystemClassDB.IsSystemsTableEmpty(); // remember if DB is empty // Force a full sync if newest data is more than 14 days old bool outoforder = SQLiteConnectionSystem.GetSettingBool("EDSMSystemsOutOfOrder", true); DateTime lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast(); if (DateTime.UtcNow.Subtract(lastmod).TotalDays >= 14) { state.performedsmsync = true; } bool edsmoreddbsync = state.performedsmsync || state.performeddbsync; // remember if we are syncing state.syncwaseddboredsm = edsmoreddbsync; if (state.performedsmsync || state.performeddbsync) { if (state.performedsmsync && !cancelRequested()) { // Download new systems try { state.performhistoryrefresh |= PerformEDSMFullSync(cancelRequested, reportProgress, logLine, logError); state.performedsmsync = false; } catch (Exception ex) { logError("GetAllEDSMSystems exception:" + ex.Message); } } if (!cancelRequested()) { logLine("Indexing systems table"); SQLiteConnectionSystem.CreateSystemsTableIndexes(); try { EliteDangerousCore.EDDB.SystemClassEDDB.PerformEDDBFullSync(cancelRequested, reportProgress, logLine, logError); state.performeddbsync = false; } catch (Exception ex) { logError("GetEDDBUpdate exception: " + ex.Message); } state.performhistoryrefresh = true; } } if (!cancelRequested()) { logLine("Indexing systems table"); SQLiteConnectionSystem.CreateSystemsTableIndexes(); lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast(); if (DateTime.UtcNow.Subtract(lastmod).TotalHours >= 1) { logLine("Checking for new EDSM systems (may take a few moments)."); EDSMClass edsm = new EDSMClass(); long updates = edsm.GetNewSystems(cancelRequested, reportProgress, logLine); logLine($"EDSM updated {updates:N0} systems."); state.performhistoryrefresh |= (updates > 0); } } reportProgress(-1, ""); }
// Called from EDDiscoveryController, in DoPerform, on back thread, to determine what sync to do.. public static long PerformEDSMFullSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight) { long updates = 0; EDSMClass edsm = new EDSMClass(); LogLine("Get hidden systems from EDSM and remove from database"); RemoveHiddenSystems(); if (PendingClose()) { return(updates); } LogLine("Download systems file from EDSM."); string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json"); LogLine("Resyncing all downloaded EDSM systems with System Database." + Environment.NewLine + "This will take a while."); // string s = edsmsystems; bool success = true; // debug, comment out next two lines bool newfile; bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) => { SQLiteConnectionSystem.CreateTempSystemsTable(); DateTime maxdate = new DateTime(2000, 1, 1); using (var reader = new StreamReader(s)) updates = ParseEDSMUpdateSystemsStream(reader, grididallow, ref maxdate, true, PendingClose, ReportProgress, useCache: false, useTempSystems: true); if (!PendingClose()) // abort, without saving time, to make it do it again { LogLine("Replacing old systems table with new systems table and re-indexing - please wait"); ReportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait"); SQLiteConnectionSystem.ReplaceSystemsTable(); SetLastEDSMRecordTimeUTC(maxdate); // record the last record seen in time ReportProgress(-1, ""); LogLine("System Database updated with EDSM data, " + updates + " systems updated."); GC.Collect(); } else { success = false; throw new OperationCanceledException(); } }); if (!success) { LogLine("Failed to download EDSM system file from server, will check next time"); } return(updates); }
// Called from Background Thread Worker at Init() private void BackgroundInit() { StarScan.LoadBodyDesignationMap(); MaterialCommodityDB.SetUpInitialTable(); SQLiteConnectionSystem.CreateSystemsTableIndexes(); // just make sure they are there.. Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Check systems"); ReportProgress(-1, ""); if (!EDDOptions.Instance.NoSystemsLoad) { // Async load of maps in another thread downloadMapsTask = DownloadMaps(this, () => PendingClose, LogLine, LogLineHighlight); // Former CheckSystems, reworked to accomodate new switches.. // Check to see what sync refreshes we need // New Galmap load - it was not doing a refresh if EDSM sync kept on happening. Now has its own timer string rwgalmaptime = SQLiteConnectionSystem.GetSettingString("EDSMGalMapLast", "2000-01-01 00:00:00"); // Latest time from RW file. DateTime galmaptime; if (!DateTime.TryParse(rwgalmaptime, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out galmaptime)) { galmaptime = new DateTime(2000, 1, 1); } if (DateTime.Now.Subtract(galmaptime).TotalDays > 14) // Over 14 days do a sync from EDSM for galmap { LogLine("Get galactic mapping from EDSM."); galacticMapping.DownloadFromEDSM(); SQLiteConnectionSystem.PutSettingString("EDSMGalMapLast", DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss")); } Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Check systems complete"); } galacticMapping.ParseData(); // at this point, gal map data has been uploaded - get it into memory SystemClassDB.AddToAutoComplete(galacticMapping.GetGMONames()); SystemNoteClass.GetAllSystemNotes(); BookmarkClass.GetAllBookmarks(); LogLine("Loaded Notes, Bookmarks and Galactic mapping."); ReportProgress(-1, ""); InvokeAsyncOnUiThread(() => OnInitialSyncComplete?.Invoke()); if (PendingClose) { return; } if (EliteDangerousCore.EDDN.EDDNClass.CheckforEDMC()) // EDMC is running { if (EDCommander.Current.SyncToEddn) // Both EDD and EDMC should not sync to EDDN. { LogLineHighlight("EDDiscovery and EDMarketConnector should not both sync to EDDN. Stop EDMC or uncheck 'send to EDDN' in settings tab!"); } } if (!EDDOptions.Instance.NoLoad) // here in this thread, we do a refresh of history. { LogLine("Reading travel history"); DoRefreshHistory(new RefreshWorkerArgs { CurrentCommander = EDCommander.CurrentCmdrID }); // kick the background refresh worker thread into action } if (PendingClose) { return; } if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if enabled { SystemClassEDSM.DetermineIfFullEDSMSyncRequired(syncstate); // ask EDSM and EDDB if they want to do a Full sync.. EliteDangerousCore.EDDB.SystemClassEDDB.DetermineIfEDDBSyncRequired(syncstate); if (syncstate.perform_eddb_sync || syncstate.perform_edsm_fullsync) { string databases = (syncstate.perform_edsm_fullsync && syncstate.perform_eddb_sync) ? "EDSM and EDDB" : ((syncstate.perform_edsm_fullsync) ? "EDSM" : "EDDB"); LogLine("Full synchronisation to the " + databases + " databases required." + Environment.NewLine + "This will take a while, up to 15 minutes, please be patient." + Environment.NewLine + "Please continue running ED Discovery until refresh is complete."); } } else { LogLine("Synchronisation to EDSM and EDDB disabled. Use Settings panel to reenable"); } InvokeAsyncOnUiThread(() => OnInitialisationComplete?.Invoke()); }
// returns no of updates + inserts, not no of items processed. Protect yourself against bad json private static long DoParseEDSMUpdateSystemsReader(JsonTextReader jr, bool[] grididallowed, ref DateTime maxdate, Func <bool> cancelRequested, Action <int, string> reportProgress, bool useCache = true, bool useTempSystems = false) { Dictionary <long, SystemClassBase> systemsByEdsmId = useCache ? GetEdsmSystemsLite() : new Dictionary <long, SystemClassBase>(); int count = 0; int updatecount = 0; int insertcount = 0; Random rnd = new Random(); string sysnamesTableName = useTempSystems ? "SystemNames_temp" : "SystemNames"; string edsmsysTableName = useTempSystems ? "EdsmSystems_temp" : "EdsmSystems"; Stopwatch sw = Stopwatch.StartNew(); const int BlockSize = 10000; while (!cancelRequested()) { bool jr_eof = false; List <EDSMDumpSystem> objs = new List <EDSMDumpSystem>(BlockSize); while (!cancelRequested()) { if (jr.Read()) { if (jr.TokenType == JsonToken.StartObject) { objs.Add(EDSMDumpSystem.Deserialize(jr)); if (objs.Count >= BlockSize) { break; } } } else { jr_eof = true; break; } } IEnumerator <EDSMDumpSystem> jo_enum = objs.GetEnumerator(); bool jo_enum_finished = false; while (!jo_enum_finished && !cancelRequested()) { int blkcount = 0; int oldinsertcnt = insertcount; int oldupdatecnt = updatecount; int oldcount = count; using (SQLExtTransactionLock <SQLiteConnectionSystem> tl = new SQLExtTransactionLock <SQLiteConnectionSystem>()) { tl.OpenWriter(); using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: SQLLiteExtensions.SQLExtConnection.AccessMode.Writer)) { using (DbTransaction txn = cn.BeginTransaction()) { DbCommand updateNameCmd = null; DbCommand updateSysCmd = null; DbCommand insertNameCmd = null; DbCommand insertSysCmd = null; DbCommand selectSysCmd = null; DbCommand selectNameCmd = null; try { updateNameCmd = cn.CreateCommand("UPDATE SystemNames SET Name=@Name WHERE EdsmId=@EdsmId", txn); updateNameCmd.AddParameter("@Name", DbType.String); updateNameCmd.AddParameter("@EdsmId", DbType.Int64); updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET X=@X, Y=@Y, Z=@Z, UpdateTimestamp=@UpdateTimestamp, VersionTimestamp=@VersionTimestamp, GridId=@GridId, RandomId=@RandomId WHERE EdsmId=@EdsmId", txn); updateSysCmd.AddParameter("@X", DbType.Int64); updateSysCmd.AddParameter("@Y", DbType.Int64); updateSysCmd.AddParameter("@Z", DbType.Int64); updateSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64); updateSysCmd.AddParameter("@VersionTimestamp", DbType.Int64); updateSysCmd.AddParameter("@GridId", DbType.Int64); updateSysCmd.AddParameter("@RandomId", DbType.Int64); updateSysCmd.AddParameter("@EdsmId", DbType.Int64); insertNameCmd = cn.CreateCommand("INSERT INTO " + sysnamesTableName + " (Name, EdsmId) VALUES (@Name, @EdsmId)", txn); insertNameCmd.AddParameter("@Name", DbType.String); insertNameCmd.AddParameter("@EdsmId", DbType.Int64); insertSysCmd = cn.CreateCommand("INSERT INTO " + edsmsysTableName + " (EdsmId, X, Y, Z, CreateTimestamp, UpdateTimestamp, VersionTimestamp, GridId, RandomId) VALUES (@EdsmId, @X, @Y, @Z, @CreateTimestamp, @UpdateTimestamp, @VersionTimestamp, @GridId, @RandomId)", txn); insertSysCmd.AddParameter("@EdsmId", DbType.Int64); insertSysCmd.AddParameter("@X", DbType.Int64); insertSysCmd.AddParameter("@Y", DbType.Int64); insertSysCmd.AddParameter("@Z", DbType.Int64); insertSysCmd.AddParameter("@CreateTimestamp", DbType.Int64); insertSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64); insertSysCmd.AddParameter("@VersionTimestamp", DbType.Int64); insertSysCmd.AddParameter("@GridId", DbType.Int64); insertSysCmd.AddParameter("@RandomId", DbType.Int64); selectSysCmd = cn.CreateCommand("SELECT Id, X, Y, Z, GridId, RandomId FROM EdsmSystems WHERE EdsmId=@EdsmId"); selectSysCmd.AddParameter("@EdsmId", DbType.Int64); selectNameCmd = cn.CreateCommand("SELECT Name FROM SystemNames WHERE EdsmId = @EdsmId"); selectNameCmd.AddParameter("@EdsmId", DbType.Int64); while (!cancelRequested()) { if (!jo_enum.MoveNext()) { reportProgress(-1, $"Syncing EDSM systems: {count:N0} processed, {insertcount:N0} new systems, {updatecount:N0} updated systems"); txn.Commit(); if (jr_eof) { System.Diagnostics.Debug.WriteLine("Maximum date was " + maxdate.ToString()); System.Diagnostics.Debug.WriteLine($"Import took {sw.ElapsedMilliseconds}ms"); return(updatecount + insertcount); } jo_enum_finished = true; break; } else if (SQLiteConnectionSystem.IsReadWaiting) { if (blkcount < objs.Count * 3 / 4) // Let the reader barge in if we've processed less than 3/4 of the items { // Reset the counts, roll back the transaction, and let the reader through... insertcount = oldinsertcnt; updatecount = oldupdatecnt; count = oldcount; jo_enum.Reset(); txn.Rollback(); break; } } EDSMDumpSystem jo = jo_enum.Current; EDSMDumpSystemCoords coords = jo.coords; if (coords != null) { DateTime updatedate = DateTime.SpecifyKind(jo.date, DateTimeKind.Utc); if (updatedate > maxdate) // even if we reject it due to grid id, keep last date up to date { maxdate = updatedate; } double x = coords.x; double z = coords.z; int gridid = GridId.Id(x, z); if (grididallowed[gridid]) // if grid allows it to be added.. { //System.Diagnostics.Debug.WriteLine("Accept due to gridid " + gridid); double y = coords.y; long edsmid = jo.id; string name = jo.name; int randomid = rnd.Next(0, 99); SystemClassBase dbsys = null; if (!useTempSystems) { if (useCache && systemsByEdsmId.ContainsKey(edsmid)) { dbsys = systemsByEdsmId[edsmid]; } if (!useCache) { selectSysCmd.Parameters["@EdsmId"].Value = edsmid; using (DbDataReader reader = selectSysCmd.ExecuteReader()) { if (reader.Read()) { dbsys = new SystemClassBase { ID = (long)reader["id"], EDSMID = edsmid }; if (System.DBNull.Value == reader["x"]) { dbsys.X = double.NaN; dbsys.Y = double.NaN; dbsys.Z = double.NaN; } else { dbsys.X = ((double)(long)reader["X"]) / SystemClassDB.XYZScalar; dbsys.Y = ((double)(long)reader["Y"]) / SystemClassDB.XYZScalar; dbsys.Z = ((double)(long)reader["Z"]) / SystemClassDB.XYZScalar; } dbsys.EDSMID = edsmid; dbsys.GridID = reader["GridId"] == DBNull.Value ? 0 : (int)((long)reader["GridId"]); dbsys.RandomID = reader["RandomId"] == DBNull.Value ? 0 : (int)((long)reader["RandomId"]); } } if (dbsys != null) { selectNameCmd.Parameters["@EdsmId"].Value = edsmid; using (DbDataReader reader = selectNameCmd.ExecuteReader()) { if (reader.Read()) { dbsys.Name = (string)reader["Name"]; } } } } } if (dbsys != null) { // see if EDSM data changed.. if (!dbsys.Name.Equals(name)) { updateNameCmd.Parameters["@Name"].Value = name; updateNameCmd.Parameters["@EdsmId"].Value = edsmid; updateNameCmd.ExecuteNonQuery(); } if (Math.Abs(dbsys.X - x) > 0.01 || Math.Abs(dbsys.Y - y) > 0.01 || Math.Abs(dbsys.Z - z) > 0.01 || dbsys.GridID != gridid) // position changed { updateSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar); updateSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; updateSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; updateSysCmd.Parameters["@GridId"].Value = gridid; updateSysCmd.Parameters["@RandomId"].Value = randomid; updateSysCmd.Parameters["@EdsmId"].Value = edsmid; updateSysCmd.ExecuteNonQuery(); updatecount++; } } else // not in database.. { insertNameCmd.Parameters["@Name"].Value = name; insertNameCmd.Parameters["@EdsmId"].Value = edsmid; insertNameCmd.ExecuteNonQuery(); insertSysCmd.Parameters["@EdsmId"].Value = edsmid; insertSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar); insertSysCmd.Parameters["@CreateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds; insertSysCmd.Parameters["@GridId"].Value = gridid; insertSysCmd.Parameters["@RandomId"].Value = randomid; insertSysCmd.ExecuteNonQuery(); insertcount++; } } else { //System.Diagnostics.Debug.WriteLine("Reject due to gridid " + gridid); } } else { System.Diagnostics.Debug.WriteLine("Reject due to coords "); } count++; blkcount++; } // WHILE END } finally { if (updateNameCmd != null) { updateNameCmd.Dispose(); } if (updateSysCmd != null) { updateSysCmd.Dispose(); } if (insertNameCmd != null) { insertNameCmd.Dispose(); } if (insertSysCmd != null) { insertSysCmd.Dispose(); } if (selectSysCmd != null) { selectSysCmd.Dispose(); } } } } } } } if (cancelRequested()) { throw new OperationCanceledException(); } return(updatecount + insertcount); }
private void DoPerformSync() // in Background worker { resyncEDSMEDDBRequestedFlag = 1; // sync is happening, stop any async requests.. Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Perform sync"); try { bool[] grids = new bool[GridId.MaxGridID]; foreach (int i in GridId.FromString(EDDConfig.Instance.EDSMGridIDs)) { grids[i] = true; } ReportProgress(-1, ""); syncstate.ClearCounters(); if (syncstate.perform_edsm_fullsync || syncstate.perform_eddb_sync) { if (syncstate.perform_edsm_fullsync && !PendingClose) { // Download new systems try { syncstate.edsm_fullsync_count = SystemClassEDSM.PerformEDSMFullSync(grids, () => PendingClose, ReportProgress, LogLine, LogLineHighlight); syncstate.perform_edsm_fullsync = false; } catch (Exception ex) { LogLineHighlight("GetAllEDSMSystems exception:" + ex.Message); } } if (!PendingClose) { SQLiteConnectionSystem.CreateSystemsTableIndexes(); // again check indexes.. sometimes SQL does not create them due to schema change try { syncstate.eddb_sync_count = EliteDangerousCore.EDDB.SystemClassEDDB.PerformEDDBFullSync(() => PendingClose, ReportProgress, LogLine, LogLineHighlight); syncstate.perform_eddb_sync = false; } catch (Exception ex) { LogLineHighlight("GetEDDBUpdate exception: " + ex.Message); } } } if (!PendingClose) { SQLiteConnectionSystem.CreateSystemsTableIndexes(); // again check indexes.. sometimes SQL does not create them due to schema change syncstate.edsm_updatesync_count = EliteDangerousCore.EDSM.SystemClassEDSM.PerformEDSMUpdateSync(grids, () => PendingClose, ReportProgress, LogLine, LogLineHighlight); } ReportProgress(-1, ""); } catch (OperationCanceledException) { // Swallow Operation Cancelled exceptions } catch (Exception ex) { LogLineHighlight("Check Systems exception: " + ex.Message + Environment.NewLine + "Trace: " + ex.StackTrace); } InvokeAsyncOnUiThread(() => PerformSyncCompleted()); }
public static void RemoveHiddenSystems(string json) // protected against bad json { try { JsonTextReader jr = new JsonTextReader(new StringReader(json)); bool jr_eof = false; while (!jr_eof) { using (SQLiteConnectionSystem cn2 = new SQLiteConnectionSystem(mode: SQLLiteExtensions.SQLExtConnection.AccessMode.Writer)) // open the db { using (DbTransaction txn = cn2.BeginTransaction()) { DbCommand infoinscmd = null; DbCommand infodelcmd = null; DbCommand namedelcmd = null; try { infoinscmd = cn2.CreateCommand("INSERT OR IGNORE INTO SystemAliases (name, id_edsm, id_edsm_mergedto) VALUES (@name, @id_edsm, @id_edsm_mergedto)", txn); infoinscmd.AddParameter("@name", DbType.String); infoinscmd.AddParameter("@id_edsm", DbType.Int64); infoinscmd.AddParameter("@id_edsm_mergedto", DbType.Int64); infodelcmd = cn2.CreateCommand("DELETE FROM EdsmSystems WHERE EdsmId=@EdsmId", txn); infodelcmd.AddParameter("@EdsmId", DbType.Int64); namedelcmd = cn2.CreateCommand("DELETE FROM SystemNames WHERE EdsmId=@EdsmId", txn); namedelcmd.AddParameter("@EdsmId", DbType.Int64); while (!SQLiteConnectionSystem.IsReadWaiting) { if (!jr.Read()) { jr_eof = true; break; } if (jr.TokenType == JsonToken.StartObject) { JObject jo = JObject.Load(jr); long edsmid = (long)jo["id"]; string name = (string)jo["system"]; string action = (string)jo["action"]; long mergedto = 0; if (jo["mergedTo"] != null) { mergedto = (long)jo["mergedTo"]; } Console.Write("Remove " + edsmid); infodelcmd.Parameters["@EdsmId"].Value = edsmid; infodelcmd.ExecuteNonQuery(); namedelcmd.Parameters["@EdsmId"].Value = edsmid; namedelcmd.ExecuteNonQuery(); if (mergedto > 0) { infoinscmd.Parameters["@name"].Value = name; infoinscmd.Parameters["@id_edsm"].Value = edsmid; infoinscmd.Parameters["@id_edsm_mergedto"].Value = mergedto; infoinscmd.ExecuteNonQuery(); } } } txn.Commit(); } finally { if (infoinscmd != null) { infoinscmd.Dispose(); } if (infodelcmd != null) { infodelcmd.Dispose(); } if (namedelcmd != null) { namedelcmd.Dispose(); } } } } } } catch (Exception e) { System.Diagnostics.Debug.WriteLine("SysClassEDSM Exception " + e.ToString()); } }
static public DateTime GetLastEDDBDownloadTime() { string timestr = SQLiteConnectionSystem.GetSettingString("EDDBSystemsTime", "0"); return(new DateTime(Convert.ToInt64(timestr), DateTimeKind.Utc)); }
static public void SetLastEDSMRecordTimeUTC(DateTime time) { SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", time.ToString(CultureInfo.InvariantCulture)); System.Diagnostics.Debug.WriteLine("Last EDSM record " + time.ToString()); }
static public void ForceEDDBFullUpdate() { SQLiteConnectionSystem.PutSettingString("EDDBSystemsTime", "0"); }
public static HistoryList LoadHistory(EDJournalClass journalmonitor, Func <bool> cancelRequested, Action <int, string> reportProgress, string NetLogPath = null, bool ForceNetLogReload = false, bool ForceJournalReload = false, int CurrentCommander = Int32.MinValue, bool Keepuievents = true, int fullhistoryloaddaylimit = 0 ) { HistoryList hist = new HistoryList(); if (CurrentCommander >= 0) { journalmonitor.ParseJournalFiles(() => cancelRequested(), (p, s) => reportProgress(p, s), forceReload: ForceJournalReload); // Parse files stop monitor.. if (NetLogPath != null) { string errstr = null; NetLogClass.ParseFiles(NetLogPath, out errstr, EliteConfigInstance.InstanceConfig.DefaultMapColour, () => cancelRequested(), (p, s) => reportProgress(p, s), ForceNetLogReload, currentcmdrid: CurrentCommander); } } Trace.WriteLine(BaseUtils.AppTicks.TickCount100 + " Files read "); reportProgress(-1, "Reading Database"); List <JournalEntry> jlist; if (fullhistoryloaddaylimit > 0) { jlist = JournalEntry.GetAll(CurrentCommander, ids: JournalEntry.EssentialEvents, allidsafter: DateTime.UtcNow.Subtract(new TimeSpan(fullhistoryloaddaylimit, 0, 0, 0)) ).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList(); } else { jlist = JournalEntry.GetAll(CurrentCommander).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList(); } Trace.WriteLine(BaseUtils.AppTicks.TickCount100 + " Database read " + jlist.Count); List <Tuple <JournalEntry, HistoryEntry> > jlistUpdated = new List <Tuple <JournalEntry, HistoryEntry> >(); using (SQLiteConnectionSystem conn = new SQLiteConnectionSystem()) { HistoryEntry prev = null; JournalEntry jprev = null; reportProgress(-1, "Creating History"); foreach (JournalEntry je in jlist) { if (MergeEntries(jprev, je)) // if we merge.. we may have updated info, so reprint. { //jprev.FillInformation(out prev.EventSummary, out prev.EventDescription, out prev.EventDetailedInfo); // need to keep this up to date.. continue; } if (je.IsUIEvent && !Keepuievents) // filter out any UI events { //System.Diagnostics.Debug.WriteLine("**** Filter out " + je.EventTypeStr + " on " + je.EventTimeLocal.ToString()); continue; } bool journalupdate = false; HistoryEntry he = HistoryEntry.FromJournalEntry(je, prev, out journalupdate, conn); prev = he; jprev = je; hist.historylist.Add(he); if (journalupdate) { jlistUpdated.Add(new Tuple <JournalEntry, HistoryEntry>(je, he)); Debug.WriteLine("Queued update requested {0} {1}", he.System.EDSMID, he.System.Name); } } } if (jlistUpdated.Count > 0) { reportProgress(-1, "Updating journal entries"); using (SQLiteConnectionUser conn = new SQLiteConnectionUser(utc: true)) { using (DbTransaction txn = conn.BeginTransaction()) { foreach (Tuple <JournalEntry, HistoryEntry> jehe in jlistUpdated) { JournalEntry je = jehe.Item1; HistoryEntry he = jehe.Item2; double dist = (je is JournalFSDJump) ? (je as JournalFSDJump).JumpDist : 0; bool updatecoord = (je is JournalLocOrJump) ? (!(je as JournalLocOrJump).HasCoordinate && he.System.HasCoordinate) : false; Debug.WriteLine("Push update {0} {1} to JE {2} HE {3}", he.System.EDSMID, he.System.Name, je.Id, he.Indexno); JournalEntry.UpdateEDSMIDPosJump(je.Id, he.System, updatecoord, dist, conn, txn); } txn.Commit(); } } } // now database has been updated due to initial fill, now fill in stuff which needs the user database hist.CommanderId = CurrentCommander; reportProgress(-1, "Updating user statistics"); hist.ProcessUserHistoryListEntries(h => h.ToList()); // here, we update the DBs in HistoryEntry and any global DBs in historylist reportProgress(-1, "Done"); return(hist); }
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, out bool journalupdate, SQLiteConnectionSystem conn = null) { ISystem isys = prev == null ? new SystemClass("Unknown") : prev.System; int indexno = prev == null ? 1 : prev.Indexno + 1; journalupdate = false; if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump) { JournalLocOrJump jl = je as JournalLocOrJump; ISystem newsys; if (jl != null && jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it { newsys = new SystemClass(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z) { EDSMID = jl.EdsmID < 0 ? 0 : jl.EdsmID, // pass across the EDSMID for the lazy load process. Faction = jl.Faction, Government = jl.EDGovernment, PrimaryEconomy = jl.EDEconomy, Security = jl.EDSecurity, Population = jl.Population ?? 0, State = jl.EDState, Allegiance = jl.EDAllegiance, UpdateDate = jl.EventTimeUTC, status = SystemStatusEnum.EDDiscovery, SystemAddress = jl.SystemAddress, }; // If it was a new system, pass the coords back to the StartJump if (prev != null && prev.journalEntry is JournalStartJump) { prev.System = newsys; // give the previous startjump our system.. } } else { // NOTE Rob: 09-JAN-2018 I've removed the Jumpstart looking up a system by name since they were using up lots of lookup time during history reading. // This is used for pre 2.2 systems without co-ords, which now should be limited. // JumpStart still gets the system when the FSD loc is processed, see above. // Jumpstart was also screwing about with the EDSM ID fill in which was broken. This is now working again. // Default one newsys = new SystemClass(jl.StarSystem); newsys.EDSMID = je.EdsmID; ISystem s = SystemCache.FindSystem(newsys, conn); // has no co-ord, did we find it? if (s != null) // found a system.. { if (jl != null && jl.HasCoordinate) // if journal Loc, and journal has a star position, use that instead of EDSM.. { s.X = Math.Round(jl.StarPos.X * 32.0) / 32.0; s.Y = Math.Round(jl.StarPos.Y * 32.0) / 32.0; s.Z = Math.Round(jl.StarPos.Z * 32.0) / 32.0; } //Debug.WriteLine("HistoryList found system {0} {1}", s.id_edsm, s.name); newsys = s; if (jl != null && je.EdsmID <= 0 && newsys.EDSMID > 0) // only update on a JL.. { journalupdate = true; Debug.WriteLine("HE EDSM ID update requested {0} {1}", newsys.EDSMID, newsys.Name); } } else { newsys.EDSMID = -1; // mark as checked but not found } } JournalFSDJump jfsd = je as JournalFSDJump; if (jfsd != null) { if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord { jfsd.JumpDist = isys.Distance(newsys); // fill it out here if (jfsd.JumpDist > 0) { journalupdate = true; Debug.WriteLine("Je Jump distance update(3) requested {0} {1} {2}", newsys.EDSMID, newsys.Name, jfsd.JumpDist); } } } isys = newsys; } HistoryEntry he = new HistoryEntry { Indexno = indexno, journalEntry = je, System = isys, EntryStatus = HistoryEntryStatus.Update(prev?.EntryStatus, je, isys.Name) }; he.TravelStatus = HistoryTravelStatus.Update(prev?.TravelStatus, prev, he); // need a real he so can't do that as part of the constructor. return(he); }
// I think we rework so we have a SyncBackgroundworker, and a refresh background worker. // loading of history is done on refresh one.. private void BackgroundWorkerThread() { readyForInitialLoad.WaitOne(); // wait for shown in form // check first and download items StarScan.LoadBodyDesignationMap(); Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Check systems"); ReportSyncProgress(""); bool checkGithub = EDDOptions.Instance.CheckGithubFiles; if (checkGithub) // not normall in debug, due to git hub chokeing { // Async load of maps in another thread DownloadMaps(() => PendingClose); // and Expedition data DownloadExpeditions(() => PendingClose); // and Exploration data DownloadExploration(() => PendingClose); } if (!EDDOptions.Instance.NoSystemsLoad) { // New Galmap load - it was not doing a refresh if EDSM sync kept on happening. Now has its own timer DateTime galmaptime = SQLiteConnectionSystem.GetSettingDate("EDSMGalMapLast", DateTime.MinValue); // Latest time from RW file. if (DateTime.Now.Subtract(galmaptime).TotalDays > 14 || !galacticMapping.GalMapFilePresent()) // Over 14 days do a sync from EDSM for galmap { LogLine("Get galactic mapping from EDSM.".Tx(this, "EDSM")); if (galacticMapping.DownloadFromEDSM()) { SQLiteConnectionSystem.PutSettingDate("EDSMGalMapLast", DateTime.UtcNow); } } Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Check systems complete"); } galacticMapping.ParseData(); // at this point, gal map data has been uploaded - get it into memory SystemCache.AddToAutoCompleteList(galacticMapping.GetGMONames()); SystemNoteClass.GetAllSystemNotes(); LogLine("Loaded Notes, Bookmarks and Galactic mapping.".Tx(this, "LN")); if (PendingClose) { return; } if (EliteDangerousCore.EDDN.EDDNClass.CheckforEDMC()) // EDMC is running { if (EDCommander.Current.SyncToEddn) // Both EDD and EDMC should not sync to EDDN. { LogLineHighlight("EDDiscovery and EDMarketConnector should not both sync to EDDN. Stop EDMC or uncheck 'send to EDDN' in settings tab!".Tx(this, "EDMC")); } } if (!EDDOptions.Instance.NoLoad) // here in this thread, we do a refresh of history. { LogLine("Reading travel history".Tx(this, "RTH")); if (EDDOptions.Instance.Commander != null) { EDCommander switchto = EDCommander.GetCommander(EDDOptions.Instance.Commander); if (switchto != null) { EDCommander.CurrentCmdrID = switchto.Nr; } } DoRefreshHistory(new RefreshWorkerArgs { CurrentCommander = EDCommander.CurrentCmdrID }); // kick the background refresh worker thread into action } if (PendingClose) { return; } CheckForSync(); // see if any EDSM/EDDB sync is needed if (PendingClose) { return; } // Now stay in loop services stuff backgroundRefreshWorker = new Thread(BackgroundHistoryRefreshWorkerThread) { Name = "Background Refresh Worker", IsBackground = true }; backgroundRefreshWorker.Start(); // start the refresh worker, another thread which does subsequenct (not the primary one) refresh work in the background.. try { if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if no system off, and EDSM download on { DoPerformSync(); // this is done after the initial history load.. } while (!PendingClose) { int wh = WaitHandle.WaitAny(new WaitHandle[] { closeRequested, resyncRequestedEvent }); if (PendingClose) { break; } switch (wh) { case 0: // Close Requested break; case 1: // Resync Requested if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if no system off, and EDSM download on { DoPerformSync(); } break; } } } catch (OperationCanceledException) { } backgroundRefreshWorker.Join(); // Now we have been ordered to close down, so go thru the process closeRequested.WaitOne(); OnBgSafeClose?.Invoke(); ReadyForFinalClose = true; InvokeAsyncOnUiThread(() => { OnFinalClose?.Invoke(); }); }
public static bool PerformEDSMFullSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError) { string rwsystime = SQLiteConnectionSystem.GetSettingString("EDSMLastSystems", "2000-01-01 00:00:00"); // Latest time from RW file. DateTime edsmdate; if (!DateTime.TryParse(rwsystime, CultureInfo.InvariantCulture, DateTimeStyles.None, out edsmdate)) { edsmdate = new DateTime(2000, 1, 1); } long updates = 0; // Delete all old systems SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", "2010-01-01 00:00:00"); SQLiteConnectionSystem.PutSettingString("EDDBSystemsTime", "0"); EDSMClass edsm = new EDSMClass(); logLine("Get hidden systems from EDSM and remove from database"); RemoveHiddenSystems(); if (cancelRequested()) { return(false); } logLine("Download systems file from EDSM."); string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json"); logLine("Resyncing all downloaded EDSM systems with local database." + Environment.NewLine + "This will take a while."); bool newfile; bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) => { SQLiteConnectionSystem.CreateTempSystemsTable(); string rwsysfiletime = "2014-01-01 00:00:00"; bool outoforder = false; using (var reader = new StreamReader(s)) updates = ParseEDSMUpdateSystemsStream(reader, ref rwsysfiletime, ref outoforder, true, cancelRequested, reportProgress, useCache: false, useTempSystems: true); if (!cancelRequested()) // abort, without saving time, to make it do it again { SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", rwsysfiletime); logLine("Replacing old systems table with new systems table and re-indexing - please wait"); reportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait"); SQLiteConnectionSystem.ReplaceSystemsTable(); SQLiteConnectionSystem.PutSettingBool("EDSMSystemsOutOfOrder", outoforder); reportProgress(-1, ""); #if !DEBUG File.Delete(edsmsystems); #endif } else { throw new OperationCanceledException(); } }); if (!success) { logLine("Failed to download EDSM system file from server, will check next time"); return(false); } // Stop if requested if (cancelRequested()) { return(false); } logLine("Local database updated with EDSM data, " + updates + " systems updated."); GC.Collect(); return(updates > 0); }