static public long ParseEDDBUpdateSystems(string filename, Action <string> logline)
{
StreamReader sr = new StreamReader(filename); // read directly from file..
if (sr == null)
{
return(0);
}
string line;
int updated = 0;
int inserted = 0;
while (!sr.EndOfStream)
{
using (SQLiteTxnLockED <SQLiteConnectionSystem> tl = new SQLiteTxnLockED <SQLiteConnectionSystem>())
{
tl.OpenWriter();
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: EDDbAccessMode.Writer)) // open the db
{
DbCommand selectCmd = null;
DbCommand insertCmd = null;
DbCommand updateCmd = null;
DbCommand updateSysCmd = null;
using (DbTransaction txn = cn.BeginTransaction())
{
try
{
selectCmd = cn.CreateCommand("SELECT EddbId, Population, EddbUpdatedAt FROM EddbSystems WHERE EdsmId = @EdsmId LIMIT 1", txn); // 1 return matching ID
selectCmd.AddParameter("@Edsmid", DbType.Int64);
insertCmd = cn.CreateCommand("INSERT INTO EddbSystems (EdsmId, EddbId, Name, Faction, Population, GovernmentId, AllegianceId, State, Security, PrimaryEconomyId, NeedsPermit, EddbUpdatedAt) " +
"VALUES (@EdsmId, @EddbId, @Name, @Faction, @Population, @GovernmentId, @AllegianceId, @State, @Security, @PrimaryEconomyid, @NeedsPermit, @EddbUpdatedAt)", txn);
insertCmd.AddParameter("@EdsmId", DbType.Int64);
insertCmd.AddParameter("@EddbId", DbType.Int64);
insertCmd.AddParameter("@Name", DbType.String);
insertCmd.AddParameter("@Faction", DbType.String);
insertCmd.AddParameter("@Population", DbType.Int64);
insertCmd.AddParameter("@GovernmentId", DbType.Int64);
insertCmd.AddParameter("@AllegianceId", DbType.Int64);
insertCmd.AddParameter("@State", DbType.Int64);
insertCmd.AddParameter("@Security", DbType.Int64);
insertCmd.AddParameter("@PrimaryEconomyId", DbType.Int64);
insertCmd.AddParameter("@NeedsPermit", DbType.Int64);
insertCmd.AddParameter("@EddbUpdatedAt", DbType.Int64);
updateCmd = cn.CreateCommand("UPDATE EddbSystems SET EddbId=@EddbId, Name=@Name, Faction=@Faction, Population=@Population, GovernmentId=@GovernmentId, AllegianceId=@AllegianceId, State=@State, Security=@Security, PrimaryEconomyId=@PrimaryEconomyId, NeedsPermit=@NeedsPermit, EddbUpdatedAt=@EddbUpdatedAt WHERE EdsmId=@EdsmId", txn);
updateCmd.AddParameter("@EdsmId", DbType.Int64);
updateCmd.AddParameter("@EddbId", DbType.Int64);
updateCmd.AddParameter("@Name", DbType.String);
updateCmd.AddParameter("@Faction", DbType.String);
updateCmd.AddParameter("@Population", DbType.Int64);
updateCmd.AddParameter("@GovernmentId", DbType.Int64);
updateCmd.AddParameter("@AllegianceId", DbType.Int64);
updateCmd.AddParameter("@State", DbType.Int64);
updateCmd.AddParameter("@Security", DbType.Int64);
updateCmd.AddParameter("@PrimaryEconomyId", DbType.Int64);
updateCmd.AddParameter("@NeedsPermit", DbType.Int64);
updateCmd.AddParameter("@EddbUpdatedAt", DbType.Int64);
updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET EddbId=@EddbId WHERE EdsmId=@EdsmId");
updateSysCmd.AddParameter("@EdsmId", DbType.Int64);
updateSysCmd.AddParameter("@EddbId", DbType.Int64);
int c = 0;
int hasinfo = 0;
int lasttc = Environment.TickCount;
while (!SQLiteConnectionSystem.IsReadWaiting)
{
line = sr.ReadLine();
if (line == null) // End of stream
{
break;
}
{
JObject jo = JObject.Parse(line);
ISystem system = SystemClassDB.FromJson(jo, SystemInfoSource.EDDB);
if (system.HasEDDBInformation) // screen out for speed any EDDB data with empty interesting fields
{
hasinfo++;
selectCmd.Parameters["@EdsmId"].Value = system.id_edsm; // EDDB carries EDSM ID, so find entry in dB
//DEBUGif ( c > 30000 ) Console.WriteLine("EDDB ID " + system.id_eddb + " EDSM ID " + system.id_edsm + " " + system.name + " Late info system");
long updated_at = 0;
long population = 0;
long eddbid = 0;
using (DbDataReader reader1 = selectCmd.ExecuteReader()) // if found (if not, we ignore EDDB system)
{
if (reader1.Read()) // its there.. check its got the right stuff in it.
{
eddbid = (long)reader1["EddbId"];
updated_at = (long)reader1["EddbUpdatedAt"];
population = (long)reader1["Population"];
}
}
updateSysCmd.Parameters["@EdsmId"].Value = system.id_edsm;
updateSysCmd.Parameters["@EddbId"].Value = system.id_eddb;
updateSysCmd.ExecuteNonQuery();
if (eddbid != 0)
{
if (updated_at != system.eddb_updated_at || population != system.population)
{
updateCmd.Parameters["@EddbId"].Value = system.id_eddb;
updateCmd.Parameters["@Name"].Value = system.name;
updateCmd.Parameters["@Faction"].Value = system.faction;
updateCmd.Parameters["@Population"].Value = system.population;
updateCmd.Parameters["@GovernmentId"].Value = system.government;
updateCmd.Parameters["@AllegianceId"].Value = system.allegiance;
updateCmd.Parameters["@State"].Value = system.state;
updateCmd.Parameters["@Security"].Value = system.security;
updateCmd.Parameters["@PrimaryEconomyId"].Value = system.primary_economy;
updateCmd.Parameters["@NeedsPermit"].Value = system.needs_permit;
updateCmd.Parameters["@EddbUpdatedAt"].Value = system.eddb_updated_at;
updateCmd.Parameters["@EdsmId"].Value = system.id_edsm;
updateCmd.ExecuteNonQuery();
updated++;
}
}
else
{
insertCmd.Parameters["@EdsmId"].Value = system.id_edsm;
insertCmd.Parameters["@EddbId"].Value = system.id_eddb;
insertCmd.Parameters["@Name"].Value = system.name;
insertCmd.Parameters["@Faction"].Value = system.faction;
insertCmd.Parameters["@Population"].Value = system.population;
insertCmd.Parameters["@GovernmentId"].Value = system.government;
insertCmd.Parameters["@AllegianceId"].Value = system.allegiance;
insertCmd.Parameters["@State"].Value = system.state;
insertCmd.Parameters["@Security"].Value = system.security;
insertCmd.Parameters["@PrimaryEconomyId"].Value = system.primary_economy;
insertCmd.Parameters["@NeedsPermit"].Value = system.needs_permit;
insertCmd.Parameters["@EddbUpdatedAt"].Value = system.eddb_updated_at;
insertCmd.ExecuteNonQuery();
inserted++;
}
}
else
{
//Console.WriteLine("EDDB ID " + system.id_eddb + " EDSM ID " + system.id_edsm + " " + system.name + " No info reject");
}
if (++c % 10000 == 0)
{
Console.WriteLine("EDDB Count " + c + " Delta " + (Environment.TickCount - lasttc) + " info " + hasinfo + " update " + updated + " new " + inserted);
lasttc = Environment.TickCount;
}
}
}
txn.Commit();
}
catch
{
ExtendedControls.MessageBoxTheme.Show("There is a problem using the EDDB systems file." + Environment.NewLine +
"Please perform a manual EDDB sync (see Admin menu) next time you run the program ", "EDDB Sync Error");
break;
}
finally
{
if (selectCmd != null)
{
selectCmd.Dispose();
}
if (updateCmd != null)
{
updateCmd.Dispose();
}
if (insertCmd != null)
{
insertCmd.Dispose();
}
}
}
}
}
}
return(updated + inserted);
}
private static long DoParseEDSMUpdateSystemsReader(JsonTextReader jr, ref string date, ref bool outoforder, Func <bool> cancelRequested, Action <int, string> reportProgress, bool useCache = true, bool useTempSystems = false)
{
DateTime maxdate;
if (!DateTime.TryParse(date, CultureInfo.InvariantCulture, DateTimeStyles.None, out maxdate))
{
maxdate = new DateTime(2010, 1, 1);
}
Dictionary <long, SystemClassBase> systemsByEdsmId = useCache ? GetEdsmSystemsLite() : new Dictionary <long, SystemClassBase>();
int count = 0;
int updatecount = 0;
int insertcount = 0;
Random rnd = new Random();
int[] histogramsystems = new int[50000];
string sysnamesTableName = useTempSystems ? "SystemNames_temp" : "SystemNames";
string edsmsysTableName = useTempSystems ? "EdsmSystems_temp" : "EdsmSystems";
Stopwatch sw = Stopwatch.StartNew();
const int BlockSize = 10000;
while (!cancelRequested())
{
bool jr_eof = false;
List <EDSMDumpSystem> objs = new List <EDSMDumpSystem>(BlockSize);
while (!cancelRequested())
{
if (jr.Read())
{
if (jr.TokenType == JsonToken.StartObject)
{
objs.Add(EDSMDumpSystem.Deserialize(jr));
if (objs.Count >= BlockSize)
{
break;
}
}
}
else
{
jr_eof = true;
break;
}
}
IEnumerator <EDSMDumpSystem> jo_enum = objs.GetEnumerator();
bool jo_enum_finished = false;
while (!jo_enum_finished && !cancelRequested())
{
int blkcount = 0;
int oldinsertcnt = insertcount;
int oldupdatecnt = updatecount;
int oldcount = count;
using (SQLiteTxnLockED <SQLiteConnectionSystem> tl = new SQLiteTxnLockED <SQLiteConnectionSystem>())
{
tl.OpenWriter();
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: EDDbAccessMode.Writer))
{
using (DbTransaction txn = cn.BeginTransaction())
{
DbCommand updateNameCmd = null;
DbCommand updateSysCmd = null;
DbCommand insertNameCmd = null;
DbCommand insertSysCmd = null;
DbCommand selectSysCmd = null;
DbCommand selectNameCmd = null;
try
{
updateNameCmd = cn.CreateCommand("UPDATE SystemNames SET Name=@Name WHERE EdsmId=@EdsmId", txn);
updateNameCmd.AddParameter("@Name", DbType.String);
updateNameCmd.AddParameter("@EdsmId", DbType.Int64);
updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET X=@X, Y=@Y, Z=@Z, UpdateTimestamp=@UpdateTimestamp, VersionTimestamp=@VersionTimestamp, GridId=@GridId, RandomId=@RandomId WHERE EdsmId=@EdsmId", txn);
updateSysCmd.AddParameter("@X", DbType.Int64);
updateSysCmd.AddParameter("@Y", DbType.Int64);
updateSysCmd.AddParameter("@Z", DbType.Int64);
updateSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64);
updateSysCmd.AddParameter("@VersionTimestamp", DbType.Int64);
updateSysCmd.AddParameter("@GridId", DbType.Int64);
updateSysCmd.AddParameter("@RandomId", DbType.Int64);
updateSysCmd.AddParameter("@EdsmId", DbType.Int64);
insertNameCmd = cn.CreateCommand("INSERT INTO " + sysnamesTableName + " (Name, EdsmId) VALUES (@Name, @EdsmId)", txn);
insertNameCmd.AddParameter("@Name", DbType.String);
insertNameCmd.AddParameter("@EdsmId", DbType.Int64);
insertSysCmd = cn.CreateCommand("INSERT INTO " + edsmsysTableName + " (EdsmId, X, Y, Z, CreateTimestamp, UpdateTimestamp, VersionTimestamp, GridId, RandomId) VALUES (@EdsmId, @X, @Y, @Z, @CreateTimestamp, @UpdateTimestamp, @VersionTimestamp, @GridId, @RandomId)", txn);
insertSysCmd.AddParameter("@EdsmId", DbType.Int64);
insertSysCmd.AddParameter("@X", DbType.Int64);
insertSysCmd.AddParameter("@Y", DbType.Int64);
insertSysCmd.AddParameter("@Z", DbType.Int64);
insertSysCmd.AddParameter("@CreateTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@VersionTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@GridId", DbType.Int64);
insertSysCmd.AddParameter("@RandomId", DbType.Int64);
selectSysCmd = cn.CreateCommand("SELECT Id, X, Y, Z, GridId, RandomId FROM EdsmSystems WHERE EdsmId=@EdsmId");
selectSysCmd.AddParameter("@EdsmId", DbType.Int64);
selectNameCmd = cn.CreateCommand("SELECT Name FROM SystemNames WHERE EdsmId = @EdsmId");
selectNameCmd.AddParameter("@EdsmId", DbType.Int64);
while (!cancelRequested())
{
if (!jo_enum.MoveNext())
{
reportProgress(-1, $"Syncing EDSM systems: {count:N0} processed, {insertcount:N0} new systems, {updatecount:N0} updated systems");
txn.Commit();
if (jr_eof)
{
date = maxdate.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
Console.WriteLine($"Import took {sw.ElapsedMilliseconds}ms");
for (int id = 0; id < histogramsystems.Length; id++)
{
if (histogramsystems[id] != 0)
{
Console.WriteLine("Id " + id + " count " + histogramsystems[id]);
}
}
return(updatecount + insertcount);
}
jo_enum_finished = true;
break;
}
else if (SQLiteConnectionSystem.IsReadWaiting)
{
if (blkcount < objs.Count * 3 / 4) // Let the reader barge in if we've processed less than 3/4 of the items
{
// Reset the counts, roll back the transaction, and let the reader through...
insertcount = oldinsertcnt;
updatecount = oldupdatecnt;
count = oldcount;
jo_enum.Reset();
txn.Rollback();
break;
}
}
EDSMDumpSystem jo = jo_enum.Current;
EDSMDumpSystemCoords coords = jo.coords;
if (coords != null)
{
double x = coords.x;
double y = coords.y;
double z = coords.z;
long edsmid = jo.id;
string name = jo.name;
int gridid = GridId.Id(x, z);
int randomid = rnd.Next(0, 99);
DateTime updatedate = jo.date;
histogramsystems[gridid]++;
if (updatedate > maxdate)
{
maxdate = updatedate;
}
else if (updatedate < maxdate - TimeSpan.FromHours(1))
{
outoforder = true;
}
SystemClassBase dbsys = null;
if (!useTempSystems)
{
if (useCache && systemsByEdsmId.ContainsKey(edsmid))
{
dbsys = systemsByEdsmId[edsmid];
}
if (!useCache)
{
selectSysCmd.Parameters["@EdsmId"].Value = edsmid;
using (DbDataReader reader = selectSysCmd.ExecuteReader())
{
if (reader.Read())
{
dbsys = new SystemClassBase
{
id = (long)reader["id"],
id_edsm = edsmid
};
if (System.DBNull.Value == reader["x"])
{
dbsys.x = double.NaN;
dbsys.y = double.NaN;
dbsys.z = double.NaN;
}
else
{
dbsys.x = ((double)(long)reader["X"]) / SystemClassDB.XYZScalar;
dbsys.y = ((double)(long)reader["Y"]) / SystemClassDB.XYZScalar;
dbsys.z = ((double)(long)reader["Z"]) / SystemClassDB.XYZScalar;
}
dbsys.id_edsm = edsmid;
dbsys.gridid = reader["GridId"] == DBNull.Value ? 0 : (int)((long)reader["GridId"]);
dbsys.randomid = reader["RandomId"] == DBNull.Value ? 0 : (int)((long)reader["RandomId"]);
}
}
if (dbsys != null)
{
selectNameCmd.Parameters["@EdsmId"].Value = edsmid;
using (DbDataReader reader = selectNameCmd.ExecuteReader())
{
if (reader.Read())
{
dbsys.name = (string)reader["Name"];
}
}
}
}
}
if (dbsys != null)
{
// see if EDSM data changed..
if (!dbsys.name.Equals(name))
{
updateNameCmd.Parameters["@Name"].Value = name;
updateNameCmd.Parameters["@EdsmId"].Value = edsmid;
updateNameCmd.ExecuteNonQuery();
}
if (Math.Abs(dbsys.x - x) > 0.01 ||
Math.Abs(dbsys.y - y) > 0.01 ||
Math.Abs(dbsys.z - z) > 0.01 ||
dbsys.gridid != gridid) // position changed
{
updateSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
updateSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
updateSysCmd.Parameters["@GridId"].Value = gridid;
updateSysCmd.Parameters["@RandomId"].Value = randomid;
updateSysCmd.Parameters["@EdsmId"].Value = edsmid;
updateSysCmd.ExecuteNonQuery();
updatecount++;
}
}
else // not in database..
{
insertNameCmd.Parameters["@Name"].Value = name;
insertNameCmd.Parameters["@EdsmId"].Value = edsmid;
insertNameCmd.ExecuteNonQuery();
insertSysCmd.Parameters["@EdsmId"].Value = edsmid;
insertSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@CreateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@GridId"].Value = gridid;
insertSysCmd.Parameters["@RandomId"].Value = randomid;
insertSysCmd.ExecuteNonQuery();
insertcount++;
}
}
count++;
blkcount++;
}
}
finally
{
if (updateNameCmd != null)
{
updateNameCmd.Dispose();
}
if (updateSysCmd != null)
{
updateSysCmd.Dispose();
}
if (insertNameCmd != null)
{
insertNameCmd.Dispose();
}
if (insertSysCmd != null)
{
insertSysCmd.Dispose();
}
if (selectSysCmd != null)
{
selectSysCmd.Dispose();
}
}
}
}
}
}
}
if (cancelRequested())
{
throw new OperationCanceledException();
}
return(updatecount + insertcount);
}
private void NamedStars() // background thread.. run after Update. Thread never deletes, only adds to its own structures
{
try // just in case someone tears us down..
{
int lylimit = (int)(_starlimitly / _lastcamera.LastZoom);
lylimit = Math.Max(lylimit, 1);
int sqlylimit = lylimit * lylimit; // in squared distance limit from viewpoint
StarGrid.TransFormInfo ti = new StarGrid.TransFormInfo(_resmat, _znear, _glControl.Width, _glControl.Height, sqlylimit, _lastcamera.LastCameraPos);
SortedDictionary <float, StarGrid.InViewInfo> inviewlist = new SortedDictionary <float, StarGrid.InViewInfo>(new DuplicateKeyComparer <float>()); // who's in view, sorted by distance
//Stopwatch sw1 = new Stopwatch();sw1.Start(); Tools.LogToFile(String.Format("starnamesest Estimate at {0} len {1}", ti.campos, sqlylimit));
_stargrids.GetSystemsInView(ref inviewlist, 2000.0F, ti); // consider all grids under 2k from current pos.
//Tools.LogToFile(String.Format("starnamesest Took {0} in view {1}", sw1.ElapsedMilliseconds, inviewlist.Count));
float textscalingw = Math.Min(_starnamemaxly, Math.Max(_starnamesizely / _lastcamera.LastZoom, _starnameminly)); // per char
float starsize = Math.Min(Math.Max(_lastcamera.LastZoom / 10F, 1.0F), 20F); // Normal stars are at 1F.
//Console.WriteLine("Per char {0} h {1} sc {2} ", textscalingw, textscalingh, starsize);
foreach (StarNames s in _starnamesbackground.Values) // all items not processed
{
s.updatedinview = false; // only items remaining will clear this
}
_needrepaint = false; // assume nothing changes
int painted = 0;
//string res = ""; // used to view whats added/removed/draw..
foreach (StarGrid.InViewInfo inview in inviewlist.Values) // for all in viewport, sorted by distance from camera position
{
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem())
{
StarNames sys = null;
bool draw = false;
if (_starnamesbackground.ContainsKey(inview.position)) // if already there..
{
sys = _starnamesbackground[inview.position];
sys.updatedinview = true;
draw = (_discson && sys.paintstar == null && sys.newstar == null) ||
(_nameson && ((sys.nametexture == null && sys.newnametexture == null)));
painted++;
}
else if (painted < maxstars)
{
ISystem sc = _formmap.FindSystem(inview.position, cn); // with the open connection, find this star..
if (sc != null) // if can't be resolved, ignore
{
sys = new StarNames(sc, inview.position); // we keep position in here using same floats as inview so it will match
_starnamesbackground.Add(inview.position, sys); // add to our database
_starnamestoforeground.Add(sys); // send to foreground for adding
draw = true;
painted++;
//Tools.LogToFile(String.Format("starnamesest: push {0}", sys.Pos));
//res += "N";
}
}
else
{
break; // no point doing any more.. got our fill of items
}
if (draw)
{
_needrepaint = true; // changed a item.. needs a repaint
if (_nameson)
{
float width = textscalingw * sys.name.Length;
Bitmap map = DatasetBuilder.DrawString(sys.name, _namecolour, _starfont);
sys.newnametexture = TexturedQuadData.FromBitmap(map,
new PointData(sys.pos.X, sys.pos.Y, sys.pos.Z),
_lastcamera.Rotation,
width, textscalingw * 4.0F, _startextoffset + width / 2, 0);
sys.rotation = _lastcamera.Rotation; // remember when we were when we draw it
sys.zoom = _lastcamera.LastZoom;
}
if (_discson)
{
sys.newstar = new PointData(sys.pos.X, sys.pos.Y, sys.pos.Z, starsize, inview.AsColor);
}
}
}
}
foreach (StarNames s in _starnamesbackground.Values) // only items above will remain.
{
//if (s.inview != s.updatedinview) res += (s.updatedinview) ? "+" : "-";
_needrepaint = _needrepaint || (s.inview != s.updatedinview); // set if we change our mind on any of the items
s.inview = s.updatedinview; // copy flag over, causes foreground to start removing them
}
//if ( _needrepaint) Console.WriteLine(String.Format("starnamesest in view {0} limit {1} repaint {2} {3}", inviewlist.Count, lylimit, _needrepaint, res));
//Tools.LogToFile(String.Format("starnamesest added all delta {0} topaint {1}", sw1.ElapsedMilliseconds, painted));
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception watcher: " + ex.Message);
System.Diagnostics.Trace.WriteLine("Trace: " + ex.StackTrace);
}
_formmap.BeginInvoke((System.Windows.Forms.MethodInvoker) delegate // kick the UI thread to process.
{
_formmap.ChangeNamedStars();
});
}
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, out bool journalupdate, SQLiteConnectionSystem conn = null, EDCommander cmdr = null)
{
ISystem isys = prev == null ? new SystemClass("Unknown") : prev.System;
int indexno = prev == null ? 1 : prev.Indexno + 1;
int mapcolour = 0;
journalupdate = false;
bool starposfromedsm = false;
bool firstdiscover = false;
if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump)
{
JournalLocOrJump jl = je as JournalLocOrJump;
ISystem newsys;
if (jl != null && jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it
{
newsys = new SystemClass(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z)
{
EDSMID = jl.EdsmID < 0 ? 0 : jl.EdsmID, // pass across the EDSMID for the lazy load process.
Faction = jl.Faction,
Government = jl.EDGovernment,
PrimaryEconomy = jl.EDEconomy,
Security = jl.EDSecurity,
Population = jl.Population ?? 0,
State = jl.EDState,
Allegiance = jl.EDAllegiance,
UpdateDate = jl.EventTimeUTC,
status = SystemStatusEnum.EDDiscovery,
SystemAddress = jl.SystemAddress,
};
// If it was a new system, pass the coords back to the StartJump
if (prev != null && prev.journalEntry is JournalStartJump)
{
prev.System = newsys; // give the previous startjump our system..
}
}
else
{
// NOTE Rob: 09-JAN-2018 I've removed the Jumpstart looking up a system by name since they were using up lots of lookup time during history reading.
// This is used for pre 2.2 systems without co-ords, which now should be limited.
// JumpStart still gets the system when the FSD loc is processed, see above.
// Jumpstart was also screwing about with the EDSM ID fill in which was broken. This is now working again.
// Default one
newsys = new SystemClass(jl.StarSystem);
newsys.EDSMID = je.EdsmID;
ISystem s = SystemCache.FindSystem(newsys, conn); // has no co-ord, did we find it?
if (s != null) // found a system..
{
if (jl != null && jl.HasCoordinate) // if journal Loc, and journal has a star position, use that instead of EDSM..
{
s.X = Math.Round(jl.StarPos.X * 32.0) / 32.0;
s.Y = Math.Round(jl.StarPos.Y * 32.0) / 32.0;
s.Z = Math.Round(jl.StarPos.Z * 32.0) / 32.0;
}
//Debug.WriteLine("HistoryList found system {0} {1}", s.id_edsm, s.name);
newsys = s;
if (jl != null && je.EdsmID <= 0 && newsys.EDSMID > 0) // only update on a JL..
{
journalupdate = true;
Debug.WriteLine("HE EDSM ID update requested {0} {1}", newsys.EDSMID, newsys.Name);
}
}
else
{
newsys.EDSMID = -1; // mark as checked but not found
}
}
JournalFSDJump jfsd = je as JournalFSDJump;
if (jfsd != null)
{
if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord
{
jfsd.JumpDist = isys.Distance(newsys); // fill it out here
if (jfsd.JumpDist > 0)
{
journalupdate = true;
Debug.WriteLine("Je Jump distance update(3) requested {0} {1} {2}", newsys.EDSMID, newsys.Name, jfsd.JumpDist);
}
}
mapcolour = jfsd.MapColor;
}
isys = newsys;
starposfromedsm = (jl != null && jl.HasCoordinate) ? jl.StarPosFromEDSM : newsys.HasCoordinate;
firstdiscover = jl == null ? false : jl.EDSMFirstDiscover;
}
string summary, info, detailed;
je.FillInformation(out summary, out info, out detailed);
HistoryEntry he = new HistoryEntry
{
Indexno = indexno,
EntryType = je.EventTypeID,
Journalid = je.Id,
journalEntry = je,
System = isys,
EventTimeUTC = je.EventTimeUTC,
MapColour = mapcolour,
EdsmSync = je.SyncedEDSM,
EDDNSync = je.SyncedEDDN,
EGOSync = je.SyncedEGO,
StartMarker = je.StartMarker,
StopMarker = je.StopMarker,
EventSummary = summary,
EventDescription = info,
EventDetailedInfo = detailed,
IsStarPosFromEDSM = starposfromedsm,
IsEDSMFirstDiscover = firstdiscover,
Commander = cmdr ?? EDCommander.GetCommander(je.CommanderId)
};
// WORK out docked/landed state
if (prev != null)
{
if (prev.docked.HasValue) // copy docked..
{
he.docked = prev.docked;
}
if (prev.landed.HasValue)
{
he.landed = prev.landed;
}
if (prev.hyperspace.HasValue)
{
he.hyperspace = prev.hyperspace;
}
if (prev.marketId != null)
{
he.marketId = prev.marketId;
}
he.stationName = prev.stationName;
he.shiptype = prev.shiptype;
he.shipid = prev.shipid;
he.whereami = prev.whereami;
he.onCrewWithCaptain = prev.onCrewWithCaptain;
he.gamemode = prev.gamemode;
he.group = prev.group;
}
if (je.EventTypeID == JournalTypeEnum.Location)
{
JournalLocation jl = je as JournalLocation;
he.docked = jl.Docked;
he.landed = jl.Latitude.HasValue;
he.whereami = jl.Docked ? jl.StationName : jl.Body;
he.hyperspace = false;
}
else if (je.EventTypeID == JournalTypeEnum.Docked)
{
JournalDocked jl = je as JournalDocked;
he.docked = true;
he.whereami = jl.StationName;
he.stationName = jl.StationName;
he.marketId = jl.MarketID;
}
else if (je.EventTypeID == JournalTypeEnum.Undocked)
{
he.docked = false;
he.stationName = null;
he.marketId = null;
}
else if (je.EventTypeID == JournalTypeEnum.Touchdown)
{
he.landed = true;
}
else if (je.EventTypeID == JournalTypeEnum.Liftoff)
{
he.landed = !(je as JournalLiftoff).PlayerControlled;
}
else if (je.EventTypeID == JournalTypeEnum.SupercruiseEntry)
{
he.whereami = (je as JournalSupercruiseEntry).StarSystem;
he.hyperspace = true;
}
else if (je.EventTypeID == JournalTypeEnum.SupercruiseExit)
{
he.whereami = (je as JournalSupercruiseExit).Body;
he.hyperspace = false;
}
else if (je.EventTypeID == JournalTypeEnum.FSDJump)
{
he.whereami = (je as JournalFSDJump).StarSystem;
he.hyperspace = true;
}
else if (je.EventTypeID == JournalTypeEnum.StartJump)
{
he.hyperspace = true; // some of these are just to make sure, as FSDJump will also set it
}
else if (je.EventTypeID == JournalTypeEnum.LoadGame)
{
JournalLoadGame jl = je as JournalLoadGame;
he.onCrewWithCaptain = null; // can't be in a crew at this point
he.gamemode = jl.GameMode; // set game mode
he.group = jl.Group; // and group, may be empty
he.landed = jl.StartLanded;
he.hyperspace = false;
if (jl.Ship.IndexOf("buggy", StringComparison.InvariantCultureIgnoreCase) == -1) // load game with buggy, can't tell what ship we get back into, so ignore
{
he.shiptype = (je as JournalLoadGame).Ship;
he.shipid = (je as JournalLoadGame).ShipId;
}
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardBuy) // BUY does not have ship id, but the new entry will that is written later - journals 8.34
{
he.shiptype = (je as JournalShipyardBuy).ShipType;
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardNew)
{
he.shiptype = (je as JournalShipyardNew).ShipType;
he.shipid = (je as JournalShipyardNew).ShipId;
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardSwap)
{
he.shiptype = (je as JournalShipyardSwap).ShipType;
he.shipid = (je as JournalShipyardSwap).ShipId;
}
else if (je.EventTypeID == JournalTypeEnum.JoinACrew)
{
he.onCrewWithCaptain = (je as JournalJoinACrew).Captain;
}
else if (je.EventTypeID == JournalTypeEnum.QuitACrew)
{
he.onCrewWithCaptain = null;
}
if (prev != null && prev.travelling) // if we are travelling..
{
he.travelled_distance = prev.travelled_distance;
he.travelled_missingjump = prev.travelled_missingjump;
he.travelled_jumps = prev.travelled_jumps;
if (he.IsFSDJump && !he.MultiPlayer) // if jump, and not multiplayer..
{
double dist = ((JournalFSDJump)je).JumpDist;
if (dist <= 0)
{
he.travelled_missingjump++;
}
else
{
he.travelled_distance += dist;
he.travelled_jumps++;
}
}
he.travelled_seconds = prev.travelled_seconds;
TimeSpan diff = he.EventTimeUTC.Subtract(prev.EventTimeUTC);
if (he.EntryType != JournalTypeEnum.LoadGame && diff < new TimeSpan(2, 0, 0)) // time between last entry and load game is not real time
{
he.travelled_seconds += diff;
}
if (he.StopMarker || he.StartMarker)
{
//Debug.WriteLine("Travelling stop at " + he.Indexno);
he.travelling = false;
he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelled " + he.travelled_distance.ToStringInvariant("0.0") + " LY"
+ ", " + he.travelled_jumps + " jumps"
+ ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") +
", time " + he.travelled_seconds;
he.travelled_distance = 0;
he.travelled_seconds = new TimeSpan(0);
}
else
{
he.travelling = true;
if (he.IsFSDJump)
{
he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelling" +
" distance " + he.travelled_distance.ToString("0.0") + " LY"
+ ", " + he.travelled_jumps + " jumps"
+ ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") +
", time " + he.travelled_seconds;
}
}
}
if (he.StartMarker)
{
//Debug.WriteLine("Travelling start at " + he.Indexno);
he.travelling = true;
}
return(he);
}
// Protected against bad JSON
public int GetLogs(DateTime?starttimeutc, DateTime?endtimeutc, out List <JournalFSDJump> log, out DateTime logstarttime, out DateTime logendtime)
{
log = new List <JournalFSDJump>();
logstarttime = DateTime.MaxValue;
logendtime = DateTime.MinValue;
if (!ValidCredentials)
{
return(0);
}
string query = "get-logs?showId=1&apiKey=" + apiKey + "&commanderName=" + HttpUtility.UrlEncode(commanderName);
if (starttimeutc != null)
{
query += "&startDateTime=" + HttpUtility.UrlEncode(starttimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture));
}
if (endtimeutc != null)
{
query += "&endDateTime=" + HttpUtility.UrlEncode(endtimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture));
}
var response = RequestGet("api-logs-v1/" + query, handleException: true);
if (response.Error)
{
return(0);
}
var json = response.Body;
if (json == null)
{
return(0);
}
try
{
JObject msg = JObject.Parse(json);
int msgnr = msg["msgnum"].Int(0);
JArray logs = (JArray)msg["logs"];
if (logs != null)
{
string startdatestr = msg["startDateTime"].Value <string>();
string enddatestr = msg["endDateTime"].Value <string>();
if (startdatestr == null || !DateTime.TryParseExact(startdatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logstarttime))
{
logstarttime = DateTime.MaxValue;
}
if (enddatestr == null || !DateTime.TryParseExact(enddatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logendtime))
{
logendtime = DateTime.MinValue;
}
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem())
{
foreach (JObject jo in logs)
{
string name = jo["system"].Value <string>();
string ts = jo["date"].Value <string>();
long id = jo["systemId"].Value <long>();
bool firstdiscover = jo["firstDiscover"].Value <bool>();
DateTime etutc = DateTime.ParseExact(ts, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); // UTC time
ISystem sc = SystemClassDB.GetSystem(id, cn, SystemClassDB.SystemIDType.EdsmId, name: name);
if (sc == null)
{
if (DateTime.UtcNow.Subtract(etutc).TotalHours < 6) // Avoid running into the rate limit
{
sc = GetSystemsByName(name)?.FirstOrDefault(s => s.EDSMID == id);
}
if (sc == null)
{
sc = new SystemClass(name)
{
EDSMID = id
};
}
}
JournalFSDJump fsd = new JournalFSDJump(etutc, sc, EliteConfigInstance.InstanceConfig.DefaultMapColour, firstdiscover, (int)SyncFlags.EDSM);
log.Add(fsd);
}
}
}
return(msgnr);
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("Failed due to " + e.ToString());
return(499); // BAD JSON
}
}
// Called from EDDiscoveryController, in DoPerform, on back thread, to determine what sync to do..
public static long PerformEDSMFullSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
EDSMClass edsm = new EDSMClass();
LogLine("Get hidden systems from EDSM and remove from database");
RemoveHiddenSystems();
if (PendingClose())
{
return(updates);
}
LogLine("Download systems file from EDSM.");
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
LogLine("Resyncing all downloaded EDSM systems with System Database." + Environment.NewLine + "This will take a while.");
// string s = edsmsystems; bool success = true; // debug, comment out next two lines
// with the new downloader, the file is stored, then processed. If the user cancels, we will use the old file again. Once complete, the file is removed
ReportProgress(-1, "Downloading star database from EDSM");
bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDSMFullSystemsURL, edsmsystems, false, out bool newfile, (n, s) =>
{
SQLiteConnectionSystem.CreateTempSystemsTable();
DateTime maxdate = new DateTime(2000, 1, 1);
try
{
using (var reader = new StreamReader(s))
updates = ParseEDSMUpdateSystemsStream(reader, grididallow, ref maxdate, true, PendingClose, ReportProgress, useCache: false, useTempSystems: true);
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.3 Exception " + e.ToString());
}
if (!PendingClose()) // abort, without saving time, to make it do it again
{
LogLine("Replacing old systems table with new systems table and re-indexing - please wait");
ReportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait");
SQLiteConnectionSystem.ReplaceSystemsTable();
SetLastEDSMRecordTimeUTC(maxdate); // record the last record seen in time
ReportProgress(-1, "");
BaseUtils.FileHelpers.DeleteFileNoError(edsmsystems); // remove file - don't hold in storage
LogLine("System Database updated with EDSM data, " + updates + " systems updated.");
GC.Collect();
}
else
{
success = false;
ReportProgress(-1, "Operation Cancelled");
throw new OperationCanceledException();
}
}, cancelRequested: PendingClose);
if (!success)
{
ReportProgress(-1, "EDSM Failed to download correctly");
LogLine("Failed to download EDSM system file from server, will check next time");
}
return(updates);
}
// Time storers
static public void ForceEDSMFullUpdate()
{
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", "2010-01-01 00:00:00");
}
public int GetLogs(DateTime?starttimeutc, DateTime?endtimeutc, out List <HistoryEntry> log, out DateTime logstarttime, out DateTime logendtime)
{
log = new List <HistoryEntry>();
logstarttime = DateTime.MaxValue;
logendtime = DateTime.MinValue;
if (!IsApiKeySet)
{
return(0);
}
string query = "get-logs?showId=1&apiKey=" + apiKey + "&commanderName=" + HttpUtility.UrlEncode(commanderName);
if (starttimeutc != null)
{
query += "&startDateTime=" + HttpUtility.UrlEncode(starttimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture));
}
if (endtimeutc != null)
{
query += "&endDateTime=" + HttpUtility.UrlEncode(endtimeutc.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture));
}
var response = RequestGet("api-logs-v1/" + query, handleException: true);
if (response.Error)
{
return(0);
}
var json = response.Body;
if (json == null)
{
return(0);
}
JObject msg = JObject.Parse(json);
int msgnr = msg["msgnum"].Int(0);
JArray logs = (JArray)msg["logs"];
if (logs != null)
{
string startdatestr = msg["startDateTime"].Value <string>();
string enddatestr = msg["endDateTime"].Value <string>();
if (startdatestr == null || !DateTime.TryParseExact(startdatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logstarttime))
{
logstarttime = DateTime.MaxValue;
}
if (enddatestr == null || !DateTime.TryParseExact(enddatestr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out logendtime))
{
logendtime = DateTime.MinValue;
}
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem())
{
foreach (JObject jo in logs)
{
string name = jo["system"].Value <string>();
string ts = jo["date"].Value <string>();
long id = jo["systemId"].Value <long>();
bool firstdiscover = jo["firstDiscover"].Value <bool>();
DateTime etutc = DateTime.ParseExact(ts, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); // UTC time
ISystem sc = SystemClassDB.GetSystem(id, cn, SystemClassDB.SystemIDType.EdsmId);
if (sc == null)
{
sc = GetSystemsByName(name).FirstOrDefault(s => s.id_edsm == id);
if (sc == null)
{
sc = new SystemClass(name)
{
id_edsm = id
};
}
}
HistoryEntry he = HistoryEntry.MakeVSEntry(sc, etutc, EliteConfigInstance.InstanceConfig.DefaultMapColour, "", "", firstdiscover: firstdiscover); // FSD jump entry
log.Add(he);
}
}
}
return(msgnr);
}
public static HistoryList LoadHistory(EDJournalClass journalmonitor, Func <bool> cancelRequested, Action <int, string> reportProgress,
string NetLogPath = null,
bool ForceNetLogReload = false,
bool ForceJournalReload = false,
bool CheckEdsm = false,
int CurrentCommander = Int32.MinValue,
bool Keepuievents = true)
{
HistoryList hist = new HistoryList();
EDCommander cmdr = null;
if (CurrentCommander >= 0)
{
cmdr = EDCommander.GetCommander(CurrentCommander);
journalmonitor.ParseJournalFiles(() => cancelRequested(), (p, s) => reportProgress(p, s), forceReload: ForceJournalReload); // Parse files stop monitor..
if (NetLogPath != null)
{
string errstr = null;
NetLogClass.ParseFiles(NetLogPath, out errstr, EliteConfigInstance.InstanceConfig.DefaultMapColour, () => cancelRequested(), (p, s) => reportProgress(p, s), ForceNetLogReload, currentcmdrid: CurrentCommander);
}
}
reportProgress(-1, "Resolving systems");
List <JournalEntry> jlist = JournalEntry.GetAll(CurrentCommander).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList();
List <Tuple <JournalEntry, HistoryEntry> > jlistUpdated = new List <Tuple <JournalEntry, HistoryEntry> >();
using (SQLiteConnectionSystem conn = new SQLiteConnectionSystem())
{
HistoryEntry prev = null;
JournalEntry jprev = null;
foreach (JournalEntry je in jlist)
{
if (MergeEntries(jprev, je)) // if we merge.. we may have updated info, so reprint.
{
jprev.FillInformation(out prev.EventSummary, out prev.EventDescription, out prev.EventDetailedInfo); // need to keep this up to date..
continue;
}
if (je.IsUIEvent && !Keepuievents) // filter out any UI events
{
System.Diagnostics.Debug.WriteLine("**** Filter out " + je.EventTypeStr + " on " + je.EventTimeLocal.ToString());
continue;
}
bool journalupdate = false;
HistoryEntry he = HistoryEntry.FromJournalEntry(je, prev, CheckEdsm, out journalupdate, conn, cmdr);
prev = he;
jprev = je;
hist.historylist.Add(he);
if (journalupdate)
{
jlistUpdated.Add(new Tuple <JournalEntry, HistoryEntry>(je, he));
}
}
}
if (jlistUpdated.Count > 0)
{
reportProgress(-1, "Updating journal entries");
using (SQLiteConnectionUser conn = new SQLiteConnectionUser(utc: true))
{
using (DbTransaction txn = conn.BeginTransaction())
{
foreach (Tuple <JournalEntry, HistoryEntry> jehe in jlistUpdated)
{
JournalEntry je = jehe.Item1;
HistoryEntry he = jehe.Item2;
JournalFSDJump jfsd = je as JournalFSDJump;
if (jfsd != null)
{
JournalEntry.UpdateEDSMIDPosJump(jfsd.Id, he.System, !jfsd.HasCoordinate && he.System.HasCoordinate, jfsd.JumpDist, conn, txn);
}
}
txn.Commit();
}
}
}
// now database has been updated due to initial fill, now fill in stuff which needs the user database
hist.CommanderId = CurrentCommander;
hist.ProcessUserHistoryListEntries(h => h.ToList()); // here, we update the DBs in HistoryEntry and any global DBs in historylist
hist.SendEDSMStatusInfo(hist.GetLast, true);
return(hist);
}
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, bool checkedsm, out bool journalupdate, SQLiteConnectionSystem conn = null, EDCommander cmdr = null)
{
ISystem isys = prev == null ? new SystemClassDB("Unknown") : prev.System;
int indexno = prev == null ? 1 : prev.Indexno + 1;
int mapcolour = 0;
journalupdate = false;
bool starposfromedsm = false;
bool firstdiscover = false;
if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump)
{
JournalLocOrJump jl = je as JournalLocOrJump;
JournalFSDJump jfsd = je as JournalFSDJump;
ISystem newsys;
if (jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it
{
newsys = new SystemClassDB(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z);
newsys.id_edsm = jl.EdsmID < 0 ? 0 : jl.EdsmID; // pass across the EDSMID for the lazy load process.
if (jfsd != null && jfsd.JumpDist <= 0 && isys.HasCoordinate) // if we don't have a jump distance (pre 2.2) but the last sys does have pos, we can compute distance and update entry
{
jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here
journalupdate = true;
}
}
else
{ // Default one
newsys = new SystemClassDB(jl.StarSystem);
newsys.id_edsm = jl.EdsmID;
if (checkedsm) // see if we can find the right system
{
SystemClassDB s = SystemClassDB.FindEDSM(newsys, conn); // has no co-ord, did we find it?
if (s != null) // yes, use, and update the journal with the esdmid, and also the position if we have a co-ord
{ // so next time we don't have to do this again..
if (jl.HasCoordinate)
{
s.x = Math.Round(jl.StarPos.X * 32.0) / 32.0;
s.y = Math.Round(jl.StarPos.Y * 32.0) / 32.0;
s.z = Math.Round(jl.StarPos.Z * 32.0) / 32.0;
}
newsys = s;
if (jfsd != null && jfsd.JumpDist <= 0 && newsys.HasCoordinate && isys.HasCoordinate) // if we don't have a jump distance (pre 2.2) but the last sys does, we can compute
{
jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here. EDSM systems always have co-ords, but we should check anyway
journalupdate = true;
}
if (jl.EdsmID <= 0 && newsys.id_edsm > 0)
{
journalupdate = true;
}
}
}
}
if (jfsd != null)
{
if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord
{
jfsd.JumpDist = SystemClassDB.Distance(isys, newsys); // fill it out here
journalupdate = true;
}
mapcolour = jfsd.MapColor;
}
isys = newsys;
starposfromedsm = jl.HasCoordinate ? jl.StarPosFromEDSM : newsys.HasCoordinate;
firstdiscover = jl.EDSMFirstDiscover;
}
string summary, info, detailed;
je.FillInformation(out summary, out info, out detailed);
HistoryEntry he = new HistoryEntry
{
Indexno = indexno,
EntryType = je.EventTypeID,
Journalid = je.Id,
journalEntry = je,
System = isys,
EventTimeUTC = je.EventTimeUTC,
MapColour = mapcolour,
EdsmSync = je.SyncedEDSM,
EDDNSync = je.SyncedEDDN,
EGOSync = je.SyncedEGO,
StartMarker = je.StartMarker,
StopMarker = je.StopMarker,
EventSummary = summary,
EventDescription = info,
EventDetailedInfo = detailed,
IsStarPosFromEDSM = starposfromedsm,
IsEDSMFirstDiscover = firstdiscover,
Commander = cmdr ?? EDCommander.GetCommander(je.CommanderId)
};
// WORK out docked/landed state
if (prev != null)
{
if (prev.docked.HasValue) // copy docked..
{
he.docked = prev.docked;
}
if (prev.landed.HasValue)
{
he.landed = prev.landed;
}
he.shiptype = prev.shiptype;
he.shipid = prev.shipid;
he.whereami = prev.whereami;
he.onCrewWithCaptain = prev.onCrewWithCaptain;
he.gamemode = prev.gamemode;
he.group = prev.group;
}
if (je.EventTypeID == JournalTypeEnum.Location)
{
JournalLocation jl = je as JournalLocation;
he.docked = jl.Docked;
he.whereami = jl.Docked ? jl.StationName : jl.Body;
}
else if (je.EventTypeID == JournalTypeEnum.Docked)
{
JournalDocked jl = je as JournalDocked;
he.docked = true;
he.whereami = jl.StationName;
}
else if (je.EventTypeID == JournalTypeEnum.Undocked)
{
he.docked = false;
}
else if (je.EventTypeID == JournalTypeEnum.Touchdown)
{
he.landed = true;
}
else if (je.EventTypeID == JournalTypeEnum.Liftoff)
{
he.landed = false;
}
else if (je.EventTypeID == JournalTypeEnum.SupercruiseEntry)
{
he.whereami = (je as JournalSupercruiseEntry).StarSystem;
}
else if (je.EventTypeID == JournalTypeEnum.SupercruiseExit)
{
he.whereami = (je as JournalSupercruiseExit).Body;
}
else if (je.EventTypeID == JournalTypeEnum.FSDJump)
{
he.whereami = (je as JournalFSDJump).StarSystem;
}
else if (je.EventTypeID == JournalTypeEnum.LoadGame)
{
JournalLoadGame jl = je as JournalLoadGame;
he.onCrewWithCaptain = null; // can't be in a crew at this point
he.gamemode = jl.GameMode; // set game mode
he.group = jl.Group; // and group, may be empty
he.landed = jl.StartLanded;
if (jl.Ship.IndexOf("buggy", StringComparison.InvariantCultureIgnoreCase) == -1) // load game with buggy, can't tell what ship we get back into, so ignore
{
he.shiptype = (je as JournalLoadGame).Ship;
he.shipid = (je as JournalLoadGame).ShipId;
}
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardBuy) // BUY does not have ship id, but the new entry will that is written later - journals 8.34
{
he.shiptype = (je as JournalShipyardBuy).ShipType;
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardNew)
{
he.shiptype = (je as JournalShipyardNew).ShipType;
he.shipid = (je as JournalShipyardNew).ShipId;
}
else if (je.EventTypeID == JournalTypeEnum.ShipyardSwap)
{
he.shiptype = (je as JournalShipyardSwap).ShipType;
he.shipid = (je as JournalShipyardSwap).ShipId;
}
else if (je.EventTypeID == JournalTypeEnum.JoinACrew)
{
he.onCrewWithCaptain = (je as JournalJoinACrew).Captain;
}
else if (je.EventTypeID == JournalTypeEnum.QuitACrew)
{
he.onCrewWithCaptain = null;
}
if (prev != null && prev.travelling) // if we are travelling..
{
he.travelled_distance = prev.travelled_distance;
he.travelled_missingjump = prev.travelled_missingjump;
he.travelled_jumps = prev.travelled_jumps;
if (he.IsFSDJump && !he.MultiPlayer) // if jump, and not multiplayer..
{
double dist = ((JournalFSDJump)je).JumpDist;
if (dist <= 0)
{
he.travelled_missingjump++;
}
else
{
he.travelled_distance += dist;
he.travelled_jumps++;
}
}
he.travelled_seconds = prev.travelled_seconds;
TimeSpan diff = he.EventTimeUTC.Subtract(prev.EventTimeUTC);
if (he.EntryType != JournalTypeEnum.LoadGame && diff < new TimeSpan(2, 0, 0)) // time between last entry and load game is not real time
{
he.travelled_seconds += diff;
}
if (he.StopMarker || he.StartMarker)
{
//Debug.WriteLine("Travelling stop at " + he.Indexno);
he.travelling = false;
he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelled " + he.travelled_distance.ToStringInvariant("0.0") + " LY"
+ ", " + he.travelled_jumps + " jumps"
+ ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") +
", time " + he.travelled_seconds;
he.travelled_distance = 0;
he.travelled_seconds = new TimeSpan(0);
}
else
{
he.travelling = true;
if (he.IsFSDJump)
{
he.EventDetailedInfo += ((he.EventDetailedInfo.Length > 0) ? Environment.NewLine : "") + "Travelling" +
" distance " + he.travelled_distance.ToString("0.0") + " LY"
+ ", " + he.travelled_jumps + " jumps"
+ ((he.travelled_missingjump > 0) ? ", " + he.travelled_missingjump + " unknown distance jumps" : "") +
", time " + he.travelled_seconds;
}
}
}
if (he.StartMarker)
{
//Debug.WriteLine("Travelling start at " + he.Indexno);
he.travelling = true;
}
return(he);
}
internal long GetNewSystems(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine)
{
string lstsyst;
DateTime lstsystdate;
// First system in EDSM is from 2015-05-01 00:39:40
DateTime gammadate = new DateTime(2015, 5, 1, 0, 0, 0, DateTimeKind.Utc);
bool outoforder = SQLiteConnectionSystem.GetSettingBool("EDSMSystemsOutOfOrder", true);
if (SystemClassDB.IsSystemsTableEmpty())
{
lstsystdate = gammadate;
}
else
{
// Get the most recent modify time returned from EDSM
DateTime lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast();
lstsystdate = lastmod - TimeSpan.FromSeconds(1);
if (lstsystdate < gammadate)
{
lstsystdate = gammadate;
}
}
lstsyst = lstsystdate.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
Console.WriteLine("EDSM Check date: " + lstsyst);
long updates = 0;
while (lstsystdate < DateTime.UtcNow)
{
if (cancelRequested())
{
return(updates);
}
DateTime enddate = lstsystdate + TimeSpan.FromHours(12);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
logLine($"Downloading systems from {lstsystdate.ToLocalTime().ToString()} to {enddate.ToLocalTime().ToString()}");
reportProgress(-1, "Requesting systems from EDSM");
string json = null;
try
{
json = RequestSystems(lstsystdate, enddate);
}
catch (WebException ex)
{
reportProgress(-1, $"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
logLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
logLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
break;
}
catch (Exception ex)
{
reportProgress(-1, $"EDSM request failed");
logLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
break;
}
if (json == null)
{
reportProgress(-1, "EDSM request failed");
logLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
break;
}
updates += SystemClassEDSM.ParseEDSMUpdateSystemsString(json, ref lstsyst, ref outoforder, false, cancelRequested, reportProgress, false);
lstsystdate += TimeSpan.FromHours(12);
}
logLine($"System download complete");
return(updates);
}
public long UpdateSync(bool[] grididallow, Func <bool> PendingClose, Action <string> ReportProgress)
{
DateTime lastrecordtime = SQLiteConnectionSystem.GetLastEDSMRecordTimeUTC();
long updates = 0;
double fetchmult = 1;
while (lastrecordtime < DateTime.UtcNow.Subtract(new TimeSpan(0, 30, 0))) // stop at X mins before now, so we don't get in a condition
{ // where we do a set, the time moves to just before now,
// and we then do another set with minimum amount of hours
if (PendingClose())
{
return(updates);
}
if (updates == 0)
{
LogLine("Checking for updated EDSM systems (may take a few moments).");
}
EDSMClass edsm = new EDSMClass();
double hourstofetch = 3;
if (lastrecordtime < ED21date.AddHours(-48))
{
hourstofetch = 48;
}
else if (lastrecordtime < ED23date.AddHours(-12))
{
hourstofetch = 12;
}
else if (lastrecordtime < ED30date.AddHours(-6))
{
hourstofetch = 6;
}
DateTime enddate = lastrecordtime + TimeSpan.FromHours(hourstofetch * fetchmult);
if (enddate > DateTime.UtcNow)
{
enddate = DateTime.UtcNow;
}
LogLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine($"Downloading systems from UTC {lastrecordtime.ToUniversalTime().ToString()} to {enddate.ToUniversalTime().ToString()}");
string json = null;
BaseUtils.ResponseData response;
try
{
Stopwatch sw = new Stopwatch();
response = edsm.RequestSystemsData(lastrecordtime, enddate, timeout: 20000);
fetchmult = Math.Max(0.1, Math.Min(Math.Min(fetchmult * 1.1, 1.0), 5000.0 / sw.ElapsedMilliseconds));
}
catch (WebException ex)
{
ReportProgress($"EDSM request failed");
if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null && ex.Response is HttpWebResponse)
{
string status = ((HttpWebResponse)ex.Response).StatusDescription;
LogLine($"Download of EDSM systems from the server failed ({status}), will try next time program is run");
}
else
{
LogLine($"Download of EDSM systems from the server failed ({ex.Status.ToString()}), will try next time program is run");
}
return(updates);
}
catch (Exception ex)
{
ReportProgress($"EDSM request failed");
LogLine($"Download of EDSM systems from the server failed ({ex.Message}), will try next time program is run");
return(updates);
}
if (response.Error)
{
if ((int)response.StatusCode == 429)
{
LogLine($"EDSM rate limit hit - waiting 2 minutes");
for (int sec = 0; sec < 120; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
else
{
LogLine($"Download of EDSM systems from the server failed ({response.StatusCode.ToString()}), will try next time program is run");
return(updates);
}
}
json = response.Body;
if (json == null)
{
ReportProgress("EDSM request failed");
LogLine("Download of EDSM systems from the server failed (no data returned), will try next time program is run");
return(updates);
}
// debug File.WriteAllText(@"c:\code\json.txt", json);
DateTime prevrectime = lastrecordtime;
System.Diagnostics.Debug.WriteLine("Last record time {0} JSON size {1}", lastrecordtime.ToUniversalTime(), json.Length);
long updated = 0;
try
{
ReportProgress($"EDSM star database update from UTC " + lastrecordtime.ToUniversalTime().ToString());
updated = SystemsDB.ParseEDSMJSONString(json, grididallow, ref lastrecordtime, PendingClose, ReportProgress, "");
System.Diagnostics.Debug.WriteLine($".. Updated {updated} to {lastrecordtime.ToUniversalTime().ToString()}");
System.Diagnostics.Debug.WriteLine("Updated to time {0}", lastrecordtime.ToUniversalTime());
// if lastrecordtime did not change (=) or worse still, EDSM somehow moved the time back (unlikely)
if (lastrecordtime <= prevrectime)
{
lastrecordtime += TimeSpan.FromHours(12); // Lets move on manually so we don't get stuck
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM.2 Exception " + e.ToString());
ReportProgress("EDSM request failed");
LogLine("Processing EDSM systems download failed, will try next time program is run");
return(updates);
}
updates += updated;
SQLiteConnectionSystem.SetLastEDSMRecordTimeUTC(lastrecordtime); // keep on storing this in case next time we get an exception
int delay = 10; // Anthor's normal delay
int ratelimitlimit;
int ratelimitremain;
int ratelimitreset;
if (response.Headers != null &&
response.Headers["X-Rate-Limit-Limit"] != null &&
response.Headers["X-Rate-Limit-Remaining"] != null &&
response.Headers["X-Rate-Limit-Reset"] != null &&
Int32.TryParse(response.Headers["X-Rate-Limit-Limit"], out ratelimitlimit) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Remaining"], out ratelimitremain) &&
Int32.TryParse(response.Headers["X-Rate-Limit-Reset"], out ratelimitreset))
{
if (ratelimitremain < ratelimitlimit * 3 / 4) // lets keep at least X remaining for other purposes later..
{
delay = ratelimitreset / (ratelimitlimit - ratelimitremain); // slow down to its pace now.. example 878/(360-272) = 10 seconds per quota
}
else
{
delay = 0;
}
System.Diagnostics.Debug.WriteLine("EDSM Delay Parameters {0} {1} {2} => {3}s", ratelimitlimit, ratelimitremain, ratelimitreset, delay);
}
for (int sec = 0; sec < delay; sec++)
{
if (!PendingClose())
{
System.Threading.Thread.Sleep(1000);
}
}
}
return(updates);
}
private void DoPerformSync() // in Background worker
{
InvokeAsyncOnUiThread.Invoke(() => OnSyncStarting?.Invoke()); // tell listeners sync is starting
resyncEDSMEDDBRequestedFlag = 1; // sync is happening, stop any async requests..
// check for 102, and if not performing a full sync, upgrade it
SQLiteConnectionSystem.UpgradeSystemTableFrom102TypeDB(() => PendingClose, ReportSyncProgress, syncstate.perform_edsm_fullsync);
Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Perform EDSM/EDDB sync");
try
{
bool[] grids = new bool[GridId.MaxGridID];
foreach (int i in GridId.FromString(EDDConfig.Instance.EDSMGridIDs))
{
grids[i] = true;
}
syncstate.ClearCounters();
if (syncstate.perform_edsm_fullsync || syncstate.perform_eddb_edsmalias_sync)
{
if (syncstate.perform_edsm_fullsync && !PendingClose)
{
// Download new systems
try
{
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
ReportSyncProgress("Performing full download of EDSM Database from server");
bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDSMFullSystemsURL, edsmsystems, false, out bool newfile);
syncstate.perform_edsm_fullsync = false;
if (success)
{
syncstate.edsm_fullsync_count = SQLiteConnectionSystem.UpgradeSystemTableFromFile(edsmsystems, grids, () => PendingClose, ReportSyncProgress);
if (syncstate.edsm_fullsync_count < 0) // this should always update something, the table is replaced. If its not, its been cancelled
{
return;
}
BaseUtils.FileHelpers.DeleteFileNoError(edsmsystems); // remove file - don't hold in storage
}
}
catch (Exception ex)
{
LogLineHighlight("GetAllEDSMSystems exception:" + ex.Message);
}
}
if (!PendingClose)
{
try
{
EDSMClass edsm = new EDSMClass();
string jsonhidden = edsm.GetHiddenSystems();
if (jsonhidden != null)
{
SystemsDB.ParseAliasString(jsonhidden);
string eddbsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "eddbsystems.json");
bool success = BaseUtils.DownloadFile.HTTPDownloadFile(EliteConfigInstance.InstanceConfig.EDDBSystemsURL, eddbsystems, false, out bool newfile);
syncstate.perform_eddb_edsmalias_sync = false;
if (success)
{
syncstate.eddb_sync_count = SystemsDB.ParseEDDBJSONFile(eddbsystems, () => PendingClose);
if (syncstate.eddb_sync_count < 0) // on a cancel or error
{
return;
}
SQLiteConnectionSystem.SetLastEDDBDownloadTime();
BaseUtils.FileHelpers.DeleteFileNoError(eddbsystems); // remove file - don't hold in storage
}
}
}
catch (Exception ex)
{
LogLineHighlight("GetEDDBUpdate exception: " + ex.Message);
}
}
}
if (!PendingClose)
{
syncstate.edsm_updatesync_count = UpdateSync(grids, () => PendingClose, ReportSyncProgress);
}
}
catch (OperationCanceledException)
{
// Swallow Operation Cancelled exceptions
}
catch (Exception ex)
{
LogLineHighlight("Check Systems exception: " + ex.Message + Environment.NewLine + "Trace: " + ex.StackTrace);
}
InvokeAsyncOnUiThread(() => PerformSyncCompletedonUI());
}
public static void PerformSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError, SystemsSyncState state) // big check.. done in a thread.
{
reportProgress(-1, "");
state.performhistoryrefresh = false;
state.syncwasfirstrun = SystemClassDB.IsSystemsTableEmpty(); // remember if DB is empty
// Force a full sync if newest data is more than 14 days old
bool outoforder = SQLiteConnectionSystem.GetSettingBool("EDSMSystemsOutOfOrder", true);
DateTime lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast();
if (DateTime.UtcNow.Subtract(lastmod).TotalDays >= 14)
{
state.performedsmsync = true;
}
bool edsmoreddbsync = state.performedsmsync || state.performeddbsync; // remember if we are syncing
state.syncwaseddboredsm = edsmoreddbsync;
if (state.performedsmsync || state.performeddbsync)
{
if (state.performedsmsync && !cancelRequested())
{
// Download new systems
try
{
state.performhistoryrefresh |= PerformEDSMFullSync(cancelRequested, reportProgress, logLine, logError);
state.performedsmsync = false;
}
catch (Exception ex)
{
logError("GetAllEDSMSystems exception:" + ex.Message);
}
}
if (!cancelRequested())
{
logLine("Indexing systems table");
SQLiteConnectionSystem.CreateSystemsTableIndexes();
try
{
EliteDangerousCore.EDDB.SystemClassEDDB.PerformEDDBFullSync(cancelRequested, reportProgress, logLine, logError);
state.performeddbsync = false;
}
catch (Exception ex)
{
logError("GetEDDBUpdate exception: " + ex.Message);
}
state.performhistoryrefresh = true;
}
}
if (!cancelRequested())
{
logLine("Indexing systems table");
SQLiteConnectionSystem.CreateSystemsTableIndexes();
lastmod = outoforder ? SystemClassDB.GetLastSystemModifiedTime() : SystemClassDB.GetLastSystemModifiedTimeFast();
if (DateTime.UtcNow.Subtract(lastmod).TotalHours >= 1)
{
logLine("Checking for new EDSM systems (may take a few moments).");
EDSMClass edsm = new EDSMClass();
long updates = edsm.GetNewSystems(cancelRequested, reportProgress, logLine);
logLine($"EDSM updated {updates:N0} systems.");
state.performhistoryrefresh |= (updates > 0);
}
}
reportProgress(-1, "");
}
// Called from EDDiscoveryController, in DoPerform, on back thread, to determine what sync to do..
public static long PerformEDSMFullSync(bool[] grididallow, Func <bool> PendingClose, Action <int, string> ReportProgress, Action <string> LogLine, Action <string> LogLineHighlight)
{
long updates = 0;
EDSMClass edsm = new EDSMClass();
LogLine("Get hidden systems from EDSM and remove from database");
RemoveHiddenSystems();
if (PendingClose())
{
return(updates);
}
LogLine("Download systems file from EDSM.");
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
LogLine("Resyncing all downloaded EDSM systems with System Database." + Environment.NewLine + "This will take a while.");
// string s = edsmsystems; bool success = true; // debug, comment out next two lines
bool newfile;
bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) =>
{
SQLiteConnectionSystem.CreateTempSystemsTable();
DateTime maxdate = new DateTime(2000, 1, 1);
using (var reader = new StreamReader(s))
updates = ParseEDSMUpdateSystemsStream(reader, grididallow, ref maxdate, true, PendingClose, ReportProgress, useCache: false, useTempSystems: true);
if (!PendingClose()) // abort, without saving time, to make it do it again
{
LogLine("Replacing old systems table with new systems table and re-indexing - please wait");
ReportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait");
SQLiteConnectionSystem.ReplaceSystemsTable();
SetLastEDSMRecordTimeUTC(maxdate); // record the last record seen in time
ReportProgress(-1, "");
LogLine("System Database updated with EDSM data, " + updates + " systems updated.");
GC.Collect();
}
else
{
success = false;
throw new OperationCanceledException();
}
});
if (!success)
{
LogLine("Failed to download EDSM system file from server, will check next time");
}
return(updates);
}
// Called from Background Thread Worker at Init()
private void BackgroundInit()
{
StarScan.LoadBodyDesignationMap();
MaterialCommodityDB.SetUpInitialTable();
SQLiteConnectionSystem.CreateSystemsTableIndexes(); // just make sure they are there..
Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Check systems");
ReportProgress(-1, "");
if (!EDDOptions.Instance.NoSystemsLoad)
{
// Async load of maps in another thread
downloadMapsTask = DownloadMaps(this, () => PendingClose, LogLine, LogLineHighlight);
// Former CheckSystems, reworked to accomodate new switches..
// Check to see what sync refreshes we need
// New Galmap load - it was not doing a refresh if EDSM sync kept on happening. Now has its own timer
string rwgalmaptime = SQLiteConnectionSystem.GetSettingString("EDSMGalMapLast", "2000-01-01 00:00:00"); // Latest time from RW file.
DateTime galmaptime;
if (!DateTime.TryParse(rwgalmaptime, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out galmaptime))
{
galmaptime = new DateTime(2000, 1, 1);
}
if (DateTime.Now.Subtract(galmaptime).TotalDays > 14) // Over 14 days do a sync from EDSM for galmap
{
LogLine("Get galactic mapping from EDSM.");
galacticMapping.DownloadFromEDSM();
SQLiteConnectionSystem.PutSettingString("EDSMGalMapLast", DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss"));
}
Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Check systems complete");
}
galacticMapping.ParseData(); // at this point, gal map data has been uploaded - get it into memory
SystemClassDB.AddToAutoComplete(galacticMapping.GetGMONames());
SystemNoteClass.GetAllSystemNotes();
BookmarkClass.GetAllBookmarks();
LogLine("Loaded Notes, Bookmarks and Galactic mapping.");
ReportProgress(-1, "");
InvokeAsyncOnUiThread(() => OnInitialSyncComplete?.Invoke());
if (PendingClose)
{
return;
}
if (EliteDangerousCore.EDDN.EDDNClass.CheckforEDMC()) // EDMC is running
{
if (EDCommander.Current.SyncToEddn) // Both EDD and EDMC should not sync to EDDN.
{
LogLineHighlight("EDDiscovery and EDMarketConnector should not both sync to EDDN. Stop EDMC or uncheck 'send to EDDN' in settings tab!");
}
}
if (!EDDOptions.Instance.NoLoad) // here in this thread, we do a refresh of history.
{
LogLine("Reading travel history");
DoRefreshHistory(new RefreshWorkerArgs {
CurrentCommander = EDCommander.CurrentCmdrID
}); // kick the background refresh worker thread into action
}
if (PendingClose)
{
return;
}
if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if enabled
{
SystemClassEDSM.DetermineIfFullEDSMSyncRequired(syncstate); // ask EDSM and EDDB if they want to do a Full sync..
EliteDangerousCore.EDDB.SystemClassEDDB.DetermineIfEDDBSyncRequired(syncstate);
if (syncstate.perform_eddb_sync || syncstate.perform_edsm_fullsync)
{
string databases = (syncstate.perform_edsm_fullsync && syncstate.perform_eddb_sync) ? "EDSM and EDDB" : ((syncstate.perform_edsm_fullsync) ? "EDSM" : "EDDB");
LogLine("Full synchronisation to the " + databases + " databases required." + Environment.NewLine +
"This will take a while, up to 15 minutes, please be patient." + Environment.NewLine +
"Please continue running ED Discovery until refresh is complete.");
}
}
else
{
LogLine("Synchronisation to EDSM and EDDB disabled. Use Settings panel to reenable");
}
InvokeAsyncOnUiThread(() => OnInitialisationComplete?.Invoke());
}
// returns no of updates + inserts, not no of items processed. Protect yourself against bad json
private static long DoParseEDSMUpdateSystemsReader(JsonTextReader jr, bool[] grididallowed, ref DateTime maxdate, Func <bool> cancelRequested, Action <int, string> reportProgress, bool useCache = true, bool useTempSystems = false)
{
Dictionary <long, SystemClassBase> systemsByEdsmId = useCache ? GetEdsmSystemsLite() : new Dictionary <long, SystemClassBase>();
int count = 0;
int updatecount = 0;
int insertcount = 0;
Random rnd = new Random();
string sysnamesTableName = useTempSystems ? "SystemNames_temp" : "SystemNames";
string edsmsysTableName = useTempSystems ? "EdsmSystems_temp" : "EdsmSystems";
Stopwatch sw = Stopwatch.StartNew();
const int BlockSize = 10000;
while (!cancelRequested())
{
bool jr_eof = false;
List <EDSMDumpSystem> objs = new List <EDSMDumpSystem>(BlockSize);
while (!cancelRequested())
{
if (jr.Read())
{
if (jr.TokenType == JsonToken.StartObject)
{
objs.Add(EDSMDumpSystem.Deserialize(jr));
if (objs.Count >= BlockSize)
{
break;
}
}
}
else
{
jr_eof = true;
break;
}
}
IEnumerator <EDSMDumpSystem> jo_enum = objs.GetEnumerator();
bool jo_enum_finished = false;
while (!jo_enum_finished && !cancelRequested())
{
int blkcount = 0;
int oldinsertcnt = insertcount;
int oldupdatecnt = updatecount;
int oldcount = count;
using (SQLExtTransactionLock <SQLiteConnectionSystem> tl = new SQLExtTransactionLock <SQLiteConnectionSystem>())
{
tl.OpenWriter();
using (SQLiteConnectionSystem cn = new SQLiteConnectionSystem(mode: SQLLiteExtensions.SQLExtConnection.AccessMode.Writer))
{
using (DbTransaction txn = cn.BeginTransaction())
{
DbCommand updateNameCmd = null;
DbCommand updateSysCmd = null;
DbCommand insertNameCmd = null;
DbCommand insertSysCmd = null;
DbCommand selectSysCmd = null;
DbCommand selectNameCmd = null;
try
{
updateNameCmd = cn.CreateCommand("UPDATE SystemNames SET Name=@Name WHERE EdsmId=@EdsmId", txn);
updateNameCmd.AddParameter("@Name", DbType.String);
updateNameCmd.AddParameter("@EdsmId", DbType.Int64);
updateSysCmd = cn.CreateCommand("UPDATE EdsmSystems SET X=@X, Y=@Y, Z=@Z, UpdateTimestamp=@UpdateTimestamp, VersionTimestamp=@VersionTimestamp, GridId=@GridId, RandomId=@RandomId WHERE EdsmId=@EdsmId", txn);
updateSysCmd.AddParameter("@X", DbType.Int64);
updateSysCmd.AddParameter("@Y", DbType.Int64);
updateSysCmd.AddParameter("@Z", DbType.Int64);
updateSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64);
updateSysCmd.AddParameter("@VersionTimestamp", DbType.Int64);
updateSysCmd.AddParameter("@GridId", DbType.Int64);
updateSysCmd.AddParameter("@RandomId", DbType.Int64);
updateSysCmd.AddParameter("@EdsmId", DbType.Int64);
insertNameCmd = cn.CreateCommand("INSERT INTO " + sysnamesTableName + " (Name, EdsmId) VALUES (@Name, @EdsmId)", txn);
insertNameCmd.AddParameter("@Name", DbType.String);
insertNameCmd.AddParameter("@EdsmId", DbType.Int64);
insertSysCmd = cn.CreateCommand("INSERT INTO " + edsmsysTableName + " (EdsmId, X, Y, Z, CreateTimestamp, UpdateTimestamp, VersionTimestamp, GridId, RandomId) VALUES (@EdsmId, @X, @Y, @Z, @CreateTimestamp, @UpdateTimestamp, @VersionTimestamp, @GridId, @RandomId)", txn);
insertSysCmd.AddParameter("@EdsmId", DbType.Int64);
insertSysCmd.AddParameter("@X", DbType.Int64);
insertSysCmd.AddParameter("@Y", DbType.Int64);
insertSysCmd.AddParameter("@Z", DbType.Int64);
insertSysCmd.AddParameter("@CreateTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@UpdateTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@VersionTimestamp", DbType.Int64);
insertSysCmd.AddParameter("@GridId", DbType.Int64);
insertSysCmd.AddParameter("@RandomId", DbType.Int64);
selectSysCmd = cn.CreateCommand("SELECT Id, X, Y, Z, GridId, RandomId FROM EdsmSystems WHERE EdsmId=@EdsmId");
selectSysCmd.AddParameter("@EdsmId", DbType.Int64);
selectNameCmd = cn.CreateCommand("SELECT Name FROM SystemNames WHERE EdsmId = @EdsmId");
selectNameCmd.AddParameter("@EdsmId", DbType.Int64);
while (!cancelRequested())
{
if (!jo_enum.MoveNext())
{
reportProgress(-1, $"Syncing EDSM systems: {count:N0} processed, {insertcount:N0} new systems, {updatecount:N0} updated systems");
txn.Commit();
if (jr_eof)
{
System.Diagnostics.Debug.WriteLine("Maximum date was " + maxdate.ToString());
System.Diagnostics.Debug.WriteLine($"Import took {sw.ElapsedMilliseconds}ms");
return(updatecount + insertcount);
}
jo_enum_finished = true;
break;
}
else if (SQLiteConnectionSystem.IsReadWaiting)
{
if (blkcount < objs.Count * 3 / 4) // Let the reader barge in if we've processed less than 3/4 of the items
{
// Reset the counts, roll back the transaction, and let the reader through...
insertcount = oldinsertcnt;
updatecount = oldupdatecnt;
count = oldcount;
jo_enum.Reset();
txn.Rollback();
break;
}
}
EDSMDumpSystem jo = jo_enum.Current;
EDSMDumpSystemCoords coords = jo.coords;
if (coords != null)
{
DateTime updatedate = DateTime.SpecifyKind(jo.date, DateTimeKind.Utc);
if (updatedate > maxdate) // even if we reject it due to grid id, keep last date up to date
{
maxdate = updatedate;
}
double x = coords.x;
double z = coords.z;
int gridid = GridId.Id(x, z);
if (grididallowed[gridid]) // if grid allows it to be added..
{
//System.Diagnostics.Debug.WriteLine("Accept due to gridid " + gridid);
double y = coords.y;
long edsmid = jo.id;
string name = jo.name;
int randomid = rnd.Next(0, 99);
SystemClassBase dbsys = null;
if (!useTempSystems)
{
if (useCache && systemsByEdsmId.ContainsKey(edsmid))
{
dbsys = systemsByEdsmId[edsmid];
}
if (!useCache)
{
selectSysCmd.Parameters["@EdsmId"].Value = edsmid;
using (DbDataReader reader = selectSysCmd.ExecuteReader())
{
if (reader.Read())
{
dbsys = new SystemClassBase
{
ID = (long)reader["id"],
EDSMID = edsmid
};
if (System.DBNull.Value == reader["x"])
{
dbsys.X = double.NaN;
dbsys.Y = double.NaN;
dbsys.Z = double.NaN;
}
else
{
dbsys.X = ((double)(long)reader["X"]) / SystemClassDB.XYZScalar;
dbsys.Y = ((double)(long)reader["Y"]) / SystemClassDB.XYZScalar;
dbsys.Z = ((double)(long)reader["Z"]) / SystemClassDB.XYZScalar;
}
dbsys.EDSMID = edsmid;
dbsys.GridID = reader["GridId"] == DBNull.Value ? 0 : (int)((long)reader["GridId"]);
dbsys.RandomID = reader["RandomId"] == DBNull.Value ? 0 : (int)((long)reader["RandomId"]);
}
}
if (dbsys != null)
{
selectNameCmd.Parameters["@EdsmId"].Value = edsmid;
using (DbDataReader reader = selectNameCmd.ExecuteReader())
{
if (reader.Read())
{
dbsys.Name = (string)reader["Name"];
}
}
}
}
}
if (dbsys != null)
{
// see if EDSM data changed..
if (!dbsys.Name.Equals(name))
{
updateNameCmd.Parameters["@Name"].Value = name;
updateNameCmd.Parameters["@EdsmId"].Value = edsmid;
updateNameCmd.ExecuteNonQuery();
}
if (Math.Abs(dbsys.X - x) > 0.01 ||
Math.Abs(dbsys.Y - y) > 0.01 ||
Math.Abs(dbsys.Z - z) > 0.01 ||
dbsys.GridID != gridid) // position changed
{
updateSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar);
updateSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
updateSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
updateSysCmd.Parameters["@GridId"].Value = gridid;
updateSysCmd.Parameters["@RandomId"].Value = randomid;
updateSysCmd.Parameters["@EdsmId"].Value = edsmid;
updateSysCmd.ExecuteNonQuery();
updatecount++;
}
}
else // not in database..
{
insertNameCmd.Parameters["@Name"].Value = name;
insertNameCmd.Parameters["@EdsmId"].Value = edsmid;
insertNameCmd.ExecuteNonQuery();
insertSysCmd.Parameters["@EdsmId"].Value = edsmid;
insertSysCmd.Parameters["@X"].Value = (long)(x * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@Y"].Value = (long)(y * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@Z"].Value = (long)(z * SystemClassDB.XYZScalar);
insertSysCmd.Parameters["@CreateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@UpdateTimestamp"].Value = updatedate.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@VersionTimestamp"].Value = DateTime.UtcNow.Subtract(new DateTime(2015, 1, 1)).TotalSeconds;
insertSysCmd.Parameters["@GridId"].Value = gridid;
insertSysCmd.Parameters["@RandomId"].Value = randomid;
insertSysCmd.ExecuteNonQuery();
insertcount++;
}
}
else
{
//System.Diagnostics.Debug.WriteLine("Reject due to gridid " + gridid);
}
}
else
{
System.Diagnostics.Debug.WriteLine("Reject due to coords ");
}
count++;
blkcount++;
} // WHILE END
}
finally
{
if (updateNameCmd != null)
{
updateNameCmd.Dispose();
}
if (updateSysCmd != null)
{
updateSysCmd.Dispose();
}
if (insertNameCmd != null)
{
insertNameCmd.Dispose();
}
if (insertSysCmd != null)
{
insertSysCmd.Dispose();
}
if (selectSysCmd != null)
{
selectSysCmd.Dispose();
}
}
}
}
}
}
}
if (cancelRequested())
{
throw new OperationCanceledException();
}
return(updatecount + insertcount);
}
private void DoPerformSync() // in Background worker
{
resyncEDSMEDDBRequestedFlag = 1; // sync is happening, stop any async requests..
Debug.WriteLine(BaseUtils.AppTicks.TickCount100 + " Perform sync");
try
{
bool[] grids = new bool[GridId.MaxGridID];
foreach (int i in GridId.FromString(EDDConfig.Instance.EDSMGridIDs))
{
grids[i] = true;
}
ReportProgress(-1, "");
syncstate.ClearCounters();
if (syncstate.perform_edsm_fullsync || syncstate.perform_eddb_sync)
{
if (syncstate.perform_edsm_fullsync && !PendingClose)
{
// Download new systems
try
{
syncstate.edsm_fullsync_count = SystemClassEDSM.PerformEDSMFullSync(grids, () => PendingClose, ReportProgress, LogLine, LogLineHighlight);
syncstate.perform_edsm_fullsync = false;
}
catch (Exception ex)
{
LogLineHighlight("GetAllEDSMSystems exception:" + ex.Message);
}
}
if (!PendingClose)
{
SQLiteConnectionSystem.CreateSystemsTableIndexes(); // again check indexes.. sometimes SQL does not create them due to schema change
try
{
syncstate.eddb_sync_count = EliteDangerousCore.EDDB.SystemClassEDDB.PerformEDDBFullSync(() => PendingClose, ReportProgress, LogLine, LogLineHighlight);
syncstate.perform_eddb_sync = false;
}
catch (Exception ex)
{
LogLineHighlight("GetEDDBUpdate exception: " + ex.Message);
}
}
}
if (!PendingClose)
{
SQLiteConnectionSystem.CreateSystemsTableIndexes(); // again check indexes.. sometimes SQL does not create them due to schema change
syncstate.edsm_updatesync_count = EliteDangerousCore.EDSM.SystemClassEDSM.PerformEDSMUpdateSync(grids, () => PendingClose, ReportProgress, LogLine, LogLineHighlight);
}
ReportProgress(-1, "");
}
catch (OperationCanceledException)
{
// Swallow Operation Cancelled exceptions
}
catch (Exception ex)
{
LogLineHighlight("Check Systems exception: " + ex.Message + Environment.NewLine + "Trace: " + ex.StackTrace);
}
InvokeAsyncOnUiThread(() => PerformSyncCompleted());
}
public static void RemoveHiddenSystems(string json) // protected against bad json
{
try
{
JsonTextReader jr = new JsonTextReader(new StringReader(json));
bool jr_eof = false;
while (!jr_eof)
{
using (SQLiteConnectionSystem cn2 = new SQLiteConnectionSystem(mode: SQLLiteExtensions.SQLExtConnection.AccessMode.Writer)) // open the db
{
using (DbTransaction txn = cn2.BeginTransaction())
{
DbCommand infoinscmd = null;
DbCommand infodelcmd = null;
DbCommand namedelcmd = null;
try
{
infoinscmd = cn2.CreateCommand("INSERT OR IGNORE INTO SystemAliases (name, id_edsm, id_edsm_mergedto) VALUES (@name, @id_edsm, @id_edsm_mergedto)", txn);
infoinscmd.AddParameter("@name", DbType.String);
infoinscmd.AddParameter("@id_edsm", DbType.Int64);
infoinscmd.AddParameter("@id_edsm_mergedto", DbType.Int64);
infodelcmd = cn2.CreateCommand("DELETE FROM EdsmSystems WHERE EdsmId=@EdsmId", txn);
infodelcmd.AddParameter("@EdsmId", DbType.Int64);
namedelcmd = cn2.CreateCommand("DELETE FROM SystemNames WHERE EdsmId=@EdsmId", txn);
namedelcmd.AddParameter("@EdsmId", DbType.Int64);
while (!SQLiteConnectionSystem.IsReadWaiting)
{
if (!jr.Read())
{
jr_eof = true;
break;
}
if (jr.TokenType == JsonToken.StartObject)
{
JObject jo = JObject.Load(jr);
long edsmid = (long)jo["id"];
string name = (string)jo["system"];
string action = (string)jo["action"];
long mergedto = 0;
if (jo["mergedTo"] != null)
{
mergedto = (long)jo["mergedTo"];
}
Console.Write("Remove " + edsmid);
infodelcmd.Parameters["@EdsmId"].Value = edsmid;
infodelcmd.ExecuteNonQuery();
namedelcmd.Parameters["@EdsmId"].Value = edsmid;
namedelcmd.ExecuteNonQuery();
if (mergedto > 0)
{
infoinscmd.Parameters["@name"].Value = name;
infoinscmd.Parameters["@id_edsm"].Value = edsmid;
infoinscmd.Parameters["@id_edsm_mergedto"].Value = mergedto;
infoinscmd.ExecuteNonQuery();
}
}
}
txn.Commit();
}
finally
{
if (infoinscmd != null)
{
infoinscmd.Dispose();
}
if (infodelcmd != null)
{
infodelcmd.Dispose();
}
if (namedelcmd != null)
{
namedelcmd.Dispose();
}
}
}
}
}
}
catch (Exception e)
{
System.Diagnostics.Debug.WriteLine("SysClassEDSM Exception " + e.ToString());
}
}
static public DateTime GetLastEDDBDownloadTime()
{
string timestr = SQLiteConnectionSystem.GetSettingString("EDDBSystemsTime", "0");
return(new DateTime(Convert.ToInt64(timestr), DateTimeKind.Utc));
}
static public void SetLastEDSMRecordTimeUTC(DateTime time)
{
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", time.ToString(CultureInfo.InvariantCulture));
System.Diagnostics.Debug.WriteLine("Last EDSM record " + time.ToString());
}
static public void ForceEDDBFullUpdate()
{
SQLiteConnectionSystem.PutSettingString("EDDBSystemsTime", "0");
}
public static HistoryList LoadHistory(EDJournalClass journalmonitor, Func <bool> cancelRequested, Action <int, string> reportProgress,
string NetLogPath = null,
bool ForceNetLogReload = false,
bool ForceJournalReload = false,
int CurrentCommander = Int32.MinValue,
bool Keepuievents = true,
int fullhistoryloaddaylimit = 0
)
{
HistoryList hist = new HistoryList();
if (CurrentCommander >= 0)
{
journalmonitor.ParseJournalFiles(() => cancelRequested(), (p, s) => reportProgress(p, s), forceReload: ForceJournalReload); // Parse files stop monitor..
if (NetLogPath != null)
{
string errstr = null;
NetLogClass.ParseFiles(NetLogPath, out errstr, EliteConfigInstance.InstanceConfig.DefaultMapColour, () => cancelRequested(), (p, s) => reportProgress(p, s), ForceNetLogReload, currentcmdrid: CurrentCommander);
}
}
Trace.WriteLine(BaseUtils.AppTicks.TickCount100 + " Files read ");
reportProgress(-1, "Reading Database");
List <JournalEntry> jlist;
if (fullhistoryloaddaylimit > 0)
{
jlist = JournalEntry.GetAll(CurrentCommander,
ids: JournalEntry.EssentialEvents,
allidsafter: DateTime.UtcNow.Subtract(new TimeSpan(fullhistoryloaddaylimit, 0, 0, 0))
).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList();
}
else
{
jlist = JournalEntry.GetAll(CurrentCommander).OrderBy(x => x.EventTimeUTC).ThenBy(x => x.Id).ToList();
}
Trace.WriteLine(BaseUtils.AppTicks.TickCount100 + " Database read " + jlist.Count);
List <Tuple <JournalEntry, HistoryEntry> > jlistUpdated = new List <Tuple <JournalEntry, HistoryEntry> >();
using (SQLiteConnectionSystem conn = new SQLiteConnectionSystem())
{
HistoryEntry prev = null;
JournalEntry jprev = null;
reportProgress(-1, "Creating History");
foreach (JournalEntry je in jlist)
{
if (MergeEntries(jprev, je)) // if we merge.. we may have updated info, so reprint.
{
//jprev.FillInformation(out prev.EventSummary, out prev.EventDescription, out prev.EventDetailedInfo); // need to keep this up to date..
continue;
}
if (je.IsUIEvent && !Keepuievents) // filter out any UI events
{
//System.Diagnostics.Debug.WriteLine("**** Filter out " + je.EventTypeStr + " on " + je.EventTimeLocal.ToString());
continue;
}
bool journalupdate = false;
HistoryEntry he = HistoryEntry.FromJournalEntry(je, prev, out journalupdate, conn);
prev = he;
jprev = je;
hist.historylist.Add(he);
if (journalupdate)
{
jlistUpdated.Add(new Tuple <JournalEntry, HistoryEntry>(je, he));
Debug.WriteLine("Queued update requested {0} {1}", he.System.EDSMID, he.System.Name);
}
}
}
if (jlistUpdated.Count > 0)
{
reportProgress(-1, "Updating journal entries");
using (SQLiteConnectionUser conn = new SQLiteConnectionUser(utc: true))
{
using (DbTransaction txn = conn.BeginTransaction())
{
foreach (Tuple <JournalEntry, HistoryEntry> jehe in jlistUpdated)
{
JournalEntry je = jehe.Item1;
HistoryEntry he = jehe.Item2;
double dist = (je is JournalFSDJump) ? (je as JournalFSDJump).JumpDist : 0;
bool updatecoord = (je is JournalLocOrJump) ? (!(je as JournalLocOrJump).HasCoordinate && he.System.HasCoordinate) : false;
Debug.WriteLine("Push update {0} {1} to JE {2} HE {3}", he.System.EDSMID, he.System.Name, je.Id, he.Indexno);
JournalEntry.UpdateEDSMIDPosJump(je.Id, he.System, updatecoord, dist, conn, txn);
}
txn.Commit();
}
}
}
// now database has been updated due to initial fill, now fill in stuff which needs the user database
hist.CommanderId = CurrentCommander;
reportProgress(-1, "Updating user statistics");
hist.ProcessUserHistoryListEntries(h => h.ToList()); // here, we update the DBs in HistoryEntry and any global DBs in historylist
reportProgress(-1, "Done");
return(hist);
}
public static HistoryEntry FromJournalEntry(JournalEntry je, HistoryEntry prev, out bool journalupdate, SQLiteConnectionSystem conn = null)
{
ISystem isys = prev == null ? new SystemClass("Unknown") : prev.System;
int indexno = prev == null ? 1 : prev.Indexno + 1;
journalupdate = false;
if (je.EventTypeID == JournalTypeEnum.Location || je.EventTypeID == JournalTypeEnum.FSDJump)
{
JournalLocOrJump jl = je as JournalLocOrJump;
ISystem newsys;
if (jl != null && jl.HasCoordinate) // LAZY LOAD IF it has a co-ord.. the front end will when it needs it
{
newsys = new SystemClass(jl.StarSystem, jl.StarPos.X, jl.StarPos.Y, jl.StarPos.Z)
{
EDSMID = jl.EdsmID < 0 ? 0 : jl.EdsmID, // pass across the EDSMID for the lazy load process.
Faction = jl.Faction,
Government = jl.EDGovernment,
PrimaryEconomy = jl.EDEconomy,
Security = jl.EDSecurity,
Population = jl.Population ?? 0,
State = jl.EDState,
Allegiance = jl.EDAllegiance,
UpdateDate = jl.EventTimeUTC,
status = SystemStatusEnum.EDDiscovery,
SystemAddress = jl.SystemAddress,
};
// If it was a new system, pass the coords back to the StartJump
if (prev != null && prev.journalEntry is JournalStartJump)
{
prev.System = newsys; // give the previous startjump our system..
}
}
else
{
// NOTE Rob: 09-JAN-2018 I've removed the Jumpstart looking up a system by name since they were using up lots of lookup time during history reading.
// This is used for pre 2.2 systems without co-ords, which now should be limited.
// JumpStart still gets the system when the FSD loc is processed, see above.
// Jumpstart was also screwing about with the EDSM ID fill in which was broken. This is now working again.
// Default one
newsys = new SystemClass(jl.StarSystem);
newsys.EDSMID = je.EdsmID;
ISystem s = SystemCache.FindSystem(newsys, conn); // has no co-ord, did we find it?
if (s != null) // found a system..
{
if (jl != null && jl.HasCoordinate) // if journal Loc, and journal has a star position, use that instead of EDSM..
{
s.X = Math.Round(jl.StarPos.X * 32.0) / 32.0;
s.Y = Math.Round(jl.StarPos.Y * 32.0) / 32.0;
s.Z = Math.Round(jl.StarPos.Z * 32.0) / 32.0;
}
//Debug.WriteLine("HistoryList found system {0} {1}", s.id_edsm, s.name);
newsys = s;
if (jl != null && je.EdsmID <= 0 && newsys.EDSMID > 0) // only update on a JL..
{
journalupdate = true;
Debug.WriteLine("HE EDSM ID update requested {0} {1}", newsys.EDSMID, newsys.Name);
}
}
else
{
newsys.EDSMID = -1; // mark as checked but not found
}
}
JournalFSDJump jfsd = je as JournalFSDJump;
if (jfsd != null)
{
if (jfsd.JumpDist <= 0 && isys.HasCoordinate && newsys.HasCoordinate) // if no JDist, its a really old entry, and if previous has a co-ord
{
jfsd.JumpDist = isys.Distance(newsys); // fill it out here
if (jfsd.JumpDist > 0)
{
journalupdate = true;
Debug.WriteLine("Je Jump distance update(3) requested {0} {1} {2}", newsys.EDSMID, newsys.Name, jfsd.JumpDist);
}
}
}
isys = newsys;
}
HistoryEntry he = new HistoryEntry
{
Indexno = indexno,
journalEntry = je,
System = isys,
EntryStatus = HistoryEntryStatus.Update(prev?.EntryStatus, je, isys.Name)
};
he.TravelStatus = HistoryTravelStatus.Update(prev?.TravelStatus, prev, he); // need a real he so can't do that as part of the constructor.
return(he);
}
// I think we rework so we have a SyncBackgroundworker, and a refresh background worker.
// loading of history is done on refresh one..
private void BackgroundWorkerThread()
{
readyForInitialLoad.WaitOne(); // wait for shown in form
// check first and download items
StarScan.LoadBodyDesignationMap();
Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Check systems");
ReportSyncProgress("");
bool checkGithub = EDDOptions.Instance.CheckGithubFiles;
if (checkGithub) // not normall in debug, due to git hub chokeing
{
// Async load of maps in another thread
DownloadMaps(() => PendingClose);
// and Expedition data
DownloadExpeditions(() => PendingClose);
// and Exploration data
DownloadExploration(() => PendingClose);
}
if (!EDDOptions.Instance.NoSystemsLoad)
{
// New Galmap load - it was not doing a refresh if EDSM sync kept on happening. Now has its own timer
DateTime galmaptime = SQLiteConnectionSystem.GetSettingDate("EDSMGalMapLast", DateTime.MinValue); // Latest time from RW file.
if (DateTime.Now.Subtract(galmaptime).TotalDays > 14 || !galacticMapping.GalMapFilePresent()) // Over 14 days do a sync from EDSM for galmap
{
LogLine("Get galactic mapping from EDSM.".Tx(this, "EDSM"));
if (galacticMapping.DownloadFromEDSM())
{
SQLiteConnectionSystem.PutSettingDate("EDSMGalMapLast", DateTime.UtcNow);
}
}
Debug.WriteLine(BaseUtils.AppTicks.TickCountLap() + " Check systems complete");
}
galacticMapping.ParseData(); // at this point, gal map data has been uploaded - get it into memory
SystemCache.AddToAutoCompleteList(galacticMapping.GetGMONames());
SystemNoteClass.GetAllSystemNotes();
LogLine("Loaded Notes, Bookmarks and Galactic mapping.".Tx(this, "LN"));
if (PendingClose)
{
return;
}
if (EliteDangerousCore.EDDN.EDDNClass.CheckforEDMC()) // EDMC is running
{
if (EDCommander.Current.SyncToEddn) // Both EDD and EDMC should not sync to EDDN.
{
LogLineHighlight("EDDiscovery and EDMarketConnector should not both sync to EDDN. Stop EDMC or uncheck 'send to EDDN' in settings tab!".Tx(this, "EDMC"));
}
}
if (!EDDOptions.Instance.NoLoad) // here in this thread, we do a refresh of history.
{
LogLine("Reading travel history".Tx(this, "RTH"));
if (EDDOptions.Instance.Commander != null)
{
EDCommander switchto = EDCommander.GetCommander(EDDOptions.Instance.Commander);
if (switchto != null)
{
EDCommander.CurrentCmdrID = switchto.Nr;
}
}
DoRefreshHistory(new RefreshWorkerArgs {
CurrentCommander = EDCommander.CurrentCmdrID
}); // kick the background refresh worker thread into action
}
if (PendingClose)
{
return;
}
CheckForSync(); // see if any EDSM/EDDB sync is needed
if (PendingClose)
{
return;
}
// Now stay in loop services stuff
backgroundRefreshWorker = new Thread(BackgroundHistoryRefreshWorkerThread)
{
Name = "Background Refresh Worker", IsBackground = true
};
backgroundRefreshWorker.Start(); // start the refresh worker, another thread which does subsequenct (not the primary one) refresh work in the background..
try
{
if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if no system off, and EDSM download on
{
DoPerformSync(); // this is done after the initial history load..
}
while (!PendingClose)
{
int wh = WaitHandle.WaitAny(new WaitHandle[] { closeRequested, resyncRequestedEvent });
if (PendingClose)
{
break;
}
switch (wh)
{
case 0: // Close Requested
break;
case 1: // Resync Requested
if (!EDDOptions.Instance.NoSystemsLoad && EDDConfig.Instance.EDSMEDDBDownload) // if no system off, and EDSM download on
{
DoPerformSync();
}
break;
}
}
}
catch (OperationCanceledException)
{
}
backgroundRefreshWorker.Join();
// Now we have been ordered to close down, so go thru the process
closeRequested.WaitOne();
OnBgSafeClose?.Invoke();
ReadyForFinalClose = true;
InvokeAsyncOnUiThread(() =>
{
OnFinalClose?.Invoke();
});
}
public static bool PerformEDSMFullSync(Func <bool> cancelRequested, Action <int, string> reportProgress, Action <string> logLine, Action <string> logError)
{
string rwsystime = SQLiteConnectionSystem.GetSettingString("EDSMLastSystems", "2000-01-01 00:00:00"); // Latest time from RW file.
DateTime edsmdate;
if (!DateTime.TryParse(rwsystime, CultureInfo.InvariantCulture, DateTimeStyles.None, out edsmdate))
{
edsmdate = new DateTime(2000, 1, 1);
}
long updates = 0;
// Delete all old systems
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", "2010-01-01 00:00:00");
SQLiteConnectionSystem.PutSettingString("EDDBSystemsTime", "0");
EDSMClass edsm = new EDSMClass();
logLine("Get hidden systems from EDSM and remove from database");
RemoveHiddenSystems();
if (cancelRequested())
{
return(false);
}
logLine("Download systems file from EDSM.");
string edsmsystems = Path.Combine(EliteConfigInstance.InstanceOptions.AppDataDirectory, "edsmsystems.json");
logLine("Resyncing all downloaded EDSM systems with local database." + Environment.NewLine + "This will take a while.");
bool newfile;
bool success = BaseUtils.DownloadFileHandler.DownloadFile(EDSMClass.ServerAddress + "dump/systemsWithCoordinates.json", edsmsystems, out newfile, (n, s) =>
{
SQLiteConnectionSystem.CreateTempSystemsTable();
string rwsysfiletime = "2014-01-01 00:00:00";
bool outoforder = false;
using (var reader = new StreamReader(s))
updates = ParseEDSMUpdateSystemsStream(reader, ref rwsysfiletime, ref outoforder, true, cancelRequested, reportProgress, useCache: false, useTempSystems: true);
if (!cancelRequested()) // abort, without saving time, to make it do it again
{
SQLiteConnectionSystem.PutSettingString("EDSMLastSystems", rwsysfiletime);
logLine("Replacing old systems table with new systems table and re-indexing - please wait");
reportProgress(-1, "Replacing old systems table with new systems table and re-indexing - please wait");
SQLiteConnectionSystem.ReplaceSystemsTable();
SQLiteConnectionSystem.PutSettingBool("EDSMSystemsOutOfOrder", outoforder);
reportProgress(-1, "");
#if !DEBUG
File.Delete(edsmsystems);
#endif
}
else
{
throw new OperationCanceledException();
}
});
if (!success)
{
logLine("Failed to download EDSM system file from server, will check next time");
return(false);
}
// Stop if requested
if (cancelRequested())
{
return(false);
}
logLine("Local database updated with EDSM data, " + updates + " systems updated.");
GC.Collect();
return(updates > 0);
}
