void Process(List <JournalFSDJump> edsmlogs, DateTime logstarttime, DateTime logendtime)
{
// Get all of the local entries now that we have the entries from EDSM
// Moved here to avoid the race that could have been causing duplicate entries
// EDSM only returns FSD entries, so only look for them. Tested 27/4/2018 after the HE optimisations
List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr, logstarttime.AddDays(-1), logendtime.AddDays(1)).
OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).
Select(je => HistoryEntry.FromJournalEntry(je, null, out bool jupdate)).ToList(); // using HE just because of the FillEDSM func
HistoryList hl = new HistoryList(hlfsdlist); // just so we can access the FillEDSM func
List <JournalFSDJump> toadd = new List <JournalFSDJump>();
int previdx = -1;
foreach (JournalFSDJump jfsd in edsmlogs) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == jfsd.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(jfsd.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.EDSMID <= 0) // if we don't have a valid EDMSID..
{
vi.v.System.EDSMID = 0;
hl.FillEDSM(vi.v);
}
if (vi.v.System.EDSMID <= 0 || vi.v.System.EDSMID == jfsd.EdsmID)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0) // its not a duplicate, add to db
{
toadd.Add(jfsd);
}
else
{ // it is a duplicate, check if the first discovery flag is set right
JournalFSDJump existingfsd = hlfsdlist[index].journalEntry as JournalFSDJump;
if (existingfsd != null && existingfsd.EDSMFirstDiscover != jfsd.EDSMFirstDiscover) // if we have a FSD one, and first discover is different
{
existingfsd.UpdateFirstDiscover(jfsd.EDSMFirstDiscover);
}
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
System.Diagnostics.Debug.WriteLine($"Adding EDSM logs count {toadd.Count}");
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = TravelLogUnit.EDSMType; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDCommander.CurrentCmdrID;
tlu.Add(); // Add to Database
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
foreach (JournalFSDJump jfsd in toadd)
{
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", jfsd.EventTimeUTC, jfsd.StarSystem));
jfsd.SetTLUCommander(tlu.id, tlu.CommanderId.Value); // update its TLU id to the TLU made above
jfsd.Add(jfsd.CreateFSDJournalEntryJson(), cn.Connection); // add it to the db with the JSON created
}
});
LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}");
OnDownloadedSystems?.Invoke();
}
}
private void FetcherThreadProc()
{
Trace.WriteLine($"EDSM Thread logs start");
bool jupdate = false;
int waittime = 2000; // Max 1 request every 2 seconds, with a backoff if the rate limit is hit
if (EDSMRequestBackoffTime > DateTime.UtcNow)
{
waittime = (int)Math.Min(EDSMMaxLogAgeMinutes * 60000, Math.Min(BackoffInterval.TotalSeconds * 1000, EDSMRequestBackoffTime.Subtract(DateTime.UtcNow).TotalSeconds * 1000));
}
// get them as of now.. since we are searching back in time it should be okay. On a refresh we would start again!
List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr).OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).Select(je => HistoryEntry.FromJournalEntry(je, null, false, out jupdate)).ToList();
while (!ExitRequested.WaitOne(waittime))
{
EDSMClass edsm = new EDSMClass {
apiKey = Commander.APIKey, commanderName = Commander.EdsmName
};
List <HistoryEntry> edsmlogs = null;
DateTime logstarttime = DateTime.MinValue;
DateTime logendtime = DateTime.MinValue;
int res = -1;
if (edsm.IsApiKeySet && Commander.SyncFromEdsm && DateTime.UtcNow > EDSMRequestBackoffTime)
{
if (DateTime.UtcNow.Subtract(LastEventTime).TotalMinutes >= EDSMMaxLogAgeMinutes)
{
//Trace.WriteLine($"Retrieving EDSM logs starting {LastEventTime}");
res = edsm.GetLogs(LastEventTime, null, out edsmlogs, out logstarttime, out logendtime);
}
else if (FirstEventTime > GammaStart)
{
//Trace.WriteLine($"Retrieving EDSM logs ending {FirstEventTime}");
res = edsm.GetLogs(null, FirstEventTime, out edsmlogs, out logstarttime, out logendtime);
}
}
if (ExitRequested.WaitOne(0))
{
return;
}
if (res == 429) // Rate Limit Exceeded
{
Trace.WriteLine($"EDSM Log request rate limit hit - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (logstarttime > LastEventTime && logendtime < FirstEventTime)
{
Trace.WriteLine($"Bad start and/or end times returned by EDSM - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (res == 100 && edsmlogs != null)
{
if (edsmlogs.Count > 0) // if anything to process..
{
//Trace.WriteLine($"Retrieving EDSM logs count {edsmlogs.Count}");
BackoffInterval = TimeSpan.FromSeconds(60);
if (logendtime > DateTime.UtcNow)
{
logendtime = DateTime.UtcNow;
}
HistoryList hl = new HistoryList(hlfsdlist);
List <DateTime> hlfsdtimes = hlfsdlist.Select(he => he.EventTimeUTC).ToList();
List <HistoryEntry> toadd = new List <HistoryEntry>();
int previdx = -1;
foreach (HistoryEntry he in edsmlogs) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(he.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.id_edsm <= 0)
{
vi.v.System.id_edsm = 0;
hl.FillEDSM(vi.v);
}
if (vi.v.System.id_edsm <= 0 || vi.v.System.id_edsm == he.System.id_edsm)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0)
{
toadd.Add(he);
}
else
{
HistoryEntry lhe = hlfsdlist[index];
if (he.IsEDSMFirstDiscover && !lhe.IsEDSMFirstDiscover)
{
lhe.SetFirstDiscover();
}
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
Trace.WriteLine($"Adding EDSM logs count {toadd.Count}");
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDCommander.CurrentCmdrID;
tlu.Add(); // Add to Database
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
foreach (HistoryEntry he in toadd)
{
JObject jo = JournalEntry.CreateFSDJournalEntryJson(he.EventTimeUTC,
he.System.name, he.System.x, he.System.y, he.System.z,
EliteConfigInstance.InstanceConfig.DefaultMapColour);
JournalEntry je =
JournalEntry.CreateFSDJournalEntry(tlu.id, tlu.CommanderId.Value,
(int)SyncFlags.EDSM, jo);
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", je.EventTimeUTC, he.System.name));
je.Add(jo, cn);
}
}
LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}");
if (logendtime > LastEventTime || logstarttime <= GammaStart)
{
if (OnDownloadedSystems != null)
{
OnDownloadedSystems();
}
}
}
}
if (logstarttime < FirstEventTime)
{
FirstEventTime = logstarttime;
}
if (logendtime > LastEventTime)
{
LastEventTime = logendtime;
}
}
}
}
private void FetcherThreadProc()
{
Trace.WriteLine($"EDSM Thread logs start");
bool jupdate = false;
DateTime lastCommentFetch = DateTime.MinValue;
int waittime = 2000; // Max 1 request every 2 seconds, with a backoff if the rate limit is hit
if (EDSMRequestBackoffTime > DateTime.UtcNow)
{
waittime = (int)Math.Min(EDSMMaxLogAgeMinutes * 60000, Math.Min(BackoffInterval.TotalSeconds * 1000, EDSMRequestBackoffTime.Subtract(DateTime.UtcNow).TotalSeconds * 1000));
}
while (!ExitRequested.WaitOne(waittime))
{
EDSMClass edsm = new EDSMClass(Commander);
if (edsm.ValidCredentials && DateTime.UtcNow > lastCommentFetch.AddHours(1))
{
edsm.GetComments(l => Trace.WriteLine(l));
lastCommentFetch = DateTime.UtcNow;
}
DateTime logstarttime = DateTime.MinValue; // return what we got..
DateTime logendtime = DateTime.MinValue;
List <JournalFSDJump> edsmlogs = null;
int res = -1; //return code
if (edsm.ValidCredentials && Commander.SyncFromEdsm && DateTime.UtcNow > EDSMRequestBackoffTime)
{
if (DateTime.UtcNow.Subtract(LastEventTime).TotalMinutes >= EDSMMaxLogAgeMinutes)
{
System.Diagnostics.Debug.WriteLine($"Retrieving EDSM logs starting {LastEventTime}");
res = edsm.GetLogs(LastEventTime, null, out edsmlogs, out logstarttime, out logendtime);
}
else if (FirstEventTime > GammaStart)
{
System.Diagnostics.Debug.WriteLine($"Retrieving EDSM logs ending {FirstEventTime}");
res = edsm.GetLogs(null, FirstEventTime, out edsmlogs, out logstarttime, out logendtime);
}
}
if (ExitRequested.WaitOne(0))
{
return;
}
if (res == 429) // Rate Limit Exceeded
{
Trace.WriteLine($"EDSM Log request rate limit hit - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (logstarttime > LastEventTime && logendtime < FirstEventTime)
{
Trace.WriteLine($"Bad start and/or end times returned by EDSM - backing off for {BackoffInterval.TotalSeconds}s");
EDSMRequestBackoffTime = DateTime.UtcNow + BackoffInterval;
BackoffInterval = BackoffInterval + TimeSpan.FromSeconds(60);
}
else if (res == 100 && edsmlogs != null)
{
if (edsmlogs.Count > 0) // if anything to process..
{
//Trace.WriteLine($"Retrieving EDSM logs count {edsmlogs.Count}");
BackoffInterval = TimeSpan.FromSeconds(60);
if (logendtime > DateTime.UtcNow)
{
logendtime = DateTime.UtcNow;
}
if (logstarttime < DateTime.MinValue.AddDays(1))
{
logstarttime = DateTime.MinValue.AddDays(1);
}
// Get all of the local entries now that we have the entries from EDSM
// Moved here to avoid the race that could have been causing duplicate entries
// EDSM only returns FSD entries, so only look for them. Tested 27/4/2018 after the HE optimisations
List <HistoryEntry> hlfsdlist = JournalEntry.GetAll(Commander.Nr, logstarttime.AddDays(-1), logendtime.AddDays(1)).
OfType <JournalLocOrJump>().OrderBy(je => je.EventTimeUTC).
Select(je => HistoryEntry.FromJournalEntry(je, null, out jupdate)).ToList(); // using HE just because of the FillEDSM func
HistoryList hl = new HistoryList(hlfsdlist); // just so we can access the FillEDSM func
List <JournalFSDJump> toadd = new List <JournalFSDJump>();
int previdx = -1;
foreach (JournalFSDJump jfsd in edsmlogs) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == jfsd.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.Name.Equals(jfsd.StarSystem, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(jfsd.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.EDSMID <= 0) // if we don't have a valid EDMSID..
{
vi.v.System.EDSMID = 0;
hl.FillEDSM(vi.v);
}
if (vi.v.System.EDSMID <= 0 || vi.v.System.EDSMID == jfsd.EdsmID)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0) // its not a duplicate, add to db
{
toadd.Add(jfsd);
}
else
{ // it is a duplicate, check if the first discovery flag is set right
JournalFSDJump existingfsd = hlfsdlist[index].journalEntry as JournalFSDJump;
if (existingfsd != null && existingfsd.EDSMFirstDiscover != jfsd.EDSMFirstDiscover) // if we have a FSD one, and first discover is different
{
existingfsd.UpdateFirstDiscover(jfsd.EDSMFirstDiscover);
}
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
Trace.WriteLine($"Adding EDSM logs count {toadd.Count}");
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDCommander.CurrentCmdrID;
tlu.Add(); // Add to Database
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
foreach (JournalFSDJump jfsd in toadd)
{
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", jfsd.EventTimeUTC, jfsd.StarSystem));
jfsd.SetTLUCommander(tlu.id, jfsd.CommanderId); // update its TLU id to the TLU made above
jfsd.Add(jfsd.CreateFSDJournalEntryJson(), cn);
}
}
LogLine($"Retrieved {toadd.Count} log entries from EDSM, from {logstarttime.ToLocalTime().ToString()} to {logendtime.ToLocalTime().ToString()}");
if (logendtime > LastEventTime || logstarttime <= GammaStart)
{
if (OnDownloadedSystems != null)
{
OnDownloadedSystems();
}
}
}
}
if (logstarttime < FirstEventTime)
{
FirstEventTime = logstarttime;
}
if (logendtime > LastEventTime)
{
LastEventTime = logendtime;
}
}
}
}
public void Sync(bool pushOnly)
{
try
{
SQLiteDBClass db = new SQLiteDBClass();
EDSMClass edsm = new EDSMClass();
edsm.apiKey = EDDiscoveryForm.EDDConfig.CurrentCommander.APIKey;
edsm.commanderName = EDDiscoveryForm.EDDConfig.CurrentCommander.Name;
//string comments = edsm.GetComments(new DateTime(2015, 1, 1));
List <SystemPosition> log;
int ret = edsm.GetLogs(new DateTime(2011, 1, 1), out log);
if (log == null)
{
log = new List <SystemPosition>();
}
// Send Unsynced system to EDSM.
List <SystemPosition> systems = (from s in mainForm.VisitedSystems where s.vs != null && s.vs.EDSM_sync == false && s.vs.Commander == EDDiscoveryForm.EDDConfig.CurrentCommander.Nr select s).ToList <SystemPosition>();
mainForm.LogLine("EDSM: Sending " + systems.Count.ToString() + " flightlog entries", Color.Black);
foreach (var system in systems)
{
string json = null;
if (Exit)
{
running = false;
return;
}
if (system.vs != null && system.vs.EDSM_sync == false)
{
// check if it exist in EDSM
SystemPosition ps2 = (from c in log where c.Name == system.Name && c.time.Ticks == system.time.Ticks select c).FirstOrDefault <SystemPosition>();
if (ps2 != null)
{
system.vs.EDSM_sync = true;
system.Update();
}
else
{
json = edsm.SetLog(system.Name, system.time);
}
if (json != null)
{
JObject msg = (JObject)JObject.Parse(json);
int msgnum = msg["msgnum"].Value <int>();
string msgstr = msg["msg"].Value <string>();
if (msgnum == 100 || msgnum == 401 || msgnum == 402 || msgnum == 403)
{
if (msgnum == 100)
{
System.Diagnostics.Trace.WriteLine("New");
}
system.vs.EDSM_sync = true;
system.Update();
}
else
{
mainForm.LogLine("EDSM sync ERROR:" + msgnum.ToString() + ":" + msgstr, Color.Red);
System.Diagnostics.Trace.WriteLine("Error sync:" + msgnum.ToString() + " : " + system.Name);
break;
}
}
}
}
TravelLogUnit tlu = null;
bool newsystem = false;
if (!pushOnly)
{
// Check for new systems from EDSM
int defaultColour = db.GetSettingInt("DefaultMap", Color.Red.ToArgb());
foreach (var system in log)
{
SystemPosition ps2 = (from c in mainForm.VisitedSystems where c.Name == system.Name && c.time.Ticks == system.time.Ticks select c).FirstOrDefault <SystemPosition>();
if (ps2 == null) // Add to local DB...
{
if (tlu == null) // If we dontt have a travellogunit yet then create it.
{
tlu = new TravelLogUnit();
tlu.type = 2; // EDSM
tlu.Path = "http://www.edsm.net/api-logs-v1/get-logs";
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Add(); // Add to Database
}
VisitedSystemsClass vs = new VisitedSystemsClass();
vs.Source = tlu.id;
vs.Unit = tlu.Name;
vs.Name = system.Name;
vs.Time = system.time;
vs.MapColour = defaultColour;
vs.EDSM_sync = true;
vs.Add(); // Add to DB;
System.Diagnostics.Trace.WriteLine("New from EDSM");
newsystem = true;
}
}
}
mainForm.LogLine("EDSM sync Done", Color.Black);
if (newsystem)
{
OnNewEDSMTravelLog(this);
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message);
mainForm.LogLine("EDSM sync Exception " + ex.Message, Color.Red);
}
}
private void NetLogMain()
{
try
{
m_Watcher = new System.IO.FileSystemWatcher();
if (Directory.Exists(GetNetLogPath()))
{
m_Watcher.Path = GetNetLogPath() + "\\";
m_Watcher.Filter = "netLog*.log";
m_Watcher.IncludeSubdirectories = true;
m_Watcher.NotifyFilter = NotifyFilters.FileName; // | NotifyFilters.Size;
m_Watcher.Changed += new FileSystemEventHandler(OnChanged);
m_Watcher.Created += new FileSystemEventHandler(OnChanged);
m_Watcher.Deleted += new FileSystemEventHandler(OnChanged);
m_Watcher.EnableRaisingEvents = true;
}
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show("Net log watcher exception : " + ex.Message, "EDDiscovery Error");
System.Diagnostics.Trace.WriteLine("NetlogMAin exception : " + ex.Message);
System.Diagnostics.Trace.WriteLine(ex.StackTrace);
}
List <TravelLogUnit> travelogUnits;
// Get TravelLogUnits;
travelogUnits = null;
TravelLogUnit tlUnit = null;
SQLiteDBClass db = new SQLiteDBClass();
int ii = 0;
while (!Exit)
{
try
{
ii++;
Thread.Sleep(2000);
EliteDangerous.CheckED();
if (NoEvents == false)
{
if (lastnfi != null)
{
FileInfo fi = new FileInfo(lastnfi.FileName);
if (fi.Length != lastnfi.fileSize || ii % 5 == 0)
{
if (tlUnit == null || !tlUnit.Name.Equals(Path.GetFileName(lastnfi.FileName))) // Create / find new travellog unit
{
travelogUnits = TravelLogUnit.GetAll();
// Check if we alreade have parse the file and stored in DB.
if (tlUnit == null)
{
tlUnit = (from c in travelogUnits where c.Name == fi.Name select c).FirstOrDefault <TravelLogUnit>();
}
if (tlUnit == null)
{
tlUnit = new TravelLogUnit();
tlUnit.Name = fi.Name;
tlUnit.Path = Path.GetDirectoryName(fi.FullName);
tlUnit.Size = 0; // Add real size after data is in DB //;(int)fi.Length;
tlUnit.type = 1;
tlUnit.Add();
travelogUnits.Add(tlUnit);
}
}
int nrsystems = visitedSystems.Count;
ParseFile(fi, visitedSystems);
if (nrsystems < visitedSystems.Count) // Om vi har fler system
{
System.Diagnostics.Trace.WriteLine("New systems " + nrsystems.ToString() + ":" + visitedSystems.Count.ToString());
for (int nr = nrsystems; nr < visitedSystems.Count; nr++) // Lägg till nya i locala databaslogen
{
VisitedSystemsClass dbsys = new VisitedSystemsClass();
dbsys.Name = visitedSystems[nr].Name;
dbsys.Time = visitedSystems[nr].time;
dbsys.Source = tlUnit.id;
dbsys.EDSM_sync = false;
dbsys.Unit = fi.Name;
dbsys.MapColour = db.GetSettingInt("DefaultMap", Color.Red.ToArgb());
dbsys.Unit = fi.Name;
dbsys.Commander = 0;
if (!tlUnit.Beta) // dont store history in DB for beta (YET)
{
dbsys.Add();
}
visitedSystems[nr].vs = dbsys;
}
}
else
{
//System.Diagnostics.Trace.WriteLine("No change");
}
}
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("NetlogMAin exception : " + ex.Message);
System.Diagnostics.Trace.WriteLine(ex.StackTrace);
}
}
}
public List <SystemPosition> ParseFiles(RichTextBox richTextBox_History, int defaultMapColour)
{
string datapath;
DirectoryInfo dirInfo;
datapath = GetNetLogPath();
if (datapath == null)
{
AppendText(richTextBox_History, "Netlog directory not found!" + Environment.NewLine + "Specify location in settings tab" + Environment.NewLine, Color.Red);
return(null);
}
if (!Directory.Exists(datapath)) // if logfiles directory is not found
{
if (richTextBox_History != null)
{
richTextBox_History.Clear();
AppendText(richTextBox_History, "Netlog directory not found!" + Environment.NewLine + "Specify location in settings tab" + Environment.NewLine, Color.Red);
//MessageBox.Show("Netlog directory not found!" + Environment.NewLine + "Specify location in settings tab", "EDDiscovery Error", MessageBoxButtons.OK);
}
return(null);
}
try
{
dirInfo = new DirectoryInfo(datapath);
}
catch (Exception ex)
{
AppendText(richTextBox_History, "Could not create Directory info: " + ex.Message + Environment.NewLine, Color.Red);
return(null);
}
// Get TravelLogUnits;
tlUnits = TravelLogUnit.GetAll();
List <VisitedSystemsClass> vsSystemsList = VisitedSystemsClass.GetAll(ActiveCommander);
visitedSystems.Clear();
// Add systems in local DB.
if (vsSystemsList != null)
{
foreach (VisitedSystemsClass vs in vsSystemsList)
{
if (visitedSystems.Count == 0)
{
visitedSystems.Add(new SystemPosition(vs));
}
else if (!visitedSystems.Last <SystemPosition>().Name.Equals(vs.Name)) // Avoid duplicate if times exist in same system from different files.
{
visitedSystems.Add(new SystemPosition(vs));
}
else
{
VisitedSystemsClass vs2 = (VisitedSystemsClass)visitedSystems.Last <SystemPosition>().vs;
vs.Commander = -2; // Move to dupe user
vs.Update();
}
}
}
FileInfo[] allFiles = dirInfo.GetFiles("netLog.*.log", SearchOption.AllDirectories).OrderBy(p => p.Name).ToArray();
NoEvents = true;
foreach (FileInfo fi in allFiles)
{
TravelLogUnit lu = null;
bool parsefile = true;
if (fi.Name.Equals("netLog.1510280152.01.log"))
{
parsefile = true;
}
// Check if we alreade have parse the file and stored in DB.
if (tlUnits != null)
{
lu = (from c in tlUnits where c.Name == fi.Name select c).FirstOrDefault <TravelLogUnit>();
}
if (lu != null)
{
if (lu.Size == fi.Length) // File is already in DB:
{
parsefile = false;
}
}
else
{
lu = new TravelLogUnit();
lu.Name = fi.Name;
lu.Path = Path.GetDirectoryName(fi.FullName);
lu.Size = 0; // Add real size after data is in DB //;(int)fi.Length;
lu.type = 1;
lu.Add();
}
if (parsefile)
{
int nr = 0;
List <SystemPosition> tempVisitedSystems = new List <SystemPosition>();
ParseFile(fi, tempVisitedSystems);
foreach (SystemPosition ps in tempVisitedSystems)
{
SystemPosition ps2;
ps2 = (from c in visitedSystems where c.Name == ps.Name && c.time == ps.time select c).FirstOrDefault <SystemPosition>();
if (ps2 == null)
{
VisitedSystemsClass dbsys = new VisitedSystemsClass();
dbsys.Name = ps.Name;
dbsys.Time = ps.time;
dbsys.Source = lu.id;
dbsys.EDSM_sync = false;
dbsys.Unit = fi.Name;
dbsys.MapColour = defaultMapColour;
dbsys.Commander = ActiveCommander;
if (!lu.Beta) // dont store history in DB for beta (YET)
{
VisitedSystemsClass last = VisitedSystemsClass.GetLast();
if (last == null || !last.Name.Equals(dbsys.Name)) // If same name as last system. Dont Add. otherwise we get a duplet with last from logfile before with different time.
{
if (!VisitedSystemsClass.Exist(dbsys.Name, dbsys.Time))
{
dbsys.Add();
visitedSystems.Add(ps);
nr++;
}
}
}
}
}
lu.Size = (int)fi.Length;
lu.Update();
AppendText(richTextBox_History, fi.Name + " " + nr.ToString() + " added to local database." + Environment.NewLine, Color.Black);
}
}
NoEvents = false;
//var result = visitedSystems.OrderByDescending(a => a.time).ToList<SystemPosition>();
return(visitedSystems);
}
private void Sync(EDSMClass edsm)
{
try
{
mainForm.LogLine("EDSM sync begin");
List <HistoryEntry> hlfsdunsyncedlist = mainForm.history.FilterByNotEDSMSyncedAndFSD; // first entry is oldest
if (_syncTo && hlfsdunsyncedlist.Count > 0) // send systems to edsm (verified with dates, 29/9/2016, utc throughout)
{
DateTime utcmin = hlfsdunsyncedlist[0].EventTimeUTC.AddDays(-1); // 1 days for margin ;-) only get them back to this date for speed..
mainForm.LogLine("EDSM: Sending " + hlfsdunsyncedlist.Count.ToString() + " flightlog entries");
List <HistoryEntry> edsmsystemlog = null;
edsm.GetLogs(utcmin, out edsmsystemlog); // always returns a log, time is in UTC as per HistoryEntry and JournalEntry
int edsmsystemssent = 0;
foreach (var he in hlfsdunsyncedlist)
{
if (Exit)
{
running = false;
return;
}
HistoryEntry ps2 = (from c in edsmsystemlog where c.System.name == he.System.name && c.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks select c).FirstOrDefault();
if (ps2 != null) // it did, just make sure EDSM sync flag is set..
{
he.SetEdsmSync();
}
else
{
string errmsg; // (verified with EDSM 29/9/2016)
// it converts to UTC inside the function, supply local for now
if (edsm.SendTravelLog(he.System.name, he.EventTimeUTC, he.System.HasCoordinate && !he.IsStarPosFromEDSM, he.System.x, he.System.y, he.System.z, out errmsg))
{
he.SetEdsmSync();
}
if (errmsg.Length > 0)
{
mainForm.LogLine(errmsg);
}
edsmsystemssent++;
}
}
mainForm.LogLine(string.Format("EDSM Systems sent {0}", edsmsystemssent));
}
// TBD Comments to edsm?
if (_syncFrom) // Verified ok with time 29/9/2016
{
var json = edsm.GetComments(new DateTime(2011, 1, 1));
if (json != null)
{
JObject msg = JObject.Parse(json);
int msgnr = msg["msgnum"].Value <int>();
JArray comments = (JArray)msg["comments"];
if (comments != null)
{
int commentsadded = 0;
foreach (JObject jo in comments)
{
string name = jo["system"].Value <string>();
string note = jo["comment"].Value <string>();
string utctime = jo["lastUpdate"].Value <string>();
int edsmid = 0;
if (!Int32.TryParse(JSONHelper.GetStringDef(jo["systemId"], "0"), out edsmid))
{
edsmid = 0;
}
DateTime localtime = DateTime.ParseExact(utctime, "yyyy-MM-dd HH:mm:ss",
CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime();
SystemNoteClass curnote = SystemNoteClass.GetNoteOnSystem(name, edsmid);
if (curnote != null)
{
// curnote uses local time to store
if (localtime.Ticks > curnote.Time.Ticks) // if newer, add on (verified with EDSM 29/9/2016)
{
curnote.Note += ". EDSM: " + note;
curnote.Time = localtime;
curnote.EdsmId = edsmid;
curnote.Update();
commentsadded++;
}
}
else
{
curnote = new SystemNoteClass();
curnote.Note = note;
curnote.Time = localtime;
curnote.Name = name;
curnote.Journalid = 0;
curnote.EdsmId = edsmid;
curnote.Add();
commentsadded++;
}
}
mainForm.LogLine(string.Format("EDSM Comments downloaded/updated {0}", commentsadded));
}
}
}
if (_syncFrom) // verified after struggle 29/9/2016
{
List <HistoryEntry> edsmsystemlog = null;
edsm.GetLogs(new DateTime(2011, 1, 1), out edsmsystemlog); // get the full list of systems
edsmsystemlog = edsmsystemlog.OrderBy(s => s.EventTimeUTC).ToList();
List <HistoryEntry> hlfsdlist = mainForm.history.FilterByFSD.OrderBy(h => h.EventTimeUTC).ToList(); // FSD jumps only
List <HistoryEntry> toadd = new List <HistoryEntry>();
int previdx = -1;
foreach (HistoryEntry he in edsmsystemlog) // find out list of ones not present
{
int index = hlfsdlist.FindIndex(x => x.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase) && x.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks);
if (index < 0)
{
// Look for any entries where DST may have thrown off the time
foreach (var vi in hlfsdlist.Select((v, i) => new { v = v, i = i }).Where(vi => vi.v.System.name.Equals(he.System.name, StringComparison.InvariantCultureIgnoreCase)))
{
if (vi.i > previdx)
{
double hdiff = vi.v.EventTimeUTC.Subtract(he.EventTimeUTC).TotalHours;
if (hdiff >= -2 && hdiff <= 2 && hdiff == Math.Floor(hdiff))
{
if (vi.v.System.id_edsm <= 0)
{
vi.v.System.id_edsm = 0;
mainForm.history.FillEDSM(vi.v);
}
if (vi.v.System.id_edsm <= 0 || vi.v.System.id_edsm == he.System.id_edsm)
{
index = vi.i;
break;
}
}
}
}
}
if (index < 0)
{
toadd.Add(he);
}
else
{
previdx = index;
}
}
if (toadd.Count > 0) // if we have any, we can add
{
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDDiscoveryForm.EDDConfig.CurrentCommander.Nr;
tlu.Add(); // Add to Database
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
foreach (HistoryEntry he in toadd)
{
EDDiscovery.EliteDangerous.JournalEntry je =
EDDiscovery.EliteDangerous.JournalEntry.CreateFSDJournalEntry(tlu.id, tlu.CommanderId.Value, he.EventTimeUTC,
he.System.name, he.System.x, he.System.y, he.System.z,
_defmapcolour, (int)EDDiscovery.EliteDangerous.SyncFlags.EDSM);
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", je.EventTimeUTC, he.System.name));
je.Add(cn);
}
}
if (OnDownloadedSystems != null)
{
OnDownloadedSystems();
}
mainForm.LogLine(string.Format("EDSM downloaded {0} systems", toadd.Count));
}
}
mainForm.LogLine("EDSM sync Done");
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message);
mainForm.LogLineHighlight("EDSM sync Exception " + ex.Message);
}
}
public void Sync()
{
try
{
EDSMClass edsm = new EDSMClass();
edsm.apiKey = EDDiscoveryForm.EDDConfig.CurrentCommander.APIKey;
edsm.commanderName = EDDiscoveryForm.EDDConfig.CurrentCommander.Name;
//string comments = edsm.GetComments(new DateTime(2015, 1, 1));
List <VisitedSystemsClass> log;
List <SystemNoteClass> notes;
int ret = edsm.GetLogs(new DateTime(2011, 1, 1), out log);
int nret = edsm.GetComments(new DateTime(2011, 1, 1), out notes);
if (log == null)
{
log = new List <VisitedSystemsClass>();
}
if (_syncTo)
{
// Send Unsynced system to EDSM.
List <VisitedSystemsClass> systems = (from s in mainForm.VisitedSystems where s.EDSM_sync == false && s.Commander == EDDiscoveryForm.EDDConfig.CurrentCommander.Nr select s).ToList <VisitedSystemsClass>();
mainForm.LogLine("EDSM: Sending " + systems.Count.ToString() + " flightlog entries");
foreach (var system in systems)
{
if (Exit)
{
running = false;
return;
}
if (system.EDSM_sync == false)
{
// check if it exist in EDSM
VisitedSystemsClass ps2 = (from c in log where c.Name == system.Name && c.Time.Ticks == system.Time.Ticks select c).FirstOrDefault <VisitedSystemsClass>();
if (ps2 != null)
{
system.EDSM_sync = true;
system.Update();
}
else
{
SendTravelLog(edsm, system, mainForm);
}
}
}
}
TravelLogUnit tlu = null;
bool newsystem = false;
if (_syncFrom)
{
// Check for new systems from EDSM
foreach (var system in log)
{
VisitedSystemsClass ps2 = mainForm?.VisitedSystems == null ? null :
(from c in mainForm.VisitedSystems where c.Name == system.Name && c.Time.Ticks == system.Time.Ticks select c).FirstOrDefault <VisitedSystemsClass>();
if (ps2 == null) // Add to local DB...
{
if (tlu == null) // If we dontt have a travellogunit yet then create it.
{
tlu = new TravelLogUnit();
tlu.type = 2; // EDSM
tlu.Path = "https://www.edsm.net/api-logs-v1/get-logs";
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Add(); // Add to Database
}
VisitedSystemsClass vs = new VisitedSystemsClass();
vs.Source = tlu.id;
vs.Unit = tlu.Name;
vs.Name = system.Name;
vs.Time = system.Time;
vs.MapColour = _defmapcolour;
vs.EDSM_sync = true;
vs.Commander = EDDiscoveryForm.EDDConfig.CurrentCommander.Nr;
vs.Add(); // Add to DB;
System.Diagnostics.Trace.WriteLine("New from EDSM");
newsystem = true;
}
}
// Sync comments from EDSM
foreach (var note in notes)
{
SystemNoteClass dbnote = SystemNoteClass.GetSystemNoteClass(note.Name.ToLower());
if (dbnote != null) // if there..
{
if (note.Time > dbnote.Time)
{
dbnote.Time = note.Time;
dbnote.Note = note.Note;
dbnote.Update();
}
}
else
{
note.Add();
}
}
}
mainForm.LogLine("EDSM sync Done");
if (newsystem)
{
OnNewEDSMTravelLog(this);
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message);
mainForm.LogLineHighlight("EDSM sync Exception " + ex.Message);
}
}