private void ScanTickWorker(object sender, System.ComponentModel.DoWorkEventArgs e)
{
var worker = sender as System.ComponentModel.BackgroundWorker;
var entries = new List<JournalEntry>();
e.Result = entries;
int netlogpos = 0;
EDJournalReader nfi = null;
try
{
string filename = null;
if (m_netLogFileQueue.TryDequeue(out filename)) // if a new one queued, we swap to using it
{
nfi = OpenFileReader(new FileInfo(filename));
lastnfi = nfi;
System.Diagnostics.Trace.WriteLine(string.Format("Change in file, scan {0}", lastnfi.FileName));
}
else if (ticksNoActivity >= 30 && (lastnfi == null || (!File.Exists(lastnfi.FileName) || lastnfi.filePos >= new FileInfo(lastnfi.FileName).Length)))
{
if (lastnfi == null)
{
Trace.Write($"No last file - scanning for journals");
}
else if (!File.Exists(lastnfi.FileName))
{
Trace.WriteLine($"File {lastnfi.FileName} not found - scanning for journals");
}
else
{
Trace.WriteLine($"No activity on {lastnfi.FileName} for 60 seconds ({lastnfi.filePos} >= {new FileInfo(lastnfi.FileName).Length} - scanning for new journals");
}
HashSet<string> tlunames = new HashSet<string>(TravelLogUnit.GetAllNames());
string[] filenames = Directory.EnumerateFiles(m_watcherfolder, "Journal.*.log", SearchOption.AllDirectories)
.Select(s => new { name = Path.GetFileName(s), fullname = s })
.Where(s => !tlunames.Contains(s.name))
.OrderBy(s => s.name)
.Select(s => s.fullname)
.ToArray();
ticksNoActivity = 0;
foreach (var name in filenames)
{
nfi = OpenFileReader(new FileInfo(name));
lastnfi = nfi;
break;
}
}
else
{
nfi = lastnfi;
}
ticksNoActivity++;
if (nfi != null)
{
if (nfi.TravelLogUnit.id == 0)
{
nfi.TravelLogUnit.type = 3;
nfi.TravelLogUnit.Add();
}
netlogpos = nfi.TravelLogUnit.Size;
List<JournalEntry> ents = nfi.ReadJournalLog().ToList();
using (SQLiteConnectionUserUTC cn = new SQLiteConnectionUserUTC())
{
using (DbTransaction txn = cn.BeginTransaction())
{
foreach (JournalEntry je in ents)
{
entries.Add(je);
je.Add(cn, txn);
ticksNoActivity = 0;
}
txn.Commit();
}
}
nfi.TravelLogUnit.Update();
}
if (worker.CancellationPending)
{
e.Cancel = true;
}
}
catch (Exception ex)
{
// Revert and re-read the failed entries
if (nfi != null && nfi.TravelLogUnit != null)
{
nfi.TravelLogUnit.Size = netlogpos;
}
System.Diagnostics.Trace.WriteLine("Net tick exception : " + ex.Message);
System.Diagnostics.Trace.WriteLine(ex.StackTrace);
throw;
}
}
public void ParseJournalFiles(Func<bool> cancelRequested, Action<int, string> updateProgress, bool forceReload = false)
{
System.Diagnostics.Trace.WriteLine("Scanned "+ m_watcherfolder);
Dictionary<string, TravelLogUnit> m_travelogUnits = TravelLogUnit.GetAll().Where(t => (t.type & 0xFF) == 3).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(m_watcherfolder, "Journal.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List<EDJournalReader> readersToUpdate = new List<EDJournalReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, m_travelogUnits);
if (!m_travelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
m_travelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.type = 3;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = reader;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
using (SQLiteConnectionUserUTC cn = new SQLiteConnectionUserUTC())
{
for (int i = 0; i < readersToUpdate.Count; i++)
{
EDJournalReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
List<JournalEntry> entries = reader.ReadJournalLog().ToList(); // this may create new commanders, and may write to the TLU db
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (JournalEntry je in entries)
{
System.Diagnostics.Trace.WriteLine(string.Format("Write Journal to db {0} {1}", je.EventTimeUTC, je.EventTypeStr));
je.Add(cn, tn);
}
tn.Commit();
}
reader.TravelLogUnit.Update();
if (updateProgress != null)
{
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
}
lastnfi = reader;
}
}
}
private void Sync()
{
try
{
mainForm.LogLine("EDSM sync begin");
EDSMClass edsm = new EDSMClass();
List<HistoryEntry> hlfsdunsyncedlist = mainForm.history.FilterByNotEDSMSyncedAndFSD; // first entry is oldest
if ( _syncTo && hlfsdunsyncedlist.Count > 0 ) // send systems to edsm (verified with dates, 29/9/2016, utc throughout)
{
DateTime utcmin = hlfsdunsyncedlist[0].EventTimeUTC.AddDays(-1); // 1 days for margin ;-) only get them back to this date for speed..
mainForm.LogLine("EDSM: Sending " + hlfsdunsyncedlist.Count.ToString() + " flightlog entries");
List<HistoryEntry> edsmsystemlog = null;
edsm.GetLogs(utcmin, out edsmsystemlog); // always returns a log, time is in UTC as per HistoryEntry and JournalEntry
int edsmsystemssent = 0;
foreach (var he in hlfsdunsyncedlist)
{
if (Exit)
{
running = false;
return;
}
HistoryEntry ps2 = (from c in edsmsystemlog where c.System.name == he.System.name && c.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks select c).FirstOrDefault();
if (ps2 != null) // it did, just make sure EDSM sync flag is set..
{
he.SetEdsmSync();
}
else
{
string errmsg; // (verified with EDSM 29/9/2016)
// it converts to UTC inside the function, supply local for now
if ( edsm.SendTravelLog(he.System.name, he.EventTimeUTC, he.System.HasCoordinate, he.System.x, he.System.y, he.System.z, out errmsg) )
he.SetEdsmSync();
if (errmsg.Length > 0)
mainForm.LogLine(errmsg);
edsmsystemssent++;
}
}
mainForm.LogLine(string.Format("EDSM Systems sent {0}", edsmsystemssent));
}
// TBD Comments to edsm?
if ( _syncFrom ) // Verified ok with time 29/9/2016
{
var json = edsm.GetComments(new DateTime(2011, 1, 1));
if (json != null)
{
JObject msg = JObject.Parse(json);
int msgnr = msg["msgnum"].Value<int>();
JArray comments = (JArray)msg["comments"];
if (comments != null)
{
int commentsadded = 0;
foreach (JObject jo in comments)
{
string name = jo["system"].Value<string>();
string note = jo["comment"].Value<string>();
string utctime = jo["lastUpdate"].Value<string>();
DateTime localtime = DateTime.ParseExact(utctime, "yyyy-MM-dd HH:mm:ss",
CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime();
SystemNoteClass curnote = SystemNoteClass.GetNoteOnSystem(name);
if (curnote != null)
{
// curnote uses local time to store
if (localtime.Ticks > curnote.Time.Ticks) // if newer, add on (verified with EDSM 29/9/2016)
{
curnote.Note += ". EDSM: " + note;
curnote.Time = localtime;
curnote.Update();
commentsadded++;
}
}
else
{
curnote = new SystemNoteClass();
curnote.Note = note;
curnote.Time = localtime;
curnote.Name = name;
curnote.Journalid = 0;
curnote.Add();
commentsadded++;
}
}
mainForm.LogLine(string.Format("EDSM Comments downloaded/updated {0}", commentsadded));
}
}
}
if (_syncFrom ) // verified after struggle 29/9/2016
{
List<HistoryEntry> edsmsystemlog = null;
edsm.GetLogs(new DateTime(2011, 1, 1), out edsmsystemlog); // get the full list of systems
List<HistoryEntry> hlfsdlist = mainForm.history.FilterByFSD; // FSD jumps only
List<HistoryEntry> toadd = new List<HistoryEntry>();
foreach (HistoryEntry he in edsmsystemlog) // find out list of ones not present
{
if (hlfsdlist.FindIndex(x => x.System.name.Equals(he.System.name) && x.EventTimeUTC.Ticks == he.EventTimeUTC.Ticks) < 0)
{
toadd.Add(he);
}
}
if ( toadd.Count >0 ) // if we have any, we can add
{
TravelLogUnit tlu = new TravelLogUnit(); // need a tlu for it
tlu.type = 2; // EDSM
tlu.Name = "EDSM-" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
tlu.Size = 0;
tlu.Path = "EDSM";
tlu.CommanderId = EDDiscoveryForm.EDDConfig.CurrentCommander.Nr;
tlu.Add(); // Add to Database
using (SQLiteConnectionUserUTC cn = new SQLiteConnectionUserUTC())
{
foreach (HistoryEntry he in toadd)
{
EDDiscovery.EliteDangerous.JournalEntry je =
EDDiscovery.EliteDangerous.JournalEntry.CreateFSDJournalEntry(tlu.id, tlu.CommanderId.Value, he.EventTimeUTC,
he.System.name, he.System.x, he.System.y, he.System.z,
_defmapcolour, (int)EDDiscovery.EliteDangerous.SyncFlags.EDSM);
System.Diagnostics.Trace.WriteLine(string.Format("Add {0} {1}", je.EventTimeUTC, he.System.name));
je.Add(cn);
}
}
if (OnDownloadedSystems != null)
OnDownloadedSystems();
mainForm.LogLine(string.Format("EDSM downloaded {0} systems", toadd.Count));
}
}
mainForm.LogLine("EDSM sync Done");
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine("Exception ex:" + ex.Message);
mainForm.LogLineHighlight("EDSM sync Exception " + ex.Message);
}
}
// DONE purposely without using any VisitedSystem code.. so we can blow it away later.
public static void TranferVisitedSystemstoJournalTable()
{
List<Object[]> ehl = new List<Object[]>();
Dictionary<string, Dictionary<string, double>> dists = new Dictionary<string, Dictionary<string, double>>(StringComparer.CurrentCultureIgnoreCase);
List<EDDiscovery2.DB.TravelLogUnit> tlus = EDDiscovery2.DB.TravelLogUnit.GetAll().Where(t => t.type == 1).ToList();
using (SQLiteConnectionOld conn = new SQLiteConnectionOld())
{
// 0 1 2
using (DbCommand cmd = conn.CreateCommand("SELECT NameA, NameB, Dist FROM Distances WHERE Status >= 1")) // any distance pairs okay
{
using (DbDataReader reader = cmd.ExecuteReader())
{
while (reader.Read())
{
object[] vals = new object[3];
reader.GetValues(vals);
string namea = (string)vals[0];
string nameb = (string)vals[1];
double dist = (double)vals[2];
if (!dists.ContainsKey(namea))
{
dists[namea] = new Dictionary<string, double>(StringComparer.CurrentCultureIgnoreCase);
}
dists[namea][nameb] = dist;
if (!dists.ContainsKey(nameb))
{
dists[nameb] = new Dictionary<string, double>(StringComparer.CurrentCultureIgnoreCase);
}
dists[nameb][namea] = dist;
}
}
}
// 0 1 2 3 4 5 6 7 8 9
using (DbCommand cmd = conn.CreateCommand("Select Name,Time,Unit,Commander,edsm_sync,Map_colour,X,Y,Z,id_edsm_assigned From VisitedSystems Order By Time"))
{
using (DbDataReader reader = cmd.ExecuteReader())
{
string prev = "";
while (reader.Read())
{
Object[] array = new Object[17];
reader.GetValues(array); // love this call.
string tluname = (string)array[2]; // 2 is in terms of its name.. look it up
EDDiscovery2.DB.TravelLogUnit tlu = tlus.Find(x => x.Name.Equals(tluname, StringComparison.InvariantCultureIgnoreCase));
array[15] = (tlu != null) ? (long)tlu.id : 0; // even if we don't find it, tlu may be screwed up, still want to import
array[16] = null;
if (prev.Length>0 && dists.ContainsKey((string)array[0]))
{
Dictionary<string, double> _dists = dists[(string)array[0]];
if (_dists.ContainsKey(prev))
{
array[16] = _dists[prev];
}
}
ehl.Add(array);
prev = (string)array[0];
}
}
}
}
using (SQLiteConnectionUserUTC conn = new SQLiteConnectionUserUTC())
{
using (DbTransaction txn = conn.BeginTransaction())
{
foreach (Object[] array in ehl)
{
using (DbCommand cmd = conn.CreateCommand(
"Insert into JournalEntries (TravelLogId,CommanderId,EventTypeId,EventType,EventTime,EventData,EdsmId,Synced) " +
"values (@tli,@cid,@eti,@et,@etime,@edata,@edsmid,@synced)", txn))
{
cmd.AddParameterWithValue("@tli", (long)array[15]);
cmd.AddParameterWithValue("@cid", (long)array[3]);
cmd.AddParameterWithValue("@eti", EDDiscovery.EliteDangerous.JournalTypeEnum.FSDJump);
cmd.AddParameterWithValue("@et", "FSDJump");
cmd.AddParameterWithValue("@etime", (DateTime)array[1]);
JObject je = new JObject();
je["timestamp"] = DateTime.SpecifyKind((DateTime)array[1], DateTimeKind.Local).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'");
je["event"] = "FSDJump";
je["StarSystem"] = ((string)array[0]);
if (System.DBNull.Value != array[6] && System.DBNull.Value != array[7] && System.DBNull.Value != array[8])
{
je["StarPos"] = new JArray() {array[6], array[7], array[8] };
}
if (array[16] != null)
{
je["JumpDist"] = (double)array[16];
}
je["EDDMapColor"] = ((long)array[5]);
cmd.AddParameterWithValue("@edata", je.ToString()); // order number - look at the dbcommand above
long edsmid = 0;
if (System.DBNull.Value != array[9])
edsmid = (long)array[9];
cmd.AddParameterWithValue("@edsmid", edsmid); // order number - look at the dbcommand above
cmd.AddParameterWithValue("@synced", ((bool)array[4] == true) ? 1 : 0);
SQLiteDBClass.SQLNonQueryText(conn, cmd);
}
}
txn.Commit();
}
}
}
public static List<JournalEntry> ParseFiles(string datapath, out string error, int defaultMapColour, Func<bool> cancelRequested, Action<int, string> updateProgress, bool forceReload = false, Dictionary<string, NetLogFileReader> netlogreaders = null, int currentcmdrid = -1)
{
error = null;
if (datapath == null)
{
error = "Netlog directory not set!";
return null;
}
if (!Directory.Exists(datapath)) // if logfiles directory is not found
{
error = "Netlog directory is not present!";
return null;
}
if (netlogreaders == null)
{
netlogreaders = new Dictionary<string, NetLogFileReader>();
}
if (currentcmdrid < 0)
{
currentcmdrid = EDDConfig.Instance.CurrentCmdrID;
}
List<JournalLocOrJump> visitedSystems = new List<JournalLocOrJump>();
Dictionary<string, TravelLogUnit> m_travelogUnits = TravelLogUnit.GetAll().Where(t => t.type == 1).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
Dictionary<long, string> travellogunitid2name = m_travelogUnits.Values.ToDictionary(t => t.id, t => t.Name);
Dictionary<string, List<JournalLocOrJump>> vsc_lookup = JournalEntry.GetAll().OfType<JournalLocOrJump>().GroupBy(v => v.TLUId).Where(g => travellogunitid2name.ContainsKey(g.Key)).ToDictionary(g => travellogunitid2name[g.Key], g => g.ToList());
HashSet<long> journalids = new HashSet<long>(m_travelogUnits.Values.Select(t => t.id).ToList());
List<JournalLocOrJump> vsSystemsList = JournalEntry.GetAll(currentcmdrid).OfType<JournalLocOrJump>().Where(j => journalids.Contains(j.TLUId)).ToList();
if (vsSystemsList != null)
{
foreach (JournalLocOrJump vs in vsSystemsList)
{
if (visitedSystems.Count == 0)
visitedSystems.Add(vs);
else if (!visitedSystems.Last().StarSystem.Equals(vs.StarSystem, StringComparison.CurrentCultureIgnoreCase)) // Avoid duplicate if times exist in same system from different files.
visitedSystems.Add(vs);
}
}
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(datapath, "netLog.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List<NetLogFileReader> readersToUpdate = new List<NetLogFileReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, m_travelogUnits, vsc_lookup, netlogreaders);
if (!m_travelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
m_travelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = reader;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
using (SQLiteConnectionUserUTC cn = new SQLiteConnectionUserUTC())
{
for (int i = 0; i < readersToUpdate.Count; i++)
{
NetLogFileReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (JObject jo in reader.ReadSystems(cancelRequested, currentcmdrid))
{
jo["EDDMapColor"] = defaultMapColour;
JournalLocOrJump je = new JournalFSDJump(jo)
{
TLUId = (int)reader.TravelLogUnit.id,
CommanderId = currentcmdrid,
};
je.Add(cn, tn);
visitedSystems.Add(je);
}
tn.Commit();
reader.TravelLogUnit.Update();
}
if (updateProgress != null)
{
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
}
}
}
return visitedSystems.ToList<JournalEntry>();
}
