public void ProcessDetectedNewFiles(List <EDJournalReader> readersToUpdate, Action <int, string> updateProgress,
Action <JournalEntry> fireback = null)
{
//System.Diagnostics.Trace.WriteLine(BaseUtils.AppTicks.TickCountLap("PJF"), "Ready to update");
for (int i = 0; i < readersToUpdate.Count; i++)
{
EDJournalReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
//System.Diagnostics.Trace.WriteLine(BaseUtils.AppTicks.TickCountLap("PJF"), i + " read ");
reader.ReadJournal(out List <JournalReaderEntry> entries, out List <UIEvent> uievents, historyrefreshparsing: true, resetOnError: true); // this may create new commanders, and may write to the TLU db
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
if (entries.Count > 0)
{
if (fireback != null)
{
foreach (JournalReaderEntry jre in entries)
{
fireback(jre.JournalEntry);
}
}
else
{
ILookup <DateTime, JournalEntry> existing = JournalEntry.GetAllByTLU(reader.TravelLogUnit.id, cn.Connection).ToLookup(e => e.EventTimeUTC);
//System.Diagnostics.Trace.WriteLine(BaseUtils.AppTicks.TickCountLap("PJF"), i + " into db");
using (DbTransaction tn = cn.Connection.BeginTransaction())
{
foreach (JournalReaderEntry jre in entries)
{
JObject jsonofentry = jre.Json;
if (!existing[jre.JournalEntry.EventTimeUTC].Any(e => JournalEntry.AreSameEntry(jre.JournalEntry, e, cn.Connection, ent1jo: jsonofentry)))
{
jre.JournalEntry.Add(jsonofentry, cn.Connection, tn);
//System.Diagnostics.Trace.WriteLine(string.Format("Write Journal to db {0} {1}", jre.JournalEntry.EventTimeUTC, jre.JournalEntry.EventTypeStr));
}
}
tn.Commit();
}
}
}
reader.TravelLogUnit.Update(cn.Connection);
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
lastnfi = reader;
});
}
updateProgress(-1, "");
}
public NetLogFileReader(TravelLogUnit tlu, List <JournalLocOrJump> vsclist = null) : base(tlu)
{
if (vsclist != null)
{
systems = vsclist;
}
else
{
systems = JournalEntry.GetAllByTLU(tlu.id).OfType <JournalLocOrJump>().ToList();
}
}
public NetLogFileReader(TravelLogUnit tlu, List <JournalLocOrJump> vsclist = null) : base(tlu)
{
if (vsclist != null)
{
systems = vsclist;
}
else
{
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
systems = JournalEntry.GetAllByTLU(tlu.id, cn.Connection).OfType <JournalLocOrJump>().ToList();
});
}
}
public void ParseJournalFiles(Func <bool> cancelRequested, Action <int, string> updateProgress, bool forceReload = false)
{
System.Diagnostics.Trace.WriteLine("Scanned " + WatcherFolder);
Dictionary <string, TravelLogUnit> m_travelogUnits = TravelLogUnit.GetAll().Where(t => (t.type & 0xFF) == 3).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(WatcherFolder, journalfilematch, SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List <EDJournalReader> readersToUpdate = new List <EDJournalReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, m_travelogUnits); // open it
if (!m_travelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
m_travelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.type = 3;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = reader;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
for (int i = 0; i < readersToUpdate.Count; i++)
{
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
EDJournalReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
List <JournalReaderEntry> entries = reader.ReadJournalLog(true).ToList(); // this may create new commanders, and may write to the TLU db
ILookup <DateTime, JournalEntry> existing = JournalEntry.GetAllByTLU(reader.TravelLogUnit.id).ToLookup(e => e.EventTimeUTC);
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (JournalReaderEntry jre in entries)
{
if (!existing[jre.JournalEntry.EventTimeUTC].Any(e => JournalEntry.AreSameEntry(jre.JournalEntry, e, ent1jo: jre.Json)))
{
jre.JournalEntry.Add(jre.Json, cn, tn);
System.Diagnostics.Trace.WriteLine(string.Format("Write Journal to db {0} {1}", jre.JournalEntry.EventTimeUTC, jre.JournalEntry.EventTypeStr));
}
}
tn.Commit();
}
reader.TravelLogUnit.Update(cn);
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
lastnfi = reader;
}
}
updateProgress(-1, "");
}
// given a list of files to reparse, read them and store to db or fire them back (and set firebacklastn to make it work)
public void ProcessDetectedNewFiles(List <EDJournalReader> readersToUpdate, Action <int, string> updateProgress,
Action <JournalEntry, int, int, int, int> fireback = null, int firebacklastn = 0)
{
for (int i = 0; i < readersToUpdate.Count; i++)
{
EDJournalReader reader = readersToUpdate[i];
List <JournalEntry> entries = new List <JournalEntry>();
List <UIEvent> uievents = new List <UIEvent>();
bool readanything = reader.ReadJournal(entries, uievents, historyrefreshparsing: true); // this may create new commanders, and may write to the TLU
if (fireback != null)
{
if (readanything) // need to update TLU pos if read anything
{
reader.TravelLogUnit.Update();
}
if (i >= readersToUpdate.Count - firebacklastn) // if within fireback window
{
for (int e = 0; e < entries.Count; e++)
{
//System.Diagnostics.Debug.WriteLine("Fire {0} {1} {2} {3} {4} {5}", entries[e].CommanderId, i, readersToUpdate.Count, e, entries.Count, entries[e].EventTypeStr );
fireback(entries[e], i, readersToUpdate.Count, e, entries.Count);
}
}
}
else
{
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
// only lookup TLUs if there is actually anything to compare against
ILookup <DateTime, JournalEntry> existing = entries.Count > 0 ? JournalEntry.GetAllByTLU(reader.ID, cn.Connection).ToLookup(e => e.EventTimeUTC) : null;
//System.Diagnostics.Trace.WriteLine(BaseUtils.AppTicks.TickCountLap("PJF"), i + " into db");
using (DbTransaction tn = cn.Connection.BeginTransaction())
{
foreach (JournalEntry jre in entries)
{
//System.Diagnostics.Trace.WriteLine(string.Format("--- Check {0} {1} Existing {2} : {3}", jre.EventTimeUTC, jre.EventTypeStr, existing[jre.EventTimeUTC].Count(), jre.GetJson().ToString()));
if (!existing[jre.EventTimeUTC].Any(e => JournalEntry.AreSameEntry(jre, e, cn.Connection, ent1jo: jre.GetJson(cn.Connection, tn))))
{
//foreach (var x in existing[jre.EventTimeUTC]) { System.Diagnostics.Trace.WriteLine(string.Format(" passed vs {0} Deepequals {1}", x.GetJson().ToString(), x.GetJson().DeepEquals(jre.GetJson()))); }
BaseUtils.JSON.JObject jo = jre.GetJson(cn.Connection, tn);
jre.Add(jo, cn.Connection, tn);
//System.Diagnostics.Trace.WriteLine(string.Format("Write Journal to db {0} {1}", jre.EventTimeUTC, jre.EventTypeStr));
}
else
{
//System.Diagnostics.Trace.WriteLine(string.Format("Duplicate Journal to db {0} {1}", jre.EventTimeUTC, jre.EventTypeStr));
}
}
if (readanything)
{
reader.TravelLogUnit.Update(cn.Connection, tn);
}
tn.Commit();
}
});
}
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.FullName);
lastnfi = reader;
}
updateProgress(-1, "");
}
static public void ParseFiles(string datapath, out string error, int defaultMapColour,
Func <bool> cancelRequested, Action <int, string> updateProgress,
bool forceReload, int currentcmdrid)
{
error = null;
if (datapath == null)
{
error = "Netlog directory not set!";
return;
}
if (!Directory.Exists(datapath)) // if logfiles directory is not found
{
error = "Netlog directory is not present!";
return;
}
// list of systems in journal, sorted by time
List <JournalLocOrJump> vsSystemsEnts = JournalEntry.GetAll(currentcmdrid).OfType <JournalLocOrJump>().OrderBy(j => j.EventTimeUTC).ToList();
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(datapath, "netLog.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List <NetLogFileReader> readersToUpdate = new List <NetLogFileReader>();
List <TravelLogUnit> tlutoadd = new List <TravelLogUnit>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi.FullName, currentcmdrid);
if (reader.ID == 0) // if not present, add to commit add list
{
tlutoadd.Add(reader.TravelLogUnit);
}
if (forceReload) // Force a reload of the travel log
{
reader.Pos = 0;
}
if (reader.Pos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
if (tlutoadd.Count > 0) // now, on spinning rust, this takes ages for 600+ log files first time, so transaction it
{
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
using (DbTransaction txn = cn.Connection.BeginTransaction())
{
foreach (var tlu in tlutoadd)
{
tlu.Add(cn.Connection, txn);
}
txn.Commit();
}
});
}
for (int i = 0; i < readersToUpdate.Count; i++)
{
UserDatabase.Instance.ExecuteWithDatabase(cn =>
{
int ji = 0;
NetLogFileReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.FullName);
var systems = JournalEntry.GetAllByTLU(reader.ID, cn.Connection).OfType <JournalLocOrJump>().ToList();
var last = systems.LastOrDefault(); // find last system recorded for this TLU, may be null if no systems..
using (DbTransaction tn = cn.Connection.BeginTransaction())
{
var ienum = reader.ReadSystems(last, cancelRequested, currentcmdrid);
System.Diagnostics.Debug.WriteLine("Scanning TLU " + reader.ID + " " + reader.FullName);
foreach (JObject jo in ienum)
{
jo["EDDMapColor"] = defaultMapColour;
JournalLocOrJump je = new JournalFSDJump(jo);
je.SetTLUCommander(reader.TravelLogUnit.ID, currentcmdrid);
while (ji < vsSystemsEnts.Count && vsSystemsEnts[ji].EventTimeUTC < je.EventTimeUTC)
{
ji++; // move to next entry which is bigger in time or equal to ours.
}
JournalLocOrJump prev = (ji > 0 && (ji - 1) < vsSystemsEnts.Count) ? vsSystemsEnts[ji - 1] : null;
JournalLocOrJump next = ji < vsSystemsEnts.Count ? vsSystemsEnts[ji] : null;
bool previssame = (prev != null && prev.StarSystem.Equals(je.StarSystem, StringComparison.CurrentCultureIgnoreCase) && (!prev.HasCoordinate || !je.HasCoordinate || (prev.StarPos - je.StarPos).LengthSquared < 0.01));
bool nextissame = (next != null && next.StarSystem.Equals(je.StarSystem, StringComparison.CurrentCultureIgnoreCase) && (!next.HasCoordinate || !je.HasCoordinate || (next.StarPos - je.StarPos).LengthSquared < 0.01));
// System.Diagnostics.Debug.WriteLine("{0} {1} {2}", ji, vsSystemsEnts[ji].EventTimeUTC, je.EventTimeUTC);
if (!(previssame || nextissame))
{
je.Add(jo, cn.Connection, tn);
System.Diagnostics.Debug.WriteLine("Add {0} {1}", je.EventTimeUTC, jo.ToString());
}
}
reader.TravelLogUnit.Update(cn.Connection, tn);
tn.Commit();
}
if (updateProgress != null)
{
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.FullName);
}
});
}
}
