public static void TranferVisitedSystemstoJournalTable() // DONE purposely without using any VisitedSystem code.. so we can blow it away later.
{
List <Object[]> ehl = new List <Object[]>();
Dictionary <string, Dictionary <string, double> > dists = new Dictionary <string, Dictionary <string, double> >(StringComparer.CurrentCultureIgnoreCase);
List <EDDiscovery2.DB.TravelLogUnit> tlus = EDDiscovery2.DB.TravelLogUnit.GetAll().Where(t => t.type == 1).ToList();
using (SQLiteConnectionOld conn = new SQLiteConnectionOld())
{
// 0 1 2
using (DbCommand cmd = conn.CreateCommand("SELECT NameA, NameB, Dist FROM Distances WHERE Status >= 1")) // any distance pairs okay
{
using (DbDataReader reader = cmd.ExecuteReader())
{
while (reader.Read())
{
object[] vals = new object[3];
reader.GetValues(vals);
string namea = (string)vals[0];
string nameb = (string)vals[1];
double dist = (double)vals[2];
if (!dists.ContainsKey(namea))
{
dists[namea] = new Dictionary <string, double>(StringComparer.CurrentCultureIgnoreCase);
}
dists[namea][nameb] = dist;
if (!dists.ContainsKey(nameb))
{
dists[nameb] = new Dictionary <string, double>(StringComparer.CurrentCultureIgnoreCase);
}
dists[nameb][namea] = dist;
}
}
}
int olddbver = SQLiteConnectionOld.GetSettingInt("DBVer", 1);
if (olddbver < 7) // 2.5.2
{
System.Diagnostics.Trace.WriteLine("Database too old - unable to migrate travel log");
return;
}
string query;
if (olddbver < 8) // 2.5.6
{
query = "Select Name,Time,Unit,Commander,edsm_sync, -65536 AS Map_colour, NULL AS X, NULL AS Y, NULL AS Z, NULL as id_edsm_assigned From VisitedSystems Order By Time";
}
else if (olddbver < 14) // 3.2.1
{
query = "Select Name,Time,Unit,Commander,edsm_sync,Map_colour, NULL AS X, NULL AS Y, NULL AS Z, NULL as id_edsm_assigned From VisitedSystems Order By Time";
}
else if (olddbver < 18) // 4.0.2
{
query = "Select Name,Time,Unit,Commander,edsm_sync,Map_colour,X,Y,Z, NULL AS id_edsm_assigned From VisitedSystems Order By Time";
}
else
{
// 0 1 2 3 4 5 6 7 8 9
query = "Select Name,Time,Unit,Commander,edsm_sync,Map_colour,X,Y,Z,id_edsm_assigned From VisitedSystems Order By Time";
}
using (DbCommand cmd = conn.CreateCommand(query))
{
using (DbDataReader reader = cmd.ExecuteReader())
{
string prev = "";
while (reader.Read())
{
Object[] array = new Object[17];
reader.GetValues(array); // love this call.
string tluname = (string)array[2]; // 2 is in terms of its name.. look it up
if (tluname.StartsWith("EDSM-")) // Don't migrate the entries that were synced from EDSM
{ // We can sync them from EDSM later.
continue;
}
EDDiscovery2.DB.TravelLogUnit tlu = tlus.Find(x => x.Name.Equals(tluname, StringComparison.InvariantCultureIgnoreCase));
array[15] = (tlu != null) ? (long)tlu.id : 0; // even if we don't find it, tlu may be screwed up, still want to import
array[16] = null;
if (prev.Length > 0 && dists.ContainsKey((string)array[0]))
{
Dictionary <string, double> _dists = dists[(string)array[0]];
if (_dists.ContainsKey(prev))
{
array[16] = _dists[prev];
}
}
ehl.Add(array);
prev = (string)array[0];
}
}
}
}
using (SQLiteConnectionUser conn = new SQLiteConnectionUser(utc: true))
{
using (DbTransaction txn = conn.BeginTransaction())
{
foreach (Object[] array in ehl)
{
using (DbCommand cmd = conn.CreateCommand(
"Insert into JournalEntries (TravelLogId,CommanderId,EventTypeId,EventType,EventTime,EventData,EdsmId,Synced) " +
"values (@tli,@cid,@eti,@et,@etime,@edata,@edsmid,@synced)", txn))
{
cmd.AddParameterWithValue("@tli", (long)array[15]);
cmd.AddParameterWithValue("@cid", (long)array[3]);
cmd.AddParameterWithValue("@eti", EDDiscovery.EliteDangerous.JournalTypeEnum.FSDJump);
cmd.AddParameterWithValue("@et", "FSDJump");
JObject je = new JObject();
DateTime eventtime = DateTime.SpecifyKind((DateTime)array[1], DateTimeKind.Local).ToUniversalTime();
je["timestamp"] = eventtime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'");
je["event"] = "FSDJump";
je["StarSystem"] = ((string)array[0]);
if (System.DBNull.Value != array[6] && System.DBNull.Value != array[7] && System.DBNull.Value != array[8])
{
je["StarPos"] = new JArray()
{
array[6], array[7], array[8]
};
}
if (array[16] != null)
{
je["JumpDist"] = (double)array[16];
}
je["EDDMapColor"] = ((long)array[5]);
cmd.AddParameterWithValue("@etime", eventtime);
cmd.AddParameterWithValue("@edata", je.ToString()); // order number - look at the dbcommand above
long edsmid = 0;
if (System.DBNull.Value != array[9])
{
edsmid = (long)array[9];
}
cmd.AddParameterWithValue("@edsmid", edsmid); // order number - look at the dbcommand above
cmd.AddParameterWithValue("@synced", ((bool)array[4] == true) ? 1 : 0);
SQLiteDBClass.SQLNonQueryText(conn, cmd);
}
}
txn.Commit();
}
}
}
public static void ParseFiles(string datapath, out string error, int defaultMapColour, Func<bool> cancelRequested, Action<int, string> updateProgress, bool forceReload = false, Dictionary<string, NetLogFileReader> netlogreaders = null, int currentcmdrid = -1)
{
error = null;
if (datapath == null)
{
error = "Netlog directory not set!";
return;
}
if (!Directory.Exists(datapath)) // if logfiles directory is not found
{
error = "Netlog directory is not present!";
return;
}
if (netlogreaders == null)
{
netlogreaders = new Dictionary<string, NetLogFileReader>();
}
if (currentcmdrid < 0)
{
currentcmdrid = EDDConfig.Instance.CurrentCmdrID;
}
// TLUs
List<TravelLogUnit> tlus = TravelLogUnit.GetAll();
Dictionary<string, TravelLogUnit> netlogtravelogUnits = tlus.Where(t => t.type == 1).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
Dictionary<long, string> travellogunitid2name = netlogtravelogUnits.Values.ToDictionary(t => t.id, t => t.Name);
Dictionary<string, List<JournalLocOrJump>> vsc_lookup = JournalEntry.GetAll().OfType<JournalLocOrJump>().GroupBy(v => v.TLUId).Where(g => travellogunitid2name.ContainsKey(g.Key)).ToDictionary(g => travellogunitid2name[g.Key], g => g.ToList());
// list of systems in journal, sorted by time
List<JournalLocOrJump> vsSystemsEnts = JournalEntry.GetAll(currentcmdrid).OfType<JournalLocOrJump>().OrderBy(j => j.EventTimeUTC).ToList();
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(datapath, "netLog.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List<NetLogFileReader> readersToUpdate = new List<NetLogFileReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, netlogtravelogUnits, vsc_lookup, netlogreaders);
if (!netlogtravelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
netlogtravelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = reader;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
for (int i = 0; i < readersToUpdate.Count; i++)
{
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
int ji = 0;
NetLogFileReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (JObject jo in reader.ReadSystems(cancelRequested, currentcmdrid))
{
jo["EDDMapColor"] = defaultMapColour;
JournalLocOrJump je = new JournalFSDJump(jo)
{
TLUId = (int)reader.TravelLogUnit.id,
CommanderId = currentcmdrid,
};
while (ji < vsSystemsEnts.Count && vsSystemsEnts[ji].EventTimeUTC < je.EventTimeUTC)
{
ji++; // move to next entry which is bigger in time or equal to ours.
}
JournalLocOrJump prev = (ji > 0 && (ji - 1) < vsSystemsEnts.Count) ? vsSystemsEnts[ji - 1] : null;
JournalLocOrJump next = ji < vsSystemsEnts.Count ? vsSystemsEnts[ji] : null;
bool previssame = (prev != null && prev.StarSystem.Equals(je.StarSystem, StringComparison.CurrentCultureIgnoreCase) && (!prev.HasCoordinate || !je.HasCoordinate || (prev.StarPos - je.StarPos).LengthSquared < 0.01));
bool nextissame = (next != null && next.StarSystem.Equals(je.StarSystem, StringComparison.CurrentCultureIgnoreCase) && (!next.HasCoordinate || !je.HasCoordinate || (next.StarPos - je.StarPos).LengthSquared < 0.01));
// System.Diagnostics.Debug.WriteLine("{0} {1} {2}", ji, vsSystemsEnts[ji].EventTimeUTC, je.EventTimeUTC);
if (!(previssame || nextissame))
{
je.Add(cn, tn);
System.Diagnostics.Debug.WriteLine("Add {0} {1}", je.EventTimeUTC, je.EventDataString);
}
}
tn.Commit();
reader.TravelLogUnit.Update();
}
if (updateProgress != null)
{
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
}
}
}
}
public void ParseJournalFiles(Func<bool> cancelRequested, Action<int, string> updateProgress, bool forceReload = false)
{
System.Diagnostics.Trace.WriteLine("Scanned "+ m_watcherfolder);
Dictionary<string, TravelLogUnit> m_travelogUnits = TravelLogUnit.GetAll().Where(t => (t.type & 0xFF) == 3).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(m_watcherfolder, "Journal.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List<EDJournalReader> readersToUpdate = new List<EDJournalReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, m_travelogUnits);
if (!m_travelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
m_travelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.type = 3;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = reader;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
for (int i = 0; i < readersToUpdate.Count; i++)
{
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
EDJournalReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
List<JournalEntry> entries = reader.ReadJournalLog().ToList(); // this may create new commanders, and may write to the TLU db
ILookup<DateTime, JournalEntry> existing = JournalEntry.GetAllByTLU(reader.TravelLogUnit.id).ToLookup(e => e.EventTimeUTC);
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (JournalEntry je in entries)
{
if (!existing[je.EventTimeUTC].Any(e => JournalEntry.AreSameEntry(je, e)))
{
System.Diagnostics.Trace.WriteLine(string.Format("Write Journal to db {0} {1}", je.EventTimeUTC, je.EventTypeStr));
je.Add(cn, tn);
}
}
tn.Commit();
}
reader.TravelLogUnit.Update(cn);
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
lastnfi = reader;
}
}
updateProgress(-1, "");
}
public List<JournalEntry> ScanForNewEntries()
{
var entries = new List<JournalEntry>();
int netlogpos = 0;
EDJournalReader nfi = null;
try
{
string filename = null;
if (m_netLogFileQueue.TryDequeue(out filename)) // if a new one queued, we swap to using it
{
nfi = OpenFileReader(new FileInfo(filename));
lastnfi = nfi;
System.Diagnostics.Trace.WriteLine(string.Format("Change in file, scan {0}", lastnfi.FileName));
}
else if (ticksNoActivity >= 30 && (lastnfi == null || (!File.Exists(lastnfi.FileName) || lastnfi.filePos >= new FileInfo(lastnfi.FileName).Length)))
{
if (lastnfi == null)
{
Trace.Write($"No last file - scanning for journals");
}
else if (!File.Exists(lastnfi.FileName))
{
Trace.WriteLine($"File {lastnfi.FileName} not found - scanning for journals");
}
else
{
Trace.WriteLine($"No activity on {lastnfi.FileName} for 60 seconds ({lastnfi.filePos} >= {new FileInfo(lastnfi.FileName).Length} - scanning for new journals");
}
HashSet<string> tlunames = new HashSet<string>(TravelLogUnit.GetAllNames());
string[] filenames = Directory.EnumerateFiles(m_watcherfolder, "Journal.*.log", SearchOption.AllDirectories)
.Select(s => new { name = Path.GetFileName(s), fullname = s })
.Where(s => !tlunames.Contains(s.name))
.OrderBy(s => s.name)
.Select(s => s.fullname)
.ToArray();
ticksNoActivity = 0;
foreach (var name in filenames)
{
nfi = OpenFileReader(new FileInfo(name));
lastnfi = nfi;
break;
}
}
else
{
nfi = lastnfi;
}
ticksNoActivity++;
if (nfi != null)
{
if (nfi.TravelLogUnit.id == 0)
{
nfi.TravelLogUnit.type = 3;
nfi.TravelLogUnit.Add();
}
netlogpos = nfi.TravelLogUnit.Size;
List<JournalEntry> ents = nfi.ReadJournalLog().ToList();
if (ents.Count > 0)
{
using (SQLiteConnectionUser cn = new SQLiteConnectionUser(utc: true))
{
using (DbTransaction txn = cn.BeginTransaction())
{
ents = ents.Where(je => JournalEntry.FindEntry(je).Count == 0).ToList();
foreach (JournalEntry je in ents)
{
entries.Add(je);
je.Add(cn, txn);
ticksNoActivity = 0;
}
nfi.TravelLogUnit.Update(cn);
txn.Commit();
}
}
}
}
return entries;
}
catch (Exception ex)
{
// Revert and re-read the failed entries
if (nfi != null && nfi.TravelLogUnit != null)
{
nfi.TravelLogUnit.Size = netlogpos;
}
System.Diagnostics.Trace.WriteLine("Net tick exception : " + ex.Message);
System.Diagnostics.Trace.WriteLine(ex.StackTrace);
throw;
}
}
// called during start up and if refresh history is pressed
public List<VisitedSystemsClass> ParseFiles(out string error, int defaultMapColour, Func<bool> cancelRequested, Action<int, string> updateProgress, bool forceReload = false)
{
error = null;
string datapath = GetNetLogPath();
if (datapath == null)
{
error = "Netlog directory not found!" + Environment.NewLine + "Specify location in settings tab";
return null;
}
if (!Directory.Exists(datapath)) // if logfiles directory is not found
{
error = "Netlog directory is not present!" + Environment.NewLine + "Specify location in settings tab";
return null;
}
List<VisitedSystemsClass> vsSystemsList = VisitedSystemsClass.GetAll(EDDConfig.Instance.CurrentCmdrID);
List<VisitedSystemsClass> visitedSystems = new List<VisitedSystemsClass>();
Dictionary<string, TravelLogUnit> m_travelogUnits = TravelLogUnit.GetAll().Where(t => t.type == 1).GroupBy(t => t.Name).Select(g => g.First()).ToDictionary(t => t.Name);
Dictionary<string, List<VisitedSystemsClass>> vsc_lookup = VisitedSystemsClass.GetAll().GroupBy(v => v.Unit).ToDictionary(g => g.Key, g => g.ToList());
if (vsSystemsList != null)
{
foreach (VisitedSystemsClass vs in vsSystemsList)
{
if (visitedSystems.Count == 0)
visitedSystems.Add(vs);
else if (!visitedSystems.Last<VisitedSystemsClass>().Name.Equals(vs.Name)) // Avoid duplicate if times exist in same system from different files.
visitedSystems.Add(vs);
else
{
VisitedSystemsClass vs2 = (VisitedSystemsClass)visitedSystems.Last<VisitedSystemsClass>();
if (vs2.id != vs.id)
{
vs.Commander = -2; // Move to dupe user
vs.Update();
}
}
}
}
// order by file write time so we end up on the last one written
FileInfo[] allFiles = Directory.EnumerateFiles(datapath, "netLog.*.log", SearchOption.AllDirectories).Select(f => new FileInfo(f)).OrderBy(p => p.LastWriteTime).ToArray();
List<NetLogFileReader> readersToUpdate = new List<NetLogFileReader>();
for (int i = 0; i < allFiles.Length; i++)
{
FileInfo fi = allFiles[i];
var reader = OpenFileReader(fi, m_travelogUnits, vsc_lookup);
if (!m_travelogUnits.ContainsKey(reader.TravelLogUnit.Name))
{
m_travelogUnits[reader.TravelLogUnit.Name] = reader.TravelLogUnit;
reader.TravelLogUnit.Add();
}
if (!netlogreaders.ContainsKey(reader.TravelLogUnit.Name))
{
netlogreaders[reader.TravelLogUnit.Name] = lastnfi;
}
if (forceReload)
{
// Force a reload of the travel log
reader.TravelLogUnit.Size = 0;
}
if (reader.filePos != fi.Length || i == allFiles.Length - 1) // File not already in DB, or is the last one
{
readersToUpdate.Add(reader);
}
}
using (SQLiteConnectionUser cn = new SQLiteConnectionUser())
{
for (int i = 0; i < readersToUpdate.Count; i++)
{
NetLogFileReader reader = readersToUpdate[i];
updateProgress(i * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
using (DbTransaction tn = cn.BeginTransaction())
{
foreach (VisitedSystemsClass ps in reader.ReadSystems(cancelRequested))
{
ps.EDSM_sync = false;
ps.MapColour = defaultMapColour;
ps.Commander = EDDConfig.Instance.CurrentCmdrID;
ps.Add(cn, tn);
visitedSystems.Add(ps);
}
reader.TravelLogUnit.Update(cn, tn);
tn.Commit();
}
if (updateProgress != null)
{
updateProgress((i + 1) * 100 / readersToUpdate.Count, reader.TravelLogUnit.Name);
}
lastnfi = reader;
}
}
return visitedSystems;
}