private static bool TryCreateNewUpdate(SectorCache cache, TableWriteData data, bool tablesareempty, ref DateTime maxdate, ref int nextsectorid, out Sector t, bool makenew = false)
{
if (data.edsm.date > maxdate) // for all, record last recorded date processed
{
maxdate = data.edsm.date;
}
Sector prev = null;
t = null;
if (!cache.SectorNameCache.ContainsKey(data.classifier.SectorName)) // if unknown to cache
{
if (!tablesareempty && !makenew)
{
return(false);
}
cache.SectorNameCache[data.classifier.SectorName] = t = new Sector(data.classifier.SectorName, gridid: data.gridid); // make a sector of sectorname and with gridID n , id == -1
}
else
{
t = cache.SectorNameCache[data.classifier.SectorName]; // find the first sector of name
while (t != null && t.GId != data.gridid) // if GID of sector disagrees
{
prev = t; // go thru list
t = t.NextSector;
}
if (t == null) // still not got it, its a new one.
{
if (!tablesareempty && !makenew)
{
return(false);
}
prev.NextSector = t = new Sector(data.classifier.SectorName, gridid: data.gridid); // make a sector of sectorname and with gridID n , id == -1
}
}
if (t.Id == -1) // if unknown sector ID..
{
if (tablesareempty) // if tables are empty, we can just presume its id
{
t.Id = nextsectorid++; // insert the sector with the guessed ID
t.insertsec = true;
cache.SectorIDCache[t.Id] = t; // and cache
//System.Diagnostics.Debug.WriteLine("Made sector " + t.Name + ":" + t.GId);
}
}
if (t.edsmdatalist == null)
{
t.edsmdatalist = new List <TableWriteData>(5000);
}
t.edsmdatalist.Add(data); // add to list of systems to process for this sector
return(true);
}
// create a new entry for insert in the sector tables
private static void CreateNewUpdate(SectorCache cache, DbCommand selectSectorCmd, EDSMFileEntry d, int gid, bool tablesareempty, ref DateTime maxdate, ref int nextsectorid)
{
TableWriteData data = new TableWriteData()
{
edsm = d, classifier = new EliteNameClassifier(d.name), gridid = gid
};
CreateNewUpdate(cache, selectSectorCmd, data, tablesareempty, ref maxdate, ref nextsectorid);
}
// create a new entry for insert in the sector tables
private static void CreateNewUpdate(DbCommand selectSectorCmd, EDSMFileEntry d, int gid, bool tablesareempty, ref DateTime maxdate, ref int nextsectorid)
{
TableWriteData data = new TableWriteData()
{
edsm = d, classifier = new EliteNameClassifier(d.name), gridid = gid
};
if (d.date > maxdate) // for all, record last recorded date processed
{
maxdate = d.date;
}
Sector t = null, prev = null;
if (!sectornamecache.ContainsKey(data.classifier.SectorName)) // if unknown to cache
{
sectornamecache[data.classifier.SectorName] = t = new Sector(data.classifier.SectorName, gridid: data.gridid); // make a sector of sectorname and with gridID n , id == -1
}
else
{
t = sectornamecache[data.classifier.SectorName]; // find the first sector of name
while (t != null && t.GId != data.gridid) // if GID of sector disagrees
{
prev = t; // go thru list
t = t.NextSector;
}
if (t == null) // still not got it, its a new one.
{
prev.NextSector = t = new Sector(data.classifier.SectorName, gridid: data.gridid); // make a sector of sectorname and with gridID n , id == -1
}
}
if (t.Id == -1) // if unknown sector ID..
{
if (tablesareempty) // if tables are empty, we can just presume its id
{
t.Id = nextsectorid++; // insert the sector with the guessed ID
t.insertsec = true;
sectoridcache[t.Id] = t; // and cache
//System.Diagnostics.Debug.WriteLine("Made sector " + t.Name + ":" + t.GId);
}
else
{
selectSectorCmd.Parameters[0].Value = t.Name;
selectSectorCmd.Parameters[1].Value = t.GId;
using (DbDataReader reader = selectSectorCmd.ExecuteReader()) // find name:gid
{
if (reader.Read()) // if found name:gid
{
t.Id = (long)reader[0];
}
else
{
t.Id = nextsectorid++; // insert the sector with the guessed ID
t.insertsec = true;
}
sectoridcache[t.Id] = t; // and cache
// System.Diagnostics.Debug.WriteLine("Made sector " + t.Name + ":" + t.GId);
}
}
}
if (t.edsmdatalist == null)
{
t.edsmdatalist = new List <TableWriteData>(5000);
}
t.edsmdatalist.Add(data); // add to list of systems to process for this sector
}
private static void CreateNewUpdate(SectorCache cache, DbCommand selectSectorCmd, TableWriteData data, bool tablesareempty, ref DateTime maxdate, ref int nextsectorid)
{
TryCreateNewUpdate(cache, data, tablesareempty, ref maxdate, ref nextsectorid, out Sector t, true);
if (t.Id == -1) // if unknown sector ID..
{
selectSectorCmd.Parameters[0].Value = t.Name;
selectSectorCmd.Parameters[1].Value = t.GId;
using (DbDataReader reader = selectSectorCmd.ExecuteReader()) // find name:gid
{
if (reader.Read()) // if found name:gid
{
t.Id = (long)reader[0];
}
else
{
t.Id = nextsectorid++; // insert the sector with the guessed ID
t.insertsec = true;
}
cache.SectorIDCache[t.Id] = t; // and cache
// System.Diagnostics.Debug.WriteLine("Made sector " + t.Name + ":" + t.GId);
}
}
}
private static int ProcessBlock(SectorCache cache,
IEnumerator <JToken> enumerator,
bool[] grididallowed, // null = all, else grid bool value
bool tablesareempty,
string tablepostfix,
ref DateTime maxdate, // updated with latest date
ref int nextsectorid,
out bool jr_eof)
{
int recordstostore = 0;
DbCommand selectSectorCmd = null;
DateTime cpmaxdate = maxdate;
int cpnextsectorid = nextsectorid;
const int BlockSize = 10000;
int Limit = int.MaxValue;
var entries = new List <TableWriteData>();
jr_eof = false;
while (true)
{
if (!enumerator.MoveNext()) // get next token, if not, stop eof
{
jr_eof = true;
break;
}
JToken t = enumerator.Current;
if (t.IsObject) // if start of object..
{
EDSMFileEntry d = new EDSMFileEntry();
if (d.Deserialize(enumerator) && d.id >= 0 && d.name.HasChars() && d.z != int.MinValue) // if we have a valid record
{
int gridid = GridId.Id128(d.x, d.z);
if (grididallowed == null || (grididallowed.Length > gridid && grididallowed[gridid])) // allows a null or small grid
{
TableWriteData data = new TableWriteData()
{
edsm = d, classifier = new EliteNameClassifier(d.name), gridid = gridid
};
if (!TryCreateNewUpdate(cache, data, tablesareempty, ref cpmaxdate, ref cpnextsectorid, out Sector sector))
{
entries.Add(data);
}
recordstostore++;
}
}
if (--Limit == 0)
{
jr_eof = true;
break;
}
if (recordstostore >= BlockSize)
{
break;
}
}
}
SystemsDatabase.Instance.ExecuteWithDatabase(action: db =>
{
try
{
var cn = db.Connection;
selectSectorCmd = cn.CreateSelect("Sectors" + tablepostfix, "id", "name = @sname AND gridid = @gid", null,
new string[] { "sname", "gid" }, new DbType[] { DbType.String, DbType.Int32 });
foreach (var entry in entries)
{
CreateNewUpdate(cache, selectSectorCmd, entry, tablesareempty, ref cpmaxdate, ref cpnextsectorid);
}
}
finally
{
if (selectSectorCmd != null)
{
selectSectorCmd.Dispose();
}
}
});
maxdate = cpmaxdate;
nextsectorid = cpnextsectorid;
return(recordstostore);
}
private static int ProcessBlock(SectorCache cache,
JsonTextReader jr,
bool[] grididallowed, // null = all, else grid bool value
bool tablesareempty,
string tablepostfix,
ref DateTime maxdate, // updated with latest date
ref int nextsectorid,
ref bool jr_eof)
{
int recordstostore = 0;
DbCommand selectSectorCmd = null;
DateTime cpmaxdate = maxdate;
int cpnextsectorid = nextsectorid;
const int BlockSize = 10000;
int Limit = int.MaxValue;
var entries = new List <TableWriteData>();
while (jr_eof == false)
{
try
{
if (jr.Read())
{
if (jr.TokenType == JsonToken.StartObject)
{
EDSMFileEntry d = new EDSMFileEntry();
if (d.Deserialize(jr) && d.id >= 0 && d.name.HasChars() && d.z != int.MinValue) // if we have a valid record
{
int gridid = GridId.Id128(d.x, d.z);
if (grididallowed == null || (grididallowed.Length > gridid && grididallowed[gridid])) // allows a null or small grid
{
TableWriteData data = new TableWriteData()
{
edsm = d, classifier = new EliteNameClassifier(d.name), gridid = gridid
};
if (!TryCreateNewUpdate(cache, data, tablesareempty, ref cpmaxdate, ref cpnextsectorid, out Sector sector))
{
entries.Add(data);
}
recordstostore++;
}
}
if (--Limit == 0)
{
jr_eof = true;
break;
}
if (recordstostore >= BlockSize)
{
break;
}
}
}
else
{
jr_eof = true;
break;
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine("EDSM JSON file exception " + ex.ToString());
jr_eof = true; // stop read, but let it continue to finish this section
}
}
SystemsDatabase.Instance.ExecuteWithDatabase(action: db =>
{
try
{
var cn = db.Connection;
selectSectorCmd = cn.CreateSelect("Sectors" + tablepostfix, "id", "name = @sname AND gridid = @gid", null,
new string[] { "sname", "gid" }, new DbType[] { DbType.String, DbType.Int32 });
foreach (var entry in entries)
{
CreateNewUpdate(cache, selectSectorCmd, entry, tablesareempty, ref cpmaxdate, ref cpnextsectorid);
}
}
finally
{
if (selectSectorCmd != null)
{
selectSectorCmd.Dispose();
}
}
});
maxdate = cpmaxdate;
nextsectorid = cpnextsectorid;
return(recordstostore);
}
