private List <Record> Consolidate(List <Record> baseList, List <Alteration> alterations, List <Record> deletions)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
Dictionary <long, Record> records = new Dictionary <long, Record>();
foreach (var x in baseList)
{
records[x.Id] = x;
}
#if FULL_CONSOLIDATE_DEBUG
Printer.PrintMessage("Initial Snapshot Contains: ");
foreach (var x in baseList)
{
Printer.PrintMessage("Initial: ({2}) {0} - {1}", x.CanonicalName, x.UniqueIdentifier, x.Id);
}
#endif
List <long> pending = new List <long>();
CacheRecords(alterations.Select(x => x.NewRecord).Where(x => x.HasValue).Select(x => x.Value));
foreach (var x in alterations.Select(x => x).Reverse())
{
Objects.Record rec = null;
switch (x.Type)
{
case AlterationType.Add:
case AlterationType.Copy:
{
var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
Printer.PrintMessage("Add: ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
records[record.Id] = record;
break;
}
case AlterationType.Move:
{
var record = GetCachedRecord(x.NewRecord.Value);
if (deletions != null)
{
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
deletions.Add(rec);
}
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Move (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
Printer.PrintMessage("Move (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception("Consistency constraint invalid!");
}
records[record.Id] = record;
break;
}
case AlterationType.Update:
{
var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Update (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
Printer.PrintMessage("Update (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
if (records.ContainsKey(record.Id))
{
Printer.PrintMessage(" (error, already present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception();
}
records[record.Id] = record;
break;
}
case AlterationType.Discard:
break;
case AlterationType.Delete:
if (deletions != null)
{
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
deletions.Add(rec);
}
#if FULL_CONSOLIDATE_DEBUG
rec = Get <Objects.Record>(x.PriorRecord);
rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
Printer.PrintMessage("Delete: ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
if (!records.ContainsKey(rec.Id))
{
Printer.PrintMessage(" (error, not present)");
}
#endif
if (!records.Remove(x.PriorRecord.Value))
{
throw new Exception("Consistency constraint invalid!");
}
break;
default:
throw new Exception();
}
}
var result = records.Select(x => x.Value).ToList();
#if DEBUG
HashSet <string> namecheck = new HashSet <string>();
foreach (var x in result)
{
if (namecheck.Contains(x.CanonicalName))
{
throw new Exception("Inconsistency in internal state!");
}
namecheck.Add(x.CanonicalName);
}
#endif
return(result);
}
private void RunConsistencyCheck()
{
Printer.PrintMessage(" - Upgrading database - running full consistency check.");
var objNames = Table <ObjectName>().ToList();
Dictionary <long, long> nameMapping = new Dictionary <long, long>();
Dictionary <string, long> nameIndexes = new Dictionary <string, long>();
DropTable <ObjectName>();
DropTable <RecordRef>();
CreateTable <ObjectName>();
int duplicateObjs = 0;
foreach (var x in objNames)
{
if (nameIndexes.ContainsKey(x.CanonicalName))
{
nameMapping[x.NameId] = nameIndexes[x.CanonicalName];
duplicateObjs++;
}
else
{
ObjectName oname = new ObjectName()
{
CanonicalName = x.CanonicalName
};
Insert(oname);
nameMapping[x.NameId] = oname.NameId;
nameIndexes[x.CanonicalName] = oname.NameId;
}
}
foreach (var x in Table <Record>().ToList())
{
x.CanonicalNameId = nameMapping[x.CanonicalNameId];
Update(x);
}
Printer.PrintMessage(" - Cleaned {0} duplicate canonical names.", duplicateObjs);
CreateTable <RecordRef>();
Dictionary <Tuple <string, long, DateTime>, Record> records = new Dictionary <Tuple <string, long, DateTime>, Record>();
foreach (var x in Table <Objects.Record>().ToList())
{
var key = new Tuple <string, long, DateTime>(x.UniqueIdentifier, x.CanonicalNameId, x.ModificationTime);
if (records.ContainsKey(key))
{
var other = records[key];
Printer.PrintDiagnostics("Found duplicate records {0} ==> {1}", x.Id, other.Id);
Printer.PrintDiagnostics(" - UID: {0}", x.UniqueIdentifier);
Printer.PrintDiagnostics(" - Time: {0}", x.ModificationTime);
Printer.PrintDiagnostics(" - Name: {0}", Get <ObjectName>(x.CanonicalNameId).CanonicalName);
int updates = 0;
foreach (var s in Table <Alteration>().Where(z => z.PriorRecord == x.Id || z.NewRecord == x.Id))
{
if (s.NewRecord.HasValue && s.NewRecord.Value == x.Id)
{
s.NewRecord = other.Id;
}
if (s.PriorRecord.HasValue && s.PriorRecord.Value == x.Id)
{
s.PriorRecord = other.Id;
}
Update(s);
updates++;
}
Delete(x);
Printer.PrintDiagnostics("Deleted record and updated {0} links.", updates);
}
else
{
records[key] = x;
}
}
int missingParents = 0;
int bonusDeletions = 0;
HashSet <long> allRecordIDs = new HashSet <long>();
var allRecords = Table <Objects.Record>().ToList();
foreach (var x in allRecords)
{
allRecordIDs.Add(x.Id);
}
foreach (var x in allRecords)
{
if (x.Parent.HasValue && !allRecordIDs.Contains(x.Parent.Value))
{
missingParents++;
x.Parent = null;
Update(x);
}
}
Printer.PrintDiagnostics("Removed {0} phantom record parents.", missingParents);
foreach (var x in Table <Objects.Version>().ToList())
{
x.Snapshot = null;
Update(x);
var alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
HashSet <long> moveAdds = new HashSet <long>();
HashSet <long> moveDeletes = new HashSet <long>();
foreach (var s in alterations)
{
if (s.Type == AlterationType.Move)
{
moveAdds.Add(s.NewRecord.Value);
moveDeletes.Add(s.PriorRecord.Value);
}
if (s.Type == AlterationType.Delete)
{
if (Find <Record>(s.PriorRecord.Value) == null)
{
bonusDeletions++;
Delete(s);
}
}
}
foreach (var s in alterations)
{
if (s.Type == AlterationType.Add && moveAdds.Contains(s.NewRecord.Value))
{
Delete(s);
Printer.PrintDiagnostics("Cleaned up extra add in v{0} for \"{1}\"", x.ShortName, Get <ObjectName>(Get <Record>(s.NewRecord.Value).CanonicalNameId).CanonicalName);
}
if (s.Type == AlterationType.Delete && moveDeletes.Contains(s.PriorRecord.Value))
{
Delete(s);
Printer.PrintDiagnostics("Cleaned up extra delete in v{0} for \"{1}\"", x.ShortName, Get <ObjectName>(Get <Record>(s.PriorRecord.Value).CanonicalNameId).CanonicalName);
}
}
}
if (bonusDeletions >= 0)
{
Printer.PrintDiagnostics("Erased {0} unreconcilable deletions.", bonusDeletions);
}
}
public List <Record> GetRecords(Objects.Version version, out List <Record> baseList, out List <Alteration> alterations, bool testFailure = false)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
Printer.PrintDiagnostics("Getting records for version {0}.", version.ID);
long?snapshotID = null;
bool readtrans = false;
try
{
BeginTransaction();
readtrans = true;
}
catch (InvalidOperationException)
{
// not an issue
}
List <Objects.Version> parents = new List <Objects.Version>();
Objects.Version snapshotVersion = version;
while (!snapshotID.HasValue)
{
parents.Add(snapshotVersion);
if (snapshotVersion.Snapshot.HasValue)
{
snapshotID = snapshotVersion.Snapshot;
}
else
{
if (!snapshotVersion.Parent.HasValue)
{
break;
}
snapshotVersion = Get <Objects.Version>(snapshotVersion.Parent);
}
}
if (!snapshotID.HasValue)
{
Printer.PrintDiagnostics(" - No snapshot.");
baseList = new List <Record>();
}
else
{
Printer.PrintDiagnostics(" - Last snapshot version: {0}", snapshotVersion.ID);
var sslist = Query <RecordRef>("SELECT * FROM RecordRef WHERE RecordRef.SnapshotID = ?", snapshotID.Value).Select(x => x.RecordID).ToList();
CacheRecords(sslist);
baseList = sslist.Select(x => GetCachedRecord(x)).ToList();
Printer.PrintDiagnostics(" - Snapshot {0} has {1} records.", snapshotID, baseList.Count);
}
alterations = GetAlterationsInternal(parents);
if (readtrans)
{
Commit();
}
Printer.PrintDiagnostics(" - Target has {0} alterations.", alterations.Count);
List <Record> finalList = null;
try
{
finalList = Consolidate(baseList, alterations, null);
}
catch
{
if (testFailure)
{
throw;
}
Printer.PrintError("Error during database operation. Deleting cached snapshots.");
BeginExclusive(true);
RunConsistencyCheck();
Commit();
Printer.PrintError("Please retry this operation.");
throw;
}
Printer.PrintDiagnostics("Record list resolved in {0} ticks.", sw.ElapsedTicks);
if (testFailure)
{
return(finalList);
}
if (baseList.Count < alterations.Count || (alterations.Count > 4096 && parents.Count > 128))
{
Printer.PrintDiagnostics(" - Attempting to build new snapshot ({0} records in base list, {1} alterations over {2} revisions)", baseList.Count, alterations.Count, parents.Count);
try
{
BeginTransaction();
try
{
Objects.Snapshot snapshot = new Snapshot();
this.InsertSafe(snapshot);
foreach (var z in finalList)
{
Objects.RecordRef rref = new RecordRef();
rref.RecordID = z.Id;
rref.SnapshotID = snapshot.Id;
this.InsertSafe(rref);
}
version.Snapshot = snapshot.Id;
this.UpdateSafe(version);
Commit();
}
catch
{
Rollback();
}
}
catch
{
// Already in a transaction
Objects.Snapshot snapshot = new Snapshot();
this.InsertSafe(snapshot);
foreach (var z in finalList)
{
Objects.RecordRef rref = new RecordRef();
rref.RecordID = z.Id;
rref.SnapshotID = snapshot.Id;
this.InsertSafe(rref);
}
version.Snapshot = snapshot.Id;
this.UpdateSafe(version);
}
}
return(finalList);
}
private WorkspaceDB(string path, SQLite.SQLiteOpenFlags flags, LocalDB localDB) : base(path, flags)
{
Printer.PrintDiagnostics("Metadata DB Open.");
EnableWAL = true;
LocalDatabase = localDB;
if (ExecuteScalar <string>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'FormatInfo';") != "FormatInfo")
{
CreateTable <Objects.FormatInfo>();
}
if (flags.HasFlag(SQLite.SQLiteOpenFlags.Create))
{
ExecuteDirect("PRAGMA main.page_size = 4096;");
ExecuteDirect("PRAGMA main.cache_size = 10240;");
ExecuteDirect("PRAGMA temp_store = MEMORY;");
ExecuteDirect("PRAGMA threads = 2;");
EnableWAL = true;
PrepareTables();
return;
}
if (!ValidForUpgrade)
{
return;
}
if (Format.InternalFormat < InternalDBVersion)
{
try
{
var fmt = Format;
int priorFormat = fmt.InternalFormat;
if (priorFormat < 42)
{
ExecuteDirect("DROP TABLE Annotation");
ExecuteDirect("DROP TABLE AnnotationJournal");
ExecuteDirect("DROP TABLE TagJournal");
ExecuteDirect("DROP TABLE Tag");
}
BeginExclusive(true);
Printer.PrintMessage("Updating workspace database version from v{0} to v{1}", Format.InternalFormat, InternalDBVersion);
PrepareTables();
if (priorFormat < 42)
{
RunConsistencyCheck();
}
if (priorFormat < 33)
{
foreach (var x in Table <Record>().ToList())
{
if (x.Parent.HasValue)
{
Record rp = Find <Record>(x.Parent.Value);
if (rp == null)
{
Printer.PrintMessage("Found missing parent for record ID {0} - parent ID: {1}, attempting to fix.", x.Id, x.Parent.Value);
Printer.PrintMessage(" - Record {0} - {1}", Find <ObjectName>(x.CanonicalNameId).CanonicalName, x.Fingerprint);
var alterations = Query <Alteration>("SELECT * from Alteration where Alteration.NewRecord = ?", x.Id);
var possibleParents = alterations.Where(z => z.PriorRecord.HasValue).Select(z => z.PriorRecord.Value).Distinct().ToList();
Printer.PrintMessage(" - Found {0} possible parents for this record.", possibleParents.Count);
if (possibleParents.Count == 1)
{
Printer.PrintMessage(" - Updating parent record to ID {0}", possibleParents[0]);
x.Parent = possibleParents[0];
}
else
{
if (possibleParents.Count > 0)
{
Printer.PrintMessage(" - Data is ambiguous, removing parent record.");
}
else
{
Printer.PrintMessage(" - No parents found, removing parent record.");
}
x.Parent = null;
}
Update(x);
}
}
}
}
if (priorFormat < 32)
{
int count = 0;
foreach (var x in Table <Objects.Version>())
{
if (x.Message != null && x.Message.StartsWith("Automatic merge of"))
{
var mergeInfos = Table <MergeInfo>().Where(y => y.DestinationVersion == x.ID).ToList();
if (mergeInfos.Count == 1)
{
mergeInfos[0].Type = MergeType.Automatic;
Update(mergeInfos[0]);
count++;
}
}
}
Printer.PrintMessage("Updated #b#{0}## merge info records.", count);
}
if (priorFormat < 31)
{
Dictionary <long, Record> recordMap = new Dictionary <long, Record>();
foreach (var x in Table <Record>().ToList())
{
recordMap[x.Id] = x;
}
foreach (var x in Table <Record>().ToList())
{
if (x.Parent.HasValue && !recordMap.ContainsKey(x.Parent.Value))
{
x.Parent = null;
Update(x);
}
}
}
if (priorFormat < 30)
{
foreach (var x in Table <Objects.Version>().ToList())
{
x.Snapshot = null;
Update(x);
var alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
Dictionary <long, bool> moveDeletes = new Dictionary <long, bool>();
HashSet <long> deletions = new HashSet <long>();
int counter = 0;
foreach (var s in alterations)
{
if (s.Type == AlterationType.Move)
{
if (moveDeletes.ContainsKey(s.PriorRecord.Value))
{
moveDeletes[s.PriorRecord.Value] = false;
}
else
{
moveDeletes[s.PriorRecord.Value] = true;
}
}
}
foreach (var s in alterations)
{
if (s.Type == AlterationType.Move)
{
if (moveDeletes[s.PriorRecord.Value] == false)
{
s.Type = AlterationType.Copy;
Update(s);
deletions.Add(s.PriorRecord.Value);
counter++;
}
}
}
foreach (var s in deletions)
{
Alteration alt = new Alteration()
{
PriorRecord = s, Type = AlterationType.Delete, Owner = x.AlterationList
};
Insert(alt);
}
if (counter > 0)
{
Printer.PrintDiagnostics("Version {0} had {1} multiple-moves that have been fixed.", x.ShortName, counter);
}
}
}
if (priorFormat < 30)
{
foreach (var x in Table <Objects.Version>().ToList())
{
x.Snapshot = null;
Update(x);
var alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
HashSet <Tuple <AlterationType, long?, long?> > duplicateAlterations = new HashSet <Tuple <AlterationType, long?, long?> >();
int counter = 0;
foreach (var s in alterations)
{
var key = new Tuple <AlterationType, long?, long?>(s.Type, s.NewRecord, s.PriorRecord);
if (duplicateAlterations.Contains(key))
{
Delete(s);
counter++;
}
else
{
duplicateAlterations.Add(key);
}
}
if (counter > 0)
{
Printer.PrintDiagnostics("Version {0} had {1} duplicated alterations that have been fixed.", x.ShortName, counter);
}
}
}
DropTable <Objects.FormatInfo>();
fmt.InternalFormat = InternalDBVersion;
CreateTable <Objects.FormatInfo>();
Insert(fmt);
Commit();
Vacuum();
}
catch (Exception e)
{
Rollback();
Printer.PrintError("Couldn't update DB: {0}", e.ToString());
}
PrepareTables();
}
}