Example #1
0
        private List <Record> Consolidate(List <Record> baseList, List <Alteration> alterations, List <Record> deletions)
        {
            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            Dictionary <long, Record> records = new Dictionary <long, Record>();

            foreach (var x in baseList)
            {
                records[x.Id] = x;
            }

#if FULL_CONSOLIDATE_DEBUG
            Printer.PrintMessage("Initial Snapshot Contains: ");
            foreach (var x in baseList)
            {
                Printer.PrintMessage("Initial: ({2}) {0} - {1}", x.CanonicalName, x.UniqueIdentifier, x.Id);
            }
#endif

            List <long> pending = new List <long>();
            CacheRecords(alterations.Select(x => x.NewRecord).Where(x => x.HasValue).Select(x => x.Value));
            foreach (var x in alterations.Select(x => x).Reverse())
            {
                Objects.Record rec = null;
                switch (x.Type)
                {
                case AlterationType.Add:
                case AlterationType.Copy:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
                    Printer.PrintMessage("Add: ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Move:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
                    if (deletions != null)
                    {
                        rec = Get <Objects.Record>(x.PriorRecord);
                        rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                        deletions.Add(rec);
                    }
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Move (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
                    Printer.PrintMessage("Move (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception("Consistency constraint invalid!");
                    }
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Update:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Update (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
                    Printer.PrintMessage("Update (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception();
                    }
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Discard:
                    break;

                case AlterationType.Delete:
                    if (deletions != null)
                    {
                        rec = Get <Objects.Record>(x.PriorRecord);
                        rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                        deletions.Add(rec);
                    }
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Delete: ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception("Consistency constraint invalid!");
                    }
                    break;

                default:
                    throw new Exception();
                }
            }
            var result = records.Select(x => x.Value).ToList();
#if DEBUG
            HashSet <string> namecheck = new HashSet <string>();
            foreach (var x in result)
            {
                if (namecheck.Contains(x.CanonicalName))
                {
                    throw new Exception("Inconsistency in internal state!");
                }
                namecheck.Add(x.CanonicalName);
            }
#endif
            return(result);
        }
Example #2
0
        private void RunConsistencyCheck()
        {
            Printer.PrintMessage(" - Upgrading database - running full consistency check.");
            var objNames = Table <ObjectName>().ToList();
            Dictionary <long, long>   nameMapping = new Dictionary <long, long>();
            Dictionary <string, long> nameIndexes = new Dictionary <string, long>();

            DropTable <ObjectName>();
            DropTable <RecordRef>();
            CreateTable <ObjectName>();
            int duplicateObjs = 0;

            foreach (var x in objNames)
            {
                if (nameIndexes.ContainsKey(x.CanonicalName))
                {
                    nameMapping[x.NameId] = nameIndexes[x.CanonicalName];
                    duplicateObjs++;
                }
                else
                {
                    ObjectName oname = new ObjectName()
                    {
                        CanonicalName = x.CanonicalName
                    };
                    Insert(oname);
                    nameMapping[x.NameId]        = oname.NameId;
                    nameIndexes[x.CanonicalName] = oname.NameId;
                }
            }
            foreach (var x in Table <Record>().ToList())
            {
                x.CanonicalNameId = nameMapping[x.CanonicalNameId];
                Update(x);
            }
            Printer.PrintMessage(" - Cleaned {0} duplicate canonical names.", duplicateObjs);
            CreateTable <RecordRef>();
            Dictionary <Tuple <string, long, DateTime>, Record> records = new Dictionary <Tuple <string, long, DateTime>, Record>();

            foreach (var x in Table <Objects.Record>().ToList())
            {
                var key = new Tuple <string, long, DateTime>(x.UniqueIdentifier, x.CanonicalNameId, x.ModificationTime);
                if (records.ContainsKey(key))
                {
                    var other = records[key];
                    Printer.PrintDiagnostics("Found duplicate records {0} ==> {1}", x.Id, other.Id);
                    Printer.PrintDiagnostics(" - UID: {0}", x.UniqueIdentifier);
                    Printer.PrintDiagnostics(" - Time: {0}", x.ModificationTime);
                    Printer.PrintDiagnostics(" - Name: {0}", Get <ObjectName>(x.CanonicalNameId).CanonicalName);

                    int updates = 0;
                    foreach (var s in Table <Alteration>().Where(z => z.PriorRecord == x.Id || z.NewRecord == x.Id))
                    {
                        if (s.NewRecord.HasValue && s.NewRecord.Value == x.Id)
                        {
                            s.NewRecord = other.Id;
                        }
                        if (s.PriorRecord.HasValue && s.PriorRecord.Value == x.Id)
                        {
                            s.PriorRecord = other.Id;
                        }
                        Update(s);
                        updates++;
                    }
                    Delete(x);
                    Printer.PrintDiagnostics("Deleted record and updated {0} links.", updates);
                }
                else
                {
                    records[key] = x;
                }
            }
            int missingParents = 0;
            int bonusDeletions = 0;

            HashSet <long> allRecordIDs = new HashSet <long>();
            var            allRecords   = Table <Objects.Record>().ToList();

            foreach (var x in allRecords)
            {
                allRecordIDs.Add(x.Id);
            }
            foreach (var x in allRecords)
            {
                if (x.Parent.HasValue && !allRecordIDs.Contains(x.Parent.Value))
                {
                    missingParents++;
                    x.Parent = null;
                    Update(x);
                }
            }
            Printer.PrintDiagnostics("Removed {0} phantom record parents.", missingParents);


            foreach (var x in Table <Objects.Version>().ToList())
            {
                x.Snapshot = null;
                Update(x);
                var            alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
                HashSet <long> moveAdds    = new HashSet <long>();
                HashSet <long> moveDeletes = new HashSet <long>();
                foreach (var s in alterations)
                {
                    if (s.Type == AlterationType.Move)
                    {
                        moveAdds.Add(s.NewRecord.Value);
                        moveDeletes.Add(s.PriorRecord.Value);
                    }
                    if (s.Type == AlterationType.Delete)
                    {
                        if (Find <Record>(s.PriorRecord.Value) == null)
                        {
                            bonusDeletions++;
                            Delete(s);
                        }
                    }
                }
                foreach (var s in alterations)
                {
                    if (s.Type == AlterationType.Add && moveAdds.Contains(s.NewRecord.Value))
                    {
                        Delete(s);
                        Printer.PrintDiagnostics("Cleaned up extra add in v{0} for \"{1}\"", x.ShortName, Get <ObjectName>(Get <Record>(s.NewRecord.Value).CanonicalNameId).CanonicalName);
                    }
                    if (s.Type == AlterationType.Delete && moveDeletes.Contains(s.PriorRecord.Value))
                    {
                        Delete(s);
                        Printer.PrintDiagnostics("Cleaned up extra delete in v{0} for \"{1}\"", x.ShortName, Get <ObjectName>(Get <Record>(s.PriorRecord.Value).CanonicalNameId).CanonicalName);
                    }
                }
            }
            if (bonusDeletions >= 0)
            {
                Printer.PrintDiagnostics("Erased {0} unreconcilable deletions.", bonusDeletions);
            }
        }
Example #3
0
        public List <Record> GetRecords(Objects.Version version, out List <Record> baseList, out List <Alteration> alterations, bool testFailure = false)
        {
            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            Printer.PrintDiagnostics("Getting records for version {0}.", version.ID);
            long?snapshotID = null;
            bool readtrans  = false;

            try
            {
                BeginTransaction();
                readtrans = true;
            }
            catch (InvalidOperationException)
            {
                // not an issue
            }
            List <Objects.Version> parents = new List <Objects.Version>();

            Objects.Version snapshotVersion = version;
            while (!snapshotID.HasValue)
            {
                parents.Add(snapshotVersion);
                if (snapshotVersion.Snapshot.HasValue)
                {
                    snapshotID = snapshotVersion.Snapshot;
                }
                else
                {
                    if (!snapshotVersion.Parent.HasValue)
                    {
                        break;
                    }
                    snapshotVersion = Get <Objects.Version>(snapshotVersion.Parent);
                }
            }
            if (!snapshotID.HasValue)
            {
                Printer.PrintDiagnostics(" - No snapshot.");
                baseList = new List <Record>();
            }
            else
            {
                Printer.PrintDiagnostics(" - Last snapshot version: {0}", snapshotVersion.ID);
                var sslist = Query <RecordRef>("SELECT * FROM RecordRef WHERE RecordRef.SnapshotID = ?", snapshotID.Value).Select(x => x.RecordID).ToList();
                CacheRecords(sslist);
                baseList = sslist.Select(x => GetCachedRecord(x)).ToList();
                Printer.PrintDiagnostics(" - Snapshot {0} has {1} records.", snapshotID, baseList.Count);
            }
            alterations = GetAlterationsInternal(parents);
            if (readtrans)
            {
                Commit();
            }
            Printer.PrintDiagnostics(" - Target has {0} alterations.", alterations.Count);
            List <Record> finalList = null;

            try
            {
                finalList = Consolidate(baseList, alterations, null);
            }
            catch
            {
                if (testFailure)
                {
                    throw;
                }
                Printer.PrintError("Error during database operation. Deleting cached snapshots.");
                BeginExclusive(true);
                RunConsistencyCheck();
                Commit();
                Printer.PrintError("Please retry this operation.");
                throw;
            }
            Printer.PrintDiagnostics("Record list resolved in {0} ticks.", sw.ElapsedTicks);
            if (testFailure)
            {
                return(finalList);
            }
            if (baseList.Count < alterations.Count || (alterations.Count > 4096 && parents.Count > 128))
            {
                Printer.PrintDiagnostics(" - Attempting to build new snapshot ({0} records in base list, {1} alterations over {2} revisions)", baseList.Count, alterations.Count, parents.Count);
                try
                {
                    BeginTransaction();
                    try
                    {
                        Objects.Snapshot snapshot = new Snapshot();
                        this.InsertSafe(snapshot);
                        foreach (var z in finalList)
                        {
                            Objects.RecordRef rref = new RecordRef();
                            rref.RecordID   = z.Id;
                            rref.SnapshotID = snapshot.Id;
                            this.InsertSafe(rref);
                        }
                        version.Snapshot = snapshot.Id;
                        this.UpdateSafe(version);
                        Commit();
                    }
                    catch
                    {
                        Rollback();
                    }
                }
                catch
                {
                    // Already in a transaction
                    Objects.Snapshot snapshot = new Snapshot();
                    this.InsertSafe(snapshot);
                    foreach (var z in finalList)
                    {
                        Objects.RecordRef rref = new RecordRef();
                        rref.RecordID   = z.Id;
                        rref.SnapshotID = snapshot.Id;
                        this.InsertSafe(rref);
                    }
                    version.Snapshot = snapshot.Id;
                    this.UpdateSafe(version);
                }
            }
            return(finalList);
        }
Example #4
0
        private WorkspaceDB(string path, SQLite.SQLiteOpenFlags flags, LocalDB localDB) : base(path, flags)
        {
            Printer.PrintDiagnostics("Metadata DB Open.");
            EnableWAL     = true;
            LocalDatabase = localDB;

            if (ExecuteScalar <string>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'FormatInfo';") != "FormatInfo")
            {
                CreateTable <Objects.FormatInfo>();
            }
            if (flags.HasFlag(SQLite.SQLiteOpenFlags.Create))
            {
                ExecuteDirect("PRAGMA main.page_size = 4096;");
                ExecuteDirect("PRAGMA main.cache_size = 10240;");
                ExecuteDirect("PRAGMA temp_store = MEMORY;");
                ExecuteDirect("PRAGMA threads = 2;");
                EnableWAL = true;
                PrepareTables();
                return;
            }

            if (!ValidForUpgrade)
            {
                return;
            }

            if (Format.InternalFormat < InternalDBVersion)
            {
                try
                {
                    var fmt         = Format;
                    int priorFormat = fmt.InternalFormat;

                    if (priorFormat < 42)
                    {
                        ExecuteDirect("DROP TABLE Annotation");
                        ExecuteDirect("DROP TABLE AnnotationJournal");
                        ExecuteDirect("DROP TABLE TagJournal");
                        ExecuteDirect("DROP TABLE Tag");
                    }
                    BeginExclusive(true);
                    Printer.PrintMessage("Updating workspace database version from v{0} to v{1}", Format.InternalFormat, InternalDBVersion);
                    PrepareTables();
                    if (priorFormat < 42)
                    {
                        RunConsistencyCheck();
                    }
                    if (priorFormat < 33)
                    {
                        foreach (var x in Table <Record>().ToList())
                        {
                            if (x.Parent.HasValue)
                            {
                                Record rp = Find <Record>(x.Parent.Value);
                                if (rp == null)
                                {
                                    Printer.PrintMessage("Found missing parent for record ID {0} - parent ID: {1}, attempting to fix.", x.Id, x.Parent.Value);
                                    Printer.PrintMessage(" - Record {0} - {1}", Find <ObjectName>(x.CanonicalNameId).CanonicalName, x.Fingerprint);

                                    var alterations     = Query <Alteration>("SELECT * from Alteration where Alteration.NewRecord = ?", x.Id);
                                    var possibleParents = alterations.Where(z => z.PriorRecord.HasValue).Select(z => z.PriorRecord.Value).Distinct().ToList();

                                    Printer.PrintMessage(" - Found {0} possible parents for this record.", possibleParents.Count);
                                    if (possibleParents.Count == 1)
                                    {
                                        Printer.PrintMessage(" - Updating parent record to ID {0}", possibleParents[0]);
                                        x.Parent = possibleParents[0];
                                    }
                                    else
                                    {
                                        if (possibleParents.Count > 0)
                                        {
                                            Printer.PrintMessage(" - Data is ambiguous, removing parent record.");
                                        }
                                        else
                                        {
                                            Printer.PrintMessage(" - No parents found, removing parent record.");
                                        }
                                        x.Parent = null;
                                    }
                                    Update(x);
                                }
                            }
                        }
                    }
                    if (priorFormat < 32)
                    {
                        int count = 0;
                        foreach (var x in Table <Objects.Version>())
                        {
                            if (x.Message != null && x.Message.StartsWith("Automatic merge of"))
                            {
                                var mergeInfos = Table <MergeInfo>().Where(y => y.DestinationVersion == x.ID).ToList();
                                if (mergeInfos.Count == 1)
                                {
                                    mergeInfos[0].Type = MergeType.Automatic;
                                    Update(mergeInfos[0]);
                                    count++;
                                }
                            }
                        }
                        Printer.PrintMessage("Updated #b#{0}## merge info records.", count);
                    }
                    if (priorFormat < 31)
                    {
                        Dictionary <long, Record> recordMap = new Dictionary <long, Record>();
                        foreach (var x in Table <Record>().ToList())
                        {
                            recordMap[x.Id] = x;
                        }

                        foreach (var x in Table <Record>().ToList())
                        {
                            if (x.Parent.HasValue && !recordMap.ContainsKey(x.Parent.Value))
                            {
                                x.Parent = null;
                                Update(x);
                            }
                        }
                    }
                    if (priorFormat < 30)
                    {
                        foreach (var x in Table <Objects.Version>().ToList())
                        {
                            x.Snapshot = null;
                            Update(x);
                            var alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
                            Dictionary <long, bool> moveDeletes = new Dictionary <long, bool>();
                            HashSet <long>          deletions   = new HashSet <long>();
                            int counter = 0;
                            foreach (var s in alterations)
                            {
                                if (s.Type == AlterationType.Move)
                                {
                                    if (moveDeletes.ContainsKey(s.PriorRecord.Value))
                                    {
                                        moveDeletes[s.PriorRecord.Value] = false;
                                    }
                                    else
                                    {
                                        moveDeletes[s.PriorRecord.Value] = true;
                                    }
                                }
                            }
                            foreach (var s in alterations)
                            {
                                if (s.Type == AlterationType.Move)
                                {
                                    if (moveDeletes[s.PriorRecord.Value] == false)
                                    {
                                        s.Type = AlterationType.Copy;
                                        Update(s);
                                        deletions.Add(s.PriorRecord.Value);
                                        counter++;
                                    }
                                }
                            }
                            foreach (var s in deletions)
                            {
                                Alteration alt = new Alteration()
                                {
                                    PriorRecord = s, Type = AlterationType.Delete, Owner = x.AlterationList
                                };
                                Insert(alt);
                            }
                            if (counter > 0)
                            {
                                Printer.PrintDiagnostics("Version {0} had {1} multiple-moves that have been fixed.", x.ShortName, counter);
                            }
                        }
                    }
                    if (priorFormat < 30)
                    {
                        foreach (var x in Table <Objects.Version>().ToList())
                        {
                            x.Snapshot = null;
                            Update(x);
                            var alterations = Table <Objects.Alteration>().Where(z => z.Owner == x.AlterationList);
                            HashSet <Tuple <AlterationType, long?, long?> > duplicateAlterations = new HashSet <Tuple <AlterationType, long?, long?> >();
                            int counter = 0;
                            foreach (var s in alterations)
                            {
                                var key = new Tuple <AlterationType, long?, long?>(s.Type, s.NewRecord, s.PriorRecord);
                                if (duplicateAlterations.Contains(key))
                                {
                                    Delete(s);
                                    counter++;
                                }
                                else
                                {
                                    duplicateAlterations.Add(key);
                                }
                            }
                            if (counter > 0)
                            {
                                Printer.PrintDiagnostics("Version {0} had {1} duplicated alterations that have been fixed.", x.ShortName, counter);
                            }
                        }
                    }

                    DropTable <Objects.FormatInfo>();
                    fmt.InternalFormat = InternalDBVersion;
                    CreateTable <Objects.FormatInfo>();
                    Insert(fmt);

                    Commit();

                    Vacuum();
                }
                catch (Exception e)
                {
                    Rollback();
                    Printer.PrintError("Couldn't update DB: {0}", e.ToString());
                }
                PrepareTables();
            }
        }