Example #1
0
        /// <summary>
        /// Inserts all documents in a schema into a single index in the schema.
        /// </summary>
        /// <param name="transaction"></param>
        /// <param name="schemaMeta"></param>
        /// <param name="indexMeta"></param>
        private void RebuildIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta)
        {
            try
            {
                var filePath        = Path.Combine(schemaMeta.DiskPath, Constants.DocumentCatalogFile);
                var documentCatalog = core.IO.GetJson <PersistDocumentCatalog>(transaction, filePath, LockOperation.Read);

                //Clear out the existing index pages.
                core.IO.PutPBuf(transaction, indexMeta.DiskPath, new PersistIndexPageCatalog());

                var indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);

                var state = new RebuildIndexItemThreadProc_ParallelState()
                {
                    TargetThreadCount = Environment.ProcessorCount * 2,
                };

                state.TargetThreadCount = 8;

                var param = new RebuildIndexItemThreadProc_Params()
                {
                    DocumentCatalog  = documentCatalog,
                    State            = state,
                    IndexMeta        = indexMeta,
                    IndexPageCatalog = indexPageCatalog,
                    SchemaMeta       = schemaMeta,
                    Transaction      = transaction
                };

                for (int i = 0; i < state.TargetThreadCount; i++)
                {
                    new Thread(RebuildIndexItemThreadProc).Start(param);
                    param.Initialized.WaitOne(Timeout.Infinite);
                }

                while (state.IsComplete == false)
                {
                    Thread.Sleep(1);
                }

                if (state.Success == false)
                {
                    throw new LeafSQLExceptionBase($"Failed to build index: {state.Exception.ToString()}");
                }

                core.IO.PutPBuf(transaction, indexMeta.DiskPath, indexPageCatalog);
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Failed to rebuild single index for process {0}.", transaction.ProcessId), ex);
                throw;
            }
        }
Example #2
0
        public void Create(Session session, string schema, Library.Payloads.Models.Index index, out Guid newId)
        {
            try
            {
                var persistIndex = PersistIndex.FromPayload(index);

                if (persistIndex.Id == Guid.Empty)
                {
                    persistIndex.Id = Guid.NewGuid();
                }
                if (persistIndex.Created == DateTime.MinValue)
                {
                    persistIndex.Created = DateTime.UtcNow;
                }
                if (persistIndex.Modfied == DateTime.MinValue)
                {
                    persistIndex.Modfied = DateTime.UtcNow;
                }

                using (var txRef = core.Transactions.Begin(session))
                {
                    var schemaMeta = core.Schemas.VirtualPathToMeta(txRef.Transaction, schema, LockOperation.Read);
                    if (schemaMeta == null || schemaMeta.Exists == false)
                    {
                        throw new LeafSQLSchemaDoesNotExistException(schema);
                    }

                    var indexCatalog = GetIndexCatalog(txRef.Transaction, schemaMeta, LockOperation.Write);
                    indexCatalog.Add(persistIndex);
                    core.IO.PutJson(txRef.Transaction, indexCatalog.DiskPath, indexCatalog);

                    persistIndex.DiskPath = Path.Combine(schemaMeta.DiskPath, MakeIndexFileName(index.Name));
                    core.IO.PutPBuf(txRef.Transaction, persistIndex.DiskPath, new PersistIndexPageCatalog());

                    RebuildIndex(txRef.Transaction, schemaMeta, persistIndex);

                    newId = persistIndex.Id;

                    txRef.Commit();
                }
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Failed to create index for process {0}.", session.ProcessId), ex);
                throw;
            }
        }
Example #3
0
        private List <string> GetIndexSearchTokens(Transaction transaction, PersistIndex indexMeta, PersistDocument document)
        {
            try
            {
                List <string> result = new List <string>();

                foreach (var indexAttribute in indexMeta.Attributes)
                {
                    var    jsonContent = JObject.Parse(document.Content);
                    JToken jToken      = null;
                    if (jsonContent.TryGetValue(indexAttribute.Name, StringComparison.CurrentCultureIgnoreCase, out jToken))
                    {
                        result.Add(jToken.ToString());
                    }
                }

                return(result);
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Failed to build index search tokens for process {0}.", transaction.ProcessId), ex);
                throw;
            }
        }
Example #4
0
 public PotentialIndex(PersistIndex index, List <IndexHandledCondition> indexHandledConditions)
 {
     this.Index = index;
     this.IndexHandledConditions = indexHandledConditions;
 }
Example #5
0
 public IndexSelection(PersistIndex index, List <IndexHandledCondition> indexHandledConditions)
 {
     this.IndexHandledConditions = indexHandledConditions;
     this.Index = index;
 }
Example #6
0
        /// <summary>
        /// Removes a document from an index.
        /// </summary>
        /// <param name="transaction"></param>
        /// <param name="schemaMeta"></param>
        /// <param name="indexMeta"></param>
        /// <param name="document"></param>
        private void DeleteDocumentFromIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, Guid documentId)
        {
            try
            {
                var persistIndexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);

                if (RemoveDocumentFromLeaves(ref persistIndexPageCatalog.Leaves, documentId))
                {
                    core.IO.PutPBuf(transaction, indexMeta.DiskPath, persistIndexPageCatalog);
                }
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Index document upsert failed for process {0}.", transaction.ProcessId), ex);
                throw;
            }
        }
Example #7
0
        /// <summary>
        /// Inserts an index entry for a single document into a single index using a long lived index page catalog.
        /// </summary>
        /// <param name="transaction"></param>
        /// <param name="schemaMeta"></param>
        /// <param name="indexMeta"></param>
        /// <param name="document"></param>
        private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document, PersistIndexPageCatalog indexPageCatalog, bool flushPageCatalog)
        {
            try
            {
                var searchTokens = GetIndexSearchTokens(transaction, indexMeta, document);
                var findResult   = FindKeyPage(transaction, indexMeta, searchTokens, indexPageCatalog);

                if (searchTokens.Count == 0)
                {
                    //None of the supplied index attributes could be found on this document.
                    return;
                }

                //If we found a full match for all supplied key values - add the document to the leaf collection.
                if (findResult.IsFullMatch)
                {
                    if (findResult.Leaf.DocumentIDs == null)
                    {
                        findResult.Leaf.DocumentIDs = new HashSet <Guid>();
                    }

                    if (indexMeta.IsUnique && findResult.Leaf.DocumentIDs.Count > 1)
                    {
                        string exceptionText = string.Format("Duplicate key violation occurred for index [{0}]/[{1}]. Values: {{{2}}}",
                                                             schemaMeta.VirtualPath, indexMeta.Name, string.Join(",", searchTokens));

                        throw new LeafSQLDuplicateKeyViolation(exceptionText);
                    }

                    findResult.Leaf.DocumentIDs.Add(document.Id);
                    if (flushPageCatalog)
                    {
                        core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog);
                    }
                }
                else
                {
                    //If we didn't find a full match for all supplied key values,
                    //  then create the tree and add the document to the lowest leaf.
                    //Note that we are going to start creating the leaf level at the findResult.ExtentLevel.
                    //  This is because we may have a partial match and don't need to create the full tree.
                    lock (indexPageCatalog)
                    {
                        for (int i = findResult.ExtentLevel; i < searchTokens.Count; i++)
                        {
                            findResult.Leaf   = findResult.Extent.AddNewleaf(searchTokens[i]);
                            findResult.Extent = findResult.Leaf.Extent;
                        }

                        if (findResult.Leaf == null || findResult.Leaf.DocumentIDs == null)
                        {
                            findResult.Leaf.DocumentIDs = new HashSet <Guid>();
                        }

                        findResult.Leaf.DocumentIDs.Add(document.Id);
                    }
                    if (flushPageCatalog)
                    {
                        core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog);
                    }
                }
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Index document insert failed for process {0}.", transaction.ProcessId), ex);
                throw;
            }
        }
Example #8
0
 /// <summary>
 /// Inserts an index entry for a single document into a single index.
 /// </summary>
 /// <param name="transaction"></param>
 /// <param name="schemaMeta"></param>
 /// <param name="indexMeta"></param>
 /// <param name="document"></param>
 private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document)
 {
     InsertDocumentIntoIndex(transaction, schemaMeta, indexMeta, document, null, true);
 }
Example #9
0
        /// <summary>
        /// Finds the appropriate index page for a set of key values using a long lived index page catalog.
        /// </summary>
        /// <param name="transaction"></param>
        /// <param name="indexMeta"></param>
        /// <param name="searchTokens"></param>
        /// <param name="indexPageCatalog"></param>
        /// <returns></returns>
        private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens, PersistIndexPageCatalog suppliedIndexPageCatalog)
        {
            try
            {
                var indexPageCatalog = suppliedIndexPageCatalog;
                if (indexPageCatalog == null)
                {
                    indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);
                }

                lock (indexPageCatalog)
                {
                    FindKeyPageResult result = new FindKeyPageResult()
                    {
                        Catalog = indexPageCatalog
                    };

                    result.Extent = result.Catalog.Leaves;
                    if (result.Extent == null || result.Extent.Count == 0)
                    {
                        //The index is empty.
                        return(result);
                    }

                    int foundExtentCount = 0;

                    foreach (var token in searchTokens)
                    {
                        bool locatedExtent = false;

                        foreach (var leaf in result.Extent.Leaves)
                        {
                            if (leaf.Key == token)
                            {
                                locatedExtent = true;
                                foundExtentCount++;
                                result.Leaf   = leaf;
                                result.Extent = leaf.Extent; //Move one level lower in the extent tree.

                                result.IsPartialMatch = true;
                                result.ExtentLevel    = foundExtentCount;

                                if (foundExtentCount == searchTokens.Count)
                                {
                                    result.IsPartialMatch = false;
                                    result.IsFullMatch    = true;
                                    return(result);
                                }
                            }
                        }

                        if (locatedExtent == false)
                        {
                            return(result);
                        }
                    }

                    return(result);
                }
            }
            catch (Exception ex)
            {
                core.Log.Write(String.Format("Failed to locate key page for process {0}.", transaction.ProcessId), ex);
                throw;
            }
        }
Example #10
0
 /// <summary>
 // Finds the appropriate index page for a set of key values.
 /// </summary>
 /// <param name="transaction"></param>
 /// <param name="indexMeta"></param>
 /// <param name="searchTokens"></param>
 /// <returns></returns>
 private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens)
 {
     return(FindKeyPage(transaction, indexMeta, searchTokens, null));
 }