public void Create(UInt64 processId, string schema, Index index, out Guid newId) { try { var persistIndex = PersistIndex.FromPayload(index); if (persistIndex.Id == Guid.Empty) { persistIndex.Id = Guid.NewGuid(); } if (persistIndex.Created == DateTime.MinValue) { persistIndex.Created = DateTime.UtcNow; } if (persistIndex.Modfied == DateTime.MinValue) { persistIndex.Modfied = DateTime.UtcNow; } using (var txRef = core.Transactions.Begin(processId)) { var schemaMeta = core.Schemas.VirtualPathToMeta(txRef.Transaction, schema, LockOperation.Read); if (schemaMeta == null || schemaMeta.Exists == false) { throw new DokdexSchemaDoesNotExistException(schema); } var indexCatalog = GetIndexCatalog(txRef.Transaction, schemaMeta, LockOperation.Write); indexCatalog.Add(persistIndex); core.IO.PutJson(txRef.Transaction, indexCatalog.DiskPath, indexCatalog); persistIndex.DiskPath = Path.Combine(schemaMeta.DiskPath, MakeIndexFileName(index.Name)); core.IO.PutPBuf(txRef.Transaction, persistIndex.DiskPath, new PersistIndexPageCatalog()); RebuildIndex(txRef.Transaction, schemaMeta, persistIndex); newId = persistIndex.Id; txRef.Commit(); } } catch (Exception ex) { core.Log.Write(String.Format("Failed to create index for process {0}.", processId), ex); throw; } }
/// <summary> /// Inserts all documents in a schema into a single index in the schema. /// </summary> /// <param name="transaction"></param> /// <param name="schemaMeta"></param> /// <param name="indexMeta"></param> private void RebuildIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta) { try { var filePath = Path.Combine(schemaMeta.DiskPath, Constants.DocumentCatalogFile); var documentCatalog = core.IO.GetJson <PersistDocumentCatalog>(transaction, filePath, LockOperation.Read); //Clear out the existing index pages. core.IO.PutPBuf(transaction, indexMeta.DiskPath, new PersistIndexPageCatalog()); var indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write); var state = new RebuildIndexItemThreadProc_ParallelState() { TargetThreadCount = Environment.ProcessorCount * 2 }; var param = new RebuildIndexItemThreadProc_Params() { DocumentCatalog = documentCatalog, State = state, IndexMeta = indexMeta, IndexPageCatalog = indexPageCatalog, SchemaMeta = schemaMeta, Transaction = transaction }; for (int i = 0; i < state.TargetThreadCount; i++) { new Thread(RebuildIndexItemThreadProc).Start(param); param.Initialized.WaitOne(Timeout.Infinite); } while (state.IsComplete == false) { Thread.Sleep(1); } core.IO.PutPBuf(transaction, indexMeta.DiskPath, indexPageCatalog); } catch (Exception ex) { core.Log.Write(String.Format("Failed to rebuild single index for process {0}.", transaction.ProcessId), ex); throw; } }
static public PersistIndex FromPayload(Index index) { var persistIndex = new PersistIndex() { Id = index.Id, Name = index.Name, Created = index.Created, Modfied = index.Modfied, IsUnique = index.IsUnique }; foreach (var indexAttribute in index.Attributes) { persistIndex.AddAttribute(PersistIndexAttribute.FromPayload(indexAttribute)); } return(persistIndex); }
private List <string> GetIndexSearchTokens(Transaction transaction, PersistIndex indexMeta, PersistDocument document) { try { List <string> result = new List <string>(); foreach (var indexAttribute in indexMeta.Attributes) { var jsonContent = JObject.Parse(document.Content); JToken jToken = null; if (jsonContent.TryGetValue(indexAttribute.Name, StringComparison.CurrentCultureIgnoreCase, out jToken)) { result.Add(jToken.ToString()); } } return(result); } catch (Exception ex) { core.Log.Write(String.Format("Failed to build index search tokens for process {0}.", transaction.ProcessId), ex); throw; } }
public void Add(PersistIndex item) { this.Collection.Add(item); }
public void Remove(PersistIndex item) { Collection.Remove(item); }
public PotentialIndex(PersistIndex index, List <string> handledKeyNames) { this.Index = index; this.HandledKeyNames = handledKeyNames; }
/// <summary> /// Removes a document from an index. /// </summary> /// <param name="transaction"></param> /// <param name="schemaMeta"></param> /// <param name="indexMeta"></param> /// <param name="document"></param> private void DeleteDocumentFromIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, Guid documentId) { try { var persistIndexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write); if (RemoveDocumentFromLeaves(ref persistIndexPageCatalog.Leaves, documentId)) { core.IO.PutPBuf(transaction, indexMeta.DiskPath, persistIndexPageCatalog); } } catch (Exception ex) { core.Log.Write(String.Format("Index document upsert failed for process {0}.", transaction.ProcessId), ex); throw; } }
/// <summary> /// Inserts an index entry for a single document into a single index using a long lived index page catalog. /// </summary> /// <param name="transaction"></param> /// <param name="schemaMeta"></param> /// <param name="indexMeta"></param> /// <param name="document"></param> private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document, PersistIndexPageCatalog indexPageCatalog, bool flushPageCatalog) { try { var searchTokens = GetIndexSearchTokens(transaction, indexMeta, document); var findResult = FindKeyPage(transaction, indexMeta, searchTokens, indexPageCatalog); //If we found a full match for all supplied key values - add the document to the leaf collection. if (findResult.IsFullMatch) { if (findResult.Leaf.DocumentIDs == null) { findResult.Leaf.DocumentIDs = new HashSet <Guid>(); } if (indexMeta.IsUnique && findResult.Leaf.DocumentIDs.Count > 1) { string exceptionText = string.Format("Duplicate key violation occurred for index [{0}]/[{1}]. Values: {{{2}}}", schemaMeta.VirtualPath, indexMeta.Name, string.Join(",", searchTokens)); throw new DokdexDuplicateKeyViolation(exceptionText); } findResult.Leaf.DocumentIDs.Add(document.Id); if (flushPageCatalog) { core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog); } } else { //If we didn't find a full match for all supplied key values, // then create the tree and add the document to the lowest leaf. //Note that we are going to start creating the leaf level at the findResult.ExtentLevel. // This is because we may have a partial match and don't need to create the full tree. lock (indexPageCatalog) { for (int i = findResult.ExtentLevel; i < searchTokens.Count; i++) { findResult.Leaf = findResult.Leaves.AddNewleaf(searchTokens[i]); findResult.Leaves = findResult.Leaf.Leaves; } if (findResult.Leaf.DocumentIDs == null) { findResult.Leaf.DocumentIDs = new HashSet <Guid>(); } findResult.Leaf.DocumentIDs.Add(document.Id); } if (flushPageCatalog) { core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog); } } } catch (Exception ex) { core.Log.Write(String.Format("Index document insert failed for process {0}.", transaction.ProcessId), ex); throw; } }
/// <summary> /// Inserts an index entry for a single document into a single index. /// </summary> /// <param name="transaction"></param> /// <param name="schemaMeta"></param> /// <param name="indexMeta"></param> /// <param name="document"></param> private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document) { InsertDocumentIntoIndex(transaction, schemaMeta, indexMeta, document, null, true); }
/// <summary> /// Finds the appropriate index page for a set of key values using a long lived index page catalog. /// </summary> /// <param name="transaction"></param> /// <param name="indexMeta"></param> /// <param name="searchTokens"></param> /// <param name="indexPageCatalog"></param> /// <returns></returns> private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens, PersistIndexPageCatalog suppliedIndexPageCatalog) { try { var indexPageCatalog = suppliedIndexPageCatalog; if (indexPageCatalog == null) { indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write); } lock (indexPageCatalog) { FindKeyPageResult result = new FindKeyPageResult() { Catalog = indexPageCatalog }; result.Leaves = result.Catalog.Leaves; if (result.Leaves == null || result.Leaves.Count == 0) { //The index is empty. return(result); } int foundExtentCount = 0; foreach (var token in searchTokens) { bool locatedExtent = false; foreach (var leaf in result.Leaves) { if (leaf.Key == token) { locatedExtent = true; foundExtentCount++; result.Leaf = leaf; result.Leaves = leaf.Leaves; //Move one level lower in the extent tree. result.IsPartialMatch = true; result.ExtentLevel = foundExtentCount; if (foundExtentCount == searchTokens.Count) { result.IsPartialMatch = false; result.IsFullMatch = true; return(result); } } } if (locatedExtent == false) { return(result); } } return(result); } } catch (Exception ex) { core.Log.Write(String.Format("Failed to locate key page for process {0}.", transaction.ProcessId), ex); throw; } }
/// <summary> // Finds the appropriate index page for a set of key values. /// </summary> /// <param name="transaction"></param> /// <param name="indexMeta"></param> /// <param name="searchTokens"></param> /// <returns></returns> private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens) { return(FindKeyPage(transaction, indexMeta, searchTokens, null)); }
public IndexSelection(PersistIndex index, List <string> handledKeyNames) { this.HandledKeyNames = handledKeyNames; this.Index = index; }