/// <summary>
/// Inserts all documents in a schema into a single index in the schema.
/// </summary>
/// <param name="transaction"></param>
/// <param name="schemaMeta"></param>
/// <param name="indexMeta"></param>
private void RebuildIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta)
{
try
{
var filePath = Path.Combine(schemaMeta.DiskPath, Constants.DocumentCatalogFile);
var documentCatalog = core.IO.GetJson <PersistDocumentCatalog>(transaction, filePath, LockOperation.Read);
//Clear out the existing index pages.
core.IO.PutPBuf(transaction, indexMeta.DiskPath, new PersistIndexPageCatalog());
var indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);
var state = new RebuildIndexItemThreadProc_ParallelState()
{
TargetThreadCount = Environment.ProcessorCount * 2,
};
state.TargetThreadCount = 8;
var param = new RebuildIndexItemThreadProc_Params()
{
DocumentCatalog = documentCatalog,
State = state,
IndexMeta = indexMeta,
IndexPageCatalog = indexPageCatalog,
SchemaMeta = schemaMeta,
Transaction = transaction
};
for (int i = 0; i < state.TargetThreadCount; i++)
{
new Thread(RebuildIndexItemThreadProc).Start(param);
param.Initialized.WaitOne(Timeout.Infinite);
}
while (state.IsComplete == false)
{
Thread.Sleep(1);
}
if (state.Success == false)
{
throw new LeafSQLExceptionBase($"Failed to build index: {state.Exception.ToString()}");
}
core.IO.PutPBuf(transaction, indexMeta.DiskPath, indexPageCatalog);
}
catch (Exception ex)
{
core.Log.Write(String.Format("Failed to rebuild single index for process {0}.", transaction.ProcessId), ex);
throw;
}
}
public void Create(Session session, string schema, Library.Payloads.Models.Index index, out Guid newId)
{
try
{
var persistIndex = PersistIndex.FromPayload(index);
if (persistIndex.Id == Guid.Empty)
{
persistIndex.Id = Guid.NewGuid();
}
if (persistIndex.Created == DateTime.MinValue)
{
persistIndex.Created = DateTime.UtcNow;
}
if (persistIndex.Modfied == DateTime.MinValue)
{
persistIndex.Modfied = DateTime.UtcNow;
}
using (var txRef = core.Transactions.Begin(session))
{
var schemaMeta = core.Schemas.VirtualPathToMeta(txRef.Transaction, schema, LockOperation.Read);
if (schemaMeta == null || schemaMeta.Exists == false)
{
throw new LeafSQLSchemaDoesNotExistException(schema);
}
var indexCatalog = GetIndexCatalog(txRef.Transaction, schemaMeta, LockOperation.Write);
indexCatalog.Add(persistIndex);
core.IO.PutJson(txRef.Transaction, indexCatalog.DiskPath, indexCatalog);
persistIndex.DiskPath = Path.Combine(schemaMeta.DiskPath, MakeIndexFileName(index.Name));
core.IO.PutPBuf(txRef.Transaction, persistIndex.DiskPath, new PersistIndexPageCatalog());
RebuildIndex(txRef.Transaction, schemaMeta, persistIndex);
newId = persistIndex.Id;
txRef.Commit();
}
}
catch (Exception ex)
{
core.Log.Write(String.Format("Failed to create index for process {0}.", session.ProcessId), ex);
throw;
}
}
private List <string> GetIndexSearchTokens(Transaction transaction, PersistIndex indexMeta, PersistDocument document)
{
try
{
List <string> result = new List <string>();
foreach (var indexAttribute in indexMeta.Attributes)
{
var jsonContent = JObject.Parse(document.Content);
JToken jToken = null;
if (jsonContent.TryGetValue(indexAttribute.Name, StringComparison.CurrentCultureIgnoreCase, out jToken))
{
result.Add(jToken.ToString());
}
}
return(result);
}
catch (Exception ex)
{
core.Log.Write(String.Format("Failed to build index search tokens for process {0}.", transaction.ProcessId), ex);
throw;
}
}
public PotentialIndex(PersistIndex index, List <IndexHandledCondition> indexHandledConditions)
{
this.Index = index;
this.IndexHandledConditions = indexHandledConditions;
}
public IndexSelection(PersistIndex index, List <IndexHandledCondition> indexHandledConditions)
{
this.IndexHandledConditions = indexHandledConditions;
this.Index = index;
}
/// <summary>
/// Removes a document from an index.
/// </summary>
/// <param name="transaction"></param>
/// <param name="schemaMeta"></param>
/// <param name="indexMeta"></param>
/// <param name="document"></param>
private void DeleteDocumentFromIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, Guid documentId)
{
try
{
var persistIndexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);
if (RemoveDocumentFromLeaves(ref persistIndexPageCatalog.Leaves, documentId))
{
core.IO.PutPBuf(transaction, indexMeta.DiskPath, persistIndexPageCatalog);
}
}
catch (Exception ex)
{
core.Log.Write(String.Format("Index document upsert failed for process {0}.", transaction.ProcessId), ex);
throw;
}
}
/// <summary>
/// Inserts an index entry for a single document into a single index using a long lived index page catalog.
/// </summary>
/// <param name="transaction"></param>
/// <param name="schemaMeta"></param>
/// <param name="indexMeta"></param>
/// <param name="document"></param>
private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document, PersistIndexPageCatalog indexPageCatalog, bool flushPageCatalog)
{
try
{
var searchTokens = GetIndexSearchTokens(transaction, indexMeta, document);
var findResult = FindKeyPage(transaction, indexMeta, searchTokens, indexPageCatalog);
if (searchTokens.Count == 0)
{
//None of the supplied index attributes could be found on this document.
return;
}
//If we found a full match for all supplied key values - add the document to the leaf collection.
if (findResult.IsFullMatch)
{
if (findResult.Leaf.DocumentIDs == null)
{
findResult.Leaf.DocumentIDs = new HashSet <Guid>();
}
if (indexMeta.IsUnique && findResult.Leaf.DocumentIDs.Count > 1)
{
string exceptionText = string.Format("Duplicate key violation occurred for index [{0}]/[{1}]. Values: {{{2}}}",
schemaMeta.VirtualPath, indexMeta.Name, string.Join(",", searchTokens));
throw new LeafSQLDuplicateKeyViolation(exceptionText);
}
findResult.Leaf.DocumentIDs.Add(document.Id);
if (flushPageCatalog)
{
core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog);
}
}
else
{
//If we didn't find a full match for all supplied key values,
// then create the tree and add the document to the lowest leaf.
//Note that we are going to start creating the leaf level at the findResult.ExtentLevel.
// This is because we may have a partial match and don't need to create the full tree.
lock (indexPageCatalog)
{
for (int i = findResult.ExtentLevel; i < searchTokens.Count; i++)
{
findResult.Leaf = findResult.Extent.AddNewleaf(searchTokens[i]);
findResult.Extent = findResult.Leaf.Extent;
}
if (findResult.Leaf == null || findResult.Leaf.DocumentIDs == null)
{
findResult.Leaf.DocumentIDs = new HashSet <Guid>();
}
findResult.Leaf.DocumentIDs.Add(document.Id);
}
if (flushPageCatalog)
{
core.IO.PutPBuf(transaction, indexMeta.DiskPath, findResult.Catalog);
}
}
}
catch (Exception ex)
{
core.Log.Write(String.Format("Index document insert failed for process {0}.", transaction.ProcessId), ex);
throw;
}
}
/// <summary>
/// Inserts an index entry for a single document into a single index.
/// </summary>
/// <param name="transaction"></param>
/// <param name="schemaMeta"></param>
/// <param name="indexMeta"></param>
/// <param name="document"></param>
private void InsertDocumentIntoIndex(Transaction transaction, PersistSchema schemaMeta, PersistIndex indexMeta, PersistDocument document)
{
InsertDocumentIntoIndex(transaction, schemaMeta, indexMeta, document, null, true);
}
/// <summary>
/// Finds the appropriate index page for a set of key values using a long lived index page catalog.
/// </summary>
/// <param name="transaction"></param>
/// <param name="indexMeta"></param>
/// <param name="searchTokens"></param>
/// <param name="indexPageCatalog"></param>
/// <returns></returns>
private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens, PersistIndexPageCatalog suppliedIndexPageCatalog)
{
try
{
var indexPageCatalog = suppliedIndexPageCatalog;
if (indexPageCatalog == null)
{
indexPageCatalog = core.IO.GetPBuf <PersistIndexPageCatalog>(transaction, indexMeta.DiskPath, LockOperation.Write);
}
lock (indexPageCatalog)
{
FindKeyPageResult result = new FindKeyPageResult()
{
Catalog = indexPageCatalog
};
result.Extent = result.Catalog.Leaves;
if (result.Extent == null || result.Extent.Count == 0)
{
//The index is empty.
return(result);
}
int foundExtentCount = 0;
foreach (var token in searchTokens)
{
bool locatedExtent = false;
foreach (var leaf in result.Extent.Leaves)
{
if (leaf.Key == token)
{
locatedExtent = true;
foundExtentCount++;
result.Leaf = leaf;
result.Extent = leaf.Extent; //Move one level lower in the extent tree.
result.IsPartialMatch = true;
result.ExtentLevel = foundExtentCount;
if (foundExtentCount == searchTokens.Count)
{
result.IsPartialMatch = false;
result.IsFullMatch = true;
return(result);
}
}
}
if (locatedExtent == false)
{
return(result);
}
}
return(result);
}
}
catch (Exception ex)
{
core.Log.Write(String.Format("Failed to locate key page for process {0}.", transaction.ProcessId), ex);
throw;
}
}
/// <summary>
// Finds the appropriate index page for a set of key values.
/// </summary>
/// <param name="transaction"></param>
/// <param name="indexMeta"></param>
/// <param name="searchTokens"></param>
/// <returns></returns>
private FindKeyPageResult FindKeyPage(Transaction transaction, PersistIndex indexMeta, List <string> searchTokens)
{
return(FindKeyPage(transaction, indexMeta, searchTokens, null));
}