/// <summary>
/// Gather statistics for one or all indices.
/// </summary>
/// <param name="name">Optional name of the index.</param>
/// <returns>Indices statistics.</returns>
public IndicesStats Stats(string name)
{
IndicesStats ret = new IndicesStats();
if (String.IsNullOrEmpty(name))
{
lock (_IndicesLock)
{
foreach (KomodoIndex curr in _Indices)
{
ret.Stats.Add(curr.Stats());
}
}
}
else
{
lock (_IndicesLock)
{
if (_Indices.Any(i => i.Name.Equals(name)))
{
KomodoIndex idx = _Indices.First(i => i.Name.Equals(name));
ret.Stats.Add(idx.Stats());
}
else
{
// cannot find
return(ret);
}
}
}
ret.Success = true;
ret.Time.End = DateTime.UtcNow;
return(ret);
}
/// <summary>
/// Remove an index, and optionally, destroy it.
/// </summary>
/// <param name="name">Name.</param>
/// <param name="destroy">True if you wish to delete its data.</param>
public async Task Remove(string name, bool destroy)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
KomodoIndex idx = null;
lock (_IndicesLock)
{
if (_Indices.Exists(i => i.Name.Equals(name)))
{
idx = _Indices.First(i => i.Name.Equals(name));
_Indices.Remove(idx);
DbExpression e = new DbExpression(
_ORM.GetColumnName <Index>(nameof(Index.GUID)),
DbOperators.Equals,
idx.GUID);
_ORM.DeleteMany <Index>(e);
}
}
if (idx != null)
{
if (destroy)
{
await idx.Destroy();
}
idx.Dispose();
}
}
/// <summary>
/// Store or store-and-index a document.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceDoc">Source document.</param>
/// <param name="data">Byte data.</param>
/// <param name="options">Text parse options.</param>
/// <param name="parse">True if the document should be parsed and indexed.</param>
/// <param name="postbackUrl">URL to which results should be POSTed.</param>
/// <returns>Index result.</returns>
public async Task <IndexResult> AddDocument(string indexName, SourceDocument sourceDoc, byte[] data, ParseOptions options, bool parse = true, string postbackUrl = null)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
KomodoIndex idx = GetIndexClient(indexName);
return(await idx.Add(sourceDoc, data, options, parse, postbackUrl));
}
/// <summary>
/// Enumerate the index.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="query">Enumeration query.</param>
/// <returns>Enumeraiton result.</returns>
public EnumerationResult Enumerate(string indexName, EnumerationQuery query)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
KomodoIndex idx = GetIndexClient(indexName);
return(idx.Enumerate(query));
}
/// <summary>
/// Search the index.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="query">Search query.</param>
/// <returns>Search result.</returns>
public SearchResult Search(string indexName, SearchQuery query)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
KomodoIndex idx = GetIndexClient(indexName);
return(idx.Search(query));
}
/// <summary>
/// Remove a document.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceGuid">Source document GUID.</param>
public void RemoveDocument(string indexName, string sourceGuid)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (String.IsNullOrEmpty(sourceGuid))
{
throw new ArgumentNullException(nameof(sourceGuid));
}
KomodoIndex idx = GetIndexClient(indexName);
idx.Remove(sourceGuid);
}
/// <summary>
/// Retrieve parse result by source document GUID.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceGuid">Source document GUID.</param>
/// <returns>Parse result.</returns>
public ParseResult GetDocumentParseResult(string indexName, string sourceGuid)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (String.IsNullOrEmpty(sourceGuid))
{
throw new ArgumentNullException(nameof(sourceGuid));
}
KomodoIndex idx = GetIndexClient(indexName);
return(idx.GetParseResult(sourceGuid));
}
/// <summary>
/// Check if a parsed document exists by the source document GUID.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceGuid">Source document GUID.</param>
/// <returns>True if exists.</returns>
public bool ParsedDocumentExists(string indexName, string sourceGuid)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (String.IsNullOrEmpty(sourceGuid))
{
throw new ArgumentNullException(nameof(sourceGuid));
}
KomodoIndex idx = GetIndexClient(indexName);
return(idx.ExistsParsed(sourceGuid));
}
/// <summary>
/// Retrieve source document metadata.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceGuid">Source document GUID.</param>
/// <returns>SourceDocument.</returns>
public SourceDocument GetSourceDocumentMetadata(string indexName, string sourceGuid)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (String.IsNullOrEmpty(sourceGuid))
{
throw new ArgumentNullException(nameof(sourceGuid));
}
KomodoIndex idx = GetIndexClient(indexName);
return(idx.GetSourceDocumentMetadata(sourceGuid));
}
/// <summary>
/// Retrieve source document content.
/// </summary>
/// <param name="indexName">Name of the index.</param>
/// <param name="sourceGuid">Source document GUID.</param>
/// <returns>DocumentContent.</returns>
public async Task <DocumentData> GetSourceDocumentContent(string indexName, string sourceGuid)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
if (String.IsNullOrEmpty(sourceGuid))
{
throw new ArgumentNullException(nameof(sourceGuid));
}
KomodoIndex idx = GetIndexClient(indexName);
return(await idx.GetSourceDocumentContent(sourceGuid));
}
private KomodoIndex GetIndexClient(string indexName)
{
if (String.IsNullOrEmpty(indexName))
{
throw new ArgumentNullException(nameof(indexName));
}
KomodoIndex idx = _Indices.GetIndexClient(indexName);
if (idx == null)
{
_Logging.Warn(_Header + "index " + indexName + " could not be found");
throw new KeyNotFoundException("No index with name '" + indexName + "' could be found.");
}
return(idx);
}
/// <summary>
/// Add an index.
/// </summary>
/// <param name="idx">Index.</param>
/// <returns>Index.</returns>
public KomodoIndex Add(Index idx)
{
if (idx == null)
{
throw new ArgumentNullException(nameof(idx));
}
lock (_IndicesLock)
{
if (_Indices.Exists(i => i.Name.Equals(idx.Name)))
{
return(_Indices.First(i => i.Name.Equals(idx.Name)));
}
KomodoIndex ki = new KomodoIndex(_DbSettings, _SourceDocsStorageSettings, _ParsedDocsStorageSettings, idx);
_ORM.Insert <Index>(idx);
_Indices.Add(ki);
return(ki);
}
}
static void Main(string[] args)
{
try
{
_DatabaseSettings = new DbSettings("test.db");
_SourceDocs = new StorageSettings(new DiskSettings("./source/"));
_ParsedDocs = new StorageSettings(new DiskSettings("./parsed/"));
_Postings = new StorageSettings(new DiskSettings("./postings/"));
_Index = new Index("test", "test", "test");
_IndexClient = new KomodoIndex(_DatabaseSettings, _SourceDocs, _ParsedDocs, _Index);
Console.WriteLine("Test cases");
Console.WriteLine(" 1 Basic indexing and queries");
Console.WriteLine(" 2 Basic indexing and enumeration");
Console.WriteLine("");
string testCase = Common.InputString("Selection:", "1", false);
switch (testCase)
{
case "1":
TestCase1();
break;
case "2":
TestCase2();
break;
default:
Console.WriteLine("Unknown test case.");
break;
}
}
catch (Exception e)
{
Console.WriteLine("Exception:" + Environment.NewLine + Common.SerializeJson(e, true));
}
}
static void Main(string[] args)
{
#region Index-Manager
Console.WriteLine("Initializing index manager");
_Indices = new KomodoIndices(
new DbSettings("./indices.db"),
new StorageSettings(new DiskSettings("./source/")),
new StorageSettings(new DiskSettings("./parsed/")));
#endregion
#region Indices
Console.WriteLine("Initializing indices");
_Index1 = new Index("default", "default", "default");
_Index2 = new Index("metadata", "default", "metadata");
_Indices.Add(_Index1);
_Indices.Add(_Index2);
#endregion
#region Adding-Documents
_IndexClient1 = _Indices.GetIndexClient("default");
_IndexClient2 = _Indices.GetIndexClient("metadata");
byte[] doc1 = File.ReadAllBytes("person1.json");
SourceDocument sd1 = new SourceDocument(
"default",
"default",
"Person 1",
"Person 1",
null,
DocType.Json,
null,
"application/json",
doc1.Length,
Common.Md5(doc1));
byte[] doc2 = File.ReadAllBytes("person2.json");
SourceDocument sd2 = new SourceDocument(
"default",
"default",
"Person 2",
"Person 2",
null,
DocType.Json,
null,
"application/json",
doc2.Length,
Common.Md5(doc2));
byte[] doc3 = File.ReadAllBytes("person3.json");
SourceDocument sd3 = new SourceDocument(
"default",
"default",
"Person 3",
"Person 3",
null,
DocType.Json,
null,
"application/json",
doc3.Length,
Common.Md5(doc3));
IndexResult r1 = _IndexClient1.Add(sd1, doc1, new ParseOptions(), true).Result;
IndexResult r2 = _IndexClient1.Add(sd2, doc2, new ParseOptions(), true).Result;
IndexResult r3 = _IndexClient1.Add(sd3, doc3, new ParseOptions(), true).Result;
#endregion
#region Blobs
_Blobs = new Blobs(new DiskSettings("./Metadata/"));
if (!Directory.Exists("./Metadata/"))
{
Directory.CreateDirectory("./Metadata/");
}
#endregion
#region Policy
byte[] bytes = File.ReadAllBytes("./policy.json");
_Policy = Common.DeserializeJson <MetadataPolicy>(File.ReadAllBytes("./policy.json"));
#endregion
#region Initialize-Metadata
Console.WriteLine("Initializing metadata processor");
_Metadata = new MetadataProcessor(_Policy, _Indices);
#endregion
#region Apply-Metadata
Console.WriteLine("Processing metadata");
_Result1 = _Metadata.ProcessDocument(
r1.SourceDocument,
r1.ParsedDocument,
r1.ParseResult).Result;
// Console.WriteLine("Document 1: " + Environment.NewLine + Common.SerializeJson(_Result1, true));
_Result2 = _Metadata.ProcessDocument(
r2.SourceDocument,
r2.ParsedDocument,
r2.ParseResult).Result;
// Console.WriteLine("Document 2: " + Environment.NewLine + Common.SerializeJson(_Result2, true));
_Result3 = _Metadata.ProcessDocument(
r3.SourceDocument,
r3.ParsedDocument,
r3.ParseResult).Result;
Console.WriteLine("Document 3: " + Environment.NewLine + Common.SerializeJson(_Result3, true));
#endregion
}
/// <summary>
/// Process a document using the configured rules.
/// </summary>
/// <param name="source">Source document.</param>
/// <param name="parsed">Parsed document.</param>
/// <param name="parseResult">Parse result.</param>
/// <returns>Metadata result.</returns>
public async Task <MetadataResult> ProcessDocument(SourceDocument source, ParsedDocument parsed, ParseResult parseResult)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (parsed == null)
{
throw new ArgumentNullException(nameof(parsed));
}
if (parseResult == null)
{
throw new ArgumentNullException(nameof(parseResult));
}
MetadataResult result = new MetadataResult();
result.Source = source;
result.Parsed = parsed;
result.ParseResult = parseResult;
List <MetadataRule> matchingRules = GetMatchingRules(source, parsed, parseResult);
if (matchingRules == null || matchingRules.Count < 1)
{
return(result);
}
foreach (MetadataRule rule in matchingRules)
{
result.MatchingRules.Add(rule);
if (rule.AddMetadataDocument != null && rule.AddMetadataDocument.Count > 0)
{
foreach (AddMetadataDocumentAction addDocAction in rule.AddMetadataDocument)
{
if (addDocAction.Properties == null || addDocAction.Properties.Count < 1)
{
continue;
}
#region Retrieve-Index-Clients
KomodoIndex src = _Indices.GetIndexClient(source.IndexGUID);
if (src == null)
{
throw new InvalidOperationException("Unable to find source index " + source.IndexGUID);
}
KomodoIndex dst = _Indices.GetIndexClient(addDocAction.IndexGUID);
if (dst == null)
{
throw new InvalidOperationException("Unable to find destination index " + addDocAction.IndexGUID);
}
#endregion
#region Generate-Derived-Metadata-Document
Dictionary <string, object> derivedDocument = new Dictionary <string, object>();
foreach (MetadataDocumentProperty prop in addDocAction.Properties)
{
if (prop.ValueAction == PropertyValueAction.CopyFromDocument)
{
string val = GetValueFromParseResult(parseResult, prop.SourceProperty);
derivedDocument.Add(prop.Key, val);
}
else if (prop.ValueAction == PropertyValueAction.Static)
{
derivedDocument.Add(prop.Key, prop.Value);
}
}
byte[] derivedDocBytes = Encoding.UTF8.GetBytes(Common.SerializeJson(derivedDocument, true));
#endregion
#region Store-in-Database
MetadataDocument metadataDoc = new MetadataDocument();
metadataDoc.Created = DateTime.UtcNow;
metadataDoc.GUID = Guid.NewGuid().ToString();
metadataDoc.IndexGUID = source.IndexGUID;
metadataDoc.OwnerGUID = source.OwnerGUID;
metadataDoc.SourceDocumentGUID = source.GUID;
metadataDoc.TargetIndexGUID = addDocAction.IndexGUID;
metadataDoc.Type = DocType.Json;
metadataDoc = src.AddMetadata(source, metadataDoc);
#endregion
#region Index
SourceDocument derivedSourceDoc = new SourceDocument(
metadataDoc.GUID,
source.OwnerGUID,
addDocAction.IndexGUID,
addDocAction.Name,
addDocAction.Title,
addDocAction.Tags,
DocType.Json,
null,
"application/json",
derivedDocBytes.Length,
Common.Md5(derivedDocBytes));
IndexResult idxResult = await dst.Add(derivedSourceDoc, derivedDocBytes, new ParseOptions(), addDocAction.Parse);
#endregion
#region Store-Results
result.MetadataDocuments.Add(metadataDoc);
result.DerivedDocuments.Add(idxResult.SourceDocument);
result.DerivedDocumentsData.Add(derivedDocument);
result.DerivedIndexResults.Add(idxResult);
#endregion
}
}
}
foreach (MetadataRule rule in matchingRules)
{
if (rule.Postback != null &&
rule.Postback.Urls != null &&
rule.Postback.Urls.Count > 0)
{
MetadataResult postbackMetadata = Common.CopyObject <MetadataResult>(result);
if (!rule.Postback.IncludeSource)
{
postbackMetadata.Source = null;
}
if (!rule.Postback.IncludeParsed)
{
postbackMetadata.Parsed = null;
}
if (!rule.Postback.IncludeParseResult)
{
postbackMetadata.ParseResult = null;
}
if (!rule.Postback.IncludeMetadata)
{
postbackMetadata.MetadataDocuments = null;
}
if (!rule.Postback.IncludeRules)
{
postbackMetadata.MatchingRules = null;
}
if (!rule.Postback.IncludeDerivedDocuments)
{
postbackMetadata.DerivedDocuments = null;
}
rule.Postback.Urls = rule.Postback.Urls.Distinct().ToList();
foreach (string url in rule.Postback.Urls)
{
if (String.IsNullOrEmpty(url))
{
continue;
}
RestRequest req = new RestRequest(
url,
HttpMethod.POST,
null,
"application/json");
RestResponse resp = req.Send(Common.SerializeJson(result, true));
result.PostbackStatusCodes.Add(url, resp.StatusCode);
}
}
}
return(result);
}
