/// <summary> /// Gather statistics for one or all indices. /// </summary> /// <param name="name">Optional name of the index.</param> /// <returns>Indices statistics.</returns> public IndicesStats Stats(string name) { IndicesStats ret = new IndicesStats(); if (String.IsNullOrEmpty(name)) { lock (_IndicesLock) { foreach (KomodoIndex curr in _Indices) { ret.Stats.Add(curr.Stats()); } } } else { lock (_IndicesLock) { if (_Indices.Any(i => i.Name.Equals(name))) { KomodoIndex idx = _Indices.First(i => i.Name.Equals(name)); ret.Stats.Add(idx.Stats()); } else { // cannot find return(ret); } } } ret.Success = true; ret.Time.End = DateTime.UtcNow; return(ret); }
/// <summary> /// Remove an index, and optionally, destroy it. /// </summary> /// <param name="name">Name.</param> /// <param name="destroy">True if you wish to delete its data.</param> public async Task Remove(string name, bool destroy) { if (String.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } KomodoIndex idx = null; lock (_IndicesLock) { if (_Indices.Exists(i => i.Name.Equals(name))) { idx = _Indices.First(i => i.Name.Equals(name)); _Indices.Remove(idx); DbExpression e = new DbExpression( _ORM.GetColumnName <Index>(nameof(Index.GUID)), DbOperators.Equals, idx.GUID); _ORM.DeleteMany <Index>(e); } } if (idx != null) { if (destroy) { await idx.Destroy(); } idx.Dispose(); } }
/// <summary> /// Store or store-and-index a document. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceDoc">Source document.</param> /// <param name="data">Byte data.</param> /// <param name="options">Text parse options.</param> /// <param name="parse">True if the document should be parsed and indexed.</param> /// <param name="postbackUrl">URL to which results should be POSTed.</param> /// <returns>Index result.</returns> public async Task <IndexResult> AddDocument(string indexName, SourceDocument sourceDoc, byte[] data, ParseOptions options, bool parse = true, string postbackUrl = null) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } KomodoIndex idx = GetIndexClient(indexName); return(await idx.Add(sourceDoc, data, options, parse, postbackUrl)); }
/// <summary> /// Enumerate the index. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="query">Enumeration query.</param> /// <returns>Enumeraiton result.</returns> public EnumerationResult Enumerate(string indexName, EnumerationQuery query) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (query == null) { throw new ArgumentNullException(nameof(query)); } KomodoIndex idx = GetIndexClient(indexName); return(idx.Enumerate(query)); }
/// <summary> /// Search the index. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="query">Search query.</param> /// <returns>Search result.</returns> public SearchResult Search(string indexName, SearchQuery query) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (query == null) { throw new ArgumentNullException(nameof(query)); } KomodoIndex idx = GetIndexClient(indexName); return(idx.Search(query)); }
/// <summary> /// Remove a document. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceGuid">Source document GUID.</param> public void RemoveDocument(string indexName, string sourceGuid) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (String.IsNullOrEmpty(sourceGuid)) { throw new ArgumentNullException(nameof(sourceGuid)); } KomodoIndex idx = GetIndexClient(indexName); idx.Remove(sourceGuid); }
/// <summary> /// Retrieve parse result by source document GUID. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceGuid">Source document GUID.</param> /// <returns>Parse result.</returns> public ParseResult GetDocumentParseResult(string indexName, string sourceGuid) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (String.IsNullOrEmpty(sourceGuid)) { throw new ArgumentNullException(nameof(sourceGuid)); } KomodoIndex idx = GetIndexClient(indexName); return(idx.GetParseResult(sourceGuid)); }
/// <summary> /// Check if a parsed document exists by the source document GUID. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceGuid">Source document GUID.</param> /// <returns>True if exists.</returns> public bool ParsedDocumentExists(string indexName, string sourceGuid) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (String.IsNullOrEmpty(sourceGuid)) { throw new ArgumentNullException(nameof(sourceGuid)); } KomodoIndex idx = GetIndexClient(indexName); return(idx.ExistsParsed(sourceGuid)); }
/// <summary> /// Retrieve source document metadata. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceGuid">Source document GUID.</param> /// <returns>SourceDocument.</returns> public SourceDocument GetSourceDocumentMetadata(string indexName, string sourceGuid) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (String.IsNullOrEmpty(sourceGuid)) { throw new ArgumentNullException(nameof(sourceGuid)); } KomodoIndex idx = GetIndexClient(indexName); return(idx.GetSourceDocumentMetadata(sourceGuid)); }
/// <summary> /// Retrieve source document content. /// </summary> /// <param name="indexName">Name of the index.</param> /// <param name="sourceGuid">Source document GUID.</param> /// <returns>DocumentContent.</returns> public async Task <DocumentData> GetSourceDocumentContent(string indexName, string sourceGuid) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } if (String.IsNullOrEmpty(sourceGuid)) { throw new ArgumentNullException(nameof(sourceGuid)); } KomodoIndex idx = GetIndexClient(indexName); return(await idx.GetSourceDocumentContent(sourceGuid)); }
private KomodoIndex GetIndexClient(string indexName) { if (String.IsNullOrEmpty(indexName)) { throw new ArgumentNullException(nameof(indexName)); } KomodoIndex idx = _Indices.GetIndexClient(indexName); if (idx == null) { _Logging.Warn(_Header + "index " + indexName + " could not be found"); throw new KeyNotFoundException("No index with name '" + indexName + "' could be found."); } return(idx); }
/// <summary> /// Add an index. /// </summary> /// <param name="idx">Index.</param> /// <returns>Index.</returns> public KomodoIndex Add(Index idx) { if (idx == null) { throw new ArgumentNullException(nameof(idx)); } lock (_IndicesLock) { if (_Indices.Exists(i => i.Name.Equals(idx.Name))) { return(_Indices.First(i => i.Name.Equals(idx.Name))); } KomodoIndex ki = new KomodoIndex(_DbSettings, _SourceDocsStorageSettings, _ParsedDocsStorageSettings, idx); _ORM.Insert <Index>(idx); _Indices.Add(ki); return(ki); } }
static void Main(string[] args) { try { _DatabaseSettings = new DbSettings("test.db"); _SourceDocs = new StorageSettings(new DiskSettings("./source/")); _ParsedDocs = new StorageSettings(new DiskSettings("./parsed/")); _Postings = new StorageSettings(new DiskSettings("./postings/")); _Index = new Index("test", "test", "test"); _IndexClient = new KomodoIndex(_DatabaseSettings, _SourceDocs, _ParsedDocs, _Index); Console.WriteLine("Test cases"); Console.WriteLine(" 1 Basic indexing and queries"); Console.WriteLine(" 2 Basic indexing and enumeration"); Console.WriteLine(""); string testCase = Common.InputString("Selection:", "1", false); switch (testCase) { case "1": TestCase1(); break; case "2": TestCase2(); break; default: Console.WriteLine("Unknown test case."); break; } } catch (Exception e) { Console.WriteLine("Exception:" + Environment.NewLine + Common.SerializeJson(e, true)); } }
static void Main(string[] args) { #region Index-Manager Console.WriteLine("Initializing index manager"); _Indices = new KomodoIndices( new DbSettings("./indices.db"), new StorageSettings(new DiskSettings("./source/")), new StorageSettings(new DiskSettings("./parsed/"))); #endregion #region Indices Console.WriteLine("Initializing indices"); _Index1 = new Index("default", "default", "default"); _Index2 = new Index("metadata", "default", "metadata"); _Indices.Add(_Index1); _Indices.Add(_Index2); #endregion #region Adding-Documents _IndexClient1 = _Indices.GetIndexClient("default"); _IndexClient2 = _Indices.GetIndexClient("metadata"); byte[] doc1 = File.ReadAllBytes("person1.json"); SourceDocument sd1 = new SourceDocument( "default", "default", "Person 1", "Person 1", null, DocType.Json, null, "application/json", doc1.Length, Common.Md5(doc1)); byte[] doc2 = File.ReadAllBytes("person2.json"); SourceDocument sd2 = new SourceDocument( "default", "default", "Person 2", "Person 2", null, DocType.Json, null, "application/json", doc2.Length, Common.Md5(doc2)); byte[] doc3 = File.ReadAllBytes("person3.json"); SourceDocument sd3 = new SourceDocument( "default", "default", "Person 3", "Person 3", null, DocType.Json, null, "application/json", doc3.Length, Common.Md5(doc3)); IndexResult r1 = _IndexClient1.Add(sd1, doc1, new ParseOptions(), true).Result; IndexResult r2 = _IndexClient1.Add(sd2, doc2, new ParseOptions(), true).Result; IndexResult r3 = _IndexClient1.Add(sd3, doc3, new ParseOptions(), true).Result; #endregion #region Blobs _Blobs = new Blobs(new DiskSettings("./Metadata/")); if (!Directory.Exists("./Metadata/")) { Directory.CreateDirectory("./Metadata/"); } #endregion #region Policy byte[] bytes = File.ReadAllBytes("./policy.json"); _Policy = Common.DeserializeJson <MetadataPolicy>(File.ReadAllBytes("./policy.json")); #endregion #region Initialize-Metadata Console.WriteLine("Initializing metadata processor"); _Metadata = new MetadataProcessor(_Policy, _Indices); #endregion #region Apply-Metadata Console.WriteLine("Processing metadata"); _Result1 = _Metadata.ProcessDocument( r1.SourceDocument, r1.ParsedDocument, r1.ParseResult).Result; // Console.WriteLine("Document 1: " + Environment.NewLine + Common.SerializeJson(_Result1, true)); _Result2 = _Metadata.ProcessDocument( r2.SourceDocument, r2.ParsedDocument, r2.ParseResult).Result; // Console.WriteLine("Document 2: " + Environment.NewLine + Common.SerializeJson(_Result2, true)); _Result3 = _Metadata.ProcessDocument( r3.SourceDocument, r3.ParsedDocument, r3.ParseResult).Result; Console.WriteLine("Document 3: " + Environment.NewLine + Common.SerializeJson(_Result3, true)); #endregion }
/// <summary> /// Process a document using the configured rules. /// </summary> /// <param name="source">Source document.</param> /// <param name="parsed">Parsed document.</param> /// <param name="parseResult">Parse result.</param> /// <returns>Metadata result.</returns> public async Task <MetadataResult> ProcessDocument(SourceDocument source, ParsedDocument parsed, ParseResult parseResult) { if (source == null) { throw new ArgumentNullException(nameof(source)); } if (parsed == null) { throw new ArgumentNullException(nameof(parsed)); } if (parseResult == null) { throw new ArgumentNullException(nameof(parseResult)); } MetadataResult result = new MetadataResult(); result.Source = source; result.Parsed = parsed; result.ParseResult = parseResult; List <MetadataRule> matchingRules = GetMatchingRules(source, parsed, parseResult); if (matchingRules == null || matchingRules.Count < 1) { return(result); } foreach (MetadataRule rule in matchingRules) { result.MatchingRules.Add(rule); if (rule.AddMetadataDocument != null && rule.AddMetadataDocument.Count > 0) { foreach (AddMetadataDocumentAction addDocAction in rule.AddMetadataDocument) { if (addDocAction.Properties == null || addDocAction.Properties.Count < 1) { continue; } #region Retrieve-Index-Clients KomodoIndex src = _Indices.GetIndexClient(source.IndexGUID); if (src == null) { throw new InvalidOperationException("Unable to find source index " + source.IndexGUID); } KomodoIndex dst = _Indices.GetIndexClient(addDocAction.IndexGUID); if (dst == null) { throw new InvalidOperationException("Unable to find destination index " + addDocAction.IndexGUID); } #endregion #region Generate-Derived-Metadata-Document Dictionary <string, object> derivedDocument = new Dictionary <string, object>(); foreach (MetadataDocumentProperty prop in addDocAction.Properties) { if (prop.ValueAction == PropertyValueAction.CopyFromDocument) { string val = GetValueFromParseResult(parseResult, prop.SourceProperty); derivedDocument.Add(prop.Key, val); } else if (prop.ValueAction == PropertyValueAction.Static) { derivedDocument.Add(prop.Key, prop.Value); } } byte[] derivedDocBytes = Encoding.UTF8.GetBytes(Common.SerializeJson(derivedDocument, true)); #endregion #region Store-in-Database MetadataDocument metadataDoc = new MetadataDocument(); metadataDoc.Created = DateTime.UtcNow; metadataDoc.GUID = Guid.NewGuid().ToString(); metadataDoc.IndexGUID = source.IndexGUID; metadataDoc.OwnerGUID = source.OwnerGUID; metadataDoc.SourceDocumentGUID = source.GUID; metadataDoc.TargetIndexGUID = addDocAction.IndexGUID; metadataDoc.Type = DocType.Json; metadataDoc = src.AddMetadata(source, metadataDoc); #endregion #region Index SourceDocument derivedSourceDoc = new SourceDocument( metadataDoc.GUID, source.OwnerGUID, addDocAction.IndexGUID, addDocAction.Name, addDocAction.Title, addDocAction.Tags, DocType.Json, null, "application/json", derivedDocBytes.Length, Common.Md5(derivedDocBytes)); IndexResult idxResult = await dst.Add(derivedSourceDoc, derivedDocBytes, new ParseOptions(), addDocAction.Parse); #endregion #region Store-Results result.MetadataDocuments.Add(metadataDoc); result.DerivedDocuments.Add(idxResult.SourceDocument); result.DerivedDocumentsData.Add(derivedDocument); result.DerivedIndexResults.Add(idxResult); #endregion } } } foreach (MetadataRule rule in matchingRules) { if (rule.Postback != null && rule.Postback.Urls != null && rule.Postback.Urls.Count > 0) { MetadataResult postbackMetadata = Common.CopyObject <MetadataResult>(result); if (!rule.Postback.IncludeSource) { postbackMetadata.Source = null; } if (!rule.Postback.IncludeParsed) { postbackMetadata.Parsed = null; } if (!rule.Postback.IncludeParseResult) { postbackMetadata.ParseResult = null; } if (!rule.Postback.IncludeMetadata) { postbackMetadata.MetadataDocuments = null; } if (!rule.Postback.IncludeRules) { postbackMetadata.MatchingRules = null; } if (!rule.Postback.IncludeDerivedDocuments) { postbackMetadata.DerivedDocuments = null; } rule.Postback.Urls = rule.Postback.Urls.Distinct().ToList(); foreach (string url in rule.Postback.Urls) { if (String.IsNullOrEmpty(url)) { continue; } RestRequest req = new RestRequest( url, HttpMethod.POST, null, "application/json"); RestResponse resp = req.Send(Common.SerializeJson(result, true)); result.PostbackStatusCodes.Add(url, resp.StatusCode); } } } return(result); }