/// <summary>
/// Add an index.
/// </summary>
/// <param name="index">Index.</param>
/// <returns>Index.</returns>
public void AddIndex(Index index)
{
if (index == null)
{
throw new ArgumentNullException(nameof(index));
}
_Indices.Add(index);
}
static void AddIndex()
{
Index index = new Index(
Common.InputString("GUID :", null, false),
Common.InputString("Owner :", null, false),
Common.InputString("Name :", null, false));
_Komodo.AddIndex(index);
}
static void GetIndex()
{
string indexName = Common.InputString("Index name:", "default", false);
Index index = _Komodo.GetIndex(indexName);
if (index == null)
{
Console.WriteLine("(none)");
}
else
{
Console.WriteLine(Common.SerializeJson(index, true));
}
}
/// <summary>
/// Add an index.
/// </summary>
/// <param name="idx">Index.</param>
/// <returns>Index.</returns>
public KomodoIndex Add(Index idx)
{
if (idx == null)
{
throw new ArgumentNullException(nameof(idx));
}
lock (_IndicesLock)
{
if (_Indices.Exists(i => i.Name.Equals(idx.Name)))
{
return(_Indices.First(i => i.Name.Equals(idx.Name)));
}
KomodoIndex ki = new KomodoIndex(_DbSettings, _SourceDocsStorageSettings, _ParsedDocsStorageSettings, idx);
_ORM.Insert <Index>(idx);
_Indices.Add(ki);
return(ki);
}
}
private static async Task PostIndices(RequestMetadata md)
{
string header = "[Komodo.Server] " + md.Http.Request.Source.IpAddress + ":" + md.Http.Request.Source.Port + " PostIndices ";
if (md.Http.Request.Data == null || md.Http.Request.ContentLength < 1)
{
md.Http.Response.StatusCode = 400;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "No request body.", null, null).ToJson(true));
return;
}
byte[] reqData = Common.StreamToBytes(md.Http.Request.Data);
Index index = Common.DeserializeJson <Index>(reqData);
index.OwnerGUID = md.User.GUID;
if (String.IsNullOrEmpty(index.GUID))
{
index.GUID = Guid.NewGuid().ToString();
}
if (_Daemon.IndexExists(index.Name))
{
_Logging.Warn(header + "index " + index.Name + " already exists");
md.Http.Response.StatusCode = 409;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(409, "Index already exists.", null, null).ToJson(true));
return;
}
_Daemon.AddIndex(index);
_Logging.Debug(header + "created index " + index.Name);
md.Http.Response.StatusCode = 201;
await md.Http.Response.Send();
return;
}
private static async Task GetIndex(RequestMetadata md)
{
string header = "[Komodo.Server] " + md.Http.Request.Source.IpAddress + ":" + md.Http.Request.Source.Port + " GetIndex ";
string indexName = md.Http.Request.Url.Elements[0];
Index index = _Daemon.GetIndex(indexName);
if (index == null)
{
_Logging.Warn(header + "index " + indexName + " does not exist");
md.Http.Response.StatusCode = 404;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(404, "Unknown index.", null, null).ToJson(true));
return;
}
md.Http.Response.StatusCode = 200;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(Common.SerializeJson(index, md.Params.Pretty));
return;
}
static void Main(string[] args)
{
try
{
_DatabaseSettings = new DbSettings("test.db");
_SourceDocs = new StorageSettings(new DiskSettings("./source/"));
_ParsedDocs = new StorageSettings(new DiskSettings("./parsed/"));
_Postings = new StorageSettings(new DiskSettings("./postings/"));
_Index = new Index("test", "test", "test");
_IndexClient = new KomodoIndex(_DatabaseSettings, _SourceDocs, _ParsedDocs, _Index);
Console.WriteLine("Test cases");
Console.WriteLine(" 1 Basic indexing and queries");
Console.WriteLine(" 2 Basic indexing and enumeration");
Console.WriteLine("");
string testCase = Common.InputString("Selection:", "1", false);
switch (testCase)
{
case "1":
TestCase1();
break;
case "2":
TestCase2();
break;
default:
Console.WriteLine("Unknown test case.");
break;
}
}
catch (Exception e)
{
Console.WriteLine("Exception:" + Environment.NewLine + Common.SerializeJson(e, true));
}
}
static void Main(string[] args)
{
#region Index-Manager
Console.WriteLine("Initializing index manager");
_Indices = new KomodoIndices(
new DbSettings("./indices.db"),
new StorageSettings(new DiskSettings("./source/")),
new StorageSettings(new DiskSettings("./parsed/")));
#endregion
#region Indices
Console.WriteLine("Initializing indices");
_Index1 = new Index("default", "default", "default");
_Index2 = new Index("metadata", "default", "metadata");
_Indices.Add(_Index1);
_Indices.Add(_Index2);
#endregion
#region Adding-Documents
_IndexClient1 = _Indices.GetIndexClient("default");
_IndexClient2 = _Indices.GetIndexClient("metadata");
byte[] doc1 = File.ReadAllBytes("person1.json");
SourceDocument sd1 = new SourceDocument(
"default",
"default",
"Person 1",
"Person 1",
null,
DocType.Json,
null,
"application/json",
doc1.Length,
Common.Md5(doc1));
byte[] doc2 = File.ReadAllBytes("person2.json");
SourceDocument sd2 = new SourceDocument(
"default",
"default",
"Person 2",
"Person 2",
null,
DocType.Json,
null,
"application/json",
doc2.Length,
Common.Md5(doc2));
byte[] doc3 = File.ReadAllBytes("person3.json");
SourceDocument sd3 = new SourceDocument(
"default",
"default",
"Person 3",
"Person 3",
null,
DocType.Json,
null,
"application/json",
doc3.Length,
Common.Md5(doc3));
IndexResult r1 = _IndexClient1.Add(sd1, doc1, new ParseOptions(), true).Result;
IndexResult r2 = _IndexClient1.Add(sd2, doc2, new ParseOptions(), true).Result;
IndexResult r3 = _IndexClient1.Add(sd3, doc3, new ParseOptions(), true).Result;
#endregion
#region Blobs
_Blobs = new Blobs(new DiskSettings("./Metadata/"));
if (!Directory.Exists("./Metadata/"))
{
Directory.CreateDirectory("./Metadata/");
}
#endregion
#region Policy
byte[] bytes = File.ReadAllBytes("./policy.json");
_Policy = Common.DeserializeJson <MetadataPolicy>(File.ReadAllBytes("./policy.json"));
#endregion
#region Initialize-Metadata
Console.WriteLine("Initializing metadata processor");
_Metadata = new MetadataProcessor(_Policy, _Indices);
#endregion
#region Apply-Metadata
Console.WriteLine("Processing metadata");
_Result1 = _Metadata.ProcessDocument(
r1.SourceDocument,
r1.ParsedDocument,
r1.ParseResult).Result;
// Console.WriteLine("Document 1: " + Environment.NewLine + Common.SerializeJson(_Result1, true));
_Result2 = _Metadata.ProcessDocument(
r2.SourceDocument,
r2.ParsedDocument,
r2.ParseResult).Result;
// Console.WriteLine("Document 2: " + Environment.NewLine + Common.SerializeJson(_Result2, true));
_Result3 = _Metadata.ProcessDocument(
r3.SourceDocument,
r3.ParsedDocument,
r3.ParseResult).Result;
Console.WriteLine("Document 3: " + Environment.NewLine + Common.SerializeJson(_Result3, true));
#endregion
}
private void RunSetup()
{
#region Variables
DateTime timestamp = DateTime.UtcNow;
Settings settings = new Settings();
#endregion
#region Welcome
Console.WriteLine(
Environment.NewLine +
Environment.NewLine +
"oooo .o8 " + Environment.NewLine +
"`888 888 " + Environment.NewLine +
" 888 oooo .ooooo. ooo. .oo. .oo. .ooooo. .oooo888 .ooooo. " + Environment.NewLine +
" 888 .8P' d88' `88b `888P'Y88bP'Y88b d88' `88b d88' `888 d88' `88b " + Environment.NewLine +
" 888888. 888 888 888 888 888 888 888 888 888 888 888 " + Environment.NewLine +
" 888 `88b. 888 888 888 888 888 888 888 888 888 888 888 " + Environment.NewLine +
"o888o o888o `Y8bod8P' o888o o888o o888o `Y8bod8P' `Y8bod88P `Y8bod8P' " + Environment.NewLine +
Environment.NewLine +
Environment.NewLine);
// ________________ 1 2 3 4 5 6 7
// ________________12345678901234567890123456789012345678901234567890123456789012345678901234567890
Console.WriteLine("Thank you for using Komodo! We'll put together a basic system configuration");
Console.WriteLine("so you can be up and running quickly.");
Console.WriteLine("");
Console.WriteLine(Common.Line(79, "-"));
Console.WriteLine("");
#endregion
#region Initial-Settings
settings.EnableConsole = true;
settings.Server = new Settings.ServerSettings();
settings.Server.HeaderApiKey = "x-api-key";
settings.Server.AdminApiKey = "komodoadmin";
settings.Server.ListenerPort = 9090;
settings.Server.ListenerHostname = "localhost";
settings.Logging = new Settings.LoggingSettings();
settings.Logging.ConsoleLogging = true;
settings.Logging.Header = "komodo";
settings.Logging.SyslogServerIp = "127.0.0.1";
settings.Logging.SyslogServerPort = 514;
settings.Logging.MinimumLevel = Severity.Info;
settings.Logging.FileLogging = true;
settings.Logging.FileDirectory = "./logs/";
settings.Logging.Filename = "Komodo.log";
if (!Directory.Exists("./data/"))
{
Directory.CreateDirectory("./data/");
}
if (!Directory.Exists("./logs/"))
{
Directory.CreateDirectory("./logs/");
}
settings.Database = new DbSettings("./data/komodo.db");
string tempDirectory = "./data/temp/";
settings.TempStorage = new StorageSettings(new DiskSettings(tempDirectory));
if (!Directory.Exists(tempDirectory))
{
Directory.CreateDirectory(tempDirectory);
}
string sourceDirectory = "./data/source/";
settings.SourceDocuments = new StorageSettings(new DiskSettings(sourceDirectory));
if (!Directory.Exists(sourceDirectory))
{
Directory.CreateDirectory(sourceDirectory);
}
string parsedDirectory = "./data/parsed/";
settings.ParsedDocuments = new StorageSettings(new DiskSettings(parsedDirectory));
if (!Directory.Exists(parsedDirectory))
{
Directory.CreateDirectory(parsedDirectory);
}
#endregion
#region Initialize-Database-and-Create-Records
WatsonORM orm = new WatsonORM(settings.Database.ToDatabaseSettings());
orm.InitializeDatabase();
orm.InitializeTable(typeof(ApiKey));
orm.InitializeTable(typeof(Index));
orm.InitializeTable(typeof(Metadata));
orm.InitializeTable(typeof(MetadataDocument));
orm.InitializeTable(typeof(Node));
orm.InitializeTable(typeof(ParsedDocument));
orm.InitializeTable(typeof(Permission));
orm.InitializeTable(typeof(SourceDocument));
orm.InitializeTable(typeof(TermDoc));
orm.InitializeTable(typeof(TermGuid));
orm.InitializeTable(typeof(User));
DbExpression e = new DbExpression("id", DbOperators.GreaterThan, 0);
User user = null;
ApiKey apiKey = null;
Permission perm = null;
Index idx = null;
List <User> users = orm.SelectMany <User>(e);
if (users == null || users.Count < 1)
{
Console.WriteLine("| Creating first user 'default'");
user = new User("default", "Default", "*****@*****.**", "default");
user = orm.Insert <User>(user);
}
else
{
Console.WriteLine("| Users already exist, not creating default user");
}
List <Index> indices = orm.SelectMany <Index>(e);
if (indices == null || indices.Count < 1)
{
Console.WriteLine("| Creating first index 'default'");
idx = new Index(user.GUID, "default");
idx = orm.Insert <Index>(idx);
}
else
{
Console.WriteLine("| Indices already exist, not creating default index");
}
List <ApiKey> keys = orm.SelectMany <ApiKey>(e);
if (keys == null || keys.Count < 1)
{
Console.WriteLine("| Creating first API key 'default'");
apiKey = new ApiKey("default", user.GUID, true);
apiKey = orm.Insert <ApiKey>(apiKey);
}
else
{
Console.WriteLine("| API keys already exist, not creating default API key");
}
List <Permission> perms = orm.SelectMany <Permission>(e);
if (perms == null || perms.Count < 1)
{
Console.WriteLine("| Creating first permission 'default'");
perm = new Permission(idx.GUID, user.GUID, apiKey.GUID, true, true, true, true, true);
perm = orm.Insert <Permission>(perm);
}
else
{
Console.WriteLine("| Permissions already exist, not creating default permissions");
}
#endregion
#region Write-System-JSON
File.WriteAllBytes("./system.json", Encoding.UTF8.GetBytes(Common.SerializeJson(settings, true)));
#endregion
#region Wrap-Up
string baseUrl = "http://localhost:" + settings.Server.ListenerPort;
// ________________ 1 2 3 4 5 6 7
// ________________12345678901234567890123456789012345678901234567890123456789012345678901234567890
Console.WriteLine("");
Console.WriteLine("All finished!");
Console.WriteLine("");
Console.WriteLine("If you ever want to return to this setup wizard, just re-run the application");
Console.WriteLine("from the terminal with the 'setup' argument.");
Console.WriteLine("");
Console.WriteLine("Verify Komodo is running in your browser using the following URL:");
Console.WriteLine("");
Console.WriteLine(" " + baseUrl);
Console.WriteLine("");
Console.WriteLine("We've created your first index for you called 'First'. Try POSTing a JSON");
Console.WriteLine("document to the index using the API key 'default' using the URL:");
Console.WriteLine("");
Console.WriteLine(" " + baseUrl + "/default?type=json&name=My+First+Document&x-api-key=default");
Console.WriteLine("");
#endregion
}
private static async Task PostIndexDocument(RequestMetadata md)
{
#region Variables
string header = "[Komodo.Server] " + md.Http.Request.Source.IpAddress + ":" + md.Http.Request.Source.Port + " PostIndexDocument ";
string tempFile = _Settings.TempStorage.Disk.Directory + Guid.NewGuid().ToString();
string indexName = md.Http.Request.Url.Elements[0];
string sourceGuid = null;
if (md.Http.Request.Url.Elements.Length == 2)
{
sourceGuid = md.Http.Request.Url.Elements[1];
}
#endregion
#region Check-Index-Existence
Index index = _Daemon.GetIndex(indexName);
if (index == null)
{
_Logging.Warn(header + "index " + indexName + " does not exist");
md.Http.Response.StatusCode = 404;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(404, "Unknown index.", null, null).ToJson(true));
return;
}
#endregion
#region Check-Supplied-GUID
if (!String.IsNullOrEmpty(sourceGuid))
{
if (_Daemon.SourceDocumentExists(indexName, sourceGuid))
{
_Logging.Warn(header + "document " + indexName + "/" + sourceGuid + " already exists");
md.Http.Response.StatusCode = 409;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(409, "Requested GUID already exists.", null, null).ToJson(true));
return;
}
}
#endregion
#region Retrieve-DocType-from-QS
if (String.IsNullOrEmpty(md.Params.Type))
{
_Logging.Warn(header + "no 'type' value found in querystring");
md.Http.Response.StatusCode = 400;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "Supply 'type' [json/xml/html/sql/text] in querystring.", null, null).ToJson(true));
return;
}
DocType docType = DocType.Json;
switch (md.Params.Type)
{
case "json":
docType = DocType.Json;
break;
case "xml":
docType = DocType.Xml;
break;
case "html":
docType = DocType.Html;
break;
case "sql":
docType = DocType.Sql;
break;
case "text":
docType = DocType.Text;
break;
case "unknown":
docType = DocType.Unknown;
break;
default:
_Logging.Warn(header + "invalid 'type' value found in querystring: " + md.Params.Type);
md.Http.Response.StatusCode = 400;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "Supply 'type' [json/xml/html/sql/text] in querystring.", null, null).ToJson(true));
return;
}
#endregion
try
{
#region Write-Temp-File
long contentLength = 0;
string md5 = null;
CrawlResult crawlResult = null;
if (!String.IsNullOrEmpty(md.Params.Url) || !String.IsNullOrEmpty(md.Params.Filename))
{
#region Crawl
if (!String.IsNullOrEmpty(md.Params.Url))
{
HttpCrawler httpCrawler = new HttpCrawler(md.Params.Url);
crawlResult = httpCrawler.Download(tempFile);
if (!crawlResult.Success)
{
_Logging.Warn(header + "failed to crawl URL " + md.Params.Url);
md.Http.Response.StatusCode = 500;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "Failed to crawl supplied URL.", null, crawlResult).ToJson(true));
return;
}
contentLength = crawlResult.ContentLength;
md5 = Common.Md5File(tempFile);
}
else
{
FileCrawler fileCrawler = new FileCrawler(md.Params.Filename);
crawlResult = fileCrawler.Download(tempFile);
if (!crawlResult.Success)
{
_Logging.Warn(header + "failed to crawl filename " + md.Params.Filename);
md.Http.Response.StatusCode = 500;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(400, "Failed to crawl supplied filename.", null, crawlResult).ToJson(true));
return;
}
contentLength = crawlResult.ContentLength;
md5 = Common.Md5(tempFile);
}
#endregion
}
else
{
using (FileStream fs = new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite))
{
await md.Http.Request.Data.CopyToAsync(fs);
}
contentLength = md.Http.Request.ContentLength;
md5 = Common.Md5File(tempFile);
}
#endregion
#region Build-Source-Document
string sourceUrl = null;
if (!String.IsNullOrEmpty(md.Params.Url))
{
sourceUrl = md.Params.Url;
}
else if (!String.IsNullOrEmpty(md.Params.Filename))
{
sourceUrl = md.Params.Filename;
}
List <string> tags = null;
if (!String.IsNullOrEmpty(md.Params.Tags))
{
tags = Common.CsvToStringList(md.Params.Tags);
}
SourceDocument src = new SourceDocument(
sourceGuid,
md.User.GUID,
index.GUID,
md.Params.Name,
md.Params.Title,
tags,
docType,
sourceUrl,
md.Http.Request.ContentType,
contentLength,
md5);
#endregion
if (!md.Params.Async)
{
#region Sync
IndexResult result = await _Daemon.AddDocument(
indexName,
src,
Common.ReadBinaryFile(tempFile),
new ParseOptions(),
!md.Params.Bypass);
if (!result.Success)
{
_Logging.Warn(header + "unable to store document in index " + indexName);
md.Http.Response.StatusCode = 500;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(new ErrorResponse(500, "Unable to store document in index '" + indexName + "'.", null, result).ToJson(true));
return;
}
md.Http.Response.StatusCode = 200;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(Common.SerializeJson(result, md.Params.Pretty));
return;
#endregion
}
else
{
#region Async
IndexResult result = new IndexResult();
result.Success = true;
result.GUID = src.GUID;
result.Type = docType;
result.ParseResult = null;
result.Time = null;
Task unawaited = _Daemon.AddDocument(
index.Name,
src,
Common.ReadBinaryFile(tempFile),
new ParseOptions(),
!md.Params.Bypass);
md.Http.Response.StatusCode = 200;
md.Http.Response.ContentType = "application/json";
await md.Http.Response.Send(Common.SerializeJson(result, md.Params.Pretty));
return;
#endregion
}
}
finally
{
if (File.Exists(tempFile))
{
File.Delete(tempFile);
}
}
}