async static Task IndexItems(LuceneServerClient client, List <IDictionary <string, object> > items, int counter)
{
Console.Write($"Index { items.Count() } items...");
if (!await client.IndexDocumentsAsync(items))
{
throw new Exception("Can't index items");
}
Console.WriteLine($"succeeded ... { counter }");
items.Clear();
}
async static Task <int> Main(string[] args)
{
try
{
string serverUrl = null, indexName = null;
IndexMapping indexMapping = new IndexMapping();
bool removeIndex = false;
for (int i = 0; i < args.Length; i++)
{
switch (args[i].ToLower())
{
case "-remove":
removeIndex = true;
break;
}
if (i < args.Length - 1)
{
switch (args[i].ToLower())
{
case "-server":
case "-s":
serverUrl = args[++i];
break;
case "-index":
case "-i":
indexName = args[++i];
break;
case "-field":
case "-f":
indexMapping.AddField(args[++i].ToFieldMapping(stored: true, index: true));
break;
case "-sfield":
indexMapping.AddField(args[++i].ToFieldMapping(stored: true, index: false));
break;
case "-primary":
indexMapping.PrimaryFields =
new List <string>(
args[++i].Split(',').Select(s => s.Trim()));
break;
}
}
}
if (String.IsNullOrEmpty(serverUrl) ||
String.IsNullOrEmpty(indexName) ||
!indexMapping.IsValid())
{
Console.WriteLine("Usage:");
Console.WriteLine("CreateIndex.exe -server[-s] server");
Console.WriteLine(" -index[-i] indexname");
Console.WriteLine(" -field[-f] fieldname[.fieldtype][.stored|not_stored] // add indexed field - defaults .TextType.stored");
Console.WriteLine(" -field ...");
Console.WriteLine(" -storedfield[-sfield] fieldname[.fieldtype] // add stored field - defaults .TextType");
Console.WriteLine(" -storedfield ...");
Console.WriteLine(" -primary primary-search-fieldname // default: first field");
Console.WriteLine(" -remove // remove existing index first");
Console.WriteLine();
Console.WriteLine($"FieldTypes: { String.Join(", ", FieldTypes.Values()) }");
return(1);
}
var client = new LuceneServerClient(serverUrl, indexName);
#region Create Index
if (removeIndex)
{
Console.WriteLine($"Delete index { indexName }...");
if (!await client.RemoveIndexAsync())
{
throw new Exception("Can't deleting index");
}
}
Console.WriteLine($"Create index { indexName }...");
if (!await client.CreateIndexAsync())
{
throw new Exception("Can't creating index");
}
#endregion
#region Mapping
Console.WriteLine($"Map index { indexName }...");
if (!await client.MapAsync(indexMapping))
{
throw new Exception("Can't map index");
}
#endregion
Console.WriteLine("finished successfully");
return(0);
}
catch (Exception ex)
{
Console.WriteLine("Exception:");
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return(1);
}
}
async static Task <int> Main(string[] args)
{
try
{
string serverUrl = null, indexName = null;
DbTypes dbType = DbTypes.Unknown;
string connectionString = null, sqlStatement = null;
Dictionary <string, string> fields = new Dictionary <string, string>();
#region Parse Arguments
for (int i = 0; i < args.Length; i++)
{
switch (args[i].ToLower())
{
//case "-remove":
// removeIndex = true;
// break;
}
if (i < args.Length - 1)
{
switch (args[i].ToLower())
{
case "-server":
case "-s":
serverUrl = args[++i];
break;
case "-index":
case "-i":
indexName = args[++i];
break;
case "-db-type":
if (!Enum.TryParse <DbTypes>(args[++i], true, out dbType))
{
throw new Exception($"Unknown Db-Type { args[i] }");
}
break;
case "-db":
case "-db-connectionstring":
connectionString = args[++i];
break;
case "-sql":
case "-sql-statement":
sqlStatement = args[++i];
break;
}
}
if (i < args.Length - 2)
{
switch (args[i].ToLower())
{
case "-f":
fields[args[++i]] = args[++i];
break;
}
}
}
if (String.IsNullOrEmpty(serverUrl) ||
String.IsNullOrEmpty(indexName) ||
dbType == DbTypes.Unknown ||
String.IsNullOrEmpty(connectionString) ||
String.IsNullOrEmpty(sqlStatement) ||
fields.Count == 0)
{
Console.WriteLine("Usage:");
Console.WriteLine("ImportDbTable.exe -server[-s] {server}");
Console.WriteLine(" -index[-i] {indexname}");
Console.WriteLine(" -db-type {dbType}");
Console.WriteLine(" -db-connectionstring[-db] {connectionString}");
Console.WriteLine(" -sql-statement[-sql] {sqlStatement}");
Console.WriteLine(" -f {indexfield} {expression} // {expression}: \"lorem {{DB_FIELD1}} ipsum {{DB_FIELD2}}\"");
Console.WriteLine();
//Console.WriteLine($"FieldTypes: { String.Join(", ", FieldTypes.Values()) }");
return(1);
}
#endregion
var startTime = DateTime.Now;
int counter = 0;
using (var client = new LuceneServerClient(serverUrl, indexName))
{
var items = new List <IDictionary <string, object> >();
string regexDbFieldsPattern = @"{{(.*?)}}";
using (var connection = DbConnectionFactory.CreateInstance(dbType, connectionString))
{
foreach (IDictionary <string, object> row in connection.Query(sqlStatement, buffered: false))
{
if (row != null)
{
var item = new Dictionary <string, object>();
foreach (var indexField in fields.Keys)
{
string expression = fields[indexField];
var matches = Regex.Matches(expression, regexDbFieldsPattern).Select(m => m.ToString().Substring(2, m.ToString().Length - 4)).ToArray();
foreach (var match in matches)
{
expression = expression.Replace($"{{{{{ match }}}}}", row[match]?.ToString() ?? String.Empty);
}
item[indexField] = expression;
}
items.Add(item);
counter++;
}
if (counter % 1000 == 0)
{
await IndexItems(client, items, counter);
}
}
}
await client.IndexDocumentsAsync(items);
}
Console.WriteLine($"{ counter } records ... { Math.Round((DateTime.Now-startTime).TotalMinutes, 2) } minutes");
Console.WriteLine("finished successfully");
return(0);
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine("Exception:");
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return(1);
}
}
async static Task <int> Main(string[] args)
{
try
{
string cmd = "fill", jsonFile = (args.Length == 1 && args[0] != "fill" ? args[0] : String.Empty), indexUrl = String.Empty, indexName = String.Empty, category = String.Empty;
string proxyUrl = String.Empty, proxyUser = String.Empty, proxyPassword = String.Empty;
string basicAuthUser = String.Empty, basicAuthPassword = String.Empty;
int packageSize = 50000;
bool replace = false;
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "fill" && i < args.Length - 1)
{
cmd = "fill";
jsonFile = args[i + 1];
i++;
}
if (args[i] == "remove-category")
{
cmd = "remove-category";
}
if (args[i] == "-s" && i < args.Length - 1)
{
indexUrl = args[i + 1];
}
if (args[i] == "-i" && i < args.Length - 1)
{
indexName = args[i + 1];
}
if (args[i] == "-c" && i < args.Length - 1)
{
category = args[i + 1];
}
if (args[i] == "-r")
{
replace = true;
}
if (args[i] == "-packagesize" && i < args.Length - 1)
{
packageSize = int.Parse(args[i + 1]);
}
#region Proxy
if (args[i] == "-proxy")
{
proxyUrl = args[i + 1];
}
if (args[i] == "-proxy-user")
{
proxyUser = args[i + 1];
}
if (args[i] == "-proxy-pwd")
{
proxyPassword = args[i + 1];
}
#endregion
#region Basic Authentication
if (args[i] == "-basic-auth-user")
{
basicAuthUser = args[i + 1];
}
if (args[i] == "-basic-auth-pwd")
{
basicAuthPassword = args[i + 1];
}
#endregion
}
if (args.Length == 0)
{
Console.WriteLine("Usage: gView.Cmd.ElasticSearch fill|remove-catetory [Options]");
return(1);
}
//gView.Framework.system.SystemVariables.CustomApplicationDirectory =
// System.IO.Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
Console.WriteLine("Environment");
Console.WriteLine("Working Directory: " + gView.Framework.system.SystemVariables.StartupDirectory);
Console.WriteLine("64Bit=" + gView.Framework.system.Wow.Is64BitProcess);
if (cmd == "fill")
{
#region Fill Index (with Json File)
var importConfig = JsonConvert.DeserializeObject <ImportConfig>(File.ReadAllText(jsonFile));
if (importConfig?.Connection == null)
{
throw new Exception("Invalid config. No connection defined");
}
var httpClientHandler = new HttpClientHandler();
if (!String.IsNullOrEmpty(proxyUrl))
{
httpClientHandler.Proxy = new WebProxy
{
Address = new Uri(proxyUrl),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(proxyUser, proxyPassword)
};
}
var httpClient = new HttpClient(handler: httpClientHandler, disposeHandler: true);
if (!String.IsNullOrEmpty(basicAuthUser))
{
var byteArray = Encoding.ASCII.GetBytes($"{ basicAuthUser }:{ basicAuthPassword }");
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
using (var luceneServerClient = new LuceneServerClient(
importConfig.Connection.Url,
importConfig.Connection.DefaultIndex,
httpClient: httpClient))
{
if (importConfig.Connection.DeleteIndex)
{
await luceneServerClient.RemoveIndexAsync();
}
if (!await luceneServerClient.CreateIndexAsync())
{
throw new Exception($"Can't create elasticsearch index { importConfig.Connection.DefaultIndex }");
}
if (!await luceneServerClient.MapAsync(new SearchIndexMapping()))
{
throw new Exception($"Can't map item in elasticsearch index { importConfig.Connection.DefaultIndex }");
}
ISpatialReference sRefTarget = SpatialReference.FromID("epsg:4326");
Console.WriteLine("Target Spatial Reference: " + sRefTarget.Name + " " + String.Join(" ", sRefTarget.Parameters));
foreach (var datasetConfig in importConfig.Datasets)
{
if (datasetConfig.FeatureClasses == null)
{
continue;
}
IDataset dataset = new PlugInManager().CreateInstance(datasetConfig.DatasetGuid) as IDataset;
if (dataset == null)
{
throw new ArgumentException("Can't load dataset with guid " + datasetConfig.DatasetGuid.ToString());
}
await dataset.SetConnectionString(datasetConfig.ConnectionString);
await dataset.Open();
foreach (var featureClassConfig in datasetConfig.FeatureClasses)
{
var itemProto = featureClassConfig.IndexItemProto;
if (itemProto == null)
{
continue;
}
string metaId = Guid.NewGuid().ToString("N").ToLower();
category = featureClassConfig.Category;
if (!String.IsNullOrWhiteSpace(category))
{
var meta = new Meta()
{
Id = metaId,
Category = category,
Descrption = featureClassConfig.Meta?.Descrption,
Sample = featureClassConfig?.Meta.Sample,
Service = featureClassConfig?.Meta.Service,
Query = featureClassConfig?.Meta.Query
};
if (!await luceneServerClient.AddCustomMetadataAsync(metaId, JsonConvert.SerializeObject(meta)))
{
throw new Exception($"Can't index meta item in elasticsearch index { importConfig.Connection.MetaIndex }");
}
}
bool useGeometry = featureClassConfig.UserGeometry;
IDatasetElement dsElement = await dataset.Element(featureClassConfig.Name);
if (dsElement == null)
{
throw new ArgumentException("Unknown dataset element " + featureClassConfig.Name);
}
IFeatureClass fc = dsElement.Class as IFeatureClass;
if (fc == null)
{
throw new ArgumentException("Dataobject is not a featureclass " + featureClassConfig.Name);
}
Console.WriteLine("Index " + fc.Name);
Console.WriteLine("=====================================================================");
QueryFilter filter = new QueryFilter();
filter.SubFields = "*";
if (!String.IsNullOrWhiteSpace(featureClassConfig.Filter))
{
filter.WhereClause = featureClassConfig.Filter;
Console.WriteLine("Filter: " + featureClassConfig.Filter);
}
List <Dictionary <string, object> > items = new List <Dictionary <string, object> >();
int count = 0;
ISpatialReference sRef = fc.SpatialReference ?? SpatialReference.FromID("epsg:" + featureClassConfig.SRefId);
Console.WriteLine("Source Spatial Reference: " + sRef.Name + " " + String.Join(" ", sRef.Parameters));
Console.WriteLine("IDField: " + fc.IDFieldName);
using (var transformer = GeometricTransformerFactory.Create())
{
if (useGeometry)
{
transformer.SetSpatialReferences(sRef, sRefTarget);
}
IFeatureCursor cursor = await fc.GetFeatures(filter);
IFeature feature;
while ((feature = await cursor.NextFeature()) != null)
{
var indexItem = ParseFeature(metaId, category, feature, itemProto, useGeometry, transformer, featureClassConfig);
items.Add(indexItem);
count++;
if (items.Count >= packageSize)
{
if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }");
}
items.Clear();
Console.Write(count + "...");
}
}
if (items.Count > 0)
{
if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }");
}
Console.WriteLine(count + "...finish");
}
}
}
}
}
#endregion
}
else if (cmd == "remove-category")
{
#region Remove Category
//RemoveCategory(indexUrl, indexName, replace ? Replace(category) : category,
// proxyUrl, proxyUser, proxyPassword,
// basicAuthUser, basicAuthPassword);
#endregion
}
return(0);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return(1);
}
}
