async static Task <int> Main(string[] args)
{
try
{
string cmd = "fill", jsonFile = (args.Length == 1 && args[0] != "fill" ? args[0] : String.Empty), indexUrl = String.Empty, indexName = String.Empty, category = String.Empty;
string proxyUrl = String.Empty, proxyUser = String.Empty, proxyPassword = String.Empty;
string basicAuthUser = String.Empty, basicAuthPassword = String.Empty;
int packageSize = 50000;
bool replace = false;
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "fill" && i < args.Length - 1)
{
cmd = "fill";
jsonFile = args[i + 1];
i++;
}
if (args[i] == "remove-category")
{
cmd = "remove-category";
}
if (args[i] == "-s" && i < args.Length - 1)
{
indexUrl = args[i + 1];
}
if (args[i] == "-i" && i < args.Length - 1)
{
indexName = args[i + 1];
}
if (args[i] == "-c" && i < args.Length - 1)
{
category = args[i + 1];
}
if (args[i] == "-r")
{
replace = true;
}
if (args[i] == "-packagesize" && i < args.Length - 1)
{
packageSize = int.Parse(args[i + 1]);
}
#region Proxy
if (args[i] == "-proxy")
{
proxyUrl = args[i + 1];
}
if (args[i] == "-proxy-user")
{
proxyUser = args[i + 1];
}
if (args[i] == "-proxy-pwd")
{
proxyPassword = args[i + 1];
}
#endregion
#region Basic Authentication
if (args[i] == "-basic-auth-user")
{
basicAuthUser = args[i + 1];
}
if (args[i] == "-basic-auth-pwd")
{
basicAuthPassword = args[i + 1];
}
#endregion
}
if (args.Length == 0)
{
Console.WriteLine("Usage: gView.Cmd.ElasticSearch fill|remove-catetory [Options]");
return(1);
}
//gView.Framework.system.SystemVariables.CustomApplicationDirectory =
// System.IO.Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
Console.WriteLine("Environment");
Console.WriteLine("Working Directory: " + gView.Framework.system.SystemVariables.StartupDirectory);
Console.WriteLine("64Bit=" + gView.Framework.system.Wow.Is64BitProcess);
if (cmd == "fill")
{
#region Fill Index (with Json File)
var importConfig = JsonConvert.DeserializeObject <ImportConfig>(File.ReadAllText(jsonFile));
if (importConfig?.Connection == null)
{
throw new Exception("Invalid config. No connection defined");
}
var httpClientHandler = new HttpClientHandler();
if (!String.IsNullOrEmpty(proxyUrl))
{
httpClientHandler.Proxy = new WebProxy
{
Address = new Uri(proxyUrl),
BypassProxyOnLocal = false,
UseDefaultCredentials = false,
Credentials = new NetworkCredential(proxyUser, proxyPassword)
};
}
var httpClient = new HttpClient(handler: httpClientHandler, disposeHandler: true);
if (!String.IsNullOrEmpty(basicAuthUser))
{
var byteArray = Encoding.ASCII.GetBytes($"{ basicAuthUser }:{ basicAuthPassword }");
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
}
using (var luceneServerClient = new LuceneServerClient(
importConfig.Connection.Url,
importConfig.Connection.DefaultIndex,
httpClient: httpClient))
{
if (importConfig.Connection.DeleteIndex)
{
await luceneServerClient.RemoveIndexAsync();
}
if (!await luceneServerClient.CreateIndexAsync())
{
throw new Exception($"Can't create elasticsearch index { importConfig.Connection.DefaultIndex }");
}
if (!await luceneServerClient.MapAsync(new SearchIndexMapping()))
{
throw new Exception($"Can't map item in elasticsearch index { importConfig.Connection.DefaultIndex }");
}
ISpatialReference sRefTarget = SpatialReference.FromID("epsg:4326");
Console.WriteLine("Target Spatial Reference: " + sRefTarget.Name + " " + String.Join(" ", sRefTarget.Parameters));
foreach (var datasetConfig in importConfig.Datasets)
{
if (datasetConfig.FeatureClasses == null)
{
continue;
}
IDataset dataset = new PlugInManager().CreateInstance(datasetConfig.DatasetGuid) as IDataset;
if (dataset == null)
{
throw new ArgumentException("Can't load dataset with guid " + datasetConfig.DatasetGuid.ToString());
}
await dataset.SetConnectionString(datasetConfig.ConnectionString);
await dataset.Open();
foreach (var featureClassConfig in datasetConfig.FeatureClasses)
{
var itemProto = featureClassConfig.IndexItemProto;
if (itemProto == null)
{
continue;
}
string metaId = Guid.NewGuid().ToString("N").ToLower();
category = featureClassConfig.Category;
if (!String.IsNullOrWhiteSpace(category))
{
var meta = new Meta()
{
Id = metaId,
Category = category,
Descrption = featureClassConfig.Meta?.Descrption,
Sample = featureClassConfig?.Meta.Sample,
Service = featureClassConfig?.Meta.Service,
Query = featureClassConfig?.Meta.Query
};
if (!await luceneServerClient.AddCustomMetadataAsync(metaId, JsonConvert.SerializeObject(meta)))
{
throw new Exception($"Can't index meta item in elasticsearch index { importConfig.Connection.MetaIndex }");
}
}
bool useGeometry = featureClassConfig.UserGeometry;
IDatasetElement dsElement = await dataset.Element(featureClassConfig.Name);
if (dsElement == null)
{
throw new ArgumentException("Unknown dataset element " + featureClassConfig.Name);
}
IFeatureClass fc = dsElement.Class as IFeatureClass;
if (fc == null)
{
throw new ArgumentException("Dataobject is not a featureclass " + featureClassConfig.Name);
}
Console.WriteLine("Index " + fc.Name);
Console.WriteLine("=====================================================================");
QueryFilter filter = new QueryFilter();
filter.SubFields = "*";
if (!String.IsNullOrWhiteSpace(featureClassConfig.Filter))
{
filter.WhereClause = featureClassConfig.Filter;
Console.WriteLine("Filter: " + featureClassConfig.Filter);
}
List <Dictionary <string, object> > items = new List <Dictionary <string, object> >();
int count = 0;
ISpatialReference sRef = fc.SpatialReference ?? SpatialReference.FromID("epsg:" + featureClassConfig.SRefId);
Console.WriteLine("Source Spatial Reference: " + sRef.Name + " " + String.Join(" ", sRef.Parameters));
Console.WriteLine("IDField: " + fc.IDFieldName);
using (var transformer = GeometricTransformerFactory.Create())
{
if (useGeometry)
{
transformer.SetSpatialReferences(sRef, sRefTarget);
}
IFeatureCursor cursor = await fc.GetFeatures(filter);
IFeature feature;
while ((feature = await cursor.NextFeature()) != null)
{
var indexItem = ParseFeature(metaId, category, feature, itemProto, useGeometry, transformer, featureClassConfig);
items.Add(indexItem);
count++;
if (items.Count >= packageSize)
{
if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }");
}
items.Clear();
Console.Write(count + "...");
}
}
if (items.Count > 0)
{
if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }");
}
Console.WriteLine(count + "...finish");
}
}
}
}
}
#endregion
}
else if (cmd == "remove-category")
{
#region Remove Category
//RemoveCategory(indexUrl, indexName, replace ? Replace(category) : category,
// proxyUrl, proxyUser, proxyPassword,
// basicAuthUser, basicAuthPassword);
#endregion
}
return(0);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
return(1);
}
}
async static Task <int> Main(string[] args)
{
string cmd = "fill", jsonFile = (args.Length == 1 && args[0] != "fill" ? args[0] : String.Empty), indexUrl = String.Empty, indexName = String.Empty, category = String.Empty;
string proxyUrl = String.Empty, proxyUser = String.Empty, proxyPassword = String.Empty;
string basicAuthUser = String.Empty, basicAuthPassword = String.Empty;
bool replace = false;
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "fill" && i < args.Length - 1)
{
cmd = "fill";
jsonFile = args[i + 1];
i++;
}
if (args[i] == "remove-category")
{
cmd = "remove-category";
}
if (args[i] == "-s" && i < args.Length - 1)
{
indexUrl = args[i + 1];
}
if (args[i] == "-i" && i < args.Length - 1)
{
indexName = args[i + 1];
}
if (args[i] == "-c" && i < args.Length - 1)
{
category = args[i + 1];
}
if (args[i] == "-r")
{
replace = true;
}
#region Proxy
if (args[i] == "-proxy")
{
proxyUrl = args[i + 1];
}
if (args[i] == "-proxy-user")
{
proxyUser = args[i + 1];
}
if (args[i] == "-proxy-pwd")
{
proxyPassword = args[i + 1];
}
#endregion
#region Basic Authentication
if (args[i] == "-basic-auth-user")
{
basicAuthUser = args[i + 1];
}
if (args[i] == "-basic-auth-pwd")
{
basicAuthPassword = args[i + 1];
}
#endregion
}
if (args.Length == 0)
{
Console.WriteLine("Usage: gView.Cmd.ElasticSearch fill|remove-catetory [Options]");
return(1);
}
if (cmd == "fill" && String.IsNullOrEmpty(jsonFile))
{
Console.WriteLine("Usage: gView.Cmd.ElasticSearch fill {json-file}");
return(1);
}
else if (cmd == "remove-category" && (String.IsNullOrWhiteSpace(indexUrl) || String.IsNullOrWhiteSpace(category)))
{
Console.WriteLine("Usage: gView.cmd.ElasticSearch remove-category -s {index-url} -i {index-name} -c {category} [-r]");
Console.WriteLine(" -r ... raplace german Umlaute:");
Console.WriteLine(" _ae_, _oe_, _ue_ => ä, ö, ü");
Console.WriteLine(" _Ae_, _Oe_, _Ue_ => Ä, Ö, Ü");
Console.WriteLine(" _sz_ => ß");
return(1);
}
try
{
//gView.Framework.system.SystemVariables.CustomApplicationDirectory =
// System.IO.Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
Console.WriteLine("Environment");
Console.WriteLine("Working Directory: " + gView.Framework.system.SystemVariables.StartupDirectory);
Console.WriteLine("64Bit=" + gView.Framework.system.Wow.Is64BitProcess);
if (cmd == "fill")
{
#region Fill Index (with Json File)
var importConfig = JsonConvert.DeserializeObject <ImportConfig>(File.ReadAllText(jsonFile));
var searchContext = new ElasticSearchContext(importConfig.Connection.Url,
importConfig.Connection.DefaultIndex,
proxyUri: proxyUrl, proxyUsername: proxyUser, proxyPassword: proxyPassword,
basicAuthUser: basicAuthUser, basicAuthPassword: basicAuthPassword);
if (importConfig.Connection.DeleteIndex)
{
searchContext.DeleteIndex();
searchContext.DeleteIndex(importConfig.Connection.MetaIndex);
}
if (!searchContext.CreateIndex <Item>())
{
throw new Exception($"Can't create elasticsearch index { importConfig.Connection.DefaultIndex }: { searchContext.LastErrorMessage }");
}
if (!searchContext.Map <Item>())
{
throw new Exception($"Can't map item in elasticsearch index { importConfig.Connection.DefaultIndex }: { searchContext.LastErrorMessage }");
}
if (!searchContext.CreateIndex <Meta>(importConfig.Connection.MetaIndex))
{
throw new Exception($"Can't create elasticsearch index { importConfig.Connection.MetaIndex }: { searchContext.LastErrorMessage }");
}
if (!searchContext.Map <Meta>(importConfig.Connection.MetaIndex))
{
throw new Exception($"Can't map item in elasticsearch index { importConfig.Connection.MetaIndex }: { searchContext.LastErrorMessage }");
}
ISpatialReference sRefTarget = SpatialReference.FromID("epsg:4326");
Console.WriteLine("Target Spatial Reference: " + sRefTarget.Name + " " + String.Join(" ", sRefTarget.Parameters));
foreach (var datasetConfig in importConfig.Datasets)
{
if (datasetConfig.FeatureClasses == null)
{
continue;
}
IDataset dataset = new PlugInManager().CreateInstance(datasetConfig.DatasetGuid) as IDataset;
if (dataset == null)
{
throw new ArgumentException("Can't load dataset with guid " + datasetConfig.DatasetGuid.ToString());
}
await dataset.SetConnectionString(datasetConfig.ConnectionString);
await dataset.Open();
foreach (var featureClassConfig in datasetConfig.FeatureClasses)
{
var itemProto = featureClassConfig.IndexItemProto;
if (itemProto == null)
{
continue;
}
string metaId = Guid.NewGuid().ToString("N").ToLower();
category = featureClassConfig.Category;
if (!String.IsNullOrWhiteSpace(category))
{
var meta = new Meta()
{
Id = metaId,
Category = category,
Descrption = featureClassConfig.Meta?.Descrption,
Sample = featureClassConfig?.Meta.Sample,
Service = featureClassConfig?.Meta.Service,
Query = featureClassConfig?.Meta.Query
};
if (!searchContext.Index <Meta>(meta, importConfig.Connection.MetaIndex))
{
throw new Exception($"Can't index meta item in elasticsearch index { importConfig.Connection.MetaIndex }");
}
}
bool useGeometry = featureClassConfig.UserGeometry;
IDatasetElement dsElement = await dataset.Element(featureClassConfig.Name);
if (dsElement == null)
{
throw new ArgumentException("Unknown dataset element " + featureClassConfig.Name);
}
IFeatureClass fc = dsElement.Class as IFeatureClass;
if (fc == null)
{
throw new ArgumentException("Dataobject is not a featureclass " + featureClassConfig.Name);
}
Console.WriteLine("Index " + fc.Name);
Console.WriteLine("=====================================================================");
QueryFilter filter = new QueryFilter();
filter.SubFields = "*";
if (!String.IsNullOrWhiteSpace(featureClassConfig.Filter))
{
filter.WhereClause = featureClassConfig.Filter;
Console.WriteLine("Filter: " + featureClassConfig.Filter);
}
List <Item> items = new List <Item>();
int count = 0;
ISpatialReference sRef = fc.SpatialReference ?? SpatialReference.FromID("epsg:" + featureClassConfig.SRefId);
Console.WriteLine("Source Spatial Reference: " + sRef.Name + " " + String.Join(" ", sRef.Parameters));
Console.WriteLine("IDField: " + fc.IDFieldName);
using (var transformer = GeometricTransformerFactory.Create())
{
if (useGeometry)
{
transformer.SetSpatialReferences(sRef, sRefTarget);
}
IFeatureCursor cursor = await fc.GetFeatures(filter);
IFeature feature;
while ((feature = await cursor.NextFeature()) != null)
{
var indexItem = ParseFeature(metaId, category, feature, itemProto, useGeometry, transformer, featureClassConfig);
items.Add(indexItem);
count++;
if (items.Count >= 500)
{
if (!searchContext.IndexManyPro <Item>(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }: { searchContext.LastErrorMessage }");
}
items.Clear();
Console.Write(count + "...");
}
}
if (items.Count > 0)
{
if (!searchContext.IndexManyPro <Item>(items.ToArray()))
{
throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }: { searchContext.LastErrorMessage }");
}
Console.WriteLine(count + "...finish");
}
}
}
}
#endregion
}
else if (cmd == "remove-category")
{
#region Remove Category
RemoveCategory(indexUrl, indexName, replace ? Replace(category) : category,
proxyUrl, proxyUser, proxyPassword,
basicAuthUser, basicAuthPassword);
#endregion
}
return(0);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(1);
}
}
