async static Task IndexItems(LuceneServerClient client, List <IDictionary <string, object> > items, int counter) { Console.Write($"Index { items.Count() } items..."); if (!await client.IndexDocumentsAsync(items)) { throw new Exception("Can't index items"); } Console.WriteLine($"succeeded ... { counter }"); items.Clear(); }
async static Task <int> Main(string[] args) { try { string serverUrl = null, indexName = null; IndexMapping indexMapping = new IndexMapping(); bool removeIndex = false; for (int i = 0; i < args.Length; i++) { switch (args[i].ToLower()) { case "-remove": removeIndex = true; break; } if (i < args.Length - 1) { switch (args[i].ToLower()) { case "-server": case "-s": serverUrl = args[++i]; break; case "-index": case "-i": indexName = args[++i]; break; case "-field": case "-f": indexMapping.AddField(args[++i].ToFieldMapping(stored: true, index: true)); break; case "-sfield": indexMapping.AddField(args[++i].ToFieldMapping(stored: true, index: false)); break; case "-primary": indexMapping.PrimaryFields = new List <string>( args[++i].Split(',').Select(s => s.Trim())); break; } } } if (String.IsNullOrEmpty(serverUrl) || String.IsNullOrEmpty(indexName) || !indexMapping.IsValid()) { Console.WriteLine("Usage:"); Console.WriteLine("CreateIndex.exe -server[-s] server"); Console.WriteLine(" -index[-i] indexname"); Console.WriteLine(" -field[-f] fieldname[.fieldtype][.stored|not_stored] // add indexed field - defaults .TextType.stored"); Console.WriteLine(" -field ..."); Console.WriteLine(" -storedfield[-sfield] fieldname[.fieldtype] // add stored field - defaults .TextType"); Console.WriteLine(" -storedfield ..."); Console.WriteLine(" -primary primary-search-fieldname // default: first field"); Console.WriteLine(" -remove // remove existing index first"); Console.WriteLine(); Console.WriteLine($"FieldTypes: { String.Join(", ", FieldTypes.Values()) }"); return(1); } var client = new LuceneServerClient(serverUrl, indexName); #region Create Index if (removeIndex) { Console.WriteLine($"Delete index { indexName }..."); if (!await client.RemoveIndexAsync()) { throw new Exception("Can't deleting index"); } } Console.WriteLine($"Create index { indexName }..."); if (!await client.CreateIndexAsync()) { throw new Exception("Can't creating index"); } #endregion #region Mapping Console.WriteLine($"Map index { indexName }..."); if (!await client.MapAsync(indexMapping)) { throw new Exception("Can't map index"); } #endregion Console.WriteLine("finished successfully"); return(0); } catch (Exception ex) { Console.WriteLine("Exception:"); Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); return(1); } }
async static Task <int> Main(string[] args) { try { string serverUrl = null, indexName = null; DbTypes dbType = DbTypes.Unknown; string connectionString = null, sqlStatement = null; Dictionary <string, string> fields = new Dictionary <string, string>(); #region Parse Arguments for (int i = 0; i < args.Length; i++) { switch (args[i].ToLower()) { //case "-remove": // removeIndex = true; // break; } if (i < args.Length - 1) { switch (args[i].ToLower()) { case "-server": case "-s": serverUrl = args[++i]; break; case "-index": case "-i": indexName = args[++i]; break; case "-db-type": if (!Enum.TryParse <DbTypes>(args[++i], true, out dbType)) { throw new Exception($"Unknown Db-Type { args[i] }"); } break; case "-db": case "-db-connectionstring": connectionString = args[++i]; break; case "-sql": case "-sql-statement": sqlStatement = args[++i]; break; } } if (i < args.Length - 2) { switch (args[i].ToLower()) { case "-f": fields[args[++i]] = args[++i]; break; } } } if (String.IsNullOrEmpty(serverUrl) || String.IsNullOrEmpty(indexName) || dbType == DbTypes.Unknown || String.IsNullOrEmpty(connectionString) || String.IsNullOrEmpty(sqlStatement) || fields.Count == 0) { Console.WriteLine("Usage:"); Console.WriteLine("ImportDbTable.exe -server[-s] {server}"); Console.WriteLine(" -index[-i] {indexname}"); Console.WriteLine(" -db-type {dbType}"); Console.WriteLine(" -db-connectionstring[-db] {connectionString}"); Console.WriteLine(" -sql-statement[-sql] {sqlStatement}"); Console.WriteLine(" -f {indexfield} {expression} // {expression}: \"lorem {{DB_FIELD1}} ipsum {{DB_FIELD2}}\""); Console.WriteLine(); //Console.WriteLine($"FieldTypes: { String.Join(", ", FieldTypes.Values()) }"); return(1); } #endregion var startTime = DateTime.Now; int counter = 0; using (var client = new LuceneServerClient(serverUrl, indexName)) { var items = new List <IDictionary <string, object> >(); string regexDbFieldsPattern = @"{{(.*?)}}"; using (var connection = DbConnectionFactory.CreateInstance(dbType, connectionString)) { foreach (IDictionary <string, object> row in connection.Query(sqlStatement, buffered: false)) { if (row != null) { var item = new Dictionary <string, object>(); foreach (var indexField in fields.Keys) { string expression = fields[indexField]; var matches = Regex.Matches(expression, regexDbFieldsPattern).Select(m => m.ToString().Substring(2, m.ToString().Length - 4)).ToArray(); foreach (var match in matches) { expression = expression.Replace($"{{{{{ match }}}}}", row[match]?.ToString() ?? String.Empty); } item[indexField] = expression; } items.Add(item); counter++; } if (counter % 1000 == 0) { await IndexItems(client, items, counter); } } } await client.IndexDocumentsAsync(items); } Console.WriteLine($"{ counter } records ... { Math.Round((DateTime.Now-startTime).TotalMinutes, 2) } minutes"); Console.WriteLine("finished successfully"); return(0); } catch (Exception ex) { Console.WriteLine(); Console.WriteLine("Exception:"); Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); return(1); } }
async static Task <int> Main(string[] args) { try { string cmd = "fill", jsonFile = (args.Length == 1 && args[0] != "fill" ? args[0] : String.Empty), indexUrl = String.Empty, indexName = String.Empty, category = String.Empty; string proxyUrl = String.Empty, proxyUser = String.Empty, proxyPassword = String.Empty; string basicAuthUser = String.Empty, basicAuthPassword = String.Empty; int packageSize = 50000; bool replace = false; for (int i = 0; i < args.Length; i++) { if (args[i] == "fill" && i < args.Length - 1) { cmd = "fill"; jsonFile = args[i + 1]; i++; } if (args[i] == "remove-category") { cmd = "remove-category"; } if (args[i] == "-s" && i < args.Length - 1) { indexUrl = args[i + 1]; } if (args[i] == "-i" && i < args.Length - 1) { indexName = args[i + 1]; } if (args[i] == "-c" && i < args.Length - 1) { category = args[i + 1]; } if (args[i] == "-r") { replace = true; } if (args[i] == "-packagesize" && i < args.Length - 1) { packageSize = int.Parse(args[i + 1]); } #region Proxy if (args[i] == "-proxy") { proxyUrl = args[i + 1]; } if (args[i] == "-proxy-user") { proxyUser = args[i + 1]; } if (args[i] == "-proxy-pwd") { proxyPassword = args[i + 1]; } #endregion #region Basic Authentication if (args[i] == "-basic-auth-user") { basicAuthUser = args[i + 1]; } if (args[i] == "-basic-auth-pwd") { basicAuthPassword = args[i + 1]; } #endregion } if (args.Length == 0) { Console.WriteLine("Usage: gView.Cmd.ElasticSearch fill|remove-catetory [Options]"); return(1); } //gView.Framework.system.SystemVariables.CustomApplicationDirectory = // System.IO.Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); Console.WriteLine("Environment"); Console.WriteLine("Working Directory: " + gView.Framework.system.SystemVariables.StartupDirectory); Console.WriteLine("64Bit=" + gView.Framework.system.Wow.Is64BitProcess); if (cmd == "fill") { #region Fill Index (with Json File) var importConfig = JsonConvert.DeserializeObject <ImportConfig>(File.ReadAllText(jsonFile)); if (importConfig?.Connection == null) { throw new Exception("Invalid config. No connection defined"); } var httpClientHandler = new HttpClientHandler(); if (!String.IsNullOrEmpty(proxyUrl)) { httpClientHandler.Proxy = new WebProxy { Address = new Uri(proxyUrl), BypassProxyOnLocal = false, UseDefaultCredentials = false, Credentials = new NetworkCredential(proxyUser, proxyPassword) }; } var httpClient = new HttpClient(handler: httpClientHandler, disposeHandler: true); if (!String.IsNullOrEmpty(basicAuthUser)) { var byteArray = Encoding.ASCII.GetBytes($"{ basicAuthUser }:{ basicAuthPassword }"); httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); } using (var luceneServerClient = new LuceneServerClient( importConfig.Connection.Url, importConfig.Connection.DefaultIndex, httpClient: httpClient)) { if (importConfig.Connection.DeleteIndex) { await luceneServerClient.RemoveIndexAsync(); } if (!await luceneServerClient.CreateIndexAsync()) { throw new Exception($"Can't create elasticsearch index { importConfig.Connection.DefaultIndex }"); } if (!await luceneServerClient.MapAsync(new SearchIndexMapping())) { throw new Exception($"Can't map item in elasticsearch index { importConfig.Connection.DefaultIndex }"); } ISpatialReference sRefTarget = SpatialReference.FromID("epsg:4326"); Console.WriteLine("Target Spatial Reference: " + sRefTarget.Name + " " + String.Join(" ", sRefTarget.Parameters)); foreach (var datasetConfig in importConfig.Datasets) { if (datasetConfig.FeatureClasses == null) { continue; } IDataset dataset = new PlugInManager().CreateInstance(datasetConfig.DatasetGuid) as IDataset; if (dataset == null) { throw new ArgumentException("Can't load dataset with guid " + datasetConfig.DatasetGuid.ToString()); } await dataset.SetConnectionString(datasetConfig.ConnectionString); await dataset.Open(); foreach (var featureClassConfig in datasetConfig.FeatureClasses) { var itemProto = featureClassConfig.IndexItemProto; if (itemProto == null) { continue; } string metaId = Guid.NewGuid().ToString("N").ToLower(); category = featureClassConfig.Category; if (!String.IsNullOrWhiteSpace(category)) { var meta = new Meta() { Id = metaId, Category = category, Descrption = featureClassConfig.Meta?.Descrption, Sample = featureClassConfig?.Meta.Sample, Service = featureClassConfig?.Meta.Service, Query = featureClassConfig?.Meta.Query }; if (!await luceneServerClient.AddCustomMetadataAsync(metaId, JsonConvert.SerializeObject(meta))) { throw new Exception($"Can't index meta item in elasticsearch index { importConfig.Connection.MetaIndex }"); } } bool useGeometry = featureClassConfig.UserGeometry; IDatasetElement dsElement = await dataset.Element(featureClassConfig.Name); if (dsElement == null) { throw new ArgumentException("Unknown dataset element " + featureClassConfig.Name); } IFeatureClass fc = dsElement.Class as IFeatureClass; if (fc == null) { throw new ArgumentException("Dataobject is not a featureclass " + featureClassConfig.Name); } Console.WriteLine("Index " + fc.Name); Console.WriteLine("====================================================================="); QueryFilter filter = new QueryFilter(); filter.SubFields = "*"; if (!String.IsNullOrWhiteSpace(featureClassConfig.Filter)) { filter.WhereClause = featureClassConfig.Filter; Console.WriteLine("Filter: " + featureClassConfig.Filter); } List <Dictionary <string, object> > items = new List <Dictionary <string, object> >(); int count = 0; ISpatialReference sRef = fc.SpatialReference ?? SpatialReference.FromID("epsg:" + featureClassConfig.SRefId); Console.WriteLine("Source Spatial Reference: " + sRef.Name + " " + String.Join(" ", sRef.Parameters)); Console.WriteLine("IDField: " + fc.IDFieldName); using (var transformer = GeometricTransformerFactory.Create()) { if (useGeometry) { transformer.SetSpatialReferences(sRef, sRefTarget); } IFeatureCursor cursor = await fc.GetFeatures(filter); IFeature feature; while ((feature = await cursor.NextFeature()) != null) { var indexItem = ParseFeature(metaId, category, feature, itemProto, useGeometry, transformer, featureClassConfig); items.Add(indexItem); count++; if (items.Count >= packageSize) { if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray())) { throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }"); } items.Clear(); Console.Write(count + "..."); } } if (items.Count > 0) { if (!await luceneServerClient.IndexDocumentsAsync(items.ToArray())) { throw new Exception($"Error on indexing { items.Count } items on elasticsearch index { importConfig.Connection.DefaultIndex }"); } Console.WriteLine(count + "...finish"); } } } } } #endregion } else if (cmd == "remove-category") { #region Remove Category //RemoveCategory(indexUrl, indexName, replace ? Replace(category) : category, // proxyUrl, proxyUser, proxyPassword, // basicAuthUser, basicAuthPassword); #endregion } return(0); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); return(1); } }