Пример #1
0
 public BlackListDataManager()
 {
     try
     {
         var elasticSearchUrl = ConfigurationManager.AppSettings["ElasticStoreConnection"];
         var index = ConfigurationManager.AppSettings["EIndex_BlackList"];
         _settings =
             new ConnectionSettings(new Uri(elasticSearchUrl)).SetDefaultIndex(index).PluralizeTypeNames();
         _client = new ElasticClient(_settings);
         var isIndexExist = _client.IndexExists(i => i.Index(index));
         if (!isIndexExist.Exists)
         {
             _client.CreateIndex(c => c.Index(index));
             _settings.SetDefaultIndex(index);
             _client = new ElasticClient(_settings);
         }
         var response =
             _client.Map<BlockedUser>(
                 h =>
                     h.MapFromAttributes(10).Properties(p => p
                         .String(s => s.Name(i => i.ByEmail).Index(FieldIndexOption.NotAnalyzed))
                         .String(s => s.Name(i => i.ToEmail).Index(FieldIndexOption.NotAnalyzed))
                         ));
         IsServerError(response);
     }
     catch (Exception ex)
     {
         Logger.LogException(ex, _soruce, "BlackListDataManager", Severity.Critical);
         throw;
     }
 }
Пример #2
0
 public GameDataManager()
 {
     try
     {
         var elasticSearchUrl = ConfigurationManager.AppSettings["ElasticStoreConnection"];
         var index = ConfigurationManager.AppSettings["EIndex_Game"];
         _settings =
             new ConnectionSettings(new Uri(elasticSearchUrl)).SetDefaultIndex(index).PluralizeTypeNames();
         _client = new ElasticClient(_settings);
         var isIndexExist = _client.IndexExists(i => i.Index(index));
         if (!isIndexExist.Exists)
         {
             _client.CreateIndex(c => c.Index(index));
             _settings.SetDefaultIndex(index);
             _client = new ElasticClient(_settings);
         }
         var response =
             _client.Map<Game>(
                 h =>
                     h.Properties(p => p
                         .String(s => s.Name(i => i.GameId).Index(FieldIndexOption.NotAnalyzed))
                         .String(s => s.Name(i => i.Status).Index(FieldIndexOption.NotAnalyzed))
                         .String(s => s.Name(i => i.TurnOfPlayer).Index(FieldIndexOption.NotAnalyzed))
                         .Object<Winner>(o => o.Name(w => w.Winner).Properties(wp => wp.String(f => f.Name(l => l.FacebookId).Index(FieldIndexOption.NotAnalyzed)))
                         )));
         IsServerError(response);
     }
     catch (Exception ex)
     {
         Logger.LogException(ex, _source, "GameDataManager", Severity.Critical);
         throw;
     }
 }
        static void Main(string[] args)
        {
            var context = new ElasticDBEntities();

            var artists = context.Artists.ToList();

            var node = "http://localhost:9200";

            var searchBoxUri = new Uri(node);
            var settings = new ConnectionSettings(searchBoxUri);
            //settings.SetDefaultIndex("sample");

            var client = new ElasticClient(settings);
            
            if (client.IndexExists("store").Exists)
            {
                client.DeleteIndex("store");
            }

            //client.CreateIndex("sample");

            foreach (var artist in artists)
            {
                //var index = client.Index(artist);
                var index = client.Index(artist, i => i.Index("store").Refresh());
            }

            // Index all documents
            //client.IndexMany<Artist>(artists);

        }
Пример #4
0
        public void RebuildIndex(string userId, int? noteId)
        {
            if (!string.IsNullOrWhiteSpace(userId))
            {
                var notes = new List<NoteModel>();

                if (noteId.HasValue)
                    notes.Add(Repository.GetById(noteId.Value));
                else
                    notes = Repository.GetByUserId(userId).ToList();
                

                var settings = new ConnectionSettings(new Uri("http://localhost:9200"), "mapnotes");
                var client = new ElasticClient(settings);


                var indexExists = client.IndexExists("mapnotes");
                if (!indexExists.Exists)
                {
                    client.CreateIndex(descriptor => descriptor.Index("mapnotes")
                            .AddMapping<NoteIndex>(m => m.Properties(p => p.GeoPoint(d => d.Name(f => f.Location).IndexLatLon()))));
                }

                foreach (var note in notes)
                {
                    client.Index(new NoteIndex
                    {
                        Id = note.Id,
                        UserId = note.UserId,
                        Title = note.Title,
                        Location = new Location(note.Latitude, note.Longitude)
                    });
                }
            }
        }
        private static void deleteTestIndices()
        {
            var elastic = new ElasticClient(new ConnectionSettings(new UriBuilder("http", "localhost", 9200, "", "").Uri, remindersIndex));

            var indexExists = elastic.IndexExists(remindersIndex);
            if (indexExists.Exists)
            {
                var deleteResponse = elastic.DeleteIndex(remindersIndex, d => d.Index(remindersIndex));
                if (!deleteResponse.IsValid)
                    throw new Exception("Initialization failed");
            }
        }
        //static constructor to ensure config settings are set
        static ElasticSearchConnection()
        {
            DropShareServerName = DataMartAppSettings.Default.dropshareServerName;
            DropShare = DataMartAppSettings.Default.dropshareFolderName;
            ElasticSearchUrl = DataMartAppSettings.Default.elasticSearchURL;
            ConnectionSettings conSettings = new ConnectionSettings(new Uri(ElasticSearchUrl));
            ESClient = new ElasticClient(conSettings);

            //Create the logging index in the Server if it doesn't already exists
            if (!ESClient.IndexExists("logs").Exists)
            {
                ESClient.CreateIndex("logs");
            }
        }
        private  void deleteTestIndices()
        {
            var ci = ElasticStorageProvider.FromConnectionString<ConnectionInfo>(TEST_CONNECTION_STRING);
            var cs = new ConnectionSettings(new UriBuilder("http", ci.Host, ci.Port, "", "").Uri, ci.Index);
            var elastic = new ElasticClient(cs);

            var indexExists = elastic.IndexExists(ci.Index);
            if (indexExists.Exists)
            {
                var deleteResponse = elastic.DeleteIndex(ci.Index, d => d.Index(ci.Index));
                if (!deleteResponse.IsValid)
                    throw new Exception("Initialization failed");
            }
        }
Пример #8
0
 public static ElasticClient Create()
 {
     var esHost = ConfigurationManager.AppSettings["esHost"];
     Log.DebugFormat("esHost = {0}", esHost);
     var esIndex = ConfigurationManager.AppSettings["esIndex"];
     Log.DebugFormat("esIndex = {0}", esIndex);
     var client = new ElasticClient(new ConnectionSettings(new Uri(esHost)).SetDefaultIndex(esIndex));
     //TODO: double-check
     lock (Locker)
     {
         if (!client.IndexExists(esIndex).Exists)
             client.CreateIndex(esIndex, new IndexSettings());
     }
     return client;
 }
Пример #9
0
        static void Main(string[] args)
        {
            var indexName = _IndexName;
              int numOfCountries = 1;
              //if (args.Length > 0 && args[0] == "full")
              {
            numOfCountries = 42;
            indexName += "-full";
              }

              settings = new ConnectionSettings(new Uri(esconnection))
            .SetDefaultIndex(indexName);
              client = new ElasticClient(settings);

              using (new Timed("CreateIndex"))
              {
            if (client.IndexExists(indexName).Exists)
            {
              client.DeleteIndex(indexName);
            }
            PricesInstaller.CreateIndex(client, indexName).AssertValid();
              }
              var accos = Timed.Do("Generating Accos", () => Generator.GenerateAccos(numOfCountries).ToList());

              using (new Timed(string.Format("Indexing {0} Accos", accos.Count)))
              {
            var sets = accos.InSetsOf(100);

            foreach (var accoSet in sets)
            {
              using (new Timed(string.Format("Indexing Acco {0}-{1}/{2}", accoSet.First().Id, accoSet.First().Id + 99, accos.Count)))
              {
            int pricesCount = 0;
            foreach (var acco in accoSet)
            {
              IndexAcco(acco);
              var accoAvailabilities = Generator.GenerateAccoAvailabilities(acco.HighSeasons);

              var prices = Generator.GeneratePrices(acco, accoAvailabilities);
              pricesCount += prices.Count;
              IndexPrices(acco, prices);
            }
            Console.WriteLine("Generated {0} prices for {1} accos", pricesCount, accoSet.Count);
              }
            }
              }
        }
Пример #10
0
        public void elasticSearch() {


            var uri = new Uri("http://localhost:9200");
            var settings = new ConnectionSettings(uri).SetDefaultIndex("contacts");
            var client = new ElasticClient(settings);

            if (client.IndexExists("contacts").Exists)
            {

                UpsertContact(client, new Contacts(1,"Andrew Kagwa", "Uganda"));
                Console.WriteLine("Indexing Successfull");
            }
            else
            {

            }



            Console.ReadKey();

            QueryContainer query = new TermQuery
            {
                Field = "Name",
                Value = "Andrew",
                 
                
            };

            var searchReq = new SearchRequest
            {

                From = 0,
                Size = 10,
                Query = query,
            };

            var result = client.Search<Contacts>(searchReq);



        }
Пример #11
0
        protected DataSet(string datasourceName, bool fireException)
        {
            this.datasetId = datasourceName.ToLower();
            this.client    = Lib.ES.Manager.GetESClient(datasetId, idxType: ES.Manager.IndexType.DataSource);


            var ret = client.IndexExists(client.ConnectionSettings.DefaultIndex);

            if (ret.Exists == false)
            {
                if (fireException)
                {
                    throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetNotFound);
                }
                else
                {
                    this.client = null;
                }
            }
        }
Пример #12
0
        private static void CreateIndex(string name, ElasticClient client)
        {
            if (client.IndexExists(x => x.Index(name)).Exists)
            {
                Log.Logger.Information("Delete index {indexName}", name);

                client.DeleteIndex(x => x.Index(name));
            }

            Log.Logger.Information("Create index {indexName}", name);
            client.CreateIndex(name, c => c
                .NumberOfReplicas(0)
                .NumberOfShards(1)
                .AddMapping<Book>(m => m.MapFromAttributes())
                .AddMapping<CD>(m => m.MapFromAttributes()));
            Log.Logger.Information("Index {indexName} created", name);

            Log.Logger.Information("Closing index {indexName}", name);

            Thread.Sleep(30000);
            var closeRes = client.CloseIndex(x => x.Index(name));

            
            if (!closeRes.Acknowledged)
                Log.Logger.Error("Could not close index: {message}",closeRes.ServerError.Error);

            Log.Logger.Information("Add analyzer to index {indexName}", name);
            var res = client.Raw.IndicesPutSettings(name, File.ReadAllText("Analyzer.json"));

            if (!res.Success)
                Log.Logger.Error("Could not create analyzer: {error}", res.OriginalException.ToString());


            Log.Logger.Information("Open index {indexName}", name);
            client.OpenIndex(x => x.Index(name));

            RandomBooks(1000, name, client);
            RandomCDs(1000, name, client);
        }
        private void WipeIndex(ElasticClient esClient, string indexName)
        {
            var indexExistsRequest = new IndexExistsRequest(indexName);
            var indexExistsResult = esClient.IndexExists(indexExistsRequest);

            if (indexExistsResult.Exists == true)
            {
                esClient.DeleteIndex(new DeleteIndexRequest(indexName));
            }

            EnsureIndexExists(esClient, indexName);
        }
        private void EnsureIndexExists(ElasticClient esClient, string indexName)
        {
            var indexExistsRequest = new IndexExistsRequest(indexName);
            var indexExistsResult = esClient.IndexExists(indexExistsRequest);

            if (indexExistsResult.Exists == false)
            {
                var createIndexRequest = new CreateIndexRequest(indexName);

                var createIndexResponse = esClient.CreateIndex(createIndexRequest);

                if (createIndexResponse.Acknowledged == false)
                {
                    throw new ApplicationException(string.Format("Kunne ikke lave {0} index", indexName));
                }
            }
        }
        public ActionResult ReIndexAll()
        {
            var documents = db.DocumentModels.ToList();

            var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
            var searchBoxUri = new Uri(uriString);

            var settings = new ConnectionSettings(searchBoxUri);
            settings.SetDefaultIndex(indexName);

            var client = new ElasticClient(settings);

            // delete index if exists at startup
            if (client.IndexExists(indexName).Exists)
            {
                client.DeleteIndex(indexName);
            }

            // Create a new "sample" index with default settings
            //client.CreateIndex("sample", new IndexSettings());
            ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName);
            iCreateIndexReq.IndexSettings = new IndexSettings();
            iCreateIndexReq.IndexSettings.NumberOfReplicas = 10;

            //client.CreateIndex(iCreateIndexReq);

            var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10));

            //client.CreateIndex()
            // Index all documents
            client.IndexMany<DocumentModel>(documents);
            //client.Index()

            ViewBag.Message = "Reindexing all database is complete!";

            return RedirectToAction("Index");
        }
        private static void Index(DocumentModel document, String operation)
        {
            var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
            var searchBoxUri = new Uri(uriString);

            var settings = new ConnectionSettings(searchBoxUri);
            settings.SetDefaultIndex(indexName);

            var client = new ElasticClient(settings);

            if (!client.IndexExists(indexName).Exists)
            {
                // Create a new "sample" index with default settings
                ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName);
                iCreateIndexReq.IndexSettings = new IndexSettings();
                iCreateIndexReq.IndexSettings.NumberOfReplicas = 10;

                //iCreateIndexReq.IndexSettings.Mappings = new List<RootObjectMapping>();
                //RootObjectMapping rootObjectMapping = new RootObjectMapping();
                //rootObjectMapping.AllFieldMapping()
                //iCreateIndexReq.IndexSettings.Mappings.
                //client.CreateIndex(iCreateIndexReq);
                //client.CreateIndex(indexName,s=>s.)
                var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10));

                //client.CreateIndex(indexName, new IndexSettings());
                //client.create
            }

            if (operation.Equals("delete"))
            {
                //client.DeleteById(indexName, "documents", document.DocumentId);
                IDeleteByQueryRequest iDeleteByQueryRequest = new DeleteByQueryRequest();
                //IDeleteIndexRequest delReq = new DeleteIndexRequest(indexName);
                //client.DeleteIndex()
                //client.DeleteByQuery(new DeleteByQueryRequest())
                client.Delete<DocumentModel>(f => f.Id(document.DocumentId).Index(indexName).Refresh());
                //var response = this.Client.Delete<ElasticsearchProject>(f=>f.Id(newDocument.Id).Index(newIndex).Refresh());
            }
            else
            {
               //IIndexRequest<DocumentModel> indexRequest = IndexRequest<DocumentModel>();
               //client.Index(i)
               //client.Index(document, indexName, "documents", document.DocumentId);
               //IndexDescriptor indexDesc = IndexDescriptor;
               //IndexRequestParameters indexParameter = new IndexRequestParameters();
               // indexParameter.Replication(1);
                client.Index(document, i => i.Id(document.DocumentId).Index(indexName));
                //client.Index();
                //client.Index()
            }
        }
Пример #17
0
        // Create index for filling data
        // @indexName : whatever
        public void ElasticSearchSetup(string indexName)
        {
            Uri node = new Uri(ElasticServer);
            ConnectionSettings settings = new ConnectionSettings(node);
            settings.DefaultIndex(indexName);
            client = new ElasticClient(settings);

            // create the index if it doesn't exist
            if (!client.IndexExists(indexName).Exists)
            {
                client.CreateIndex(indexName);
            }
        }
Пример #18
0
        public static IElasticClient ConfigureElastic()
        {
            Logger.Info("Setting up Elastic");
            var connectionString = ConfigurationManager.ConnectionStrings["Elastic"];
            if (connectionString == null)
                throw new ArgumentException("No elastic connection string found");

            var data = connectionString.ConnectionString.Split(';');

            var url = data.FirstOrDefault(x => x.StartsWith("Url", StringComparison.CurrentCultureIgnoreCase));
            if (url == null)
                throw new ArgumentException("No URL parameter in elastic connection string");
            var index = data.FirstOrDefault(x => x.StartsWith("DefaultIndex", StringComparison.CurrentCultureIgnoreCase));
            if (string.IsNullOrEmpty(index))
                throw new ArgumentException("No DefaultIndex parameter in elastic connection string");

            index = index.Substring(13);

            var node = new Uri(url.Substring(4));
            var pool = new Elasticsearch.Net.SingleNodeConnectionPool(node);

            var regex = new Regex("([+\\-!\\(\\){}\\[\\]^\"~*?:\\\\\\/>< ]|[&\\|]{2}|AND|OR|NOT)", RegexOptions.Compiled);
            var settings = new Nest.ConnectionSettings(pool, (_) => new JsonNetSerializer(_))
                .DefaultIndex(index)
                .DefaultTypeNameInferrer(type =>
                        type.FullName
                        .Replace("Pulse.Presentation.ServiceStack.", "")
                        .Replace("Pulse.Application.Elastic.", "")
                        .Replace('.', '_')
                        )
                .DefaultFieldNameInferrer(field => regex.Replace(field, ""));
#if DEBUG
            settings = settings.DisableDirectStreaming();
#endif

            var client = new ElasticClient(settings);


            if (!client.IndexExists(index).Exists)
                client.CreateIndex(index, i => i
                    .Settings(s => s
                        .NumberOfShards(8)
                        .NumberOfReplicas(0)
                        .Analysis(analysis => analysis
                        .TokenFilters(f => f.NGram("ngram", d => d.MinGram(1).MaxGram(15)))
                        .Analyzers(a => a
                            .Custom("default_index", t => t.Tokenizer("keyword").Filters(new[] { "standard", "lowercase", "asciifolding", "kstem", "ngram" }))
                            .Custom("suffix", t => t.Tokenizer("keyword").Filters(new[] { "standard", "lowercase", "asciifolding", "reverse" }))
                        )
                    )));

            return client;
        }
Пример #19
0
        public void Setup()
        {
            settings = new ConnectionSettings(new Uri(esconnection))
            .SetDefaultIndex(indexName)
            .UsePrettyResponses();
              client = new ElasticClient(settings);

              if (client.IndexExists(indexName).Exists)
            client.DeleteIndex(indexName);

              PricesInstaller.CreateIndex(client, indexName).AssertValid();

              client.Index(new Acco { Id = 1, LocationCode = "AA" }).AssertValid();
              client.Index(new Acco { Id = 2, LocationCode = "BB" }).AssertValid();
              client.Index(new Acco { Id = 3, LocationCode = "CC" }).AssertValid();

              client.Index(new Price { Id = 1, CareType = "AI", DepartureDate = DateTime.Today, DurationDays = 10, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 1.ToString(CultureInfo.InvariantCulture) }).AssertValid();
              client.Index(new Price { Id = 2, CareType = "HP", DepartureDate = DateTime.Today, DurationDays = 11, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 1.ToString(CultureInfo.InvariantCulture) }).AssertValid();
              client.Index(new Price { Id = 3, CareType = "AI", DepartureDate = DateTime.Today, DurationDays = 12, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 2.ToString(CultureInfo.InvariantCulture) }).AssertValid();
              client.Refresh().AssertValid();
        }
Пример #20
0
        public IElasticClient ConfigureElastic()
        {
            var connectionString = ConfigurationManager.ConnectionStrings["Elastic"];
            if (connectionString == null)
                throw new ArgumentException("No elastic connection string found");

            var data = connectionString.ConnectionString.Split(';');

            var url = data.FirstOrDefault(x => x.StartsWith("Url", StringComparison.CurrentCultureIgnoreCase));
            if (url == null)
                throw new ArgumentException("No URL parameter in elastic connection string");
            var index = data.FirstOrDefault(x => x.StartsWith("DefaultIndex", StringComparison.CurrentCultureIgnoreCase));
            if (index.IsNullOrEmpty())
                throw new ArgumentException("No DefaultIndex parameter in elastic connection string");

            index = index.Substring(13);

            var node = new Uri(url.Substring(4));

            var settings = new Nest.ConnectionSettings(node);
            settings.SetDefaultIndex(index);

            settings.SetDefaultTypeNameInferrer(type => type.FullName.Replace("Demo.Application.ServiceStack.", "").Replace('.', '_'));
            // Disable camel case field names (need to match out POCO field names)
            settings.SetDefaultPropertyNameInferrer(field => field);

            var client = new ElasticClient(settings);
            if (!client.IndexExists(index).Exists)
                client.CreateIndex(index, i => i
                        .Analysis(analysis => analysis
                            .TokenFilters(f => f
                                .Add("ngram", new Nest.NgramTokenFilter { MinGram = 2, MaxGram = 15 })
                                )
                            .Analyzers(a => a
                                .Add(
                                    "default_index",
                                    new Nest.CustomAnalyzer
                                    {
                                        Tokenizer = "standard",
                                        Filter = new[] { "standard", "lowercase", "asciifolding", "kstem", "ngram" }
                                    }
                                )
                                .Add(
                                    "suffix",
                                    new Nest.CustomAnalyzer
                                    {
                                        Tokenizer = "keyword",
                                        Filter = new[] { "standard", "lowercase", "asciifolding", "reverse" }
                                    }
                                )
                            )
                        ));

            return client;
        }
        private static void Index(Document document, String operation)
        {
            var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
            var searchBoxUri = new Uri(uriString);

            var settings = new ConnectionSettings(searchBoxUri);
            settings.SetDefaultIndex("sample");

            var client = new ElasticClient(settings);

            if (!client.IndexExists("sample").Exists)
            {
                // Create a new "sample" index with default settings
                client.CreateIndex("sample", new IndexSettings());
            }

            if (operation.Equals("delete"))
            {
                client.DeleteById("sample", "documents", document.DocumentId);
            }
            else
            {
                client.Index(document, "sample", "documents", document.DocumentId);
            }
        }
        public ActionResult ReIndexAll()
        {
            var documents = db.Documents.ToList();

            var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
            var searchBoxUri = new Uri(uriString);

            var settings = new ConnectionSettings(searchBoxUri);
            settings.SetDefaultIndex("sample");

            var client = new ElasticClient(settings);

            // delete index if exists at startup
            if (client.IndexExists("sample").Exists)
            {
                client.DeleteIndex("sample");
            }

            // Create a new "sample" index with default settings
            client.CreateIndex("sample", new IndexSettings());

            // Index all documents
            client.IndexMany<Document>(documents);

            ViewBag.Message = "Reindexing all database is complete!";

            return RedirectToAction("Index");
        }