public BlackListDataManager()
{
try
{
var elasticSearchUrl = ConfigurationManager.AppSettings["ElasticStoreConnection"];
var index = ConfigurationManager.AppSettings["EIndex_BlackList"];
_settings =
new ConnectionSettings(new Uri(elasticSearchUrl)).SetDefaultIndex(index).PluralizeTypeNames();
_client = new ElasticClient(_settings);
var isIndexExist = _client.IndexExists(i => i.Index(index));
if (!isIndexExist.Exists)
{
_client.CreateIndex(c => c.Index(index));
_settings.SetDefaultIndex(index);
_client = new ElasticClient(_settings);
}
var response =
_client.Map<BlockedUser>(
h =>
h.MapFromAttributes(10).Properties(p => p
.String(s => s.Name(i => i.ByEmail).Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name(i => i.ToEmail).Index(FieldIndexOption.NotAnalyzed))
));
IsServerError(response);
}
catch (Exception ex)
{
Logger.LogException(ex, _soruce, "BlackListDataManager", Severity.Critical);
throw;
}
}
public GameDataManager()
{
try
{
var elasticSearchUrl = ConfigurationManager.AppSettings["ElasticStoreConnection"];
var index = ConfigurationManager.AppSettings["EIndex_Game"];
_settings =
new ConnectionSettings(new Uri(elasticSearchUrl)).SetDefaultIndex(index).PluralizeTypeNames();
_client = new ElasticClient(_settings);
var isIndexExist = _client.IndexExists(i => i.Index(index));
if (!isIndexExist.Exists)
{
_client.CreateIndex(c => c.Index(index));
_settings.SetDefaultIndex(index);
_client = new ElasticClient(_settings);
}
var response =
_client.Map<Game>(
h =>
h.Properties(p => p
.String(s => s.Name(i => i.GameId).Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name(i => i.Status).Index(FieldIndexOption.NotAnalyzed))
.String(s => s.Name(i => i.TurnOfPlayer).Index(FieldIndexOption.NotAnalyzed))
.Object<Winner>(o => o.Name(w => w.Winner).Properties(wp => wp.String(f => f.Name(l => l.FacebookId).Index(FieldIndexOption.NotAnalyzed)))
)));
IsServerError(response);
}
catch (Exception ex)
{
Logger.LogException(ex, _source, "GameDataManager", Severity.Critical);
throw;
}
}
static void Main(string[] args)
{
var context = new ElasticDBEntities();
var artists = context.Artists.ToList();
var node = "http://localhost:9200";
var searchBoxUri = new Uri(node);
var settings = new ConnectionSettings(searchBoxUri);
//settings.SetDefaultIndex("sample");
var client = new ElasticClient(settings);
if (client.IndexExists("store").Exists)
{
client.DeleteIndex("store");
}
//client.CreateIndex("sample");
foreach (var artist in artists)
{
//var index = client.Index(artist);
var index = client.Index(artist, i => i.Index("store").Refresh());
}
// Index all documents
//client.IndexMany<Artist>(artists);
}
public void RebuildIndex(string userId, int? noteId)
{
if (!string.IsNullOrWhiteSpace(userId))
{
var notes = new List<NoteModel>();
if (noteId.HasValue)
notes.Add(Repository.GetById(noteId.Value));
else
notes = Repository.GetByUserId(userId).ToList();
var settings = new ConnectionSettings(new Uri("http://localhost:9200"), "mapnotes");
var client = new ElasticClient(settings);
var indexExists = client.IndexExists("mapnotes");
if (!indexExists.Exists)
{
client.CreateIndex(descriptor => descriptor.Index("mapnotes")
.AddMapping<NoteIndex>(m => m.Properties(p => p.GeoPoint(d => d.Name(f => f.Location).IndexLatLon()))));
}
foreach (var note in notes)
{
client.Index(new NoteIndex
{
Id = note.Id,
UserId = note.UserId,
Title = note.Title,
Location = new Location(note.Latitude, note.Longitude)
});
}
}
}
private static void deleteTestIndices()
{
var elastic = new ElasticClient(new ConnectionSettings(new UriBuilder("http", "localhost", 9200, "", "").Uri, remindersIndex));
var indexExists = elastic.IndexExists(remindersIndex);
if (indexExists.Exists)
{
var deleteResponse = elastic.DeleteIndex(remindersIndex, d => d.Index(remindersIndex));
if (!deleteResponse.IsValid)
throw new Exception("Initialization failed");
}
}
//static constructor to ensure config settings are set
static ElasticSearchConnection()
{
DropShareServerName = DataMartAppSettings.Default.dropshareServerName;
DropShare = DataMartAppSettings.Default.dropshareFolderName;
ElasticSearchUrl = DataMartAppSettings.Default.elasticSearchURL;
ConnectionSettings conSettings = new ConnectionSettings(new Uri(ElasticSearchUrl));
ESClient = new ElasticClient(conSettings);
//Create the logging index in the Server if it doesn't already exists
if (!ESClient.IndexExists("logs").Exists)
{
ESClient.CreateIndex("logs");
}
}
private void deleteTestIndices()
{
var ci = ElasticStorageProvider.FromConnectionString<ConnectionInfo>(TEST_CONNECTION_STRING);
var cs = new ConnectionSettings(new UriBuilder("http", ci.Host, ci.Port, "", "").Uri, ci.Index);
var elastic = new ElasticClient(cs);
var indexExists = elastic.IndexExists(ci.Index);
if (indexExists.Exists)
{
var deleteResponse = elastic.DeleteIndex(ci.Index, d => d.Index(ci.Index));
if (!deleteResponse.IsValid)
throw new Exception("Initialization failed");
}
}
public static ElasticClient Create()
{
var esHost = ConfigurationManager.AppSettings["esHost"];
Log.DebugFormat("esHost = {0}", esHost);
var esIndex = ConfigurationManager.AppSettings["esIndex"];
Log.DebugFormat("esIndex = {0}", esIndex);
var client = new ElasticClient(new ConnectionSettings(new Uri(esHost)).SetDefaultIndex(esIndex));
//TODO: double-check
lock (Locker)
{
if (!client.IndexExists(esIndex).Exists)
client.CreateIndex(esIndex, new IndexSettings());
}
return client;
}
static void Main(string[] args)
{
var indexName = _IndexName;
int numOfCountries = 1;
//if (args.Length > 0 && args[0] == "full")
{
numOfCountries = 42;
indexName += "-full";
}
settings = new ConnectionSettings(new Uri(esconnection))
.SetDefaultIndex(indexName);
client = new ElasticClient(settings);
using (new Timed("CreateIndex"))
{
if (client.IndexExists(indexName).Exists)
{
client.DeleteIndex(indexName);
}
PricesInstaller.CreateIndex(client, indexName).AssertValid();
}
var accos = Timed.Do("Generating Accos", () => Generator.GenerateAccos(numOfCountries).ToList());
using (new Timed(string.Format("Indexing {0} Accos", accos.Count)))
{
var sets = accos.InSetsOf(100);
foreach (var accoSet in sets)
{
using (new Timed(string.Format("Indexing Acco {0}-{1}/{2}", accoSet.First().Id, accoSet.First().Id + 99, accos.Count)))
{
int pricesCount = 0;
foreach (var acco in accoSet)
{
IndexAcco(acco);
var accoAvailabilities = Generator.GenerateAccoAvailabilities(acco.HighSeasons);
var prices = Generator.GeneratePrices(acco, accoAvailabilities);
pricesCount += prices.Count;
IndexPrices(acco, prices);
}
Console.WriteLine("Generated {0} prices for {1} accos", pricesCount, accoSet.Count);
}
}
}
}
public void elasticSearch() {
var uri = new Uri("http://localhost:9200");
var settings = new ConnectionSettings(uri).SetDefaultIndex("contacts");
var client = new ElasticClient(settings);
if (client.IndexExists("contacts").Exists)
{
UpsertContact(client, new Contacts(1,"Andrew Kagwa", "Uganda"));
Console.WriteLine("Indexing Successfull");
}
else
{
}
Console.ReadKey();
QueryContainer query = new TermQuery
{
Field = "Name",
Value = "Andrew",
};
var searchReq = new SearchRequest
{
From = 0,
Size = 10,
Query = query,
};
var result = client.Search<Contacts>(searchReq);
}
protected DataSet(string datasourceName, bool fireException)
{
this.datasetId = datasourceName.ToLower();
this.client = Lib.ES.Manager.GetESClient(datasetId, idxType: ES.Manager.IndexType.DataSource);
var ret = client.IndexExists(client.ConnectionSettings.DefaultIndex);
if (ret.Exists == false)
{
if (fireException)
{
throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetNotFound);
}
else
{
this.client = null;
}
}
}
private static void CreateIndex(string name, ElasticClient client)
{
if (client.IndexExists(x => x.Index(name)).Exists)
{
Log.Logger.Information("Delete index {indexName}", name);
client.DeleteIndex(x => x.Index(name));
}
Log.Logger.Information("Create index {indexName}", name);
client.CreateIndex(name, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.AddMapping<Book>(m => m.MapFromAttributes())
.AddMapping<CD>(m => m.MapFromAttributes()));
Log.Logger.Information("Index {indexName} created", name);
Log.Logger.Information("Closing index {indexName}", name);
Thread.Sleep(30000);
var closeRes = client.CloseIndex(x => x.Index(name));
if (!closeRes.Acknowledged)
Log.Logger.Error("Could not close index: {message}",closeRes.ServerError.Error);
Log.Logger.Information("Add analyzer to index {indexName}", name);
var res = client.Raw.IndicesPutSettings(name, File.ReadAllText("Analyzer.json"));
if (!res.Success)
Log.Logger.Error("Could not create analyzer: {error}", res.OriginalException.ToString());
Log.Logger.Information("Open index {indexName}", name);
client.OpenIndex(x => x.Index(name));
RandomBooks(1000, name, client);
RandomCDs(1000, name, client);
}
private void WipeIndex(ElasticClient esClient, string indexName)
{
var indexExistsRequest = new IndexExistsRequest(indexName);
var indexExistsResult = esClient.IndexExists(indexExistsRequest);
if (indexExistsResult.Exists == true)
{
esClient.DeleteIndex(new DeleteIndexRequest(indexName));
}
EnsureIndexExists(esClient, indexName);
}
private void EnsureIndexExists(ElasticClient esClient, string indexName)
{
var indexExistsRequest = new IndexExistsRequest(indexName);
var indexExistsResult = esClient.IndexExists(indexExistsRequest);
if (indexExistsResult.Exists == false)
{
var createIndexRequest = new CreateIndexRequest(indexName);
var createIndexResponse = esClient.CreateIndex(createIndexRequest);
if (createIndexResponse.Acknowledged == false)
{
throw new ApplicationException(string.Format("Kunne ikke lave {0} index", indexName));
}
}
}
public ActionResult ReIndexAll()
{
var documents = db.DocumentModels.ToList();
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex(indexName);
var client = new ElasticClient(settings);
// delete index if exists at startup
if (client.IndexExists(indexName).Exists)
{
client.DeleteIndex(indexName);
}
// Create a new "sample" index with default settings
//client.CreateIndex("sample", new IndexSettings());
ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName);
iCreateIndexReq.IndexSettings = new IndexSettings();
iCreateIndexReq.IndexSettings.NumberOfReplicas = 10;
//client.CreateIndex(iCreateIndexReq);
var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10));
//client.CreateIndex()
// Index all documents
client.IndexMany<DocumentModel>(documents);
//client.Index()
ViewBag.Message = "Reindexing all database is complete!";
return RedirectToAction("Index");
}
private static void Index(DocumentModel document, String operation)
{
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex(indexName);
var client = new ElasticClient(settings);
if (!client.IndexExists(indexName).Exists)
{
// Create a new "sample" index with default settings
ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName);
iCreateIndexReq.IndexSettings = new IndexSettings();
iCreateIndexReq.IndexSettings.NumberOfReplicas = 10;
//iCreateIndexReq.IndexSettings.Mappings = new List<RootObjectMapping>();
//RootObjectMapping rootObjectMapping = new RootObjectMapping();
//rootObjectMapping.AllFieldMapping()
//iCreateIndexReq.IndexSettings.Mappings.
//client.CreateIndex(iCreateIndexReq);
//client.CreateIndex(indexName,s=>s.)
var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10));
//client.CreateIndex(indexName, new IndexSettings());
//client.create
}
if (operation.Equals("delete"))
{
//client.DeleteById(indexName, "documents", document.DocumentId);
IDeleteByQueryRequest iDeleteByQueryRequest = new DeleteByQueryRequest();
//IDeleteIndexRequest delReq = new DeleteIndexRequest(indexName);
//client.DeleteIndex()
//client.DeleteByQuery(new DeleteByQueryRequest())
client.Delete<DocumentModel>(f => f.Id(document.DocumentId).Index(indexName).Refresh());
//var response = this.Client.Delete<ElasticsearchProject>(f=>f.Id(newDocument.Id).Index(newIndex).Refresh());
}
else
{
//IIndexRequest<DocumentModel> indexRequest = IndexRequest<DocumentModel>();
//client.Index(i)
//client.Index(document, indexName, "documents", document.DocumentId);
//IndexDescriptor indexDesc = IndexDescriptor;
//IndexRequestParameters indexParameter = new IndexRequestParameters();
// indexParameter.Replication(1);
client.Index(document, i => i.Id(document.DocumentId).Index(indexName));
//client.Index();
//client.Index()
}
}
// Create index for filling data
// @indexName : whatever
public void ElasticSearchSetup(string indexName)
{
Uri node = new Uri(ElasticServer);
ConnectionSettings settings = new ConnectionSettings(node);
settings.DefaultIndex(indexName);
client = new ElasticClient(settings);
// create the index if it doesn't exist
if (!client.IndexExists(indexName).Exists)
{
client.CreateIndex(indexName);
}
}
public static IElasticClient ConfigureElastic()
{
Logger.Info("Setting up Elastic");
var connectionString = ConfigurationManager.ConnectionStrings["Elastic"];
if (connectionString == null)
throw new ArgumentException("No elastic connection string found");
var data = connectionString.ConnectionString.Split(';');
var url = data.FirstOrDefault(x => x.StartsWith("Url", StringComparison.CurrentCultureIgnoreCase));
if (url == null)
throw new ArgumentException("No URL parameter in elastic connection string");
var index = data.FirstOrDefault(x => x.StartsWith("DefaultIndex", StringComparison.CurrentCultureIgnoreCase));
if (string.IsNullOrEmpty(index))
throw new ArgumentException("No DefaultIndex parameter in elastic connection string");
index = index.Substring(13);
var node = new Uri(url.Substring(4));
var pool = new Elasticsearch.Net.SingleNodeConnectionPool(node);
var regex = new Regex("([+\\-!\\(\\){}\\[\\]^\"~*?:\\\\\\/>< ]|[&\\|]{2}|AND|OR|NOT)", RegexOptions.Compiled);
var settings = new Nest.ConnectionSettings(pool, (_) => new JsonNetSerializer(_))
.DefaultIndex(index)
.DefaultTypeNameInferrer(type =>
type.FullName
.Replace("Pulse.Presentation.ServiceStack.", "")
.Replace("Pulse.Application.Elastic.", "")
.Replace('.', '_')
)
.DefaultFieldNameInferrer(field => regex.Replace(field, ""));
#if DEBUG
settings = settings.DisableDirectStreaming();
#endif
var client = new ElasticClient(settings);
if (!client.IndexExists(index).Exists)
client.CreateIndex(index, i => i
.Settings(s => s
.NumberOfShards(8)
.NumberOfReplicas(0)
.Analysis(analysis => analysis
.TokenFilters(f => f.NGram("ngram", d => d.MinGram(1).MaxGram(15)))
.Analyzers(a => a
.Custom("default_index", t => t.Tokenizer("keyword").Filters(new[] { "standard", "lowercase", "asciifolding", "kstem", "ngram" }))
.Custom("suffix", t => t.Tokenizer("keyword").Filters(new[] { "standard", "lowercase", "asciifolding", "reverse" }))
)
)));
return client;
}
public void Setup()
{
settings = new ConnectionSettings(new Uri(esconnection))
.SetDefaultIndex(indexName)
.UsePrettyResponses();
client = new ElasticClient(settings);
if (client.IndexExists(indexName).Exists)
client.DeleteIndex(indexName);
PricesInstaller.CreateIndex(client, indexName).AssertValid();
client.Index(new Acco { Id = 1, LocationCode = "AA" }).AssertValid();
client.Index(new Acco { Id = 2, LocationCode = "BB" }).AssertValid();
client.Index(new Acco { Id = 3, LocationCode = "CC" }).AssertValid();
client.Index(new Price { Id = 1, CareType = "AI", DepartureDate = DateTime.Today, DurationDays = 10, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 1.ToString(CultureInfo.InvariantCulture) }).AssertValid();
client.Index(new Price { Id = 2, CareType = "HP", DepartureDate = DateTime.Today, DurationDays = 11, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 1.ToString(CultureInfo.InvariantCulture) }).AssertValid();
client.Index(new Price { Id = 3, CareType = "AI", DepartureDate = DateTime.Today, DurationDays = 12, PricePerPerson = 100, TransportFrom = "AMS", TransportType = "EV" }, new IndexParameters { Parent = 2.ToString(CultureInfo.InvariantCulture) }).AssertValid();
client.Refresh().AssertValid();
}
public IElasticClient ConfigureElastic()
{
var connectionString = ConfigurationManager.ConnectionStrings["Elastic"];
if (connectionString == null)
throw new ArgumentException("No elastic connection string found");
var data = connectionString.ConnectionString.Split(';');
var url = data.FirstOrDefault(x => x.StartsWith("Url", StringComparison.CurrentCultureIgnoreCase));
if (url == null)
throw new ArgumentException("No URL parameter in elastic connection string");
var index = data.FirstOrDefault(x => x.StartsWith("DefaultIndex", StringComparison.CurrentCultureIgnoreCase));
if (index.IsNullOrEmpty())
throw new ArgumentException("No DefaultIndex parameter in elastic connection string");
index = index.Substring(13);
var node = new Uri(url.Substring(4));
var settings = new Nest.ConnectionSettings(node);
settings.SetDefaultIndex(index);
settings.SetDefaultTypeNameInferrer(type => type.FullName.Replace("Demo.Application.ServiceStack.", "").Replace('.', '_'));
// Disable camel case field names (need to match out POCO field names)
settings.SetDefaultPropertyNameInferrer(field => field);
var client = new ElasticClient(settings);
if (!client.IndexExists(index).Exists)
client.CreateIndex(index, i => i
.Analysis(analysis => analysis
.TokenFilters(f => f
.Add("ngram", new Nest.NgramTokenFilter { MinGram = 2, MaxGram = 15 })
)
.Analyzers(a => a
.Add(
"default_index",
new Nest.CustomAnalyzer
{
Tokenizer = "standard",
Filter = new[] { "standard", "lowercase", "asciifolding", "kstem", "ngram" }
}
)
.Add(
"suffix",
new Nest.CustomAnalyzer
{
Tokenizer = "keyword",
Filter = new[] { "standard", "lowercase", "asciifolding", "reverse" }
}
)
)
));
return client;
}
private static void Index(Document document, String operation)
{
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex("sample");
var client = new ElasticClient(settings);
if (!client.IndexExists("sample").Exists)
{
// Create a new "sample" index with default settings
client.CreateIndex("sample", new IndexSettings());
}
if (operation.Equals("delete"))
{
client.DeleteById("sample", "documents", document.DocumentId);
}
else
{
client.Index(document, "sample", "documents", document.DocumentId);
}
}
public ActionResult ReIndexAll()
{
var documents = db.Documents.ToList();
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex("sample");
var client = new ElasticClient(settings);
// delete index if exists at startup
if (client.IndexExists("sample").Exists)
{
client.DeleteIndex("sample");
}
// Create a new "sample" index with default settings
client.CreateIndex("sample", new IndexSettings());
// Index all documents
client.IndexMany<Document>(documents);
ViewBag.Message = "Reindexing all database is complete!";
return RedirectToAction("Index");
}