public void JsonSerialization_ExpectedLogStashFormat() { var request = mocks.StrictMock<HttpRequestBase>(); var timestamp = DateTime.Parse("2013-07-18T11:13:32.3149976+01:00"); using (mocks.Record()) { SetupResult.For(request.UserHostAddress).Return("http://192.168.0.1"); SetupResult.For(request.HttpMethod).Return("GET"); SetupResult.For(request.UserAgent).Return("Chrome"); SetupResult.For(request.Url).Return(new Uri("http://192.168.0.2/Some/Path/?query=somewhere")); } using (mocks.Playback()) { target = new Entry(request, timestamp) { Message = "My Message", Severity = "Trace" }; var serializer = new JsonNetSerializer(); var result = serializer.Serialize(target); var expected = "{\"@message\":\"My Message\",\"@type\":\"Trace\",\"@timestamp\":\"2013-07-18T11:13:32.3149976+01:00\",\"@source_host\":\"http://192.168.0.1\",\"@source_path\":\"GET /Some/Path/?query=somewhere\",\"@fields\":{\"request\":\"GET /Some/Path/?query=somewhere\",\"user-agent\":\"Chrome\"}}"; Assert.AreEqual(expected, result); } }
public void Execute() { Console.WriteLine("PLAIN SAMPLE"); var connection = new ElasticConnection("localhost", 9200); var serializer = new JsonNetSerializer(); var tweet = new Tweet { User = "******", Message = "trying out Elastic Search" }; var anotherTweet = new Tweet { User = "******", Message = "one more message" }; IndexTweet(tweet, "1", connection, serializer); IndexTweet(anotherTweet, "2", connection, serializer); GetTweet("1", serializer, connection); SearchTweets(connection, serializer); DeleteTweeterIndex(connection, serializer); Console.WriteLine("Press any key"); Console.ReadKey(); }
private static void IndexTweet(Tweet tweet, string id, ElasticConnection connection, JsonNetSerializer serializer) { /* $ curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty=true' -d '{ "User" : "testUser", "Message" : "trying out Elastic Search" }' */ // This is url that will be requested from ES. We can grab it and put to any ES admin console (like ElasticHead) to debug ES behavior. string indexCommand = Commands.Index(index: "twitter", type: "tweet", id: id) .Refresh(true) .Pretty(); // this will generate: twitter/tweet/1?pretty=true // This variable contains JSON of serialized tweet, thus we can check if our object serialized correctly // or use it directly in ES admin console. string tweetJson = serializer.ToJson(tweet); var result = connection.Put(indexCommand, tweetJson); // Parse index result. IndexResult indexResult = serializer.ToIndexResult(result); PrintIndexCommand(result, indexResult, indexCommand, tweetJson); }
void Log_Events_Expected() { // Dev note - this builds the index, but need to run tests again to work var connection = new ElasticConnection("localhost", 9200); if (IsIndexExists(ElasticSearchStorage.GetIndex(), connection)) { connection.Delete(new DeleteCommand(ElasticSearchStorage.GetIndex())); } for (var i = 1; i <= 100; i++) { Log.Trace("Entry Message : {0}", i); Log.Info("Entry Message : {0}", i); Log.Error("Entry Message : {0}", i); } Thread.Sleep(TimeSpan.FromSeconds(5)); var serializer = new JsonNetSerializer(); // Build the Search Query var query = new QueryBuilder<Entry>().Build(); // Execute the search string result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query); var searchResult = serializer.ToSearchResult<Entry>(result); // Check all log entries in search index Assert.AreEqual(300, searchResult.hits.total); // Secondary check, use the instance based loger using (var log = Log.GetLogger()) { for (var i = 1; i <= 100; i++) { log.Trace("Entry Message : {0}", i); log.Info("Entry Message : {0}", i); log.Error("Entry Message : {0}", i); } Thread.Sleep(TimeSpan.FromSeconds(5)); // Build the Search Query query = new QueryBuilder<Entry>().Build(); // Execute the search result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query); searchResult = serializer.ToSearchResult<Entry>(result); // Check all log entries in search index Assert.AreEqual(600, searchResult.hits.total); } }
private static void CreateIndexAlias(ElasticConnection connection, JsonNetSerializer serializer) { /* * curl -XPUT http://localhost:9200/twitter/_alias/twitter_alias */ string indexAliasCommand = Commands.IndexAlias("twitter", "twitter_alias") .Pretty(); var result = connection.Put(indexAliasCommand); // Parse index result. var indexAliasResult = serializer.ToCommandResult(result); PrintIndexAliasResult(indexAliasResult, indexAliasCommand, result); }
private static IEnumerable<Tweet> SearchTweets(ElasticConnection connection, JsonNetSerializer serializer) { string searchCommand = Commands.Search("twitter", "tweet").Pretty(); /* { "query": { "term": { "User": { "value": "testuser", "boost": "5" } } } } */ string query = new QueryBuilder<Tweet>() .Query(qry => qry .Term(term => term .Field(tweet => tweet.User) .Value("testUser".ToLower()) // by default terms query requires lowercased values. .Boost(5) ) // Alternate way //.Custom(" 'term': {{ '{0}': {{ 'value': '{1}', 'boost': '5' }} }}".AltQuote(), "User", "testuser") ).BuildBeautified(); var results = connection.Post(searchCommand, query); var searchResult = serializer.ToSearchResult<Tweet>(results); PrintSearchResults(searchResult, searchCommand, query, results); return searchResult.Documents; }
private static void DeleteTweeterIndex(ElasticConnection connection, JsonNetSerializer serializer) { /* $ curl -XDELETE 'http://localhost:9200/twitter?pretty=true' */ string deleteCommand = Commands.Delete(index: "twitter").Pretty(); var result = connection.Delete(deleteCommand); DeleteResult deleteResult = serializer.ToDeleteResult(result); PrintDeleteCommand(deleteCommand, deleteResult, result); }
private static Tweet GetTweet(string id, JsonNetSerializer serializer, ElasticConnection connection) { /* $ curl -XGET 'http://localhost:9200/twitter/tweet/1?pretty=true' */ String getCommand = Commands.Get(index: "twitter", type: "tweet", id: id).Pretty(); // this will generate: twitter/tweet/1?pretty=true var result = connection.Get(getCommand); // Deserialize Get command result to GetResult object. var getResult = serializer.ToGetResult<Tweet>(result); var getTweet = getResult.Document; PrintGetCommand(getTweet, result, getCommand); return getTweet; }
private static void BulkTweetIndex(IEnumerable<Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer) { string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh(); int id = 10; // start adding tweets from id = 10 string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Index(data: tweet, id: (id++).AsString()) ); string result = connection.Post(bulkCommand, bulkJson); //Parse bulk result; BulkResult bulkResult = serializer.ToBulkResult(result); PrintBulkCommand(bulkCommand, bulkJson, bulkResult); }
public static void BulkType(List <Bic_Iban_Codes> ModelData, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index) { string bulkCommand = new BulkCommand(index: _index, type: "iban_bic").Refresh(); string bulkJson = new BulkBuilder(serializer) .BuildCollection(ModelData, (builder, pro) => builder.Index(data: pro, id: pro.CodeID.ToString()) ); string result = connection.Post(bulkCommand, bulkJson); BulkResult bulkResult = serializer.ToBulkResult(result); connection.Post(_index + "/_refresh"); }
public static void BulkType <T>(List <T> Model, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index, string _type) where T : IElasticMapper { string bulkCommand = ""; bulkCommand = new BulkCommand(index: _index, type: _type).Refresh(); string bulkJson = new BulkBuilder(serializer) .BuildCollection(Model, (builder, pro) => builder.Index(data: pro, id: pro.id) ); string result = connection.Post(bulkCommand, bulkJson); BulkResult bulkResult = serializer.ToBulkResult(result); connection.Post(_index + "/_refresh"); }
private static Task<IEnumerable<Tweet>> SearchTweetsAsync(ElasticConnection connection, JsonNetSerializer serializer) { string searchCommand = Commands.Search("twitter", "tweet").Pretty(); string query = new QueryBuilder<Tweet>() .Query(qry => qry .Term(term => term .Field(tweet => tweet.User) .Value("testUser".ToLower()) // by default terms query requires lowercased values. .Boost(5) ) ).BuildBeautified(); return connection.PostAsync(searchCommand, query) // process search results asynchronously .ContinueWith( searchTask => { OperationResult results = searchTask.Result; var searchResult = serializer.ToSearchResult<Tweet>(results); Console.WriteLine("ASYNC Search Results: \r\n"); PrintSearchResults(searchResult, searchCommand, query, results); return searchResult.Documents; }); }
public ElasticSearchDbLogger(ITime time) { _time = time; var host = Environment.GetEnvironmentVariable("ElasticSearchHost"); if (string.IsNullOrEmpty(host)) { throw new ConfigurationErrorsException(@"Please define an environment variable of ElasticSearchHost with the hostname of your ES server."); } _connection = new ElasticConnection(host); _serializer = new JsonNetSerializer(); }
void Log_Disabled_Events_Expected() { Settings.LoggingEnabled = false; // Dev note - this builds the index, but need to run tests again to work var connection = new ElasticConnection("localhost", 9200); for (var i = 1; i <= 100; i++) { Log.Trace("Entry Message : {0}", i); Log.Info("Entry Message : {0}", i); Log.Error("Entry Message : {0}", i); } Thread.Sleep(TimeSpan.FromSeconds(5)); var serializer = new JsonNetSerializer(); // Build the Search Query var query = new QueryBuilder<Entry>().Build(); // Execute the search string result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query); var searchResult = serializer.ToSearchResult<Entry>(result); // Check all log entries in search index Assert.AreEqual(600, searchResult.hits.total); }
private static void ListIndexAliases(ElasticConnection connection, JsonNetSerializer serializer) { /* * curl -XGET http://localhost:9200/twitter/_aliases */ string indexAliasCommand = Commands.IndexAliases("twitter") .Pretty(); var result = connection.Get(indexAliasCommand); // Parse index result. var indexAliasResult = serializer.ToIndexAliasesResult(result); PrintIndexAliasListResult(indexAliasResult, indexAliasCommand, result); }
private static long CountTweets(ElasticConnection connection, JsonNetSerializer serializer) { string countCommand = Commands.Count("twitter", "tweet").Pretty(); string query = new SingleQueryBuilder<Tweet>() .Term(t => t .Field(x => x.User) .Value("testuser") ) .BuildBeautified(); // or .Buid(); to get condensed single line query. /* or alternatively query = new TermQuery<Tweet>() .Field(x => x.User) .Value("testuser") .BuildBeautified(); // or .Buid(); to get condensed single line query. */ var results = connection.Post(countCommand, query); var searchResult = serializer.ToCountResult(results); PrintCountResults(searchResult, countCommand, query, results); return searchResult.count; }
public override void Import(DateTime @from) { var logQuery = new LogQueryClass(); var inputFormat = new COMW3CInputContextClass(); string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date, s-ip as sourceIP, cs-method as method, cs-uri-stem as uri, cs-uri-query as query, s-port as port, c-ip as clientIP, cs(User-Agent) as userAgent, cs-host as clientToServerHost, sc-status as statusCode, sc-substatus as subStatus, sc-win32-status as win32Status, sc-bytes as serverToClientBytes, cs-bytes as clientToServerBytes, time-taken as duration FROM {0}", "SOME FILE"); ILogRecordset results = logQuery.Execute(strQuery, inputFormat); var tweets = new List<Entry>(); while (!results.atEnd()) { ILogRecord logRecord = results.getRecord(); dynamic date = logRecord.getValue("date"); dynamic sourceIP = logRecord.getValue("sourceIP"); dynamic method = logRecord.getValue("method"); dynamic uri = logRecord.getValue("uri"); dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query"); dynamic port = logRecord.getValue("port"); dynamic clientIP = logRecord.getValue("clientIP") is DBNull ? string.Empty : logRecord.getValue("clientIP"); dynamic userAgent = logRecord.getValue("userAgent") is DBNull ? string.Empty : logRecord.getValue("userAgent"); dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull ? string.Empty : logRecord.getValue("clientToServerHost"); dynamic statusCode = logRecord.getValue("statusCode"); dynamic subStatus = logRecord.getValue("subStatus"); dynamic win32Status = logRecord.getValue("win32Status"); dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes"); dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes"); dynamic duration = logRecord.getValue("duration"); tweets.Add(new Entry { Date = date, SourceIP = sourceIP, Method = method, Uri = uri, Query = query, Port = port, ClientIP = clientIP, UserAgent = userAgent, ClientToServerHost = clientToServerHost, StatusCode = statusCode, SubStatus = subStatus, Win32Status = win32Status, ServerToClientBytes = serverToClientBytes, ClientToServerBytes = clientToServerBytes, Duration = duration }); results.moveNext(); } var serializer = new JsonNetSerializer(); string bulkCommand = new BulkCommand(index: "log", type: "iis"); string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Create(tweet) ); _connection.Post(bulkCommand, bulkJson); }
public void Execute() { Console.WriteLine("PLAIN SAMPLE"); var connection = new ElasticConnection("localhost", 9200); var serializer = new JsonNetSerializer(); var tweet = new Tweet { User = "******", Message = "trying out Elastic Search" }; var anotherTweet = new Tweet { User = "******", Message = "one more message" }; var tweets = new List<Tweet> { new Tweet { User = "******", Message = "first bulk tweet" }, new Tweet { User = "******", Message = "second bulk tweet" }, new Tweet { User = "******", Message = "third bulk tweet" }, }; IndexTweet(tweet, "1", connection, serializer); IndexTweet(anotherTweet, "2", connection, serializer); BulkTweetIndex(tweets, connection, serializer); CreateIndexAlias(connection, serializer); ListIndexAliases(connection, serializer); GetTweet("1", serializer, connection); SearchTweets(connection, serializer); SearchTweetsAsync(connection, serializer).Wait(); CountTweets(connection, serializer); DeleteTweeterIndex(connection, serializer); Console.WriteLine("Press any key"); Console.ReadKey(); }