public void JsonSerialization_ExpectedLogStashFormat()
{
var request = mocks.StrictMock<HttpRequestBase>();
var timestamp = DateTime.Parse("2013-07-18T11:13:32.3149976+01:00");
using (mocks.Record())
{
SetupResult.For(request.UserHostAddress).Return("http://192.168.0.1");
SetupResult.For(request.HttpMethod).Return("GET");
SetupResult.For(request.UserAgent).Return("Chrome");
SetupResult.For(request.Url).Return(new Uri("http://192.168.0.2/Some/Path/?query=somewhere"));
}
using (mocks.Playback())
{
target = new Entry(request, timestamp)
{
Message = "My Message",
Severity = "Trace"
};
var serializer = new JsonNetSerializer();
var result = serializer.Serialize(target);
var expected = "{\"@message\":\"My Message\",\"@type\":\"Trace\",\"@timestamp\":\"2013-07-18T11:13:32.3149976+01:00\",\"@source_host\":\"http://192.168.0.1\",\"@source_path\":\"GET /Some/Path/?query=somewhere\",\"@fields\":{\"request\":\"GET /Some/Path/?query=somewhere\",\"user-agent\":\"Chrome\"}}";
Assert.AreEqual(expected, result);
}
}
public void Execute()
{
Console.WriteLine("PLAIN SAMPLE");
var connection = new ElasticConnection("localhost", 9200);
var serializer = new JsonNetSerializer();
var tweet = new Tweet
{
User = "******",
Message = "trying out Elastic Search"
};
var anotherTweet = new Tweet
{
User = "******",
Message = "one more message"
};
IndexTweet(tweet, "1", connection, serializer);
IndexTweet(anotherTweet, "2", connection, serializer);
GetTweet("1", serializer, connection);
SearchTweets(connection, serializer);
DeleteTweeterIndex(connection, serializer);
Console.WriteLine("Press any key");
Console.ReadKey();
}
private static void IndexTweet(Tweet tweet, string id, ElasticConnection connection, JsonNetSerializer serializer)
{
/*
$ curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty=true' -d '{
"User" : "testUser",
"Message" : "trying out Elastic Search"
}'
*/
// This is url that will be requested from ES. We can grab it and put to any ES admin console (like ElasticHead) to debug ES behavior.
string indexCommand = Commands.Index(index: "twitter", type: "tweet", id: id)
.Refresh(true)
.Pretty(); // this will generate: twitter/tweet/1?pretty=true
// This variable contains JSON of serialized tweet, thus we can check if our object serialized correctly
// or use it directly in ES admin console.
string tweetJson = serializer.ToJson(tweet);
var result = connection.Put(indexCommand, tweetJson);
// Parse index result.
IndexResult indexResult = serializer.ToIndexResult(result);
PrintIndexCommand(result, indexResult, indexCommand, tweetJson);
}
void Log_Events_Expected()
{
// Dev note - this builds the index, but need to run tests again to work
var connection = new ElasticConnection("localhost", 9200);
if (IsIndexExists(ElasticSearchStorage.GetIndex(), connection))
{
connection.Delete(new DeleteCommand(ElasticSearchStorage.GetIndex()));
}
for (var i = 1; i <= 100; i++)
{
Log.Trace("Entry Message : {0}", i);
Log.Info("Entry Message : {0}", i);
Log.Error("Entry Message : {0}", i);
}
Thread.Sleep(TimeSpan.FromSeconds(5));
var serializer = new JsonNetSerializer();
// Build the Search Query
var query = new QueryBuilder<Entry>().Build();
// Execute the search
string result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query);
var searchResult = serializer.ToSearchResult<Entry>(result);
// Check all log entries in search index
Assert.AreEqual(300, searchResult.hits.total);
// Secondary check, use the instance based loger
using (var log = Log.GetLogger())
{
for (var i = 1; i <= 100; i++)
{
log.Trace("Entry Message : {0}", i);
log.Info("Entry Message : {0}", i);
log.Error("Entry Message : {0}", i);
}
Thread.Sleep(TimeSpan.FromSeconds(5));
// Build the Search Query
query = new QueryBuilder<Entry>().Build();
// Execute the search
result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query);
searchResult = serializer.ToSearchResult<Entry>(result);
// Check all log entries in search index
Assert.AreEqual(600, searchResult.hits.total);
}
}
private static void CreateIndexAlias(ElasticConnection connection, JsonNetSerializer serializer)
{
/*
* curl -XPUT http://localhost:9200/twitter/_alias/twitter_alias
*/
string indexAliasCommand = Commands.IndexAlias("twitter", "twitter_alias")
.Pretty();
var result = connection.Put(indexAliasCommand);
// Parse index result.
var indexAliasResult = serializer.ToCommandResult(result);
PrintIndexAliasResult(indexAliasResult, indexAliasCommand, result);
}
private static IEnumerable<Tweet> SearchTweets(ElasticConnection connection, JsonNetSerializer serializer)
{
string searchCommand = Commands.Search("twitter", "tweet").Pretty();
/*
{
"query": {
"term": {
"User": {
"value": "testuser",
"boost": "5"
}
}
}
}
*/
string query = new QueryBuilder<Tweet>()
.Query(qry => qry
.Term(term => term
.Field(tweet => tweet.User)
.Value("testUser".ToLower()) // by default terms query requires lowercased values.
.Boost(5)
)
// Alternate way
//.Custom(" 'term': {{ '{0}': {{ 'value': '{1}', 'boost': '5' }} }}".AltQuote(), "User", "testuser")
).BuildBeautified();
var results = connection.Post(searchCommand, query);
var searchResult = serializer.ToSearchResult<Tweet>(results);
PrintSearchResults(searchResult, searchCommand, query, results);
return searchResult.Documents;
}
private static void DeleteTweeterIndex(ElasticConnection connection, JsonNetSerializer serializer)
{
/*
$ curl -XDELETE 'http://localhost:9200/twitter?pretty=true'
*/
string deleteCommand = Commands.Delete(index: "twitter").Pretty();
var result = connection.Delete(deleteCommand);
DeleteResult deleteResult = serializer.ToDeleteResult(result);
PrintDeleteCommand(deleteCommand, deleteResult, result);
}
private static Tweet GetTweet(string id, JsonNetSerializer serializer, ElasticConnection connection)
{
/*
$ curl -XGET 'http://localhost:9200/twitter/tweet/1?pretty=true'
*/
String getCommand = Commands.Get(index: "twitter", type: "tweet", id: id).Pretty(); // this will generate: twitter/tweet/1?pretty=true
var result = connection.Get(getCommand);
// Deserialize Get command result to GetResult object.
var getResult = serializer.ToGetResult<Tweet>(result);
var getTweet = getResult.Document;
PrintGetCommand(getTweet, result, getCommand);
return getTweet;
}
private static void BulkTweetIndex(IEnumerable<Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer)
{
string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh();
int id = 10; // start adding tweets from id = 10
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Index(data: tweet, id: (id++).AsString())
);
string result = connection.Post(bulkCommand, bulkJson);
//Parse bulk result;
BulkResult bulkResult = serializer.ToBulkResult(result);
PrintBulkCommand(bulkCommand, bulkJson, bulkResult);
}
public static void BulkType(List <Bic_Iban_Codes> ModelData, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index)
{
string bulkCommand = new BulkCommand(index: _index, type: "iban_bic").Refresh();
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(ModelData, (builder, pro) => builder.Index(data: pro, id: pro.CodeID.ToString())
);
string result = connection.Post(bulkCommand, bulkJson);
BulkResult bulkResult = serializer.ToBulkResult(result);
connection.Post(_index + "/_refresh");
}
public static void BulkType <T>(List <T> Model, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index, string _type) where T : IElasticMapper
{
string bulkCommand = "";
bulkCommand = new BulkCommand(index: _index, type: _type).Refresh();
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(Model, (builder, pro) => builder.Index(data: pro, id: pro.id)
);
string result = connection.Post(bulkCommand, bulkJson);
BulkResult bulkResult = serializer.ToBulkResult(result);
connection.Post(_index + "/_refresh");
}
private static Task<IEnumerable<Tweet>> SearchTweetsAsync(ElasticConnection connection, JsonNetSerializer serializer)
{
string searchCommand = Commands.Search("twitter", "tweet").Pretty();
string query = new QueryBuilder<Tweet>()
.Query(qry => qry
.Term(term => term
.Field(tweet => tweet.User)
.Value("testUser".ToLower()) // by default terms query requires lowercased values.
.Boost(5)
)
).BuildBeautified();
return connection.PostAsync(searchCommand, query)
// process search results asynchronously
.ContinueWith( searchTask => {
OperationResult results = searchTask.Result;
var searchResult = serializer.ToSearchResult<Tweet>(results);
Console.WriteLine("ASYNC Search Results: \r\n");
PrintSearchResults(searchResult, searchCommand, query, results);
return searchResult.Documents;
});
}
public ElasticSearchDbLogger(ITime time)
{
_time = time;
var host = Environment.GetEnvironmentVariable("ElasticSearchHost");
if (string.IsNullOrEmpty(host))
{
throw new ConfigurationErrorsException(@"Please define an environment variable of ElasticSearchHost with the hostname of your ES server.");
}
_connection = new ElasticConnection(host);
_serializer = new JsonNetSerializer();
}
void Log_Disabled_Events_Expected()
{
Settings.LoggingEnabled = false;
// Dev note - this builds the index, but need to run tests again to work
var connection = new ElasticConnection("localhost", 9200);
for (var i = 1; i <= 100; i++)
{
Log.Trace("Entry Message : {0}", i);
Log.Info("Entry Message : {0}", i);
Log.Error("Entry Message : {0}", i);
}
Thread.Sleep(TimeSpan.FromSeconds(5));
var serializer = new JsonNetSerializer();
// Build the Search Query
var query = new QueryBuilder<Entry>().Build();
// Execute the search
string result = connection.Post(Commands.Search(ElasticSearchStorage.GetIndex(), "Log"), query);
var searchResult = serializer.ToSearchResult<Entry>(result);
// Check all log entries in search index
Assert.AreEqual(600, searchResult.hits.total);
}
private static void ListIndexAliases(ElasticConnection connection, JsonNetSerializer serializer)
{
/*
* curl -XGET http://localhost:9200/twitter/_aliases
*/
string indexAliasCommand = Commands.IndexAliases("twitter")
.Pretty();
var result = connection.Get(indexAliasCommand);
// Parse index result.
var indexAliasResult = serializer.ToIndexAliasesResult(result);
PrintIndexAliasListResult(indexAliasResult, indexAliasCommand, result);
}
private static long CountTweets(ElasticConnection connection, JsonNetSerializer serializer)
{
string countCommand = Commands.Count("twitter", "tweet").Pretty();
string query = new SingleQueryBuilder<Tweet>()
.Term(t => t
.Field(x => x.User)
.Value("testuser")
)
.BuildBeautified(); // or .Buid(); to get condensed single line query.
/* or alternatively
query = new TermQuery<Tweet>()
.Field(x => x.User)
.Value("testuser")
.BuildBeautified(); // or .Buid(); to get condensed single line query.
*/
var results = connection.Post(countCommand, query);
var searchResult = serializer.ToCountResult(results);
PrintCountResults(searchResult, countCommand, query, results);
return searchResult.count;
}
public override void Import(DateTime @from)
{
var logQuery = new LogQueryClass();
var inputFormat = new COMW3CInputContextClass();
string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date,
s-ip as sourceIP,
cs-method as method,
cs-uri-stem as uri,
cs-uri-query as query,
s-port as port,
c-ip as clientIP,
cs(User-Agent) as userAgent,
cs-host as clientToServerHost,
sc-status as statusCode,
sc-substatus as subStatus,
sc-win32-status as win32Status,
sc-bytes as serverToClientBytes,
cs-bytes as clientToServerBytes,
time-taken as duration FROM {0}", "SOME FILE");
ILogRecordset results = logQuery.Execute(strQuery, inputFormat);
var tweets = new List<Entry>();
while (!results.atEnd())
{
ILogRecord logRecord = results.getRecord();
dynamic date = logRecord.getValue("date");
dynamic sourceIP = logRecord.getValue("sourceIP");
dynamic method = logRecord.getValue("method");
dynamic uri = logRecord.getValue("uri");
dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query");
dynamic port = logRecord.getValue("port");
dynamic clientIP = logRecord.getValue("clientIP") is DBNull
? string.Empty
: logRecord.getValue("clientIP");
dynamic userAgent = logRecord.getValue("userAgent") is DBNull
? string.Empty
: logRecord.getValue("userAgent");
dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull
? string.Empty
: logRecord.getValue("clientToServerHost");
dynamic statusCode = logRecord.getValue("statusCode");
dynamic subStatus = logRecord.getValue("subStatus");
dynamic win32Status = logRecord.getValue("win32Status");
dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes");
dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes");
dynamic duration = logRecord.getValue("duration");
tweets.Add(new Entry
{
Date = date,
SourceIP = sourceIP,
Method = method,
Uri = uri,
Query = query,
Port = port,
ClientIP = clientIP,
UserAgent = userAgent,
ClientToServerHost = clientToServerHost,
StatusCode = statusCode,
SubStatus = subStatus,
Win32Status = win32Status,
ServerToClientBytes = serverToClientBytes,
ClientToServerBytes = clientToServerBytes,
Duration = duration
});
results.moveNext();
}
var serializer = new JsonNetSerializer();
string bulkCommand = new BulkCommand(index: "log", type: "iis");
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Create(tweet)
);
_connection.Post(bulkCommand, bulkJson);
}
public void Execute()
{
Console.WriteLine("PLAIN SAMPLE");
var connection = new ElasticConnection("localhost", 9200);
var serializer = new JsonNetSerializer();
var tweet = new Tweet
{
User = "******",
Message = "trying out Elastic Search"
};
var anotherTweet = new Tweet
{
User = "******",
Message = "one more message"
};
var tweets = new List<Tweet> {
new Tweet
{
User = "******",
Message = "first bulk tweet"
},
new Tweet
{
User = "******",
Message = "second bulk tweet"
},
new Tweet
{
User = "******",
Message = "third bulk tweet"
},
};
IndexTweet(tweet, "1", connection, serializer);
IndexTweet(anotherTweet, "2", connection, serializer);
BulkTweetIndex(tweets, connection, serializer);
CreateIndexAlias(connection, serializer);
ListIndexAliases(connection, serializer);
GetTweet("1", serializer, connection);
SearchTweets(connection, serializer);
SearchTweetsAsync(connection, serializer).Wait();
CountTweets(connection, serializer);
DeleteTweeterIndex(connection, serializer);
Console.WriteLine("Press any key");
Console.ReadKey();
}
