public void parse_json_string_as_object2(bool flatten)
{
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new JsonFilter()
{
FlattenJson = flatten
});
appender.ActivateOptions();
});
string json =
"{\"InnerMessage\":\"Starting.\", \"Data\":{\"Type\":\"Url\", \"Host\":\"localhost\", \"Array\":[\"One\", \"Two\"]}}";
log4net.LogicalThreadContext.Properties["JsonRaw"] = json;
_log.Info("Info");
Client.Refresh(TestIndex);
var res = Client.Search <JObject>(s => s.AllIndices().Type("LogEvent").Take(1));
var doc = res.Documents.First();
QueryConfiguration(appender =>
{
appender.ElasticFilters = oldFilters;
appender.ActivateOptions();
});
JToken actualObj;
string innerMessage;
string dataType;
string dataHost;
string dataArrayFirst;
if (flatten)
{
actualObj = doc;
innerMessage = actualObj["InnerMessage"].ToString();
dataType = actualObj["Data.Type"].ToString();
dataHost = actualObj["Data.Host"].ToString();
dataArrayFirst = actualObj["Data.Array.0"].ToString();
}
else
{
actualObj = doc["JsonRaw"];
innerMessage = actualObj["InnerMessage"].ToString();
dataType = actualObj["Data"]["Type"].ToString();
dataHost = actualObj["Data"]["Host"].ToString();
dataArrayFirst = actualObj["Data"]["Array"][0].ToString();
}
Assert.IsNotNull(actualObj);
Assert.AreEqual("Starting.", innerMessage);
Assert.AreEqual("Url", dataType);
Assert.AreEqual("localhost", dataHost);
Assert.AreEqual("One", dataArrayFirst);
}
public void Can_read_KvFilter_properties(string[] fieldSplit, string[] valueSplit, string trim,
bool expectAnotherToBeNull)
{
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new KvFilter()
{
FieldSplit = string.Join("", fieldSplit),
ValueSplit = string.Join("", valueSplit),
TrimKey = trim,
TrimValue = trim
});
appender.ActivateOptions();
});
_log.InfoFormat(
"this is message{1}key{0}value{1}another {0} 'another'{1}object{0}[this is object :)]",
valueSplit[0].TrimStart('\\'), fieldSplit[0].TrimStart('\\'));
Client.Refresh(TestIndex);
var searchResults = Client.Search <dynamic>(s => s.AllIndices().Type("LogEvent").Take(1));
var entry = searchResults.Documents.First();
QueryConfiguration(appender =>
{
appender.ElasticFilters = oldFilters;
appender.ActivateOptions();
});
Assert.IsNotNull(entry.key);
Assert.AreEqual("value", entry.key.ToString());
Assert.IsNotNull(entry["object"]);
Assert.AreEqual("this is object :)", entry["object"].ToString());
if (entry.another == null)
{
Assert.IsTrue(expectAnotherToBeNull, "only on the spaces issues test this 'another' should be null");
}
else
{
Assert.AreEqual("another", entry.another.ToString());
}
}
public void Setup()
{
_elasticClientFactory = Substitute.For <IElasticClientFactory>();
_elasticClient = Substitute.For <IElasticsearchClient>();
_timer = Substitute.For <IIndexingTimer>();
_tolerateCallsFactory = Substitute.For <ITolerateCallsFactory>();
_bulk = Substitute.For <ILogBulkSet>();
_logEventConverterFactory = Substitute.For <ILogEventConverterFactory>();
_logEventConverter = Substitute.For <ILogEventConverter>();
_elasticFilters = new ElasticAppenderFilters();
_fileAccessor = Substitute.For <IFileAccessor>();
_eventWriter = Substitute.For <IExternalEventWriter>();
_elasticClientFactory.CreateClient(null, 0, false, false, null).ReturnsForAnyArgs(_elasticClient);
_logEventConverterFactory.Create(FixFlags.All, false).ReturnsForAnyArgs(_logEventConverter);
}
public void test_ttl(string ttlValue, int expectation)
{
const string ttlTemplateName = "ttltemplate";
const int toWaitMillisec = 3000;
Client.PutIndexTemplate(ttlTemplateName,
descriptor =>
descriptor.Template("*")
.Settings(settings => settings.Setting("indices.ttl.interval", "1s").Setting("index.ttl.interval", "1s"))
.Mappings(mapping => mapping.Map("_default_", desc => desc.TtlField(ttlField => ttlField.Enable().Default("1d")))));
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(
appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new AddValueFilter()
{
Key = "_ttl", Value = ttlValue
});
});
_log.Info("test");
Client.Refresh(TestIndex);
var res = Client.Search <dynamic>(s => s.AllIndices().Type("LogEvent"));
Assert.AreEqual(1, res.Total);
// "Magic". The time of deletion isn't consistent :/
int numOfTries = 20;
while (--numOfTries > 0)
{
Client.Refresh(TestIndex);
Client.Optimize(TestIndex);
res = Client.Search <dynamic>(s => s.AllTypes().AllIndices());
numOfTries = res.Total == expectation ? 0 : numOfTries;
Thread.Sleep(toWaitMillisec);
}
res = Client.Search <dynamic>(s => s.AllIndices().Type("LogEvent"));
Client.DeleteIndexTemplate(ttlTemplateName);
QueryConfiguration(appender => appender.ElasticFilters = oldFilters);
Assert.AreEqual(expectation, res.Total);
}
public ElasticSearchAppender()
{
FixedFields = FixFlags.Partial;
BulkSize = 2000;
BulkIdleTimeout = 5000;
TimeoutToWaitForTimer = 5000;
Server = "localhost";
Port = 9200;
IndexName = "LogEvent-%{+yyyy.MM.dd}";
IndexType = "LogEvent";
IndexAsync = true;
MaxAsyncConnections = 10;
Template = null;
_timer = new Timer(TimerElapsed, null, Timeout.Infinite, Timeout.Infinite);
ElasticFilters = new ElasticAppenderFilters();
}
public static void Performance()
{
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
appender.BulkSize = 4000;
appender.BulkIdleTimeout = -1;
oldFilters = appender.ElasticFilters;
//appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new GrokFilter()
{
Pattern = "testNum: {INT:testNum}, name is {WORD:name} and guid {UUID:guid}"
});
});
PerformanceTests.Test(1, 32000);
QueryConfiguration(appender => appender.ElasticFilters = oldFilters);
}
public void can_parse_log4net_context_stacks()
{
const string sourceKey = "UserName";
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
var convert = new ConvertFilter();
convert.AddToString(sourceKey);
var toArray = new ConvertToArrayFilter {
SourceKey = sourceKey
};
convert.AddToArray(toArray);
appender.ElasticFilters.AddConvert(convert);
});
LogicalThreadContext.Stacks[sourceKey].Push("name1");
LogicalThreadContext.Stacks[sourceKey].Push("name2");
_log.Info("hi");
Client.Refresh(TestIndex);
var res = Client.Search <JObject>(s => s.AllIndices().Type("LogEvent").Take(1));
var doc = res.Documents.First();
var usrName = doc[sourceKey];
Assert.NotNull(usrName);
Assert.AreEqual("name1", usrName[0].Value <string>());
Assert.AreEqual("name2", usrName[1].Value <string>());
QueryConfiguration(appender =>
{
appender.ElasticFilters = oldFilters;
});
}
public void parse_json_string_as_object()
{
const string sourceKey = "jsonObject";
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new JsonFilter()
{
SourceKey = sourceKey
});
});
var jObject = new JObject {
{ "key", "value\r\nnewline" }, { "arr", new JArray(Enumerable.Range(0, 5)) }
};
LogicalThreadContext.Properties[sourceKey] = jObject.ToString();
_log.Info("logging jsonObject");
Client.Refresh(TestIndex);
var res = Client.Search <JObject>(s => s.AllIndices().Type("LogEvent").Take(1));
var doc = res.Documents.First();
var jsonObject = doc[sourceKey];
QueryConfiguration(appender => appender.ElasticFilters = oldFilters);
Assert.NotNull(jsonObject);
Assert.AreEqual(jsonObject["key"].Value <string>(), "value\r\nnewline");
var arr = jsonObject["arr"].ToArray();
foreach (var i in Enumerable.Range(0, 5))
{
Assert.AreEqual(arr[i].Value <int>(), i);
}
}
public void parse_xml_string_as_object(bool flatten)
{
const string separator = "_";
const string sourceKey = "xmlObject";
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new XmlFilter {
SourceKey = sourceKey, FlattenXml = flatten, Separator = separator
});
});
var xmlDoc = new XmlDocument();
var parentNode = xmlDoc.CreateElement("Parent");
var parentAttribute = xmlDoc.CreateAttribute("key");
parentAttribute.Value = "value\r\nnewline";
parentNode.Attributes.Append(parentAttribute);
xmlDoc.AppendChild(parentNode);
foreach (var i in Enumerable.Range(0, 5))
{
var childNode = xmlDoc.CreateElement("Child");
var childAttribute = xmlDoc.CreateAttribute("id");
childAttribute.Value = i.ToString();
childNode.Attributes.Append(childAttribute);
parentNode.AppendChild(childNode);
}
var xmlString = ConvertXmlToString(xmlDoc);
LogicalThreadContext.Properties[sourceKey] = xmlString;
_log.Info("logging xmlObject");
Client.Refresh(TestIndex);
var res = Client.Search <JObject>(s => s.AllIndices().Type("LogEvent").Take(1));
var doc = res.Documents.First();
QueryConfiguration(appender =>
{
appender.ElasticFilters = oldFilters;
});
if (flatten)
{
Assert.NotNull(doc);
Assert.AreEqual(doc["Parent" + separator + "@key"].ToString(), "value\r\nnewline");
Assert.AreEqual(doc["Parent" + separator + "Child" + separator + "0" + separator + "@id"].ToString(), "0");
Assert.AreEqual(doc["Parent" + separator + "Child" + separator + "1" + separator + "@id"].ToString(), "1");
}
else
{
var jsonObject = doc[sourceKey];
Assert.NotNull(jsonObject);
Assert.AreEqual(jsonObject["Parent"]["@key"].Value <string>(), "value\r\nnewline");
var arr = jsonObject["Parent"]["Child"].ToArray();
foreach (var i in Enumerable.Range(0, 5))
{
Assert.AreEqual(arr[i]["@id"].Value <int>(), i);
}
}
}
public void parse_json_string_as_object(bool flatten, string separator = "_")
{
const string sourceKey = "jsonObject";
ElasticAppenderFilters oldFilters = null;
QueryConfiguration(appender =>
{
oldFilters = appender.ElasticFilters;
appender.ElasticFilters = new ElasticAppenderFilters();
appender.ElasticFilters.AddFilter(new JsonFilter()
{
FlattenJson = flatten, Separator = separator, SourceKey = sourceKey
});
});
var jObject = new JObject
{
{ "key", "value\r\nnewline" },
{ "Data", new JObject {
{ "Type", "Url" }, { "Host", "localhost" }, { "Array", new JArray(Enumerable.Range(0, 5)) }
} }
};
log4net.LogicalThreadContext.Properties[sourceKey] = jObject.ToString();
_log.Info("logging jsonObject");
Client.Refresh(TestIndex);
var res = Client.Search <JObject>(s => s.AllIndices().Type("LogEvent").Take(1));
var doc = res.Documents.First();
QueryConfiguration(appender =>
{
appender.ElasticFilters = oldFilters;
});
JToken actualObj;
string key;
string dataType;
string dataHost;
string dataArrayFirst;
string dataArrayLast;
if (flatten)
{
actualObj = doc;
key = actualObj["key"].ToString();
dataType = actualObj["Data" + separator + "Type"].ToString();
dataHost = actualObj["Data" + separator + "Host"].ToString();
dataArrayFirst = actualObj["Data" + separator + "Array" + separator + "0"].ToString();
dataArrayLast = actualObj["Data" + separator + "Array" + separator + "4"].ToString();
}
else
{
actualObj = doc[sourceKey];
key = actualObj["key"].ToString();
dataType = actualObj["Data"]["Type"].ToString();
dataHost = actualObj["Data"]["Host"].ToString();
dataArrayFirst = actualObj["Data"]["Array"][0].ToString();
dataArrayLast = actualObj["Data"]["Array"][4].ToString();
}
Assert.IsNotNull(actualObj);
Assert.AreEqual("value\r\nnewline", key);
Assert.AreEqual("Url", dataType);
Assert.AreEqual("localhost", dataHost);
Assert.AreEqual("0", dataArrayFirst);
Assert.AreEqual("4", dataArrayLast);
}
