protected override void Act()
{
base.Act();
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create(eventId: 1, payloadNames: new[] { "arg1" },
payload: new object[] { "value arg1" })));
Assert.IsTrue(this.sink.FlushAsync().Wait(TimeSpan.FromSeconds(45)));
}
protected override void Act()
{
base.Act();
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 1,
payloadNames: new string[] { "arg1" },
payload: new object[] { "value arg1" })));
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 2,
payloadNames: new string[] { "arg2" },
payload: new object[] { "value arg2" })));
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 3,
payloadNames: new string[] { "arg3" },
payload: new object[] { "value arg3" })));
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 4)));
}
public void when_several_payload_values_then_takes_first_ones_as_columns()
{
int numberOfAllowedItems = 200;
var entity = new CloudEventEntry(EventEntryTestHelper.Create(
payloadNames: Enumerable.Range(0, 300).Select(i => "arg" + i),
payload: Enumerable.Range(0, 300).Select(i => (object)i)));
entity.CreateKey(true, 0);
var dict = entity.CreateTableEntity().WriteEntity(null);
for (int i = 0; i < numberOfAllowedItems; i++)
{
Assert.IsTrue(dict.ContainsKey("Payload_arg" + i), i.ToString());
Assert.AreEqual <int>(i, dict["Payload_arg" + i].Int32Value.Value);
}
for (int i = numberOfAllowedItems; i < 300; i++)
{
Assert.IsFalse(dict.ContainsKey("Payload_arg" + i), i.ToString());
}
var deserializedPayloadField = JsonConvert.DeserializeObject <Dictionary <string, object> >(dict["Payload"].StringValue);
foreach (var payloadItem in entity.Payload)
{
Assert.IsTrue(deserializedPayloadField.ContainsKey(payloadItem.Key));
Assert.AreEqual <int>((int)payloadItem.Value, (int)(long)deserializedPayloadField[payloadItem.Key]);
}
}
public void ShouldRaiseFlushFailedOnFlushAsyncWhenHttpClientFails()
{
var httpClient = HttpClientTestHelper.Create();
httpClient.When(client => client.PostAsync(Arg.Any <string>(), Arg.Any <HttpContent>())).Do(action => { throw new Exception(); });
using (var sink = new EventHubHttpSink(httpClient, "eventHubNameNs", "eventhubName", "pubId", "token", Buffering.DefaultBufferingInterval, Buffering.DefaultBufferingCount, Buffering.DefaultMaxBufferSize, TimeSpan.Zero))
using (var collectErrorsListener = new MockEventListener())
{
collectErrorsListener.EnableEvents(SemanticLoggingEventSource.Log, EventLevel.Error, Keywords.All);
sink.OnNext(EventEntryTestHelper.Create());
try
{
sink.FlushAsync().Wait();
Assert.Fail("AggregateException should be thrown.");
}
catch (AggregateException ex)
{
Assert.IsInstanceOfType(ex.InnerException, typeof(FlushFailedException));
}
Assert.IsTrue(collectErrorsListener.WrittenEntries.Any(x => x.EventId == 1));
}
}
public void RolledFileWithOverwriteWillFallBackToUniqueNameIfDateTemplateMatchesButArchiveFileIsInUse()
{
string targetArchiveFile = fileNameWithoutExtension + ".2007" + Extension;
using (FileStream stream = File.Open(targetArchiveFile, FileMode.CreateNew, FileAccess.Write, FileShare.Read))
{
using (var sink
= new RollingFlatFileSink(fileName, 0, "yyyy", RollFileExistsBehavior.Overwrite, RollInterval.Day, 0, new SimpleMessageFormatter(), false))
{
sink.RollingHelper.DateTimeProvider = dateTimeProvider;
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "1234567890"));
Assert.IsTrue(sink.RollingHelper.UpdateRollingInformationIfNecessary());
sink.RollingHelper.PerformRoll(new DateTime(2007, 01, 01));
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "12345"));
}
}
Assert.IsTrue(File.Exists(fileName));
Assert.AreEqual("12345", File.ReadAllText(fileName));
Assert.IsTrue(File.Exists(targetArchiveFile));
Assert.AreEqual(string.Empty, File.ReadAllText(targetArchiveFile)); // couldn't archive
string[] archiveFiles = Directory.GetFiles(".", targetArchiveFile + "*");
Assert.AreEqual(2, archiveFiles.Length);
foreach (string archiveFile in archiveFiles)
{
if (!Path.GetFileName(archiveFile).Equals(targetArchiveFile))
{
Assert.AreEqual("1234567890", File.ReadAllText(archiveFile));
}
}
}
[Ignore] // TODO fix race condition
public void ConcurrentAppendsEntriesToFlatFileWhenUsingAsync()
{
AppDomain.CurrentDomain.SetData("APPBASE", Environment.CurrentDirectory);
const int TimesRepeated = 50;
const int NumberOfEntries = 100;
for (int repeat = 0; repeat < TimesRepeated; repeat++)
{
this.fileName = Path.ChangeExtension(Guid.NewGuid().ToString("N"), ".log");
try
{
using (var sink = new FlatFileSink(this.fileName, new SimpleMessageFormatter(), isAsync: true))
{
Parallel.For(0, NumberOfEntries, i => sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "|" + i)));
sink.FlushAsync().Wait();
}
var entriesStr = ReadFileWithoutLock(this.fileName);
var entries = entriesStr.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual <int>(NumberOfEntries, entries.Length, this.fileName + "|" + entries.Length + " " + entriesStr);
}
finally
{
if (File.Exists(this.fileName))
{
File.Delete(this.fileName);
}
}
}
}
public async Task ShouldWritePropertiesForBatchMessage()
{
var httpClient = HttpClientTestHelper.Create();
httpClient.PostAsync(Arg.Any <string>(), Arg.Any <HttpContent>()).Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var entry = EventEntryTestHelper.Create();
using (var sink = new EventHubHttpSink(httpClient, "eventHubNameNs", "eventhubName", "pubId", "token", Buffering.DefaultBufferingInterval, Buffering.DefaultBufferingCount, Buffering.DefaultMaxBufferSize, TimeSpan.Zero))
{
sink.OnNext(entry);
sink.OnNext(entry);
await sink.FlushAsync();
}
IList <EventEntry> entries = new List <EventEntry> {
entry, entry
};
var messages = entries.Select(c => c.ToBatchMessage());
var sendMessage = new ServiceBusHttpMessage
{
Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(messages))
};
var byteRepresentation = sendMessage.Body;
await httpClient.Received().PostAsync(Arg.Any <string>(), Arg.Is <HttpContent>(c => byteRepresentation.SequenceEqual(c.ReadAsByteArrayAsync().Result)));
}
public void when_400_error_is_returned_then_batch_fails_and_logs_exception_without_timeout()
{
var mockHttpListener = new MockHttpListener();
using (var collectErrorsListener = new MockEventListener())
{
collectErrorsListener.EnableEvents(SemanticLoggingEventSource.Log, EventLevel.Error, Keywords.All);
var endpoint = mockHttpListener.Start(new MockHttpListenerResponse()
{
ResponseCode = 400,
ContentType = "application/json",
Content = "{ \"error\": \"InvalidIndexNameException[[log,stash] Invalid index name [log,stash], must not contain the following characters [\\\\, /, *, ?, \\\", <, >, |, , ,]]\",\"status\": 400}"
});
var sink = new ElasticsearchSink("instance", endpoint, "slabtest", "etw", true, TimeSpan.FromSeconds(1), 100, 800, TimeSpan.FromMinutes(1));
sink.OnNext(EventEntryTestHelper.Create());
var flushCompleteInTime = sink.FlushAsync().Wait(TimeSpan.FromSeconds(45));
mockHttpListener.Stop();
// Make sure the exception is logged
Assert.IsTrue(collectErrorsListener.WrittenEntries.First().Payload.Single(m => m.ToString().Contains("InvalidIndexNameException")) != null);
Assert.IsTrue(flushCompleteInTime);
}
}
public void RolledFileWithIncrementWillCreateArchiveFileWithMaxSequenceIfDateTemplateDoesMatch()
{
using (var sink
= new RollingFlatFileSink(fileName, 0, "yyyy", RollFileExistsBehavior.Increment, RollInterval.Day, 0, new SimpleMessageFormatter(), false))
{
sink.RollingHelper.DateTimeProvider = dateTimeProvider;
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "1234567890"));
Assert.IsTrue(sink.RollingHelper.UpdateRollingInformationIfNecessary());
sink.RollingHelper.PerformRoll(new DateTime(2007, 01, 01));
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "12345"));
Assert.IsTrue(sink.RollingHelper.UpdateRollingInformationIfNecessary());
sink.RollingHelper.PerformRoll(new DateTime(2007, 01, 02));
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "abcde"));
}
Assert.IsTrue(File.Exists(fileName));
Assert.AreEqual("abcde", File.ReadAllText(fileName));
Assert.IsTrue(File.Exists(fileNameWithoutExtension + ".2007.2" + Extension));
Assert.AreEqual("12345", File.ReadAllText(fileNameWithoutExtension + ".2007.2" + Extension));
Assert.IsTrue(File.Exists(fileNameWithoutExtension + ".2007.1" + Extension));
Assert.AreEqual("1234567890", File.ReadAllText(fileNameWithoutExtension + ".2007.1" + Extension));
string[] archiveFiles = Directory.GetFiles(".", fileNameWithoutExtension + ".2007*" + Extension + "*");
Assert.AreEqual(2, archiveFiles.Length);
}
protected override void Act()
{
base.Act();
this.entry = new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 12,
providerId: Guid.NewGuid(),
providerName: "Provider Name",
timestamp: new DateTimeOffset(2013, 4, 10, 16, 0, 0, TimeSpan.Zero),
keywords: (EventKeywords)16L,
level: EventLevel.Informational,
formattedMessage: "My message",
opcode: (EventOpcode)4,
task: (EventTask)24,
version: 2,
payloadNames: new string[] { "arg1" },
payload: new object[] { "value arg1" },
processId: 200,
threadId: 300,
activityId: Guid.Parse("{562D0422-F427-4849-A6CD-7990A46F1223}"),
relatedActivityId: Guid.Parse("{23408E19-3133-47E1-9307-C99A4F9AC8CC}")))
{
InstanceName = "Instance Name"
};
sink.OnNext(this.entry);
}
public void ShouldRaiseFlushFailedOnFlushAsyncWhenEventHubClientFails()
{
var eventHubClient = Substitute.For <IEventHubClient>();
using (var sink = new EventHubAmqpSink(eventHubClient, Buffering.DefaultBufferingInterval, Buffering.DefaultBufferingCount, Buffering.DefaultMaxBufferSize, TimeSpan.Zero))
using (var collectErrorsListener = new MockEventListener())
{
collectErrorsListener.EnableEvents(SemanticLoggingEventSource.Log, EventLevel.Error, Keywords.All);
eventHubClient.When(client => client.SendBatchAsync(Arg.Any <IEnumerable <EventData> >()))
.Do(action => { throw new Exception(); });
sink.OnNext(EventEntryTestHelper.Create());
try
{
sink.FlushAsync().Wait();
Assert.Fail("AggregateException should be thrown.");
}
catch (AggregateException ex)
{
Assert.IsInstanceOfType(ex.InnerException, typeof(FlushFailedException));
}
Assert.IsTrue(collectErrorsListener.WrittenEntries.Any(x => x.EventId == 1));
}
}
public void when_having_big_message_value_then_truncates()
{
var entity = new CloudEventEntry(EventEntryTestHelper.Create(formattedMessage: new string('a', 500000)));
entity.CreateKey(true, 0);
var dict = entity.CreateTableEntity().WriteEntity(null);
Assert.AreEqual(new string('a', 30000) + "--TRUNCATED--", dict["Message"].StringValue);
}
protected override void Act()
{
base.Act();
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 10,
payloadNames: new[] { "arg1", "arg2", "arg3" },
payload: new object[] { MyLongEnum.Value1, MyIntEnum.Value2, MyShortEnum.Value3 })));
}
public void when_having_big_payload_then_truncates()
{
var entity = new CloudEventEntry(EventEntryTestHelper.Create(payloadNames: new string[] { "arg1" }, payload: new object[] { new string('a', 500000) }));
entity.CreateKey(true, 0);
var dict = entity.CreateTableEntity().WriteEntity(null);
StringAssert.Contains(dict["Payload"].StringValue, "'payload_serialization_error'");
Assert.IsFalse(dict.ContainsKey("Payload_arg1"));
}
public void when_writing_entity_adds_payload_to_dictionary()
{
var entity = new CloudEventEntry(EventEntryTestHelper.Create(payloadNames: new string[] { "message1", "message2" }, payload: new object[] { "value1", "value2" }));
entity.CreateKey(true, 0);
var dict = entity.CreateTableEntity().WriteEntity(null);
Assert.IsTrue(dict.ContainsKey("Payload_message1"));
Assert.IsTrue(dict.ContainsKey("Payload_message2"));
}
public void CreatesFlatFile()
{
sink = new FlatFileSink(this.fileName, new SimpleMessageFormatter(), false);
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "|1"));
Assert.IsTrue(File.Exists(this.fileName));
var entries = ReadFileWithoutLock(this.fileName).Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual <int>(1, entries.Count());
}
public void ConcurrentAppendsEntriesToRollingFlatFile()
{
const int NumberOfEntries = 300;
using (var sink = new RollingFlatFileSink(fileName, 0, "yyyy", RollFileExistsBehavior.Increment, RollInterval.Day, 0, new SimpleMessageFormatter(), false))
{
Parallel.For(0, NumberOfEntries, i => sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "Info-" + i + ":")));
}
Assert.AreEqual <int>(NumberOfEntries, File.ReadAllText(fileName).Split(new string[] { ":" }, StringSplitOptions.RemoveEmptyEntries).Length);
}
public void when_generating_key_then_prefixes_with_instance_name()
{
var entity = new CloudEventEntry(EventEntryTestHelper.Create(
timestamp: DateTimeOffset.UtcNow))
{
InstanceName = "MyInstanceName"
};
entity.CreateKey(false, 0);
StringAssert.StartsWith(entity.RowKey, "MyInstanceName");
}
public void when_having_big_overall_payloads_then_stores_warning_and_does_not_contain_payload()
{
var entity = new CloudEventEntry(EventEntryTestHelper.Create(payloadNames: Enumerable.Range(0, 50).Select(i => "arg" + i), payload: Enumerable.Range(0, 50).Select(i => new string('a', 2000))));
entity.CreateKey(true, 0);
var dict = entity.CreateTableEntity().WriteEntity(null);
StringAssert.Contains(dict["Payload"].StringValue, "'payload_serialization_error'");
Assert.AreEqual(0, dict.Keys.Count(x => x.StartsWith("Payload_")));
}
private static EventEntry CreateEventEntry()
{
var payload = new Dictionary <string, object> {
{ "msg", "the message" }, { "date", DateTime.UtcNow }
};
var logObject =
EventEntryTestHelper.Create(
timestamp: DateTimeOffset.UtcNow,
payloadNames: payload.Keys,
payload: payload.Values);
return(logObject);
}
protected EventEntry CreateEventEntry(string msgPropertyValue)
{
var payload = new Dictionary <string, object> {
{ "msg", msgPropertyValue }, { "date", DateTime.UtcNow }
};
var logObject =
EventEntryTestHelper.Create(
timestamp: DateTimeOffset.UtcNow,
payloadNames: payload.Keys,
payload: payload.Values);
return(logObject);
}
protected override void Act()
{
base.Act();
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 10,
opcode: EventOpcode.Reply,
level: EventLevel.Informational,
version: 2,
payloadNames: new string[] { "arg1", "arg2", "arg3" },
payload: new object[] { 1, "2", true })));
}
protected override void Act()
{
base.Act();
for (int i = 0; i < NumberOfEntries; i++)
{
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 10,
payloadNames: new string[] { "arg" },
payload: new object[] { i })));
}
}
public void AppendsEntriesToFlatFile()
{
sink = new FlatFileSink(this.fileName, new SimpleMessageFormatter(), false);
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "|1"));
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "|2"));
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "|3"));
var entries = ReadFileWithoutLock(this.fileName).Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
Assert.AreEqual <int>(3, entries.Length);
Assert.AreEqual("1", entries[0]);
Assert.AreEqual("2", entries[1]);
Assert.AreEqual("3", entries[2]);
}
public void WriterKeepsTally()
{
using (var sink
= new RollingFlatFileSink(fileName, 10, "yyyy", RollFileExistsBehavior.Overwrite, RollInterval.Day, 0, new SimpleMessageFormatter(), false))
{
sink.RollingHelper.DateTimeProvider = this.dateTimeProvider;
Assert.IsTrue(sink.RollingHelper.UpdateRollingInformationIfNecessary());
sink.OnNext(EventEntryTestHelper.Create(formattedMessage: "12345"));
Assert.AreEqual(5L, sink.Tally);
}
}
protected override void Act()
{
base.Act();
for (int i = 0; i < 100; i++)
{
sink.OnNext(
new CloudEventEntry(EventEntryTestHelper.Create(
eventId: 30,
formattedMessage: new string('b', 30000),
payloadNames: new string[] { "Medium" },
payload: new object[] { new string('a', 20000) })));
}
}
protected override void Act()
{
base.Act();
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create(eventId: 10,
payloadNames: Enumerable.Range(0, 500).Select(x => "arg" + x),
payload: Enumerable.Range(0, 500).Select(x => (object)x))));
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create(eventId: 20, payloadNames: new string[] { "Large" },
payload: new object[] { new string('a', 500000) })));
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create(eventId: 30, formattedMessage: new string('b', 500000))));
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create(eventId: 40,
payloadNames: Enumerable.Range(0, 50).Select(x => "arg" + x),
payload: Enumerable.Range(0, 50).Select(x => (object)new string('c', 1000)))));
}
public async Task ShouldWriteEntriesOnFlushAsync()
{
var eventHubClient = Substitute.For <IEventHubClient>();
using (var sink = new EventHubAmqpSink(eventHubClient, Buffering.DefaultBufferingInterval, Buffering.DefaultBufferingCount, Buffering.DefaultMaxBufferSize, TimeSpan.Zero))
{
sink.OnNext(EventEntryTestHelper.Create());
sink.OnNext(EventEntryTestHelper.Create());
await sink.FlushAsync();
}
eventHubClient.Received().SendBatchAsync(Arg.Is <IEnumerable <EventData> >(l => l.Count() == 2)).Wait();
}
// Ignoring because the storage client library considers inexistant account as transient so it retries and the exponential back-off could take time
public void when_cannot_connect_to_storage_account_then_on_completed_should_not_stall_or_throw()
{
const string ValidNotExisting = "DefaultEndpointsProtocol=https;AccountName=InexistantDoesntReallyMatter;AccountKey=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa==";
using (var sink = new WindowsAzureTableSink("instanceName", ValidNotExisting, "Table", TimeSpan.FromSeconds(1), 5000, TimeSpan.FromSeconds(20)))
using (var collectErrorsListener = new MockEventListener())
{
collectErrorsListener.EnableEvents(SemanticLoggingEventSource.Log, EventLevel.Error, Keywords.All);
sink.OnNext(new CloudEventEntry(EventEntryTestHelper.Create()));
Assert.IsTrue(Task.Run(() => sink.OnCompleted()).Wait(TimeSpan.FromSeconds(15)));
Assert.IsTrue(collectErrorsListener.WrittenEntries.Any(x => x.EventId == 500));
}
}
public void when_cannot_connect_to_database_then_on_completed_should_not_stall_or_throw()
{
const string ValidNotExisting = @"Data Source=(localdb)\v11.0; AttachDBFilename='|DataDirectory|\DoesNotExist.mdf';Initial Catalog=SemanticLoggingTests;Integrated Security=True";
using (var sink = new SqlDatabaseSink("test", ValidNotExisting, "tableName", Buffering.DefaultBufferingInterval, Buffering.DefaultBufferingCount, Buffering.DefaultMaxBufferSize, TimeSpan.FromSeconds(20)))
using (var collectErrorsListener = new MockEventListener())
{
collectErrorsListener.EnableEvents(SemanticLoggingEventSource.Log, EventLevel.Error, Keywords.All);
var closure = sink;
closure.OnNext(EventEntryTestHelper.Create());
Assert.IsTrue(Task.Run(() => closure.OnCompleted()).Wait(TimeSpan.FromSeconds(5)));
Assert.IsTrue(collectErrorsListener.WrittenEntries.Any(x => x.EventId == 101));
}
}