public void InitialiseLogStream_Verify_CreateLogStream()
{
//arrange
var container = new AutoMoqer();
var amazonclientMock = container.GetMock <IAmazonCloudWatchLogs>();
amazonclientMock.SetupSequence(x => x.DescribeLogStreams(It.IsAny <DescribeLogStreamsRequest>()))
.Returns(new DescribeLogStreamsResponse
{
LogStreams = new List <LogStream>()
})
.Returns(new DescribeLogStreamsResponse
{
LogStreams = new List <LogStream>
{
new LogStream
{
LogStreamName = Logstream
}
}
});
var target = new CloudWatchLogsClientWrapper(amazonclientMock.Object, LogGroup, Logstream);
//act
target.InitialiseLogStream();
//assert
amazonclientMock.Verify(x => x.CreateLogStream(It.IsAny <CreateLogStreamRequest>()));
}
public void AddLogRequest_InitialiseLogStream_ShouldBeCalledINoLogStream()
{
//arrange
var cloudwatchClient = InitialiseClient();
var logstream = Logstream + Guid.NewGuid();
var cloudwatvhLogClient = new CloudWatchLogsClientWrapper(
cloudwatchClient,
LogGroup,
logstream);
var logevents = Builder <InputLogEvent>
.CreateListOfSize(20)
.All()
.With(x => x.Timestamp = DateTime.Now.AddDays(-1))
.Build()
.ToList();
//act
try
{
cloudwatvhLogClient.AddLogRequest(logevents);
}
catch (ApplicationException ex)
{
if (ex.Message == "LogStream doesn't exist")
{
//assert
Assert.Pass();
}
}
}
public async Task WriteAsync_Should_Retry_Failed_Request_3_Times()
{
// arrange
int actualRetries = 0, expectedRetries = 3;
var clientMock = new Mock <IAmazonCloudWatchLogs>().Init();
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
actualRetries++;
return(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = actualRetries <= (expectedRetries - 1) ? HttpStatusCode.BadGateway : HttpStatusCode.OK
}));
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(_logGroup, _logStream, CreateIntervalProvider())
);
// act
await target.WriteAsync(CreateEvents());
// assert
Assert.Equal(expectedRetries, actualRetries);
}
public void AddLogRequest_AllShouldbeCommited()
{
//arrange
var cloudwatchClient = InitialiseClient();
var logstream = Logstream + Guid.NewGuid();
var cloudwatvhLogClient = new CloudWatchLogsClientWrapper(
cloudwatchClient,
LogGroup,
logstream);
cloudwatvhLogClient.InitialiseLogStream();
var logevents = Builder <InputLogEvent>
.CreateListOfSize(20)
.All()
.With(x => x.Timestamp = DateTime.Now.AddDays(-1))
.Build()
.ToList();
//act
cloudwatvhLogClient.AddLogRequest(logevents);
Thread.Sleep(2000);
var results = cloudwatchClient.GetLogEvents(new GetLogEventsRequest(
LogGroup,
logstream));
//assert
results.Events.Count.Should().Equal(20);
results.Events.All(x => string.IsNullOrEmpty(x.Message));
results.Events.All(x => x.Timestamp != new DateTime());
}
public async Task WriteAsync_Should_Queue_Log_Requests_And_Preserve_Their_Order()
{
// arrange
var rand = new Random();
var clientMock = new Mock <IAmazonCloudWatchLogs>().Init();
var expectedSequence = Enumerable.Range(0, 20);
List <int> actualSequence = new List <int>(), retried = new List <int>();
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
return(Task
.Delay(rand.Next() % 500) // Simulates delay on the API side
.ContinueWith(t =>
{
PutLogEventsResponse result;
var messageInt = int.Parse(r.LogEvents.First().Message);
var shouldRetry = (messageInt % 5 == 0) && !retried.Contains(messageInt); // retry every fifth element once.
if (shouldRetry)
{
retried.Add(messageInt);
result = new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.BadRequest
};
}
else
{
actualSequence.Add(messageInt);
result = new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK
};
}
return result;
}));
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(_logGroup, _logStream, CreateIntervalProvider())
);
// act
await Task.WhenAll(expectedSequence
.Select(i => target.WriteAsync(new[] { new InputLogEvent {
Message = i.ToString()
} }))
.ToArray());
// assert
Assert.True(expectedSequence.SequenceEqual(actualSequence), "Message sequence should be preserved.");
Assert.True(expectedSequence.Where(i => i % 5 == 0).SequenceEqual(retried), "Expected and actual retries should match.");
}
public override void ActivateOptions()
{
base.ActivateOptions();
_client = new CloudWatchLogsClientWrapper(EndPoint, AccessKey, Secret, ClientConfig);
_eventProcessor = new LogEventProcessor(_configOverrides, GroupName, StreamName, Timestamp, Message);
if (Layout == null)
{
Layout = new PatternLayout("%message");
}
}
public async Task WriteAsync_Should_Reinit_Sequence_Token_If_All_Retries_Fail()
{
// arrange
string logGroup = Guid.NewGuid().ToString(), logStream = Guid.NewGuid().ToString();
var clientMock = new Mock <IAmazonCloudWatchLogs>().InitGroup().InitCreateStream();
int token = 1;
string successfullToken = null;
clientMock
.Setup(m => m.DescribeLogStreamsAsync(It.IsAny <DescribeLogStreamsRequest>(), It.IsAny <CancellationToken>()))
.Returns <DescribeLogStreamsRequest, CancellationToken>((r, c) => Task.FromResult(
new DescribeLogStreamsResponse
{
HttpStatusCode = HttpStatusCode.OK,
LogStreams = new List <LogStream> {
new LogStream {
LogStreamName = r.LogStreamNamePrefix, UploadSequenceToken = token++.ToString()
}
}
}
));
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
if (r.SequenceToken == "1")
{
throw new InvalidSequenceTokenException("Invalid token.");
}
successfullToken = r.SequenceToken;
return(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK
}));
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, CreateIntervalProvider())
);
// act
await Assert.ThrowsAsync <InvalidSequenceTokenException>(() => target.WriteAsync(CreateEvents())); // first time should fail.
await target.WriteAsync(CreateEvents()); // second time should be successful.
// assert
Assert.Equal("2", successfullToken);
}
public void AddLogRequest_Verify()
{
//arrange
var container = new AutoMoqer();
var amazonclientMock = container.GetMock <IAmazonCloudWatchLogs>();
amazonclientMock.Setup(
x => x.DescribeLogStreams(
It.IsAny <DescribeLogStreamsRequest>()
)
).Returns(new DescribeLogStreamsResponse
{
LogStreams = new List <LogStream>
{
new LogStream
{
LogStreamName = Logstream
}
}
});
amazonclientMock.Setup(x => x.PutLogEvents(It.IsAny <PutLogEventsRequest>()))
.Returns(new PutLogEventsResponse());
var target = new CloudWatchLogsClientWrapper(amazonclientMock.Object, LogGroup, Logstream);
var logEvents = new List <InputLogEvent>()
{
new InputLogEvent
{
Message = "TestMessage",
Timestamp = DateTime.UtcNow
}
};
//act
target.AddLogRequest(logEvents);
//assert
amazonclientMock.Verify(x => x.PutLogEvents(
It.Is <PutLogEventsRequest>(y =>
y.LogGroupName == LogGroup &&
y.LogStreamName == Logstream &&
y.LogEvents.Single().Message == logEvents.Single().Message&&
y.LogEvents.Single().Timestamp == logEvents.Single().Timestamp
))
, Times.Once()
);
}
public void InitialiseLogStream_Test()
{
//arrange
var cloudwatchClient = InitialiseClient();
var cloudwatvchLogClient = new CloudWatchLogsClientWrapper(cloudwatchClient, LogGroup, Logstream);
cloudwatvchLogClient.InitialiseLogStream();
//act
var results = cloudwatchClient.DescribeLogStreams(new DescribeLogStreamsRequest(LogGroup));
//assert
results.LogStreams.Count().Should().Equal(1);
results.LogStreams.Any(x => x.LogStreamName == Logstream).Should().Be.True();
}
public override void ActivateOptions()
{
base.ActivateOptions();
EventMessageParser = EventMessageParser ?? new LogsEventMessageParser(_configOverrides);
_client = new CloudWatchLogsClientWrapper(EndPoint, AccessKey, Secret, ClientConfig);
_eventProcessor = new LogEventProcessor(_groupName, _streamName, _timestamp, _message)
{
EventMessageParser = EventMessageParser
};
if (Layout == null)
{
Layout = new PatternLayout("%message");
}
}
public async Task Init_Should_Retry_Requests_Resulting_In_Amazon_Exceptions()
{
// arrange
int actualRetries = 0, expectedRetries = 3;
string logGroup = Guid.NewGuid().ToString(), logStream = Guid.NewGuid().ToString();
var clientMock = new Mock <IAmazonCloudWatchLogs>().InitStream();
clientMock
.Setup(m => m.DescribeLogGroupsAsync(It.IsAny <DescribeLogGroupsRequest>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(new DescribeLogGroupsResponse {
HttpStatusCode = HttpStatusCode.OK
}));
clientMock
.Setup(m => m.CreateLogGroupAsync(It.IsAny <CreateLogGroupRequest>(), It.IsAny <CancellationToken>()))
.Returns <CreateLogGroupRequest, CancellationToken>((r, c) =>
{
actualRetries++;
if (actualRetries <= (expectedRetries - 1))
{
throw new OperationAbortedException("something happened.");
}
return(Task.FromResult(new CreateLogGroupResponse {
HttpStatusCode = HttpStatusCode.OK
}));
});
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK
}));
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, CreateIntervalProvider())
);
// act
await target.WriteAsync(CreateEvents());
// assert
Assert.Equal(expectedRetries, actualRetries);
}
public async Task WriteAsync_Should_Throw_InvalidSequenceTokenException()
{
// arrange
var clientMock = new Mock <IAmazonCloudWatchLogs>().Init();
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
throw new InvalidSequenceTokenException("invalid token");
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(_logGroup, _logStream, CreateIntervalProvider())
);
// act
await Assert.ThrowsAsync <InvalidSequenceTokenException>(() => target.WriteAsync(CreateEvents()));
}
public async Task Init_Should_Retry_Failed_Requests()
{
// arrange
int actualRetries = 0, expectedRetries = 3;
string logGroup = Guid.NewGuid().ToString(), logStream = Guid.NewGuid().ToString();
var clientMock = new Mock <IAmazonCloudWatchLogs>().InitCreateGroup().InitStream();
clientMock
.Setup(m => m.DescribeLogGroupsAsync(It.IsAny <DescribeLogGroupsRequest>(), It.IsAny <CancellationToken>()))
.Returns <DescribeLogGroupsRequest, CancellationToken>((r, c) =>
{
actualRetries++;
bool success = actualRetries > (expectedRetries - 1);
return(Task.FromResult(new DescribeLogGroupsResponse {
HttpStatusCode = success ? HttpStatusCode.OK : HttpStatusCode.BadGateway,
LogGroups = success ? new List <LogGroup> {
new LogGroup {
LogGroupName = r.LogGroupNamePrefix
}
} : null
}));
});
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK
}));
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, CreateIntervalProvider())
);
// act
await target.WriteAsync(CreateEvents());
// assert
Assert.Equal(expectedRetries, actualRetries);
}
public async Task WriteAsync_Should_Throw_AWSFailedRequestException()
{
// arrange
var clientMock = new Mock <IAmazonCloudWatchLogs>().Init();
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
return(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.BadGateway
}));
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(_logGroup, _logStream, CreateIntervalProvider())
);
// act
await Assert.ThrowsAsync <AWSFailedRequestException>(() => target.WriteAsync(CreateEvents()));
}
public async Task WriteAsync_Should_Order_Log_Events_Chronologically_Before_Sending()
{
// arange
var clientMock = new Mock <IAmazonCloudWatchLogs>().Init();
var events = new[]
{
new InputLogEvent {
Timestamp = DateTime.UtcNow
},
new InputLogEvent {
Timestamp = DateTime.UtcNow.AddSeconds(-1)
}
};
List <InputLogEvent> actual = null, expected = events.OrderBy(e => e.Timestamp).ToList();
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
actual = r.LogEvents;
return(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK
}));
});
var target = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(_logGroup, _logStream, CreateIntervalProvider())
);
// act
await target.WriteAsync(events);
// assert
Assert.True(expected.SequenceEqual(actual), "Actual events sequence should be ordered chronologically.");
}
protected override void ResetClient()
{
_client = null;
}
public async Task WriteAsync_Should_Handle_Concurent_Requests_From_Multiple_Target_Insances()
{
// arange
string logGroup = Guid.NewGuid().ToString(), logStream = Guid.NewGuid().ToString();
var clientMock = new Mock <IAmazonCloudWatchLogs>().InitGroup().InitCreateStream();
var tokens = new List <int>();
clientMock
.Setup(m => m.DescribeLogStreamsAsync(It.IsAny <DescribeLogStreamsRequest>(), It.IsAny <CancellationToken>()))
.Returns((DescribeLogStreamsRequest req, CancellationToken token) => Task.FromResult(new DescribeLogStreamsResponse
{
HttpStatusCode = HttpStatusCode.OK,
LogStreams = new List <LogStream> {
new LogStream {
LogStreamName = req.LogStreamNamePrefix, UploadSequenceToken = "1"
}
}
}));
clientMock
.Setup(m => m.PutLogEventsAsync(It.IsAny <PutLogEventsRequest>(), It.IsAny <CancellationToken>()))
.Returns <PutLogEventsRequest, CancellationToken>((r, c) =>
{
int tokenInt = int.Parse(r.SequenceToken);
lock (tokens)
{
if (tokens.Any(t => t == tokenInt))
{
throw new InvalidSequenceTokenException("Token already used.");
}
tokens.Add(tokenInt);
}
return(Task.FromResult(new PutLogEventsResponse {
HttpStatusCode = HttpStatusCode.OK, NextSequenceToken = (tokenInt + 1).ToString()
}));
});
var intervalProvider = CreateIntervalProvider();
var target1 = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, intervalProvider)
);
var target2 = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, intervalProvider)
);
var target3 = new CloudWatchLogsClientWrapper(
clientMock.Object,
new CloudWatchLogsWrapperSettings(logGroup, logStream, intervalProvider)
);
// act
await Task.WhenAll(new []
{
target1.WriteAsync(CreateEvents()),
target2.WriteAsync(CreateEvents()),
target3.WriteAsync(CreateEvents())
});
// assert
var actual = tokens.OrderBy(i => i);
var expected = Enumerable.Range(1, 3);
Assert.True(expected.SequenceEqual(actual), $"Expected token sequence: {String.Join(",", expected)}, actual: {String.Join(",", actual)}");
}
