public async Task GetLogsAsTextWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, ModuleLogFilter.Empty);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
string logsText = Encoding.UTF8.GetString(decompressedBytes);
Assert.Equal(expectedLogText, logsText);
}
public async Task GetLogsAsTextTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.None <bool>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
string logsText = Encoding.UTF8.GetString(bytes);
Assert.Equal(expectedLogText, logsText);
}
public async Task HandleWithCancellationTest()
{
// Arrange
var buffer = DockerFraming.Frame(TestLogTexts);
string id = "m1";
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None <int>(), Option.None <string>(), Option.None <string>(), Option.None <bool>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new MemoryStream(buffer));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProcessor = new LogsProcessor(new LogMessageParser("testIotHub", "d1"));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logRequestItem = new LogRequestItem(id, ModuleLogFilter.Empty);
var logsStreamRequest = new LogsStreamRequest("1.0", new List <LogRequestItem> {
logRequestItem
}, LogsContentEncoding.None, LogsContentType.Text);
byte[] logsStreamRequestBytes = logsStreamRequest.ToBytes();
var logsStreamRequestArraySeg = new ArraySegment <byte>(logsStreamRequestBytes);
var clientWebSocket = new Mock <IClientWebSocket>();
clientWebSocket.Setup(c => c.ReceiveAsync(It.IsAny <ArraySegment <byte> >(), It.IsAny <CancellationToken>()))
.Callback <ArraySegment <byte>, CancellationToken>((a, c) => logsStreamRequestArraySeg.CopyTo(a))
.ReturnsAsync(new WebSocketReceiveResult(logsStreamRequestBytes.Length, WebSocketMessageType.Binary, true));
var receivedBytes = new List <byte>();
clientWebSocket.Setup(c => c.SendAsync(It.IsAny <ArraySegment <byte> >(), WebSocketMessageType.Binary, true, It.IsAny <CancellationToken>()))
.Callback <ArraySegment <byte>, WebSocketMessageType, bool, CancellationToken>((a, w, f, c) => receivedBytes.AddRange(a.Array))
.Returns(async() => await Task.Delay(TimeSpan.FromSeconds(3)));
clientWebSocket.SetupGet(c => c.State).Returns(WebSocketState.Open);
// Act
var logsStreamRequestHandler = new LogsStreamRequestHandler(logsProvider, runtimeInfoProvider.Object);
Task handleTask = logsStreamRequestHandler.Handle(clientWebSocket.Object, CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(10));
clientWebSocket.SetupGet(c => c.State).Returns(WebSocketState.Closed);
// Assert
await Task.Delay(TimeSpan.FromSeconds(5));
Assert.True(handleTask.IsCompleted);
runtimeInfoProvider.VerifyAll();
clientWebSocket.VerifyAll();
Assert.True(receivedBytes.Count < buffer.Length);
IList <string> receivedChunks = SimpleFraming.Parse(receivedBytes.ToArray())
.Select(r => Encoding.UTF8.GetString(r))
.ToList();
Assert.Equal(TestLogTexts.Take(receivedChunks.Count), receivedChunks);
}
public async Task HandleTest()
{
// Arrange
var random = new Random();
var buffer = new byte[1024 * 128];
random.NextBytes(buffer);
string id = "m1";
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(id, true, Option.None <int>(), Option.None <int>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new MemoryStream(buffer));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, Mock.Of <ILogsProcessor>());
var logRequestItem = new LogRequestItem(id, ModuleLogFilter.Empty);
var logsStreamRequest = new LogsStreamRequest("1.0", new List <LogRequestItem> {
logRequestItem
}, LogsContentEncoding.None, LogsContentType.Text);
byte[] logsStreamRequestBytes = logsStreamRequest.ToBytes();
var logsStreamRequestArraySeg = new ArraySegment <byte>(logsStreamRequestBytes);
var clientWebSocket = new Mock <IClientWebSocket>();
clientWebSocket.Setup(c => c.ReceiveAsync(It.IsAny <ArraySegment <byte> >(), It.IsAny <CancellationToken>()))
.Callback <ArraySegment <byte>, CancellationToken>((a, c) => logsStreamRequestArraySeg.CopyTo(a))
.Returns(
async() =>
{
await Task.Yield();
return(new WebSocketReceiveResult(logsStreamRequestBytes.Length, WebSocketMessageType.Binary, true));
});
var receivedBytes = new List <byte>();
clientWebSocket.Setup(c => c.SendAsync(It.IsAny <ArraySegment <byte> >(), WebSocketMessageType.Binary, true, It.IsAny <CancellationToken>()))
.Callback <ArraySegment <byte>, WebSocketMessageType, bool, CancellationToken>((a, w, f, c) => receivedBytes.AddRange(a.Array))
.Returns(async() => await Task.Yield());
clientWebSocket.SetupGet(c => c.State).Returns(WebSocketState.Open);
// Act
var logsStreamRequestHandler = new LogsStreamRequestHandler(logsProvider);
await logsStreamRequestHandler.Handle(clientWebSocket.Object, CancellationToken.None);
// Assert
runtimeInfoProvider.VerifyAll();
clientWebSocket.VerifyAll();
Assert.Equal(buffer, receivedBytes.ToArray());
}
public async Task GetLogsStreamWithFiltersTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.Some(10);
Option <int> since = Option.Some(1552887267);
CancellationToken cancellationToken = CancellationToken.None;
byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var filter = new ModuleLogFilter(tail, since, Option.Some(6), Option.Some("Starting"));
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter);
var receivedBytes = new List <byte>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.AddRange(bytes.ToArray());
return(Task.CompletedTask);
}
// Act
await logsProvider.GetLogsStream(moduleId, logOptions, Callback, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(3));
// Assert
Assert.NotEmpty(receivedBytes);
string receivedText = Compression.DecompressFromGzip(receivedBytes.ToArray())
.Skip(8)
.ToArray()
.FromBytes();
Assert.Equal(TestLogTexts[0], receivedText);
}
public async Task GetLogsAsJsonWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.Some(true);
CancellationToken cancellationToken = CancellationToken.None;
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
// Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None <int>(), includeTimestamp, Option.None <string>());
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
var logMessages = decompressedBytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsAsJsonWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, cancellationToken))
.ReturnsAsync(new MemoryStream(GetDockerLogsStream(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(moduleId, LogsContentEncoding.Gzip, LogsContentType.Json);
// Act
byte[] bytes = await logsProvider.GetLogs(logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
var logMessages = decompressedBytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsAsJsonTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
var logMessages = bytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsStreamTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev3";
string moduleId = "mod3";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.None <bool>();
CancellationToken cancellationToken = CancellationToken.None;
byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), true);
var receivedBytes = new List <byte>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.AddRange(bytes.ToArray());
return(Task.CompletedTask);
}
// Act
await logsProvider.GetLogsStream(moduleId, logOptions, Callback, cancellationToken);
// Assert
Assert.NotEmpty(receivedBytes);
Assert.Equal(string.Join(string.Empty, TestLogTexts).ToBytes(), receivedBytes);
}
public void GetMatchingIdsTest(string regex, IList <string> moduleIds, IList <string> expectedList)
{
ISet <string> actualModules = LogsProvider.GetMatchingIds(regex, moduleIds);
Assert.Equal(expectedList.OrderBy(i => i), actualModules.OrderBy(i => i));
}
public void NeedToProcessStreamTest(ModuleLogOptions logOptions, bool expectedResult)
{
Assert.Equal(expectedResult, LogsProvider.NeedToProcessStream(logOptions));
}
public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string moduleId1 = "mod1";
string moduleId2 = "mod2";
var filter = new ModuleLogFilter(tail, since, Option.None <int>(), Option.Some("bad"));
byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts);
byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts);
var modulesList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(moduleId1, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(moduleId2, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1));
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(modulesList);
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
string regex = "mod";
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter);
var receivedBytes = new List <byte[]>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.Add(bytes.ToArray());
return(Task.CompletedTask);
}
var expectedTextLines = new List <string> {
TestLogTexts[3], TestLogTexts[4], TestLogTexts[3], TestLogTexts[4]
};
expectedTextLines.Sort();
// Act
await logsProvider.GetLogsStream(regex, logOptions, Callback, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(6));
// Assert
Assert.NotEmpty(receivedBytes);
List <string> receivedText = receivedBytes
.Select(
r =>
Compression.DecompressFromGzip(r)
.Skip(8)
.ToArray()
.FromBytes())
.ToList();
receivedText.Sort();
Assert.Equal(expectedTextLines, receivedText);
}
public LogsController(LogsProvider logsProvider)
{
LogsProvider = logsProvider;
}
public LogsProviderTests()
{
_sut = new LogsProvider();
}