public async Task GetLogsAsTextTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.None <bool>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
string logsText = Encoding.UTF8.GetString(bytes);
Assert.Equal(expectedLogText, logsText);
}
protected override async Task <Option <object> > HandleRequestInternal(Option <LogsUploadRequest> payloadOption, CancellationToken cancellationToken)
{
LogsUploadRequest payload = payloadOption.Expect(() => new ArgumentException("Request payload not found"));
async Task <IList <string> > GetModuleIds()
{
if (payload.Id == Constants.AllModulesIdentifier)
{
IEnumerable <ModuleRuntimeInfo> modules = await this.runtimeInfoProvider.GetModules(cancellationToken);
return(modules.Select(m => m.Name).ToList());
}
else
{
return(new List <string> {
payload.Id
});
}
}
var moduleLogOptions = new ModuleLogOptions(payload.Encoding, payload.ContentType, payload.Filter);
IList <string> moduleIds = await GetModuleIds();
IEnumerable <Task> uploadTasks = moduleIds.Select(m => this.UploadLogs(payload.SasUrl, m, moduleLogOptions, cancellationToken));
await Task.WhenAll(uploadTasks);
return(Option.None <object>());
}
public async Task TestLogsUploadAllTaskRequest()
{
// Arrange
string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}";
var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR"));
LogsContentEncoding contentEncoding = LogsContentEncoding.None;
LogsContentType contentType = LogsContentType.Json;
string payload = @"{""id"": ""all"", ""sasUrl"": ""<sasurl>"", ""filter"": <filter>}".Replace("<sasurl>", sasUrl).Replace("<filter>", filter.ToJson());
string mod1 = "m1";
string mod2 = "m2";
string mod3 = "m3";
var moduleRuntimeInfoList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(moduleRuntimeInfoList);
var logsUploader = new Mock <ILogsUploader>();
var logsProvider = new Mock <ILogsProvider>();
var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter);
var mod1LogBytes = new byte[100];
logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod1LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter);
var mod2LogBytes = new byte[80];
logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod2LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter);
var mod3LogBytes = new byte[120];
logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod3LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
// Act
var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider.Object);
Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.False(response.HasValue);
logsProvider.VerifyAll();
logsUploader.VerifyAll();
runtimeInfoProvider.VerifyAll();
}
public async Task GetLogsAsTextWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, ModuleLogFilter.Empty);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
string logsText = Encoding.UTF8.GetString(decompressedBytes);
Assert.Equal(expectedLogText, logsText);
}
public async Task TestLogsUploadRequest(string payload, string id, string sasUrl, LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter)
{
// Arrange
var logsUploader = new Mock <IRequestsUploader>();
var logsProvider = new Mock <ILogsProvider>();
var uploadBytes = new byte[100];
var moduleLogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
if (contentType == LogsContentType.Text)
{
Func <ArraySegment <byte>, Task> getLogsCallback = bytes => Task.CompletedTask;
logsUploader.Setup(l => l.GetLogsUploaderCallback(sasUrl, id, contentEncoding, contentType))
.ReturnsAsync(getLogsCallback);
logsProvider.Setup(l => l.GetLogsStream(id, moduleLogOptions, getLogsCallback, It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
}
else
{
logsProvider.Setup(l => l.GetLogs(id, It.IsAny <ModuleLogOptions>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(uploadBytes);
logsUploader.Setup(l => l.UploadLogs(sasUrl, id, uploadBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
}
IEnumerable <ModuleRuntimeInfo> moduleRuntimeInfoList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = Mock.Of <IRuntimeInfoProvider>(r => r.GetModules(It.IsAny <CancellationToken>()) == Task.FromResult(moduleRuntimeInfoList));
// Act
var logsUploadRequestHandler = new ModuleLogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider);
Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.True(response.HasValue);
var taskStatusResponse = response.OrDefault().FromJson <TaskStatusResponse>();
Assert.NotNull(taskStatusResponse);
Assert.NotEmpty(taskStatusResponse.CorrelationId);
Assert.Equal(string.Empty, taskStatusResponse.Message);
await WaitForBackgroundTaskCompletion(taskStatusResponse.CorrelationId).TimeoutAfter(TimeSpan.FromSeconds(5));
logsProvider.VerifyAll();
logsUploader.VerifyAll();
Mock.Get(runtimeInfoProvider).VerifyAll();
}
async Task UploadLogs(string sasUrl, string id, ModuleLogOptions moduleLogOptions, CancellationToken token)
{
if (moduleLogOptions.ContentType == LogsContentType.Json)
{
byte[] logBytes = await this.logsProvider.GetLogs(id, moduleLogOptions, token);
await this.logsUploader.Upload(sasUrl, id, logBytes, moduleLogOptions.ContentEncoding, moduleLogOptions.ContentType);
}
else if (moduleLogOptions.ContentType == LogsContentType.Text)
{
Func <ArraySegment <byte>, Task> uploaderCallback = await this.logsUploader.GetUploaderCallback(sasUrl, id, moduleLogOptions.ContentEncoding, moduleLogOptions.ContentType);
await this.logsProvider.GetLogsStream(id, moduleLogOptions, uploaderCallback, token);
}
}
public async Task GetLogsStreamWithFiltersTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.Some(10);
Option <int> since = Option.Some(1552887267);
CancellationToken cancellationToken = CancellationToken.None;
byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var filter = new ModuleLogFilter(tail, since, Option.Some(6), Option.Some("Starting"));
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter);
var receivedBytes = new List <byte>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.AddRange(bytes.ToArray());
return(Task.CompletedTask);
}
// Act
await logsProvider.GetLogsStream(moduleId, logOptions, Callback, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(3));
// Assert
Assert.NotEmpty(receivedBytes);
string receivedText = Compression.DecompressFromGzip(receivedBytes.ToArray())
.Skip(8)
.ToArray()
.FromBytes();
Assert.Equal(TestLogTexts[0], receivedText);
}
protected override async Task <Option <IEnumerable <ModuleLogsResponse> > > HandleRequestInternal(Option <ModuleLogsRequest> payloadOption, CancellationToken cancellationToken)
{
ModuleLogsRequest payload = payloadOption.Expect(() => new ArgumentException("Request payload not found"));
if (ExpectedSchemaVersion.CompareMajorVersion(payload.SchemaVersion, "logs upload request schema") != 0)
{
Events.MismatchedMinorVersions(payload.SchemaVersion, ExpectedSchemaVersion);
}
Events.ProcessingRequest(payload);
ILogsRequestToOptionsMapper requestToOptionsMapper = new LogsRequestToOptionsMapper(
this.runtimeInfoProvider,
payload.Encoding,
payload.ContentType,
LogOutputFraming.None,
Option.None <LogsOutputGroupingConfig>(),
false);
IList <(string id, ModuleLogOptions logOptions)> logOptionsList = await requestToOptionsMapper.MapToLogOptions(payload.Items, cancellationToken);
IEnumerable <Task <ModuleLogsResponse> > uploadLogsTasks = logOptionsList.Select(
async l =>
{
Events.ReceivedLogOptions(l);
ModuleLogOptions logOptions = l.logOptions.Filter.Tail
.Filter(t => t < MaxTailValue)
.Map(t => l.logOptions)
.GetOrElse(
() =>
{
var filter = new ModuleLogFilter(Option.Some(MaxTailValue), l.logOptions.Filter.Since, l.logOptions.Filter.Until, l.logOptions.Filter.LogLevel, l.logOptions.Filter.RegexString);
return(new ModuleLogOptions(l.logOptions.ContentEncoding, l.logOptions.ContentType, filter, l.logOptions.OutputFraming, l.logOptions.OutputGroupingConfig, l.logOptions.Follow));
});
byte[] moduleLogs = await this.logsProvider.GetLogs(l.id, logOptions, cancellationToken);
Events.ReceivedModuleLogs(moduleLogs, l.id);
return(logOptions.ContentEncoding == LogsContentEncoding.Gzip
? new ModuleLogsResponse(l.id, moduleLogs)
: new ModuleLogsResponse(l.id, moduleLogs.FromBytes()));
});
IEnumerable <ModuleLogsResponse> response = await Task.WhenAll(uploadLogsTasks);
return(Option.Some(response));
}
public async Task GetLogsAsJsonWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.Some(true);
CancellationToken cancellationToken = CancellationToken.None;
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
// Note: EdgeAgent automatically includes the timestamp for log parsing by default for content type JSON
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
ModuleLogFilter filter = new ModuleLogFilter(tail, since, until, Option.None <int>(), includeTimestamp, Option.None <string>());
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
var logMessages = decompressedBytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsAsJsonWithCompressionTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string expectedLogText = TestLogTexts.Join(string.Empty);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, cancellationToken))
.ReturnsAsync(new MemoryStream(GetDockerLogsStream(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(moduleId, LogsContentEncoding.Gzip, LogsContentType.Json);
// Act
byte[] bytes = await logsProvider.GetLogs(logOptions, cancellationToken);
// Assert
byte[] decompressedBytes = Compression.DecompressFromGzip(bytes);
var logMessages = decompressedBytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsAsJsonTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
string moduleId = "mod1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, false, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(DockerFraming.Frame(TestLogTexts)));
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
// Act
byte[] bytes = await logsProvider.GetLogs(moduleId, logOptions, cancellationToken);
// Assert
var logMessages = bytes.FromBytes <List <ModuleLogMessage> >();
Assert.NotNull(logMessages);
Assert.Equal(TestLogTexts.Length, logMessages.Count);
for (int i = 0; i < logMessages.Count; i++)
{
ModuleLogMessage logMessage = logMessages[i];
(int logLevel, Option <DateTime> timeStamp, string text) = LogMessageParser.ParseLogText(TestLogTexts[i]);
Assert.Equal(logLevel, logMessage.LogLevel);
Assert.Equal(timeStamp.HasValue, logMessage.TimeStamp.HasValue);
Assert.Equal(timeStamp.OrDefault(), logMessage.TimeStamp.OrDefault());
Assert.Equal(text, logMessage.Text);
Assert.Equal(iotHub, logMessage.IoTHub);
Assert.Equal(deviceId, logMessage.DeviceId);
Assert.Equal(moduleId, logMessage.ModuleId);
}
}
public async Task GetLogsStreamTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev3";
string moduleId = "mod3";
Option <int> tail = Option.None <int>();
Option <string> since = Option.None <string>();
Option <string> until = Option.None <string>();
Option <bool> includeTimestamp = Option.None <bool>();
CancellationToken cancellationToken = CancellationToken.None;
byte[] dockerLogsStreamBytes = DockerFraming.Frame(TestLogTexts);
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId, true, tail, since, until, includeTimestamp, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(new[] { new ModuleRuntimeInfo(moduleId, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()) });
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
var logOptions = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), true);
var receivedBytes = new List <byte>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.AddRange(bytes.ToArray());
return(Task.CompletedTask);
}
// Act
await logsProvider.GetLogsStream(moduleId, logOptions, Callback, cancellationToken);
// Assert
Assert.NotEmpty(receivedBytes);
Assert.Equal(string.Join(string.Empty, TestLogTexts).ToBytes(), receivedBytes);
}
public async Task TestLogsUploadRequest(string payload, string id, string sasUrl, LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter)
{
// Arrange
var logsUploader = new Mock <ILogsUploader>();
var logsProvider = new Mock <ILogsProvider>();
var uploadBytes = new byte[100];
var moduleLogOptions = new ModuleLogOptions(contentEncoding, contentType, filter);
logsProvider.Setup(l => l.GetLogs(id, moduleLogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(uploadBytes);
logsUploader.Setup(l => l.Upload(sasUrl, id, uploadBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
// Act
var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, Mock.Of <IRuntimeInfoProvider>());
Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.False(response.HasValue);
logsProvider.VerifyAll();
logsUploader.VerifyAll();
}
public static IEnumerable <object[]> GetIdsToProcessTestData()
{
var logOptions1 = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Json, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var logOptions2 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, ModuleLogFilter.Empty, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var logOptions3 = new ModuleLogOptions(LogsContentEncoding.None, LogsContentType.Text, new ModuleLogFilter(Option.Some(100), Option.None <string>(), Option.None <string>(), Option.None <int>(), Option.None <bool>(), Option.None <string>()), LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
yield return(new object[]
{
new List <(string id, ModuleLogOptions logOptions)>
{
("edgeAgent", logOptions1),
("edgeHub", logOptions2),
("tempSensor", logOptions3)
},
new List <string> {
"edgeAgent", "edgeHub", "tempSensor", "eModule2"
},
new Dictionary <string, ModuleLogOptions>
{
["edgeAgent"] = logOptions1,
["edgeHub"] = logOptions2,
["tempSensor"] = logOptions3
}
});
public async Task GetTextLogsTest()
{
var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR"));
LogsContentEncoding contentEncoding = LogsContentEncoding.None;
LogsContentType contentType = LogsContentType.Text;
string payload =
@"{
""schemaVersion"": ""1.0"",
""items"": {
""id"": ""m1"",
""filter"": <filter>
},
""encoding"": ""none"",
""contentType"": ""text""
}"
.Replace("<filter>", filter.ToJson());
string mod1 = "m1";
string mod2 = "m2";
string mod3 = "m3";
var moduleRuntimeInfoList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(moduleRuntimeInfoList);
var logsProvider = new Mock <ILogsProvider>();
var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
string mod1Logs = new[]
{
"Log line 1\n",
"Log line 2\n",
"Log line 3\n"
}.Join(string.Empty);
logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod1Logs.ToBytes());
// Act
var logsRequestHandler = new LogsRequestHandler(logsProvider.Object, runtimeInfoProvider.Object);
Option <string> response = await logsRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.True(response.HasValue);
logsProvider.VerifyAll();
runtimeInfoProvider.VerifyAll();
var logsResponseList = response.OrDefault().FromJson <List <LogsResponse> >();
Assert.NotNull(logsResponseList);
Assert.Single(logsResponseList);
LogsResponse logsResponse = logsResponseList[0];
Assert.Equal(mod1, logsResponse.Id);
Assert.True(logsResponse.Payload.HasValue);
Assert.False(logsResponse.PayloadBytes.HasValue);
Assert.Equal(mod1Logs, logsResponse.Payload.OrDefault());
}
public async Task TestLogsUploadAllTaskRequest()
{
// Arrange
string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}";
var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR"));
LogsContentEncoding contentEncoding = LogsContentEncoding.None;
LogsContentType contentType = LogsContentType.Json;
string payload =
@"{
""schemaVersion"": ""1.0"",
""sasUrl"": ""<sasurl>"",
""items"": {
""id"": "".*"",
""filter"": <filter>
},
""encoding"": ""none"",
""contentType"": ""json""
}"
.Replace("<sasurl>", sasUrl)
.Replace("<filter>", filter.ToJson());
string mod1 = "m1";
string mod2 = "m2";
string mod3 = "m3";
var moduleRuntimeInfoList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(moduleRuntimeInfoList);
var logsUploader = new Mock <ILogsUploader>();
var logsProvider = new Mock <ILogsProvider>();
var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var mod1LogBytes = new byte[100];
logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod1LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var mod2LogBytes = new byte[80];
logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod2LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var mod3LogBytes = new byte[120];
logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod3LogBytes);
logsUploader.Setup(l => l.Upload(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType))
.Returns(Task.CompletedTask);
// Act
var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider.Object);
Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.True(response.HasValue);
var taskStatusResponse = response.OrDefault().FromJson <TaskStatusResponse>();
Assert.NotNull(taskStatusResponse);
Assert.NotEmpty(taskStatusResponse.CorrelationId);
Assert.Equal(string.Empty, taskStatusResponse.Message);
await WaitForBackgroundTaskCompletion(taskStatusResponse.CorrelationId).TimeoutAfter(TimeSpan.FromSeconds(5));
logsProvider.VerifyAll();
logsUploader.VerifyAll();
runtimeInfoProvider.VerifyAll();
}
async Task UploadLogs(string sasUrl, string id, ModuleLogOptions moduleLogOptions, CancellationToken token)
{
byte[] logBytes = await this.logsProvider.GetLogs(id, moduleLogOptions, token);
await this.logsUploader.Upload(sasUrl, id, logBytes, moduleLogOptions.ContentEncoding, moduleLogOptions.ContentType);
}
public async Task GetLogsStreamWithMultipleModulesWithRegexMatchTest()
{
// Arrange
string iotHub = "foo.azure-devices.net";
string deviceId = "dev1";
Option <int> tail = Option.None <int>();
Option <int> since = Option.None <int>();
CancellationToken cancellationToken = CancellationToken.None;
string moduleId1 = "mod1";
string moduleId2 = "mod2";
var filter = new ModuleLogFilter(tail, since, Option.None <int>(), Option.Some("bad"));
byte[] dockerLogsStreamBytes1 = DockerFraming.Frame(TestLogTexts);
byte[] dockerLogsStreamBytes2 = DockerFraming.Frame(TestLogTexts);
var modulesList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(moduleId1, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(moduleId2, "docker", ModuleStatus.Running, "foo", 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId1, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes1));
runtimeInfoProvider.Setup(r => r.GetModuleLogs(moduleId2, true, tail, since, cancellationToken))
.ReturnsAsync(new MemoryStream(dockerLogsStreamBytes2));
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(modulesList);
var logsProcessor = new LogsProcessor(new LogMessageParser(iotHub, deviceId));
var logsProvider = new LogsProvider(runtimeInfoProvider.Object, logsProcessor);
string regex = "mod";
var logOptions = new ModuleLogOptions(LogsContentEncoding.Gzip, LogsContentType.Text, filter);
var receivedBytes = new List <byte[]>();
Task Callback(ArraySegment <byte> bytes)
{
receivedBytes.Add(bytes.ToArray());
return(Task.CompletedTask);
}
var expectedTextLines = new List <string> {
TestLogTexts[3], TestLogTexts[4], TestLogTexts[3], TestLogTexts[4]
};
expectedTextLines.Sort();
// Act
await logsProvider.GetLogsStream(regex, logOptions, Callback, cancellationToken);
await Task.Delay(TimeSpan.FromSeconds(6));
// Assert
Assert.NotEmpty(receivedBytes);
List <string> receivedText = receivedBytes
.Select(
r =>
Compression.DecompressFromGzip(r)
.Skip(8)
.ToArray()
.FromBytes())
.ToList();
receivedText.Sort();
Assert.Equal(expectedTextLines, receivedText);
}
public async Task GetJsonGzipLogsTest()
{
var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None <string>(), Option.Some(3), Option.None <bool>(), Option.Some("ERR"));
LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip;
LogsContentType contentType = LogsContentType.Json;
string payload =
@"{
""schemaVersion"": ""1.0"",
""items"": {
""id"": ""m1"",
""filter"": <filter>
},
""encoding"": ""gzip"",
""contentType"": ""json""
}"
.Replace("<filter>", filter.ToJson());
string iotHub = "foo.azure-devices.net";
string deviceId = "d1";
string mod1 = "m1";
string mod2 = "m2";
string mod3 = "m3";
var moduleRuntimeInfoList = new List <ModuleRuntimeInfo>
{
new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()),
new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>())
};
var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>();
runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>()))
.ReturnsAsync(moduleRuntimeInfoList);
var logsProvider = new Mock <ILogsProvider>();
var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false);
var mod1Logs = new List <ModuleLogMessage>
{
new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 1"),
new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 2"),
new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 3")
};
byte[] mod1LogBytes = Compression.CompressToGzip(mod1Logs.ToBytes());
logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>()))
.ReturnsAsync(mod1LogBytes);
// Act
var logsRequestHandler = new ModuleLogsRequestHandler(logsProvider.Object, runtimeInfoProvider.Object);
Option <string> response = await logsRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None);
// Assert
Assert.True(response.HasValue);
logsProvider.VerifyAll();
runtimeInfoProvider.VerifyAll();
var logsResponseList = response.OrDefault().FromJson <List <ModuleLogsResponse> >();
Assert.NotNull(logsResponseList);
Assert.Single(logsResponseList);
ModuleLogsResponse logsResponse = logsResponseList[0];
Assert.Equal(mod1, logsResponse.Id);
Assert.False(logsResponse.Payload.HasValue);
Assert.True(logsResponse.PayloadBytes.HasValue);
Assert.Equal(mod1LogBytes, logsResponse.PayloadBytes.OrDefault());
}
public void NeedToProcessStreamTest(ModuleLogOptions logOptions, bool expectedResult)
{
Assert.Equal(expectedResult, LogsProvider.NeedToProcessStream(logOptions));
}
