internal string GetBlobName(string id, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) { string extension = GetExtension(logsContentEncoding, logsContentType); string blobName = $"{id}-{DateTime.UtcNow.ToString("yyyy-MM-dd--HH-mm-ss", CultureInfo.InvariantCulture)}"; return($"{this.iotHubName}/{this.deviceId}/{blobName}.{extension}"); }
public LogsStreamRequest(string schemaVersion, List <LogRequestItem> items, LogsContentEncoding encoding, LogsContentType contentType) { this.SchemaVersion = schemaVersion; this.Items = Preconditions.CheckNotNull(items, nameof(items)); this.Encoding = encoding; this.ContentType = contentType; }
public async Task Upload(string uri, string id, byte[] payload, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) { Preconditions.CheckNonWhiteSpace(uri, nameof(uri)); Preconditions.CheckNonWhiteSpace(id, nameof(id)); Preconditions.CheckNotNull(payload, nameof(payload)); try { var containerUri = new Uri(uri); string blobName = this.GetBlobName(id, logsContentEncoding, logsContentType); var container = new CloudBlobContainer(containerUri); Events.Uploading(blobName, container.Name); await ExecuteWithRetry( () => { IAzureBlob blob = this.azureBlobUploader.GetBlob(containerUri, blobName); SetContentEncoding(blob, logsContentEncoding); SetContentType(blob, logsContentType); return(blob.UploadFromByteArrayAsync(payload)); }, r => Events.UploadErrorRetrying(blobName, container.Name, r)); Events.UploadSuccess(blobName, container.Name); } catch (Exception e) { Events.UploadError(e, id); throw; } }
public async Task TestLogsUploadAllTaskRequest() { // Arrange string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; string payload = @"{""id"": ""all"", ""sasUrl"": ""<sasurl>"", ""filter"": <filter>}".Replace("<sasurl>", sasUrl).Replace("<filter>", filter.ToJson()); string mod1 = "m1"; string mod2 = "m2"; string mod3 = "m3"; var moduleRuntimeInfoList = new List <ModuleRuntimeInfo> { new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()) }; var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>(); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>())) .ReturnsAsync(moduleRuntimeInfoList); var logsUploader = new Mock <ILogsUploader>(); var logsProvider = new Mock <ILogsProvider>(); var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter); var mod1LogBytes = new byte[100]; logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod1LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter); var mod2LogBytes = new byte[80]; logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod2LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter); var mod3LogBytes = new byte[120]; logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod3LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); // Act var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider.Object); Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.False(response.HasValue); logsProvider.VerifyAll(); logsUploader.VerifyAll(); runtimeInfoProvider.VerifyAll(); }
public ModuleLogsUploadRequest( string schemaVersion, List <LogRequestItem> items, LogsContentEncoding encoding, LogsContentType contentType, string sasUrl) : this(schemaVersion, items, encoding, contentType, sasUrl, new NestedEdgeParentUriParser()) { }
static void SetContentEncoding(IAzureBlob blob, LogsContentEncoding logsContentEncoding) { switch (logsContentEncoding) { case LogsContentEncoding.Gzip: blob.BlobProperties.ContentEncoding = "gzip"; break; } }
static Option <string> GetContentEncoding(LogsContentEncoding logsContentEncoding) { switch (logsContentEncoding) { case LogsContentEncoding.Gzip: return(Option.Some("gzip")); default: return(Option.None <string>()); } }
internal static string GetLogsExtension(LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) { if (logsContentEncoding == LogsContentEncoding.Gzip) { return(logsContentType == LogsContentType.Json ? "json.gz" : "log.gz"); } else { return(logsContentType == LogsContentType.Json ? "json" : "log"); } }
public ModuleLogsRequest( string schemaVersion, List <LogRequestItem> items, LogsContentEncoding encoding, LogsContentType contentType) { this.SchemaVersion = Preconditions.CheckNonWhiteSpace(schemaVersion, nameof(schemaVersion)); this.Encoding = encoding; this.ContentType = contentType; this.Items = Preconditions.CheckNotNull(items, nameof(items)); }
public LogsUploadRequest( string schemaVersion, List <LogRequestItem> items, LogsContentEncoding encoding, LogsContentType contentType, string sasUrl) { this.SchemaVersion = Preconditions.CheckNonWhiteSpace(schemaVersion, nameof(schemaVersion)); this.Encoding = encoding; this.ContentType = contentType; this.SasUrl = Preconditions.CheckNotNull(sasUrl, nameof(sasUrl)); this.Items = Preconditions.CheckNotNull(items, nameof(items)); }
public LogsUploadRequest( string id, LogsContentEncoding encoding, LogsContentType contentType, string sasUrl, ModuleLogFilter filter) { this.Id = Preconditions.CheckNonWhiteSpace(id, nameof(id)); this.Encoding = encoding; this.ContentType = contentType; this.SasUrl = sasUrl; this.Filter = filter ?? ModuleLogFilter.Empty; }
internal static string GetExtension(LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) { if (logsContentEncoding == LogsContentEncoding.Gzip) { return("gz"); } if (logsContentType == LogsContentType.Json) { return("json"); } return("log"); }
public LogsRequestToOptionsMapper( IRuntimeInfoProvider runtimeInfoProvider, LogsContentEncoding contentEncoding, LogsContentType contentType, LogOutputFraming outputFraming, Option <LogsOutputGroupingConfig> outputGroupingConfig, bool follow) { this.runtimeInfoProvider = Preconditions.CheckNotNull(runtimeInfoProvider, nameof(runtimeInfoProvider)); this.contentType = contentType; this.contentEncoding = contentEncoding; this.outputFraming = outputFraming; this.outputGroupingConfig = outputGroupingConfig; this.follow = follow; }
public ModuleLogOptions( LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter, LogOutputFraming outputFraming, Option <LogsOutputGroupingConfig> outputGroupingConfig, bool follow) { this.ContentEncoding = contentEncoding; this.ContentType = contentType; this.Filter = Preconditions.CheckNotNull(filter, nameof(filter)); this.OutputFraming = outputFraming; this.OutputGroupingConfig = outputGroupingConfig; this.Follow = follow; }
public ModuleLogsUploadRequest( string schemaVersion, List <LogRequestItem> items, LogsContentEncoding encoding, LogsContentType contentType, string sasUrl, INestedEdgeParentUriParser parser) { this.SchemaVersion = Preconditions.CheckNonWhiteSpace(schemaVersion, nameof(schemaVersion)); this.Encoding = encoding; this.ContentType = contentType; this.Items = Preconditions.CheckNotNull(items, nameof(items)); this.SasUrl = Preconditions.CheckNotNull(sasUrl, nameof(sasUrl)); Option <string> url = parser.ParseURI(this.SasUrl); this.SasUrl = url.GetOrElse(this.SasUrl); }
static byte[] ProcessByContentEncoding(byte[] bytes, LogsContentEncoding contentEncoding) => contentEncoding == LogsContentEncoding.Gzip ? Compression.CompressToGzip(bytes) : bytes;
public ModuleLogOptions(LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter) { this.ContentEncoding = contentEncoding; this.ContentType = contentType; this.Filter = Preconditions.CheckNotNull(filter, nameof(filter)); }
public void GetExtensionTest(LogsContentEncoding contentEncoding, LogsContentType contentType, string expectedExtension) { Assert.Equal(expectedExtension, AzureBlobRequestsUploader.GetLogsExtension(contentEncoding, contentType)); }
public async Task GetTextLogsTest() { var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Text; string payload = @"{ ""schemaVersion"": ""1.0"", ""items"": { ""id"": ""m1"", ""filter"": <filter> }, ""encoding"": ""none"", ""contentType"": ""text"" }" .Replace("<filter>", filter.ToJson()); string mod1 = "m1"; string mod2 = "m2"; string mod3 = "m3"; var moduleRuntimeInfoList = new List <ModuleRuntimeInfo> { new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()) }; var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>(); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>())) .ReturnsAsync(moduleRuntimeInfoList); var logsProvider = new Mock <ILogsProvider>(); var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); string mod1Logs = new[] { "Log line 1\n", "Log line 2\n", "Log line 3\n" }.Join(string.Empty); logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod1Logs.ToBytes()); // Act var logsRequestHandler = new LogsRequestHandler(logsProvider.Object, runtimeInfoProvider.Object); Option <string> response = await logsRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.True(response.HasValue); logsProvider.VerifyAll(); runtimeInfoProvider.VerifyAll(); var logsResponseList = response.OrDefault().FromJson <List <LogsResponse> >(); Assert.NotNull(logsResponseList); Assert.Single(logsResponseList); LogsResponse logsResponse = logsResponseList[0]; Assert.Equal(mod1, logsResponse.Id); Assert.True(logsResponse.Payload.HasValue); Assert.False(logsResponse.PayloadBytes.HasValue); Assert.Equal(mod1Logs, logsResponse.Payload.OrDefault()); }
public async Task GetJsonGzipLogsTest() { var filter = new ModuleLogFilter(Option.Some(100), Option.Some("1501000"), Option.None <string>(), Option.Some(3), Option.None <bool>(), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.Gzip; LogsContentType contentType = LogsContentType.Json; string payload = @"{ ""schemaVersion"": ""1.0"", ""items"": { ""id"": ""m1"", ""filter"": <filter> }, ""encoding"": ""gzip"", ""contentType"": ""json"" }" .Replace("<filter>", filter.ToJson()); string iotHub = "foo.azure-devices.net"; string deviceId = "d1"; string mod1 = "m1"; string mod2 = "m2"; string mod3 = "m3"; var moduleRuntimeInfoList = new List <ModuleRuntimeInfo> { new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()) }; var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>(); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>())) .ReturnsAsync(moduleRuntimeInfoList); var logsProvider = new Mock <ILogsProvider>(); var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); var mod1Logs = new List <ModuleLogMessage> { new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 1"), new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 2"), new ModuleLogMessage(iotHub, deviceId, mod1, "0", 6, Option.None <DateTime>(), "log line 3") }; byte[] mod1LogBytes = Compression.CompressToGzip(mod1Logs.ToBytes()); logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod1LogBytes); // Act var logsRequestHandler = new ModuleLogsRequestHandler(logsProvider.Object, runtimeInfoProvider.Object); Option <string> response = await logsRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.True(response.HasValue); logsProvider.VerifyAll(); runtimeInfoProvider.VerifyAll(); var logsResponseList = response.OrDefault().FromJson <List <ModuleLogsResponse> >(); Assert.NotNull(logsResponseList); Assert.Single(logsResponseList); ModuleLogsResponse logsResponse = logsResponseList[0]; Assert.Equal(mod1, logsResponse.Id); Assert.False(logsResponse.Payload.HasValue); Assert.True(logsResponse.PayloadBytes.HasValue); Assert.Equal(mod1LogBytes, logsResponse.PayloadBytes.OrDefault()); }
public ModuleLogOptions(string id, LogsContentEncoding contentEncoding, LogsContentType contentType) { this.Id = Preconditions.CheckNonWhiteSpace(id, nameof(id)); this.ContentEncoding = contentEncoding; this.ContentType = contentType; }
public async Task TestLogsUploadAllTaskRequest() { // Arrange string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; var filter = new ModuleLogFilter(Option.Some(100), Option.Some(1501000), Option.Some(3), Option.Some("ERR")); LogsContentEncoding contentEncoding = LogsContentEncoding.None; LogsContentType contentType = LogsContentType.Json; string payload = @"{ ""schemaVersion"": ""1.0"", ""sasUrl"": ""<sasurl>"", ""items"": { ""id"": "".*"", ""filter"": <filter> }, ""encoding"": ""none"", ""contentType"": ""json"" }" .Replace("<sasurl>", sasUrl) .Replace("<filter>", filter.ToJson()); string mod1 = "m1"; string mod2 = "m2"; string mod3 = "m3"; var moduleRuntimeInfoList = new List <ModuleRuntimeInfo> { new ModuleRuntimeInfo(mod1, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod2, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()), new ModuleRuntimeInfo(mod3, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()) }; var runtimeInfoProvider = new Mock <IRuntimeInfoProvider>(); runtimeInfoProvider.Setup(r => r.GetModules(It.IsAny <CancellationToken>())) .ReturnsAsync(moduleRuntimeInfoList); var logsUploader = new Mock <ILogsUploader>(); var logsProvider = new Mock <ILogsProvider>(); var module1LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); var mod1LogBytes = new byte[100]; logsProvider.Setup(l => l.GetLogs(mod1, module1LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod1LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod1, mod1LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module2LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); var mod2LogBytes = new byte[80]; logsProvider.Setup(l => l.GetLogs(mod2, module2LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod2LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod2, mod2LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); var module3LogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); var mod3LogBytes = new byte[120]; logsProvider.Setup(l => l.GetLogs(mod3, module3LogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(mod3LogBytes); logsUploader.Setup(l => l.Upload(sasUrl, mod3, mod3LogBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); // Act var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider.Object); Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.True(response.HasValue); var taskStatusResponse = response.OrDefault().FromJson <TaskStatusResponse>(); Assert.NotNull(taskStatusResponse); Assert.NotEmpty(taskStatusResponse.CorrelationId); Assert.Equal(string.Empty, taskStatusResponse.Message); await WaitForBackgroundTaskCompletion(taskStatusResponse.CorrelationId).TimeoutAfter(TimeSpan.FromSeconds(5)); logsProvider.VerifyAll(); logsUploader.VerifyAll(); runtimeInfoProvider.VerifyAll(); }
public async Task TestLogsUploadRequest(string payload, string id, string sasUrl, LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter) { // Arrange var logsUploader = new Mock <ILogsUploader>(); var logsProvider = new Mock <ILogsProvider>(); var uploadBytes = new byte[100]; var moduleLogOptions = new ModuleLogOptions(contentEncoding, contentType, filter); logsProvider.Setup(l => l.GetLogs(id, moduleLogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(uploadBytes); logsUploader.Setup(l => l.Upload(sasUrl, id, uploadBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); // Act var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, Mock.Of <IRuntimeInfoProvider>()); Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.False(response.HasValue); logsProvider.VerifyAll(); logsUploader.VerifyAll(); }
// This method returns a func instead of IAzureAppendBlob interface to keep the ILogsUploader interface non Azure specific. public async Task <Func <ArraySegment <byte>, Task> > GetUploaderCallback(string uri, string id, LogsContentEncoding logsContentEncoding, LogsContentType logsContentType) { Preconditions.CheckNonWhiteSpace(uri, nameof(uri)); Preconditions.CheckNonWhiteSpace(id, nameof(id)); var containerUri = new Uri(uri); string blobName = this.GetBlobName(id, logsContentEncoding, logsContentType); var container = new CloudBlobContainer(containerUri); Events.Uploading(blobName, container.Name); IAzureAppendBlob blob = await this.azureBlobUploader.GetAppendBlob(containerUri, blobName, GetContentType(logsContentType), GetContentEncoding(logsContentEncoding)); return(blob.AppendByteArray); }
public async Task TestLogsUploadRequest(string payload, string id, string sasUrl, LogsContentEncoding contentEncoding, LogsContentType contentType, ModuleLogFilter filter) { // Arrange var logsUploader = new Mock <ILogsUploader>(); var logsProvider = new Mock <ILogsProvider>(); var uploadBytes = new byte[100]; var moduleLogOptions = new ModuleLogOptions(contentEncoding, contentType, filter, LogOutputFraming.None, Option.None <LogsOutputGroupingConfig>(), false); if (contentType == LogsContentType.Text) { Func <ArraySegment <byte>, Task> getLogsCallback = bytes => Task.CompletedTask; logsUploader.Setup(l => l.GetUploaderCallback(sasUrl, id, contentEncoding, contentType)) .ReturnsAsync(getLogsCallback); logsProvider.Setup(l => l.GetLogsStream(id, moduleLogOptions, getLogsCallback, It.IsAny <CancellationToken>())) .Returns(Task.CompletedTask); } else { logsProvider.Setup(l => l.GetLogs(id, moduleLogOptions, It.IsAny <CancellationToken>())) .ReturnsAsync(uploadBytes); logsUploader.Setup(l => l.Upload(sasUrl, id, uploadBytes, contentEncoding, contentType)) .Returns(Task.CompletedTask); } IEnumerable <ModuleRuntimeInfo> moduleRuntimeInfoList = new List <ModuleRuntimeInfo> { new ModuleRuntimeInfo(id, "docker", ModuleStatus.Running, string.Empty, 0, Option.None <DateTime>(), Option.None <DateTime>()) }; var runtimeInfoProvider = Mock.Of <IRuntimeInfoProvider>(r => r.GetModules(It.IsAny <CancellationToken>()) == Task.FromResult(moduleRuntimeInfoList)); // Act var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object, runtimeInfoProvider); Option <string> response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload), CancellationToken.None); // Assert Assert.False(response.HasValue); logsProvider.VerifyAll(); logsUploader.VerifyAll(); Mock.Get(runtimeInfoProvider).VerifyAll(); }