private async Task <string> Download(RunFromPackageContext pkgContext) { var zipUri = new Uri(pkgContext.Url); if (!Utility.TryCleanUrl(zipUri.AbsoluteUri, out string cleanedUrl)) { throw new Exception("Invalid url for the package"); } var tmpPath = Path.GetTempPath(); var fileName = Path.GetFileName(zipUri.AbsolutePath); var filePath = Path.Combine(tmpPath, fileName); if (pkgContext.PackageContentLength != null && pkgContext.PackageContentLength > 100 * 1024 * 1024) { _logger.LogInformation($"Downloading zip contents from '{cleanedUrl}' using aria2c'"); AriaDownload(tmpPath, fileName, zipUri); } else { _logger.LogInformation($"Downloading zip contents from '{cleanedUrl}' using httpclient'"); await HttpClientDownload(filePath, zipUri); } return(filePath); }
private async Task ApplyContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)"); assignmentContext.ApplyAppSettings(_environment); // We need to get the non-PlaceholderMode script Path so we can unzip to the correct location. // This asks the factory to skip the PlaceholderMode check when configuring options. var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); if ((pkgContext.IsScmRunFromPackage() && await pkgContext.BlobExistsAsync(_logger)) || (!pkgContext.IsScmRunFromPackage() && !string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1")) { await ApplyBlobPackageContext(pkgContext, options.ScriptPath); } else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString)) { await _meshInitServiceClient.MountCifs(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare, "/home"); } // BYOS var storageVolumes = assignmentContext.GetBYOSEnvironmentVariables() .Select(AzureStorageInfoValue.FromEnvironmentVariable).ToList(); var mountedVolumes = (await Task.WhenAll(storageVolumes.Where(v => v != null).Select(MountStorageAccount))).Where( result => result).ToList(); if (mountedVolumes.Count != storageVolumes.Count) { _logger.LogWarning( $"Successfully mounted {mountedVolumes.Count} / {storageVolumes.Count} BYOS storage accounts"); } }
public async Task <string> Download(RunFromPackageContext pkgContext) { var zipUri = new Uri(pkgContext.Url); if (!Utility.TryCleanUrl(zipUri.AbsoluteUri, out string cleanedUrl)) { throw new InvalidOperationException("Invalid url for the package"); } var tmpPath = Path.GetTempPath(); var fileName = Path.GetFileName(zipUri.AbsolutePath); var filePath = Path.Combine(tmpPath, fileName); if (pkgContext.PackageContentLength != null && pkgContext.PackageContentLength > AriaDownloadThreshold) { _logger.LogDebug($"Downloading zip contents from '{cleanedUrl}' using aria2c'"); AriaDownload(tmpPath, fileName, zipUri, pkgContext.IsWarmUpRequest); } else { _logger.LogDebug($"Downloading zip contents from '{cleanedUrl}' using httpclient'"); await HttpClientDownload(filePath, zipUri, pkgContext.IsWarmUpRequest); } return(filePath); }
public async Task <string> Download(RunFromPackageContext pkgContext) { if (Utility.TryCleanUrl(pkgContext.Url, out var cleanedUrl)) { _logger.LogDebug("Downloading app contents from '{cleanedUrl}'", cleanedUrl); } else { throw new InvalidOperationException("Invalid url for the package"); } var isWarmupRequest = pkgContext.IsWarmUpRequest; var needsManagedIdentityToken = !isWarmupRequest && await IsAuthenticationTokenNecessary(pkgContext.Url); _logger.LogDebug( "{nameof(PackageDownloadHandler)}: Needs ManagedIdentity Token = '{needsManagedIdentityToken}' IsWarmupRequest = '{isWarmupRequest}'", nameof(PackageDownloadHandler), needsManagedIdentityToken, isWarmupRequest); var zipUri = new Uri(pkgContext.Url); var token = needsManagedIdentityToken ? await _managedIdentityTokenProvider.GetManagedIdentityToken(zipUri.AbsoluteUri) : string.Empty; return(await Download(pkgContext, zipUri, token)); }
private void UnpackPackage(string filePath, string scriptPath, RunFromPackageContext pkgContext) { CodePackageType packageType; using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationGetPackageType)) { packageType = GetPackageType(filePath, pkgContext); } if (packageType == CodePackageType.Squashfs) { // default to mount for squashfs images if (_environment.IsMountDisabled()) { UnsquashImage(filePath, scriptPath); } else { MountFsImage(filePath, scriptPath); } } else if (packageType == CodePackageType.Zip) { // default to unzip for zip packages if (_environment.IsMountEnabled()) { MountZipFile(filePath, scriptPath); } else { UnzipPackage(filePath, scriptPath); } } }
public async Task MountsSquashFsFileIfMountEnabledUsingFileCommand(bool azureFilesMounted, string isMountEnabled) { var isWarmUpRequest = false; int packageLength = 100; var extension = "unknown"; _environment.SetEnvironmentVariable(EnvironmentSettingNames.MountEnabled, isMountEnabled); _environment.SetEnvironmentVariable(EnvironmentSettingNames.MeshInitURI, MeshInitUri); // httpDownload var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClientFactory = TestHelpers.CreateHttpClientFactory(handlerMock.Object); var packageDownloadHandler = new PackageDownloadHandler(_httpClientFactory, new Mock <IManagedIdentityTokenProvider>(MockBehavior.Strict).Object, _bashCmdHandlerMock.Object, NullLogger <PackageDownloadHandler> .Instance, _metricsLogger); _runFromPackageHandler = new RunFromPackageHandler(_environment, _meshServiceClientMock.Object, _bashCmdHandlerMock.Object, _zipHandler.Object, packageDownloadHandler, _metricsLogger, _logger); var url = $"http://url/zip-file.{extension}"; var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, packageLength, isWarmUpRequest); _bashCmdHandlerMock.Setup(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith(BashCommandHandler.FileCommand) && url.EndsWith(Path.GetFileName(s), StringComparison.OrdinalIgnoreCase)), MetricEventNames.LinuxContainerSpecializationFileCommand)).Returns(("squashfs", string.Empty, 0)); _meshServiceClientMock.Setup(m => m.MountFuse(MeshServiceClient.SquashFsOperation, It.Is <string>(s => url.EndsWith(Path.GetFileName(s))), TargetScriptPath)) .Returns(Task.FromResult(true)); var applyBlobContextResult = await _runFromPackageHandler.ApplyBlobPackageContext(runFromPackageContext, TargetScriptPath, azureFilesMounted, true); Assert.True(applyBlobContextResult); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(r => IsZipDownloadRequest(r, url)), ItExpr.IsAny <CancellationToken>()); _bashCmdHandlerMock.Verify(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith(BashCommandHandler.FileCommand) && url.EndsWith(Path.GetFileName(s), StringComparison.OrdinalIgnoreCase)), MetricEventNames.LinuxContainerSpecializationFileCommand), Times.Once); _meshServiceClientMock.Verify(m => m.MountFuse(MeshServiceClient.SquashFsOperation, It.Is <string>(s => url.EndsWith(Path.GetFileName(s))), TargetScriptPath), Times.Once); }
private async Task ApplyContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)"); assignmentContext.ApplyAppSettings(_environment); // We need to get the non-PlaceholderMode script Path so we can unzip to the correct location. // This asks the factory to skip the PlaceholderMode check when configuring options. var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); if ((pkgContext.IsScmRunFromPackage() && await pkgContext.BlobExistsAsync(_logger)) || (!pkgContext.IsScmRunFromPackage() && !string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1")) { await ApplyBlobPackageContext(pkgContext, options.ScriptPath); } else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString)) { await MountCifs(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare, "/home"); } // Irrespective of deployment mechanism mount the share for user data files. if (assignmentContext.IsUserDataMountEnabled()) { if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString) && !string.IsNullOrEmpty(assignmentContext.AzureFilesContentShare)) { await MountUserData(assignmentContext); } else { _logger.LogWarning( $"{EnvironmentSettingNames.AzureFilesConnectionString} or {EnvironmentSettingNames.AzureFilesContentShare} is empty. User data share will not be mounted"); } } }
public async Task ThrowsExceptionForInvalidUrls(string url) { var downloader = new PackageDownloadHandler(_httpClientFactory, _managedIdentityTokenProvider.Object, _bashCmdHandlerMock.Object, _logger, _metricsLogger); var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, null, true); await Assert.ThrowsAsync <InvalidOperationException>(async() => await downloader.Download(runFromPackageContext)); }
public async Task DownloadsZipsUsingHttpClient(bool isWarmupRequest) { FileUtility.Instance = GetFileSystem().Object; var url = $"http://url/{ZipFileName}"; const int fileSize = RunFromPackageHandler.AriaDownloadThreshold - 1; var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClient = new HttpClient(handlerMock.Object); _runFromPackageHandler = new RunFromPackageHandler(_environment, _httpClient, _meshServiceClientMock.Object, _bashCmdHandlerMock.Object, _zipHandler.Object, _metricsLogger, _logger); var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, fileSize, isWarmupRequest); var filePath = await _runFromPackageHandler.Download(runFromPackageContext); Assert.Equal(ZipFileName, Path.GetFileName(filePath), StringComparer.Ordinal); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(r => IsZipDownloadRequest(r, url)), ItExpr.IsAny <CancellationToken>()); }
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}'. IsWarmup: '{assignmentContext.IsWarmupRequest}')"); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); _logger.LogInformation($"Will be using {pkgContext.EnvironmentVariableName} app setting as zip url. IsWarmup: '{assignmentContext.IsWarmupRequest}')"); if (pkgContext.IsScmRunFromPackage()) { // Not user assigned so limit validation return(null); } else if (!string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1") { // In AppService, ZipUrl == 1 means the package is hosted in azure files. // Otherwise we expect zipUrl to be a blobUri to a zip or a squashfs image (var error, var contentLength) = await ValidateBlobPackageContext(pkgContext); if (string.IsNullOrEmpty(error)) { assignmentContext.PackageContentLength = contentLength; } return(error); } else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString)) { return(await ValidateAzureFilesContext(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare)); } else { _logger.LogError($"Missing ZipUrl and AzureFiles config. Continue with empty root."); return(null); } }
public void Returns_IsRunFromPackage(string environmentVariableName, string url, bool blobExists, bool scmRunFromPackageConfigured) { var runFromPackageContext = new RunFromPackageContext(environmentVariableName, url, 10, false); var options = new ScriptApplicationHostOptions { IsScmRunFromPackage = Url.Equals(url) ? blobExists : false }; Assert.Equal(scmRunFromPackageConfigured, runFromPackageContext.IsRunFromPackage(options, NullLogger.Instance)); }
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}'. IsWarmup: '{assignmentContext.IsWarmupRequest}')"); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); _logger.LogInformation($"Will be using {pkgContext.EnvironmentVariableName} app setting as zip url. IsWarmup: '{assignmentContext.IsWarmupRequest}')"); if (pkgContext.IsScmRunFromPackage()) { // Not user assigned so limit validation return(null); } else if (!string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1") { if (Uri.TryCreate(pkgContext.Url, UriKind.Absolute, out var uri)) { if (Utility.IsResourceAzureBlobWithoutSas(uri)) { // Note: this also means we skip validation for publicly available blobs _logger.LogDebug("Skipping validation for '{pkgContext.EnvironmentVariableName}' with no SAS token", pkgContext.EnvironmentVariableName); return(null); } else { // In AppService, ZipUrl == 1 means the package is hosted in azure files. // Otherwise we expect zipUrl to be a blobUri to a zip or a squashfs image var(error, contentLength) = await ValidateBlobPackageContext(pkgContext); if (string.IsNullOrEmpty(error)) { assignmentContext.PackageContentLength = contentLength; } return(error); } } else { var invalidUrlError = $"Invalid url for specified for {pkgContext.EnvironmentVariableName}"; _logger.LogError(invalidUrlError); // For now we return null here instead of the actual error since this validation is new. // Eventually this could return the error message. return(null); } } else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString)) { return(await ValidateAzureFilesContext(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare)); } else { _logger.LogError("Missing ZipUrl and AzureFiles config. Continue with empty root."); return(null); } }
public async Task DownloadFailsForInvalidUrl(bool isWarmupRequest, bool isLargeZip) { var fileSize = isLargeZip ? RunFromPackageHandler.AriaDownloadThreshold + 1 : RunFromPackageHandler.AriaDownloadThreshold - 1; const string url = "invalid-url"; var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, fileSize, isWarmupRequest); await Assert.ThrowsAsync <UriFormatException>(async() => await _runFromPackageHandler.Download(runFromPackageContext)); }
private async Task ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath) { // download zip and extract var filePath = await Download(pkgContext); await UnpackPackage(filePath, targetPath, pkgContext); string bundlePath = Path.Combine(targetPath, "worker-bundle"); if (Directory.Exists(bundlePath)) { _logger.LogInformation($"Python worker bundle detected"); } }
public async Task DownloadsLargeZipsUsingAria2c(bool isWarmupRequest) { var url = $"http://url/{ZipFileName}"; FileUtility.Instance = GetFileSystem().Object; const int fileSize = PackageDownloadHandler.AriaDownloadThreshold + 1; var expectedMetricName = isWarmupRequest ? MetricEventNames.LinuxContainerSpecializationZipDownloadWarmup : MetricEventNames.LinuxContainerSpecializationZipDownload; var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); if (isWarmupRequest) { handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.Is <HttpRequestMessage>(s => MatchesVerb(s, HttpMethod.Get) && MatchesTargetUri(s, url)), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClientFactory = TestHelpers.CreateHttpClientFactory(handlerMock.Object); } else { _bashCmdHandlerMock.Setup(b => b.RunBashCommand(It.Is <string>(s => s.StartsWith(PackageDownloadHandler.Aria2CExecutable)), expectedMetricName)).Returns(("", "", 0)); } var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, fileSize, isWarmupRequest); var packageDownloadHandler = new PackageDownloadHandler(_httpClientFactory, _managedIdentityTokenProvider.Object, _bashCmdHandlerMock.Object, _logger, _metricsLogger); var filePath = await packageDownloadHandler.Download(runFromPackageContext); Assert.Equal(ZipFileName, Path.GetFileName(filePath), StringComparer.Ordinal); if (isWarmupRequest) { handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()); } else { _bashCmdHandlerMock.Verify(b => b.RunBashCommand(It.Is <string>(s => s.StartsWith(PackageDownloadHandler.Aria2CExecutable)), expectedMetricName), Times.Once); } }
public async Task ApplyContextFailsForUnknownFileTypes(bool azureFilesMounted, string isMountEnabled) { var isWarmUpRequest = false; int packageLength = 100; var extension = "unknown"; _environment.SetEnvironmentVariable(EnvironmentSettingNames.MountEnabled, isMountEnabled); _environment.SetEnvironmentVariable(EnvironmentSettingNames.MeshInitURI, MeshInitUri); // httpDownload var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClient = new HttpClient(handlerMock.Object); _runFromPackageHandler = new RunFromPackageHandler(_environment, _httpClient, _meshServiceClientMock.Object, _bashCmdHandlerMock.Object, _zipHandler.Object, _metricsLogger, _logger); var url = $"http://url/zip-file.{extension}"; var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, packageLength, isWarmUpRequest); _bashCmdHandlerMock.Setup(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith(BashCommandHandler.FileCommand) && url.EndsWith(Path.GetFileName(s), StringComparison.OrdinalIgnoreCase)), MetricEventNames.LinuxContainerSpecializationFileCommand)).Returns(("unknown", string.Empty, 0)); await Assert.ThrowsAsync <InvalidOperationException>(async() => await _runFromPackageHandler.ApplyBlobPackageContext(runFromPackageContext, TargetScriptPath, azureFilesMounted, true)); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(r => IsZipDownloadRequest(r, url)), ItExpr.IsAny <CancellationToken>()); _bashCmdHandlerMock.Verify(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith(BashCommandHandler.FileCommand) && url.EndsWith(Path.GetFileName(s), StringComparison.OrdinalIgnoreCase)), MetricEventNames.LinuxContainerSpecializationFileCommand), Times.Once); }
private async Task UnpackPackage(string filePath, string scriptPath, RunFromPackageContext pkgContext, string localSitePackagesPath) { var useLocalSitePackages = !string.IsNullOrEmpty(localSitePackagesPath); CodePackageType packageType; using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationGetPackageType)) { packageType = GetPackageType(filePath, pkgContext); } if (packageType == CodePackageType.Squashfs) { // default to mount for squashfs images if (_environment.IsMountDisabled()) { if (useLocalSitePackages) { UnsquashImage(filePath, localSitePackagesPath); await CreateBindMount(localSitePackagesPath, scriptPath); } else { UnsquashImage(filePath, scriptPath); } } else { await _meshServiceClient.MountFuse(MeshServiceClient.SquashFsOperation, filePath, scriptPath); } } else if (packageType == CodePackageType.Zip) { // default to unzip for zip packages if (_environment.IsMountEnabled()) { await _meshServiceClient.MountFuse(MeshServiceClient.ZipOperation, filePath, scriptPath); } else { if (useLocalSitePackages) { _unZipHandler.UnzipPackage(filePath, localSitePackagesPath); await CreateBindMount(localSitePackagesPath, scriptPath); } else { _unZipHandler.UnzipPackage(filePath, scriptPath); } } } }
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')"); string error = null; HttpResponseMessage response = null; try { RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); _logger.LogInformation($"Will be using {pkgContext.EnvironmentVariableName} app setting as zip url"); if (pkgContext.IsScmRunFromPackage()) { // Not user assigned so limit validation return(null); } var zipUrl = pkgContext.Url; if (!string.IsNullOrEmpty(zipUrl)) { // make sure the zip uri is valid and accessible await Utility.InvokeWithRetriesAsync(async() => { try { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipHead)) { var request = new HttpRequestMessage(HttpMethod.Head, zipUrl); response = await _client.SendAsync(request); response.EnsureSuccessStatusCode(); } } catch (Exception e) { _logger.LogError(e, $"{MetricEventNames.LinuxContainerSpecializationZipHead} failed"); throw; } }, maxRetries : 2, retryInterval : TimeSpan.FromSeconds(0.3)); // Keep this less than ~1s total } } catch (Exception e) { error = $"Invalid zip url specified (StatusCode: {response?.StatusCode})"; _logger.LogError(e, "ValidateContext failed"); } return(error); }
public async Task Returns_IsRunFromPackage(string environmentVariableName, string url, bool blobExists, bool scmRunFromPackageConfigured) { var cloudBlockBlobService = new Mock <RunFromPackageCloudBlockBlobService>(MockBehavior.Strict); cloudBlockBlobService .Setup(c => c.BlobExists(Url, environmentVariableName, NullLogger.Instance)) .ReturnsAsync(blobExists); cloudBlockBlobService .Setup(c => c.BlobExists(It.Is <string>(s => !string.Equals(s, Url, StringComparison.OrdinalIgnoreCase)), environmentVariableName, NullLogger.Instance)) .ReturnsAsync(false); var runFromPackageContext = new RunFromPackageContext(environmentVariableName, url, 10, false, cloudBlockBlobService.Object); Assert.Equal(scmRunFromPackageConfigured, await runFromPackageContext.IsRunFromPackage(NullLogger.Instance)); }
private async Task <(string, long?)> ValidateBlobPackageContext(RunFromPackageContext context) { string blobUri = context.Url; string eventName = context.IsWarmUpRequest ? MetricEventNames.LinuxContainerSpecializationZipHeadWarmup : MetricEventNames.LinuxContainerSpecializationZipHead; string error = null; HttpResponseMessage response = null; long?contentLength = null; try { if (!string.IsNullOrEmpty(blobUri)) { // make sure the zip uri is valid and accessible await Utility.InvokeWithRetriesAsync(async() => { try { using (_metricsLogger.LatencyEvent(eventName)) { var request = new HttpRequestMessage(HttpMethod.Head, blobUri); response = await _client.SendAsync(request); response.EnsureSuccessStatusCode(); if (response.Content != null && response.Content.Headers != null) { contentLength = response.Content.Headers.ContentLength; } } } catch (Exception e) { _logger.LogError(e, $"{eventName} failed"); throw; } }, maxRetries : 2, retryInterval : TimeSpan.FromSeconds(0.3)); // Keep this less than ~1s total } } catch (Exception e) { error = $"Invalid zip url specified (StatusCode: {response?.StatusCode})"; _logger.LogError(e, $"ValidateContext failed. IsWarmupRequest = {context.IsWarmUpRequest}"); } return(error, contentLength); }
private async Task ApplyContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)"); assignmentContext.ApplyAppSettings(_environment); // We need to get the non-PlaceholderMode script Path so we can unzip to the correct location. // This asks the factory to skip the PlaceholderMode check when configuring options. var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); var zipPath = pkgContext.Url; if (!string.IsNullOrEmpty(zipPath)) { // download zip and extract var zipUri = new Uri(zipPath); string filePath = null; if (pkgContext.IsScmRunFromPackage()) { bool blobExists = await pkgContext.BlobExistsAsync(_logger); if (blobExists) { filePath = await DownloadAsync(zipUri); } else { return; } } else { filePath = await DownloadAsync(zipUri); } UnpackPackage(filePath, options.ScriptPath); string bundlePath = Path.Combine(options.ScriptPath, "worker-bundle"); if (Directory.Exists(bundlePath)) { _logger.LogInformation($"Python worker bundle detected"); } } }
public async Task MountsZipFileIfMountEnabled(bool azureFilesMounted) { var isWarmUpRequest = false; var extension = "zip"; _environment.SetEnvironmentVariable(EnvironmentSettingNames.MountEnabled, "1"); _environment.SetEnvironmentVariable(EnvironmentSettingNames.MeshInitURI, MeshInitUri); // httpDownload var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClient = new HttpClient(handlerMock.Object); _runFromPackageHandler = new RunFromPackageHandler(_environment, _httpClient, _meshServiceClientMock.Object, _bashCmdHandlerMock.Object, _zipHandler.Object, _metricsLogger, _logger); var url = $"http://url/zip-file.{extension}"; var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, DefaultPackageLength, isWarmUpRequest); _meshServiceClientMock .Setup(m => m.MountFuse(MeshServiceClient.ZipOperation, It.Is <string>(s => url.EndsWith(Path.GetFileName(s))), TargetScriptPath)) .Returns(Task.FromResult(true)); var applyBlobContextResult = await _runFromPackageHandler.ApplyBlobPackageContext(runFromPackageContext, TargetScriptPath, azureFilesMounted, true); Assert.True(applyBlobContextResult); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(r => IsZipDownloadRequest(r, url)), ItExpr.IsAny <CancellationToken>()); _meshServiceClientMock .Setup(m => m.MountFuse(MeshServiceClient.ZipOperation, It.Is <string>(s => url.EndsWith(Path.GetFileName(s))), TargetScriptPath)) .Returns(Task.FromResult(true)); }
private async Task <string> Download(RunFromPackageContext pkgContext, Uri zipUri, string token) { if (pkgContext.IsWarmUpRequest && !string.IsNullOrEmpty(token)) { throw new Exception("Warmup requests do not support ManagedIdentity token"); } var tmpPath = Path.GetTempPath(); var fileName = Path.GetFileName(zipUri.AbsolutePath); var filePath = Path.Combine(tmpPath, fileName); string downloadMetricName; if (!string.IsNullOrEmpty(token)) { downloadMetricName = MetricEventNames.LinuxContainerSpecializationZipDownloadUsingManagedIdentity; } else { downloadMetricName = pkgContext.IsWarmUpRequest ? MetricEventNames.LinuxContainerSpecializationZipDownloadWarmup : MetricEventNames.LinuxContainerSpecializationZipDownload; } var tokenPrefix = token == null ? "Null" : token.Substring(0, Math.Min(token.Length, 3)); // Aria download doesn't support MI Token or warmup requests if (pkgContext.PackageContentLength != null && pkgContext.PackageContentLength > AriaDownloadThreshold && string.IsNullOrEmpty(token) && !pkgContext.IsWarmUpRequest) { _logger.LogDebug( "Downloading zip contents using aria2c. IsWarmupRequest = '{pkgContext.IsWarmUpRequest}'. Managed Identity TokenPrefix = '{tokenPrefix}'", pkgContext.IsWarmUpRequest, tokenPrefix); AriaDownload(tmpPath, fileName, zipUri, pkgContext.IsWarmUpRequest, downloadMetricName); } else { _logger.LogDebug( "Downloading zip contents using httpclient. IsWarmupRequest = '{pkgContext.IsWarmUpRequest}'. Managed Identity TokenPrefix = '{tokenPrefix}'", pkgContext.IsWarmUpRequest, tokenPrefix); await HttpClientDownload(filePath, zipUri, pkgContext.IsWarmUpRequest, token, downloadMetricName); } return(filePath); }
private async Task ApplyContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)"); assignmentContext.ApplyAppSettings(_environment); // We need to get the non-PlaceholderMode script Path so we can unzip to the correct location. // This asks the factory to skip the PlaceholderMode check when configuring options. var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder); RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext(); if ((pkgContext.IsScmRunFromPackage() && await pkgContext.BlobExistsAsync(_logger)) || (!pkgContext.IsScmRunFromPackage() && !string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1")) { await ApplyBlobPackageContext(pkgContext, options.ScriptPath); } else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString)) { ApplyAzureFilesContext(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare, "/home"); } }
public async Task DownloadsFileUsingManagedIdentity() { var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.Is <HttpRequestMessage>(s => MatchesVerb(s, HttpMethod.Head)), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.Unauthorized, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.Is <HttpRequestMessage>(s => MatchesVerb(s, HttpMethod.Get) && HasBearerToken(s) && MatchesTargetUri(s, UriWithNoSasToken)), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClientFactory = TestHelpers.CreateHttpClientFactory(handlerMock.Object); _managedIdentityTokenProvider.Setup(p => p.GetManagedIdentityToken(UriWithNoSasToken)).Returns(Task.FromResult(BearerToken)); var downloader = new PackageDownloadHandler(_httpClientFactory, _managedIdentityTokenProvider.Object, _bashCmdHandlerMock.Object, _logger, _metricsLogger); var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, UriWithNoSasToken, 0, false); await downloader.Download(runFromPackageContext); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(s => MatchesVerb(s, HttpMethod.Head)), ItExpr.IsAny <CancellationToken>()); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(s => MatchesVerb(s, HttpMethod.Get) && HasBearerToken(s) && MatchesTargetUri(s, UriWithNoSasToken)), ItExpr.IsAny <CancellationToken>()); _managedIdentityTokenProvider.Verify(p => p.GetManagedIdentityToken(UriWithNoSasToken), Times.Once); }
public async Task DownloadsLargeZipsUsingAria2c(bool isWarmupRequest) { var url = $"http://url/{ZipFileName}"; FileUtility.Instance = GetFileSystem().Object; const int fileSize = RunFromPackageHandler.AriaDownloadThreshold + 1; var expectedMetricName = isWarmupRequest ? MetricEventNames.LinuxContainerSpecializationZipDownloadWarmup : MetricEventNames.LinuxContainerSpecializationZipDownload; _bashCmdHandlerMock.Setup(b => b.RunBashCommand(It.Is <string>(s => s.StartsWith(RunFromPackageHandler.Aria2CExecutable)), expectedMetricName)).Returns(("", "", 0)); var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, fileSize, isWarmupRequest); var filePath = await _runFromPackageHandler.Download(runFromPackageContext); Assert.Equal(ZipFileName, Path.GetFileName(filePath), StringComparer.Ordinal); _bashCmdHandlerMock.Verify(b => b.RunBashCommand(It.Is <string>(s => s.StartsWith(RunFromPackageHandler.Aria2CExecutable)), expectedMetricName), Times.Once); }
private CodePackageType GetPackageType(string filePath, RunFromPackageContext pkgContext) { // cloud build always builds squashfs if (pkgContext.IsScmRunFromPackage()) { return(CodePackageType.Squashfs); } var uri = new Uri(pkgContext.Url); // check file name since it'll be faster than running `file` if (FileIsAny(".squashfs", ".sfs", ".sqsh", ".img", ".fs")) { return(CodePackageType.Squashfs); } else if (FileIsAny(".zip")) { return(CodePackageType.Zip); } // Check file magic-number using `file` command. (var output, _, _) = RunBashCommand($"file -b {filePath}", MetricEventNames.LinuxContainerSpecializationFileCommand); if (output.StartsWith("Squashfs", StringComparison.OrdinalIgnoreCase)) { return(CodePackageType.Squashfs); } else if (output.StartsWith("Zip", StringComparison.OrdinalIgnoreCase)) { return(CodePackageType.Zip); } else { throw new InvalidOperationException($"Can't find CodePackageType to match {filePath}"); } bool FileIsAny(params string[] options) => options.Any(o => uri.AbsolutePath.EndsWith(o, StringComparison.OrdinalIgnoreCase)); }
public async Task <bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted, bool throwOnFailure = true) { try { // If Azure Files are mounted, /home will point to a shared remote dir // So extracting to /home/site/wwwroot can interfere with other running instances // Instead extract to localSitePackagesPath and bind to /home/site/wwwroot // home will continue to point to azure file share var localSitePackagesPath = azureFilesMounted ? _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.LocalSitePackages, EnvironmentSettingNames.DefaultLocalSitePackagesPath) : string.Empty; // download zip and extract var filePath = await _packageDownloadHandler.Download(pkgContext); await UnpackPackage(filePath, targetPath, pkgContext, localSitePackagesPath); string bundlePath = Path.Combine(targetPath, "worker-bundle"); if (Directory.Exists(bundlePath)) { _logger.LogInformation("Python worker bundle detected"); } return(true); } catch (Exception e) { _logger.LogDebug(e, nameof(ApplyBlobPackageContext)); if (throwOnFailure) { throw; } return(false); } }
public void Returns_ScmRunFromPackage(string environmentVariableName, bool isScmRunFromPackage) { var runFromPackageContext = new RunFromPackageContext(environmentVariableName, "", 0, false); Assert.Equal(isScmRunFromPackage, runFromPackageContext.IsScmRunFromPackage()); }
public async Task UnsquashesSquashFsFileIfMountDisabled(bool azureFilesMounted) { var isWarmUpRequest = false; var extension = "squashfs"; _environment.SetEnvironmentVariable(EnvironmentSettingNames.MountEnabled, "0"); _environment.SetEnvironmentVariable(EnvironmentSettingNames.MeshInitURI, MeshInitUri); // httpDownload var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict); handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync", ItExpr.IsAny <HttpRequestMessage>(), ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage { StatusCode = HttpStatusCode.OK, Content = new ReadOnlyMemoryContent(ReadOnlyMemory <byte> .Empty) }); _httpClient = new HttpClient(handlerMock.Object); _runFromPackageHandler = new RunFromPackageHandler(_environment, _httpClient, _meshServiceClientMock.Object, _bashCmdHandlerMock.Object, _zipHandler.Object, _metricsLogger, _logger); if (azureFilesMounted) { _bashCmdHandlerMock .Setup(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith($"{RunFromPackageHandler.UnsquashFSExecutable} -f -d '{EnvironmentSettingNames.DefaultLocalSitePackagesPath}'")), MetricEventNames.LinuxContainerSpecializationUnsquash)).Returns((string.Empty, string.Empty, 0)); _meshServiceClientMock .Setup(m => m.CreateBindMount(EnvironmentSettingNames.DefaultLocalSitePackagesPath, TargetScriptPath)) .Returns(Task.FromResult(true)); } else { _bashCmdHandlerMock .Setup(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith($"{RunFromPackageHandler.UnsquashFSExecutable} -f -d '{TargetScriptPath}'")), MetricEventNames.LinuxContainerSpecializationUnsquash)).Returns((string.Empty, string.Empty, 0)); } var url = $"http://url/zip-file.{extension}"; var runFromPackageContext = new RunFromPackageContext(EnvironmentSettingNames.AzureWebsiteRunFromPackage, url, DefaultPackageLength, isWarmUpRequest); var applyBlobContextResult = await _runFromPackageHandler.ApplyBlobPackageContext(runFromPackageContext, TargetScriptPath, azureFilesMounted, true); Assert.True(applyBlobContextResult); handlerMock.Protected().Verify <Task <HttpResponseMessage> >("SendAsync", Times.Once(), ItExpr.Is <HttpRequestMessage>(r => IsZipDownloadRequest(r, url)), ItExpr.IsAny <CancellationToken>()); if (azureFilesMounted) { _bashCmdHandlerMock .Verify(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith( $"{RunFromPackageHandler.UnsquashFSExecutable} -f -d '{EnvironmentSettingNames.DefaultLocalSitePackagesPath}'")), MetricEventNames.LinuxContainerSpecializationUnsquash), Times.Once); _meshServiceClientMock .Verify(m => m.CreateBindMount(EnvironmentSettingNames.DefaultLocalSitePackagesPath, TargetScriptPath), Times.Once); } else { _bashCmdHandlerMock .Verify(b => b.RunBashCommand( It.Is <string>(s => s.StartsWith($"{RunFromPackageHandler.UnsquashFSExecutable} -f -d '{TargetScriptPath}'")), MetricEventNames.LinuxContainerSpecializationUnsquash), Times.Once); } }