public async Task SpecializeMSISidecar_EmptyMSIEndpoint_NoOp()
{
var environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.MsiEndpoint, "" }
};
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
string error = await _instanceManager.SpecializeMSISidecar(assignmentContext);
Assert.Null(error);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs, p => Assert.StartsWith("MSI enabled status: False", p));
}
public async Task StartAssignment_Succeeds_With_No_RunFromPackage_AppSetting()
{
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>()
};
bool result = _instanceManager.StartAssignment(context);
Assert.True(result);
Assert.True(_scriptWebEnvironment.InStandbyMode);
await TestHelpers.Await(() => !_scriptWebEnvironment.InStandbyMode, timeout : 5000);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Starting Assignment", p),
p => Assert.StartsWith("Applying 0 app setting(s)", p),
p => Assert.StartsWith("Triggering specialization", p));
}
public async Task <bool> MountAzureFileShare(HostAssignmentContext assignmentContext)
{
try
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMountCifs))
{
var targetPath = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHomePath);
_logger.LogDebug($"Mounting {EnvironmentSettingNames.AzureFilesContentShare} at {targetPath}");
bool succeeded = await _meshServiceClient.MountCifs(assignmentContext.AzureFilesConnectionString,
assignmentContext.AzureFilesContentShare, targetPath);
_logger.LogInformation($"Mounted {EnvironmentSettingNames.AzureFilesContentShare} at {targetPath} Success = {succeeded}");
return(succeeded);
}
}
catch (Exception e)
{
_logger.LogWarning(e, nameof(MountAzureFileShare));
return(false);
}
}
private InstanceManager GetInstanceManagerForMSISpecialization(HostAssignmentContext hostAssignmentContext,
HttpStatusCode httpStatusCode, IMeshServiceClient meshServiceClient)
{
var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict);
var msiEndpoint = hostAssignmentContext.Environment[EnvironmentSettingNames.MsiEndpoint] + ScriptConstants.LinuxMSISpecializationStem;
handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync",
ItExpr.Is <HttpRequestMessage>(request => request.Method == HttpMethod.Post &&
request.RequestUri.AbsoluteUri.Equals(msiEndpoint) &&
request.Content != null),
ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage
{
StatusCode = httpStatusCode
});
InstanceManager.Reset();
return(new InstanceManager(_optionsFactory, new HttpClient(handlerMock.Object), _scriptWebEnvironment, _environment,
_loggerFactory.CreateLogger <InstanceManager>(), new TestMetricsLogger(), meshServiceClient));
}
public bool StartAssignment(HostAssignmentContext context)
{
if (!_webHostEnvironment.InStandbyMode)
{
_logger.LogError("Assign called while host is not in placeholder mode");
return(false);
}
if (_assignmentContext == null)
{
lock (_assignmentLock)
{
if (_assignmentContext != null)
{
return(_assignmentContext.Equals(context));
}
_assignmentContext = context;
}
_logger.LogInformation("Starting Assignment");
// set a flag which will cause any incoming http requests to buffer
// until specialization is complete
// the host is guaranteed not to receive any requests until AFTER assign
// has been initiated, so setting this flag here is sufficient to ensure
// that any subsequent incoming requests while the assign is in progress
// will be delayed until complete
_webHostEnvironment.DelayRequests();
// start the specialization process in the background
Task.Run(async() => await Assign(context));
return(true);
}
else
{
// No lock needed here since _assignmentContext is not null when we are here
return(_assignmentContext.Equals(context));
}
}
public async void StartAssignment_Succeeds_With_NonEmpty_ScmRunFromPackage_Blob()
{
var contentRoot = Path.Combine(Path.GetTempPath(), @"FunctionsTest");
var zipFilePath = Path.Combine(contentRoot, "content.zip");
await TestHelpers.CreateContentZip(contentRoot, zipFilePath, Path.Combine(@"TestScripts", "DotNet"));
IConfiguration configuration = TestHelpers.GetTestConfiguration();
string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
Uri sasUri = await TestHelpers.CreateBlobSas(connectionString, zipFilePath, "scm-run-from-pkg-test", "NonEmpty.zip");
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.ScmRunFromPackage, sasUri.ToString() }
}
};
bool result = _instanceManager.StartAssignment(context, isWarmup: false);
Assert.True(result);
Thread.Sleep(assignmentWaitPeriod);
Assert.False(_scriptWebEnvironment.InStandbyMode);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Starting Assignment", p),
p => Assert.StartsWith("Applying 1 app setting(s)", p),
p => Assert.StartsWith("Downloading zip contents from", p),
p => Assert.EndsWith(" bytes downloaded", p),
p => Assert.EndsWith(" bytes written", p),
p => Assert.StartsWith("Running: ", p),
p => Assert.StartsWith("Output:", p),
p => Assert.StartsWith("bash:", p),
p => Assert.StartsWith("exitCode:", p),
p => Assert.StartsWith("Triggering specialization", p));
}
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext)
{
_logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')");
string error = null;
HttpResponseMessage response = null;
try
{
var zipUrl = assignmentContext.ZipUrl;
if (!string.IsNullOrEmpty(zipUrl))
{
// make sure the zip uri is valid and accessible
await Utility.InvokeWithRetriesAsync(async() =>
{
try
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipHead))
{
var request = new HttpRequestMessage(HttpMethod.Head, zipUrl);
response = await _client.SendAsync(request);
response.EnsureSuccessStatusCode();
}
}
catch (Exception e)
{
_logger.LogError(e, $"{MetricEventNames.LinuxContainerSpecializationZipHead} failed");
throw;
}
}, maxRetries : 2, retryInterval : TimeSpan.FromSeconds(0.3)); // Keep this less than ~1s total
}
}
catch (Exception e)
{
error = $"Invalid zip url specified (StatusCode: {response?.StatusCode})";
_logger.LogError(e, "ValidateContext failed");
}
return(error);
}
private async Task Assign(HostAssignmentContext assignmentContext)
{
try
{
// first make all environment and file system changes required for
// the host to be specialized
assignmentContext.ApplyAppSettings();
}
catch (Exception)
{
throw;
}
finally
{
// all assignment settings/files have been applied so we can flip
// the switch now on specialization
// even if there are failures applying context above, we want to
// leave placeholder mode
_linuxConsumptionEnv.FlagAsSpecializedAndReady();
_linuxConsumptionEnv.ResumeRequests();
}
}
private async Task Assign(string encryptionKey)
{
// create a zip package
var contentRoot = Path.Combine(Path.GetTempPath(), @"FunctionsTest");
var sourcePath = Path.Combine(Directory.GetCurrentDirectory(), @"TestScripts\Node\HttpTrigger");
var zipFilePath = Path.Combine(contentRoot, "content.zip");
await TestHelpers.CreateContentZip(contentRoot, zipFilePath, @"TestScripts\Node\HttpTrigger");
// upload the blob and get a SAS uri
var configuration = _httpServer.Host.Services.GetService <IConfiguration>();
string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
var sasUri = await TestHelpers.CreateBlobSas(connectionString, zipFilePath, "azure-functions-test", "appcontents.zip");
// Now specialize the host by invoking assign
var secretManager = _httpServer.Host.Services.GetService <ISecretManagerProvider>().Current;
var masterKey = (await secretManager.GetHostSecretsAsync()).MasterKey;
string uri = "admin/instance/assign";
var request = new HttpRequestMessage(HttpMethod.Post, uri);
var environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, sasUri.ToString() },
{ RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName, "~2" },
{ RpcWorkerConstants.FunctionWorkerRuntimeSettingName, "node" }
};
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestApp",
Environment = environment
};
var encryptedAssignmentContext = EncryptedHostAssignmentContext.Create(assignmentContext, encryptionKey);
string json = JsonConvert.SerializeObject(encryptedAssignmentContext);
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, masterKey);
var response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
}
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext)
{
_logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')");
var zipUrl = assignmentContext.ZipUrl;
if (!string.IsNullOrEmpty(zipUrl))
{
// make sure the zip uri is valid and accessible
var request = new HttpRequestMessage(HttpMethod.Head, zipUrl);
var response = await _client.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
string error = $"Invalid zip url specified (StatusCode: {response.StatusCode})";
_logger.LogError(error);
return(error);
}
}
return(null);
}
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext, bool isWarmup)
{
if (isWarmup)
{
return(null);
}
_logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')");
RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
_logger.LogInformation($"Will be using {pkgContext.EnvironmentVariableName} app setting as zip url");
if (pkgContext.IsScmRunFromPackage())
{
// Not user assigned so limit validation
return(null);
}
else if (!string.IsNullOrEmpty(pkgContext.Url) && pkgContext.Url != "1")
{
// In AppService, ZipUrl == 1 means the package is hosted in azure files.
// Otherwise we expect zipUrl to be a blobUri to a zip or a squashfs image
(var error, var contentLength) = await ValidateBlobPackageContext(pkgContext.Url);
if (string.IsNullOrEmpty(error))
{
assignmentContext.PackageContentLength = contentLength;
}
return(error);
}
else if (!string.IsNullOrEmpty(assignmentContext.AzureFilesConnectionString))
{
return(await ValidateAzureFilesContext(assignmentContext.AzureFilesConnectionString, assignmentContext.AzureFilesContentShare));
}
else
{
_logger.LogError($"Missing ZipUrl and AzureFiles config. Continue with empty root.");
return(null);
}
}
public async Task ValidateContext_Succeeds()
{
var environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://microsoft.com" }
};
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
string error = await _instanceManager.ValidateContext(assignmentContext);
Assert.Null(error);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Validating host assignment context (SiteId: 1234, SiteName: 'TestSite')", p));
}
public async Task StartAssignment_Failure_ExitsPlaceholderMode()
{
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>
{
// force the assignment to fail
{ "throw", "test" }
}
};
bool result = _instanceManager.StartAssignment(context);
Assert.True(result);
Assert.True(_scriptWebEnvironment.InStandbyMode);
await TestHelpers.Await(() => !_scriptWebEnvironment.InStandbyMode, timeout : 5000);
var error = _loggerProvider.GetAllLogMessages().First(p => p.Level == LogLevel.Error);
Assert.Equal("Assign failed", error.FormattedMessage);
Assert.Equal("Kaboom!", error.Exception.Message);
}
private async Task ApplyContext(HostAssignmentContext assignmentContext)
{
_logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)");
assignmentContext.ApplyAppSettings(_environment);
// We need to get the non-PlaceholderMode script path so we can unzip to the correct location.
// This asks the factory to skip the PlaceholderMode check when configuring options.
var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder);
var zipPath = assignmentContext.ZipUrl;
if (!string.IsNullOrEmpty(zipPath))
{
// download zip and extract
var zipUri = new Uri(zipPath);
var filePath = Path.GetTempFileName();
await DownloadAsync(zipUri, filePath);
_logger.LogInformation($"Extracting files to '{options.ScriptPath}'");
ZipFile.ExtractToDirectory(filePath, options.ScriptPath, overwriteFiles: true);
_logger.LogInformation($"Zip extraction complete");
}
}
public async Task ValidateContext_InvalidZipUrl_ReturnsError()
{
var environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, "http://invalid.com/invalid/dne" }
};
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
string error = await _instanceManager.ValidateContext(assignmentContext);
Assert.Equal("Invalid zip url specified (StatusCode: NotFound)", error);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Validating host assignment context (SiteId: 1234, SiteName: 'TestSite')", p),
p => Assert.StartsWith("Invalid zip url specified (StatusCode: NotFound)", p));
}
public async Task DoesNotSpecializeMSISidecar_WhenMSIContextNull()
{
var environment = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.MsiEndpoint, "http://localhost:8081" },
{ EnvironmentSettingNames.MsiSecret, "secret" }
};
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment,
IsWarmupRequest = false,
MSIContext = null
};
var meshServiceClient = new Mock <IMeshServiceClient>(MockBehavior.Strict);
meshServiceClient.Setup(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
It.Is <Type>(t => t == typeof(InstanceManager)), "Could not specialize MSI sidecar")).Returns(Task.CompletedTask);
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.BadRequest, meshServiceClient.Object);
string error = await instanceManager.SpecializeMSISidecar(assignmentContext);
Assert.Null(error);
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("MSI enabled status: True", p),
p => Assert.StartsWith("Skipping specialization of MSI sidecar since MSIContext was absent", p));
meshServiceClient.Verify(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
It.Is <Type>(t => t == typeof(InstanceManager)), "Could not specialize MSI sidecar"), Times.Once);
}
public async Task <string> SpecializeMSISidecar(HostAssignmentContext context)
{
string endpoint;
var msiEnabled = context.IsMSIEnabled(out endpoint);
_logger.LogInformation($"MSI enabled status: {msiEnabled}");
if (msiEnabled)
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMSIInit))
{
var uri = new Uri(endpoint);
var address = $"http://{uri.Host}:{uri.Port}{ScriptConstants.LinuxMSISpecializationStem}";
_logger.LogDebug($"Specializing sidecar at {address}");
var requestMessage = new HttpRequestMessage(HttpMethod.Post, address)
{
Content = new StringContent(JsonConvert.SerializeObject(context.MSIContext),
Encoding.UTF8, "application/json")
};
var response = await _client.SendAsync(requestMessage);
_logger.LogInformation($"Specialize MSI sidecar returned {response.StatusCode}");
if (!response.IsSuccessStatusCode)
{
var message = $"Specialize MSI sidecar call failed. StatusCode={response.StatusCode}";
_logger.LogError(message);
return(message);
}
}
}
return(null);
}
public async Task MountsAzureFileShare(bool mountResult)
{
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHomePath, HomeDirectory);
var hostAssignmentContext = new HostAssignmentContext {
Environment = new Dictionary <string, string>()
};
const string connectionString = "connection-string";
hostAssignmentContext.Environment[EnvironmentSettingNames.AzureFilesConnectionString] = connectionString;
const string contentShare = "content-share";
hostAssignmentContext.Environment[EnvironmentSettingNames.AzureFilesContentShare] = contentShare;
_meshServiceClientMock.Setup(m => m.MountCifs(connectionString, contentShare, HomeDirectory))
.ReturnsAsync(mountResult);
var actualMountResult = await _runFromPackageHandler.MountAzureFileShare(hostAssignmentContext);
Assert.Equal(mountResult, actualMountResult);
_meshServiceClientMock.Verify(m => m.MountCifs(connectionString, contentShare, HomeDirectory), Times.Once);
}
public void Returns_BYOS_EnvironmentVariables()
{
var hostAssignmentContext = new HostAssignmentContext()
{
Environment = new Dictionary <string, string>
{
[EnvironmentSettingNames.MsiSecret] = "secret",
["AZUREFILESSTORAGE_storage1"] = "storage1",
["AzureFilesStorage_storage2"] = "storage2",
["AZUREBLOBSTORAGE_blob1"] = "blob1",
["AzureBlobStorage_blob2"] = "blob2",
[EnvironmentSettingNames.MsiEndpoint] = "endpoint",
}
};
var byosEnvironmentVariables = hostAssignmentContext.GetBYOSEnvironmentVariables();
Assert.Equal(4, byosEnvironmentVariables.Count());
Assert.Equal("storage1", byosEnvironmentVariables.First(env => env.Key == "AZUREFILESSTORAGE_storage1").Value);
Assert.Equal("storage2", byosEnvironmentVariables.First(env => env.Key == "AzureFilesStorage_storage2").Value);
Assert.Equal("blob1", byosEnvironmentVariables.First(env => env.Key == "AZUREBLOBSTORAGE_blob1").Value);
Assert.Equal("blob2", byosEnvironmentVariables.First(env => env.Key == "AzureBlobStorage_blob2").Value);
}
public void Does_Not_SetContext_AppliesHostAssignmentContext_For_Warmup_Request()
{
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>(),
SiteName = "TestSite",
Secrets = _secrets,
IsWarmupRequest = true
};
string json = JsonConvert.SerializeObject(context);
string encrypted = SimpleWebTokenHelper.Encrypt(json, environment: _environment);
var encryptedContext = new EncryptedHostAssignmentContext {
EncryptedContext = encrypted
};
var result = _startupContextProvider.SetContext(encryptedContext);
Assert.Equal(context.SiteName, result.SiteName);
Assert.Equal(_secrets.Host.Master, result.Secrets.Host.Master);
var secrets = _startupContextProvider.GetHostSecretsOrNull();
Assert.Null(secrets);
}
public async Task StartAssignment_AppliesAssignmentContext()
{
var loggerFactory = MockNullLogerFactory.CreateLoggerFactory();
var settingsManager = new ScriptSettingsManager();
var instanceManager = new InstanceManager(settingsManager, null, loggerFactory, null);
var envValue = new
{
Name = Path.GetTempFileName().Replace(".", string.Empty),
Value = Guid.NewGuid().ToString()
};
settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
WebScriptHostManager.ResetStandbyMode();
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>
{
{ envValue.Name, envValue.Value }
}
};
bool result = instanceManager.StartAssignment(context);
Assert.True(result);
// specialization is done in the background
await Task.Delay(500);
var value = Environment.GetEnvironmentVariable(envValue.Name);
Assert.Equal(value, envValue.Value);
// calling again should return false, since we're no longer
// in placeholder mode
result = instanceManager.StartAssignment(context);
Assert.False(result);
}
public async Task Assignment_Fails_Without_Encryption_Key()
{
var environment = new TestEnvironment();
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var scriptWebEnvironment = new ScriptWebHostEnvironment(environment);
var loggerFactory = new LoggerFactory();
var loggerProvider = new TestLoggerProvider();
loggerFactory.AddProvider(loggerProvider);
var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict);
handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync",
ItExpr.IsAny <HttpRequestMessage>(),
ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage
{
StatusCode = HttpStatusCode.OK
});
var instanceManager = new InstanceManager(_optionsFactory, TestHelpers.CreateHttpClientFactory(handlerMock.Object),
scriptWebEnvironment, environment, loggerFactory.CreateLogger <InstanceManager>(),
new TestMetricsLogger(), null, new Mock <IRunFromPackageHandler>().Object,
new Mock <IPackageDownloadHandler>(MockBehavior.Strict).Object);
var startupContextProvider = new StartupContextProvider(environment, loggerFactory.CreateLogger <StartupContextProvider>());
InstanceManager.Reset();
const string podEncryptionKey = "/a/vXvWJ3Hzgx4PFxlDUJJhQm5QVyGiu0NNLFm/ZMMg=";
var podController = new KubernetesPodController(environment, instanceManager, loggerFactory, startupContextProvider);
var hostAssignmentContext = new HostAssignmentContext
{
Environment = new Dictionary <string, string>()
{
[EnvironmentSettingNames.AzureWebsiteRunFromPackage] = "http://localhost:1234"
}
};
hostAssignmentContext.Secrets = new FunctionAppSecrets();
hostAssignmentContext.IsWarmupRequest = false;
var encryptedHostAssignmentValue = SimpleWebTokenHelper.Encrypt(JsonConvert.SerializeObject(hostAssignmentContext), podEncryptionKey.ToKeyBytes());
var encryptedHostAssignmentContext = new EncryptedHostAssignmentContext()
{
EncryptedContext = encryptedHostAssignmentValue
};
environment.SetEnvironmentVariable(EnvironmentSettingNames.KubernetesServiceHost, "http://localhost:80");
environment.SetEnvironmentVariable(EnvironmentSettingNames.PodNamespace, "k8se-apps");
var ex = await Assert.ThrowsAsync <System.Exception>(async() =>
{
await(podController.Assign(encryptedHostAssignmentContext));
});
Assert.Null(startupContextProvider.Context);
}
// for testing
internal static void Reset()
{
_assignmentContext = null;
}
public async Task StandbyMode_EndToEnd_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.ContainerName, "TestContainer" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
var httpConfig = new HttpConfiguration();
var testRootPath = Path.Combine(Path.GetTempPath(), "StandbyModeTest_Linux");
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var webHostSettings = new WebHostSettings
{
IsSelfHost = true,
LogPath = Path.Combine(testRootPath, "Logs"),
SecretsPath = Path.Combine(testRootPath, "Secrets"),
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
};
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var webHostBuilder = Program.CreateWebHostBuilder()
.ConfigureServices(c =>
{
c.AddSingleton(webHostSettings)
.AddSingleton <ILoggerProviderFactory>(loggerProviderFactory)
.AddSingleton <ILoggerFactory>(loggerFactory);
});
var httpServer = new TestServer(webHostBuilder);
var httpClient = httpServer.CreateClient();
httpClient.BaseAddress = new Uri("https://localhost/");
TestHelpers.WaitForWebHost(httpClient);
var traces = loggerProvider.GetAllLogMessages().ToArray();
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
// issue warmup request and verify
var request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
var response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
string responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// issue warmup request with restart and verify
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup?restart=1");
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// Now specialize the host by invoking assign
var secretManager = httpServer.Host.Services.GetService <ISecretManager>();
var masterKey = (await secretManager.GetHostSecretsAsync()).MasterKey;
string uri = "admin/instance/assign";
request = new HttpRequestMessage(HttpMethod.Post, uri);
var environment = new Dictionary <string, string>();
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
var encryptedAssignmentContext = EncryptedHostAssignmentContext.Create(assignmentContext, encryptionKey);
string json = JsonConvert.SerializeObject(encryptedAssignmentContext);
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, masterKey);
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
// give time for the specialization to happen
string[] logLines = null;
await TestHelpers.Await(() =>
{
// wait for the trace indicating that the host has been specialized
logLines = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
return(logLines.Contains("Generating 0 job function(s)"));
}, userMessageCallback : () => string.Join(Environment.NewLine, loggerProvider.GetAllLogMessages().Select(p => $"[{p.Timestamp.ToString("HH:mm:ss.fff")}] {p.FormattedMessage}")));
httpServer.Dispose();
httpClient.Dispose();
await Task.Delay(2000);
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(webHostSettings, true);
var expectedHostId = hostConfig.HostConfig.HostId;
// verify the rest of the expected logs
string text = string.Join(Environment.NewLine, logLines);
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={expectedHostId}")));
Assert.Contains("Generating 0 job function(s)", logLines);
WebScriptHostManager.ResetStandbyMode();
}
}
public async Task Assign_MSISpecializationFailure_ReturnsError()
{
var environment = new TestEnvironment();
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var scriptWebEnvironment = new ScriptWebHostEnvironment(environment);
var loggerFactory = new LoggerFactory();
var loggerProvider = new TestLoggerProvider();
loggerFactory.AddProvider(loggerProvider);
var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict);
handlerMock.Protected().Setup <Task <HttpResponseMessage> >("SendAsync",
ItExpr.IsAny <HttpRequestMessage>(),
ItExpr.IsAny <CancellationToken>()).ReturnsAsync(new HttpResponseMessage
{
StatusCode = HttpStatusCode.BadRequest
});
var meshServiceClient = new Mock <IMeshServiceClient>(MockBehavior.Strict);
meshServiceClient.Setup(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
It.Is <Type>(t => t == typeof(InstanceManager)), "Failed to specialize MSI sidecar")).Returns(Task.CompletedTask);
var instanceManager = new InstanceManager(_optionsFactory, TestHelpers.CreateHttpClientFactory(handlerMock.Object),
scriptWebEnvironment, environment, loggerFactory.CreateLogger <InstanceManager>(),
new TestMetricsLogger(), meshServiceClient.Object, _runFromPackageHandler.Object, new Mock <IPackageDownloadHandler>(MockBehavior.Strict).Object);
var startupContextProvider = new StartupContextProvider(environment, loggerFactory.CreateLogger <StartupContextProvider>());
InstanceManager.Reset();
var instanceController = new InstanceController(environment, instanceManager, loggerFactory, startupContextProvider);
const string containerEncryptionKey = "/a/vXvWJ3Hzgx4PFxlDUJJhQm5QVyGiu0NNLFm/ZMMg=";
var hostAssignmentContext = new HostAssignmentContext
{
Environment = new Dictionary <string, string>(),
MSIContext = new MSIContext()
};
hostAssignmentContext.Environment[EnvironmentSettingNames.MsiEndpoint] = "http://localhost:8081";
hostAssignmentContext.Environment[EnvironmentSettingNames.MsiSecret] = "secret";
var encryptedHostAssignmentValue = SimpleWebTokenHelper.Encrypt(JsonConvert.SerializeObject(hostAssignmentContext), containerEncryptionKey.ToKeyBytes());
var encryptedHostAssignmentContext = new EncryptedHostAssignmentContext()
{
EncryptedContext = encryptedHostAssignmentValue
};
environment.SetEnvironmentVariable(EnvironmentSettingNames.ContainerEncryptionKey, containerEncryptionKey);
IActionResult result = await instanceController.Assign(encryptedHostAssignmentContext);
var objectResult = result as ObjectResult;
Assert.Equal(objectResult.StatusCode, 500);
Assert.Equal(objectResult.Value, "Specialize MSI sidecar call failed. StatusCode=BadRequest");
meshServiceClient.Verify(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
It.Is <Type>(t => t == typeof(InstanceManager)), "Failed to specialize MSI sidecar"), Times.Once);
}
public async Task StartAssignment_AppliesAssignmentContext()
{
var envValue = new
{
Name = Path.GetTempFileName().Replace(".", string.Empty),
Value = Guid.NewGuid().ToString()
};
var allowedOrigins = new string[]
{
"https://functions.azure.com",
"https://functions-staging.azure.com",
"https://functions-next.azure.com"
};
var supportCredentials = true;
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
var context = new HostAssignmentContext
{
Environment = new Dictionary <string, string>
{
{ envValue.Name, envValue.Value }
},
CorsSettings = new CorsSettings
{
AllowedOrigins = allowedOrigins,
SupportCredentials = supportCredentials,
},
IsWarmupRequest = false
};
bool result = _instanceManager.StartAssignment(context);
Assert.True(result);
Assert.True(_scriptWebEnvironment.InStandbyMode);
// specialization is done in the background
await Task.Delay(500);
var value = _environment.GetEnvironmentVariable(envValue.Name);
Assert.Equal(value, envValue.Value);
var supportCredentialsValue = _environment.GetEnvironmentVariable(EnvironmentSettingNames.CorsSupportCredentials);
Assert.Equal(supportCredentialsValue, supportCredentials.ToString());
var allowedOriginsValue = _environment.GetEnvironmentVariable(EnvironmentSettingNames.CorsAllowedOrigins);
Assert.Equal(allowedOriginsValue, JsonConvert.SerializeObject(allowedOrigins));
// verify logs
var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Starting Assignment", p),
p => Assert.StartsWith("Applying 1 app setting(s)", p),
p => Assert.StartsWith("Triggering specialization", p));
// calling again should return false, since we're no longer
// in placeholder mode
_loggerProvider.ClearAllLogMessages();
result = _instanceManager.StartAssignment(context);
Assert.False(result);
logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
Assert.Collection(logs,
p => Assert.StartsWith("Assign called while host is not in placeholder mode", p));
}
public async Task Mounts_Valid_BYOS_Accounts()
{
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
const string account1 = "storageaccount1";
const string share1 = "share1";
const string accessKey1 = "key1key1key1==";
const string targetPath1 = "/data";
const string account2 = "storageaccount2";
const string share2 = "share2";
const string accessKey2 = "key2key2key2==";
const string targetPath2 = "/data/store2";
const string account3 = "storageaccount3";
const string share3 = "share3";
const string accessKey3 = "key3key3key3==";
const string targetPath3 = "/somepath";
var hostAssignmentContext = new HostAssignmentContext()
{
Environment = new Dictionary <string, string>
{
[EnvironmentSettingNames.MsiSecret] = "secret",
["AZUREFILESSTORAGE_storage1"] = $"{account1}|{share1}|{accessKey1}|{targetPath1}",
["AZUREFILESSTORAGE_storage2"] = $"{account2}|{share2}|{accessKey2}|{targetPath2}",
["AZUREBLOBSTORAGE_blob1"] = $"{account3}|{share3}|{accessKey3}|{targetPath3}",
[EnvironmentSettingNames.MsiEndpoint] = "endpoint",
},
SiteId = 1234,
SiteName = "TestSite",
IsWarmupRequest = false
};
var meshInitServiceClient = new Mock <IMeshServiceClient>(MockBehavior.Strict);
meshInitServiceClient.Setup(client =>
client.MountCifs(Utility.BuildStorageConnectionString(account1, accessKey1, CloudConstants.AzureStorageSuffix), share1, targetPath1))
.Throws(new Exception("Mount failure"));
meshInitServiceClient.Setup(client =>
client.MountCifs(Utility.BuildStorageConnectionString(account2, accessKey2, CloudConstants.AzureStorageSuffix), share2, targetPath2)).Returns(Task.FromResult(true));
meshInitServiceClient.Setup(client =>
client.MountBlob(Utility.BuildStorageConnectionString(account3, accessKey3, CloudConstants.AzureStorageSuffix), share3, targetPath3)).Returns(Task.FromResult(true));
var instanceManager = new InstanceManager(_optionsFactory, _httpClient, _scriptWebEnvironment, _environment,
_loggerFactory.CreateLogger <InstanceManager>(), new TestMetricsLogger(), meshInitServiceClient.Object);
instanceManager.StartAssignment(hostAssignmentContext);
await TestHelpers.Await(() => !_scriptWebEnvironment.InStandbyMode, timeout : 5000);
meshInitServiceClient.Verify(
client => client.MountCifs(Utility.BuildStorageConnectionString(account1, accessKey1, CloudConstants.AzureStorageSuffix), share1,
targetPath1), Times.Exactly(2));
meshInitServiceClient.Verify(
client => client.MountCifs(Utility.BuildStorageConnectionString(account2, accessKey2, CloudConstants.AzureStorageSuffix), share2,
targetPath2), Times.Once);
meshInitServiceClient.Verify(
client => client.MountBlob(Utility.BuildStorageConnectionString(account3, accessKey3, CloudConstants.AzureStorageSuffix), share3,
targetPath3), Times.Once);
}
private async Task ApplyContext(HostAssignmentContext assignmentContext)
{
_logger.LogInformation($"Applying {assignmentContext.Environment.Count} app setting(s)");
assignmentContext.ApplyAppSettings(_environment, _logger);
// We need to get the non-PlaceholderMode script Path so we can unzip to the correct location.
// This asks the factory to skip the PlaceholderMode check when configuring options.
var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder);
RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
if (_environment.SupportsAzureFileShareMount())
{
var azureFilesMounted = false;
if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
{
azureFilesMounted = await _runFromPackageHandler.MountAzureFileShare(assignmentContext);
}
else
{
_logger.LogError(
$"No {nameof(EnvironmentSettingNames.AzureFilesConnectionString)} or {nameof(EnvironmentSettingNames.AzureFilesContentShare)} configured. Azure FileShare will not be mounted. For PowerShell Functions, Managed Dependencies will not persisted across functions host instances.");
}
if (pkgContext.IsRunFromPackage(options, _logger))
{
if (azureFilesMounted)
{
_logger.LogWarning("App is configured to use both Run-From-Package and AzureFiles. Run-From-Package will take precedence");
}
var blobContextApplied =
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath,
azureFilesMounted, false);
if (!blobContextApplied && azureFilesMounted)
{
_logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyBlobPackageContext)}. Attempting to use local disk instead");
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
}
}
else
{
_logger.LogInformation($"No {nameof(EnvironmentSettingNames.AzureWebsiteRunFromPackage)} configured");
}
}
else
{
if (pkgContext.IsRunFromPackage(options, _logger))
{
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
}
else if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
{
await _runFromPackageHandler.MountAzureFileShare(assignmentContext);
}
}
// BYOS
var storageVolumes = assignmentContext.GetBYOSEnvironmentVariables()
.Select(AzureStorageInfoValue.FromEnvironmentVariable).ToList();
var mountedVolumes =
(await Task.WhenAll(storageVolumes.Where(v => v != null).Select(MountStorageAccount))).Where(
result => result).ToList();
if (storageVolumes.Any())
{
if (mountedVolumes.Count != storageVolumes.Count)
{
_logger.LogWarning(
$"Successfully mounted {mountedVolumes.Count} / {storageVolumes.Count} BYOS storage accounts");
}
else
{
_logger.LogInformation(
$"Successfully mounted {storageVolumes.Count} BYOS storage accounts");
}
}
}
