internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, bool noBundler, string additionalPackages)
{
if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt)))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var externalPythonPackages = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(externalPythonPackages))
{
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages));
}
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
return(await InternalPreparePythonDeploymentInDocker(files, functionAppRoot, additionalPackages, noBundler));
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else
{
return(await InternalPreparePythonDeployment(files, functionAppRoot));
}
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, BuildOption buildOption, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match
// If build option is remote, we don't need to verify if packages are in sync, as we need to delete them regardless
if (buildOption != BuildOption.Remote && await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(packagesLocation);
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
// Only one of the remote build or build-native-deps flag can be chosen
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else if (buildOption == BuildOption.Remote)
{
// No-ops, python packages will be resolved on the server side
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// No need to generate and compare .md5 when using remote build
if (buildOption != BuildOption.Remote)
{
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
}
return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
private static async Task <Stream> InternalPreparePythonDeploymentInDocker(IEnumerable <string> files, string functionAppRoot, string additionalPackages)
{
var appContentPath = CopyToTemp(files, functionAppRoot);
await DockerHelpers.DockerPull(Constants.DockerImages.LinuxPythonImageAmd64);
var containerId = string.Empty;
try
{
containerId = await DockerHelpers.DockerRun(Constants.DockerImages.LinuxPythonImageAmd64);
await DockerHelpers.ExecInContainer(containerId, "mkdir -p /home/site/wwwroot/");
await DockerHelpers.CopyToContainer(containerId, $"{appContentPath}/.", "/home/site/wwwroot");
var scriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n"));
var bundleScriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(bundleScriptFilePath, (await StaticResources.PythonBundleScript).Replace("\r\n", "\n"));
if (!string.IsNullOrWhiteSpace(additionalPackages))
{
// Give the container time to start up
await Task.Delay(TimeSpan.FromSeconds(3));
await DockerHelpers.ExecInContainer(containerId, $"apt-get update");
await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}");
}
await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild);
await DockerHelpers.CopyToContainer(containerId, bundleScriptFilePath, Constants.StaticResourcesNames.PythonBundleScript);
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonBundleScript}");
await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}");
var tempDir = Path.Combine(Path.GetTempPath(), Path.GetTempFileName().Replace(".", ""));
FileSystemHelpers.EnsureDirectory(tempDir);
await DockerHelpers.CopyFromContainer(containerId, $"/app.zip", tempDir);
return(FileSystemHelpers.OpenFile(Path.Combine(tempDir, "app.zip"), FileMode.Open));
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await DockerHelpers.KillContainer(containerId, ignoreError : true);
}
}
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
return(await InternalPreparePythonDeployment(files, functionAppRoot));
}
else
{
throw new CliException("Docker is required to publish python function apps");
}
}
private static async Task <Stream> InternalPreparePythonDeployment(IEnumerable <string> files, string functionAppRoot)
{
if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt)))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var appContentPath = CopyToTemp(files, functionAppRoot);
await DockerHelpers.DockerPull(Constants.DockerImages.LinuxPythonImageAmd64);
var containerId = string.Empty;
try
{
containerId = await DockerHelpers.DockerRun(Constants.DockerImages.LinuxPythonImageAmd64);
await DockerHelpers.ExecInContainer(containerId, "mkdir -p /home/site/wwwroot/");
await DockerHelpers.CopyToContainer(containerId, $"{appContentPath}/.", "/home/site/wwwroot");
var scriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n"));
await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild);
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}");
var tempDir = Path.Combine(Path.GetTempPath(), Path.GetTempFileName().Replace(".", ""));
FileSystemHelpers.EnsureDirectory(tempDir);
await DockerHelpers.CopyFromContainer(containerId, $"/app.zip", tempDir);
return(FileSystemHelpers.OpenFile(Path.Combine(tempDir, "app.zip"), FileMode.Open));
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await DockerHelpers.KillContainer(containerId, ignoreError : true);
}
}
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match or a sync is forced
if (await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages));
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
private static async Task RestorePythonRequirementsDocker(string functionAppRoot, string packagesLocation, string additionalPackages)
{
var dockerImage = string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting))
? Constants.DockerImages.LinuxPythonImageAmd64
: Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting);
await DockerHelpers.DockerPull(dockerImage);
var containerId = string.Empty;
try
{
containerId = await DockerHelpers.DockerRun(dockerImage);
await DockerHelpers.CopyToContainer(containerId, Constants.RequirementsTxt, $"/{Constants.RequirementsTxt}");
var scriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n"));
if (!string.IsNullOrWhiteSpace(additionalPackages))
{
// Give the container time to start up
await Task.Delay(TimeSpan.FromSeconds(3));
await DockerHelpers.ExecInContainer(containerId, $"apt-get update");
await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}");
}
await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild);
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.CopyFromContainer(containerId, $"/{Constants.ExternalPythonPackages}/.", packagesLocation);
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await DockerHelpers.KillContainer(containerId, ignoreError : true);
}
}
}
public static async Task <Stream> ZipToSquashfsStream(Stream stream)
{
var tmpFile = Path.GetTempFileName();
using (stream)
using (var fileStream = FileSystemHelpers.OpenFile(tmpFile, FileMode.OpenOrCreate, FileAccess.Write))
{
stream.Seek(0, SeekOrigin.Begin);
await stream.CopyToAsync(fileStream);
}
string containerId = null;
try
{
string dockerImage = await ChoosePythonBuildEnvImage();
containerId = await DockerHelpers.DockerRun(dockerImage, command : "sleep infinity");
await DockerHelpers.CopyToContainer(containerId, tmpFile, $"/file.zip");
var scriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.ZipToSquashfsScript).Replace("\r\n", "\n"));
await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.ZipToSquashfs);
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.ZipToSquashfs}");
await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.ZipToSquashfs}");
await DockerHelpers.CopyFromContainer(containerId, $"/file.squashfs", tmpFile);
const int defaultBufferSize = 4096;
return(new FileStream(tmpFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, defaultBufferSize, FileOptions.DeleteOnClose));
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await DockerHelpers.KillContainer(containerId, ignoreError : true);
}
}
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps)
{
if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt)))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
return(await InternalPreparePythonDeploymentInDocker(files, functionAppRoot));
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else
{
return(await InternalPreparePythonDeployment(files, functionAppRoot));
}
}
private static async Task RestorePythonRequirementsDocker(string functionAppRoot, string packagesLocation, string additionalPackages)
{
// Configurable settings
var pythonDockerImageSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting);
var dockerSkipPullFlagSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerImageSkipPull);
var dockerRunSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerRunCommand);
string dockerImage = pythonDockerImageSetting;
if (string.IsNullOrEmpty(dockerImage))
{
dockerImage = await ChoosePythonBuildEnvImage();
}
if (string.IsNullOrEmpty(dockerSkipPullFlagSetting) ||
!(dockerSkipPullFlagSetting.Equals("true", StringComparison.OrdinalIgnoreCase) || dockerSkipPullFlagSetting == "1"))
{
await DockerHelpers.DockerPull(dockerImage);
}
var containerId = string.Empty;
try
{
if (string.IsNullOrEmpty(dockerRunSetting))
{
containerId = await DockerHelpers.DockerRun(dockerImage, command : "sleep infinity");
}
else
{
(var output, _, _) = await DockerHelpers.RunDockerCommand(dockerRunSetting);
containerId = output.ToString().Trim();
}
await DockerHelpers.CopyToContainer(containerId, Constants.RequirementsTxt, $"/{Constants.RequirementsTxt}");
var scriptFilePath = Path.GetTempFileName();
await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n"));
if (!string.IsNullOrWhiteSpace(additionalPackages))
{
// Give the container time to start up
await Task.Delay(TimeSpan.FromSeconds(3));
await DockerHelpers.ExecInContainer(containerId, $"apt-get update");
await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}");
}
await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild);
await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}");
await DockerHelpers.CopyFromContainer(containerId, $"/{Constants.ExternalPythonPackages}/.", packagesLocation);
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await DockerHelpers.KillContainer(containerId, ignoreError : true);
}
}
}