internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, bool noBundler, string additionalPackages) { if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt))) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var externalPythonPackages = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages); if (FileSystemHelpers.DirectoryExists(externalPythonPackages)) { ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory"); FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages)); } if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { return(await InternalPreparePythonDeploymentInDocker(files, functionAppRoot, additionalPackages, noBundler)); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else { return(await InternalPreparePythonDeployment(files, functionAppRoot)); } }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, BuildOption buildOption, string additionalPackages) { var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt); if (!FileSystemHelpers.FileExists(reqTxtFile)) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages); if (FileSystemHelpers.DirectoryExists(packagesLocation)) { // Only update packages if checksum of requirements.txt does not match // If build option is remote, we don't need to verify if packages are in sync, as we need to delete them regardless if (buildOption != BuildOption.Remote && await ArePackagesInSync(reqTxtFile, packagesLocation)) { ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies...")); return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); } ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory"); FileSystemHelpers.DeleteDirectorySafe(packagesLocation); } FileSystemHelpers.EnsureDirectory(packagesLocation); // Only one of the remote build or build-native-deps flag can be chosen if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else if (buildOption == BuildOption.Remote) { // No-ops, python packages will be resolved on the server side } else { await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation); } // No need to generate and compare .md5 when using remote build if (buildOption != BuildOption.Remote) { // Store a checksum of requirements.txt var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5"); await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile)); } return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); }
private static async Task <Stream> InternalPreparePythonDeploymentInDocker(IEnumerable <string> files, string functionAppRoot, string additionalPackages) { var appContentPath = CopyToTemp(files, functionAppRoot); await DockerHelpers.DockerPull(Constants.DockerImages.LinuxPythonImageAmd64); var containerId = string.Empty; try { containerId = await DockerHelpers.DockerRun(Constants.DockerImages.LinuxPythonImageAmd64); await DockerHelpers.ExecInContainer(containerId, "mkdir -p /home/site/wwwroot/"); await DockerHelpers.CopyToContainer(containerId, $"{appContentPath}/.", "/home/site/wwwroot"); var scriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n")); var bundleScriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(bundleScriptFilePath, (await StaticResources.PythonBundleScript).Replace("\r\n", "\n")); if (!string.IsNullOrWhiteSpace(additionalPackages)) { // Give the container time to start up await Task.Delay(TimeSpan.FromSeconds(3)); await DockerHelpers.ExecInContainer(containerId, $"apt-get update"); await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}"); } await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild); await DockerHelpers.CopyToContainer(containerId, bundleScriptFilePath, Constants.StaticResourcesNames.PythonBundleScript); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonBundleScript}"); await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}"); var tempDir = Path.Combine(Path.GetTempPath(), Path.GetTempFileName().Replace(".", "")); FileSystemHelpers.EnsureDirectory(tempDir); await DockerHelpers.CopyFromContainer(containerId, $"/app.zip", tempDir); return(FileSystemHelpers.OpenFile(Path.Combine(tempDir, "app.zip"), FileMode.Open)); } finally { if (!string.IsNullOrEmpty(containerId)) { await DockerHelpers.KillContainer(containerId, ignoreError : true); } } }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { return(await InternalPreparePythonDeployment(files, functionAppRoot)); } else { throw new CliException("Docker is required to publish python function apps"); } }
private static async Task <Stream> InternalPreparePythonDeployment(IEnumerable <string> files, string functionAppRoot) { if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt))) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var appContentPath = CopyToTemp(files, functionAppRoot); await DockerHelpers.DockerPull(Constants.DockerImages.LinuxPythonImageAmd64); var containerId = string.Empty; try { containerId = await DockerHelpers.DockerRun(Constants.DockerImages.LinuxPythonImageAmd64); await DockerHelpers.ExecInContainer(containerId, "mkdir -p /home/site/wwwroot/"); await DockerHelpers.CopyToContainer(containerId, $"{appContentPath}/.", "/home/site/wwwroot"); var scriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n")); await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}"); var tempDir = Path.Combine(Path.GetTempPath(), Path.GetTempFileName().Replace(".", "")); FileSystemHelpers.EnsureDirectory(tempDir); await DockerHelpers.CopyFromContainer(containerId, $"/app.zip", tempDir); return(FileSystemHelpers.OpenFile(Path.Combine(tempDir, "app.zip"), FileMode.Open)); } finally { if (!string.IsNullOrEmpty(containerId)) { await DockerHelpers.KillContainer(containerId, ignoreError : true); } } }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, string additionalPackages) { var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt); if (!FileSystemHelpers.FileExists(reqTxtFile)) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages); if (FileSystemHelpers.DirectoryExists(packagesLocation)) { // Only update packages if checksum of requirements.txt does not match or a sync is forced if (await ArePackagesInSync(reqTxtFile, packagesLocation)) { ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies...")); return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); } ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory"); FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages)); } FileSystemHelpers.EnsureDirectory(packagesLocation); if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else { await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation); } // Store a checksum of requirements.txt var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5"); await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile)); return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); }
private static async Task RestorePythonRequirementsDocker(string functionAppRoot, string packagesLocation, string additionalPackages) { var dockerImage = string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting)) ? Constants.DockerImages.LinuxPythonImageAmd64 : Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting); await DockerHelpers.DockerPull(dockerImage); var containerId = string.Empty; try { containerId = await DockerHelpers.DockerRun(dockerImage); await DockerHelpers.CopyToContainer(containerId, Constants.RequirementsTxt, $"/{Constants.RequirementsTxt}"); var scriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n")); if (!string.IsNullOrWhiteSpace(additionalPackages)) { // Give the container time to start up await Task.Delay(TimeSpan.FromSeconds(3)); await DockerHelpers.ExecInContainer(containerId, $"apt-get update"); await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}"); } await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.CopyFromContainer(containerId, $"/{Constants.ExternalPythonPackages}/.", packagesLocation); } finally { if (!string.IsNullOrEmpty(containerId)) { await DockerHelpers.KillContainer(containerId, ignoreError : true); } } }
public static async Task <Stream> ZipToSquashfsStream(Stream stream) { var tmpFile = Path.GetTempFileName(); using (stream) using (var fileStream = FileSystemHelpers.OpenFile(tmpFile, FileMode.OpenOrCreate, FileAccess.Write)) { stream.Seek(0, SeekOrigin.Begin); await stream.CopyToAsync(fileStream); } string containerId = null; try { string dockerImage = await ChoosePythonBuildEnvImage(); containerId = await DockerHelpers.DockerRun(dockerImage, command : "sleep infinity"); await DockerHelpers.CopyToContainer(containerId, tmpFile, $"/file.zip"); var scriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.ZipToSquashfsScript).Replace("\r\n", "\n")); await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.ZipToSquashfs); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.ZipToSquashfs}"); await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.ZipToSquashfs}"); await DockerHelpers.CopyFromContainer(containerId, $"/file.squashfs", tmpFile); const int defaultBufferSize = 4096; return(new FileStream(tmpFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, defaultBufferSize, FileOptions.DeleteOnClose)); } finally { if (!string.IsNullOrEmpty(containerId)) { await DockerHelpers.KillContainer(containerId, ignoreError : true); } } }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps) { if (!FileSystemHelpers.FileExists(Path.Combine(functionAppRoot, Constants.RequirementsTxt))) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { return(await InternalPreparePythonDeploymentInDocker(files, functionAppRoot)); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else { return(await InternalPreparePythonDeployment(files, functionAppRoot)); } }
private static async Task RestorePythonRequirementsDocker(string functionAppRoot, string packagesLocation, string additionalPackages) { // Configurable settings var pythonDockerImageSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerImageVersionSetting); var dockerSkipPullFlagSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerImageSkipPull); var dockerRunSetting = Environment.GetEnvironmentVariable(Constants.PythonDockerRunCommand); string dockerImage = pythonDockerImageSetting; if (string.IsNullOrEmpty(dockerImage)) { dockerImage = await ChoosePythonBuildEnvImage(); } if (string.IsNullOrEmpty(dockerSkipPullFlagSetting) || !(dockerSkipPullFlagSetting.Equals("true", StringComparison.OrdinalIgnoreCase) || dockerSkipPullFlagSetting == "1")) { await DockerHelpers.DockerPull(dockerImage); } var containerId = string.Empty; try { if (string.IsNullOrEmpty(dockerRunSetting)) { containerId = await DockerHelpers.DockerRun(dockerImage, command : "sleep infinity"); } else { (var output, _, _) = await DockerHelpers.RunDockerCommand(dockerRunSetting); containerId = output.ToString().Trim(); } await DockerHelpers.CopyToContainer(containerId, Constants.RequirementsTxt, $"/{Constants.RequirementsTxt}"); var scriptFilePath = Path.GetTempFileName(); await FileSystemHelpers.WriteAllTextToFileAsync(scriptFilePath, (await StaticResources.PythonDockerBuildScript).Replace("\r\n", "\n")); if (!string.IsNullOrWhiteSpace(additionalPackages)) { // Give the container time to start up await Task.Delay(TimeSpan.FromSeconds(3)); await DockerHelpers.ExecInContainer(containerId, $"apt-get update"); await DockerHelpers.ExecInContainer(containerId, $"apt-get install -y {additionalPackages}"); } await DockerHelpers.CopyToContainer(containerId, scriptFilePath, Constants.StaticResourcesNames.PythonDockerBuild); await DockerHelpers.ExecInContainer(containerId, $"chmod +x /{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.ExecInContainer(containerId, $"/{Constants.StaticResourcesNames.PythonDockerBuild}"); await DockerHelpers.CopyFromContainer(containerId, $"/{Constants.ExternalPythonPackages}/.", packagesLocation); } finally { if (!string.IsNullOrEmpty(containerId)) { await DockerHelpers.KillContainer(containerId, ignoreError : true); } } }