private async Task <T> GetKeyObjectFromFile <T>(string name, IKeyJsonOps <T> keyOp) { string keyPath = GetFunctionSecretsFilePath(name); string key = null; if (!FileSystemHelpers.FileExists(keyPath) || FileSystemHelpers.FileInfoFromFileName(keyPath).Length == 0) { FileSystemHelpers.EnsureDirectory(Path.GetDirectoryName(keyPath)); try { using (var fileStream = FileSystemHelpers.OpenFile(keyPath, FileMode.Create, FileAccess.Write, FileShare.None)) // getting the lock early (instead of acquire the lock at "new StreamWriter(fileStream)") // so no redundant work is being done (generate secrets) { string jsonContent = keyOp.GenerateKeyJson(SecurityUtility.GenerateSecretStringsKeyPair(keyOp.NumberOfKeysInDefaultFormat), FunctionSiteExtensionVersion, out key); using (var sw = new StringWriter()) using (var sr = new System.IO.StringReader(jsonContent)) { // write json to memory // since JsonConvert has no method to format a json string new JsonTextWriter(sw) { Formatting = Formatting.Indented }.WriteToken(new JsonTextReader(sr)); using (var streamWriter = new StreamWriter(fileStream)) { await streamWriter.WriteAsync(sw.ToString()); await streamWriter.FlushAsync(); } } } return(keyOp.GenerateKeyObject(key, name)); } catch (IOException) { // failed to open file => function runtime has the handler // fallback to read key files } } string jsonStr = null; int timeOut = 5; while (true) { try { jsonStr = await FileSystemHelpers.ReadAllTextFromFileAsync(keyPath); break; } catch (Exception) { if (timeOut == 0) { throw new TimeoutException($"Fail to read {keyPath}, the file is being held by another process"); } timeOut--; await Task.Delay(250); } } bool isEncrypted; key = keyOp.GetKeyValueFromJson(jsonStr, out isEncrypted); if (isEncrypted) { key = SecurityUtility.DecryptSecretString(key); } return(keyOp.GenerateKeyObject(key, name)); }
private async Task OneDeployFetch(IRepository repository, DeploymentInfoBase deploymentInfo, string targetBranch, ILogger logger, ITracer tracer) { var artifactDeploymentInfo = (ArtifactDeploymentInfo)deploymentInfo; // For this kind of deployment, RepositoryUrl is a local path. var sourceZipFile = artifactDeploymentInfo.RepositoryUrl; // This is the path where the artifact being deployed is staged, before it is copied to the final target location var artifactDirectoryStagingPath = repository.RepositoryPath; var info = FileSystemHelpers.FileInfoFromFileName(sourceZipFile); var sizeInMb = (info.Length / (1024f * 1024f)).ToString("0.00", CultureInfo.InvariantCulture); var message = String.Format( CultureInfo.InvariantCulture, "Cleaning up temp folders from previous zip deployments and extracting pushed zip file {0} ({1} MB) to {2}", info.FullName, sizeInMb, artifactDirectoryStagingPath); using (tracer.Step(message)) { var targetInfo = FileSystemHelpers.DirectoryInfoFromDirectoryName(artifactDirectoryStagingPath); if (targetInfo.Exists) { // If the staging path already exists, rename it so we can delete it later var moveTarget = Path.Combine(targetInfo.Parent.FullName, Path.GetRandomFileName()); using (tracer.Step(string.Format("Renaming ({0}) to ({1})", targetInfo.FullName, moveTarget))) { targetInfo.MoveTo(moveTarget); } } // // We want to create a directory structure under 'extractTargetDirectory' // such that it exactly matches the directory structure specified // by deploymentInfo.TargetSubDirectoryRelativePath // string stagingSubDirPath = artifactDirectoryStagingPath; if (!string.IsNullOrWhiteSpace(artifactDeploymentInfo.TargetSubDirectoryRelativePath)) { stagingSubDirPath = Path.Combine(artifactDirectoryStagingPath, artifactDeploymentInfo.TargetSubDirectoryRelativePath); } // Create artifact staging directory hierarchy before later use Directory.CreateDirectory(stagingSubDirPath); var artifactFileStagingPath = Path.Combine(stagingSubDirPath, deploymentInfo.TargetFileName); var srcInfo = FileSystemHelpers.DirectoryInfoFromDirectoryName(deploymentInfo.RepositoryUrl); using (tracer.Step(string.Format("Moving {0} to {1}", targetInfo.FullName, artifactFileStagingPath))) { srcInfo.MoveTo(artifactFileStagingPath); } // Deletes all files and directories except for artifactFileStagingPath and artifactDirectoryStagingPath DeleteFilesAndDirsExcept(artifactFileStagingPath, artifactDirectoryStagingPath, tracer); // The deployment flow expects at least 1 commit in the IRepository commit, refer to CommitRepo() for more info CommitRepo(repository, artifactDeploymentInfo); } }
private async Task LocalZipFetch(IRepository repository, DeploymentInfoBase deploymentInfo, string targetBranch, ILogger logger, ITracer tracer) { var zipDeploymentInfo = (ArtifactDeploymentInfo)deploymentInfo; // If this was a request with a Zip URL in the JSON, we need to deploy the zip locally and get the path // Otherwise, for this kind of deployment, RepositoryUrl is a local path. var sourceZipFile = !string.IsNullOrEmpty(zipDeploymentInfo.RemoteURL) ? await DeployZipLocally(zipDeploymentInfo, tracer) : zipDeploymentInfo.RepositoryUrl; var artifactFileStagingDirectory = repository.RepositoryPath; var info = FileSystemHelpers.FileInfoFromFileName(sourceZipFile); var sizeInMb = (info.Length / (1024f * 1024f)).ToString("0.00", CultureInfo.InvariantCulture); var message = String.Format( CultureInfo.InvariantCulture, "Cleaning up temp folders from previous zip deployments and extracting pushed zip file {0} ({1} MB) to {2}", info.FullName, sizeInMb, artifactFileStagingDirectory); logger.Log(message); using (tracer.Step(message)) { // If extractTargetDirectory already exists, rename it so we can delete it concurrently with // the unzip (along with any other junk in the folder) var targetInfo = FileSystemHelpers.DirectoryInfoFromDirectoryName(artifactFileStagingDirectory); if (targetInfo.Exists) { var moveTarget = Path.Combine(targetInfo.Parent.FullName, Path.GetRandomFileName()); using (tracer.Step(string.Format("Renaming extractTargetDirectory({0}) to tempDirectory({1})", targetInfo.FullName, moveTarget))) { targetInfo.MoveTo(moveTarget); } } var cleanTask = Task.Run(() => DeleteFilesAndDirsExcept(sourceZipFile, artifactFileStagingDirectory, tracer)); var extractTask = Task.Run(() => { // // We want to create a directory structure under 'artifactFileStagingDirectory' // such that it exactly matches the directory structure specified // by deploymentInfo.TargetSubDirectoryRelativePath // string extractSubDirectoryPath = artifactFileStagingDirectory; if (!string.IsNullOrWhiteSpace(deploymentInfo.TargetSubDirectoryRelativePath) && deploymentInfo.Deployer == Constants.OneDeploy) { extractSubDirectoryPath = Path.Combine(artifactFileStagingDirectory, deploymentInfo.TargetSubDirectoryRelativePath); } FileSystemHelpers.CreateDirectory(extractSubDirectoryPath); using (var file = info.OpenRead()) using (var zip = new ZipArchive(file, ZipArchiveMode.Read)) { zip.Extract(extractSubDirectoryPath, tracer, _settings.GetZipDeployDoNotPreserveFileTime()); } }); await Task.WhenAll(cleanTask, extractTask); } CommitRepo(repository, zipDeploymentInfo); }
private Task LocalZipFetch(IRepository repository, DeploymentInfoBase deploymentInfo, string targetBranch, ILogger logger, ITracer tracer) { var zipDeploymentInfo = (ArtifactDeploymentInfo)deploymentInfo; // For this kind of deployment, RepositoryUrl is a local path. var sourceZipFile = zipDeploymentInfo.RepositoryUrl; var extractTargetDirectory = repository.RepositoryPath; var info = FileSystemHelpers.FileInfoFromFileName(sourceZipFile); var sizeInMb = (info.Length / (1024f * 1024f)).ToString("0.00", CultureInfo.InvariantCulture); var message = String.Format( CultureInfo.InvariantCulture, "Cleaning up temp folders from previous zip deployments and extracting pushed zip file {0} ({1} MB) to {2}", info.FullName, sizeInMb, extractTargetDirectory); logger.Log(message); using (tracer.Step(message)) { // If extractTargetDirectory already exists, rename it so we can delete it concurrently with // the unzip (along with any other junk in the folder) var targetInfo = FileSystemHelpers.DirectoryInfoFromDirectoryName(extractTargetDirectory); if (targetInfo.Exists) { var moveTarget = Path.Combine(targetInfo.Parent.FullName, Path.GetRandomFileName()); targetInfo.MoveTo(moveTarget); } DeleteFilesAndDirsExcept(sourceZipFile, extractTargetDirectory, tracer); // // We want to create a directory structure under 'extractTargetDirectory' // such that it exactly matches the directory structure specified // by deploymentInfo.TargetSubDirectoryRelativePath // string extractSubDirectoryPath = extractTargetDirectory; if (!string.IsNullOrWhiteSpace(deploymentInfo.TargetSubDirectoryRelativePath) && deploymentInfo.Deployer == Constants.OneDeploy) { extractSubDirectoryPath = Path.Combine(extractTargetDirectory, deploymentInfo.TargetSubDirectoryRelativePath); } FileSystemHelpers.CreateDirectory(extractSubDirectoryPath); using (var file = info.OpenRead()) using (var zip = new ZipArchive(file, ZipArchiveMode.Read)) { deploymentInfo.repositorySymlinks = zip.Extract(extractSubDirectoryPath, preserveSymlinks: ShouldPreserveSymlinks()); CreateZipSymlinks(deploymentInfo.repositorySymlinks, extractSubDirectoryPath); PermissionHelper.ChmodRecursive("777", extractSubDirectoryPath, tracer, TimeSpan.FromMinutes(1)); } } CommitRepo(repository, zipDeploymentInfo); return(Task.CompletedTask); }