private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory) { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); Directory.CreateDirectory(tempDirectory); #if OS_WINDOWS string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); #else string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif string link = actionDownloadDetails.ArchiveLink; Trace.Info($"Save archive '{link}' into {archiveFile}."); try { int retryCount = 0; // Allow up to 20 * 60s for any action to be downloaded from github graph. int timeoutSeconds = 20 * 60; while (retryCount < 3) { using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) { try { //open zip stream in async mode using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(link)) { if (response.IsSuccessStatusCode) { using (var result = await response.Content.ReadAsStreamAsync()) { await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); await fs.FlushAsync(actionDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } else if (response.StatusCode == HttpStatusCode.NotFound) { // It doesn't make sense to retry in this case, so just stop throw new ActionNotFoundException(new Uri(link)); } else { // Something else bad happened, let's go to our retry logic response.EnsureSuccessStatusCode(); } } } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info("Action download has been cancelled."); throw; } catch (ActionNotFoundException) { Trace.Info($"The action at '{link}' does not exist"); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}"); Trace.Error(ex); if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds."); } else { executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}"); } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); executionContext.Debug($"Download '{link}' to '{archiveFile}'"); var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); #if OS_WINDOWS ZipFile.ExtractToDirectory(archiveFile, stagingDirectory); #else string tar = WhichUtil.Which("tar", require: true, trace: Trace); // tar -xzf using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Info(args.Data); } }); processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Error(args.Data); } }); int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken); if (exitCode != 0) { throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}."); } } #endif // repository archive from github always contains a nested folder var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories(); if (subDirectories.Length != 1) { throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories"); } else { executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'"); IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken); } Trace.Verbose("Create watermark file indicate action download succeed."); string watermarkFile = GetWatermarkFilePath(destDirectory); File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); Trace.Info("Finished getting action repository."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting action temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); } } }
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference; ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) { Trace.Info($"Repository action is in 'self' repository."); return; } if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase)) { throw new NotSupportedException(repositoryReference.RepositoryType); } ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name)); ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref)); string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref); string watermarkFile = GetWatermarkFilePath(destDirectory); if (File.Exists(watermarkFile)) { executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'."); return; } else { // make sure we get a clean folder ready to use. IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); Directory.CreateDirectory(destDirectory); executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'"); } var configurationStore = HostContext.GetService <IConfigurationStore>(); var isHostedServer = configurationStore.GetSettings().IsHostedServer; if (isHostedServer) { string apiUrl = GetApiUrl(executionContext); string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext); await DownloadRepositoryActionAsync(executionContext, downloadDetails, destDirectory); return; } else { string apiUrl = GetApiUrl(executionContext); // URLs to try: var downloadAttempts = new List <ActionDownloadDetails> { // A built-in action or an action the user has created, on their GHES instance // Example: https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1 new ActionDownloadDetails( BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), ConfigureAuthorizationFromContext), // The same action, on GitHub.com // Example: https://api.github.com/repos/my-org/my-action/tarball/v1 new ActionDownloadDetails( BuildLinkToActionArchive(_dotcomApiUrl, repositoryReference.Name, repositoryReference.Ref), configureAuthorization: (e, h) => { /* no authorization for dotcom */ }) }; foreach (var downloadAttempt in downloadAttempts) { Trace.Info($"Download archive '{downloadAttempt.ArchiveLink}' to '{destDirectory}'."); try { await DownloadRepositoryActionAsync(executionContext, downloadAttempt, destDirectory); return; } catch (ActionNotFoundException) { Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {downloadAttempt.ArchiveLink}"); continue; } } throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", downloadAttempts.Select(d => d.ArchiveLink))}"); } }