public async Task RunAsync(TransferReport transferReport) { if (!_importConfiguration.Enabled) { return; } // // Validate importPipeline if (_importConfiguration.Blobs.Count > 0) { var importPipeline = await _transferClient.GetImportPipelineAsync(_importConfiguration.ImportPipelineName).ConfigureAwait(false); if (!importPipeline.ProvisioningState.Equals("succeeded", StringComparison.OrdinalIgnoreCase)) { throw new Exception($"ImportPipeline:{_importConfiguration.ImportPipelineName} is in non-success provisioning state {importPipeline.ProvisioningState}."); } } var importJobs = await CreateImportJobs().ConfigureAwait(false); await importJobs.ThrottledWhenAll( async (job) => await ExecuteAsync(job).ConfigureAwait(false), _importConfiguration.MaxConcurrency).ConfigureAwait(false); foreach (var job in importJobs) { if (job.Status == TransferJobStatus.Succeeded) { transferReport.ImportArtifacts.Succeeded.AddRange(job.Images); if (job.SourceType == ImportSourceType.AzureStorageBlob) { transferReport.ImportBlobs.Succeeded.Add(job.ImportBlobName); } } else if (job.Status == TransferJobStatus.Failed) { transferReport.ImportArtifacts.Failed.AddRange(job.Images); if (job.SourceType == ImportSourceType.AzureStorageBlob) { transferReport.ImportBlobs.Failed.Add(job.ImportBlobName); } } } }
public async Task RunAsync(TransferReport transferReport) { if (!_exportConfiguration.Enabled) { return; } // // Include imported images var importedImages = new List <string>(); if (_exportConfiguration.IncludeImportedArtifacts && transferReport.ImportArtifacts.Succeeded.Count > 0) { importedImages.AddRange(transferReport.ImportArtifacts.Succeeded); } if (importedImages.Count == 0 && _exportConfiguration.Repositories.Count == 0 && _exportConfiguration.Tags.Count == 0) { return; } // // Validate exportPipeline var exportPipeline = await _transferClient.GetExportPipelineAsync(_exportConfiguration.ExportPipelineName).ConfigureAwait(false); if (!exportPipeline.ProvisioningState.Equals("succeeded", StringComparison.OrdinalIgnoreCase)) { throw new Exception($"ExportPipeline:{_exportConfiguration.ExportPipelineName} is in non-success provisioning state {exportPipeline.ProvisioningState}."); } // // Export images var exportJobs = await CreateExportJobsAsync(importedImages).ConfigureAwait(false); await exportJobs.ThrottledWhenAll( async (job) => await ExecuteExportAsync(job).ConfigureAwait(false), _exportConfiguration.MaxConcurrency).ConfigureAwait(false); foreach (var job in exportJobs) { if (job.Status == TransferJobStatus.Succeeded) { transferReport.ExportArtifacts.Succeeded.AddRange(job.Images); transferReport.ExportBlobs.Succeeded.Add(job.ExportBlobName); } else if (job.Status == TransferJobStatus.Failed) { transferReport.ExportArtifacts.Failed.AddRange(job.Images); transferReport.ExportBlobs.Failed.Add(job.ExportBlobName); } } // // Copy exported blobs if (_exportConfiguration.CopyBlobs != null && _exportConfiguration.CopyBlobs.Enabled && transferReport.ExportBlobs.Succeeded.Count > 0) { var sourceContainerSas = new Uri(await GetSourceSasUriAsync().ConfigureAwait(false)); var targetContainerSas = new Uri(_exportConfiguration.CopyBlobs.DestContainerSasUri); var blobCopier = new BlobCopier(sourceContainerSas, targetContainerSas, _logger); foreach (var blobName in transferReport.ExportBlobs.Succeeded) { _logger.LogInformation($"Blob {blobName}: starting to copy ."); try { await blobCopier.CopyAsync(blobName).ConfigureAwait(false); transferReport.CopyBlobs.Succeeded.Add(blobName); _logger.LogInformation($"Blob {blobName}: Successfully copied."); } catch (Exception e) { transferReport.CopyBlobs.Failed.Add(blobName); _logger.LogError($"Blob {blobName}: failed to copy, exception: {e}."); } } } }
public static async Task Main(string[] args) { #region Global HTTP settings // https://github.com/Azure/azure-storage-net-data-movement#best-practice ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8; ServicePointManager.Expect100Continue = false; #endregion #region Logger var logFileName = LogFileNamePrefix + "_" + string.Format("{0:yyyy-MM-dd_HH-mm-ss-fff}", DateTimeOffset.Now) + ".txt"; var loggerFactory = LoggerFactory.Create(builder => builder.AddSerilog(new LoggerConfiguration() .WriteTo.File(logFileName) .WriteTo.ColoredConsole() .CreateLogger())); var logger = loggerFactory.CreateLogger("RegistryArtifactTransfer"); #endregion #region Configurations string transferDefinitionFile = args.Length > 0 ? args[0] : Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "transferdefinition.json"); var transferDefinition = GetConfig(transferDefinitionFile); transferDefinition.Validate(); var azureEnvironmentConfiguration = transferDefinition.AzureEnvironment; var identityConfiguration = transferDefinition.Identity; var registryConfiguration = transferDefinition.Registry; #endregion #region TransferClient var transferClient = new TransferClient( azureEnvironmentConfiguration, identityConfiguration, registryConfiguration); #endregion #region Process transfer var report = new TransferReport(); var importWorker = new ImportWorker( transferDefinition.Import, registryConfiguration, transferClient, loggerFactory.CreateLogger <ImportWorker>()); var exportWorker = new ExportWorker( transferDefinition.Export, registryConfiguration, identityConfiguration, transferClient, loggerFactory.CreateLogger <ExportWorker>()); await importWorker.RunAsync(report); await exportWorker.RunAsync(report); #endregion #region Report results logger.LogInformation($"Total artifacts successfully imported: {report.ImportArtifacts.Succeeded.Count}."); logger.LogInformation($"Total artifacts failed to import: {report.ImportArtifacts.Failed.Count}."); logger.LogInformation($"Total blobs successfully exported: {report.ImportBlobs.Succeeded.Count}."); logger.LogInformation($"Total blobs failed to export: {report.ImportBlobs.Failed.Count}."); logger.LogInformation($"Total artifacts successfully exported: {report.ExportArtifacts.Succeeded.Count}."); logger.LogInformation($"Total artifacts failed in exporting: {report.ExportArtifacts.Failed.Count}."); logger.LogInformation($"Total blobs successfully exported: {report.ExportBlobs.Succeeded.Count}."); logger.LogInformation($"Total blobs failed in exporting: {report.ExportBlobs.Failed.Count}."); logger.LogInformation($"Total blobs successfully copied: {report.CopyBlobs.Succeeded.Count}."); logger.LogInformation($"Total blobs failed in copying: {report.CopyBlobs.Failed.Count}."); var reportFileName = ReportFileNamePrefix + "_" + string.Format("{0:yyyy-MM-dd_HH-mm-ss-fff}", DateTimeOffset.Now) + ".json"; await File.WriteAllTextAsync(reportFileName, JsonConvert.SerializeObject(report, Formatting.Indented)); #endregion logger.LogInformation("Done!"); }