public async Task RunAsync(TransferReport transferReport)
{
if (!_importConfiguration.Enabled)
{
return;
}
//
// Validate importPipeline
if (_importConfiguration.Blobs.Count > 0)
{
var importPipeline = await _transferClient.GetImportPipelineAsync(_importConfiguration.ImportPipelineName).ConfigureAwait(false);
if (!importPipeline.ProvisioningState.Equals("succeeded", StringComparison.OrdinalIgnoreCase))
{
throw new Exception($"ImportPipeline:{_importConfiguration.ImportPipelineName} is in non-success provisioning state {importPipeline.ProvisioningState}.");
}
}
var importJobs = await CreateImportJobs().ConfigureAwait(false);
await importJobs.ThrottledWhenAll(
async (job) => await ExecuteAsync(job).ConfigureAwait(false),
_importConfiguration.MaxConcurrency).ConfigureAwait(false);
foreach (var job in importJobs)
{
if (job.Status == TransferJobStatus.Succeeded)
{
transferReport.ImportArtifacts.Succeeded.AddRange(job.Images);
if (job.SourceType == ImportSourceType.AzureStorageBlob)
{
transferReport.ImportBlobs.Succeeded.Add(job.ImportBlobName);
}
}
else if (job.Status == TransferJobStatus.Failed)
{
transferReport.ImportArtifacts.Failed.AddRange(job.Images);
if (job.SourceType == ImportSourceType.AzureStorageBlob)
{
transferReport.ImportBlobs.Failed.Add(job.ImportBlobName);
}
}
}
}
public async Task RunAsync(TransferReport transferReport)
{
if (!_exportConfiguration.Enabled)
{
return;
}
//
// Include imported images
var importedImages = new List <string>();
if (_exportConfiguration.IncludeImportedArtifacts && transferReport.ImportArtifacts.Succeeded.Count > 0)
{
importedImages.AddRange(transferReport.ImportArtifacts.Succeeded);
}
if (importedImages.Count == 0 &&
_exportConfiguration.Repositories.Count == 0 &&
_exportConfiguration.Tags.Count == 0)
{
return;
}
//
// Validate exportPipeline
var exportPipeline = await _transferClient.GetExportPipelineAsync(_exportConfiguration.ExportPipelineName).ConfigureAwait(false);
if (!exportPipeline.ProvisioningState.Equals("succeeded", StringComparison.OrdinalIgnoreCase))
{
throw new Exception($"ExportPipeline:{_exportConfiguration.ExportPipelineName} is in non-success provisioning state {exportPipeline.ProvisioningState}.");
}
//
// Export images
var exportJobs = await CreateExportJobsAsync(importedImages).ConfigureAwait(false);
await exportJobs.ThrottledWhenAll(
async (job) => await ExecuteExportAsync(job).ConfigureAwait(false),
_exportConfiguration.MaxConcurrency).ConfigureAwait(false);
foreach (var job in exportJobs)
{
if (job.Status == TransferJobStatus.Succeeded)
{
transferReport.ExportArtifacts.Succeeded.AddRange(job.Images);
transferReport.ExportBlobs.Succeeded.Add(job.ExportBlobName);
}
else if (job.Status == TransferJobStatus.Failed)
{
transferReport.ExportArtifacts.Failed.AddRange(job.Images);
transferReport.ExportBlobs.Failed.Add(job.ExportBlobName);
}
}
//
// Copy exported blobs
if (_exportConfiguration.CopyBlobs != null &&
_exportConfiguration.CopyBlobs.Enabled &&
transferReport.ExportBlobs.Succeeded.Count > 0)
{
var sourceContainerSas = new Uri(await GetSourceSasUriAsync().ConfigureAwait(false));
var targetContainerSas = new Uri(_exportConfiguration.CopyBlobs.DestContainerSasUri);
var blobCopier = new BlobCopier(sourceContainerSas, targetContainerSas, _logger);
foreach (var blobName in transferReport.ExportBlobs.Succeeded)
{
_logger.LogInformation($"Blob {blobName}: starting to copy .");
try
{
await blobCopier.CopyAsync(blobName).ConfigureAwait(false);
transferReport.CopyBlobs.Succeeded.Add(blobName);
_logger.LogInformation($"Blob {blobName}: Successfully copied.");
}
catch (Exception e)
{
transferReport.CopyBlobs.Failed.Add(blobName);
_logger.LogError($"Blob {blobName}: failed to copy, exception: {e}.");
}
}
}
}
public static async Task Main(string[] args)
{
#region Global HTTP settings
// https://github.com/Azure/azure-storage-net-data-movement#best-practice
ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
ServicePointManager.Expect100Continue = false;
#endregion
#region Logger
var logFileName = LogFileNamePrefix + "_" + string.Format("{0:yyyy-MM-dd_HH-mm-ss-fff}", DateTimeOffset.Now) + ".txt";
var loggerFactory = LoggerFactory.Create(builder => builder.AddSerilog(new LoggerConfiguration()
.WriteTo.File(logFileName)
.WriteTo.ColoredConsole()
.CreateLogger()));
var logger = loggerFactory.CreateLogger("RegistryArtifactTransfer");
#endregion
#region Configurations
string transferDefinitionFile = args.Length > 0
? args[0]
: Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "transferdefinition.json");
var transferDefinition = GetConfig(transferDefinitionFile);
transferDefinition.Validate();
var azureEnvironmentConfiguration = transferDefinition.AzureEnvironment;
var identityConfiguration = transferDefinition.Identity;
var registryConfiguration = transferDefinition.Registry;
#endregion
#region TransferClient
var transferClient = new TransferClient(
azureEnvironmentConfiguration,
identityConfiguration,
registryConfiguration);
#endregion
#region Process transfer
var report = new TransferReport();
var importWorker = new ImportWorker(
transferDefinition.Import,
registryConfiguration,
transferClient,
loggerFactory.CreateLogger <ImportWorker>());
var exportWorker = new ExportWorker(
transferDefinition.Export,
registryConfiguration,
identityConfiguration,
transferClient,
loggerFactory.CreateLogger <ExportWorker>());
await importWorker.RunAsync(report);
await exportWorker.RunAsync(report);
#endregion
#region Report results
logger.LogInformation($"Total artifacts successfully imported: {report.ImportArtifacts.Succeeded.Count}.");
logger.LogInformation($"Total artifacts failed to import: {report.ImportArtifacts.Failed.Count}.");
logger.LogInformation($"Total blobs successfully exported: {report.ImportBlobs.Succeeded.Count}.");
logger.LogInformation($"Total blobs failed to export: {report.ImportBlobs.Failed.Count}.");
logger.LogInformation($"Total artifacts successfully exported: {report.ExportArtifacts.Succeeded.Count}.");
logger.LogInformation($"Total artifacts failed in exporting: {report.ExportArtifacts.Failed.Count}.");
logger.LogInformation($"Total blobs successfully exported: {report.ExportBlobs.Succeeded.Count}.");
logger.LogInformation($"Total blobs failed in exporting: {report.ExportBlobs.Failed.Count}.");
logger.LogInformation($"Total blobs successfully copied: {report.CopyBlobs.Succeeded.Count}.");
logger.LogInformation($"Total blobs failed in copying: {report.CopyBlobs.Failed.Count}.");
var reportFileName = ReportFileNamePrefix + "_" + string.Format("{0:yyyy-MM-dd_HH-mm-ss-fff}", DateTimeOffset.Now) + ".json";
await File.WriteAllTextAsync(reportFileName, JsonConvert.SerializeObject(report, Formatting.Indented));
#endregion
logger.LogInformation("Done!");
}