public async Task Migrate(List <MigrationJob> migrationJobs)
{
foreach (MigrationJob job in migrationJobs)
{
_bucketProvider.SetBucketKeyFromOld(job.bucketOld.BucketKey);
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await job.bucketOld.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(job.projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " cannot be copied\nException:" + e.Message);
continue;
}
try
{
await _projectWork.AdoptAsync(job.projectInfo, signedUrlNew);
_logger.LogInformation("Project " + job.projectInfo.Name + " was adopted");
}
catch (Exception e)
{
_logger.LogError("Project " + job.projectInfo.Name + " was not adopted\nException:" + e.Message);
}
}
}
private async Task CopyAndAdopt(string logId)
{
_bucketProvider.SetBucketKeyFromOld(bucket.BucketKey);
_logger.LogInformation($"{logId}: Processing new project {projectInfo.Name} in bucket {bucket.BucketKey}");
OssBucket bucketNew = await _userResolver.GetBucketAsync(true);
string signedUrlOld = await bucket.CreateSignedUrlAsync(projectUrl, ObjectAccess.Read);
string signedUrlNew = await bucketNew.CreateSignedUrlAsync(projectUrl, ObjectAccess.ReadWrite);
try
{
await _projectWork.FileTransferAsync(signedUrlOld, signedUrlNew);
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} cannot be copied.");
return;
}
try
{
await _projectWork.AdoptAsync(projectInfo, signedUrlNew);
_logger.LogInformation($"{logId}: Project {projectInfo.Name} was adopted");
}
catch (Exception e)
{
_logger.LogError(e, $"{logId}: Project {projectInfo.Name} was not adopted");
}
}
public async Task InitializeAsync()
{
using var scope = _logger.BeginScope("Init");
_logger.LogInformation("Initializing base data");
// OSS bucket might fail to create, so repeat attempts
var createBucketPolicy = Policy
.Handle <ApiException>()
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot create OSS bucket. Repeating")
);
await Task.WhenAll(
// create bundles and activities
_fdaClient.InitializeAsync(),
// create the bucket
createBucketPolicy.ExecuteAsync(async() => await _bucket.CreateAsync())
);
_logger.LogInformation($"Bucket {_bucket.BucketKey} created");
// OSS bucket might be not ready yet, so repeat attempts
var waitForBucketPolicy = Policy
.Handle <ApiException>(e => e.ErrorCode == StatusCodes.Status404NotFound)
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot get fresh OSS bucket. Repeating")
);
// publish default project files (specified by the appsettings.json)
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
var projectUrl = defaultProjectConfig.Url;
var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
string signedUrl = await waitForBucketPolicy.ExecuteAsync(async() => await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from s3 to oss
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
}
_logger.LogInformation("Added default projects.");
}
public async Task <string> TransferProjectToOssAsync(OssBucket bucket, DefaultProjectConfiguration projectConfig)
{
_logger.LogInformation($"Bucket {bucket.BucketKey} created");
var projectUrl = projectConfig.Url;
var project = await _userResolver.GetProjectAsync(projectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
// OSS bucket might be not ready yet, so repeat attempts
string signedUrl = await _waitForBucketPolicy.ExecuteAsync(async() =>
await bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from outside URL to temporary oss url
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
return(signedUrl);
}
