public async Task InitializeAsync()
{
using var scope = _logger.BeginScope("Init");
_logger.LogInformation("Initializing base data");
// OSS bucket might fail to create, so repeat attempts
var createBucketPolicy = Policy
.Handle <ApiException>()
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot create OSS bucket. Repeating")
);
await Task.WhenAll(
// create bundles and activities
_fdaClient.InitializeAsync(),
// create the bucket
createBucketPolicy.ExecuteAsync(async() => await _bucket.CreateAsync())
);
_logger.LogInformation($"Bucket {_bucket.BucketKey} created");
// OSS bucket might be not ready yet, so repeat attempts
var waitForBucketPolicy = Policy
.Handle <ApiException>(e => e.ErrorCode == StatusCodes.Status404NotFound)
.WaitAndRetryAsync(
retryCount: 4,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) => _logger.LogWarning("Cannot get fresh OSS bucket. Repeating")
);
// publish default project files (specified by the appsettings.json)
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
var projectUrl = defaultProjectConfig.Url;
var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
string signedUrl = await waitForBucketPolicy.ExecuteAsync(async() => await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from s3 to oss
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
}
_logger.LogInformation("Added default projects.");
}
public async Task <string> AdoptDefaultOnly(bool RemoveCached)
{
string returnValue = "";
if (RemoveCached)
{
List <ObjectDetails> ossFiles = await _forgeOSS.GetBucketObjectsAsync(_bucket.BucketKey, "cache/");
foreach (ObjectDetails file in ossFiles)
{
returnValue += "Removing cache file " + file.ObjectKey + "\n";
try
{
await _forgeOSS.DeleteAsync(_bucket.BucketKey, file.ObjectKey);
} catch (Exception e)
{
returnValue += "Removing cache file " + file.ObjectKey + " failed\nException:" + e.Message + "\n";
}
}
}
foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects)
{
returnValue += "Project " + defaultProjectConfig.Name + " is being adopted\n";
var projectUrl = defaultProjectConfig.Url;
var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name);
string signedUrl = await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite);
try
{
await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl);
returnValue += "Project " + defaultProjectConfig.Name + " was adopted\n";
}
catch (Exception e)
{
returnValue += "Project " + defaultProjectConfig.Name + " was not adopted\nException:" + e.Message + "\n";
}
}
return(returnValue);
}
public async Task <string> TransferProjectToOssAsync(OssBucket bucket, DefaultProjectConfiguration projectConfig)
{
_logger.LogInformation($"Bucket {bucket.BucketKey} created");
var projectUrl = projectConfig.Url;
var project = await _userResolver.GetProjectAsync(projectConfig.Name);
_logger.LogInformation($"Launching 'TransferData' for {projectUrl}");
// OSS bucket might be not ready yet, so repeat attempts
string signedUrl = await _waitForBucketPolicy.ExecuteAsync(async() =>
await bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite));
// TransferData from outside URL to temporary oss url
await _projectWork.FileTransferAsync(projectUrl, signedUrl);
_logger.LogInformation($"'TransferData' for {projectUrl} is done.");
return(signedUrl);
}
