public async Task InitializeAsync() { using var scope = _logger.BeginScope("Init"); _logger.LogInformation("Initializing base data"); // OSS bucket might fail to create, so repeat attempts var createBucketPolicy = Policy .Handle <ApiException>() .WaitAndRetryAsync( retryCount: 4, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (exception, timeSpan) => _logger.LogWarning("Cannot create OSS bucket. Repeating") ); await Task.WhenAll( // create bundles and activities _fdaClient.InitializeAsync(), // create the bucket createBucketPolicy.ExecuteAsync(async() => await _bucket.CreateAsync()) ); _logger.LogInformation($"Bucket {_bucket.BucketKey} created"); // OSS bucket might be not ready yet, so repeat attempts var waitForBucketPolicy = Policy .Handle <ApiException>(e => e.ErrorCode == StatusCodes.Status404NotFound) .WaitAndRetryAsync( retryCount: 4, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (exception, timeSpan) => _logger.LogWarning("Cannot get fresh OSS bucket. Repeating") ); // publish default project files (specified by the appsettings.json) foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects) { var projectUrl = defaultProjectConfig.Url; var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name); _logger.LogInformation($"Launching 'TransferData' for {projectUrl}"); string signedUrl = await waitForBucketPolicy.ExecuteAsync(async() => await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite)); // TransferData from s3 to oss await _projectWork.FileTransferAsync(projectUrl, signedUrl); _logger.LogInformation($"'TransferData' for {projectUrl} is done."); await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl); } _logger.LogInformation("Added default projects."); }
public async Task <string> AdoptDefaultOnly(bool RemoveCached) { string returnValue = ""; if (RemoveCached) { List <ObjectDetails> ossFiles = await _forgeOSS.GetBucketObjectsAsync(_bucket.BucketKey, "cache/"); foreach (ObjectDetails file in ossFiles) { returnValue += "Removing cache file " + file.ObjectKey + "\n"; try { await _forgeOSS.DeleteAsync(_bucket.BucketKey, file.ObjectKey); } catch (Exception e) { returnValue += "Removing cache file " + file.ObjectKey + " failed\nException:" + e.Message + "\n"; } } } foreach (DefaultProjectConfiguration defaultProjectConfig in _defaultProjectsConfiguration.Projects) { returnValue += "Project " + defaultProjectConfig.Name + " is being adopted\n"; var projectUrl = defaultProjectConfig.Url; var project = await _userResolver.GetProjectAsync(defaultProjectConfig.Name); string signedUrl = await _bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite); try { await _projectWork.AdoptAsync(defaultProjectConfig, signedUrl); returnValue += "Project " + defaultProjectConfig.Name + " was adopted\n"; } catch (Exception e) { returnValue += "Project " + defaultProjectConfig.Name + " was not adopted\nException:" + e.Message + "\n"; } } return(returnValue); }
public async Task <string> TransferProjectToOssAsync(OssBucket bucket, DefaultProjectConfiguration projectConfig) { _logger.LogInformation($"Bucket {bucket.BucketKey} created"); var projectUrl = projectConfig.Url; var project = await _userResolver.GetProjectAsync(projectConfig.Name); _logger.LogInformation($"Launching 'TransferData' for {projectUrl}"); // OSS bucket might be not ready yet, so repeat attempts string signedUrl = await _waitForBucketPolicy.ExecuteAsync(async() => await bucket.CreateSignedUrlAsync(project.OSSSourceModel, ObjectAccess.ReadWrite)); // TransferData from outside URL to temporary oss url await _projectWork.FileTransferAsync(projectUrl, signedUrl); _logger.LogInformation($"'TransferData' for {projectUrl} is done."); return(signedUrl); }