protected override Yield Stop(Result result)
{
_s3Client.Dispose();
_s3Client = null;
yield return(Coroutine.Invoke(base.Stop, new Result()));
result.Return();
}
public void SetUp()
{
bucket = "bucket";
resource = "resource";
filePath = "file";
url = "s3.com/file";
fileResponse = new FileResponse("file", contentStream: new MemoryStream());
cancellationToken = CancellationToken.None;
client = Substitute.For <IAwsS3Client>();
client.GenerateUrl(bucket, resource).Returns(url);
uploader = new S3FileUploader(client);
}
public void SetUp()
{
bucket = "test-bucket";
existingResource = "test-resource";
notFoundResource = "not-found";
files = new[] { "file1", "file2" };
cancellationToken = CancellationToken.None;
s3Client = Substitute.For <IAwsS3Client>();
s3Client.List(bucket, existingResource, cancellationToken).Returns(files);
s3Client.List(bucket, notFoundResource, cancellationToken).Returns(new string[] { });
s3Client.Download(bucket, resource: Arg.Any <string>(), destinationPath: Arg.Any <string>(), cancellationToken);
s3Client.Download(bucket, resourceKey: Arg.Any <string>(), cancellationToken).Returns(new FileResponse("key", default));
fileRetriever = new S3FileRetriever(s3Client);
}
public S3FileUploadMonitoringProcess(
IFileUploadMessageMapper mapper,
IUploadConfiguration configuration,
IAwsQueueClient queueClient,
IAwsS3Client s3Client,
ILogger <S3FileUploadMonitoringProcess> logger)
{
this._mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
this._configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
this._queueClient = queueClient ?? throw new ArgumentNullException(nameof(queueClient));
this._s3Client = s3Client ?? throw new ArgumentNullException(nameof(s3Client));
this._logger = logger ?? throw new ArgumentNullException(nameof(logger));
this._dataImportTradeFileDirectoryPatternRegex = new Regex(this._configuration.DataImportTradeFileDirectoryPattern, RegexOptions.Compiled | RegexOptions.Singleline);
this._dataImportAllocationFileDirectoryPatternRegex = new Regex(this._configuration.DataImportAllocationFileDirectoryPattern, RegexOptions.Compiled | RegexOptions.Singleline);
this._dataImportEtlFileDirectoryPatternRegex = new Regex(this._configuration.DataImportEtlFileDirectoryPattern, RegexOptions.Compiled | RegexOptions.Singleline);
this._dataImportIgnoreFileDirectoryPatternRegex = new Regex(this._configuration.DataImportIgnoreFileDirectoryPattern, RegexOptions.Compiled | RegexOptions.Singleline);
}
//--- Methods ---
protected override Yield Start(XDoc config, ILifetimeScope container, Result result)
{
yield return(Coroutine.Invoke(base.Start, config, new Result()));
// are we a private storage service?
_private = config["sid"].Contents == "sid://mindtouch.com/2010/10/dream/s3.storage.private";
_log.DebugFormat("storage is {0}", _private ? "private" : "public");
// is the root blocked from access?
_privateRoot = config["private-root"].AsBool.GetValueOrDefault();
_log.DebugFormat("storage root is {0}accessible", _privateRoot ? "not " : "");
// set up S3 client
var s3Config = new AwsS3ClientConfig()
{
Endpoint = AwsEndpoint.GetEndpoint(config["endpoint"].AsText) ?? AwsEndpoint.Default,
Bucket = config["bucket"].AsText,
Delimiter = "/",
RootPath = config["folder"].AsText,
PrivateKey = config["privatekey"].AsText,
PublicKey = config["publickey"].AsText,
Timeout = TimeSpan.FromSeconds(config["timeout"].AsDouble ?? DEFAULT_S3_TIMEOUT)
};
if (string.IsNullOrEmpty(s3Config.Bucket))
{
throw new ArgumentException("missing configuration parameter 'bucket'");
}
if (string.IsNullOrEmpty(s3Config.PrivateKey))
{
throw new ArgumentException("missing configuration parameter 'privatekey'");
}
if (string.IsNullOrEmpty(s3Config.PublicKey))
{
throw new ArgumentException("missing configuration parameter 'publickey'");
}
_s3Client = container.Resolve <IAwsS3Client>(TypedParameter.From(s3Config));
result.Return();
}
public SaveJsonToS3Gateway(IAwsS3Client awsS3Client)
{
_awsS3Client = awsS3Client;
}
public S3FileUploader(IAwsS3Client client)
{
this.client = client;
}
public S3FileRetriever(IAwsS3Client s3Client)
{
this.s3Client = s3Client;
this.log = LogProvider.GetCurrentClassLogger();
}
protected override Yield Stop(Result result)
{
_s3Client.Dispose();
_s3Client = null;
yield return Coroutine.Invoke(base.Stop, new Result());
result.Return();
}
//--- Methods ---
protected override Yield Start(XDoc config, ILifetimeScope container, Result result)
{
yield return Coroutine.Invoke(base.Start, config, new Result());
// are we a private storage service?
_private = config["sid"].Contents == "sid://mindtouch.com/2010/10/dream/s3.storage.private";
_log.DebugFormat("storage is {0}", _private ? "private" : "public");
// is the root blocked from access?
_privateRoot = config["private-root"].AsBool.GetValueOrDefault();
_log.DebugFormat("storage root is {0}accessible", _privateRoot ? "not " : "");
// set up S3 client
var s3Config = new AwsS3ClientConfig() {
Endpoint = AwsEndpoint.GetEndpoint(config["endpoint"].AsText) ?? AwsEndpoint.Default,
Bucket = config["bucket"].AsText,
Delimiter = "/",
RootPath = config["folder"].AsText,
PrivateKey = config["privatekey"].AsText,
PublicKey = config["publickey"].AsText,
Timeout = TimeSpan.FromSeconds(config["timeout"].AsDouble ?? DEFAULT_S3_TIMEOUT)
};
if(string.IsNullOrEmpty(s3Config.Bucket)) {
throw new ArgumentException("missing configuration parameter 'bucket'");
}
if(string.IsNullOrEmpty(s3Config.PrivateKey)) {
throw new ArgumentException("missing configuration parameter 'privatekey'");
}
if(string.IsNullOrEmpty(s3Config.PublicKey)) {
throw new ArgumentException("missing configuration parameter 'publickey'");
}
_s3Client = container.Resolve<IAwsS3Client>(TypedParameter.From(s3Config));
result.Return();
}
