public static Stream GetS3ReadStream(IS3Client s3Client, string bucketName, string fileName, long position) { var connector = ConnectUtilities.GetS3ConnectFunc(bucketName, fileName, s3Client); var stream = ConnectUtilities.ConnectWithRetries(connector, position, MaxRetryCount); return(new PersistentStream(stream, connector, position)); }
public ConnectionTests() { _loggerMock = new Mock <ILogger <IS3Client> >(); _bucketOperationsClientMock = new Mock <IBucketOperations>(); _bucketOperationsClientMock.Setup(c => c.BucketExistsAsync(It.IsAny <string>(), It.IsAny <CancellationToken>())).ThrowsAsync(new InternalClientException("Connection refused")); _bucketOperationsClientMock.Setup(c => c.RemoveBucketAsync(It.IsAny <string>(), It.IsAny <CancellationToken>())).ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock = new Mock <IObjectOperations>(); _objectOperationsClientMock.Setup(c => c.StatObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <ServerSideEncryption>(), It.IsAny <CancellationToken>())) .ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock.Setup(c => c.GetObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <Action <Stream> >(), It.IsAny <ServerSideEncryption>(), It.IsAny <CancellationToken>())) .ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock.Setup(c => c.PresignedGetObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <DateTime?>())) .ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock.Setup(c => c.PresignedGetObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <int>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <DateTime?>())) .ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock.Setup(c => c.PutObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <Stream>(), It.IsAny <long>(), It.IsAny <string>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <ServerSideEncryption>(), It.IsAny <CancellationToken>())) .ThrowsAsync(new InternalClientException("Connection refused")); _objectOperationsClientMock.Setup(c => c.RemoveObjectAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .ThrowsAsync(new InternalClientException("Connection refused")); _s3Client = new MinioS3Client(_bucketOperationsClientMock.Object, _objectOperationsClientMock.Object, "127.0.0.1:1", "test", _loggerMock.Object, 0, 0); }
public ArticleService(IDbContextMongoDb dbContext, ISettings settings, IMapper mapper, IS3Client s3Client) { _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); _settings = settings ?? throw new ArgumentNullException(nameof(settings)); _mapper = mapper ?? throw new ArgumentNullException(nameof(mapper)); _s3Client = s3Client ?? throw new ArgumentNullException(nameof(s3Client)); }
public S3ClientPrometheusDecorator(IS3Client s3Client, MetricsProvider metricsProvider, string backendLabel) { _s3Client = s3Client; _backendLabel = backendLabel; _requestDurationMsMetric = metricsProvider.GetRequestDurationMsMetric(); _requestErrorsMetric = metricsProvider.GetRequestErrorsMetric(); }
public TemplatesStorageReader(CephOptions cephOptions, IS3Client s3Client, IMemoryCache memoryCache) { _s3Client = s3Client; _memoryCache = memoryCache; _bucketName = cephOptions.TemplatesBucketName; _degreeOfParallelism = cephOptions.DegreeOfParallelism; }
public S3FileExistsCommand(IS3Client client, string bucketName, string fileName) : base("s3", "s3-read", TimeSpan.FromSeconds(5)) { _client = client; _bucketName = bucketName; _fileName = fileName; }
private void Publish(IS3Client client, IEnumerable <ITaskItem> sourceFiles, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds, String uploadIfNotExists = null) { if (uploadIfNotExists != null) { uploadIfNotExists = uploadIfNotExists.Replace(".", "\\.").Replace("*", ".*"); } foreach (var fileItem in sourceFiles.Where(taskItem => taskItem != null && !string.IsNullOrEmpty(taskItem.GetMetadata("Identity")))) { var info = new FileInfo(fileItem.GetMetadata("Identity")); var headers = MsBuildHelpers.GetCustomItemMetadata(fileItem); Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", info.FullName)); if (uploadIfNotExists != null && Regex.IsMatch(info.Name, uploadIfNotExists)) { if (!client.FileExists(bucket, CreateRelativePath(destinationFolder, info.Name))) { client.PutFile(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, headers, publicRead, timeoutMilliseconds); } } else { client.PutFile(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, headers, publicRead, timeoutMilliseconds); } } }
private static void DoCreateTasks(Configuration cfg, ILoggerFactory _loggerFactory) { IS3Client s3Client = serviceProvider.GetService <IS3Client>(); string dbFile = Path.Combine(cfg.outputPath, TASK_DB_FILE_NAME); Log.Information($"Store recognition tasks list into {dbFile}"); SkTaskDb taskDb = new SkTaskDb(dbFile, Log.Logger); RecognitionSpec rSpec = new RecognitionSpec() { LanguageCode = cfg.lang, ProfanityFilter = true, Model = cfg.model, LiteratureText = cfg.punctuation, PartialResults = false, //возвращать только финальные результаты false AudioEncoding = cfg.audioEncoding, SampleRateHertz = cfg.sampleRate }; SkRecognitionClient recognitionClient = new SkRecognitionClient(new Uri("https://stt.api.cloud.yandex.net:443"), cfg.folderId, cfg.iamToken, rSpec, _loggerFactory, taskDb); s3Client.ProcessBucket(cfg.bucket, recognitionClient); Log.Information($"Found {taskDb.Count} objects"); }
private void ValidateCredentials(IS3Client s3Client, bool isDirectory) { try { if (isDirectory) { var putRequest = new PutObjectRequest { BucketName = bucketName, Key = path }; s3Client.PutObjectAsync(putRequest).Wait(); } else { var getRequest = new GetObjectRequest { BucketName = bucketName, Key = path, ByteRange = new ByteRange(0, 1) }; s3Client.GetObjectAsync(getRequest).Wait(); } } catch (Exception exception) { var processedException = ExceptionUtilities.ProcessAmazonS3Exception(exception, this); if (processedException is UserErrorException) { throw processedException; } } }
public void Dispose() { if (null != _client) { _client.Dispose(); } _client = null; }
protected void TearDownClient() { if (_client != null) { _client.Dispose(); } _client = null; }
public HubService(Env env, ILambdaClient lambdaClient, IS3Client s3Client, IFileHelper fileHelper) { this.env = env; this.lambdaClient = lambdaClient; this.s3Client = s3Client; this.fileHelper = fileHelper; this.hubMessageConverter = new HubMessageConverter(env, fileHelper); }
public BucketOperationTests() { _loggerMock = new Mock <ILogger <IS3Client> >(); _bucketOperationsClientMock = new Mock <IBucketOperations>(); _bucketOperationsClientMock.Setup(c => c.BucketExistsAsync("existingbucket", It.IsAny <CancellationToken>())).ReturnsAsync(value: true); _bucketOperationsClientMock.Setup(c => c.BucketExistsAsync("nonexistingbucket", It.IsAny <CancellationToken>())).ReturnsAsync(value: false); _s3Client = new MinioS3Client(_bucketOperationsClientMock.Object, null, "127.0.0.1:1", "test", _loggerMock.Object, 0, 0); }
public S3UploadFileCommand(IS3Client client, string bucketName, string localFile, string key, string contentType, string statContentType) : base("s3", "s3-write", TimeSpan.FromSeconds(5)) { _client = client; _bucketName = bucketName; _localFile = localFile; _key = key; _contentType = contentType; _statContentType = statContentType; }
public ImagePreviewService( CephOptions cephOptions, ThrottlingOptions throttlingOptions, IS3Client s3Client, MemoryBasedRequestLimiter requestLimiter) { _bucketName = cephOptions.FilesBucketName; _requestTimeout = throttlingOptions.RequestTimeout; _s3Client = s3Client; _requestLimiter = requestLimiter; }
public static void DecryptUpload(this IS3Client s3Client, string bucketName, string key, string filePath, AesCryptoServiceProvider aes, FileMetadata metadata, int retryDelay = 1000) { while (true) { if (s3Client.TryDecryptUpload(bucketName, key, filePath, aes, metadata)) { return; } Thread.Sleep(retryDelay); } }
public TemplatesManagementService( UploadFileOptions uploadFileOptions, CephOptions cephOptions, IS3Client s3Client, ITemplatesStorageReader templatesStorageReader, DistributedLockManager distributedLockManager) { _s3Client = s3Client; _templatesStorageReader = templatesStorageReader; _distributedLockManager = distributedLockManager; _bucketName = cephOptions.TemplatesBucketName; _maxBinarySize = uploadFileOptions.MaxBinarySize; }
public ObjectsStorageReader( CephOptions cephOptions, VStoreOptions vStoreOptions, IS3Client s3Client, TemplatesStorageReader templatesStorageReader, DistributedLockManager distributedLockManager) { _s3Client = s3Client; _templatesStorageReader = templatesStorageReader; _distributedLockManager = distributedLockManager; _bucketName = cephOptions.ObjectsBucketName; _degreeOfParallelism = cephOptions.DegreeOfParallelism; _fileStorageEndpoint = vStoreOptions.FileStorageEndpoint; }
public ObjectsStorageReader( CephOptions cephOptions, CdnOptions cdnOptions, IS3Client s3Client, ITemplatesStorageReader templatesStorageReader, DistributedLockManager distributedLockManager) { _cdnOptions = cdnOptions; _s3Client = s3Client; _templatesStorageReader = templatesStorageReader; _distributedLockManager = distributedLockManager; _bucketName = cephOptions.ObjectsBucketName; _degreeOfParallelism = cephOptions.DegreeOfParallelism; }
private void ValidateCredentials(IS3Client s3Client, bool isDirectory) { int maxRetryCount = MaxRetryCount; while (true) { try { if (isDirectory) { var putRequest = new PutObjectRequest { BucketName = bucketName, Key = path }; s3Client.PutObjectAsync(putRequest).Wait(); } else { var getRequest = new GetObjectRequest { BucketName = bucketName, Key = path, ByteRange = new ByteRange(0, 1) }; s3Client.GetObjectAsync(getRequest).Wait(); } // validation successful. Break and return. break; } catch (Exception exception) { var processedException = AwsExceptionUtilities.TryConvertUserException(exception, this); if (processedException is UserErrorException) { throw processedException; } Logger.WriteLine($"Failed to validate S3 credentials\n{processedException.Message}"); maxRetryCount--; if (maxRetryCount >= 0) { continue; } Logger.WriteLine("Max retry limit reached for validating S3 credentials."); throw processedException; } } }
private void Publish(IS3Client client, IEnumerable<string> sourceFiles, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { foreach (var f in sourceFiles) { if (string.IsNullOrEmpty(f)) continue; var info = new FileInfo(f); Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", info.FullName)); client.PutFile(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, publicRead, timeoutMilliseconds); } }
private void Publish(IS3Client client, IEnumerable <ITaskItem> sourceFiles, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { foreach (var fileItem in sourceFiles.Where(taskItem => taskItem != null && !string.IsNullOrEmpty(taskItem.GetMetadata("Identity")))) { var info = new FileInfo(fileItem.GetMetadata("Identity")); var headers = MsBuildHelpers.GetCustomItemMetadata(fileItem); Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", info.FullName)); client.PutFileWithHeaders(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, headers, publicRead, timeoutMilliseconds); } }
private void Publish(IS3Client client, IEnumerable<ITaskItem> sourceFiles, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { foreach (var fileItem in sourceFiles.Where(taskItem => taskItem != null && !string.IsNullOrEmpty(taskItem.GetMetadata("Identity")))) { var info = new FileInfo(fileItem.GetMetadata("Identity")); var headers = MsBuildHelpers.GetCustomItemMetadata(fileItem); Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", info.FullName)); client.PutFileWithHeaders(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, headers, publicRead, timeoutMilliseconds); } }
public Idrivee2Backend(string url, Dictionary <string, string> options) { var uri = new Utility.Uri(url); m_bucket = uri.Host; m_prefix = uri.Path; m_prefix = m_prefix.Trim(); if (m_prefix.Length != 0) { m_prefix = Util.AppendDirSeparator(m_prefix, "/"); } string accessKeyId = null; string accessKeySecret = null; if (options.ContainsKey("auth-username")) { accessKeyId = options["auth-username"]; } if (options.ContainsKey("auth-password")) { accessKeySecret = options["auth-password"]; } if (options.ContainsKey("access_key_id")) { accessKeyId = options["access_key_id"]; } if (options.ContainsKey("secret_access_key")) { accessKeySecret = options["secret_access_key"]; } if (string.IsNullOrEmpty(accessKeyId)) { throw new UserInformationException(Strings.Idrivee2Backend.NoKeyIdError, "Idrivee2NoKeyId"); } if (string.IsNullOrEmpty(accessKeySecret)) { throw new UserInformationException(Strings.Idrivee2Backend.NoKeySecretError, "Idrivee2NoKeySecret"); } string host = GetRegionEndpoint("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId); m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, options); }
private void Publish(IS3Client client, IEnumerable <string> sourceFiles, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { foreach (var f in sourceFiles) { if (string.IsNullOrEmpty(f)) { continue; } var info = new FileInfo(f); Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", info.FullName)); client.PutFile(bucket, CreateRelativePath(destinationFolder, info.Name), info.FullName, publicRead, timeoutMilliseconds); } }
public ObjectsManagementService( CephOptions cephOptions, KafkaOptions kafkaOptions, IS3Client s3Client, TemplatesStorageReader templatesStorageReader, ObjectsStorageReader objectsStorageReader, SessionStorageReader sessionStorageReader, DistributedLockManager distributedLockManager, EventSender eventSender, MetricsProvider metricsProvider) { _s3Client = s3Client; _templatesStorageReader = templatesStorageReader; _objectsStorageReader = objectsStorageReader; _sessionStorageReader = sessionStorageReader; _distributedLockManager = distributedLockManager; _eventSender = eventSender; _bucketName = cephOptions.ObjectsBucketName; _objectEventsTopic = kafkaOptions.ObjectEventsTopic; _referencedBinariesMetric = metricsProvider.GetReferencedBinariesMetric(); }
private void Publish(IS3Client client, string sourceFolder, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { var dirInfo = new DirectoryInfo(sourceFolder); var files = dirInfo.GetFiles(); foreach (var f in files) { Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", f.FullName)); client.PutFile(bucket, CreateRelativePath(destinationFolder, f.Name), f.FullName, publicRead, timeoutMilliseconds); } var dirs = dirInfo.GetDirectories(); foreach (var d in dirs) { Publish(client, d.FullName, bucket, CreateRelativePath(destinationFolder, d.Name), publicRead, timeoutMilliseconds); } }
private void Publish(IS3Client client, ITaskItem sourceFolder, string bucket, string destinationFolder, bool publicRead, int timeoutMilliseconds) { var dirInfo = new DirectoryInfo(sourceFolder.GetMetadata("Identity")); var headers = MsBuildHelpers.GetCustomItemMetadata(sourceFolder); var files = dirInfo.GetFiles(); foreach (var f in files) { Logger.LogMessage(MessageImportance.Normal, string.Format("Copying file {0}", f.FullName)); client.PutFileWithHeaders(bucket, CreateRelativePath(destinationFolder, f.Name), f.FullName, headers, publicRead, timeoutMilliseconds); } var dirs = dirInfo.GetDirectories(); foreach (var d in dirs) { Publish(client, new TaskItem(d.FullName, sourceFolder.CloneCustomMetadata()), bucket, CreateRelativePath(destinationFolder, d.Name), publicRead, timeoutMilliseconds); } }
internal static bool TryDecryptUpload(this IS3Client s3Client, string bucketName, string key, string filePath, AesCryptoServiceProvider aes, FileMetadata metadata) { try { using (var fileStream = FileUtilities.GetReadStream(filePath)) using (var cryptoStream = new CryptoStream(fileStream, aes.CreateDecryptor(), CryptoStreamMode.Read)) using (var lengthStream = new LengthStream(cryptoStream, metadata.Length)) { string md5String = Convert.ToBase64String(metadata.MD5); var request = new PutObjectRequest { BucketName = bucketName, Key = key, InputStream = lengthStream, MD5Digest = md5String }; s3Client.PutObjectAsync(request).Wait(); } return(true); } catch (Exception exception) { var processedException = ExceptionUtilities.ProcessAmazonS3Exception(exception, null); if (processedException is UserErrorException) { throw processedException; } Logger.LogLine($"Exception: {exception.Message}."); return(false); } }
public AmazonS3Client(IS3Client s3Client) : base(s3Client) { }
protected void TearDownClient() { if (_client != null) _client.Dispose(); _client = null; }
public void Dispose() { if (null != _client) _client.Dispose(); _client = null; }
public void Dispose() { s3Client?.Dispose(); s3Client = null; }
public S3(string url, Dictionary <string, string> options) { var uri = new Utility.Uri(url); uri.RequireHost(); string host = uri.Host; m_prefix = uri.Path; string awsID = null; string awsKey = null; if (options.ContainsKey("auth-username")) { awsID = options["auth-username"]; } if (options.ContainsKey("auth-password")) { awsKey = options["auth-password"]; } if (options.ContainsKey("aws_access_key_id")) { awsID = options["aws_access_key_id"]; } if (options.ContainsKey("aws_secret_access_key")) { awsKey = options["aws_secret_access_key"]; } if (!string.IsNullOrEmpty(uri.Username)) { awsID = uri.Username; } if (!string.IsNullOrEmpty(uri.Password)) { awsKey = uri.Password; } if (string.IsNullOrEmpty(awsID)) { throw new UserInformationException(Strings.S3Backend.NoAMZUserIDError, "S3NoAmzUserID"); } if (string.IsNullOrEmpty(awsKey)) { throw new UserInformationException(Strings.S3Backend.NoAMZKeyError, "S3NoAmzKey"); } bool euBuckets = Utility.Utility.ParseBoolOption(options, EU_BUCKETS_OPTION); bool useRRS = Utility.Utility.ParseBoolOption(options, RRS_OPTION); bool useSSL = Utility.Utility.ParseBoolOption(options, SSL_OPTION); string locationConstraint; options.TryGetValue(LOCATION_OPTION, out locationConstraint); if (!string.IsNullOrEmpty(locationConstraint) && euBuckets) { throw new UserInformationException(Strings.S3Backend.OptionsAreMutuallyExclusiveError(LOCATION_OPTION, EU_BUCKETS_OPTION), "S3CannotMixLocationAndEuOptions"); } if (euBuckets) { locationConstraint = S3_EU_REGION_NAME; } string storageClass; options.TryGetValue(STORAGECLASS_OPTION, out storageClass); if (string.IsNullOrWhiteSpace(storageClass) && useRRS) { storageClass = S3_RRS_CLASS_NAME; } string s3host; options.TryGetValue(SERVER_NAME, out s3host); if (string.IsNullOrEmpty(s3host)) { s3host = DEFAULT_S3_HOST; //Change in S3, now requires that you use location specific endpoint if (!string.IsNullOrEmpty(locationConstraint)) { if (DEFAULT_S3_LOCATION_BASED_HOSTS.TryGetValue(locationConstraint, out var s3hostmatch)) { s3host = s3hostmatch; } } } //Fallback to previous formats if (host.Contains(DEFAULT_S3_HOST)) { Uri u = new Uri(url); host = u.Host; m_prefix = ""; if (String.Equals(host, s3host, StringComparison.OrdinalIgnoreCase)) { m_bucket = Utility.Uri.UrlDecode(u.PathAndQuery); if (m_bucket.StartsWith("/", StringComparison.Ordinal)) { m_bucket = m_bucket.Substring(1); } if (m_bucket.Contains("/")) { m_prefix = m_bucket.Substring(m_bucket.IndexOf("/", StringComparison.Ordinal) + 1); m_bucket = m_bucket.Substring(0, m_bucket.IndexOf("/", StringComparison.Ordinal)); } } else { //Subdomain type lookup if (host.EndsWith("." + s3host, StringComparison.OrdinalIgnoreCase)) { m_bucket = host.Substring(0, host.Length - ("." + s3host).Length); host = s3host; m_prefix = Utility.Uri.UrlDecode(u.PathAndQuery); if (m_prefix.StartsWith("/", StringComparison.Ordinal)) { m_prefix = m_prefix.Substring(1); } } else { throw new UserInformationException(Strings.S3Backend.UnableToDecodeBucketnameError(url), "S3CannotDecodeBucketName"); } } Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedS3Format", null, Strings.S3Backend.DeprecatedUrlFormat("s3://" + m_bucket + "/" + m_prefix)); } else { //The new simplified url style s3://bucket/prefix m_bucket = host; host = s3host; } m_prefix = m_prefix.Trim(); if (m_prefix.Length != 0) { m_prefix = Util.AppendDirSeparator(m_prefix, "/"); } // Auto-disable dns lookup for non AWS configurations var hasForcePathStyle = options.ContainsKey("s3-ext-forcepathstyle"); if (!hasForcePathStyle && !DEFAULT_S3_LOCATION_BASED_HOSTS.Any(x => string.Equals(x.Value, host, StringComparison.OrdinalIgnoreCase)) && !string.Equals(host, "s3.amazonaws.com", StringComparison.OrdinalIgnoreCase)) { options["s3-ext-forcepathstyle"] = "true"; } options.TryGetValue(S3_CLIENT_OPTION, out var s3ClientOptionValue); if (s3ClientOptionValue == "aws" || s3ClientOptionValue == null) { s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, host, storageClass, useSSL, options); } else { s3Client = new S3MinioClient(awsID, awsKey, locationConstraint, host, storageClass, useSSL, options); } }
public AwsS3ImageStorage(IS3Client client) { _client = client; }