public ObjectOperateThread(IOss client, string fileName, int index, ManualResetEvent doneEvent) { _client = client; _fileName = fileName; _index = index; _doneEvent = doneEvent; }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); //upload sample object as source object _sourceObjectKey = OssTestUtils.GetObjectKey(_className); var metadata = new ObjectMetadata(); var poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _sourceObjectKey, Config.UploadSampleFile, metadata); _sourceObjectETag = poResult.ETag; //upload multipart sample object as source object _sourceBigObjectKey = _sourceObjectKey + ".js"; metadata = new ObjectMetadata(); poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _sourceBigObjectKey, Config.MultiUploadSampleFile, metadata); _sourceBigObjectETag = poResult.ETag; }
public static void DownloadObjectUsingRange(IOss ossClient, string bucketName, string objectKeyName, string targetFile) { var length = ossClient.GetObjectMetadata(bucketName, objectKeyName).ContentLength; var goRequest = new GetObjectRequest(bucketName, objectKeyName); goRequest.SetRange(0, length / 2); var originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.OpenOrCreate)) { WriteTo(requestStream, localfile); } } goRequest.SetRange(length / 2 + 1, length - 1); originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.Append)) { WriteTo(requestStream, localfile); } } }
public static void EnsureBucketExist(IOss oss, string bucketName) { if (!BucketExists(oss, bucketName)) { oss.CreateBucket(bucketName); } }
public static void CleanBucket(IOss oss, string bucketName) { if (!BucketExists(oss, bucketName)) { return; } //abort in progress multipart uploading var multipartListing = oss.ListMultipartUploads(new ListMultipartUploadsRequest(bucketName)); foreach (var upload in multipartListing.MultipartUploads) { var abortRequest = new AbortMultipartUploadRequest(bucketName, upload.Key, upload.UploadId); oss.AbortMultipartUpload(abortRequest); } // Clean up objects var objects = oss.ListObjects(bucketName); foreach (var obj in objects.ObjectSummaries) { oss.DeleteObject(bucketName, obj.Key); } // Delete the bucket. oss.DeleteBucket(bucketName); }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); _ossPayerClient = new OssClient(Config.Endpoint, Config.PayerAccessKeyId, Config.PayerAccessKeySecret); //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); string policy = "{\"Version\":\"1\",\"Statement\":[{\"Action\":[\"oss:*\"],\"Effect\": \"Allow\"," + "\"Principal\":[\"" + Config.PayerUid + "\"]," + "\"Resource\": [\"acs:oss:*:*:" + _bucketName + "\",\"acs:oss:*:*:" + _bucketName + "/*\"]}]}"; _ossClient.SetBucketPolicy(new SetBucketPolicyRequest(_bucketName, policy)); _ossClient.SetBucketRequestPayment(new SetBucketRequestPaymentRequest(_bucketName, RequestPayer.Requester)); _archiveBucketName = _bucketName + "archive"; _ossClient.CreateBucket(_bucketName); _ossClient.CreateBucket(_archiveBucketName, StorageClass.Archive); policy = "{\"Version\":\"1\",\"Statement\":[{\"Action\":[\"oss:*\"],\"Effect\": \"Allow\"," + "\"Principal\":[\"" + Config.PayerUid + "\"]," + "\"Resource\": [\"acs:oss:*:*:" + _archiveBucketName + "\",\"acs:oss:*:*:" + _archiveBucketName + "/*\"]}]}"; _ossClient.SetBucketPolicy(new SetBucketPolicyRequest(_archiveBucketName, policy)); _ossClient.SetBucketRequestPayment(new SetBucketRequestPaymentRequest(_archiveBucketName, RequestPayer.Requester)); }
/// <summary> /// 重试上传 OSS 失败的简历 /// </summary> /// <param name="newOss"></param> /// <param name="newBucket"></param> private static void RetryUploadOss(IOss newOss, string newBucket) { while (true) { Tuple <string, FormaterResume, string> resume; if (!retryResumeQueue.TryDequeue(out resume)) { continue; } using (var stream = new MemoryStream(GZip.Compress(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(resume.Item2))))) { try { newOss.PutObject(newBucket, $"{ConfigurationManager.AppSettings["Oss.New.ResumePath"]}{resume.Item1}", stream); var uploadSuccessPath = ConfigurationManager.AppSettings["51Job_UploadSuccessFiles_Path"] + companyName + Path.GetFileName(resume.Item3); if (File.Exists(uploadSuccessPath)) { File.Delete(uploadSuccessPath); } File.Move(resume.Item3, uploadSuccessPath); } catch (Exception) { retryResumeQueue.Enqueue(resume); } } } }
public AliyunOssStorageUploadProcessor(AliyunOssStorageConfigure configure, IOss client, IFileNameGenerator fileNameGenerator, ISubdirectoryGenerator subdirectoryGenerator, IFileValidator fileValidator) { Configure = configure; Client = client; FileNameGenerator = fileNameGenerator; SubdirectoryGenerator = subdirectoryGenerator; FileValidator = fileValidator; }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); //prefix of bucket name used in current test class _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); }
public static PutObjectResult UploadObject(IOss ossClient, string bucketName, string objectKeyName, string originalFile) { using (Stream fs = File.OpenRead(originalFile)) { return(ossClient.PutObject(bucketName, objectKeyName, fs)); } }
public void MultipartUploadComplexStepTestWithoutCrc() { Common.ClientConfiguration config = new Common.ClientConfiguration(); config.EnableCrcCheck = false; IOss ossClient = OssClientFactory.CreateOssClient(config); MultipartUploadComplexStepTest(ossClient); }
public static void ClassInitialize() { //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); _ossClient = OssClientFactory.CreateOssClient(); _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); }
public AliyunOssStorageChunkedUploadProcessor(ChunkedUploadAliyunOssStorageConfigure configure, IFileValidator fileValidator, IOss client, IMemoryCache memoryCache, IFileNameGenerator fileNameGenerator, ISubdirectoryGenerator subdirectoryGenerator) { Configure = configure; FileValidator = fileValidator; MemoryCache = memoryCache; FileNameGenerator = fileNameGenerator; SubdirectoryGenerator = subdirectoryGenerator; Client = client; }
public static IAsyncResult BeginUploadObject(IOss ossClient, string bucketName, string objectKeyName, string originalFile, AsyncCallback callback, object callbackObject) { var metadata = new ObjectMetadata(); metadata.CacheControl = "No-Cache"; metadata.ContentType = "text/html"; return(ossClient.BeginPutObject(bucketName, objectKeyName, originalFile, metadata, callback, callbackObject)); }
public static PutObjectResult UploadObject(IOss ossClient, string bucketName, string objectKeyName, string originalFile, ObjectMetadata metadata) { //using (var fs = File.Open(originalFile, FileMode.Open)) //{ // return ossClient.PutObject(bucketName, objectKeyName, fs, metadata); //} return(ossClient.PutObject(bucketName, objectKeyName, originalFile, metadata)); }
public ExcelIEService(IDbService dbService, IExcelIEDomainService excelIEDomainService, ICapPublisher capPublisher, IRedisOperation redis, IExcelExporter IExcelExporter, IOss oss, ILogger <ExcelIEService> logger) { _dbService = dbService; _excelIEDomainService = excelIEDomainService; _capPublisher = capPublisher; _redis = redis; _iExcelExporter = IExcelExporter; _oss = oss; _logger = logger; }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); }
public AliyunStorageService(IConfiguration configuration, ILogger <AliyunStorageService> logger) { this.configuration = configuration; this.logger = logger; var endPoint = configuration["CloudStorage:Aliyun:EndPoint"]; var accessKeyId = configuration["CloudStorage:Aliyun:AccessKeyId"]; var accessKeySecret = configuration["CloudStorage:Aliyun:AccessKeySecret"]; ossClient = new OssClient(endPoint, accessKeyId, accessKeySecret); }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); //create the bucket _bucketName = OssTestUtils.GetBucketName("bucket-inventory"); _bucketName2 = _bucketName + "-dest"; _ossClient.CreateBucket(_bucketName); _ossClient.CreateBucket(_bucketName2); }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); }
public static void DownloadObject(IOss ossClient, string bucketName, string objectKeyName, string targetFile) { var originalObject = ossClient.GetObject(bucketName, objectKeyName); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.OpenOrCreate)) { WriteTo(requestStream, localfile); } } }
private async Task <bool> BlobExistsAsync(IOss ossClient, BlobProviderArgs args, string blobName) { var bucketExists = await BucketExistsAsync(ossClient, args); if (bucketExists) { var objectExists = ossClient.DoesObjectExist(GetBucketName(args), blobName); return(objectExists); } return(false); }
private void AssertRequestTimeValidatity(ClientConfiguration conf) { _ossClient = new OssClient(new Uri(Config.Endpoint), Config.AccessKeyId, Config.AccessKeySecret, conf); try { _ossClient.ListBuckets(); } catch (Exception e) { Assert.Fail(e.Message); } }
private void AssertRequestTimeTooSkewed(ClientConfiguration conf) { _ossClient = new OssClient(new Uri(Config.Endpoint), Config.AccessKeyId, Config.AccessKeySecret, conf); try { _ossClient.ListBuckets(); } catch (OssException e) { Assert.AreEqual(OssErrorCode.RequestTimeTooSkewed, e.ErrorCode); } }
/// <summary> /// 获取Oss简历 /// </summary> /// <param name="client"></param> /// <param name="bucketName"></param> private static void ListObject(IOss client, string bucketName) { while (true) { try { ObjectListing result; var nextMarker = "";//File.ReadAllText("NextMarker.txt"); do { var listObjectsRequest = new ListObjectsRequest(bucketName) { Prefix = "Zhaopin/", Marker = nextMarker, MaxKeys = 100 }; result = client.ListObjects(listObjectsRequest); foreach (var summary in result.ObjectSummaries) { fileQueue.Enqueue(summary.Key); } while (true) { if (fileQueue.Count != 0) { Thread.Sleep(100); continue; } break; } nextMarker = result.NextMarker; //File.WriteAllText("NextMarker.txt", nextMarker); } while (result.IsTruncated); } catch (Exception ex) { Trace.TraceError("List object failed. {0}", ex.Message); } Thread.Sleep(5000); } }
public static void MultiPartUpload(IOss ossClient, string bucketName, string objectKeyName, string originalFile, int numberOfParts, int totalSize) { var initRequest = new InitiateMultipartUploadRequest(bucketName, objectKeyName); var initResult = ossClient.InitiateMultipartUpload(initRequest); var partSize = totalSize % numberOfParts == 0 ? totalSize / numberOfParts : totalSize / numberOfParts + 1; LogUtility.LogMessage("Each part size is {0} KB", partSize); //change to Byte partSize *= 1024; var partFile = new FileInfo(originalFile); // Create a list to save the result var partETags = new List <PartETag>(); //upload the file using (Stream fs = new FileStream(partFile.FullName, FileMode.Open)) { for (var i = 0; i < numberOfParts; i++) { // Skip to the start position long skipBytes = partSize * i; fs.Position = skipBytes; // Calculate the part size long size = partSize < partFile.Length - skipBytes ? partSize : partFile.Length - skipBytes; // Create a UploadPartRequest, uploading the part var uploadPartRequest = new UploadPartRequest(bucketName, objectKeyName, initResult.UploadId); uploadPartRequest.InputStream = fs; uploadPartRequest.PartSize = size; uploadPartRequest.PartNumber = (i + 1); var uploadPartResult = ossClient.UploadPart(uploadPartRequest); // Save the result partETags.Add(uploadPartResult.PartETag); } } var completeRequest = new CompleteMultipartUploadRequest(bucketName, objectKeyName, initResult.UploadId); foreach (var partETag in partETags) { completeRequest.PartETags.Add(partETag); } ossClient.CompleteMultipartUpload(completeRequest); }
public static void MultiPartUpload(IOss ossClient, string bucketName, string objectKeyName, string originalFile, int numberOfParts, int totalSize) { var initRequest = new InitiateMultipartUploadRequest(bucketName, objectKeyName); var initResult = ossClient.InitiateMultipartUpload(initRequest); var partSize = totalSize % numberOfParts == 0 ? totalSize / numberOfParts : totalSize / numberOfParts + 1; LogUtility.LogMessage("Each part size is {0} KB", partSize); //change to Byte partSize *= 1024; var partFile = new FileInfo(originalFile); // 新建一个List保存每个分块上传后的ETag和PartNumber var partETags = new List <PartETag>(); //upload the file using (var fs = new FileStream(partFile.FullName, FileMode.Open)) { for (var i = 0; i < numberOfParts; i++) { // 跳到每个分块的开头 long skipBytes = partSize * i; fs.Position = skipBytes; // 计算每个分块的大小 long size = partSize < partFile.Length - skipBytes ? partSize : partFile.Length - skipBytes; // 创建UploadPartRequest,上传分块 var uploadPartRequest = new UploadPartRequest(bucketName, objectKeyName, initResult.UploadId); uploadPartRequest.InputStream = fs; uploadPartRequest.PartSize = size; uploadPartRequest.PartNumber = (i + 1); var uploadPartResult = ossClient.UploadPart(uploadPartRequest); // 将返回的PartETag保存到List中。 partETags.Add(uploadPartResult.PartETag); } } var completeRequest = new CompleteMultipartUploadRequest(bucketName, objectKeyName, initResult.UploadId); foreach (var partETag in partETags) { completeRequest.PartETags.Add(partETag); } ossClient.CompleteMultipartUpload(completeRequest); }
private void AssertRequestTimeValidatity(ClientConfiguration conf) { var uri = new Uri(Config.Endpoint.ToLower().Trim().StartsWith("http") ? Config.Endpoint.Trim() : "http://" + Config.Endpoint.Trim()); _ossClient = new OssClient(uri, Config.AccessKeyId, Config.AccessKeySecret, conf); try { _ossClient.ListBuckets(); } catch (Exception e) { Assert.Fail(e.Message); } }
private void AssertRequestTimeTooSkewed(ClientConfiguration conf) { var uri = new Uri(Config.Endpoint.ToLower().Trim().StartsWith("http") ? Config.Endpoint.Trim() : "http://" + Config.Endpoint.Trim()); _ossClient = new OssClient(uri, Config.AccessKeyId, Config.AccessKeySecret, conf); try { _ossClient.ListBuckets(); } catch (OssException e) { Assert.AreEqual(OssErrorCode.RequestTimeTooSkewed, e.ErrorCode); } }
public static void ClassInitialize() { _config = new ClientConfiguration(); //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(_config); //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); //create sample object _objectKey = OssTestUtils.GetObjectKey(_className); OssTestUtils.UploadObject(_ossClient, _bucketName, _objectKey, Config.UploadTestFile, new ObjectMetadata()); }
private static void PutAndGetObject(IOss ossClient, string originalFile, int i) { //prepare the object key var key = OssTestUtils.GetObjectKey(_className); key = string.Format("{0}_{1}", key, i); //prepare the file for download (GET) var targetFile = OssTestUtils.GetTargetFileName(_className); targetFile = string.Format("{0}_{1}", targetFile, i); targetFile = Path.Combine(Config.DownloadFolder, targetFile); try { LogUtility.LogMessage("Thread {0}--Begin to put object", i); var sWatch = new Stopwatch(); sWatch.Start(); OssTestUtils.UploadObject(ossClient, _bucketName, key, originalFile); sWatch.Stop(); var duration = sWatch.Elapsed; LogUtility.LogMessage("Thread {0}--Put Duration is {1:00}:{2:00}:{3:00}", i, duration.Hours, duration.Minutes, duration.Seconds); LogUtility.LogMessage("Finish to put object"); Assert.IsTrue(OssTestUtils.ObjectExists(ossClient, _bucketName, key)); LogUtility.LogMessage("Thread {0}--Begin to get object", i); sWatch = new Stopwatch(); sWatch.Start(); OssTestUtils.DownloadObject(ossClient, _bucketName, key, targetFile); sWatch.Stop(); duration = sWatch.Elapsed; LogUtility.LogMessage("Thread {0}--Get Duration is {1:00}:{2:00}:{3:00}", i, duration.Hours, duration.Minutes, duration.Seconds); LogUtility.LogMessage("Thread {0}--Finish to get object", i); } finally { FileUtils.DeleteFile(originalFile); FileUtils.DeleteFile(targetFile); if (OssTestUtils.ObjectExists(ossClient, _bucketName, key)) { ossClient.DeleteObject(_bucketName, key); } } }
public static void ClassInitialize() { //get a OSS client object _ossClient = OssClientFactory.CreateOssClient(); //get current class name, which is prefix of bucket/object _className = TestContext.CurrentContext.Test.FullName; _className = _className.Substring(_className.LastIndexOf('.') + 1).ToLowerInvariant(); //create the bucket _bucketName = OssTestUtils.GetBucketName(_className); _ossClient.CreateBucket(_bucketName); //create sample object _keyName = OssTestUtils.GetObjectKey(_className); _keyName += ".jpg"; _process = "image/resize,m_fixed,w_100,h_100"; _localImageFile = "../../example.jpg"; _processedKey = "process/image" + _keyName; _imageInfo = "{\n \"FileSize\": {\"value\": \"3587\"},\n \"Format\": {\"value\": \"jpg\"},\n \"ImageHeight\": {\"value\": \"100\"},\n \"ImageWidth\": {\"value\": \"100\"}}"; }
public static void CleanBucket(IOss oss, string bucketName) { if (!BucketExists(oss, bucketName)) return; //abort in progress multipart uploading var multipartListing = oss.ListMultipartUploads(new ListMultipartUploadsRequest(bucketName)); foreach (var upload in multipartListing.MultipartUploads) { var abortRequest = new AbortMultipartUploadRequest(bucketName, upload.Key, upload.UploadId); oss.AbortMultipartUpload(abortRequest); } // Clean up objects var objects = oss.ListObjects(bucketName); foreach(var obj in objects.ObjectSummaries) { oss.DeleteObject(bucketName, obj.Key); } // Delete the bucket. oss.DeleteBucket(bucketName); }
public static void DownloadObjectUsingRange(IOss ossClient, string bucketName, string objectKeyName, string targetFile) { var length = ossClient.GetObjectMetadata(bucketName, objectKeyName).ContentLength; var goRequest = new GetObjectRequest(bucketName, objectKeyName); goRequest.SetRange(0, length / 2); var originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.OpenOrCreate)) { WriteTo(requestStream, localfile); } } goRequest.SetRange(length / 2 + 1, length -1); originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.Append)) { WriteTo(requestStream, localfile); } } }
public static void MultiPartUpload(IOss ossClient, string bucketName, string objectKeyName, string originalFile, int numberOfParts, int totalSize) { var initRequest = new InitiateMultipartUploadRequest(bucketName, objectKeyName); var initResult = ossClient.InitiateMultipartUpload(initRequest); var partSize = totalSize%numberOfParts == 0 ? totalSize/numberOfParts : totalSize/numberOfParts + 1; LogUtility.LogMessage("Each part size is {0} KB", partSize); //change to Byte partSize *= 1024; var partFile = new FileInfo(originalFile); // 新建一个List保存每个分块上传后的ETag和PartNumber var partETags = new List<PartETag>(); //upload the file using (var fs = new FileStream(partFile.FullName, FileMode.Open)) { for (var i = 0; i < numberOfParts; i++) { // 跳到每个分块的开头 long skipBytes = partSize * i; fs.Position = skipBytes; // 计算每个分块的大小 long size = partSize < partFile.Length - skipBytes ? partSize : partFile.Length - skipBytes; // 创建UploadPartRequest,上传分块 var uploadPartRequest = new UploadPartRequest(bucketName, objectKeyName, initResult.UploadId); uploadPartRequest.InputStream = fs; uploadPartRequest.PartSize = size; uploadPartRequest.PartNumber = (i + 1); var uploadPartResult = ossClient.UploadPart(uploadPartRequest); // 将返回的PartETag保存到List中。 partETags.Add(uploadPartResult.PartETag); } } var completeRequest = new CompleteMultipartUploadRequest(bucketName, objectKeyName, initResult.UploadId); foreach (var partETag in partETags) { completeRequest.PartETags.Add(partETag); } ossClient.CompleteMultipartUpload(completeRequest); }
public static bool ObjectExists(IOss ossClient, string bucketName, string objectKeyName) { return ossClient.ListObjects(bucketName).ObjectSummaries.Select( e => e.Key).Contains(objectKeyName); }
public static PutObjectResult UploadObject(IOss ossClient, string bucketName, string objectKeyName, string originalFile, ObjectMetadata metadata) { //using (var fs = File.Open(originalFile, FileMode.Open)) //{ // return ossClient.PutObject(bucketName, objectKeyName, fs, metadata); //} return ossClient.PutObject(bucketName, objectKeyName, originalFile, metadata); }
public static PutObjectResult UploadObject(IOss ossClient, string bucketName, string objectKeyName, string originalFile) { using (var fs = File.Open(originalFile, FileMode.Open)) { return ossClient.PutObject(bucketName, objectKeyName, fs); } }
public static bool BucketExists(IOss oss, string bucketName) { return oss.DoesBucketExist(bucketName); }