public static void DownloadObjectUsingRange(IOss ossClient, string bucketName, string objectKeyName, string targetFile) { var length = ossClient.GetObjectMetadata(bucketName, objectKeyName).ContentLength; var goRequest = new GetObjectRequest(bucketName, objectKeyName); goRequest.SetRange(0, length / 2); var originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.OpenOrCreate)) { WriteTo(requestStream, localfile); } } goRequest.SetRange(length / 2 + 1, length - 1); originalObject = ossClient.GetObject(goRequest); using (var requestStream = originalObject.Content) { using (var localfile = File.Open(targetFile, FileMode.Append)) { WriteTo(requestStream, localfile); } } }
/// <summary> /// 分块下载 /// </summary> /// <param name="bufferedStream"></param> /// <param name="startPos"></param> /// <param name="endPos"></param> /// <param name="localFilePath"></param> /// <param name="bucketName"></param> /// <param name="fileKey"></param> private static void Download(BufferedStream bufferedStream, long startPos, long endPos, String localFilePath, String bucketName, String fileKey) { Stream contentStream = null; try { var getObjectRequest = new GetObjectRequest(bucketName, fileKey); getObjectRequest.SetRange(startPos, endPos); var ossObject = client.GetObject(getObjectRequest); byte[] buffer = new byte[1024 * 1024]; var bytesRead = 0; bufferedStream.Seek(startPos, SeekOrigin.Begin); contentStream = ossObject.Content; while ((bytesRead = contentStream.Read(buffer, 0, buffer.Length)) > 0) { bufferedStream.Write(buffer, 0, bytesRead); } } finally { if (contentStream != null) { contentStream.Dispose(); } } }
public static DownloadResult DownloadObject(DownloadRecord record, String targetFolder, long start = 0, long end = 199) { GetObjectRequest getObjectRequest = new GetObjectRequest() { BucketName = BosConfig.BucketName, Key = record.CloudFile.Key }; // 获取 范围内的数据 getObjectRequest.SetRange(start, end); // 获取Object,返回结果为BosObject对象 try { BosObject bosObject = GetBosClient().GetObject(getObjectRequest); Stream stream = bosObject.ObjectContent; return(FileUtil.saveFileContent(stream, record.FileName, targetFolder)); } catch (BceServiceException BceServiceException) { if (BceServiceException.Message.IndexOf("The requested range cannot be satisfied") != -1) { DownloadResult result2 = new DownloadResult(); result2.state = 1; return(result2); } Console.WriteLine("exception:" + BceServiceException); } DownloadResult result = new DownloadResult(); result.state = 2; return(result); }
/// <summary> /// 分块下载 /// </summary> /// <param name="bufferedStream"></param> /// <param name="startPos"></param> /// <param name="endPos"></param> /// <param name="localFilePath"></param> /// <param name="bucketName"></param> /// <param name="fileKey"></param> public void Download(string downloadPath, long startPos, long endPos, String fileKey, string createFileName) { Stream contentStream = null; using (var fileStream = new FileStream(downloadPath + createFileName, FileMode.OpenOrCreate)) { var bufferedStream = new BufferedStream(fileStream); try { OssClient client = new OssClient(m_endPoint, m_keyID, m_keySecret); var getObjectRequest = new GetObjectRequest(m_bucket, fileKey); getObjectRequest.SetRange(startPos, endPos); var ossObject = client.GetObject(getObjectRequest); byte[] buffer = new byte[1024 * 1024]; var bytesRead = 0; bufferedStream.Seek(startPos, SeekOrigin.Begin); contentStream = ossObject.Content; while ((bytesRead = contentStream.Read(buffer, 0, buffer.Length)) > 0) { bufferedStream.Write(buffer, 0, bytesRead); } } finally { if (contentStream != null) { bufferedStream.Flush(); contentStream.Dispose(); } } } }
/// <summary> /// 分块下载 /// </summary> /// <param name="bufferedStream"></param> /// <param name="startPos"></param> /// <param name="endPos"></param> /// <param name="localFilePath"></param> /// <param name="bucketName"></param> /// <param name="fileKey"></param> private void GetObject(BufferedStream bufferedStream, long startPos, long endPos, String localFilePath, String bucketName, String fileKey) { Stream contentStream = null; try { var getObjectRequest = new GetObjectRequest(bucketName, fileKey); getObjectRequest.SetRange(startPos, endPos); var ossObject = Client.GetObject(getObjectRequest); byte[] buffer = new byte[1024 * 1024]; var bytesRead = 0; bufferedStream.Seek(startPos, SeekOrigin.Begin); contentStream = ossObject.Content; while ((bytesRead = contentStream.Read(buffer, 0, buffer.Length)) > 0) { bufferedStream.Write(buffer, 0, bytesRead); } } finally { if (contentStream != null) { contentStream.Dispose(); } } }
/// <summary> /// 读取文件 /// </summary> /// <param name="fileName"></param> /// <param name="offset"></param> /// <param name="count"></param> /// <returns></returns> public byte[] Read(OssClient client, string fileName, long offset, int count) { try { fileName = handlePath(fileName); byte[] buffer = new byte[1024]; byte[] result = new byte[count]; int resultIndex = 0; var length = 0; var getObjectRequest = new GetObjectRequest("oa-oss", fileName); getObjectRequest.SetRange(offset, offset + count - 1); var obj = client.GetObject(getObjectRequest); var contentLength = obj.ContentLength; using (var requestStream = obj.Content) { while ((length = requestStream.Read(buffer, 0, 1024)) != 0) { if (length < 1024) { var aaa = 111; } Array.Copy(buffer, 0, result, resultIndex, length); resultIndex += length; } } return(result); } catch (Exception ex) { throw new FileNotFoundException("获取文件失败:" + ex.Message); } }
private void Download(MemoryStream memoryStream, long startPos, long endPos, String bucketName, String fileKey, OssClient client) { System.IO.Stream contentStream = null; try { var getObjectRequest = new GetObjectRequest(bucketName, fileKey); getObjectRequest.SetRange(startPos, endPos); var ossObject = client.GetObject(getObjectRequest); byte[] buffer = new byte[1024 * 1024]; var bytesRead = 0; memoryStream.Seek(startPos, SeekOrigin.Begin); contentStream = ossObject.Content; while ((bytesRead = contentStream.Read(buffer, 0, buffer.Length)) > 0) { memoryStream.Write(buffer, 0, bytesRead); } } catch (Exception) { _commonFun.AlertLongText("操作异常,请重试。-->NotifiMngViewModel"); return; } finally { if (contentStream != null) { contentStream.Dispose(); } } }
public void TestGetRange() { string path = "put_object_ordinary.txt"; File.WriteAllText(path, "data"); FileInfo fileInfo = new FileInfo(path); string key = "te%%st "; PutObjectRequest request = new PutObjectRequest() { BucketName = this.bucketName, Key = key, FileInfo = fileInfo }; this.client.PutObject(request); GetObjectRequest getObjectRequest = new GetObjectRequest() { BucketName = this.bucketName, Key = key }; getObjectRequest.SetRange(0, 0); BosObject bosObject = this.client.GetObject(getObjectRequest); String content = Encoding.Default.GetString(IOUtils.StreamToBytes(bosObject.ObjectContent, bosObject.ObjectMetadata.ContentLength, 8192)); Assert.AreEqual(content, "d"); }
/// <summary> /// 分段下载 /// </summary> /// <param name="key"></param> /// <param name="targetFolder"></param> /// <param name="start"></param> /// <param name="end"></param> /// <returns></returns> public static DownloadResult DownloadObject(DownloadRecord record, string targetFolder, long start, long end) { DownloadResult downloadResult = new DownloadResult(); Random reum = new Random(); int randomdata = reum.Next(100000); string tempFileName = DateUtil.currentTimeMillis(new DateTime()) + randomdata + ".temp"; try { GetObjectRequest request = new GetObjectRequest(BosConfig.BucketName, record.CloudFile.Key, targetFolder, tempFileName); request.SetRange(start, end); //设置签名有效时长 request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); //执行请求 GetObjectResult result = getCosXmlServer().GetObject(request); Console.WriteLine("msg" + result.GetResultInfo()); if (String.Equals("OK", result.httpMessage)) { } else if (String.Equals("Requested Range Not Satisfiable", result.httpMessage)) { downloadResult.state = 1; File.Delete(targetFolder + @"\" + tempFileName); return(downloadResult); } else if (String.Equals("Not Found", result.httpMessage)) { downloadResult.state = 2; File.Delete(targetFolder + @"\" + tempFileName); return(downloadResult); } Stream stream = new FileStream(targetFolder + @"\" + tempFileName, FileMode.Open); downloadResult = FileUtil.saveFileContent(stream, record.FileName, targetFolder); File.Delete(targetFolder + @"\" + tempFileName); } catch (COSXML.CosException.CosClientException clientEx) { //请求失败 Console.WriteLine("CosClientException: " + clientEx.Message); downloadResult.state = 2; } catch (COSXML.CosException.CosServerException serverEx) { //请求失败 Console.WriteLine("CosServerException: " + serverEx.GetInfo()); downloadResult.state = 2; } return(downloadResult); }
/// <summary> /// 下载任务详情 /// </summary> /// <param name="task"></param> /// <returns></returns> private object DownloadItem(OssTaskDO task) { //获取索引 var p = DownloadReadAsync(task); if (p < 0) { return(null); } try { //阿里云下载 var ossPath = CheckOssPath(task.DownloadOssPath + @"\" + task.DownloadOssName); GetObjectRequest request = new GetObjectRequest(aliyunOSSConfig.BucketName, ossPath); request.SetRange(p * task.DebrisSize, p + 1 >= task.DownloadDebrisTotalCount ? task.DownloadFileLength : (p + 1) * task.DebrisSize); if (p + 1 >= task.DownloadDebrisTotalCount) { request.SetRange(p * task.DebrisSize, task.DownloadFileLength - 1); } //保存到磁盘 OssObject result = client.GetObject(request); if (result.HttpStatusCode.ToString() != "OK" && result.HttpStatusCode.ToString() != "PartialContent") { throw new Exception("Response NOT OK"); } DownloadWriteAsync(task, result.Content, p); } catch (Exception e) { Console.WriteLine(e.StackTrace); DownloadWriteReduction(task, p); } return(true); }
private static void Main(string[] args) { BosClient client = GenerateBosClient(); const string bucketName = "sample-bucket-getobject"; //示例Bucket名称 // 初始化:创建示例Bucket和Object client.CreateBucket(bucketName); //指定Bucket名称 string objectName = "sample"; client.PutObject(bucketName, objectName, "sampledata"); // 1. 获取BosObject对象并通过BosObject的输入流获取内容 BosObject bosObject = client.GetObject(bucketName, objectName); Stream objectContent = bosObject.ObjectContent; string content = new StreamReader(objectContent).ReadToEnd(); Console.WriteLine(content); // "sampledata" // 2. 通过GetObjectRequest只获取部分数据 GetObjectRequest getObjectRequest = new GetObjectRequest() { BucketName = bucketName, Key = objectName }; getObjectRequest.SetRange(0, 5); bosObject = client.GetObject(getObjectRequest); objectContent = bosObject.ObjectContent; content = new StreamReader(objectContent).ReadToEnd(); Console.WriteLine(content); // "sample" // 3. 直接通过GetObjectContent获取byte[]内容 byte[] bytes = client.GetObjectContent(bucketName, objectName); content = Encoding.Default.GetString(bytes); Console.WriteLine(content); // "sampledata" // 4. 将object内容下载到文件 FileInfo fileInfo = new FileInfo("d:\\lzb\\sample.txt"); client.GetObject(bucketName, objectName, fileInfo); content = File.ReadAllText(fileInfo.FullName); Console.WriteLine(content); // "sampledata" // 5. 只获取object的meta,不获取内容 ObjectMetadata objectMetadata = client.GetObjectMetadata(bucketName, objectName); Console.WriteLine(objectMetadata.ContentLength); // "10" Console.ReadKey(); }
private void GetObject() { getObjectRequest = new GetObjectRequest(bucket, key, localDir, localFileName); getObjectRequest.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600); if (progressCallback != null) { getObjectRequest.SetCosProgressCallback(progressCallback); } getObjectRequest.SetRange(rangeStart, rangeEnd); getObjectRequest.SetLocalFileOffset(localFileOffset); cosXmlServer.GetObject(getObjectRequest, delegate(CosResult result) { lock (syncExit) { if (isExit) { return; } } if (UpdateTaskState(TaskState.COMPLETED)) { GetObjectResult getObjectResult = result as GetObjectResult; DownloadTaskResult downloadTaskResult = new DownloadTaskResult(); downloadTaskResult.SetResult(getObjectResult); if (successCallback != null) { successCallback(downloadTaskResult); } } }, delegate(CosClientException clientEx, CosServerException serverEx) { lock (syncExit) { if (isExit) { return; } } if (UpdateTaskState(TaskState.FAILED)) { if (failCallback != null) { failCallback(clientEx, serverEx); } } }); }
/// <summary> /// 通过GetObjectRequest 可以支持断点续传和分段下载 /// </summary> /// <param name="bucketName"></param> /// <param name="key"></param> public void GetObjectMult(String bucketName, string key, string fileToDownload) { var getObjectRequest = new GetObjectRequest(bucketName, key); getObjectRequest.SetRange(20, 100); var o = ossClient.GetObject(getObjectRequest); using (var requestStream = o.Content) { byte[] buf = new byte[1024]; var fs = File.Open(fileToDownload, FileMode.OpenOrCreate); var len = 0; while ((len = requestStream.Read(buf, 0, 1024)) != 0) { fs.Write(buf, 0, len); } fs.Close(); } }
public static void GetObjectByRequest(string bucketName) { try { client.PutObject(bucketName, key, fileToUpload); var request = new GetObjectRequest(bucketName, key); request.SetRange(0, 100); var result = client.GetObject(request); Console.WriteLine("Get object succeeded, length:{0}", result.Metadata.ContentLength); } catch (OssException ex) { Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}\tHostID:{3}", ex.ErrorCode, ex.Message, ex.RequestId, ex.HostId); } catch (Exception ex) { Console.WriteLine("Failed with error info: {0}", ex.Message); } }
// actual get object requests with concurrency control private void GetObject(string crc64ecma) { lock (syncExit) { if (isExit) { return; } } // create dir if not exist DirectoryInfo dirInfo = new DirectoryInfo(localDir); if (!dirInfo.Exists) { dirInfo.Create(); } // concurrency control AutoResetEvent resetEvent = new AutoResetEvent(false); int retries = 0; // return last response GetObjectResult downloadResult = null; if (sliceToRemove == null) { sliceToRemove = new HashSet <int>(); } while (sliceList.Count != 0 && retries < maxRetries) { retries += 1; foreach (int partNumber in sliceList.Keys) { if (sliceToRemove.Contains(partNumber)) { continue; } DownloadSliceStruct slice; bool get_state = sliceList.TryGetValue(partNumber, out slice); if (activeTasks >= maxTasks) { resetEvent.WaitOne(); } lock (syncExit) { if (isExit) { return; } } string tmpFileName = "." + localFileName + ".cosresumable." + slice.partNumber; // clear remainance FileInfo tmpFileInfo = new FileInfo(localDir + tmpFileName); if (tmpFileInfo.Exists && tmpFileInfo.Length != (slice.sliceEnd - slice.sliceStart + 1) && localFileOffset != 0) { System.IO.File.Delete(localDir + tmpFileName); } getObjectRequest = new GetObjectRequest(bucket, key, localDir, tmpFileName); tmpFilePaths.Add(localDir + tmpFileName); getObjectRequest.SetRange(slice.sliceStart, slice.sliceEnd); if (progressCallback != null && this.sliceList.Count == 1) { getObjectRequest.SetCosProgressCallback(delegate(long completed, long total) { progressCallback(completed, total); } ); } Interlocked.Increment(ref activeTasks); cosXmlServer.GetObject(getObjectRequest, delegate(CosResult result) { Interlocked.Decrement(ref activeTasks); lock (syncExit) { if (isExit) { return; } } sliceToRemove.Add(partNumber); if (progressCallback != null && this.sliceList.Count > 1) { long completed = sliceToRemove.Count * this.sliceSize; long total = rangeEnd - rangeStart; if (completed > total) { completed = total; } progressCallback(completed, total); } downloadResult = result as GetObjectResult; resetEvent.Set(); if (resumable) { // flush done parts this.resumableInfo.slicesDownloaded.Add(slice); this.resumableInfo.Persist(resumableTaskFile); } }, delegate(CosClientException clientEx, CosServerException serverEx) { Interlocked.Decrement(ref activeTasks); lock (syncExit) { if (isExit) { return; } } // server 4xx throw and stop if (serverEx != null && serverEx.statusCode < 500) { throw serverEx; if (failCallback != null) { failCallback(null, serverEx); } return; } // client cannot connect, just retry if (clientEx != null) { gClientExp = clientEx; } resetEvent.Set(); } ); } while (activeTasks != 0) { Thread.Sleep(100); } // remove success parts foreach (int partNumber in sliceToRemove) { sliceList.Remove(partNumber); } } if (this.sliceList.Count != 0) { if (gClientExp != null) { throw gClientExp; } COSXML.CosException.CosClientException clientEx = new COSXML.CosException.CosClientException ((int)CosClientError.InternalError, "max retries " + retries + " excceed, download fail"); throw clientEx; if (UpdateTaskState(TaskState.Failed)) { if (failCallback != null) { failCallback(clientEx, null); } } return; } // file merge FileMode fileMode = FileMode.OpenOrCreate; FileInfo localFileInfo = new FileInfo(localDir + localFileName); if (localFileInfo.Exists && localFileOffset == 0 && localFileInfo.Length != rangeEnd - rangeStart + 1) { fileMode = FileMode.Truncate; } using (var outputStream = File.Open(localDir + localFileName, fileMode)) { outputStream.Seek(localFileOffset, 0); // sort List <string> tmpFileList = new List <string>(this.tmpFilePaths); tmpFileList.Sort(delegate(string x, string y){ int partNumber1 = int.Parse(x.Split(new string[] { "cosresumable." }, StringSplitOptions.None)[1]); int partNumber2 = int.Parse(y.Split(new string[] { "cosresumable." }, StringSplitOptions.None)[1]); return(partNumber1 - partNumber2); }); foreach (var inputFilePath in tmpFileList) { // tmp not exist, clear everything and ask for retry if (!File.Exists(inputFilePath)) { // check if download already completed if (File.Exists(localDir + localFileName)) { FileInfo fileInfo = new FileInfo(localDir + localFileName); if (fileInfo.Length == rangeEnd - rangeStart + 1) { foreach (var tmpFile in tmpFileList) { System.IO.File.Delete(tmpFile); } if (resumableTaskFile != null) { System.IO.File.Delete(resumableTaskFile); } break; } } // not completed, report fatal error foreach (var tmpFile in tmpFileList) { System.IO.File.Delete(tmpFile); } if (resumableTaskFile != null) { System.IO.File.Delete(resumableTaskFile); } if (File.Exists(localDir + localFileName)) { System.IO.File.Delete(localDir + localFileName); } COSXML.CosException.CosClientException clientEx = new COSXML.CosException.CosClientException ((int)CosClientError.InternalError, "local tmp file not exist, could be concurrent writing same file" + inputFilePath + " download again"); throw clientEx; if (failCallback != null) { failCallback(clientEx, null); } break; } using (var inputStream = File.OpenRead(inputFilePath)) { FileInfo info = new FileInfo(inputFilePath); inputStream.CopyTo(outputStream); } } tmpFileList.Clear(); tmpFilePaths.Clear(); if (UpdateTaskState(TaskState.Completed)) { var dir = new DirectoryInfo(localDir); foreach (var file in dir.EnumerateFiles("." + localFileName + ".cosresumable.*")) { file.Delete(); } if (resumableTaskFile != null) { FileInfo info = new FileInfo(resumableTaskFile); if (info.Exists) { info.Delete(); } } DownloadTaskResult downloadTaskResult = new DownloadTaskResult(); downloadTaskResult.SetResult(downloadResult); outputStream.Close(); if (successCallback != null) { successCallback(downloadTaskResult); } return; } else { // 容灾 return DownloadTaskResult downloadTaskResult = new DownloadTaskResult(); downloadTaskResult.SetResult(downloadResult); outputStream.Close(); if (successCallback != null) { successCallback(downloadTaskResult); } } } return; }
public static bool DownloadFile(this BosClient bosClient, BOSDownloadRequest bosDownloadRequest, IProgress <int> onProgressPercentChanged) { Debug.Assert(bosDownloadRequest.RequestInfo.DownloadSize > 0); Debug.Assert(bosDownloadRequest.RequestInfo.DownloadedBytes % bosDownloadRequest.RequestInfo.DownloadSize == 0); var filePath = bosDownloadRequest.RequestInfo.FilePath; var getObjectRequest = new GetObjectRequest { BucketName = bosDownloadRequest.RequestInfo.Bucket, Key = bosDownloadRequest.RequestInfo.ObjectKey, }; var objectMetadata = bosClient.GetObjectMetadata(getObjectRequest); var fileLength = objectMetadata.ContentLength; Debug.Assert(bosDownloadRequest.RequestInfo.DownloadedBytes < fileLength); using (var stream = File.OpenWrite(filePath)) { if (bosDownloadRequest.RequestInfo.DownloadedBytes == 0) { // This doesn't work: //stream.Seek(fileLength, SeekOrigin.Begin); stream.SetLength(fileLength); } stream.Seek(bosDownloadRequest.RequestInfo.DownloadedBytes, SeekOrigin.Begin); int parts = (int)(fileLength / bosDownloadRequest.RequestInfo.DownloadSize); if (fileLength % bosDownloadRequest.RequestInfo.DownloadSize != 0) { parts += 1; } for (int i = (int)(bosDownloadRequest.RequestInfo.DownloadedBytes / bosDownloadRequest.RequestInfo.DownloadSize); i < parts; ++i) { if (bosDownloadRequest.PauseCancellationToken.IsCancellationRequested) { return(false); } if (bosDownloadRequest.AbortCancellationToken.IsCancellationRequested) { stream.Close(); return(false); } var startBytes = bosDownloadRequest.RequestInfo.DownloadSize * i; var endBytes = startBytes + bosDownloadRequest.RequestInfo.DownloadSize - 1; if (endBytes > fileLength) { endBytes = fileLength; } onProgressPercentChanged?.Report((int)(((double)startBytes / fileLength) * 100)); getObjectRequest.SetRange(startBytes, endBytes); var bosObject = bosClient.GetObject(getObjectRequest); bosObject.ObjectContent.CopyTo(stream); bosDownloadRequest.RequestInfo.DownloadedBytes += endBytes + 1 - startBytes; } onProgressPercentChanged?.Report(100); } return(true); }