/// <summary> /// Computes the content crc64. /// </summary> /// <returns>The content crc64.</returns> /// <param name="input">Input.</param> /// <param name="length">stream length</param> public static string ComputeContentCrc64(Stream input, long length) { using (Crc64Stream crcStream = new Crc64Stream(input, null, length)) { byte[] buffer = new byte[32 * 1024]; int readCount = 0; while (readCount < length) { int read = crcStream.Read(buffer, 0, buffer.Length); if (read == 0) { break; } readCount += read; } if (crcStream.CalculatedHash == null) { crcStream.CalculateHash(); } if (crcStream.CalculatedHash == null || crcStream.CalculatedHash.Length == 0) { return(string.Empty); } else { return(BitConverter.ToUInt64(crcStream.CalculatedHash, 0).ToString()); } } }
public override OssObject Deserialize(ServiceResponse xmlStream) { OssObject ossObject = new OssObject(_getObjectRequest.Key) { BucketName = _getObjectRequest.BucketName, ResponseStream = xmlStream.Content, Metadata = DeserializerFactory.GetFactory() .CreateGetObjectMetadataResultDeserializer().Deserialize(xmlStream) }; DeserializeGeneric(xmlStream, ossObject); var conf = OssUtils.GetClientConfiguration(_serviceClient); var originalStream = ossObject.ResponseStream; var streamLength = ossObject.ContentLength; // setup progress var callback = _getObjectRequest.StreamTransferProgress; if (callback != null) { originalStream = OssUtils.SetupProgressListeners(originalStream, streamLength, conf.ProgressUpdateInterval, _serviceClient, callback); ossObject.ResponseStream = originalStream; } // wrap response stream in MD5Stream if (conf.EnalbeMD5Check) { byte[] expectedHashDigest = null; if (xmlStream.Headers.ContainsKey(HttpHeaders.ContentMd5)) { var md5OfResponse = xmlStream.Headers[HttpHeaders.ContentMd5]; expectedHashDigest = Convert.FromBase64String(md5OfResponse); } var hashStream = new MD5Stream(originalStream, expectedHashDigest, streamLength); ossObject.ResponseStream = hashStream; } else if (conf.EnableCrcCheck && _getObjectRequest.Range == null && string.IsNullOrEmpty(_getObjectRequest.Process)) { byte[] expectedHashDigest = null; if (xmlStream.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma)) { var crcString = xmlStream.Headers[HttpHeaders.HashCrc64Ecma]; ulong crcVal; if (ulong.TryParse(crcString, out crcVal)) { expectedHashDigest = BitConverter.GetBytes(crcVal); var hashStream = new Crc64Stream(originalStream, expectedHashDigest, streamLength); ossObject.ResponseStream = hashStream; } } } return(ossObject); }
public static PutObjectCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context, PutObjectRequest putObjectRequest) { OssUtils.CheckBucketName(putObjectRequest.BucketName); OssUtils.CheckObjectKey(putObjectRequest.Key); if (putObjectRequest.Content == null) { throw new ArgumentNullException("content"); } // handle upload callback error 203 if (putObjectRequest.IsCallbackRequest()) { context.ResponseHandlers.Add(new CallbackResponseHandler()); } var conf = OssUtils.GetClientConfiguration(client); var originalStream = putObjectRequest.Content; // setup progress var callback = putObjectRequest.StreamTransferProgress; if (callback != null) { originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback); putObjectRequest.Content = originalStream; } // wrap input stream in MD5Stream if (conf.EnalbeMD5Check) { var streamLength = originalStream.CanSeek ? originalStream.Length : -1; var hashStream = new MD5Stream(originalStream, null, streamLength); putObjectRequest.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } else if (conf.EnableCrcCheck) { var streamLength = originalStream.CanSeek ? originalStream.Length : -1; var hashStream = new Crc64Stream(originalStream, null, streamLength); putObjectRequest.Content = hashStream; context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); } return(new PutObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreatePutObjectReusltDeserializer(putObjectRequest), putObjectRequest)); }
public static AppendObjectCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context, AppendObjectRequest request) { OssUtils.CheckBucketName(request.BucketName); OssUtils.CheckObjectKey(request.Key); if (request.Content == null) { throw new ArgumentNullException("request.Content"); } request.ObjectMetadata = request.ObjectMetadata ?? new ObjectMetadata(); if (request.ObjectMetadata.ContentType == null) { request.ObjectMetadata.ContentType = HttpUtils.GetContentType(request.Key, null); } var conf = OssUtils.GetClientConfiguration(client); var originalStream = request.Content; var streamLength = request.Content.Length; // setup progress var callback = request.StreamTransferProgress; if (callback != null) { originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback); request.Content = originalStream; } // wrap input stream in MD5Stream if (conf.EnalbeMD5Check) { var hashStream = new MD5Stream(originalStream, null, streamLength); request.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } else if (conf.EnableCrcCheck && request.InitCrc != null) { var hashStream = new Crc64Stream(originalStream, null, streamLength, request.InitCrc.Value); request.Content = hashStream; context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); } return(new AppendObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateAppendObjectReusltDeserializer(), request)); }
public override void Handle(ServiceResponse response) { if (_inputStream is Crc64Stream) { Crc64Stream stream = (Crc64Stream)_inputStream; if (stream.CalculatedHash == null) { stream.CalculateHash(); } if (response.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma) && stream.CalculatedHash != null && stream.CalculatedHash.Length != 0) { var sdkCalculatedHash = BitConverter.ToUInt64(stream.CalculatedHash, 0); var ossCalculatedHashStr = response.Headers[HttpHeaders.HashCrc64Ecma]; if (!sdkCalculatedHash.ToString().Equals(ossCalculatedHashStr)) { response.Dispose(); throw new ClientException("Crc64 validation failed. Expected hash not equal to calculated hash"); } } } }
private void DoResumableDownloadSingleThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext, EventHandler <StreamTransferProgressArgs> downloadProgressCallback) { _downloadedBytes = resumableContext.GetDownloadedBytes(); if (!request.MatchingETagConstraints.Contains(resumableContext.ETag)) { request.MatchingETagConstraints.Add(resumableContext.ETag); } long totalBytes = resumableContext.GetTotalBytes(); foreach (var part in resumableContext.PartContextList) { if (part.IsCompleted) { // is CRC is enabled and part.Crc64 is 0, then redownload the data if (!_conf.EnableCrcCheck || part.Crc64 != 0) { continue; } } using (Stream fs = File.Open(GetTempDownloadFile(request), FileMode.OpenOrCreate)) { fs.Seek(part.Position, SeekOrigin.Begin); var originalStream = fs; if (downloadProgressCallback != null) { originalStream = _ossClient.SetupDownloadProgressListeners(fs, totalBytes, _downloadedBytes, _conf.ProgressUpdateInterval, downloadProgressCallback); } if (_conf.EnableCrcCheck) { originalStream = new Crc64Stream(originalStream, null, part.Length); } var getPartRequest = request.ToGetObjectRequest(); getPartRequest.SetRange(part.Position, part.Length + part.Position - 1); var partResult = _ossClient.GetObject(getPartRequest); WriteTo(partResult.Content, originalStream); if (originalStream is Crc64Stream) { Crc64Stream crcStream = originalStream as Crc64Stream; if (crcStream.CalculatedHash == null) { crcStream.CalculateHash(); } part.Crc64 = BitConverter.ToUInt64(crcStream.CalculatedHash, 0); } } part.IsCompleted = true; resumableContext.Dump(); _downloadedBytes += part.Length; } Validate(request, resumableContext); }
private void DownloadPart(object state) { DownloadTaskParam taskParam = state as DownloadTaskParam; if (taskParam == null) { throw new ClientException("Internal error. The taskParam should be type of DownloadTaskParam"); } DownloadObjectRequest request = taskParam.Request; ResumablePartContext part = taskParam.Part; EventHandler <StreamTransferProgressArgs> downloadProgressCallback = taskParam.DownloadProgressCallback; try { string fileName = GetTempDownloadFile(request); if (part.IsCompleted && File.Exists(fileName)) { // is CRC is enabled and part.Crc64 is 0, then redownload the data if (!_conf.EnableCrcCheck || part.Crc64 != 0) { return; } } const int retryCount = 3; for (int i = 0; i < retryCount; i++) { try { GetObjectRequest partRequest = request.ToGetObjectRequest(); partRequest.SetRange(part.Position, part.Position + part.Length - 1); using (var partResult = _ossClient.GetObject(partRequest)) { Crc64Stream crcStream = null; if (_conf.EnableCrcCheck) { crcStream = new Crc64Stream(partResult.Content, null, part.Length, 0); } using (var fs = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite)) { fs.Seek(part.Position, SeekOrigin.Begin); long totalBytes = WriteTo(crcStream ?? partResult.Content, fs); if (totalBytes != part.Length) { throw new OssException(string.Format("Part {0} returns {1} bytes. Expected size is {2} bytes", part.PartId, totalBytes, part.Length)); } Interlocked.Add(ref _downloadedBytes, partResult.ContentLength); } part.IsCompleted = true; if (crcStream != null) { if (crcStream.CalculatedHash == null) { crcStream.CalculateHash(); } part.Crc64 = BitConverter.ToUInt64(crcStream.CalculatedHash, 0); } return; } } catch (Exception ex) // when the connection is closed while sending the data, it will run into ObjectDisposedException. { if (!(ex is ObjectDisposedException || ex is WebException) || i == retryCount - 1) { throw; } } } throw new ClientException("DownloadPart runs into internal error"); } catch (Exception e) { taskParam.Error = e; } finally { taskParam.DownloadFinished.Set(); } }
public static UploadPartCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context, UploadPartRequest uploadPartRequest) { OssUtils.CheckBucketName(uploadPartRequest.BucketName); OssUtils.CheckObjectKey(uploadPartRequest.Key); if (string.IsNullOrEmpty(uploadPartRequest.UploadId)) { throw new ArgumentException("uploadId should be specified"); } if (!uploadPartRequest.PartNumber.HasValue) { throw new ArgumentException("partNumber should be specified"); } if (!uploadPartRequest.PartSize.HasValue) { throw new ArgumentException("partSize should be specified"); } if (uploadPartRequest.InputStream == null) { throw new ArgumentException("inputStream should be specified"); } if (uploadPartRequest.PartSize < 0 || uploadPartRequest.PartSize > OssUtils.MaxFileSize) { throw new ArgumentException("partSize not live in valid range"); } if (!OssUtils.IsPartNumberInRange(uploadPartRequest.PartNumber)) { throw new ArgumentException("partNumber not live in valid range"); } var conf = OssUtils.GetClientConfiguration(client); var originalStream = uploadPartRequest.InputStream; var streamLength = uploadPartRequest.PartSize.Value; // wrap input stream in PartialWrapperStream originalStream = new PartialWrapperStream(originalStream, streamLength); // setup progress var callback = uploadPartRequest.StreamTransferProgress; if (callback != null) { originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback); uploadPartRequest.InputStream = originalStream; } // wrap input stream in MD5Stream if (conf.EnalbeMD5Check) { var hashStream = new MD5Stream(originalStream, null, streamLength); uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } else if (conf.EnableCrcCheck) { var hashStream = new Crc64Stream(originalStream, null, streamLength); uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); } return(new UploadPartCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value, streamLength), uploadPartRequest)); }