/// <summary>
/// Computes the content crc64.
/// </summary>
/// <returns>The content crc64.</returns>
/// <param name="input">Input.</param>
/// <param name="length">stream length</param>
public static string ComputeContentCrc64(Stream input, long length)
{
using (Crc64Stream crcStream = new Crc64Stream(input, null, length))
{
byte[] buffer = new byte[32 * 1024];
int readCount = 0;
while (readCount < length)
{
int read = crcStream.Read(buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
readCount += read;
}
if (crcStream.CalculatedHash == null)
{
crcStream.CalculateHash();
}
if (crcStream.CalculatedHash == null || crcStream.CalculatedHash.Length == 0)
{
return(string.Empty);
}
else
{
return(BitConverter.ToUInt64(crcStream.CalculatedHash, 0).ToString());
}
}
}
public override OssObject Deserialize(ServiceResponse xmlStream)
{
OssObject ossObject = new OssObject(_getObjectRequest.Key)
{
BucketName = _getObjectRequest.BucketName,
ResponseStream = xmlStream.Content,
Metadata = DeserializerFactory.GetFactory()
.CreateGetObjectMetadataResultDeserializer().Deserialize(xmlStream)
};
DeserializeGeneric(xmlStream, ossObject);
var conf = OssUtils.GetClientConfiguration(_serviceClient);
var originalStream = ossObject.ResponseStream;
var streamLength = ossObject.ContentLength;
// setup progress
var callback = _getObjectRequest.StreamTransferProgress;
if (callback != null)
{
originalStream = OssUtils.SetupProgressListeners(originalStream, streamLength, conf.ProgressUpdateInterval, _serviceClient, callback);
ossObject.ResponseStream = originalStream;
}
// wrap response stream in MD5Stream
if (conf.EnalbeMD5Check)
{
byte[] expectedHashDigest = null;
if (xmlStream.Headers.ContainsKey(HttpHeaders.ContentMd5))
{
var md5OfResponse = xmlStream.Headers[HttpHeaders.ContentMd5];
expectedHashDigest = Convert.FromBase64String(md5OfResponse);
}
var hashStream = new MD5Stream(originalStream, expectedHashDigest, streamLength);
ossObject.ResponseStream = hashStream;
}
else if (conf.EnableCrcCheck && _getObjectRequest.Range == null && string.IsNullOrEmpty(_getObjectRequest.Process))
{
byte[] expectedHashDigest = null;
if (xmlStream.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma))
{
var crcString = xmlStream.Headers[HttpHeaders.HashCrc64Ecma];
ulong crcVal;
if (ulong.TryParse(crcString, out crcVal))
{
expectedHashDigest = BitConverter.GetBytes(crcVal);
var hashStream = new Crc64Stream(originalStream, expectedHashDigest, streamLength);
ossObject.ResponseStream = hashStream;
}
}
}
return(ossObject);
}
public static PutObjectCommand Create(IServiceClient client, Uri endpoint,
ExecutionContext context,
PutObjectRequest putObjectRequest)
{
OssUtils.CheckBucketName(putObjectRequest.BucketName);
OssUtils.CheckObjectKey(putObjectRequest.Key);
if (putObjectRequest.Content == null)
{
throw new ArgumentNullException("content");
}
// handle upload callback error 203
if (putObjectRequest.IsCallbackRequest())
{
context.ResponseHandlers.Add(new CallbackResponseHandler());
}
var conf = OssUtils.GetClientConfiguration(client);
var originalStream = putObjectRequest.Content;
// setup progress
var callback = putObjectRequest.StreamTransferProgress;
if (callback != null)
{
originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback);
putObjectRequest.Content = originalStream;
}
// wrap input stream in MD5Stream
if (conf.EnalbeMD5Check)
{
var streamLength = originalStream.CanSeek ? originalStream.Length : -1;
var hashStream = new MD5Stream(originalStream, null, streamLength);
putObjectRequest.Content = hashStream;
context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream));
}
else if (conf.EnableCrcCheck)
{
var streamLength = originalStream.CanSeek ? originalStream.Length : -1;
var hashStream = new Crc64Stream(originalStream, null, streamLength);
putObjectRequest.Content = hashStream;
context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream));
}
return(new PutObjectCommand(client, endpoint, context,
DeserializerFactory.GetFactory().CreatePutObjectReusltDeserializer(putObjectRequest),
putObjectRequest));
}
public static AppendObjectCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context,
AppendObjectRequest request)
{
OssUtils.CheckBucketName(request.BucketName);
OssUtils.CheckObjectKey(request.Key);
if (request.Content == null)
{
throw new ArgumentNullException("request.Content");
}
request.ObjectMetadata = request.ObjectMetadata ?? new ObjectMetadata();
if (request.ObjectMetadata.ContentType == null)
{
request.ObjectMetadata.ContentType = HttpUtils.GetContentType(request.Key, null);
}
var conf = OssUtils.GetClientConfiguration(client);
var originalStream = request.Content;
var streamLength = request.Content.Length;
// setup progress
var callback = request.StreamTransferProgress;
if (callback != null)
{
originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback);
request.Content = originalStream;
}
// wrap input stream in MD5Stream
if (conf.EnalbeMD5Check)
{
var hashStream = new MD5Stream(originalStream, null, streamLength);
request.Content = hashStream;
context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream));
}
else if (conf.EnableCrcCheck && request.InitCrc != null)
{
var hashStream = new Crc64Stream(originalStream, null, streamLength, request.InitCrc.Value);
request.Content = hashStream;
context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream));
}
return(new AppendObjectCommand(client, endpoint, context,
DeserializerFactory.GetFactory().CreateAppendObjectReusltDeserializer(),
request));
}
public override void Handle(ServiceResponse response)
{
if (_inputStream is Crc64Stream)
{
Crc64Stream stream = (Crc64Stream)_inputStream;
if (stream.CalculatedHash == null)
{
stream.CalculateHash();
}
if (response.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma) && stream.CalculatedHash != null && stream.CalculatedHash.Length != 0)
{
var sdkCalculatedHash = BitConverter.ToUInt64(stream.CalculatedHash, 0);
var ossCalculatedHashStr = response.Headers[HttpHeaders.HashCrc64Ecma];
if (!sdkCalculatedHash.ToString().Equals(ossCalculatedHashStr))
{
response.Dispose();
throw new ClientException("Crc64 validation failed. Expected hash not equal to calculated hash");
}
}
}
}
private void DoResumableDownloadSingleThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext,
EventHandler <StreamTransferProgressArgs> downloadProgressCallback)
{
_downloadedBytes = resumableContext.GetDownloadedBytes();
if (!request.MatchingETagConstraints.Contains(resumableContext.ETag))
{
request.MatchingETagConstraints.Add(resumableContext.ETag);
}
long totalBytes = resumableContext.GetTotalBytes();
foreach (var part in resumableContext.PartContextList)
{
if (part.IsCompleted)
{
// is CRC is enabled and part.Crc64 is 0, then redownload the data
if (!_conf.EnableCrcCheck || part.Crc64 != 0)
{
continue;
}
}
using (Stream fs = File.Open(GetTempDownloadFile(request), FileMode.OpenOrCreate))
{
fs.Seek(part.Position, SeekOrigin.Begin);
var originalStream = fs;
if (downloadProgressCallback != null)
{
originalStream = _ossClient.SetupDownloadProgressListeners(fs,
totalBytes,
_downloadedBytes,
_conf.ProgressUpdateInterval,
downloadProgressCallback);
}
if (_conf.EnableCrcCheck)
{
originalStream = new Crc64Stream(originalStream, null, part.Length);
}
var getPartRequest = request.ToGetObjectRequest();
getPartRequest.SetRange(part.Position, part.Length + part.Position - 1);
var partResult = _ossClient.GetObject(getPartRequest);
WriteTo(partResult.Content, originalStream);
if (originalStream is Crc64Stream)
{
Crc64Stream crcStream = originalStream as Crc64Stream;
if (crcStream.CalculatedHash == null)
{
crcStream.CalculateHash();
}
part.Crc64 = BitConverter.ToUInt64(crcStream.CalculatedHash, 0);
}
}
part.IsCompleted = true;
resumableContext.Dump();
_downloadedBytes += part.Length;
}
Validate(request, resumableContext);
}
private void DownloadPart(object state)
{
DownloadTaskParam taskParam = state as DownloadTaskParam;
if (taskParam == null)
{
throw new ClientException("Internal error. The taskParam should be type of DownloadTaskParam");
}
DownloadObjectRequest request = taskParam.Request;
ResumablePartContext part = taskParam.Part;
EventHandler <StreamTransferProgressArgs> downloadProgressCallback = taskParam.DownloadProgressCallback;
try
{
string fileName = GetTempDownloadFile(request);
if (part.IsCompleted && File.Exists(fileName))
{
// is CRC is enabled and part.Crc64 is 0, then redownload the data
if (!_conf.EnableCrcCheck || part.Crc64 != 0)
{
return;
}
}
const int retryCount = 3;
for (int i = 0; i < retryCount; i++)
{
try
{
GetObjectRequest partRequest = request.ToGetObjectRequest();
partRequest.SetRange(part.Position, part.Position + part.Length - 1);
using (var partResult = _ossClient.GetObject(partRequest))
{
Crc64Stream crcStream = null;
if (_conf.EnableCrcCheck)
{
crcStream = new Crc64Stream(partResult.Content, null, part.Length, 0);
}
using (var fs = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite))
{
fs.Seek(part.Position, SeekOrigin.Begin);
long totalBytes = WriteTo(crcStream ?? partResult.Content, fs);
if (totalBytes != part.Length)
{
throw new OssException(string.Format("Part {0} returns {1} bytes. Expected size is {2} bytes",
part.PartId, totalBytes, part.Length));
}
Interlocked.Add(ref _downloadedBytes, partResult.ContentLength);
}
part.IsCompleted = true;
if (crcStream != null)
{
if (crcStream.CalculatedHash == null)
{
crcStream.CalculateHash();
}
part.Crc64 = BitConverter.ToUInt64(crcStream.CalculatedHash, 0);
}
return;
}
}
catch (Exception ex) // when the connection is closed while sending the data, it will run into ObjectDisposedException.
{
if (!(ex is ObjectDisposedException || ex is WebException) || i == retryCount - 1)
{
throw;
}
}
}
throw new ClientException("DownloadPart runs into internal error");
}
catch (Exception e)
{
taskParam.Error = e;
}
finally
{
taskParam.DownloadFinished.Set();
}
}
public static UploadPartCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context,
UploadPartRequest uploadPartRequest)
{
OssUtils.CheckBucketName(uploadPartRequest.BucketName);
OssUtils.CheckObjectKey(uploadPartRequest.Key);
if (string.IsNullOrEmpty(uploadPartRequest.UploadId))
{
throw new ArgumentException("uploadId should be specified");
}
if (!uploadPartRequest.PartNumber.HasValue)
{
throw new ArgumentException("partNumber should be specified");
}
if (!uploadPartRequest.PartSize.HasValue)
{
throw new ArgumentException("partSize should be specified");
}
if (uploadPartRequest.InputStream == null)
{
throw new ArgumentException("inputStream should be specified");
}
if (uploadPartRequest.PartSize < 0 || uploadPartRequest.PartSize > OssUtils.MaxFileSize)
{
throw new ArgumentException("partSize not live in valid range");
}
if (!OssUtils.IsPartNumberInRange(uploadPartRequest.PartNumber))
{
throw new ArgumentException("partNumber not live in valid range");
}
var conf = OssUtils.GetClientConfiguration(client);
var originalStream = uploadPartRequest.InputStream;
var streamLength = uploadPartRequest.PartSize.Value;
// wrap input stream in PartialWrapperStream
originalStream = new PartialWrapperStream(originalStream, streamLength);
// setup progress
var callback = uploadPartRequest.StreamTransferProgress;
if (callback != null)
{
originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback);
uploadPartRequest.InputStream = originalStream;
}
// wrap input stream in MD5Stream
if (conf.EnalbeMD5Check)
{
var hashStream = new MD5Stream(originalStream, null, streamLength);
uploadPartRequest.InputStream = hashStream;
context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream));
}
else if (conf.EnableCrcCheck)
{
var hashStream = new Crc64Stream(originalStream, null, streamLength);
uploadPartRequest.InputStream = hashStream;
context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream));
}
return(new UploadPartCommand(client, endpoint, context,
DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value, streamLength),
uploadPartRequest));
}
