public void TestChecksumStreamingWithMultiStreams()
{
const string bucketName = "TestChecksumStreamingWithMultiStreams";
try
{
// Creates a bucket if it does not already exist.
_helpers.EnsureBucketExists(bucketName);
/* using 1GB file with 100MB blobs size so each stream will have 1 chunk with 10 blobs */
const long streamLength = 1L * 1024L * 1024L * 1024L;
const long blobSize = 100L * 1024L * 1024L;
var directoryObjects = new List<Ds3Object>
{
new Ds3Object("bigFile1", streamLength),
new Ds3Object("bigFile2", streamLength),
new Ds3Object("bigFile3", streamLength)
};
// Test the PUT
var putJob = _helpers.StartWriteJob(bucketName, directoryObjects, new Ds3WriteJobOptions { MaxUploadSize = blobSize }, new WriteStreamHelperStrategy());
var putCryptoStreams = new Dictionary<string, CryptoStream>();
var putMd5s = new Dictionary<string, MD5>();
directoryObjects.ForEach(obj =>
{
var md5 = MD5.Create();
var fileStream = new ChecksumStream(streamLength, this._copyBufferSize.Value);
var md5Stream = new CryptoStream(fileStream, md5, CryptoStreamMode.Read);
putCryptoStreams.Add(obj.Name, md5Stream);
putMd5s.Add(obj.Name, md5);
});
putJob.Transfer(fileName => putCryptoStreams[fileName]);
foreach (var stream in putCryptoStreams.Select(pair => pair.Value).Where(stream => !stream.HasFlushedFinalBlock))
{
stream.FlushFinalBlock();
}
foreach (var md5 in putMd5s.Select(pair => pair.Value))
{
Assert.AreEqual("Rt83cCvGZHQGu3eRIdfJIQ==", Convert.ToBase64String(md5.Hash));
}
// Test the GET
var getJob = _helpers.StartReadAllJob(bucketName, helperStrategy: new ReadStreamHelperStrategy());
var getCryptoStreams = new Dictionary<string, CryptoStream>();
var getMd5s = new Dictionary<string, MD5>();
directoryObjects.ForEach(obj =>
{
var md5 = MD5.Create();
var fileStream = new ChecksumStream(streamLength, this._copyBufferSize.Value);
var md5Stream = new CryptoStream(fileStream, md5, CryptoStreamMode.Write);
getCryptoStreams.Add(obj.Name, md5Stream);
getMd5s.Add(obj.Name, md5);
});
getJob.Transfer(fileName => getCryptoStreams[fileName]);
foreach (var stream in getCryptoStreams.Select(pair => pair.Value).Where(stream => !stream.HasFlushedFinalBlock))
{
stream.FlushFinalBlock();
}
foreach (var md5 in getMd5s.Select(pair => pair.Value))
{
Assert.AreEqual("Rt83cCvGZHQGu3eRIdfJIQ==", Convert.ToBase64String(md5.Hash));
}
}
finally
{
Ds3TestUtils.DeleteBucket(_client, bucketName);
}
}
public void TestChecksumStreamingWithMultiChunks()
{
const string bucketName = "TestChecksumStreamingWithMultiChunks";
try
{
// Creates a bucket if it does not already exist.
_helpers.EnsureBucketExists(bucketName);
const long streamLength = 150L * 1024L * 1024L * 1024L; //that way we enforce that we get 2 chunks
var directoryObjects = new List<Ds3Object> { new Ds3Object("bigFile", streamLength) };
const long blobSize = 1L * 1024L * 1024L * 1024L;
// Test the PUT
var putJob = _helpers.StartWriteJob(bucketName, directoryObjects, new Ds3WriteJobOptions { MaxUploadSize = blobSize }, new WriteStreamHelperStrategy());
using (var fileStream = new ChecksumStream(streamLength, this._copyBufferSize.Value))
{
var md5 = MD5.Create();
using (var md5Stream = new CryptoStream(fileStream, md5, CryptoStreamMode.Read))
{
putJob.Transfer(foo => md5Stream);
if (!md5Stream.HasFlushedFinalBlock)
{
md5Stream.FlushFinalBlock();
}
Assert.AreEqual("6pqugiiIUgxPkHfKKgq52A==", Convert.ToBase64String(md5.Hash));
}
}
// Test the GET
var getJob = _helpers.StartReadAllJob(bucketName, helperStrategy: new ReadStreamHelperStrategy());
using (Stream fileStream = new ChecksumStream(streamLength, this._copyBufferSize.Value))
{
var md5 = MD5.Create();
using (var md5Stream = new CryptoStream(fileStream, md5, CryptoStreamMode.Write))
{
getJob.Transfer(foo => md5Stream);
if (!md5Stream.HasFlushedFinalBlock)
{
md5Stream.FlushFinalBlock();
}
Assert.AreEqual("6pqugiiIUgxPkHfKKgq52A==", Convert.ToBase64String(md5.Hash));
}
}
}
finally
{
Ds3TestUtils.DeleteBucket(_client, bucketName);
}
}