public void TestZipUnzipRandomSucceeds()
{
//// ARRANGE
var source = new byte[992];
var r = new Random(42);
r.NextBytes(source);
//// ACT
var zippedSource = GZipHelper.Zip(source);
Assert.AreEqual(1015, zippedSource.Length);
var unzipped = GZipHelper.Unzip(zippedSource);
//// ASSERT
Assert.AreEqual(992, unzipped.Length);
for (int i = 0; i < 992; i++)
{
Assert.AreEqual(source[i], unzipped[i]);
}
}
public void TestZipUnzipSucceeds()
{
//// ARRANGE
var source = new byte[992];
for (int i = 0; i < source.Length; i++)
{
source[i] = 42;
}
//// ACT
var zippedSource = GZipHelper.Zip(source);
Assert.AreEqual(29, zippedSource.Length);
var unzipped = GZipHelper.Unzip(zippedSource);
//// ASSERT
Assert.AreEqual(992, unzipped.Length);
for (int i = 0; i < 992; i++)
{
Assert.AreEqual(source[i], unzipped[i]);
}
}
public void DownloadFiles(string baseDirectoryUri,
IList <string> blobs,
string destination,
string storageConnectionString,
CancellationToken cancelToken,
Action <string> fileDownloadedAction,
Action <string, Exception> fileFailedAction,
Action completedAction)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var baseBlobLength = 0;
if (!string.IsNullOrEmpty(baseDirectoryUri))
{
baseBlobLength = baseDirectoryUri.Length + (baseDirectoryUri.EndsWith("/") ? 0 : 1);
}
else
{
var shortestBlob = blobs.OrderBy(s => s.Length).First();
baseBlobLength = shortestBlob.LastIndexOf("/") + 1;
}
try
{
Parallel.ForEach(
blobs,
new ParallelOptions {
MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken
},
(blobName, loopState) =>
{
var blob = blobClient.GetBlobReference(blobName);
var blockBlob = blob.ToBlockBlob;
var destPath = Path.Combine(destination, blobName.Substring(baseBlobLength));
var fileDownloaded = false;
while (!fileDownloaded && !loopState.IsStopped)
{
try
{
blob.FetchAttributes();
var isGZip = blob.Properties.ContentEncoding != null && blob.Properties.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase);
if (File.Exists(destPath))
{
var hash = string.Empty;
if (isGZip)
{
using (var fileStream = GZipHelper.Zip(destPath))
{
hash = HashHelper.ComputeMD5(fileStream);
}
}
else
{
using (var fileStream = File.Open(destPath, FileMode.Open))
{
hash = HashHelper.ComputeMD5(fileStream);
}
}
if (blob.Attributes.Properties.ContentMD5 == hash)
{
fileDownloaded = true;
}
else
{
//delete obsolete existing file
File.Delete(destPath);
}
}
if (!fileDownloaded)
{
Directory.CreateDirectory(Path.GetDirectoryName(destPath));
if (isGZip)
{
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream, new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
});
stream.Seek(0, SeekOrigin.Begin);
GZipHelper.Unzip(stream, destPath);
}
}
else
{
blob.DownloadToFile(destPath, new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
});
}
fileDownloaded = true;
}
if (!loopState.IsStopped && fileDownloaded && fileDownloadedAction != null)
{
fileDownloadedAction(blobName);
}
}
catch (Exception ex)
{
if (!loopState.IsStopped && fileFailedAction != null)
{
fileFailedAction(blobName, ex);
}
Thread.Sleep(ArgSettings.RetryInterval);
}
}
});
if (completedAction != null)
{
completedAction();
}
}
catch (OperationCanceledException)
{
}
}
private void UploadFiles(string basePath, IList <string> files, CancellationToken cancelToken, Action <string> fileUploadedAction, Action <string, Exception> fileFailedAction, Action completedAction)
{
try
{
Parallel.ForEach(
files,
new ParallelOptions {
MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken
},
(filePath, loopState) =>
{
var blobName = Path.GetFileName(filePath);
if (!string.IsNullOrEmpty(basePath))
{
if (basePath.EndsWith("\\") || basePath.EndsWith("/"))
{
blobName = filePath.Substring(basePath.Length);
}
else
{
blobName = filePath.Substring(basePath.Length + 1);
}
}
var blob = this.GetBlobReference(blobName);
var fileUploaded = false;
Stream fileStream = null;
while (!fileUploaded && !loopState.IsStopped)
{
try
{
if (ArgSettings.UseGZip)
{
blob.Properties.ContentEncoding = "gzip";
blob.Properties.ContentType = MimeTypeHelper.MimeType(filePath);
fileStream = GZipHelper.Zip(filePath);
}
else
{
fileStream = File.Open(filePath, FileMode.Open);
}
var hash = HashHelper.ComputeMD5(fileStream);
// check blob is already uploaded and is the same content
if (blob.Exists())
{
if (blob.Attributes.Properties.ContentMD5 == hash && blob.Attributes.Properties.Length == fileStream.Length)
{
fileUploaded = true;
}
else
{
//delete obsolete existing blob
blob.Delete();
}
}
if (!fileUploaded)
{
blob.Properties.ContentMD5 = hash;
blob.UploadFromStream(fileStream,
new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
});
fileUploaded = true;
}
if (!loopState.IsStopped && fileUploaded && fileUploadedAction != null)
{
fileUploadedAction(blobName);
}
}
catch (Exception ex)
{
if (!loopState.IsStopped && fileFailedAction != null)
{
fileFailedAction(blobName, ex);
}
Thread.Sleep(ArgSettings.RetryInterval);
}
finally
{
if (!loopState.IsStopped && fileStream != null)
{
fileStream.Close();
}
}
}
});
if (completedAction != null)
{
completedAction();
}
}
catch (OperationCanceledException)
{
}
}
