public void TestZipUnzipRandomSucceeds() { //// ARRANGE var source = new byte[992]; var r = new Random(42); r.NextBytes(source); //// ACT var zippedSource = GZipHelper.Zip(source); Assert.AreEqual(1015, zippedSource.Length); var unzipped = GZipHelper.Unzip(zippedSource); //// ASSERT Assert.AreEqual(992, unzipped.Length); for (int i = 0; i < 992; i++) { Assert.AreEqual(source[i], unzipped[i]); } }
public void TestZipUnzipSucceeds() { //// ARRANGE var source = new byte[992]; for (int i = 0; i < source.Length; i++) { source[i] = 42; } //// ACT var zippedSource = GZipHelper.Zip(source); Assert.AreEqual(29, zippedSource.Length); var unzipped = GZipHelper.Unzip(zippedSource); //// ASSERT Assert.AreEqual(992, unzipped.Length); for (int i = 0; i < 992; i++) { Assert.AreEqual(source[i], unzipped[i]); } }
public void DownloadFiles(string baseDirectoryUri, IList <string> blobs, string destination, string storageConnectionString, CancellationToken cancelToken, Action <string> fileDownloadedAction, Action <string, Exception> fileFailedAction, Action completedAction) { var storageAccount = CloudStorageAccount.Parse(storageConnectionString); var blobClient = storageAccount.CreateCloudBlobClient(); var baseBlobLength = 0; if (!string.IsNullOrEmpty(baseDirectoryUri)) { baseBlobLength = baseDirectoryUri.Length + (baseDirectoryUri.EndsWith("/") ? 0 : 1); } else { var shortestBlob = blobs.OrderBy(s => s.Length).First(); baseBlobLength = shortestBlob.LastIndexOf("/") + 1; } try { Parallel.ForEach( blobs, new ParallelOptions { MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken }, (blobName, loopState) => { var blob = blobClient.GetBlobReference(blobName); var blockBlob = blob.ToBlockBlob; var destPath = Path.Combine(destination, blobName.Substring(baseBlobLength)); var fileDownloaded = false; while (!fileDownloaded && !loopState.IsStopped) { try { blob.FetchAttributes(); var isGZip = blob.Properties.ContentEncoding != null && blob.Properties.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase); if (File.Exists(destPath)) { var hash = string.Empty; if (isGZip) { using (var fileStream = GZipHelper.Zip(destPath)) { hash = HashHelper.ComputeMD5(fileStream); } } else { using (var fileStream = File.Open(destPath, FileMode.Open)) { hash = HashHelper.ComputeMD5(fileStream); } } if (blob.Attributes.Properties.ContentMD5 == hash) { fileDownloaded = true; } else { //delete obsolete existing file File.Delete(destPath); } } if (!fileDownloaded) { Directory.CreateDirectory(Path.GetDirectoryName(destPath)); if (isGZip) { using (var stream = new MemoryStream()) { blob.DownloadToStream(stream, new BlobRequestOptions() { RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval)) }); stream.Seek(0, SeekOrigin.Begin); GZipHelper.Unzip(stream, destPath); } } else { blob.DownloadToFile(destPath, new BlobRequestOptions() { RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval)) }); } fileDownloaded = true; } if (!loopState.IsStopped && fileDownloaded && fileDownloadedAction != null) { fileDownloadedAction(blobName); } } catch (Exception ex) { if (!loopState.IsStopped && fileFailedAction != null) { fileFailedAction(blobName, ex); } Thread.Sleep(ArgSettings.RetryInterval); } } }); if (completedAction != null) { completedAction(); } } catch (OperationCanceledException) { } }
private void UploadFiles(string basePath, IList <string> files, CancellationToken cancelToken, Action <string> fileUploadedAction, Action <string, Exception> fileFailedAction, Action completedAction) { try { Parallel.ForEach( files, new ParallelOptions { MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken }, (filePath, loopState) => { var blobName = Path.GetFileName(filePath); if (!string.IsNullOrEmpty(basePath)) { if (basePath.EndsWith("\\") || basePath.EndsWith("/")) { blobName = filePath.Substring(basePath.Length); } else { blobName = filePath.Substring(basePath.Length + 1); } } var blob = this.GetBlobReference(blobName); var fileUploaded = false; Stream fileStream = null; while (!fileUploaded && !loopState.IsStopped) { try { if (ArgSettings.UseGZip) { blob.Properties.ContentEncoding = "gzip"; blob.Properties.ContentType = MimeTypeHelper.MimeType(filePath); fileStream = GZipHelper.Zip(filePath); } else { fileStream = File.Open(filePath, FileMode.Open); } var hash = HashHelper.ComputeMD5(fileStream); // check blob is already uploaded and is the same content if (blob.Exists()) { if (blob.Attributes.Properties.ContentMD5 == hash && blob.Attributes.Properties.Length == fileStream.Length) { fileUploaded = true; } else { //delete obsolete existing blob blob.Delete(); } } if (!fileUploaded) { blob.Properties.ContentMD5 = hash; blob.UploadFromStream(fileStream, new BlobRequestOptions() { RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval)) }); fileUploaded = true; } if (!loopState.IsStopped && fileUploaded && fileUploadedAction != null) { fileUploadedAction(blobName); } } catch (Exception ex) { if (!loopState.IsStopped && fileFailedAction != null) { fileFailedAction(blobName, ex); } Thread.Sleep(ArgSettings.RetryInterval); } finally { if (!loopState.IsStopped && fileStream != null) { fileStream.Close(); } } } }); if (completedAction != null) { completedAction(); } } catch (OperationCanceledException) { } }