public override int Read(byte[] buffer, int offset, int count) { var readCount = this.innerStream.Read(buffer, offset, count); readStatus.AddToProcessedBytes(readCount); return(readCount); }
public bool Synchronize() { var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism)) using (new ProgressTracker(uploadStatus)) { var loopResult = Parallel.ForEach(dataWithRanges, () => new PSPageBlobClient(pageBlob.Uri), (dwr, b) => { using (dwr) { var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { b.UploadPages(stream, dwr.Range.StartIndex); } } uploadStatus.AddToProcessedBytes((int)dwr.Range.Length); }, this.maxParallelism); if (loopResult.IsExceptional) { if (loopResult.Exceptions.Any()) { Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); throw new AggregateException(loopResult.Exceptions); } } } return(true); }
private void CopyBaseImageToDestination() { var source = this.blobObjectFactory.Create(baseVhdBlobUri); source.FetchAttributes(); var copyStatus = new ProgressStatus(0, source.Properties.Length); using (new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete,TimeSpan.FromSeconds(1))) { destinationBlob.StartCopyFromBlob(source); destinationBlob.FetchAttributes(); while (true) { if (destinationBlob.CopyState.BytesCopied != null) { copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed); } if (destinationBlob.CopyState.Status == CopyStatus.Success) { break; } if (destinationBlob.CopyState.Status == CopyStatus.Pending) { Thread.Sleep(TimeSpan.FromSeconds(1)); } else { throw new ApplicationException( string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri, destinationBlob.Uri, destinationBlob.CopyState)); } destinationBlob.FetchAttributes(); } } }
public bool Synchronize() { var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); using (new ServicePointHandler(blob.Uri, this.maxParallelism)) using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus)) { Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() => { return(Threading.Parallel.ForEach(dataWithRanges, () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials), (dwr, b) => { using (dwr) { var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { b.Properties.ContentMD5 = md5HashOfDataChunk; b.WritePagesAsync(stream, dwr.Range.StartIndex, contentMD5: null) .ConfigureAwait(false).GetAwaiter().GetResult(); } } uploadStatus.AddToProcessedBytes((int)dwr.Range.Length); }, this.maxParallelism)); }); while (!task.Wait(TimeSpan.FromSeconds(1))) { progressTracker.Update(); } LoopResult loopResult = task.Result; if (loopResult.IsExceptional) { if (loopResult.Exceptions.Any()) { Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); throw new AggregateException(loopResult.Exceptions); } } else { using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials))) { bdms.Current.SetBlobMd5Hash(md5Hash); bdms.Current.CleanUpUploadMetaData(); bdms.Complete(); } } } return(true); }
public bool Synchronize() { var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); using(new ServicePointHandler(blob.Uri, this.maxParallelism)) using(new ProgressTracker(uploadStatus)) { var loopResult = Parallel.ForEach(dataWithRanges, () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials), (dwr, b) => { using (dwr) { var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int) dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { b.Properties.ContentMD5 = md5HashOfDataChunk; b.WritePages(stream, dwr.Range.StartIndex); } } uploadStatus.AddToProcessedBytes((int) dwr.Range.Length); }, this.maxParallelism); if(loopResult.IsExceptional) { if (loopResult.Exceptions.Any()) { Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); //TODO: throw an AggregateException return false; } } else { using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials))) { bdms.Current.SetBlobMd5Hash(md5Hash); bdms.Current.CleanUpUploadMetaData(); bdms.Complete(); } } } return true; }
public bool Synchronize() { var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); using (new ServicePointHandler(blob.Uri, this.maxParallelism)) using (new ProgressTracker(uploadStatus)) { var loopResult = Parallel.ForEach(dataWithRanges, () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials), (dwr, b) => { using (dwr) { var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { b.Properties.ContentMD5 = md5HashOfDataChunk; b.WritePages(stream, dwr.Range.StartIndex); } } uploadStatus.AddToProcessedBytes((int)dwr.Range.Length); }, this.maxParallelism); if (loopResult.IsExceptional) { if (loopResult.Exceptions.Any()) { Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); //TODO: throw an AggregateException return(false); } } else { using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials))) { bdms.Current.SetBlobMd5Hash(md5Hash); bdms.Current.CleanUpUploadMetaData(); bdms.Complete(); } } } return(true); }
public bool Synchronize(ComputeTokenCredential tokenCredential) { var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats()); using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism)) using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus)) { Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() => { return(Threading.Parallel.ForEach(dataWithRanges, () => new PSPageBlobClient(pageBlob.Uri, tokenCredential), (dwr, b) => { using (dwr) { var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { b.UploadPages(stream, dwr.Range.StartIndex); } } uploadStatus.AddToProcessedBytes((int)dwr.Range.Length); }, this.maxParallelism)); }); while (!task.Wait(TimeSpan.FromSeconds(1))) { progressTracker.Update(); } LoopResult loopResult = task.Result; if (loopResult.IsExceptional) { if (loopResult.Exceptions.Any()) { Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions); throw new AggregateException(loopResult.Exceptions); } } } return(true); }
private void CopyBaseImageToDestination() { var source = this.blobObjectFactory.Create(baseVhdBlobUri); source.FetchAttributesAsync() .ConfigureAwait(false).GetAwaiter().GetResult(); var copyStatus = new ProgressStatus(0, source.Properties.Length); using (ProgressTracker progressTracker = new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete)) { destinationBlob.StartCopyAsync(source).ConfigureAwait(false).GetAwaiter().GetResult(); destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult(); while (true) { if (destinationBlob.CopyState.BytesCopied != null) { copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed); } if (destinationBlob.CopyState.Status == CopyStatus.Success) { break; } if (destinationBlob.CopyState.Status == CopyStatus.Pending) { progressTracker.Update(); Thread.Sleep(TimeSpan.FromSeconds(1)); } else { throw new ApplicationException( string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri, destinationBlob.Uri, destinationBlob.CopyState)); } destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult(); } } }
public void Download() { if (parameters.OverWrite) { DeleteTempVhdIfExist(parameters.LocalFilePath); } else { if (File.Exists(parameters.LocalFilePath)) { var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath); throw new ArgumentException(message); } } var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey); if (parameters.ValidateFreeDiskSpace) { TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length); } const int megaByte = 1024 * 1024; var ranges = blobHandle.GetUploadableRanges(); var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte); var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats()); Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length))); Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName); var fileStreamLock = new object(); using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit)) { using (ProgressTracker progressTracker = new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete)) { using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough)) { fileStream.SetLength(0); fileStream.SetLength(blobHandle.Length); Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() => { return(Threading.Parallel.ForEach <IndexRange, Stream>(ranges, blobHandle.OpenStream, (r, b) => { b.Seek(r.StartIndex, SeekOrigin.Begin); byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager); lock (fileStreamLock) { Trace.WriteLine(String.Format("Range:{0}", r)); fileStream.Seek(r.StartIndex, SeekOrigin.Begin); fileStream.Write(buffer, 0, (int)r.Length); fileStream.Flush(); } downloadStatus.AddToProcessedBytes((int)r.Length); }, pbwlf => { pbwlf.Dispose(); }, parameters.ConnectionLimit)); }); while (!task.Wait(TimeSpan.FromSeconds(1))) { progressTracker.Update(); } LoopResult lr = task.Result; if (lr.IsExceptional) { throw new AggregateException(lr.Exceptions); } } } } Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName); }
public void Download() { if (parameters.OverWrite) { DeleteTempVhdIfExist(parameters.LocalFilePath); } else { if (File.Exists(parameters.LocalFilePath)) { var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath); throw new ArgumentException(message); } } var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey); if (parameters.ValidateFreeDiskSpace) { TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length); } const int megaByte = 1024 * 1024; var ranges = blobHandle.GetUploadableRanges(); var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte); var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats()); Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length))); Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName); var fileStreamLock = new object(); using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit)) { using (new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete, TimeSpan.FromSeconds(1))) { using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough)) { fileStream.SetLength(0); fileStream.SetLength(blobHandle.Length); LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges, blobHandle.OpenStream, (r, b) => { b.Seek(r.StartIndex, SeekOrigin.Begin); byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager); lock (fileStreamLock) { Trace.WriteLine(String.Format("Range:{0}", r)); fileStream.Seek(r.StartIndex, SeekOrigin.Begin); fileStream.Write(buffer, 0, (int)r.Length); fileStream.Flush(); } downloadStatus.AddToProcessedBytes((int)r.Length); }, pbwlf => { pbwlf.Dispose(); }, parameters.ConnectionLimit); if (lr.IsExceptional) { throw new AggregateException(lr.Exceptions); } } } } Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName); }
public void Download(string destination) { DeleteTempVhdIfExist(destination); Console.WriteLine("\t\tDownloading blob '{0}' ...", blobUri.BlobName); Console.WriteLine("\t\tImage download start time: '{0}'", DateTime.UtcNow.ToString("o")); var blobHandle = new BlobHandle(blobUri, this.StorageAccountKey); const int megaByte = 1024 * 1024; var ranges = blobHandle.GetUploadableRanges(); var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte); var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats()); Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length))); const int maxParallelism = 24; using (new ServicePointHandler(this.blobUri.Uri, maxParallelism)) using (new ProgressTracker(downloadStatus, Program.SyncOutput.ProgressUploadStatus, Program.SyncOutput.ProgressUploadComplete, TimeSpan.FromSeconds(1))) { // if(SparseFile.VolumeSupportsSparseFiles(destination)) // { // using(var fileStream = SparseFile.Create(destination)) // { // foreach (var emptyRange in blobHandle.GetEmptyRanges()) // { // SparseFile.SetSparseRange(fileStream.SafeFileHandle, emptyRange.StartIndex, emptyRange.Length); // } // } // } using (var fileStream = new FileStream(destination, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough)) { fileStream.SetLength(0); fileStream.SetLength(blobHandle.Length); LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges, blobHandle.OpenStream, (r, b) => { b.Seek(r.StartIndex, SeekOrigin.Begin); byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager); lock (fileStream) { Trace.WriteLine(String.Format("Range:{0}", r)); fileStream.Seek(r.StartIndex, SeekOrigin.Begin); fileStream.Write(buffer, 0, (int)r.Length); fileStream.Flush(); } downloadStatus.AddToProcessedBytes((int)r.Length); }, pbwlf => { pbwlf.Dispose(); }, maxParallelism); if (lr.IsExceptional) { Console.WriteLine("\t\tException(s) happened"); for (int i = 0; i < lr.Exceptions.Count; i++) { Console.WriteLine("{0} -> {1}", i, lr.Exceptions[i]); } } } } Console.WriteLine("\t\tImage download end time : '{0}'", DateTime.UtcNow.ToString("o")); }