Exemple #1
0
        public override int Read(byte[] buffer, int offset, int count)
        {
            var readCount = this.innerStream.Read(buffer, offset, count);

            readStatus.AddToProcessedBytes(readCount);
            return(readCount);
        }
        public bool Synchronize()
        {
            var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());

            using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism))
                using (new ProgressTracker(uploadStatus))
                {
                    var loopResult = Parallel.ForEach(dataWithRanges,
                                                      () => new PSPageBlobClient(pageBlob.Uri),
                                                      (dwr, b) =>
                    {
                        using (dwr)
                        {
                            var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
                            using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
                            {
                                b.UploadPages(stream, dwr.Range.StartIndex);
                            }
                        }
                        uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
                    }, this.maxParallelism);
                    if (loopResult.IsExceptional)
                    {
                        if (loopResult.Exceptions.Any())
                        {
                            Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);

                            throw new AggregateException(loopResult.Exceptions);
                        }
                    }
                }
            return(true);
        }
        private void CopyBaseImageToDestination()
        {
            var source = this.blobObjectFactory.Create(baseVhdBlobUri);
            source.FetchAttributes();

            var copyStatus = new ProgressStatus(0, source.Properties.Length);
            using (new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete,TimeSpan.FromSeconds(1)))
            {
                destinationBlob.StartCopyFromBlob(source);
                destinationBlob.FetchAttributes();

                while (true)
                {
                    if (destinationBlob.CopyState.BytesCopied != null)
                    {
                        copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed);
                    }
                    if (destinationBlob.CopyState.Status == CopyStatus.Success)
                    {
                        break;
                    }
                    if (destinationBlob.CopyState.Status == CopyStatus.Pending)
                    {
                        Thread.Sleep(TimeSpan.FromSeconds(1));
                    }
                    else
                    {
                        throw new ApplicationException(
                            string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri,
                                          destinationBlob.Uri, destinationBlob.CopyState));
                    }
                    destinationBlob.FetchAttributes();
                }
            }
        }
        public bool Synchronize()
        {
            var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());

            using (new ServicePointHandler(blob.Uri, this.maxParallelism))
                using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
                {
                    Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
                    {
                        return(Threading.Parallel.ForEach(dataWithRanges,
                                                          () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
                                                          (dwr, b) =>
                        {
                            using (dwr)
                            {
                                var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
                                using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
                                {
                                    b.Properties.ContentMD5 = md5HashOfDataChunk;
                                    b.WritePagesAsync(stream, dwr.Range.StartIndex, contentMD5: null)
                                    .ConfigureAwait(false).GetAwaiter().GetResult();
                                }
                            }
                            uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
                        }, this.maxParallelism));
                    });

                    while (!task.Wait(TimeSpan.FromSeconds(1)))
                    {
                        progressTracker.Update();
                    }

                    LoopResult loopResult = task.Result;
                    if (loopResult.IsExceptional)
                    {
                        if (loopResult.Exceptions.Any())
                        {
                            Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);

                            throw new AggregateException(loopResult.Exceptions);
                        }
                    }
                    else
                    {
                        using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
                        {
                            bdms.Current.SetBlobMd5Hash(md5Hash);
                            bdms.Current.CleanUpUploadMetaData();
                            bdms.Complete();
                        }
                    }
                }
            return(true);
        }
        public bool Synchronize()
        {
            var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());

            using(new ServicePointHandler(blob.Uri, this.maxParallelism))
            using(new ProgressTracker(uploadStatus))
            {
                var loopResult = Parallel.ForEach(dataWithRanges,
                                                  () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
                                                  (dwr, b) =>
                                                      {
                                                          using (dwr)
                                                          {
                                                              var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int) dwr.Range.Length);
                                                              using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
                                                              {
                                                                  b.Properties.ContentMD5 = md5HashOfDataChunk;
                                                                  b.WritePages(stream, dwr.Range.StartIndex);
                                                              }
                                                          }
                                                          uploadStatus.AddToProcessedBytes((int) dwr.Range.Length);
                                                      }, this.maxParallelism);
                if(loopResult.IsExceptional)
                {
                    if (loopResult.Exceptions.Any())
                    {
                        Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
                        //TODO: throw an AggregateException
                        return false;
                    }
                }
                else
                {
                    using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
                    {
                        bdms.Current.SetBlobMd5Hash(md5Hash);
                        bdms.Current.CleanUpUploadMetaData();
                        bdms.Complete();
                    }
                }
            }
            return true;
        }
        public bool Synchronize()
        {
            var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());

            using (new ServicePointHandler(blob.Uri, this.maxParallelism))
                using (new ProgressTracker(uploadStatus))
                {
                    var loopResult = Parallel.ForEach(dataWithRanges,
                                                      () => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
                                                      (dwr, b) =>
                    {
                        using (dwr)
                        {
                            var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
                            using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
                            {
                                b.Properties.ContentMD5 = md5HashOfDataChunk;
                                b.WritePages(stream, dwr.Range.StartIndex);
                            }
                        }
                        uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
                    }, this.maxParallelism);
                    if (loopResult.IsExceptional)
                    {
                        if (loopResult.Exceptions.Any())
                        {
                            Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
                            //TODO: throw an AggregateException
                            return(false);
                        }
                    }
                    else
                    {
                        using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
                        {
                            bdms.Current.SetBlobMd5Hash(md5Hash);
                            bdms.Current.CleanUpUploadMetaData();
                            bdms.Complete();
                        }
                    }
                }
            return(true);
        }
Exemple #7
0
        public bool Synchronize(ComputeTokenCredential tokenCredential)
        {
            var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());

            using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism))
                using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
                {
                    Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
                    {
                        return(Threading.Parallel.ForEach(dataWithRanges,
                                                          () => new PSPageBlobClient(pageBlob.Uri, tokenCredential),
                                                          (dwr, b) =>
                        {
                            using (dwr)
                            {
                                var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
                                using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
                                {
                                    b.UploadPages(stream, dwr.Range.StartIndex);
                                }
                            }
                            uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
                        }, this.maxParallelism));
                    });

                    while (!task.Wait(TimeSpan.FromSeconds(1)))
                    {
                        progressTracker.Update();
                    }

                    LoopResult loopResult = task.Result;
                    if (loopResult.IsExceptional)
                    {
                        if (loopResult.Exceptions.Any())
                        {
                            Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);

                            throw new AggregateException(loopResult.Exceptions);
                        }
                    }
                }
            return(true);
        }
Exemple #8
0
        private void CopyBaseImageToDestination()
        {
            var source = this.blobObjectFactory.Create(baseVhdBlobUri);

            source.FetchAttributesAsync()
            .ConfigureAwait(false).GetAwaiter().GetResult();

            var copyStatus = new ProgressStatus(0, source.Properties.Length);

            using (ProgressTracker progressTracker = new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete))
            {
                destinationBlob.StartCopyAsync(source).ConfigureAwait(false).GetAwaiter().GetResult();
                destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult();

                while (true)
                {
                    if (destinationBlob.CopyState.BytesCopied != null)
                    {
                        copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed);
                    }
                    if (destinationBlob.CopyState.Status == CopyStatus.Success)
                    {
                        break;
                    }
                    if (destinationBlob.CopyState.Status == CopyStatus.Pending)
                    {
                        progressTracker.Update();
                        Thread.Sleep(TimeSpan.FromSeconds(1));
                    }
                    else
                    {
                        throw new ApplicationException(
                                  string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri,
                                                destinationBlob.Uri, destinationBlob.CopyState));
                    }
                    destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult();
                }
            }
        }
        public void Download()
        {
            if (parameters.OverWrite)
            {
                DeleteTempVhdIfExist(parameters.LocalFilePath);
            }
            else
            {
                if (File.Exists(parameters.LocalFilePath))
                {
                    var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath);
                    throw new ArgumentException(message);
                }
            }

            var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey);

            if (parameters.ValidateFreeDiskSpace)
            {
                TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length);
            }

            const int megaByte = 1024 * 1024;

            var ranges         = blobHandle.GetUploadableRanges();
            var bufferManager  = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
            var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());

            Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));

            Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName);

            var fileStreamLock = new object();

            using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit))
            {
                using (ProgressTracker progressTracker = new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete))
                {
                    using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
                    {
                        fileStream.SetLength(0);
                        fileStream.SetLength(blobHandle.Length);

                        Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
                        {
                            return(Threading.Parallel.ForEach <IndexRange, Stream>(ranges,
                                                                                   blobHandle.OpenStream,
                                                                                   (r, b) =>
                            {
                                b.Seek(r.StartIndex, SeekOrigin.Begin);

                                byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);

                                lock (fileStreamLock)
                                {
                                    Trace.WriteLine(String.Format("Range:{0}", r));
                                    fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
                                    fileStream.Write(buffer, 0, (int)r.Length);
                                    fileStream.Flush();
                                }

                                downloadStatus.AddToProcessedBytes((int)r.Length);
                            },
                                                                                   pbwlf =>
                            {
                                pbwlf.Dispose();
                            },
                                                                                   parameters.ConnectionLimit));
                        });

                        while (!task.Wait(TimeSpan.FromSeconds(1)))
                        {
                            progressTracker.Update();
                        }

                        LoopResult lr = task.Result;

                        if (lr.IsExceptional)
                        {
                            throw new AggregateException(lr.Exceptions);
                        }
                    }
                }
            }
            Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName);
        }
Exemple #10
0
        public void Download()
        {
            if (parameters.OverWrite)
            {
                DeleteTempVhdIfExist(parameters.LocalFilePath);
            }
            else
            {
                if (File.Exists(parameters.LocalFilePath))
                {
                    var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath);
                    throw new ArgumentException(message);
                }
            }

            var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey);

            if (parameters.ValidateFreeDiskSpace)
            {
                TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length);
            }

            const int megaByte = 1024 * 1024;

            var ranges = blobHandle.GetUploadableRanges();
            var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
            var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());

            Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));

            Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName);

            var fileStreamLock = new object();
            using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit))
            {
                using (new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete, TimeSpan.FromSeconds(1)))
                {
                    using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
                    {
                        fileStream.SetLength(0);
                        fileStream.SetLength(blobHandle.Length);

                        LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges,
                                    blobHandle.OpenStream,
                                    (r, b) =>
                                        {
                                            b.Seek(r.StartIndex, SeekOrigin.Begin);

                                            byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);

                                            lock (fileStreamLock)
                                            {
                                                Trace.WriteLine(String.Format("Range:{0}", r));
                                                fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
                                                fileStream.Write(buffer, 0, (int)r.Length);
                                                fileStream.Flush();
                                            }

                                            downloadStatus.AddToProcessedBytes((int)r.Length);
                                        },
                                    pbwlf =>
                                        {
                                            pbwlf.Dispose();
                                        },
                                    parameters.ConnectionLimit);

                        if (lr.IsExceptional)
                        {
                            throw new AggregateException(lr.Exceptions);
                        }
                    }
                }
            }
            Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName);
        }
Exemple #11
0
        public void Download(string destination)
        {
            DeleteTempVhdIfExist(destination);

            Console.WriteLine("\t\tDownloading blob '{0}' ...", blobUri.BlobName);
            Console.WriteLine("\t\tImage download start time: '{0}'", DateTime.UtcNow.ToString("o"));

            var blobHandle = new BlobHandle(blobUri, this.StorageAccountKey);

            const int megaByte = 1024 * 1024;

            var ranges = blobHandle.GetUploadableRanges();
            var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
            var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());

            Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));

            const int maxParallelism = 24;
            using (new ServicePointHandler(this.blobUri.Uri, maxParallelism))
            using (new ProgressTracker(downloadStatus, Program.SyncOutput.ProgressUploadStatus, Program.SyncOutput.ProgressUploadComplete, TimeSpan.FromSeconds(1)))
            {
            //                if(SparseFile.VolumeSupportsSparseFiles(destination))
            //                {
            //                   using(var fileStream = SparseFile.Create(destination))
            //                   {
            //                       foreach (var emptyRange in blobHandle.GetEmptyRanges())
            //                       {
            //                           SparseFile.SetSparseRange(fileStream.SafeFileHandle, emptyRange.StartIndex, emptyRange.Length);
            //                       }
            //                   }
            //                }

                using (var fileStream = new FileStream(destination, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
                {
                    fileStream.SetLength(0);
                    fileStream.SetLength(blobHandle.Length);

                    LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges,
                                     blobHandle.OpenStream,
                                     (r, b) =>
                                     {
                                         b.Seek(r.StartIndex, SeekOrigin.Begin);

                                         byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);

                                         lock (fileStream)
                                         {
                                             Trace.WriteLine(String.Format("Range:{0}", r));
                                             fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
                                             fileStream.Write(buffer, 0, (int)r.Length);
                                             fileStream.Flush();
                                         }

                                         downloadStatus.AddToProcessedBytes((int)r.Length);
                                     },
                                     pbwlf =>
                                     {
                                         pbwlf.Dispose();
                                     },
                                     maxParallelism);
                    if (lr.IsExceptional)
                    {
                        Console.WriteLine("\t\tException(s) happened");
                        for (int i = 0; i < lr.Exceptions.Count; i++)
                        {
                            Console.WriteLine("{0} -> {1}", i, lr.Exceptions[i]);
                        }
                    }
                }
            }

            Console.WriteLine("\t\tImage download end time  : '{0}'", DateTime.UtcNow.ToString("o"));
        }