public override int Read(byte[] buffer, int offset, int count)
{
var readCount = this.innerStream.Read(buffer, offset, count);
readStatus.AddToProcessedBytes(readCount);
return(readCount);
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism))
using (new ProgressTracker(uploadStatus))
{
var loopResult = Parallel.ForEach(dataWithRanges,
() => new PSPageBlobClient(pageBlob.Uri),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.UploadPages(stream, dwr.Range.StartIndex);
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism);
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
throw new AggregateException(loopResult.Exceptions);
}
}
}
return(true);
}
private void CopyBaseImageToDestination()
{
var source = this.blobObjectFactory.Create(baseVhdBlobUri);
source.FetchAttributes();
var copyStatus = new ProgressStatus(0, source.Properties.Length);
using (new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete,TimeSpan.FromSeconds(1)))
{
destinationBlob.StartCopyFromBlob(source);
destinationBlob.FetchAttributes();
while (true)
{
if (destinationBlob.CopyState.BytesCopied != null)
{
copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed);
}
if (destinationBlob.CopyState.Status == CopyStatus.Success)
{
break;
}
if (destinationBlob.CopyState.Status == CopyStatus.Pending)
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
else
{
throw new ApplicationException(
string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri,
destinationBlob.Uri, destinationBlob.CopyState));
}
destinationBlob.FetchAttributes();
}
}
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(blob.Uri, this.maxParallelism))
using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
{
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach(dataWithRanges,
() => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.Properties.ContentMD5 = md5HashOfDataChunk;
b.WritePagesAsync(stream, dwr.Range.StartIndex, contentMD5: null)
.ConfigureAwait(false).GetAwaiter().GetResult();
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult loopResult = task.Result;
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
throw new AggregateException(loopResult.Exceptions);
}
}
else
{
using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
{
bdms.Current.SetBlobMd5Hash(md5Hash);
bdms.Current.CleanUpUploadMetaData();
bdms.Complete();
}
}
}
return(true);
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using(new ServicePointHandler(blob.Uri, this.maxParallelism))
using(new ProgressTracker(uploadStatus))
{
var loopResult = Parallel.ForEach(dataWithRanges,
() => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int) dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.Properties.ContentMD5 = md5HashOfDataChunk;
b.WritePages(stream, dwr.Range.StartIndex);
}
}
uploadStatus.AddToProcessedBytes((int) dwr.Range.Length);
}, this.maxParallelism);
if(loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
//TODO: throw an AggregateException
return false;
}
}
else
{
using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
{
bdms.Current.SetBlobMd5Hash(md5Hash);
bdms.Current.CleanUpUploadMetaData();
bdms.Complete();
}
}
}
return true;
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(blob.Uri, this.maxParallelism))
using (new ProgressTracker(uploadStatus))
{
var loopResult = Parallel.ForEach(dataWithRanges,
() => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.Properties.ContentMD5 = md5HashOfDataChunk;
b.WritePages(stream, dwr.Range.StartIndex);
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism);
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
//TODO: throw an AggregateException
return(false);
}
}
else
{
using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
{
bdms.Current.SetBlobMd5Hash(md5Hash);
bdms.Current.CleanUpUploadMetaData();
bdms.Complete();
}
}
}
return(true);
}
public bool Synchronize(ComputeTokenCredential tokenCredential)
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism))
using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
{
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach(dataWithRanges,
() => new PSPageBlobClient(pageBlob.Uri, tokenCredential),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.UploadPages(stream, dwr.Range.StartIndex);
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult loopResult = task.Result;
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
throw new AggregateException(loopResult.Exceptions);
}
}
}
return(true);
}
private void CopyBaseImageToDestination()
{
var source = this.blobObjectFactory.Create(baseVhdBlobUri);
source.FetchAttributesAsync()
.ConfigureAwait(false).GetAwaiter().GetResult();
var copyStatus = new ProgressStatus(0, source.Properties.Length);
using (ProgressTracker progressTracker = new ProgressTracker(copyStatus, Program.SyncOutput.ProgressCopyStatus, Program.SyncOutput.ProgressCopyComplete))
{
destinationBlob.StartCopyAsync(source).ConfigureAwait(false).GetAwaiter().GetResult();
destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult();
while (true)
{
if (destinationBlob.CopyState.BytesCopied != null)
{
copyStatus.AddToProcessedBytes(destinationBlob.CopyState.BytesCopied.Value - copyStatus.BytesProcessed);
}
if (destinationBlob.CopyState.Status == CopyStatus.Success)
{
break;
}
if (destinationBlob.CopyState.Status == CopyStatus.Pending)
{
progressTracker.Update();
Thread.Sleep(TimeSpan.FromSeconds(1));
}
else
{
throw new ApplicationException(
string.Format("Cannot copy source '{0}' to destination '{1}', copy state is '{2}'", source.Uri,
destinationBlob.Uri, destinationBlob.CopyState));
}
destinationBlob.FetchAttributesAsync().ConfigureAwait(false).GetAwaiter().GetResult();
}
}
}
public void Download()
{
if (parameters.OverWrite)
{
DeleteTempVhdIfExist(parameters.LocalFilePath);
}
else
{
if (File.Exists(parameters.LocalFilePath))
{
var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath);
throw new ArgumentException(message);
}
}
var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey);
if (parameters.ValidateFreeDiskSpace)
{
TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length);
}
const int megaByte = 1024 * 1024;
var ranges = blobHandle.GetUploadableRanges();
var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());
Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));
Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName);
var fileStreamLock = new object();
using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit))
{
using (ProgressTracker progressTracker = new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete))
{
using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
{
fileStream.SetLength(0);
fileStream.SetLength(blobHandle.Length);
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach <IndexRange, Stream>(ranges,
blobHandle.OpenStream,
(r, b) =>
{
b.Seek(r.StartIndex, SeekOrigin.Begin);
byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);
lock (fileStreamLock)
{
Trace.WriteLine(String.Format("Range:{0}", r));
fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
fileStream.Write(buffer, 0, (int)r.Length);
fileStream.Flush();
}
downloadStatus.AddToProcessedBytes((int)r.Length);
},
pbwlf =>
{
pbwlf.Dispose();
},
parameters.ConnectionLimit));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult lr = task.Result;
if (lr.IsExceptional)
{
throw new AggregateException(lr.Exceptions);
}
}
}
}
Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName);
}
public void Download()
{
if (parameters.OverWrite)
{
DeleteTempVhdIfExist(parameters.LocalFilePath);
}
else
{
if (File.Exists(parameters.LocalFilePath))
{
var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath);
throw new ArgumentException(message);
}
}
var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey);
if (parameters.ValidateFreeDiskSpace)
{
TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length);
}
const int megaByte = 1024 * 1024;
var ranges = blobHandle.GetUploadableRanges();
var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());
Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));
Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName);
var fileStreamLock = new object();
using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit))
{
using (new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete, TimeSpan.FromSeconds(1)))
{
using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
{
fileStream.SetLength(0);
fileStream.SetLength(blobHandle.Length);
LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges,
blobHandle.OpenStream,
(r, b) =>
{
b.Seek(r.StartIndex, SeekOrigin.Begin);
byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);
lock (fileStreamLock)
{
Trace.WriteLine(String.Format("Range:{0}", r));
fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
fileStream.Write(buffer, 0, (int)r.Length);
fileStream.Flush();
}
downloadStatus.AddToProcessedBytes((int)r.Length);
},
pbwlf =>
{
pbwlf.Dispose();
},
parameters.ConnectionLimit);
if (lr.IsExceptional)
{
throw new AggregateException(lr.Exceptions);
}
}
}
}
Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName);
}
public void Download(string destination)
{
DeleteTempVhdIfExist(destination);
Console.WriteLine("\t\tDownloading blob '{0}' ...", blobUri.BlobName);
Console.WriteLine("\t\tImage download start time: '{0}'", DateTime.UtcNow.ToString("o"));
var blobHandle = new BlobHandle(blobUri, this.StorageAccountKey);
const int megaByte = 1024 * 1024;
var ranges = blobHandle.GetUploadableRanges();
var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());
Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));
const int maxParallelism = 24;
using (new ServicePointHandler(this.blobUri.Uri, maxParallelism))
using (new ProgressTracker(downloadStatus, Program.SyncOutput.ProgressUploadStatus, Program.SyncOutput.ProgressUploadComplete, TimeSpan.FromSeconds(1)))
{
// if(SparseFile.VolumeSupportsSparseFiles(destination))
// {
// using(var fileStream = SparseFile.Create(destination))
// {
// foreach (var emptyRange in blobHandle.GetEmptyRanges())
// {
// SparseFile.SetSparseRange(fileStream.SafeFileHandle, emptyRange.StartIndex, emptyRange.Length);
// }
// }
// }
using (var fileStream = new FileStream(destination, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
{
fileStream.SetLength(0);
fileStream.SetLength(blobHandle.Length);
LoopResult lr = Parallel.ForEach<IndexRange, Stream>(ranges,
blobHandle.OpenStream,
(r, b) =>
{
b.Seek(r.StartIndex, SeekOrigin.Begin);
byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);
lock (fileStream)
{
Trace.WriteLine(String.Format("Range:{0}", r));
fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
fileStream.Write(buffer, 0, (int)r.Length);
fileStream.Flush();
}
downloadStatus.AddToProcessedBytes((int)r.Length);
},
pbwlf =>
{
pbwlf.Dispose();
},
maxParallelism);
if (lr.IsExceptional)
{
Console.WriteLine("\t\tException(s) happened");
for (int i = 0; i < lr.Exceptions.Count; i++)
{
Console.WriteLine("{0} -> {1}", i, lr.Exceptions[i]);
}
}
}
}
Console.WriteLine("\t\tImage download end time : '{0}'", DateTime.UtcNow.ToString("o"));
}