private async Task UploadFile(AsyncLimiter limiter, string localPath)
{
using (await limiter.AcquireOneAsync())
{
using (Stream source = File.OpenRead(localPath))
{
try
{
string localRelativePath = localPath.Substring(_rootSourceFolder.Length);
string destPath = StoragePath.Combine(_targetRootPath, localRelativePath.Replace(Path.DirectorySeparatorChar, StoragePath.PathSeparator));
await _storage.WriteAsync(destPath, source);
_copied += new FileInfo(localPath).Length;
SendUpdate();
}
catch (Exception ex)
{
_failedBlobs[localPath] = ex;
}
}
_filesCopied += 1;
}
SendUpdate();
}
private async Task CopyBlobAsync(Blob blob, AsyncLimiter limiter)
{
string destPath = blob.FullPath.Substring(_sourceRoot.Length);
destPath = StoragePath.Combine(_destinationRootPath, destPath);
try
{
using (await limiter.AcquireOneAsync())
{
using (Stream src = await _sourceStorage.OpenReadAsync(blob))
{
if (src != null)
{
await _destinationStorage.WriteAsync(destPath, src);
_bytesComplete += blob.Size.Value;
UpdateProgress(_bytesComplete, _totalSizeBytes);
}
}
}
}
catch (Exception ex)
{
_failedBlobs.Add(blob);
Log.Error(ex, "filed to copy");
}
_filesComplete += 1;
Message = $"{_filesComplete} of {"blob".ToQuantity(_allSourceBlobs.Count)} copied";
}
private async Task CopyBlobsAsync(IReadOnlyCollection <Blob> files)
{
using (var limiter = new AsyncLimiter(GlobalSettings.Default.MaxParallelUploads))
{
await Task.WhenAll(files.Where(f => f.Kind == BlobItemKind.File).Select(b => CopyBlobAsync(b, limiter)));
}
}
public override async Task ExecuteAsync()
{
IsIndeterminate = true;
Message = "scanning file list";
foreach (string path in _localPaths)
{
AnalysePath(path);
}
IsIndeterminate = false;
using (var limiter = new AsyncLimiter(GlobalSettings.Default.MaxParallelUploads))
{
await Task.WhenAll(_allFiles.Select(path => UploadFile(limiter, path)));
}
SendUpdate();
Messenger.Default.Send(new FolderUpdatedMessage(_storage, _targetRootPath));
//Message = $"uploaded {"file".ToQuantity(_filesCopied)} and {_totalSize.ToFileSizeUiString()}";
}
public AzureContainerBrowser(BlobContainerClient client, bool prependContainerName, int maxTasks)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_prependContainerName = prependContainerName;
_asyncLimiter = new AsyncLimiter(maxTasks);
}
public ConsumerConcurrencyPipeline(AsyncLimiter limiter, IConsumerPipeline pipeline)
{
_limiter = limiter;
_pipeline = pipeline;
}
public ConsumerConcurrencyPipelineFactory(ConcurrencyDefinition definition)
{
_limiter = new AsyncLimiter(definition.MaxConcurrency, definition.MaxRate, definition.RateInterval);
}
public PublisherConcurrencyPipeline(AsyncLimiter limiter, IPublisherPipeline publisher)
{
_limiter = limiter;
_publisher = publisher;
}
