public async Task <bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
if (BlobNameExtension != null)
{
blobNames = blobNames.Where(b => Path.GetExtension(b) == BlobNameExtension).ToList <string>();
}
if (!Directory.Exists(DownloadDirectory))
{
Directory.CreateDirectory(DownloadDirectory);
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
using (HttpClient client = new HttpClient())
{
client.Timeout = TimeSpan.FromMinutes(10);
await Task.WhenAll(blobNames.Select(item => DownloadItem(client, ct, item, clientThrottle)));
}
}
catch (Exception e)
{
Log.LogError(e.ToString());
}
return(!Log.HasLoggedErrors);
}
private string[] GetBlobList(string path)
{
return(ListAzureBlobs.Execute(
AccountName,
AccountKey,
ConnectionString,
ContainerName,
path,
BuildEngine,
HostObject));
}
public static string [] Execute(string accountName,
string accountKey,
string connectionString,
string containerName,
string filterBlobNames,
IBuildEngine buildengine,
ITaskHost taskHost)
{
ListAzureBlobs getAzureBlobList = new ListAzureBlobs()
{
AccountName = accountName,
AccountKey = accountKey,
ContainerName = containerName,
FilterBlobNames = filterBlobNames,
BuildEngine = buildengine,
HostObject = taskHost
};
getAzureBlobList.Execute();
return(getAzureBlobList.BlobNames);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
// track the number of blobs that fail to download
int failureCount = 0;
using (HttpClient client = new HttpClient())
{
foreach (string blob in blobNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
string filename = Path.Combine(downloadBlobDirectory, blobFilename);
var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}