public override bool Execute()
{
ParseConnectionString();
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, BlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> headerBlobType = new Tuple <string, string>("x-ms-blob-type", "BlockBlob");
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
headerBlobType
};
if (!string.IsNullOrEmpty(ContentType))
{
additionalHeaders.Add(new Tuple <string, string>(AzureHelper.ContentTypeString, ContentType));
}
var request = AzureHelper.RequestMessage("PUT", blobUrl, AccountName, AccountKey, additionalHeaders, Content);
AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult();
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(!Log.HasLoggedErrors);
}
public AzureBlobLease(string accountName, string accountKey, string connectionString, string containerName, string blobName, Microsoft.Build.Utilities.TaskLoggingHelper log, string maxWait = null, string delay = null)
{
_accountName = accountName;
_accountKey = accountKey;
_connectionString = connectionString;
_containerName = containerName;
_blobName = blobName;
_maxWait = !string.IsNullOrWhiteSpace(maxWait) ? TimeSpan.Parse(maxWait) : TimeSpan.FromSeconds(s_MaxWaitDefault);
_delay = !string.IsNullOrWhiteSpace(delay) ? TimeSpan.Parse(delay) : TimeSpan.FromMilliseconds(s_DelayDefault);
_log = log;
_leaseUrl = $"{AzureHelper.GetBlobRestUrl(_accountName, _containerName, _blobName)}?comp=lease";
}
public async Task <bool> FileEqualsExistingBlobAsync(
string accountName,
string accountKey,
string containerName,
string filePath,
string destinationBlob,
int uploadTimeout)
{
using (var client = new HttpClient
{
Timeout = TimeSpan.FromMinutes(uploadTimeout)
})
{
log.LogMessage(
MessageImportance.Low,
$"Downloading blob {destinationBlob} to check if identical.");
string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob);
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(
log,
client,
createRequest))
{
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException(
$"Failed to retrieve existing blob {destinationBlob}, " +
$"status code {response.StatusCode}.");
}
byte[] existingBytes = await response.Content.ReadAsByteArrayAsync();
byte[] localBytes = File.ReadAllBytes(filePath);
bool equal = localBytes.SequenceEqual(existingBytes);
if (equal)
{
log.LogMessage(
MessageImportance.Normal,
"Item exists in blob storage, and is verified to be identical. " +
$"File: '{filePath}' Blob: '{destinationBlob}'");
}
return(equal);
}
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return(false);
}
string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName);
string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire");
Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */);
Tuple <string, string> headerSource = new Tuple <string, string>("x-ms-copy-source", sourceUrl);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, leaseDuration, headerSource
};
var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request))
{
if (response.IsSuccessStatusCode)
{
return(true);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(false);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
// track the number of blobs that fail to download
int failureCount = 0;
using (HttpClient client = new HttpClient())
{
foreach (string blob in blobNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
string filename = Path.Combine(downloadBlobDirectory, blobFilename);
var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle)
{
await clientThrottle.WaitAsync();
string filename = string.Empty;
try {
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob));
if (!DownloadFlatFiles)
{
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
filename = Path.Combine(downloadBlobDirectory, blobFilename);
}
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
}
}
}
catch (PathTooLongException)
{
Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}");
}
catch (Exception ex)
{
Log.LogError(ex.ToString());
}
finally
{
clientThrottle.Release();
}
}