private async Task <string> AcquireLeaseOnBlobAsync()
{
_log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{_containerName}/{_blobName}'.");
string leaseId = string.Empty;
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire");
Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, leaseDuration
};
var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(_log, client, request))
{
leaseId = response.Headers.GetValues("x-ms-lease-id").FirstOrDefault();
}
}
catch (Exception e)
{
_log.LogErrorFromException(e, true);
}
}
return(leaseId);
}
public override bool Execute()
{
ParseConnectionString();
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, BlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> headerBlobType = new Tuple <string, string>("x-ms-blob-type", "BlockBlob");
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
headerBlobType
};
if (!string.IsNullOrEmpty(ContentType))
{
additionalHeaders.Add(new Tuple <string, string>(AzureHelper.ContentTypeString, ContentType));
}
var request = AzureHelper.RequestMessage("PUT", blobUrl, AccountName, AccountKey, additionalHeaders, Content);
AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult();
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return(false);
}
string deleteUrl = $"https://{AccountName}.blob.core.windows.net/{ContainerName}/{BlobName}";
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> snapshots = new Tuple <string, string>("x-ms-lease-delete-snapshots", "include");
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
snapshots
};
var request = AzureHelper.RequestMessage("DELETE", deleteUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult())
{
return(response.IsSuccessStatusCode);
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(!Log.HasLoggedErrors);
}
public static async Task <List <string> > ListBlobs(TaskLoggingHelper Log, string AccountName, string AccountKey, string ContainerName, string FilterBlobNames)
{
List <string> blobsNames = new List <string>();
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
if (!string.IsNullOrWhiteSpace(FilterBlobNames))
{
urlListBlobs += $"&prefix={FilterBlobNames}";
}
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
XmlDocument responseFile;
string nextMarker = string.Empty;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
while (!string.IsNullOrEmpty(nextMarker))
{
urlListBlobs = $"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}";
if (!string.IsNullOrWhiteSpace(FilterBlobNames))
{
urlListBlobs += $"&prefix={FilterBlobNames}";
}
var nextRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
using (HttpResponseMessage nextResponse = AzureHelper.RequestWithRetry(Log, client, nextRequest).GetAwaiter().GetResult())
{
responseFile = new XmlDocument();
responseFile.LoadXml(await nextResponse.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}
}
return(blobsNames);
}
public async Task <bool> FileEqualsExistingBlobAsync(
string accountName,
string accountKey,
string containerName,
string filePath,
string destinationBlob,
int uploadTimeout)
{
using (var client = new HttpClient
{
Timeout = TimeSpan.FromMinutes(uploadTimeout)
})
{
log.LogMessage(
MessageImportance.Low,
$"Downloading blob {destinationBlob} to check if identical.");
string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob);
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(
log,
client,
createRequest))
{
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException(
$"Failed to retrieve existing blob {destinationBlob}, " +
$"status code {response.StatusCode}.");
}
byte[] existingBytes = await response.Content.ReadAsByteArrayAsync();
byte[] localBytes = File.ReadAllBytes(filePath);
bool equal = localBytes.SequenceEqual(existingBytes);
if (equal)
{
log.LogMessage(
MessageImportance.Normal,
"Item exists in blob storage, and is verified to be identical. " +
$"File: '{filePath}' Blob: '{destinationBlob}'");
}
return(equal);
}
}
}
private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Build.Utilities.TaskLoggingHelper log)
{
TimeSpan maxWait = TimeSpan.FromSeconds(s_MaxWaitDefault);
TimeSpan delay = TimeSpan.FromMilliseconds(s_DelayDefault);
TimeSpan waitFor = maxWait;
CancellationToken token = instance._cancellationTokenSource.Token;
while (true)
{
token.ThrowIfCancellationRequested();
try
{
log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{instance._containerName}/{instance._blobName}'.");
using (HttpClient client = new HttpClient())
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "renew");
Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", instance._leaseId);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, headerLeaseId
};
var request = AzureHelper.RequestMessage("PUT", instance._leaseUrl, instance._accountName, instance._accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
throw new Exception("Unable to acquire lease.");
}
}
}
waitFor = maxWait;
}
catch (Exception e)
{
Console.WriteLine($"Rerying lease renewal on {instance._containerName}, {e.Message}");
waitFor = delay;
}
token.ThrowIfCancellationRequested();
Task.Delay(waitFor, token).Wait();
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return(false);
}
string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName);
string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire");
Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */);
Tuple <string, string> headerSource = new Tuple <string, string>("x-ms-copy-source", sourceUrl);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, leaseDuration, headerSource
};
var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request))
{
if (response.IsSuccessStatusCode)
{
return(true);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(false);
}
public void Release()
{
// Cancel the lease renewal task since we are about to release the lease.
ResetLeaseRenewalTaskState();
using (HttpClient client = new HttpClient())
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "release");
Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", _leaseId);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, headerLeaseId
};
var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(_log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
_log.LogMessage($"Unable to release lease on container/blob {_containerName}/{_blobName}.");
}
}
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
// track the number of blobs that fail to download
int failureCount = 0;
using (HttpClient client = new HttpClient())
{
foreach (string blob in blobNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
string filename = Path.Combine(downloadBlobDirectory, blobFilename);
var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(
MessageImportance.Normal,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return(false);
}
// first check what blobs are present
string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list";
HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey);
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Listing Azure containers in storage account '{0}'.", AccountName);
string url = $"https://{AccountName}.blob.core.windows.net/?comp=list";
Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName);
List <ITaskItem> discoveredContainers = new List <ITaskItem>();
using (HttpClient client = new HttpClient())
{
string nextMarker = string.Empty;
try
{
do
{
string urlToUse = url;
if (!string.IsNullOrEmpty(nextMarker))
{
urlToUse = $"{url}&marker={nextMarker}";
}
var createRequest = AzureHelper.RequestMessage("GET", urlToUse, AccountName, AccountKey);
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
discoveredContainers.AddRange(from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText));
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}while (!string.IsNullOrEmpty(nextMarker));
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
ContainerNames = discoveredContainers.ToArray();
if (ContainerNames.Length == 0)
{
Log.LogWarning("No containers were found.");
}
else
{
Log.LogMessage("Found {0} containers.", ContainerNames.Length);
}
return(!Log.HasLoggedErrors);
}
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle)
{
await clientThrottle.WaitAsync();
string filename = string.Empty;
try {
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob));
if (!DownloadFlatFiles)
{
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
filename = Path.Combine(downloadBlobDirectory, blobFilename);
}
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
}
}
}
catch (PathTooLongException)
{
Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}");
}
catch (Exception ex)
{
Log.LogError(ex.ToString());
}
finally
{
clientThrottle.Release();
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("PUT", url, AccountName, AccountKey);
Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) =>
{
// the Conflict status (409) indicates that the container already exists, so
// if FailIfExists is set to false and we get a 409 don't fail the task.
return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict));
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate))
{
try
{
Log.LogMessage(
MessageImportance.Low,
"Received response to create Container {0}: Status Code: {1} {2}",
ContainerName, response.StatusCode, response.Content.ToString());
// specifying zero is valid, it means "I don't want a token"
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
// specifying zero is valid, it means "I don't want a token"
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
}
return(!Log.HasLoggedErrors);
}
