public async Task <bool> ExecuteAsync()
{
using (FileStream fs = new FileStream(EventData, FileMode.Open, FileAccess.Read))
{
using (StreamReader streamReader = new StreamReader(fs))
{
using (HttpClient client = new HttpClient())
{
string url = String.IsNullOrEmpty(PartitionKey) ? EventHubPath + "/messages" : EventHubPath + "/partitions/" + PartitionKey + "/messages";
Func <HttpRequestMessage> createRequest = () =>
{
var req = new HttpRequestMessage(HttpMethod.Post, url);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, ConstructSharedAccessToken());
contentBytes = Encoding.UTF8.GetBytes(streamReader.ReadToEnd());
Stream postStream = new MemoryStream();
postStream.Write(this.contentBytes, 0, this.contentBytes.Length);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
Log.LogMessage(MessageImportance.High, "Sending {0} to event hub {1}", EventData, url);
try
{
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
Log.LogMessage(MessageImportance.Low, "Received response to send event to event hub");
}
Log.LogMessage(
MessageImportance.Normal,
"Successfully sent notification to event hub path {0}.",
EventHubPath);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
}
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName);
DateTime dateTime = DateTime.UtcNow;
string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName);
Log.LogMessage(MessageImportance.Normal, "Sending request to list containers in account '{0}'.", AccountName);
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url))
{
try
{
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
XmlDocument responseFile;
using (HttpResponseMessage response = await client.SendAsync(request))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
ContainerNames = (from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText)).ToArray();
}
}
catch (Exception e)
{
Log.LogError("Failed to retrieve information.\n" + e.Message);
return(false);
}
}
}
return(true);
}
private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Build.Utilities.TaskLoggingHelper log)
{
TimeSpan maxWait = TimeSpan.FromSeconds(s_MaxWaitDefault);
TimeSpan delay = TimeSpan.FromMilliseconds(s_DelayDefault);
TimeSpan waitFor = maxWait;
CancellationToken token = instance._cancellationTokenSource.Token;
while (true)
{
token.ThrowIfCancellationRequested();
try
{
log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{instance._containerName}/{instance._blobName}'.");
using (HttpClient client = new HttpClient())
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "renew");
Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", instance._leaseId);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, headerLeaseId
};
var request = AzureHelper.RequestMessage("PUT", instance._leaseUrl, instance._accountName, instance._accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
throw new Exception("Unable to acquire lease.");
}
}
}
waitFor = maxWait;
}
catch (Exception e)
{
Console.WriteLine($"Rerying lease renewal on {instance._containerName}, {e.Message}");
waitFor = delay;
}
token.ThrowIfCancellationRequested();
Task.Delay(waitFor, token).Wait();
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return(false);
}
string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName);
string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName);
using (HttpClient client = new HttpClient())
{
try
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire");
Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */);
Tuple <string, string> headerSource = new Tuple <string, string>("x-ms-copy-source", sourceUrl);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, leaseDuration, headerSource
};
var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request))
{
if (response.IsSuccessStatusCode)
{
return(true);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(false);
}
public void Release()
{
// Cancel the lease renewal task since we are about to release the lease.
ResetLeaseRenewalTaskState();
using (HttpClient client = new HttpClient())
{
Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "release");
Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", _leaseId);
List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >()
{
leaseAction, headerLeaseId
};
var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders);
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(_log, client, request).GetAwaiter().GetResult())
{
if (!response.IsSuccessStatusCode)
{
_log.LogMessage($"Unable to release lease on container/blob {_containerName}/{_blobName}.");
}
}
}
}
// This code is duplicated in BuildTools task DownloadFromAzure, and that code should be refactored to permit blob listing.
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
if (Log.HasLoggedErrors)
{
return(false);
}
List <string> blobsNames = new List <string>();
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
if (!string.IsNullOrWhiteSpace(FilterBlobNames))
{
urlListBlobs += $"&prefix={FilterBlobNames}";
}
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
try
{
var createRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
XmlDocument responseFile;
string nextMarker = string.Empty;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
while (!string.IsNullOrEmpty(nextMarker))
{
urlListBlobs = string.Format($"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}");
var nextRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey);
using (HttpResponseMessage nextResponse = AzureHelper.RequestWithRetry(Log, client, nextRequest).GetAwaiter().GetResult())
{
responseFile = new XmlDocument();
responseFile.LoadXml(await nextResponse.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}
BlobNames = blobsNames.ToArray();
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
}
public async Task <bool> ExecuteAsync(CancellationToken ct)
{
Log.LogMessage(
MessageImportance.High,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return(false);
}
// first check what blobs are present
string checkListUrl = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container&comp=list",
AccountName,
ContainerName);
HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Get, checkListUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dt,
req));
return(req);
};
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.High, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
return(true);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
public async Task UploadBlockBlobAsync(
CancellationToken ct,
string AccountName,
string AccountKey,
string ContainerName,
string filePath,
string destinationBlob)
{
string resourceUrl = string.Format("https://{0}.blob.core.windows.net/{1}", AccountName, ContainerName);
string fileName = destinationBlob;
fileName = fileName.Replace("\\", "/");
string blobUploadUrl = resourceUrl + "/" + fileName;
int size = (int)new FileInfo(filePath).Length;
int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
int bytesLeft = size;
List <string> blockIds = new List <string>();
int numberOfBlocks = (size / blockSize) + 1;
int countForId = 0;
using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open))
{
int offset = 0;
while (bytesLeft > 0)
{
int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
byte[] fileBytes = new byte[blockSize];
int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
if (nextBytesToRead != read)
{
throw new Exception(string.Format(
"Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
read, fileName, nextBytesToRead));
}
string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
blockIds.Add(blockId);
string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Clear();
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
req.Headers.Add(
AzureHelper.DateHeaderString,
dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req,
string.Empty,
string.Empty,
nextBytesToRead.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(fileBytes, 0, nextBytesToRead);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
countForId,
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
offset += read;
bytesLeft -= nextBytesToRead;
countForId += 1;
}
}
string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt1 = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>");
foreach (object item in blockIds)
{
body.AppendFormat("<Latest>{0}</Latest>", item);
}
body.Append("</BlockList>");
byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt1,
req,
string.Empty,
string.Empty,
bodyData.Length.ToString(),
""));
Stream postStream = new MemoryStream();
postStream.Write(bodyData, 0, bodyData.Length);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
// track the number of blobs that fail to download
int failureCount = 0;
using (HttpClient client = new HttpClient())
{
foreach (string blob in blobNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
string filename = Path.Combine(downloadBlobDirectory, blobFilename);
var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
List <string> blobsNames = new List <string>();
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
XmlDocument responseFile;
string nextMarker = string.Empty;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
while (!string.IsNullOrEmpty(nextMarker))
{
urlListBlobs = string.Format($"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}");
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, createRequest).GetAwaiter().GetResult())
{
responseFile = new XmlDocument();
responseFile.LoadXml(response.Content.ReadAsStringAsync().GetAwaiter().GetResult());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}
// track the number of blobs that fail to download
int failureCount = 0;
if (blobsNames.Count == 0)
{
Log.LogWarning("No blobs were found.");
}
else
{
Log.LogMessage(MessageImportance.Low, $"{blobsNames.Count} blobs found.");
}
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
}
public async Task <bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(
MessageImportance.Normal,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return(false);
}
// first check what blobs are present
string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list";
HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey);
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
DateTime dt = DateTime.UtcNow;
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Normal, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Put, url))
{
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req));
byte[] bytestoWrite = new byte[0];
int bytesToWriteLength = 0;
Stream postStream = new MemoryStream();
postStream.Write(bytestoWrite, 0, bytesToWriteLength);
req.Content = new StreamContent(postStream);
using (HttpResponseMessage response = await client.SendAsync(req))
{
this.Log.LogMessage(
MessageImportance.Normal,
"Received response to create Container {0}: Status Code: {1} {2}",
this.ContainerName, response.StatusCode, response.Content);
}
}
}
try
{
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
else
{
Log.LogWarning("{0} will not allow read permissions - ReadOnlyTokenDaysValid must be greater than 0.", StorageUri);
}
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
else
{
Log.LogWarning("{0} will not allow write permissions - WriteOnlyTokenDaysValid must be greater than 0.", StorageUri);
}
}
catch (ArgumentOutOfRangeException e)
{
Log.LogError("ArgumentOutOfRangeException: Invalid AzureHelper.SasAccessType :" + e.Message);
return(false);
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Listing Azure containers in storage account '{0}'.", AccountName);
string url = $"https://{AccountName}.blob.core.windows.net/?comp=list";
Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName);
List <ITaskItem> discoveredContainers = new List <ITaskItem>();
using (HttpClient client = new HttpClient())
{
string nextMarker = string.Empty;
try
{
do
{
string urlToUse = url;
if (!string.IsNullOrEmpty(nextMarker))
{
urlToUse = $"{url}&marker={nextMarker}";
}
var createRequest = AzureHelper.RequestMessage("GET", urlToUse, AccountName, AccountKey);
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
discoveredContainers.AddRange(from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText));
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}while (!string.IsNullOrEmpty(nextMarker));
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
ContainerNames = discoveredContainers.ToArray();
if (ContainerNames.Length == 0)
{
Log.LogWarning("No containers were found.");
}
else
{
Log.LogMessage("Found {0} containers.", ContainerNames.Length);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName);
string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName);
Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
// TODO: This task has a bug, it needs to continue when there are > 5000 containers in a storage acccount.
// Fix is something like the one made to DownloadFromAzure, but not pressing since it looks like GetLatestContainerNameFromAzure is rarely / not used.
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
ContainerNames = (from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText)).ToArray();
if (ContainerNames.Length == 0)
{
Log.LogWarning("No containers were found.");
}
else
{
Log.LogMessage("Found {0} containers.", ContainerNames.Length);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
List <string> blobsNames = null;
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames = elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList();
if (blobsNames.Count == 0)
{
Log.LogWarning("No blobs were found.");
}
}
// track the number of blobs that fail to download
int failureCount = 0;
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
// if no blobs failed to download the task succeeded
return(failureCount == 0);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "Downloading container {0} from storage account '{1}'.", ContainerName, AccountName);
DateTime dateTime = DateTime.UtcNow;
List <string> blobsNames = null;
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Normal, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs))
{
try
{
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
XmlDocument responseFile;
using (HttpResponseMessage response = await client.SendAsync(request))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames = elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList();
}
}
catch (Exception e)
{
Log.LogError("Failed to retrieve information.\n" + e.Message);
return(false);
}
}
DownloadDirectory = DownloadDirectory ?? Directory.GetCurrentDirectory();
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Normal, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob))
{
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
using (HttpResponseMessage response = await client.SendAsync(request))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
}
}
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, url);
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req));
byte[] bytestoWrite = new byte[0];
int bytesToWriteLength = 0;
Stream postStream = new MemoryStream();
postStream.Write(bytestoWrite, 0, bytesToWriteLength);
req.Content = new StreamContent(postStream);
return(req);
};
Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) =>
{
// the Conflict status (409) indicates that the container already exists, so
// if FailIfExists is set to false and we get a 409 don't fail the task.
return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict));
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate))
{
try
{
Log.LogMessage(
MessageImportance.Low,
"Received response to create Container {0}: Status Code: {1} {2}",
ContainerName, response.StatusCode, response.Content.ToString());
// specifying zero is valid, it means "I don't want a token"
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
// specifying zero is valid, it means "I don't want a token"
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
}
return(true);
}
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle)
{
await clientThrottle.WaitAsync();
string filename = string.Empty;
try {
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob);
filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob));
if (!DownloadFlatFiles)
{
int dirIndex = blob.LastIndexOf("/");
string blobDirectory = string.Empty;
string blobFilename = string.Empty;
if (dirIndex == -1)
{
blobFilename = blob;
}
else
{
blobDirectory = blob.Substring(0, dirIndex);
blobFilename = blob.Substring(dirIndex + 1);
// Trim blob name prefix (directory part) from download to blob directory
if (BlobNamePrefix != null)
{
if (BlobNamePrefix.Length > dirIndex)
{
BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex);
}
blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length);
}
}
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
filename = Path.Combine(downloadBlobDirectory, blobFilename);
}
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
}
}
}
catch (PathTooLongException)
{
Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}");
}
catch (Exception ex)
{
Log.LogError(ex.ToString());
}
finally
{
clientThrottle.Release();
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("PUT", url, AccountName, AccountKey);
Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) =>
{
// the Conflict status (409) indicates that the container already exists, so
// if FailIfExists is set to false and we get a 409 don't fail the task.
return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict));
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate))
{
try
{
Log.LogMessage(
MessageImportance.Low,
"Received response to create Container {0}: Status Code: {1} {2}",
ContainerName, response.StatusCode, response.Content.ToString());
// specifying zero is valid, it means "I don't want a token"
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
// specifying zero is valid, it means "I don't want a token"
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
}
return(!Log.HasLoggedErrors);
}