public async Task <bool> ExecuteAsync(CancellationToken ct)
{
Log.LogMessage(
MessageImportance.High,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
throw new ArgumentException("No items were provided for upload.");
}
// first check what blobs are present
string checkListUrl = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container&comp=list",
AccountName,
ContainerName);
DateTime dt = DateTime.UtcNow;
HashSet <string> blobsPresent = new HashSet <string>();
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Get, checkListUrl))
{
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dt,
req));
Log.LogMessage(MessageImportance.Normal, "Sending request to check whether Container blobs exist");
XmlDocument doc;
using (HttpResponseMessage response = await client.SendAsync(req, ct))
{
doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
}
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Normal, "Received response to check whether Container blobs exist");
}
}
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent)));
Log.LogMessage(MessageImportance.High, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
return(true);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName);
string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName);
Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
ContainerNames = (from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText)).ToArray();
if (ContainerNames.Length == 0)
{
Log.LogWarning("No containers were found.");
}
else
{
Log.LogMessage("Found {0} containers.", ContainerNames.Length);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
return(true);
}
public static Func <HttpRequestMessage> RequestMessage(string method, string url, string accountName, string accountKey, List <Tuple <string, string> > additionalHeaders = null, string body = null)
{
Func <HttpRequestMessage> requestFunc = () =>
{
HttpMethod httpMethod = HttpMethod.Get;
if (method == "PUT")
{
httpMethod = HttpMethod.Put;
}
else if (method == "DELETE")
{
httpMethod = HttpMethod.Delete;
}
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(httpMethod, url);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (additionalHeaders != null)
{
foreach (Tuple <string, string> additionalHeader in additionalHeaders)
{
request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2);
}
}
if (body != null)
{
request.Content = new StringContent(body);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
accountName,
accountKey,
method,
dateTime,
request,
"",
"",
request.Content.Headers.ContentLength.ToString(),
request.Content.Headers.ContentType.ToString()));
}
else
{
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
accountName,
accountKey,
method,
dateTime,
request));
}
return(request);
};
return(requestFunc);
}
public async Task <bool> ExecuteAsync(CancellationToken ct)
{
Log.LogMessage(
MessageImportance.High,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return(false);
}
// first check what blobs are present
string checkListUrl = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container&comp=list",
AccountName,
ContainerName);
HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Get, checkListUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dt,
req));
return(req);
};
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.High, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
return(true);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
public async Task UploadBlockBlobAsync(
CancellationToken ct,
string AccountName,
string AccountKey,
string ContainerName,
string filePath,
string destinationBlob)
{
string resourceUrl = string.Format("https://{0}.blob.core.windows.net/{1}", AccountName, ContainerName);
string fileName = destinationBlob;
fileName = fileName.Replace("\\", "/");
string blobUploadUrl = resourceUrl + "/" + fileName;
int size = (int)new FileInfo(filePath).Length;
int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
int bytesLeft = size;
List <string> blockIds = new List <string>();
int numberOfBlocks = (size / blockSize) + 1;
int countForId = 0;
using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open))
{
int offset = 0;
while (bytesLeft > 0)
{
int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
byte[] fileBytes = new byte[blockSize];
int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
if (nextBytesToRead != read)
{
throw new Exception(string.Format(
"Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
read, fileName, nextBytesToRead));
}
string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
blockIds.Add(blockId);
string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Clear();
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
req.Headers.Add(
AzureHelper.DateHeaderString,
dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req,
string.Empty,
string.Empty,
nextBytesToRead.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(fileBytes, 0, nextBytesToRead);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
countForId,
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
offset += read;
bytesLeft -= nextBytesToRead;
countForId += 1;
}
}
string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt1 = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>");
foreach (object item in blockIds)
{
body.AppendFormat("<Latest>{0}</Latest>", item);
}
body.Append("</BlockList>");
byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt1,
req,
string.Empty,
string.Empty,
bodyData.Length.ToString(),
""));
Stream postStream = new MemoryStream();
postStream.Write(bodyData, 0, bodyData.Length);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
List <string> blobsNames = new List <string>();
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
XmlDocument responseFile;
string nextMarker = string.Empty;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
while (!string.IsNullOrEmpty(nextMarker))
{
urlListBlobs = string.Format($"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}");
using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, createRequest).GetAwaiter().GetResult())
{
responseFile = new XmlDocument();
responseFile.LoadXml(response.Content.ReadAsStringAsync().GetAwaiter().GetResult());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames.AddRange(elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList());
nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText;
}
}
// track the number of blobs that fail to download
int failureCount = 0;
if (blobsNames.Count == 0)
{
Log.LogWarning("No blobs were found.");
}
else
{
Log.LogMessage(MessageImportance.Low, $"{blobsNames.Count} blobs found.");
}
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
// Blobs can be files but have the name of a directory. We'll skip those and log something weird happened.
if (!string.IsNullOrEmpty(Path.GetFileName(filename)))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed.");
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
Log.LogMessage($"{failureCount} errors seen downloading blobs.");
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
DateTime dt = DateTime.UtcNow;
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Normal, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Put, url))
{
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req));
byte[] bytestoWrite = new byte[0];
int bytesToWriteLength = 0;
Stream postStream = new MemoryStream();
postStream.Write(bytestoWrite, 0, bytesToWriteLength);
req.Content = new StreamContent(postStream);
using (HttpResponseMessage response = await client.SendAsync(req))
{
this.Log.LogMessage(
MessageImportance.Normal,
"Received response to create Container {0}: Status Code: {1} {2}",
this.ContainerName, response.StatusCode, response.Content);
}
}
}
try
{
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
else
{
Log.LogWarning("{0} will not allow read permissions - ReadOnlyTokenDaysValid must be greater than 0.", StorageUri);
}
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
else
{
Log.LogWarning("{0} will not allow write permissions - WriteOnlyTokenDaysValid must be greater than 0.", StorageUri);
}
}
catch (ArgumentOutOfRangeException e)
{
Log.LogError("ArgumentOutOfRangeException: Invalid AzureHelper.SasAccessType :" + e.Message);
return(false);
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName);
string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName);
Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
// TODO: This task has a bug, it needs to continue when there are > 5000 containers in a storage acccount.
// Fix is something like the one made to DownloadFromAzure, but not pressing since it looks like GetLatestContainerNameFromAzure is rarely / not used.
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
ContainerNames = (from x in elemList.Cast <XmlNode>()
where x.InnerText.Contains(Prefix)
select new TaskItem(x.InnerText)).ToArray();
if (ContainerNames.Length == 0)
{
Log.LogWarning("No containers were found.");
}
else
{
Log.LogMessage("Found {0} containers.", ContainerNames.Length);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(
MessageImportance.High,
"Creating container named '{0}' in storage account {1}.",
ContainerName,
AccountName);
string url = string.Format(
"https://{0}.blob.core.windows.net/{1}?restype=container",
AccountName,
ContainerName);
StorageUri = string.Format(
"https://{0}.blob.core.windows.net/{1}/",
AccountName,
ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to create Container");
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, url);
req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req));
byte[] bytestoWrite = new byte[0];
int bytesToWriteLength = 0;
Stream postStream = new MemoryStream();
postStream.Write(bytestoWrite, 0, bytesToWriteLength);
req.Content = new StreamContent(postStream);
return(req);
};
Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) =>
{
// the Conflict status (409) indicates that the container already exists, so
// if FailIfExists is set to false and we get a 409 don't fail the task.
return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict));
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate))
{
try
{
Log.LogMessage(
MessageImportance.Low,
"Received response to create Container {0}: Status Code: {1} {2}",
ContainerName, response.StatusCode, response.Content.ToString());
// specifying zero is valid, it means "I don't want a token"
if (ReadOnlyTokenDaysValid > 0)
{
ReadOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Read,
ReadOnlyTokenDaysValid);
}
// specifying zero is valid, it means "I don't want a token"
if (WriteOnlyTokenDaysValid > 0)
{
WriteOnlyToken = AzureHelper.CreateContainerSasToken(
AccountName,
ContainerName,
AccountKey,
AzureHelper.SasAccessType.Write,
WriteOnlyTokenDaysValid);
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "Downloading container {0} from storage account '{1}'.", ContainerName, AccountName);
DateTime dateTime = DateTime.UtcNow;
List <string> blobsNames = null;
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Normal, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs))
{
try
{
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
XmlDocument responseFile;
using (HttpResponseMessage response = await client.SendAsync(request))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames = elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList();
}
}
catch (Exception e)
{
Log.LogError("Failed to retrieve information.\n" + e.Message);
return(false);
}
}
DownloadDirectory = DownloadDirectory ?? Directory.GetCurrentDirectory();
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Normal, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob))
{
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
using (HttpResponseMessage response = await client.SendAsync(request))
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
}
}
}
return(true);
}
public async Task <bool> ExecuteAsync()
{
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
List <string> blobsNames = null;
string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName);
Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName);
using (HttpClient client = new HttpClient())
{
try
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
XmlDocument responseFile;
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
responseFile = new XmlDocument();
responseFile.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList elemList = responseFile.GetElementsByTagName("Name");
blobsNames = elemList.Cast <XmlNode>()
.Select(x => x.InnerText)
.ToList();
if (blobsNames.Count == 0)
{
Log.LogWarning("No blobs were found.");
}
}
// track the number of blobs that fail to download
int failureCount = 0;
foreach (string blob in blobsNames)
{
Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob);
string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob);
string filename = Path.Combine(DownloadDirectory, blob);
string blobDirectory = blob.Substring(0, blob.LastIndexOf("/"));
string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory);
if (!Directory.Exists(downloadBlobDirectory))
{
Directory.CreateDirectory(downloadBlobDirectory);
}
createRequest = () =>
{
DateTime dateTime = DateTime.UtcNow;
var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob);
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"GET",
dateTime,
request));
return(request);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
if (response.IsSuccessStatusCode)
{
Stream responseStream = await response.Content.ReadAsStreamAsync();
using (FileStream sourceStream = File.Open(filename, FileMode.Create))
{
responseStream.CopyTo(sourceStream);
}
}
else
{
Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode);
++failureCount;
}
}
}
// if no blobs failed to download the task succeeded
return(failureCount == 0);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
}
