public async Task <bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(
MessageImportance.Normal,
"Begin uploading blobs to Azure account {0} in container {1}.",
AccountName,
ContainerName);
if (Items.Length == 0)
{
Log.LogError("No items were provided for upload.");
return(false);
}
// first check what blobs are present
string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list";
HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
try
{
using (HttpClient client = new HttpClient())
{
var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey);
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
{
var doc = new XmlDocument();
doc.LoadXml(await response.Content.ReadAsStringAsync());
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
foreach (XmlNode node in nodes)
{
blobsPresent.Add(node["Name"].InnerText);
}
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
}
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
{
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
}
Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task UploadBlockBlobAsync(
CancellationToken ct,
string AccountName,
string AccountKey,
string ContainerName,
string filePath,
string destinationBlob,
string contentType,
int uploadTimeout,
string leaseId = "")
{
string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName);
string fileName = destinationBlob;
fileName = fileName.Replace("\\", "/");
string blobUploadUrl = resourceUrl + "/" + fileName;
int size = (int)new FileInfo(filePath).Length;
int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
int bytesLeft = size;
List <string> blockIds = new List <string>();
int numberOfBlocks = (size / blockSize) + 1;
int countForId = 0;
using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
int offset = 0;
while (bytesLeft > 0)
{
int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
byte[] fileBytes = new byte[blockSize];
int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
if (nextBytesToRead != read)
{
throw new Exception(string.Format(
"Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
read, fileName, nextBytesToRead));
}
string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
blockIds.Add(blockId);
string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Clear();
// In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB.
client.Timeout = TimeSpan.FromMinutes(uploadTimeout);
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
req.Headers.Add(
AzureHelper.DateHeaderString,
dt.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (!string.IsNullOrWhiteSpace(leaseId))
{
log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}");
req.Headers.Add("x-ms-lease-id", leaseId);
}
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt,
req,
string.Empty,
string.Empty,
nextBytesToRead.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(fileBytes, 0, nextBytesToRead);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
countForId,
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
offset += read;
bytesLeft -= nextBytesToRead;
countForId += 1;
}
}
string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
using (HttpClient client = new HttpClient())
{
Func <HttpRequestMessage> createRequest = () =>
{
DateTime dt1 = DateTime.UtcNow;
var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
if (string.IsNullOrEmpty(contentType))
{
contentType = DetermineContentTypeBasedOnFileExtension(filePath);
}
if (!string.IsNullOrEmpty(contentType))
{
req.Headers.Add(AzureHelper.ContentTypeString, contentType);
}
string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath);
if (!string.IsNullOrEmpty(cacheControl))
{
req.Headers.Add(AzureHelper.CacheControlString, cacheControl);
}
var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>");
foreach (object item in blockIds)
{
body.AppendFormat("<Latest>{0}</Latest>", item);
}
body.Append("</BlockList>");
byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
if (!string.IsNullOrWhiteSpace(leaseId))
{
log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}");
req.Headers.Add("x-ms-lease-id", leaseId);
}
req.Headers.Add(
AzureHelper.AuthorizationHeaderString,
AzureHelper.AuthorizationHeader(
AccountName,
AccountKey,
"PUT",
dt1,
req,
string.Empty,
string.Empty,
bodyData.Length.ToString(),
string.Empty));
Stream postStream = new MemoryStream();
postStream.Write(bodyData, 0, bodyData.Length);
postStream.Seek(0, SeekOrigin.Begin);
req.Content = new StreamContent(postStream);
return(req);
};
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
{
log.LogMessage(
MessageImportance.Low,
"Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
fileName,
response.StatusCode,
await response.Content.ReadAsStringAsync());
}
}
}