public async Task <bool> ExecuteAsync() { using (FileStream fs = new FileStream(EventData, FileMode.Open, FileAccess.Read)) { using (StreamReader streamReader = new StreamReader(fs)) { using (HttpClient client = new HttpClient()) { string url = String.IsNullOrEmpty(PartitionKey) ? EventHubPath + "/messages" : EventHubPath + "/partitions/" + PartitionKey + "/messages"; Func <HttpRequestMessage> createRequest = () => { var req = new HttpRequestMessage(HttpMethod.Post, url); req.Headers.Add(AzureHelper.AuthorizationHeaderString, ConstructSharedAccessToken()); contentBytes = Encoding.UTF8.GetBytes(streamReader.ReadToEnd()); Stream postStream = new MemoryStream(); postStream.Write(this.contentBytes, 0, this.contentBytes.Length); postStream.Seek(0, SeekOrigin.Begin); req.Content = new StreamContent(postStream); return(req); }; Log.LogMessage(MessageImportance.High, "Sending {0} to event hub {1}", EventData, url); try { using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { Log.LogMessage(MessageImportance.Low, "Received response to send event to event hub"); } Log.LogMessage( MessageImportance.Normal, "Successfully sent notification to event hub path {0}.", EventHubPath); } catch (Exception e) { Log.LogErrorFromException(e, true); return(false); } } } } return(true); }
public async Task <bool> ExecuteAsync() { Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName); DateTime dateTime = DateTime.UtcNow; string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName); Log.LogMessage(MessageImportance.Normal, "Sending request to list containers in account '{0}'.", AccountName); using (HttpClient client = new HttpClient()) { using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url)) { try { request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); XmlDocument responseFile; using (HttpResponseMessage response = await client.SendAsync(request)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); ContainerNames = (from x in elemList.Cast <XmlNode>() where x.InnerText.Contains(Prefix) select new TaskItem(x.InnerText)).ToArray(); } } catch (Exception e) { Log.LogError("Failed to retrieve information.\n" + e.Message); return(false); } } } return(true); }
private static void AutoRenewLeaseOnBlob(AzureBlobLease instance, Microsoft.Build.Utilities.TaskLoggingHelper log) { TimeSpan maxWait = TimeSpan.FromSeconds(s_MaxWaitDefault); TimeSpan delay = TimeSpan.FromMilliseconds(s_DelayDefault); TimeSpan waitFor = maxWait; CancellationToken token = instance._cancellationTokenSource.Token; while (true) { token.ThrowIfCancellationRequested(); try { log.LogMessage(MessageImportance.Low, $"Requesting lease for container/blob '{instance._containerName}/{instance._blobName}'."); using (HttpClient client = new HttpClient()) { Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "renew"); Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", instance._leaseId); List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >() { leaseAction, headerLeaseId }; var request = AzureHelper.RequestMessage("PUT", instance._leaseUrl, instance._accountName, instance._accountKey, additionalHeaders); using (HttpResponseMessage response = AzureHelper.RequestWithRetry(log, client, request).GetAwaiter().GetResult()) { if (!response.IsSuccessStatusCode) { throw new Exception("Unable to acquire lease."); } } } waitFor = maxWait; } catch (Exception e) { Console.WriteLine($"Rerying lease renewal on {instance._containerName}, {e.Message}"); waitFor = delay; } token.ThrowIfCancellationRequested(); Task.Delay(waitFor, token).Wait(); } }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); if (Log.HasLoggedErrors) { return(false); } string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName); string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName); using (HttpClient client = new HttpClient()) { try { Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire"); Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */); Tuple <string, string> headerSource = new Tuple <string, string>("x-ms-copy-source", sourceUrl); List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >() { leaseAction, leaseDuration, headerSource }; var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request)) { if (response.IsSuccessStatusCode) { return(true); } } } catch (Exception e) { Log.LogErrorFromException(e, true); } } return(false); }
public void Release() { // Cancel the lease renewal task since we are about to release the lease. ResetLeaseRenewalTaskState(); using (HttpClient client = new HttpClient()) { Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "release"); Tuple <string, string> headerLeaseId = new Tuple <string, string>("x-ms-lease-id", _leaseId); List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >() { leaseAction, headerLeaseId }; var request = AzureHelper.RequestMessage("PUT", _leaseUrl, _accountName, _accountKey, additionalHeaders); using (HttpResponseMessage response = AzureHelper.RequestWithRetry(_log, client, request).GetAwaiter().GetResult()) { if (!response.IsSuccessStatusCode) { _log.LogMessage($"Unable to release lease on container/blob {_containerName}/{_blobName}."); } } } }
// This code is duplicated in BuildTools task DownloadFromAzure, and that code should be refactored to permit blob listing. public async Task <bool> ExecuteAsync() { ParseConnectionString(); if (Log.HasLoggedErrors) { return(false); } List <string> blobsNames = new List <string>(); string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); if (!string.IsNullOrWhiteSpace(FilterBlobNames)) { urlListBlobs += $"&prefix={FilterBlobNames}"; } Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName); using (HttpClient client = new HttpClient()) { try { var createRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey); XmlDocument responseFile; string nextMarker = string.Empty; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames.AddRange(elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList()); nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText; } while (!string.IsNullOrEmpty(nextMarker)) { urlListBlobs = string.Format($"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}"); var nextRequest = AzureHelper.RequestMessage("GET", urlListBlobs, AccountName, AccountKey); using (HttpResponseMessage nextResponse = AzureHelper.RequestWithRetry(Log, client, nextRequest).GetAwaiter().GetResult()) { responseFile = new XmlDocument(); responseFile.LoadXml(await nextResponse.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames.AddRange(elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList()); nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText; } } BlobNames = blobsNames.ToArray(); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); } }
public async Task <bool> ExecuteAsync(CancellationToken ct) { Log.LogMessage( MessageImportance.High, "Begin uploading blobs to Azure account {0} in container {1}.", AccountName, ContainerName); if (Items.Length == 0) { Log.LogError("No items were provided for upload."); return(false); } // first check what blobs are present string checkListUrl = string.Format( "https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase); try { using (HttpClient client = new HttpClient()) { Func <HttpRequestMessage> createRequest = () => { DateTime dt = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Get, checkListUrl); req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dt, req)); return(req); }; Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { var doc = new XmlDocument(); doc.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); foreach (XmlNode node in nodes) { blobsPresent.Add(node["Name"].InnerText); } Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); } } using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) { await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); } Log.LogMessage(MessageImportance.High, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); return(true); } catch (Exception e) { Log.LogErrorFromException(e, true); return(false); } }
public async Task UploadBlockBlobAsync( CancellationToken ct, string AccountName, string AccountKey, string ContainerName, string filePath, string destinationBlob) { string resourceUrl = string.Format("https://{0}.blob.core.windows.net/{1}", AccountName, ContainerName); string fileName = destinationBlob; fileName = fileName.Replace("\\", "/"); string blobUploadUrl = resourceUrl + "/" + fileName; int size = (int)new FileInfo(filePath).Length; int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob int bytesLeft = size; List <string> blockIds = new List <string>(); int numberOfBlocks = (size / blockSize) + 1; int countForId = 0; using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open)) { int offset = 0; while (bytesLeft > 0) { int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; byte[] fileBytes = new byte[blockSize]; int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); if (nextBytesToRead != read) { throw new Exception(string.Format( "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", read, fileName, nextBytesToRead)); } string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); blockIds.Add(blockId); string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); using (HttpClient client = new HttpClient()) { client.DefaultRequestHeaders.Clear(); Func <HttpRequestMessage> createRequest = () => { DateTime dt = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); req.Headers.Add( AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); req.Headers.Add( AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt, req, string.Empty, string.Empty, nextBytesToRead.ToString(), string.Empty)); Stream postStream = new MemoryStream(); postStream.Write(fileBytes, 0, nextBytesToRead); postStream.Seek(0, SeekOrigin.Begin); req.Content = new StreamContent(postStream); return(req); }; log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) { log.LogMessage( MessageImportance.Low, "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", countForId, fileName, response.StatusCode, await response.Content.ReadAsStringAsync()); } } offset += read; bytesLeft -= nextBytesToRead; countForId += 1; } } string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; using (HttpClient client = new HttpClient()) { Func <HttpRequestMessage> createRequest = () => { DateTime dt1 = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>"); foreach (object item in blockIds) { body.AppendFormat("<Latest>{0}</Latest>", item); } body.Append("</BlockList>"); byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); req.Headers.Add( AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt1, req, string.Empty, string.Empty, bodyData.Length.ToString(), "")); Stream postStream = new MemoryStream(); postStream.Write(bodyData, 0, bodyData.Length); postStream.Seek(0, SeekOrigin.Begin); req.Content = new StreamContent(postStream); return(req); }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) { log.LogMessage( MessageImportance.Low, "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", fileName, response.StatusCode, await response.Content.ReadAsStringAsync()); } } }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.", ContainerName, AccountName, DownloadDirectory); try { List <string> blobNames = new List <string>(); if (BlobNames == null) { ListAzureBlobs listAzureBlobs = new ListAzureBlobs() { AccountName = AccountName, AccountKey = AccountKey, ContainerName = ContainerName, FilterBlobNames = BlobNamePrefix, BuildEngine = this.BuildEngine, HostObject = this.HostObject }; listAzureBlobs.Execute(); blobNames = listAzureBlobs.BlobNames.ToList(); } else { blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>(); if (BlobNamePrefix != null) { blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>(); } } // track the number of blobs that fail to download int failureCount = 0; using (HttpClient client = new HttpClient()) { foreach (string blob in blobNames) { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); int dirIndex = blob.LastIndexOf("/"); string blobDirectory = string.Empty; string blobFilename = string.Empty; if (dirIndex == -1) { blobFilename = blob; } else { blobDirectory = blob.Substring(0, dirIndex); blobFilename = blob.Substring(dirIndex + 1); // Trim blob name prefix (directory part) from download to blob directory if (BlobNamePrefix != null) { if (BlobNamePrefix.Length > dirIndex) { BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex); } blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length); } } string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } string filename = Path.Combine(downloadBlobDirectory, blobFilename); var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. if (!string.IsNullOrEmpty(Path.GetFileName(filename))) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); ++failureCount; } } } } Log.LogMessage($"{failureCount} errors seen downloading blobs."); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.", ContainerName, AccountName, DownloadDirectory); List <string> blobsNames = new List <string>(); string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName); using (HttpClient client = new HttpClient()) { try { Func <HttpRequestMessage> createRequest = () => { DateTime dateTime = DateTime.UtcNow; var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs); request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); return(request); }; XmlDocument responseFile; string nextMarker = string.Empty; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames.AddRange(elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList()); nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText; } while (!string.IsNullOrEmpty(nextMarker)) { urlListBlobs = string.Format($"https://{AccountName}.blob.core.windows.net/{ContainerName}?restype=container&comp=list&marker={nextMarker}"); using (HttpResponseMessage response = AzureHelper.RequestWithRetry(Log, client, createRequest).GetAwaiter().GetResult()) { responseFile = new XmlDocument(); responseFile.LoadXml(response.Content.ReadAsStringAsync().GetAwaiter().GetResult()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames.AddRange(elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList()); nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText; } } // track the number of blobs that fail to download int failureCount = 0; if (blobsNames.Count == 0) { Log.LogWarning("No blobs were found."); } else { Log.LogMessage(MessageImportance.Low, $"{blobsNames.Count} blobs found."); } foreach (string blob in blobsNames) { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob); string filename = Path.Combine(DownloadDirectory, blob); string blobDirectory = blob.Substring(0, blob.LastIndexOf("/")); string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } createRequest = () => { DateTime dateTime = DateTime.UtcNow; var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob); request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); return(request); }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. if (!string.IsNullOrEmpty(Path.GetFileName(filename))) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); ++failureCount; } } } Log.LogMessage($"{failureCount} errors seen downloading blobs."); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); } }
public async Task <bool> ExecuteAsync(CancellationToken ct) { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage( MessageImportance.Normal, "Begin uploading blobs to Azure account {0} in container {1}.", AccountName, ContainerName); if (Items.Length == 0) { Log.LogError("No items were provided for upload."); return(false); } // first check what blobs are present string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; HashSet <string> blobsPresent = new HashSet <string>(StringComparer.OrdinalIgnoreCase); try { using (HttpClient client = new HttpClient()) { var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { var doc = new XmlDocument(); doc.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); foreach (XmlNode node in nodes) { blobsPresent.Add(node["Name"].InnerText); } Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); } } using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) { await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); } Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); }
public async Task <bool> ExecuteAsync() { Log.LogMessage( MessageImportance.High, "Creating container named '{0}' in storage account {1}.", ContainerName, AccountName); DateTime dt = DateTime.UtcNow; string url = string.Format( "https://{0}.blob.core.windows.net/{1}?restype=container", AccountName, ContainerName); StorageUri = string.Format( "https://{0}.blob.core.windows.net/{1}/", AccountName, ContainerName); Log.LogMessage(MessageImportance.Normal, "Sending request to create Container"); using (HttpClient client = new HttpClient()) { using (HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Put, url)) { req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt, req)); byte[] bytestoWrite = new byte[0]; int bytesToWriteLength = 0; Stream postStream = new MemoryStream(); postStream.Write(bytestoWrite, 0, bytesToWriteLength); req.Content = new StreamContent(postStream); using (HttpResponseMessage response = await client.SendAsync(req)) { this.Log.LogMessage( MessageImportance.Normal, "Received response to create Container {0}: Status Code: {1} {2}", this.ContainerName, response.StatusCode, response.Content); } } } try { if (ReadOnlyTokenDaysValid > 0) { ReadOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Read, ReadOnlyTokenDaysValid); } else { Log.LogWarning("{0} will not allow read permissions - ReadOnlyTokenDaysValid must be greater than 0.", StorageUri); } if (WriteOnlyTokenDaysValid > 0) { WriteOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Write, WriteOnlyTokenDaysValid); } else { Log.LogWarning("{0} will not allow write permissions - WriteOnlyTokenDaysValid must be greater than 0.", StorageUri); } } catch (ArgumentOutOfRangeException e) { Log.LogError("ArgumentOutOfRangeException: Invalid AzureHelper.SasAccessType :" + e.Message); return(false); } return(true); }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage(MessageImportance.Normal, "Listing Azure containers in storage account '{0}'.", AccountName); string url = $"https://{AccountName}.blob.core.windows.net/?comp=list"; Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName); List <ITaskItem> discoveredContainers = new List <ITaskItem>(); using (HttpClient client = new HttpClient()) { string nextMarker = string.Empty; try { do { string urlToUse = url; if (!string.IsNullOrEmpty(nextMarker)) { urlToUse = $"{url}&marker={nextMarker}"; } var createRequest = AzureHelper.RequestMessage("GET", urlToUse, AccountName, AccountKey); XmlDocument responseFile; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); discoveredContainers.AddRange(from x in elemList.Cast <XmlNode>() where x.InnerText.Contains(Prefix) select new TaskItem(x.InnerText)); nextMarker = responseFile.GetElementsByTagName("NextMarker").Cast <XmlNode>().FirstOrDefault()?.InnerText; } }while (!string.IsNullOrEmpty(nextMarker)); } catch (Exception e) { Log.LogErrorFromException(e, true); } } ContainerNames = discoveredContainers.ToArray(); if (ContainerNames.Length == 0) { Log.LogWarning("No containers were found."); } else { Log.LogMessage("Found {0} containers.", ContainerNames.Length); } return(!Log.HasLoggedErrors); }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage(MessageImportance.Normal, "List of Azure containers in storage account '{0}'.", AccountName); string url = string.Format("https://{0}.blob.core.windows.net/?comp=list", AccountName); Log.LogMessage(MessageImportance.Low, "Sending request to list containers in account '{0}'.", AccountName); using (HttpClient client = new HttpClient()) { try { Func <HttpRequestMessage> createRequest = () => { DateTime dateTime = DateTime.UtcNow; var request = new HttpRequestMessage(HttpMethod.Get, url); request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); return(request); }; // TODO: This task has a bug, it needs to continue when there are > 5000 containers in a storage acccount. // Fix is something like the one made to DownloadFromAzure, but not pressing since it looks like GetLatestContainerNameFromAzure is rarely / not used. XmlDocument responseFile; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); ContainerNames = (from x in elemList.Cast <XmlNode>() where x.InnerText.Contains(Prefix) select new TaskItem(x.InnerText)).ToArray(); if (ContainerNames.Length == 0) { Log.LogWarning("No containers were found."); } else { Log.LogMessage("Found {0} containers.", ContainerNames.Length); } } } catch (Exception e) { Log.LogErrorFromException(e, true); } } return(!Log.HasLoggedErrors); }
public async Task <bool> ExecuteAsync() { Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.", ContainerName, AccountName, DownloadDirectory); List <string> blobsNames = null; string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); Log.LogMessage(MessageImportance.Low, "Sending request to list blobsNames for container '{0}'.", ContainerName); using (HttpClient client = new HttpClient()) { try { Func <HttpRequestMessage> createRequest = () => { DateTime dateTime = DateTime.UtcNow; var request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs); request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); return(request); }; XmlDocument responseFile; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames = elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList(); if (blobsNames.Count == 0) { Log.LogWarning("No blobs were found."); } } // track the number of blobs that fail to download int failureCount = 0; foreach (string blob in blobsNames) { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob); string filename = Path.Combine(DownloadDirectory, blob); string blobDirectory = blob.Substring(0, blob.LastIndexOf("/")); string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } createRequest = () => { DateTime dateTime = DateTime.UtcNow; var request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob); request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); return(request); }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); ++failureCount; } } } // if no blobs failed to download the task succeeded return(failureCount == 0); } catch (Exception e) { Log.LogErrorFromException(e, true); return(false); } } }
public async Task <bool> ExecuteAsync() { Log.LogMessage(MessageImportance.Normal, "Downloading container {0} from storage account '{1}'.", ContainerName, AccountName); DateTime dateTime = DateTime.UtcNow; List <string> blobsNames = null; string urlListBlobs = string.Format("https://{0}.blob.core.windows.net/{1}?restype=container&comp=list", AccountName, ContainerName); Log.LogMessage(MessageImportance.Normal, "Sending request to list blobsNames for container '{0}'.", ContainerName); using (HttpClient client = new HttpClient()) { using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlListBlobs)) { try { request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); XmlDocument responseFile; using (HttpResponseMessage response = await client.SendAsync(request)) { responseFile = new XmlDocument(); responseFile.LoadXml(await response.Content.ReadAsStringAsync()); XmlNodeList elemList = responseFile.GetElementsByTagName("Name"); blobsNames = elemList.Cast <XmlNode>() .Select(x => x.InnerText) .ToList(); } } catch (Exception e) { Log.LogError("Failed to retrieve information.\n" + e.Message); return(false); } } DownloadDirectory = DownloadDirectory ?? Directory.GetCurrentDirectory(); foreach (string blob in blobsNames) { Log.LogMessage(MessageImportance.Normal, "Downloading BLOB - {0}", blob); string urlGetBlob = string.Format("https://{0}.blob.core.windows.net/{1}/{2}", AccountName, ContainerName, blob); string filename = Path.Combine(DownloadDirectory, blob); string blobDirectory = blob.Substring(0, blob.LastIndexOf("/")); string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, urlGetBlob)) { request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "GET", dateTime, request)); using (HttpResponseMessage response = await client.SendAsync(request)) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } } } } return(true); }
public async Task <bool> ExecuteAsync() { Log.LogMessage( MessageImportance.High, "Creating container named '{0}' in storage account {1}.", ContainerName, AccountName); string url = string.Format( "https://{0}.blob.core.windows.net/{1}?restype=container", AccountName, ContainerName); StorageUri = string.Format( "https://{0}.blob.core.windows.net/{1}/", AccountName, ContainerName); Log.LogMessage(MessageImportance.Low, "Sending request to create Container"); using (HttpClient client = new HttpClient()) { Func <HttpRequestMessage> createRequest = () => { DateTime dt = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Put, url); req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt, req)); byte[] bytestoWrite = new byte[0]; int bytesToWriteLength = 0; Stream postStream = new MemoryStream(); postStream.Write(bytestoWrite, 0, bytesToWriteLength); req.Content = new StreamContent(postStream); return(req); }; Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) => { // the Conflict status (409) indicates that the container already exists, so // if FailIfExists is set to false and we get a 409 don't fail the task. return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict)); }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate)) { try { Log.LogMessage( MessageImportance.Low, "Received response to create Container {0}: Status Code: {1} {2}", ContainerName, response.StatusCode, response.Content.ToString()); // specifying zero is valid, it means "I don't want a token" if (ReadOnlyTokenDaysValid > 0) { ReadOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Read, ReadOnlyTokenDaysValid); } // specifying zero is valid, it means "I don't want a token" if (WriteOnlyTokenDaysValid > 0) { WriteOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Write, WriteOnlyTokenDaysValid); } } catch (Exception e) { Log.LogErrorFromException(e, true); return(false); } } } return(true); }
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle) { await clientThrottle.WaitAsync(); string filename = string.Empty; try { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob)); if (!DownloadFlatFiles) { int dirIndex = blob.LastIndexOf("/"); string blobDirectory = string.Empty; string blobFilename = string.Empty; if (dirIndex == -1) { blobFilename = blob; } else { blobDirectory = blob.Substring(0, dirIndex); blobFilename = blob.Substring(dirIndex + 1); // Trim blob name prefix (directory part) from download to blob directory if (BlobNamePrefix != null) { if (BlobNamePrefix.Length > dirIndex) { BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex); } blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length); } } string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } filename = Path.Combine(downloadBlobDirectory, blobFilename); } var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. if (!string.IsNullOrEmpty(Path.GetFileName(filename))) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); } } } catch (PathTooLongException) { Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}"); } catch (Exception ex) { Log.LogError(ex.ToString()); } finally { clientThrottle.Release(); } }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage( MessageImportance.High, "Creating container named '{0}' in storage account {1}.", ContainerName, AccountName); string url = string.Format( "https://{0}.blob.core.windows.net/{1}?restype=container", AccountName, ContainerName); StorageUri = string.Format( "https://{0}.blob.core.windows.net/{1}/", AccountName, ContainerName); Log.LogMessage(MessageImportance.Low, "Sending request to create Container"); using (HttpClient client = new HttpClient()) { var createRequest = AzureHelper.RequestMessage("PUT", url, AccountName, AccountKey); Func <HttpResponseMessage, bool> validate = (HttpResponseMessage response) => { // the Conflict status (409) indicates that the container already exists, so // if FailIfExists is set to false and we get a 409 don't fail the task. return(response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict)); }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate)) { try { Log.LogMessage( MessageImportance.Low, "Received response to create Container {0}: Status Code: {1} {2}", ContainerName, response.StatusCode, response.Content.ToString()); // specifying zero is valid, it means "I don't want a token" if (ReadOnlyTokenDaysValid > 0) { ReadOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Read, ReadOnlyTokenDaysValid); } // specifying zero is valid, it means "I don't want a token" if (WriteOnlyTokenDaysValid > 0) { WriteOnlyToken = AzureHelper.CreateContainerSasToken( AccountName, ContainerName, AccountKey, AzureHelper.SasAccessType.Write, WriteOnlyTokenDaysValid); } } catch (Exception e) { Log.LogErrorFromException(e, true); } } } return(!Log.HasLoggedErrors); }