public override bool Execute() { ParseConnectionString(); string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, BlobName); using (HttpClient client = new HttpClient()) { try { Tuple <string, string> headerBlobType = new Tuple <string, string>("x-ms-blob-type", "BlockBlob"); List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >() { headerBlobType }; if (!string.IsNullOrEmpty(ContentType)) { additionalHeaders.Add(new Tuple <string, string>(AzureHelper.ContentTypeString, ContentType)); } var request = AzureHelper.RequestMessage("PUT", blobUrl, AccountName, AccountKey, additionalHeaders, Content); AzureHelper.RequestWithRetry(Log, client, request).GetAwaiter().GetResult(); } catch (Exception e) { Log.LogErrorFromException(e, true); } } return(!Log.HasLoggedErrors); }
public AzureBlobLease(string accountName, string accountKey, string connectionString, string containerName, string blobName, Microsoft.Build.Utilities.TaskLoggingHelper log, string maxWait = null, string delay = null) { _accountName = accountName; _accountKey = accountKey; _connectionString = connectionString; _containerName = containerName; _blobName = blobName; _maxWait = !string.IsNullOrWhiteSpace(maxWait) ? TimeSpan.Parse(maxWait) : TimeSpan.FromSeconds(s_MaxWaitDefault); _delay = !string.IsNullOrWhiteSpace(delay) ? TimeSpan.Parse(delay) : TimeSpan.FromMilliseconds(s_DelayDefault); _log = log; _leaseUrl = $"{AzureHelper.GetBlobRestUrl(_accountName, _containerName, _blobName)}?comp=lease"; }
public async Task <bool> FileEqualsExistingBlobAsync( string accountName, string accountKey, string containerName, string filePath, string destinationBlob, int uploadTimeout) { using (var client = new HttpClient { Timeout = TimeSpan.FromMinutes(uploadTimeout) }) { log.LogMessage( MessageImportance.Low, $"Downloading blob {destinationBlob} to check if identical."); string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( log, client, createRequest)) { if (!response.IsSuccessStatusCode) { throw new HttpRequestException( $"Failed to retrieve existing blob {destinationBlob}, " + $"status code {response.StatusCode}."); } byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); byte[] localBytes = File.ReadAllBytes(filePath); bool equal = localBytes.SequenceEqual(existingBytes); if (equal) { log.LogMessage( MessageImportance.Normal, "Item exists in blob storage, and is verified to be identical. " + $"File: '{filePath}' Blob: '{destinationBlob}'"); } return(equal); } } }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); if (Log.HasLoggedErrors) { return(false); } string sourceUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, SourceBlobName); string destinationUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, DestinationBlobName); using (HttpClient client = new HttpClient()) { try { Tuple <string, string> leaseAction = new Tuple <string, string>("x-ms-lease-action", "acquire"); Tuple <string, string> leaseDuration = new Tuple <string, string>("x-ms-lease-duration", "60" /* seconds */); Tuple <string, string> headerSource = new Tuple <string, string>("x-ms-copy-source", sourceUrl); List <Tuple <string, string> > additionalHeaders = new List <Tuple <string, string> >() { leaseAction, leaseDuration, headerSource }; var request = AzureHelper.RequestMessage("PUT", destinationUrl, AccountName, AccountKey, additionalHeaders); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, request)) { if (response.IsSuccessStatusCode) { return(true); } } } catch (Exception e) { Log.LogErrorFromException(e, true); } } return(false); }
public async Task <bool> ExecuteAsync() { ParseConnectionString(); // If the connection string AND AccountKey & AccountName are provided, error out. if (Log.HasLoggedErrors) { return(false); } Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.", ContainerName, AccountName, DownloadDirectory); try { List <string> blobNames = new List <string>(); if (BlobNames == null) { ListAzureBlobs listAzureBlobs = new ListAzureBlobs() { AccountName = AccountName, AccountKey = AccountKey, ContainerName = ContainerName, FilterBlobNames = BlobNamePrefix, BuildEngine = this.BuildEngine, HostObject = this.HostObject }; listAzureBlobs.Execute(); blobNames = listAzureBlobs.BlobNames.ToList(); } else { blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>(); if (BlobNamePrefix != null) { blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>(); } } // track the number of blobs that fail to download int failureCount = 0; using (HttpClient client = new HttpClient()) { foreach (string blob in blobNames) { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string urlGetBlob = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); int dirIndex = blob.LastIndexOf("/"); string blobDirectory = string.Empty; string blobFilename = string.Empty; if (dirIndex == -1) { blobFilename = blob; } else { blobDirectory = blob.Substring(0, dirIndex); blobFilename = blob.Substring(dirIndex + 1); // Trim blob name prefix (directory part) from download to blob directory if (BlobNamePrefix != null) { if (BlobNamePrefix.Length > dirIndex) { BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex); } blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length); } } string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } string filename = Path.Combine(downloadBlobDirectory, blobFilename); var createRequest = AzureHelper.RequestMessage("GET", urlGetBlob, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. if (!string.IsNullOrEmpty(Path.GetFileName(filename))) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); ++failureCount; } } } } Log.LogMessage($"{failureCount} errors seen downloading blobs."); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); }
private async Task DownloadItem(HttpClient client, CancellationToken ct, string blob, SemaphoreSlim clientThrottle) { await clientThrottle.WaitAsync(); string filename = string.Empty; try { Log.LogMessage(MessageImportance.Low, "Downloading BLOB - {0}", blob); string blobUrl = AzureHelper.GetBlobRestUrl(AccountName, ContainerName, blob); filename = Path.Combine(DownloadDirectory, Path.GetFileName(blob)); if (!DownloadFlatFiles) { int dirIndex = blob.LastIndexOf("/"); string blobDirectory = string.Empty; string blobFilename = string.Empty; if (dirIndex == -1) { blobFilename = blob; } else { blobDirectory = blob.Substring(0, dirIndex); blobFilename = blob.Substring(dirIndex + 1); // Trim blob name prefix (directory part) from download to blob directory if (BlobNamePrefix != null) { if (BlobNamePrefix.Length > dirIndex) { BlobNamePrefix = BlobNamePrefix.Substring(0, dirIndex); } blobDirectory = blobDirectory.Substring(BlobNamePrefix.Length); } } string downloadBlobDirectory = Path.Combine(DownloadDirectory, blobDirectory); if (!Directory.Exists(downloadBlobDirectory)) { Directory.CreateDirectory(downloadBlobDirectory); } filename = Path.Combine(downloadBlobDirectory, blobFilename); } var createRequest = AzureHelper.RequestMessage("GET", blobUrl, AccountName, AccountKey); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) { if (response.IsSuccessStatusCode) { // Blobs can be files but have the name of a directory. We'll skip those and log something weird happened. if (!string.IsNullOrEmpty(Path.GetFileName(filename))) { Stream responseStream = await response.Content.ReadAsStreamAsync(); using (FileStream sourceStream = File.Open(filename, FileMode.Create)) { responseStream.CopyTo(sourceStream); } } else { Log.LogWarning($"Unable to download blob '{blob}' as it has a directory-like name. This may cause problems if it was needed."); } } else { Log.LogError("Failed to retrieve blob {0}, the status code was {1}", blob, response.StatusCode); } } } catch (PathTooLongException) { Log.LogError($"Unable to download blob as it exceeds the maximum allowed path length. Path: {filename}. Length:{filename.Length}"); } catch (Exception ex) { Log.LogError(ex.ToString()); } finally { clientThrottle.Release(); } }