public void Reload() { Exception = null; Index = null; IndexStats = Array.Empty <RepoStats>(); Task.Run(async() => { try { _logger.LogInformation("Loading index"); var azureConnectionString = _configuration["AzureStorageConnectionString"]; var indexName = "index.cicache"; var blobClient = new BlobClient(azureConnectionString, "index", indexName); if (!_environment.IsDevelopment()) { using var memoryStream = new MemoryStream(); ProgressText = $"Downloading index..."; await blobClient.DownloadToAsync(memoryStream); memoryStream.Position = 0; ProgressText = "Loading index..."; Index = await CrawledIndex.LoadAsync(memoryStream); } else { var binDirectory = Path.GetDirectoryName(GetType().Assembly.Location); var indexFile = Path.Combine(binDirectory, indexName); if (!File.Exists(indexFile)) { ProgressText = $"Downloading index..."; await blobClient.DownloadToAsync(indexFile); } ProgressText = "Loading index..."; Index = await CrawledIndex.LoadAsync(indexFile); } Exception = null; ProgressText = null; } catch (Exception ex) when(!Debugger.IsAttached) { _logger.LogError(ex, "Error during index loading"); Exception = ex; Index = new CrawledIndex(); ProgressText = string.Empty; } }); }
public async Task MaximumExecutionTime() { string connectionString = this.ConnectionString; string data = "hello world"; //setup blob string containerName = Randomize("sample-container"); string blobName = Randomize("sample-file"); var containerClient = new BlobContainerClient(ConnectionString, containerName); try { await containerClient.CreateIfNotExistsAsync(); await containerClient.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(data)); #region Snippet:SampleSnippetsBlobMigration_MaximumExecutionTime BlobClient blobClient = containerClient.GetBlobClient(blobName); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); Stream targetStream = new MemoryStream(); await blobClient.DownloadToAsync(targetStream, cancellationTokenSource.Token); #endregion } finally { await containerClient.DeleteIfExistsAsync(); } Assert.Pass(); }
public async Task DownloadBlob() { string data = "hello world"; //setup blob string containerName = Randomize("sample-container"); string blobName = Randomize("sample-file"); var containerClient = new BlobContainerClient(ConnectionString, containerName); string downloadFilePath = this.CreateTempPath(); try { containerClient.Create(); containerClient.GetBlobClient(blobName).Upload(new MemoryStream(Encoding.UTF8.GetBytes(data))); #region Snippet:SampleSnippetsBlobMigration_DownloadBlob BlobClient blobClient = containerClient.GetBlobClient(blobName); await blobClient.DownloadToAsync(downloadFilePath); #endregion FileStream fs = File.OpenRead(downloadFilePath); string downloadedData = await new StreamReader(fs).ReadToEndAsync(); fs.Close(); Assert.AreEqual(data, downloadedData); } finally { await containerClient.DeleteIfExistsAsync(); } }
public static async Task <byte[]> DownloadFileFromSAS(string blobSas) { var blobClientOptions = new BlobClientOptions() { Retry = { MaxRetries = 3, Delay = TimeSpan.FromSeconds(5), Mode = Azure.Core.RetryMode.Fixed } }; var blob = new BlobClient(new Uri(blobSas), blobClientOptions); byte[] data; using (var memoryStream = new MemoryStream()) { await blob.DownloadToAsync(memoryStream).ConfigureAwait(false); data = new byte[memoryStream.Length]; memoryStream.Position = 0; memoryStream.Read(data, 0, data.Length); } return(data); }
public void Dump() { // Get the blob reference BlobContainerClient blobContainer = new BlobContainerClient(StorageConnectionString, EventHubsCaptureAvroBlobContainer); BlobClient blob = blobContainer.GetBlobClient(EventHubsCaptureAvroBlobName); // Download the content to a memory stream using (Stream blobStream = new MemoryStream()) { blob.DownloadToAsync(blobStream); using (var dataTable = GetWindTurbineMetricsTable()) { // Parse the Avro File using (var avroReader = DataFileReader <GenericRecord> .OpenReader(blobStream)) { while (avroReader.HasNext()) { GenericRecord r = avroReader.Next(); byte[] body = (byte[])r["Body"]; var windTurbineMeasure = DeserializeToWindTurbineMeasure(body); // Add the row to in memory table AddWindTurbineMetricToTable(dataTable, windTurbineMeasure); } } if (dataTable.Rows.Count > 0) { BatchInsert(dataTable); } } } }
private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath, CancellationToken ct) { ct.ThrowIfCancellationRequested(); if (workItem.TryGetMetadata("DownloadFilesFromResults", out string files)) { string workItemName = workItem.GetMetadata("Identity"); string[] filesToDownload = files.Split(';'); DirectoryInfo destinationDir = Directory.CreateDirectory(Path.Combine(directoryPath, workItemName)); foreach (var file in filesToDownload) { try { string destinationFile = Path.Combine(destinationDir.FullName, file); Log.LogMessage(MessageImportance.Normal, $"Downloading {file} => {destinationFile}..."); // Currently the blob storage includes the retry iteration, however there is no good way // to get the "best" iteration number to download the files from. For now, use always iteration // 1 until helix provides an API to get result files from the "good" iteration run. // https://github.com/dotnet/core-eng/issues/13983 var uri = new Uri($"{ResultsContainer}{workItemName}/1/{file}"); BlobClient blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new BlobClient(uri) : new BlobClient(uri, new AzureSasCredential(ResultsContainerReadSAS)); await blob.DownloadToAsync(destinationFile); } catch (RequestFailedException rfe) { Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {rfe.Message}"); } } } ; return; }
private async Task WriteRebuiltFile(string fileRebuildSas) { BlobClient rebuiltFileBlobClient = new BlobClient(new Uri(fileRebuildSas)); using FileStream downloadFileStream = File.OpenWrite(_appConfiguration.OutputFilepath); await rebuiltFileBlobClient.DownloadToAsync(downloadFileStream); }
private static async Task <string> GetObjectAsync(BlobContainerClient client, string fileName, CancellationToken token) { try { BlobClient blobClient = client.GetBlobClient(fileName); if (await blobClient.ExistsAsync()) { using (System.IO.MemoryStream ms = new System.IO.MemoryStream()) { await blobClient.DownloadToAsync(ms, token); ms.Position = 0; using (System.IO.StreamReader sr = new System.IO.StreamReader(ms)) return(await sr.ReadToEndAsync()); } } } catch (Exception ex) { throw; } return(null); }
public async Task RetryPolicy() { string connectionString = this.ConnectionString; string data = "hello world"; //setup blob string containerName = Randomize("sample-container"); string blobName = Randomize("sample-file"); var containerClient = new BlobContainerClient(ConnectionString, containerName); await containerClient.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(data)); #region Snippet:SampleSnippetsBlobMigration_RetryPolicy BlobClientOptions blobClientOptions = new BlobClientOptions(); blobClientOptions.Retry.Mode = RetryMode.Exponential; blobClientOptions.Retry.Delay = TimeSpan.FromSeconds(10); blobClientOptions.Retry.MaxRetries = 6; BlobServiceClient service = new BlobServiceClient(connectionString, blobClientOptions); BlobClient blobClient = service.GetBlobContainerClient(containerName).GetBlobClient(blobName); Stream targetStream = new MemoryStream(); await blobClient.DownloadToAsync(targetStream); #endregion Assert.Pass(); }
private static async Task DownloadBlob(string connString, string blobName) { // Download the blob to a local file, using the reference created earlier. // Append the string "_DOWNLOADED" before the .txt extension so that you // can see both files in MyDocuments. string destinationFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "BlobDemo_DOWNLOADED.txt"); Console.WriteLine("Downloading blob to {0}", destinationFile); try { BlobContainerClient containerClient = new BlobContainerClient(connString, "my-container"); await containerClient.CreateIfNotExistsAsync(); try { BlobClient blobClient = containerClient.GetBlobClient(blobName); await blobClient.DownloadToAsync(destinationFile); } catch (Exception e) { Console.WriteLine("Blob Exception: " + e.Message); } } catch (Exception e) { Console.WriteLine("OH SNAP!: " + e.Message); } }
public async Task <IActionResult> GetAsync(string userName, string fileId) { try { BlobClient blobClient = this.uploadsContainer.GetBlobClient($"{userName}/{fileId}"); BlobProperties props = await blobClient.GetPropertiesAsync().ConfigureAwait(false); MemoryStream blobStream = new MemoryStream(); var file = await blobClient.DownloadToAsync(blobStream).ConfigureAwait(false); blobStream.Position = 0; return(File(blobStream, props.ContentType)); } catch (Azure.RequestFailedException ex) { logger.LogError($"Received Exception {ex}"); if (ex.Status == (int)HttpStatusCode.NotFound) { return(NotFound(ex)); } return(StatusCode(501)); } catch (Exception ex) { logger.LogError($"File not present for the resource {ex}"); return(StatusCode(500, ex)); } }
public async Task DownloadAsync() { // Create a temporary Lorem Ipsum file on disk that we can upload string originalPath = CreateTempFile(SampleFileContent); // Get a temporary path on disk where we can download the file string downloadPath = CreateTempPath(); // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobContainerClient container = new BlobContainerClient(connectionString, Randomize("sample-container")); await container.CreateAsync(); try { // Get a reference to a blob named "sample-file" BlobClient blob = container.GetBlobClient(Randomize("sample-file")); // First upload something the blob so we have something to download await blob.UploadAsync(File.OpenRead(originalPath)); // Download the blob's contents and save it to a file await blob.DownloadToAsync(downloadPath); // Verify the contents Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); } finally { // Clean up after the test when we're finished await container.DeleteAsync(); } }
public async Task Execute(IReadOnlyCollection <string> filenames) { var containerName = "fileupload"; using (var targetStream = File.Create(Path.Combine(Config.DownloadFilesAbsolutePath, "files.zip"))) { using (var zipOutputStream = new ZipOutputStream(targetStream)) { foreach (var filename in filenames) { var entry = new ZipEntry(filename) { DateTime = DateTime.UtcNow, }; zipOutputStream.PutNextEntry(entry); var blobClient = new BlobClient(connectionString: Config.AzureStorageConnectionString, blobContainerName: containerName, blobName: filename); await blobClient.DownloadToAsync(zipOutputStream); await targetStream.FlushAsync(); } zipOutputStream.Finish(); zipOutputStream.Close(); } } }
public async Task <MemoryStream> GetFile(string filename) { using var ms = new MemoryStream(); BlobClient blobClient = _blobContainerClient.GetBlobClient(filename); await blobClient.DownloadToAsync(ms); return(ms); }
public async Task <ActionResult> OnGetDownloadFileAsync(string id) { BlobClient client = _blobContainerA.GetBlobClient(id); string filePath = Path.GetTempFileName(); await client.DownloadToAsync(filePath); return(File(System.IO.File.ReadAllBytes(filePath), "application/octet-stream", id)); }
/// <summary> /// Return a bool indicating whether a local file's content is the same as /// the content of a given blob. /// /// If the blob has the ContentHash property set, the comparison is performed using /// that (MD5 hash). All recently-uploaded blobs or those uploaded by these libraries /// should; some blob clients older than ~2012 may upload without the property set. /// /// When the ContentHash property is unset, a byte-by-byte comparison is performed. /// </summary> public async Task <bool> IsFileIdenticalToBlobAsync(string localFileFullPath, BlobClient blob) { BlobProperties properties = await blob.GetPropertiesAsync(); if (properties.ContentHash != null) { var localMD5 = CalculateMD5(localFileFullPath); var blobMD5 = Convert.ToBase64String(properties.ContentHash); return(blobMD5.Equals(localMD5, StringComparison.OrdinalIgnoreCase)); } else { int bytesPerMegabyte = 1 * 1024 * 1024; if (properties.ContentLength < bytesPerMegabyte) { byte[] existingBytes = new byte[properties.ContentLength]; byte[] localBytes = File.ReadAllBytes(localFileFullPath); using (MemoryStream stream = new MemoryStream(existingBytes, true)) { await blob.DownloadToAsync(stream).ConfigureAwait(false); } return(localBytes.SequenceEqual(existingBytes)); } else { using (Stream localFileStream = File.OpenRead(localFileFullPath)) { byte[] localBuffer = new byte[bytesPerMegabyte]; byte[] remoteBuffer = new byte[bytesPerMegabyte]; int bytesLocalFile = 0; do { long start = localFileStream.Position; int localBytesRead = await localFileStream.ReadAsync(localBuffer, 0, bytesPerMegabyte); HttpRange range = new HttpRange(start, localBytesRead); BlobDownloadInfo download = await blob.DownloadAsync(range).ConfigureAwait(false); if (download.ContentLength != localBytesRead) { return(false); } using (MemoryStream stream = new MemoryStream(remoteBuffer, true)) { await download.Content.CopyToAsync(stream).ConfigureAwait(false); } if (!remoteBuffer.SequenceEqual(localBuffer)) { return(false); } }while (bytesLocalFile > 0); } return(true); } } }
private static async Task <string> GetStringFromBlobAsync(BlobContainerClient storageContainer, string ismManifestFileName) { BlobClient blobClient = storageContainer.GetBlobClient(ismManifestFileName); using var ms = new MemoryStream(); await blobClient.DownloadToAsync(ms); return(System.Text.Encoding.UTF8.GetString(ms.ToArray())); }
public static async Task <string> LoadAsText(this BlobClient blobClient) { using var memoryStream = new MemoryStream(); await blobClient.DownloadToAsync(memoryStream); var text = memoryStream.ToArray().ToStringFromUtf8(); return(text); }
public async Task <Stream> DownloadFile(string userId) { var stream = new MemoryStream(); BlobClient blobClient = containerClient.GetBlobClient($"{userId}.zip"); await blobClient.DownloadToAsync(stream); return(stream); }
private static async Task DownloadAsync(string remoteFileName) { // new a BlobClient every time seems stupid... var client = new BlobClient(Options.ConnectionString, Options.ContainerName, remoteFileName); var newFilePath = Path.Combine(Options.LocalFolderPath, remoteFileName); await client.DownloadToAsync(newFilePath); WriteMessage($"[{DateTime.Now}] downloaded {remoteFileName}."); }
public async Task <bool> DownloadAsync(string path, Stream destination, CancellationToken cancellationToken = default) { var container = GetContainerName(path); var blobPath = GetPathWithoutContainer(path); var client = new BlobClient(_connectionString, container, blobPath); var res = await client.DownloadToAsync(destination, cancellationToken); return(res.Status > 199 && res.Status < 300); }
public async Task <byte[]> ReadAsync(IFileEntry fileEntry, CancellationToken cancellationToken = default) { BlobClient blob = _container.GetBlobClient(GetBlobName(fileEntry)); using var stream = new MemoryStream(); await blob.DownloadToAsync(stream, cancellationToken); return(stream.ToArray()); }
public async Task <Response> DownloadPrivateBlob(string connectionString, string containerName, string blobName, Stream downloadToPath) { BlobContainerClient container = new BlobContainerClient(connectionString, containerName); // Get a reference to a blob named "sample-file" in a container named "sample-container" BlobClient blob = container.GetBlobClient(blobName); // Upload local file return(await blob.DownloadToAsync(downloadToPath)); }
public async Task <Stream> DownloadBlobAsStreamAsync(string blobName) { BlobClient blobClient = _blobContainerClient.GetBlobClient(blobName); var stream = new MemoryStream(); await blobClient.DownloadToAsync(stream); return(stream); }
public async Task DownloadModelAsync(Guid runId, Stream destination) { BlobClient blobClient = this.modelRepositoryClient.GetBlobClient(this.modelPathGenerator.GetModelName(runId)); if (!await blobClient.ExistsAsync()) { throw new FileNotFoundException($"No model exists for Run ID {runId}"); } await blobClient.DownloadToAsync(destination); }
private async Task <Stream> DownloadToStreamAsync(string org, string fileName) { BlobClient blockBlob = await CreateBlobClient(org, fileName); var memoryStream = new MemoryStream(); await blockBlob.DownloadToAsync(memoryStream); memoryStream.Position = 0; return(memoryStream); }
public async Task <Stream> DownloadAsync(string containerName, string fileName) { this.logger.Info($"Call: {nameof(DownloadAsync)}('{containerName}', '{fileName}')"); BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName); BlobClient blobClient = containerClient.GetBlobClient(fileName); MemoryStream ms = new MemoryStream(); await blobClient.DownloadToAsync(ms); return(ms); }
public async Task DownloadBlobAsync(AvailableBlobEvent availableBlobEvent, FileSystemInfo blobLocalStorage, CancellationToken cancellationToken) { var sasUri = new UriBuilder(availableBlobEvent.Uri) { Query = FindToken(availableBlobEvent.Uri) }; var client = new BlobClient(sasUri.Uri); await client.DownloadToAsync(blobLocalStorage.FullName, cancellationToken); }
public async Task <string> GetFilesByRef(string blobRef) { BlobContainerClient container = await GetBlobContainerClient(); BlobClient blob = container.GetBlobClient(blobRef); var path = Path.GetTempPath() + Path.GetRandomFileName(); await blob.DownloadToAsync(path); return(path); }
/// <summary> /// Downloads the model file from disk into the provided stream. /// </summary> /// <param name="runId">Run ID to download model for</param> /// <param name="destination">Destination stream to write model into</param> /// <returns>Task with result of download operation</returns> public async Task DownloadModelAsync(Guid runId, Stream destination) { await this.blobContainerClient.CreateIfNotExistsAsync(); BlobClient blobClient = this.blobContainerClient.GetBlobClient($"{runId}{fileExtension}"); if (!await blobClient.ExistsAsync()) { throw new FileNotFoundException($"No model exists for Run ID {runId}"); } await blobClient.DownloadToAsync(destination); }