public void Reload()
{
Exception = null;
Index = null;
IndexStats = Array.Empty <RepoStats>();
Task.Run(async() =>
{
try
{
_logger.LogInformation("Loading index");
var azureConnectionString = _configuration["AzureStorageConnectionString"];
var indexName = "index.cicache";
var blobClient = new BlobClient(azureConnectionString, "index", indexName);
if (!_environment.IsDevelopment())
{
using var memoryStream = new MemoryStream();
ProgressText = $"Downloading index...";
await blobClient.DownloadToAsync(memoryStream);
memoryStream.Position = 0;
ProgressText = "Loading index...";
Index = await CrawledIndex.LoadAsync(memoryStream);
}
else
{
var binDirectory = Path.GetDirectoryName(GetType().Assembly.Location);
var indexFile = Path.Combine(binDirectory, indexName);
if (!File.Exists(indexFile))
{
ProgressText = $"Downloading index...";
await blobClient.DownloadToAsync(indexFile);
}
ProgressText = "Loading index...";
Index = await CrawledIndex.LoadAsync(indexFile);
}
Exception = null;
ProgressText = null;
}
catch (Exception ex) when(!Debugger.IsAttached)
{
_logger.LogError(ex, "Error during index loading");
Exception = ex;
Index = new CrawledIndex();
ProgressText = string.Empty;
}
});
}
public async Task MaximumExecutionTime()
{
string connectionString = this.ConnectionString;
string data = "hello world";
//setup blob
string containerName = Randomize("sample-container");
string blobName = Randomize("sample-file");
var containerClient = new BlobContainerClient(ConnectionString, containerName);
try
{
await containerClient.CreateIfNotExistsAsync();
await containerClient.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(data));
#region Snippet:SampleSnippetsBlobMigration_MaximumExecutionTime
BlobClient blobClient = containerClient.GetBlobClient(blobName);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(30));
Stream targetStream = new MemoryStream();
await blobClient.DownloadToAsync(targetStream, cancellationTokenSource.Token);
#endregion
}
finally
{
await containerClient.DeleteIfExistsAsync();
}
Assert.Pass();
}
public async Task DownloadBlob()
{
string data = "hello world";
//setup blob
string containerName = Randomize("sample-container");
string blobName = Randomize("sample-file");
var containerClient = new BlobContainerClient(ConnectionString, containerName);
string downloadFilePath = this.CreateTempPath();
try
{
containerClient.Create();
containerClient.GetBlobClient(blobName).Upload(new MemoryStream(Encoding.UTF8.GetBytes(data)));
#region Snippet:SampleSnippetsBlobMigration_DownloadBlob
BlobClient blobClient = containerClient.GetBlobClient(blobName);
await blobClient.DownloadToAsync(downloadFilePath);
#endregion
FileStream fs = File.OpenRead(downloadFilePath);
string downloadedData = await new StreamReader(fs).ReadToEndAsync();
fs.Close();
Assert.AreEqual(data, downloadedData);
}
finally
{
await containerClient.DeleteIfExistsAsync();
}
}
public static async Task <byte[]> DownloadFileFromSAS(string blobSas)
{
var blobClientOptions = new BlobClientOptions()
{
Retry =
{
MaxRetries = 3,
Delay = TimeSpan.FromSeconds(5),
Mode = Azure.Core.RetryMode.Fixed
}
};
var blob = new BlobClient(new Uri(blobSas), blobClientOptions);
byte[] data;
using (var memoryStream = new MemoryStream())
{
await blob.DownloadToAsync(memoryStream).ConfigureAwait(false);
data = new byte[memoryStream.Length];
memoryStream.Position = 0;
memoryStream.Read(data, 0, data.Length);
}
return(data);
}
public void Dump()
{
// Get the blob reference
BlobContainerClient blobContainer = new BlobContainerClient(StorageConnectionString, EventHubsCaptureAvroBlobContainer);
BlobClient blob = blobContainer.GetBlobClient(EventHubsCaptureAvroBlobName);
// Download the content to a memory stream
using (Stream blobStream = new MemoryStream())
{
blob.DownloadToAsync(blobStream);
using (var dataTable = GetWindTurbineMetricsTable())
{
// Parse the Avro File
using (var avroReader = DataFileReader <GenericRecord> .OpenReader(blobStream))
{
while (avroReader.HasNext())
{
GenericRecord r = avroReader.Next();
byte[] body = (byte[])r["Body"];
var windTurbineMeasure = DeserializeToWindTurbineMeasure(body);
// Add the row to in memory table
AddWindTurbineMetricToTable(dataTable, windTurbineMeasure);
}
}
if (dataTable.Rows.Count > 0)
{
BatchInsert(dataTable);
}
}
}
}
private async Task DownloadFilesForWorkItem(ITaskItem workItem, string directoryPath, CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
if (workItem.TryGetMetadata("DownloadFilesFromResults", out string files))
{
string workItemName = workItem.GetMetadata("Identity");
string[] filesToDownload = files.Split(';');
DirectoryInfo destinationDir = Directory.CreateDirectory(Path.Combine(directoryPath, workItemName));
foreach (var file in filesToDownload)
{
try
{
string destinationFile = Path.Combine(destinationDir.FullName, file);
Log.LogMessage(MessageImportance.Normal, $"Downloading {file} => {destinationFile}...");
// Currently the blob storage includes the retry iteration, however there is no good way
// to get the "best" iteration number to download the files from. For now, use always iteration
// 1 until helix provides an API to get result files from the "good" iteration run.
// https://github.com/dotnet/core-eng/issues/13983
var uri = new Uri($"{ResultsContainer}{workItemName}/1/{file}");
BlobClient blob = string.IsNullOrEmpty(ResultsContainerReadSAS) ? new BlobClient(uri) : new BlobClient(uri, new AzureSasCredential(ResultsContainerReadSAS));
await blob.DownloadToAsync(destinationFile);
}
catch (RequestFailedException rfe)
{
Log.LogWarning($"Failed to download {workItemName}/1/{file} blob from results container: {rfe.Message}");
}
}
}
;
return;
}
private async Task WriteRebuiltFile(string fileRebuildSas)
{
BlobClient rebuiltFileBlobClient = new BlobClient(new Uri(fileRebuildSas));
using FileStream downloadFileStream = File.OpenWrite(_appConfiguration.OutputFilepath);
await rebuiltFileBlobClient.DownloadToAsync(downloadFileStream);
}
private static async Task <string> GetObjectAsync(BlobContainerClient client, string fileName, CancellationToken token)
{
try
{
BlobClient blobClient = client.GetBlobClient(fileName);
if (await blobClient.ExistsAsync())
{
using (System.IO.MemoryStream ms = new System.IO.MemoryStream())
{
await blobClient.DownloadToAsync(ms, token);
ms.Position = 0;
using (System.IO.StreamReader sr = new System.IO.StreamReader(ms))
return(await sr.ReadToEndAsync());
}
}
}
catch (Exception ex)
{
throw;
}
return(null);
}
public async Task RetryPolicy()
{
string connectionString = this.ConnectionString;
string data = "hello world";
//setup blob
string containerName = Randomize("sample-container");
string blobName = Randomize("sample-file");
var containerClient = new BlobContainerClient(ConnectionString, containerName);
await containerClient.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(data));
#region Snippet:SampleSnippetsBlobMigration_RetryPolicy
BlobClientOptions blobClientOptions = new BlobClientOptions();
blobClientOptions.Retry.Mode = RetryMode.Exponential;
blobClientOptions.Retry.Delay = TimeSpan.FromSeconds(10);
blobClientOptions.Retry.MaxRetries = 6;
BlobServiceClient service = new BlobServiceClient(connectionString, blobClientOptions);
BlobClient blobClient = service.GetBlobContainerClient(containerName).GetBlobClient(blobName);
Stream targetStream = new MemoryStream();
await blobClient.DownloadToAsync(targetStream);
#endregion
Assert.Pass();
}
private static async Task DownloadBlob(string connString, string blobName)
{
// Download the blob to a local file, using the reference created earlier.
// Append the string "_DOWNLOADED" before the .txt extension so that you
// can see both files in MyDocuments.
string destinationFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "BlobDemo_DOWNLOADED.txt");
Console.WriteLine("Downloading blob to {0}", destinationFile);
try
{
BlobContainerClient containerClient = new BlobContainerClient(connString, "my-container");
await containerClient.CreateIfNotExistsAsync();
try
{
BlobClient blobClient = containerClient.GetBlobClient(blobName);
await blobClient.DownloadToAsync(destinationFile);
}
catch (Exception e)
{
Console.WriteLine("Blob Exception: " + e.Message);
}
}
catch (Exception e)
{
Console.WriteLine("OH SNAP!: " + e.Message);
}
}
public async Task <IActionResult> GetAsync(string userName, string fileId)
{
try
{
BlobClient blobClient = this.uploadsContainer.GetBlobClient($"{userName}/{fileId}");
BlobProperties props = await blobClient.GetPropertiesAsync().ConfigureAwait(false);
MemoryStream blobStream = new MemoryStream();
var file = await blobClient.DownloadToAsync(blobStream).ConfigureAwait(false);
blobStream.Position = 0;
return(File(blobStream, props.ContentType));
}
catch (Azure.RequestFailedException ex)
{
logger.LogError($"Received Exception {ex}");
if (ex.Status == (int)HttpStatusCode.NotFound)
{
return(NotFound(ex));
}
return(StatusCode(501));
}
catch (Exception ex)
{
logger.LogError($"File not present for the resource {ex}");
return(StatusCode(500, ex));
}
}
public async Task DownloadAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobContainerClient container = new BlobContainerClient(connectionString, Randomize("sample-container"));
await container.CreateAsync();
try
{
// Get a reference to a blob named "sample-file"
BlobClient blob = container.GetBlobClient(Randomize("sample-file"));
// First upload something the blob so we have something to download
await blob.UploadAsync(File.OpenRead(originalPath));
// Download the blob's contents and save it to a file
await blob.DownloadToAsync(downloadPath);
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
await container.DeleteAsync();
}
}
public async Task Execute(IReadOnlyCollection <string> filenames)
{
var containerName = "fileupload";
using (var targetStream = File.Create(Path.Combine(Config.DownloadFilesAbsolutePath, "files.zip")))
{
using (var zipOutputStream = new ZipOutputStream(targetStream))
{
foreach (var filename in filenames)
{
var entry = new ZipEntry(filename)
{
DateTime = DateTime.UtcNow,
};
zipOutputStream.PutNextEntry(entry);
var blobClient = new BlobClient(connectionString: Config.AzureStorageConnectionString, blobContainerName: containerName, blobName: filename);
await blobClient.DownloadToAsync(zipOutputStream);
await targetStream.FlushAsync();
}
zipOutputStream.Finish();
zipOutputStream.Close();
}
}
}
public async Task <MemoryStream> GetFile(string filename)
{
using var ms = new MemoryStream();
BlobClient blobClient = _blobContainerClient.GetBlobClient(filename);
await blobClient.DownloadToAsync(ms);
return(ms);
}
public async Task <ActionResult> OnGetDownloadFileAsync(string id)
{
BlobClient client = _blobContainerA.GetBlobClient(id);
string filePath = Path.GetTempFileName();
await client.DownloadToAsync(filePath);
return(File(System.IO.File.ReadAllBytes(filePath), "application/octet-stream", id));
}
/// <summary>
/// Return a bool indicating whether a local file's content is the same as
/// the content of a given blob.
///
/// If the blob has the ContentHash property set, the comparison is performed using
/// that (MD5 hash). All recently-uploaded blobs or those uploaded by these libraries
/// should; some blob clients older than ~2012 may upload without the property set.
///
/// When the ContentHash property is unset, a byte-by-byte comparison is performed.
/// </summary>
public async Task <bool> IsFileIdenticalToBlobAsync(string localFileFullPath, BlobClient blob)
{
BlobProperties properties = await blob.GetPropertiesAsync();
if (properties.ContentHash != null)
{
var localMD5 = CalculateMD5(localFileFullPath);
var blobMD5 = Convert.ToBase64String(properties.ContentHash);
return(blobMD5.Equals(localMD5, StringComparison.OrdinalIgnoreCase));
}
else
{
int bytesPerMegabyte = 1 * 1024 * 1024;
if (properties.ContentLength < bytesPerMegabyte)
{
byte[] existingBytes = new byte[properties.ContentLength];
byte[] localBytes = File.ReadAllBytes(localFileFullPath);
using (MemoryStream stream = new MemoryStream(existingBytes, true))
{
await blob.DownloadToAsync(stream).ConfigureAwait(false);
}
return(localBytes.SequenceEqual(existingBytes));
}
else
{
using (Stream localFileStream = File.OpenRead(localFileFullPath))
{
byte[] localBuffer = new byte[bytesPerMegabyte];
byte[] remoteBuffer = new byte[bytesPerMegabyte];
int bytesLocalFile = 0;
do
{
long start = localFileStream.Position;
int localBytesRead = await localFileStream.ReadAsync(localBuffer, 0, bytesPerMegabyte);
HttpRange range = new HttpRange(start, localBytesRead);
BlobDownloadInfo download = await blob.DownloadAsync(range).ConfigureAwait(false);
if (download.ContentLength != localBytesRead)
{
return(false);
}
using (MemoryStream stream = new MemoryStream(remoteBuffer, true))
{
await download.Content.CopyToAsync(stream).ConfigureAwait(false);
}
if (!remoteBuffer.SequenceEqual(localBuffer))
{
return(false);
}
}while (bytesLocalFile > 0);
}
return(true);
}
}
}
private static async Task <string> GetStringFromBlobAsync(BlobContainerClient storageContainer, string ismManifestFileName)
{
BlobClient blobClient = storageContainer.GetBlobClient(ismManifestFileName);
using var ms = new MemoryStream();
await blobClient.DownloadToAsync(ms);
return(System.Text.Encoding.UTF8.GetString(ms.ToArray()));
}
public static async Task <string> LoadAsText(this BlobClient blobClient)
{
using var memoryStream = new MemoryStream();
await blobClient.DownloadToAsync(memoryStream);
var text = memoryStream.ToArray().ToStringFromUtf8();
return(text);
}
public async Task <Stream> DownloadFile(string userId)
{
var stream = new MemoryStream();
BlobClient blobClient = containerClient.GetBlobClient($"{userId}.zip");
await blobClient.DownloadToAsync(stream);
return(stream);
}
private static async Task DownloadAsync(string remoteFileName)
{
// new a BlobClient every time seems stupid...
var client = new BlobClient(Options.ConnectionString, Options.ContainerName, remoteFileName);
var newFilePath = Path.Combine(Options.LocalFolderPath, remoteFileName);
await client.DownloadToAsync(newFilePath);
WriteMessage($"[{DateTime.Now}] downloaded {remoteFileName}.");
}
public async Task <bool> DownloadAsync(string path, Stream destination, CancellationToken cancellationToken = default)
{
var container = GetContainerName(path);
var blobPath = GetPathWithoutContainer(path);
var client = new BlobClient(_connectionString, container, blobPath);
var res = await client.DownloadToAsync(destination, cancellationToken);
return(res.Status > 199 && res.Status < 300);
}
public async Task <byte[]> ReadAsync(IFileEntry fileEntry, CancellationToken cancellationToken = default)
{
BlobClient blob = _container.GetBlobClient(GetBlobName(fileEntry));
using var stream = new MemoryStream();
await blob.DownloadToAsync(stream, cancellationToken);
return(stream.ToArray());
}
public async Task <Response> DownloadPrivateBlob(string connectionString, string containerName, string blobName, Stream downloadToPath)
{
BlobContainerClient container = new BlobContainerClient(connectionString, containerName);
// Get a reference to a blob named "sample-file" in a container named "sample-container"
BlobClient blob = container.GetBlobClient(blobName);
// Upload local file
return(await blob.DownloadToAsync(downloadToPath));
}
public async Task <Stream> DownloadBlobAsStreamAsync(string blobName)
{
BlobClient blobClient = _blobContainerClient.GetBlobClient(blobName);
var stream = new MemoryStream();
await blobClient.DownloadToAsync(stream);
return(stream);
}
public async Task DownloadModelAsync(Guid runId, Stream destination)
{
BlobClient blobClient = this.modelRepositoryClient.GetBlobClient(this.modelPathGenerator.GetModelName(runId));
if (!await blobClient.ExistsAsync())
{
throw new FileNotFoundException($"No model exists for Run ID {runId}");
}
await blobClient.DownloadToAsync(destination);
}
private async Task <Stream> DownloadToStreamAsync(string org, string fileName)
{
BlobClient blockBlob = await CreateBlobClient(org, fileName);
var memoryStream = new MemoryStream();
await blockBlob.DownloadToAsync(memoryStream);
memoryStream.Position = 0;
return(memoryStream);
}
public async Task <Stream> DownloadAsync(string containerName, string fileName)
{
this.logger.Info($"Call: {nameof(DownloadAsync)}('{containerName}', '{fileName}')");
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName);
BlobClient blobClient = containerClient.GetBlobClient(fileName);
MemoryStream ms = new MemoryStream();
await blobClient.DownloadToAsync(ms);
return(ms);
}
public async Task DownloadBlobAsync(AvailableBlobEvent availableBlobEvent, FileSystemInfo blobLocalStorage, CancellationToken cancellationToken)
{
var sasUri = new UriBuilder(availableBlobEvent.Uri)
{
Query = FindToken(availableBlobEvent.Uri)
};
var client = new BlobClient(sasUri.Uri);
await client.DownloadToAsync(blobLocalStorage.FullName, cancellationToken);
}
public async Task <string> GetFilesByRef(string blobRef)
{
BlobContainerClient container = await GetBlobContainerClient();
BlobClient blob = container.GetBlobClient(blobRef);
var path = Path.GetTempPath() + Path.GetRandomFileName();
await blob.DownloadToAsync(path);
return(path);
}
/// <summary>
/// Downloads the model file from disk into the provided stream.
/// </summary>
/// <param name="runId">Run ID to download model for</param>
/// <param name="destination">Destination stream to write model into</param>
/// <returns>Task with result of download operation</returns>
public async Task DownloadModelAsync(Guid runId, Stream destination)
{
await this.blobContainerClient.CreateIfNotExistsAsync();
BlobClient blobClient = this.blobContainerClient.GetBlobClient($"{runId}{fileExtension}");
if (!await blobClient.ExistsAsync())
{
throw new FileNotFoundException($"No model exists for Run ID {runId}");
}
await blobClient.DownloadToAsync(destination);
}
