public async Task UploadBackupAsync(string fileName)
{
var storageCredentials = new StorageCredentials(
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_NAME"),
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_KEY")
);
//upload the backup to Azure blob
try
{
_logger.Info("Attempting to upload backup to Azure, blobname: {0}", fileName);
CloudBlobClient cloudBlobClient = new CloudStorageAccount(storageCredentials, useHttps: true).CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_CONTAINER"));
await cloudBlobContainer.CreateIfNotExistsAsync(_resillientRequestOptions, null);
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(fileName);
await cloudBlockBlob.UploadFromFileAsync(fileName, null, _resillientRequestOptions, null);
_logger.Info("Succesfully uploaded backup to Azure, blobname: {0}", fileName);
}
catch (Exception e)
{
_logger.Fatal("Failed to upload backup to Azure! {0}", e.Message.ToString());
throw e;
}
}
private static Version GetOnlineVersion(Options options)
{
CloudBlobClient client = new CloudStorageAccount(new StorageCredentials(options.StorageAccountName, options.StorageAccountKey), true).CreateCloudBlobClient();
Version result = new Version("0.0.0");
var container = client.GetContainerReference(options.StorageContainer);
if (container.Exists())
{
var blob = container.GetBlockBlobReference(options.GetBlobName());
if (blob?.Exists() ?? false)
{
try
{
result = new Version(blob.Metadata["version"]);
}
catch
{
// do nothing
}
}
}
return(result);
}
private static void Upload(Options options, string version)
{
CloudBlobClient client = new CloudStorageAccount(new StorageCredentials(options.StorageAccountName, options.StorageAccountKey), true).CreateCloudBlobClient();
var container = client.GetContainerReference(options.StorageContainer);
container.CreateIfNotExists();
string fileName = Path.GetFileName(options.InstallerExe);
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
blob.Metadata.Add("version", version);
blob.UploadFromFile(options.InstallerExe);
}
internal static async ValueTask <CloudBlobContainer> GetBlockBlobReferenceAsync(this CloudStorageAccount storageAccount, string container)
{
var containerRef = storageAccount.GetContainerReference(container);
if (!await containerRef.ExistsAsync())
{
await containerRef.CreateIfNotExistsAsync();
}
var permissions = await containerRef.GetPermissionsAsync(null, RequestOptions, null);
permissions.PublicAccess = BlobContainerPublicAccessType.Off;
await containerRef.SetPermissionsAsync(permissions, null, RequestOptions, null);
return(containerRef);
}
public async Task <String> DownloadLatestBackupAsync(string DestinationDirectory)
{
var storageCredentials = new StorageCredentials(
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_NAME"),
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_KEY")
);
try
{
CloudBlobClient cloudBlobClient = new CloudStorageAccount(storageCredentials, useHttps: true).CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_CONTAINER"));
BlobContinuationToken blobContinuationToken = null;
BlobResultSegment results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, 100, blobContinuationToken, _resillientRequestOptions, null);
CloudBlob latestbackup = null;
foreach (CloudBlob blob in results.Results)
{
await blob.FetchAttributesAsync(null, _resillientRequestOptions, null);
if (latestbackup == null)
{
latestbackup = blob;
}
if (blob.Properties.Created > latestbackup.Properties.Created)
{
latestbackup = blob;
}
}
_logger.Info("Found {0} backup(s) on Azure blob storage", results.Results.Count <IListBlobItem>());
_logger.Info("Attempting to download latest backup {0} from Azure blob..", latestbackup.Name);
await latestbackup.DownloadToFileAsync(String.Concat(DestinationDirectory, @"/", latestbackup.Name), System.IO.FileMode.CreateNew, null, _resillientRequestOptions, null);
_logger.Info("Downloaded latest backup {0} succesfully from Azure blob!", latestbackup.Name);
return(String.Concat(DestinationDirectory, @"/", latestbackup.Name));
}
catch (Exception e)
{
_logger.Fatal("Failed to download latest backup from Azure blob storage: {0}", e.Message.ToString());
throw e;
}
}
public async Task <string> Upload(byte[] bytes, string imageName)
{
try
{
var creds = new StorageCredentials(ApiConstants.AzureSAS);
var client = new CloudStorageAccount(creds, ApiConstants.AzureAccountName, "", true).CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(ApiConstants.AzureContainer);
//// Create the container if it doesn't already exist.
await container.CreateIfNotExistsAsync();
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference(imageName);
using (Stream stream = new MemoryStream(bytes))
{
try
{
await blockBlob.UploadFromStreamAsync(stream);
bytes = null;
return(blockBlob.Uri.AbsolutePath);
}
catch (Microsoft.WindowsAzure.Storage.StorageException storageException)
{
int i = 0;
}
catch (System.Net.ProtocolViolationException protocolViolationException)
{
int i = 0;
}
catch (Exception exception)
{
int i = 0;
}
}
}
catch (Exception ex)
{
int i = 1;
}
return(null);
}
internal void UploadJson(CloudStorageAccount cloudStorageAccount, FeedsMetadata feedsMeta, ProgramArgs args)
{
var root = feedsMeta.ToRootDirectory(args);
var rootInfo = new DirectoryInfo(root);
var container = cloudStorageAccount.GetContainerReference("studio-dash");
var tasks = feedsMeta.Feeds.Keys.Select(i =>
{
var jsonFile = rootInfo.ToCombinedPath($"{i}.json");
if (!File.Exists(jsonFile))
{
traceSource?.TraceWarning($"The expected file, `{jsonFile}`, is not here.");
return(Task.FromResult(0));
}
traceSource?.TraceVerbose($"uploading {jsonFile}...");
return(container.UploadBlobAsync(jsonFile, string.Empty));
}).ToArray();
Task.WaitAll(tasks);
}
public async Task RemoveOldBackupsAsync(int maximumAllowedBackups)
{
var storageCredentials = new StorageCredentials(
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_NAME"),
Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_KEY")
);
//checks the number of backups on Azure and remove oldest if above maximum
try
{
CloudBlobClient cloudBlobClient = new CloudStorageAccount(storageCredentials, useHttps: true).CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(Environment.GetEnvironmentVariable("AZURE_STORAGEACCOUNT_CONTAINER"));
BlobContinuationToken blobContinuationToken = null;
List <Tuple <Uri, DateTimeOffset?> > Containerblobs = new List <Tuple <Uri, DateTimeOffset?> >();
BlobResultSegment results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, 100, blobContinuationToken, _resillientRequestOptions, null);
foreach (IListBlobItem item in results.Results)
{
CloudBlockBlob blockblob = null;
try
{
blockblob = cloudBlobContainer.GetBlockBlobReference(new CloudBlockBlob(item.Uri).Name);
await blockblob.FetchAttributesAsync(null, _resillientRequestOptions, null);
Containerblobs.Add(new Tuple <Uri, DateTimeOffset?>(item.Uri, blockblob.Properties.Created));
}
catch (Exception e)
{
_logger.Warn("failed to fetch attribues for blob {0} with uri {1}: {2}", blockblob.Name, blockblob.StorageUri, e.Message.ToString());
}
}
_logger.Info("Found {0} backups on Azure blob storage", Containerblobs.Count);
if (Containerblobs.Count > maximumAllowedBackups)
{
_logger.Info("{0} backups on Azure blob storage exceed the maximum allowed number of backups {1}", Containerblobs.Count, maximumAllowedBackups);
var numberOfBackupsToRemove = Containerblobs.Count - maximumAllowedBackups;
_logger.Info("{0} backups need to be removed from Azure blob storage", numberOfBackupsToRemove);
IEnumerable <Tuple <Uri, DateTimeOffset?> > blobsToRemove = Containerblobs.OrderBy(x => x.Item2).Take(numberOfBackupsToRemove);
foreach (Tuple <Uri, DateTimeOffset?> blobItem in blobsToRemove)
{
_logger.Info("Attempting to remove backup blob {0} ...", blobItem.Item1.ToString());
var blockblob = cloudBlobContainer.GetBlockBlobReference(new CloudBlockBlob(blobItem.Item1).Name);
await blockblob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, null, _resillientRequestOptions, null);
_logger.Info("backup blob {0} removed succesfully!", blobItem.Item1.ToString());
}
}
else
{
_logger.Info("{0} backups on Azure blob storage does not exceed the maximum allowed number of backups {1}, no backups were removed...", Containerblobs.Count, maximumAllowedBackups);
}
}
catch (Exception e)
{
_logger.Warn("Failed to remove old backups from Azure blob storage: {0}", e.Message.ToString());
throw e;
}
}
static void Main(string[] args)
{
// Parse Arguments
Helper.SETTING g_setting = new Helper.SETTING();
if (!Helper.parse_arg(args.Length, args, out g_setting))
{
Helper.print_usage(args.Length, args);
// Console.ReadKey();
return;
}
Console.WriteLine();
// Get the uri.
var uri = g_setting.uri;
Console.WriteLine(" Processing: {0}", uri);
Console.WriteLine("");
try
{
// Init client and blob list. Endpoint is set to Mooncake by default.
var client = new CloudStorageAccount(new StorageCredentials(g_setting.accountName, g_setting.storagekey), ConfigurationManager.AppSettings[g_setting.environment.ToLower()], false).CreateCloudBlobClient();
var isBlobUri = false;
var blobs = new List <CloudPageBlob>();
// It's an uri.
if (uri.StartsWith("http://") || uri.StartsWith("https://"))
{
var blob = client.GetBlobReferenceFromServer(new Uri(uri)) as CloudPageBlob;
if (blob == null)
{
throw new FileNotFoundException("Unable to find the Page Blob.");
}
blobs.Add(blob);
isBlobUri = true;
}
else
{
// It's a container.
var container = client.GetContainerReference(uri);
if (!container.Exists())
{
throw new InvalidOperationException("Container does not exist: " + uri);
}
blobs.AddRange(container.ListBlobs().OfType <CloudPageBlob>());
}
// Show blob sizes.
foreach (var blob in blobs)
{
if (!isBlobUri)
{
Console.WriteLine(" Blob: {0}", blob.Uri);
}
// Display length.
Console.WriteLine(" > Size: {0} ({1} bytes)", PageBlobExtensions.GetFormattedDiskSize(blob.Properties.Length), blob.Properties.Length);
// Calculate size.
if (Int64.Parse(g_setting.pageRangeSize) > 0)
{
var size = blob.GetActualDiskSizeAsync(Int64.Parse(g_setting.pageRangeSize));
Console.WriteLine(" > Actual/Billable Size: {0} ({1} bytes)", PageBlobExtensions.GetFormattedDiskSize(size), size);
Console.WriteLine("");
}
else
{
var size = blob.GetActualDiskSize();
Console.WriteLine(" > Actual/Billable Size: {0} ({1} bytes)", PageBlobExtensions.GetFormattedDiskSize(size), size);
Console.WriteLine("");
}
}
Console.WriteLine();
Console.WriteLine("Calulation completed, press any key to continue...");
Console.ReadLine();
}
catch (Exception ex)
{
Console.WriteLine(" Error:");
Console.WriteLine(" {0}", ex);
Console.ReadLine();
}
}
static async Task Main(string[] args)
{
var authMode = AuthMode.ConnectionString;
// Create a CloudBlobClient using various auth methods
CloudBlobClient cloudBlobClient = null;
switch (authMode)
{
case AuthMode.ConnectionString:
// Account Key
// Local Emulator
// Technically can do SAS here as well
// Local Emulator (direct)
//cloudBlobClient = CloudStorageAccount.DevelopmentStorageAccount
// .CreateCloudBlobClient();
//cloudBlobClient = CloudStorageAccount.Parse("UseDevelopmentStorage=true")
// .CreateCloudBlobClient();
// Account Key-Based Connection String
cloudBlobClient = CloudStorageAccount.Parse($"DefaultEndpointsProtocol=https;AccountName={AccountName};AccountKey={AccountKey};EndpointSuffix=core.windows.net")
.CreateCloudBlobClient();
break;
case AuthMode.SasToken:
var sasCredentials = new StorageCredentials(ReadWriteSasKey);
cloudBlobClient = new CloudStorageAccount(sasCredentials, AccountName, endpointSuffix: null, useHttps: true).CreateCloudBlobClient();
break;
case AuthMode.AppRegistration:
case AuthMode.ManagedIdentity:
// Get an access token
string accessToken = null;
if (AuthMode.ManagedIdentity == authMode)
{
// MSI when on Azure
// Visual Studio (Azure Service Authentication)
// Command Line (az login)
var msiTokenProvider = new AzureServiceTokenProvider();
accessToken = await msiTokenProvider.GetAccessTokenAsync("https://storage.azure.com/");
// No emulator support
}
else
{
// Anywhere, with the app's own ID
var clientApp = ConfidentialClientApplicationBuilder.Create(ClientId)
.WithClientSecret(ClientSecret)
.WithAuthority(Authority)
.Build();
var authResult = await clientApp.AcquireTokenForClient(new[] { "https://storage.azure.com/.default" }).ExecuteAsync();
accessToken = authResult.AccessToken;
// No emulator support
}
var tokenCredentials = new StorageCredentials(new TokenCredential(accessToken));
cloudBlobClient = new CloudStorageAccount(tokenCredentials, AccountName, endpointSuffix: null, useHttps: true).CreateCloudBlobClient();
break;
default:
throw new InvalidOperationException();
}
// Grab reference to the container we'll use for all the samples
var container = cloudBlobClient.GetContainerReference("my-files");
// List all files in the container
Console.WriteLine("LISTING FILES in example-container...");
var blobNames = new List <string>();
// Loop through the blob metadata grabbing the name
BlobContinuationToken bct = null;
do
{
var result = await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.None, null, bct, null, null);
blobNames.AddRange(result.Results.OfType <CloudBlob>().Select(b => b.Name));
bct = result.ContinuationToken;
} while (bct != null);
// Sort them for my OCD purposes
blobNames.Sort();
foreach (var filename in blobNames)
{
Console.WriteLine(filename);
}
Console.WriteLine("...done" + Environment.NewLine);
// Add a new text file
Console.WriteLine("Uploading text file my-text.txt...");
// Grab a reference to the blob and upload (overwritting if exists)
var newBlob = container.GetBlockBlobReference("my-text.txt");
await newBlob.UploadTextAsync("I really love blobs!");
Console.WriteLine("...done" + Environment.NewLine);
// Read the file
Console.WriteLine("Reading text file my-text.txt...");
// Grab a reference to the blob and download
var existingBlob = container.GetBlockBlobReference("my-text.txt");
Console.WriteLine(await existingBlob.DownloadTextAsync());
Console.WriteLine("...done" + Environment.NewLine);
// Move that file to a diffent name
Console.WriteLine("Moving my-text.txt to your-text.txt...");
// Get a handle on both files in the container
var sourceBlob = container.GetBlockBlobReference("my-text.txt");
newBlob = container.GetBlockBlobReference("your-text.txt");
// Move the blob
if (await sourceBlob.ExistsAsync())
{
// This does a server-side async copy
await newBlob.StartCopyAsync(sourceBlob);
// Wait for the copy to complete
while (CopyStatus.Pending == newBlob.CopyState.Status)
{
await Task.Delay(100);
await newBlob.FetchAttributesAsync();
}
// 86 the original file
await sourceBlob.DeleteIfExistsAsync();
}
Console.WriteLine("...done" + Environment.NewLine);
// Read the moved file
Console.WriteLine("Reading text file your-text.txt...");
// Grab a reference to the blob and download
existingBlob = container.GetBlockBlobReference("your-text.txt");
Console.WriteLine(await existingBlob.DownloadTextAsync());
Console.WriteLine("...done");
}
static void Main(string[] args)
{
// Header.
Console.WriteLine("");
Console.WriteLine(" Windows Azure VHD Size v1.0 (2013-12-18) - Sandrino Di Mattia");
Console.WriteLine(" Usage: wazvhdsize.exe <accountName> <accountKey> <vhdUrl|containerName>");
Console.WriteLine("");
// Validate args.
if (args.Length != 3)
{
Console.WriteLine(" Invalid number of arguments.");
Console.WriteLine("");
return;
}
// Get the uri.
var uri = args[2];
Console.WriteLine(" Processing: {0}", uri);
Console.WriteLine("");
try
{
// Init client and blob list.
var client = new CloudStorageAccount(new StorageCredentials(args[0], args[1]), false).CreateCloudBlobClient();
var isBlobUri = false;
var blobs = new List <CloudPageBlob>();
// It's an uri.
if (uri.StartsWith("http://") || uri.StartsWith("https://"))
{
var blob = client.GetBlobReferenceFromServer(new Uri(uri)) as CloudPageBlob;
if (blob == null)
{
throw new FileNotFoundException("Unable to find the Page Blob.");
}
blobs.Add(blob);
isBlobUri = true;
}
else
{
// It's a container.
var container = client.GetContainerReference(uri);
if (!container.Exists())
{
throw new InvalidOperationException("Container does not exist: " + uri);
}
blobs.AddRange(container.ListBlobs().OfType <CloudPageBlob>());
}
// Show blob sizes.
foreach (var blob in blobs)
{
if (!isBlobUri)
{
Console.WriteLine(" Blob: {0}", blob.Uri);
}
// Display length.
Console.WriteLine(" > Size: {0} ({1} bytes)", PageBlobExtensions.GetFormattedDiskSize(blob.Properties.Length), blob.Properties.Length);
// Calculate size.
var size = blob.GetActualDiskSize();
Console.WriteLine(" > Actual/Billable Size: {0} ({1} bytes)", PageBlobExtensions.GetFormattedDiskSize(size), size);
Console.WriteLine("");
}
}
catch (Exception ex)
{
Console.WriteLine(" Error:");
Console.WriteLine(" {0}", ex);
}
}
