private void CreateCloudFileDirectoryRecursively(CloudFileDirectory dir)
{
if (null == dir)
{
return;
}
CloudFileDirectory parent = dir.Parent;
// null == parent means dir is root directory,
// we should not call CreateIfNotExists in that case
if (null != parent)
{
CreateCloudFileDirectoryRecursively(parent);
try
{
// create anyway, ignore 409 and 403
dir.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null).Wait();
}
catch (AggregateException e)
{
StorageException innnerException = e.Flatten().InnerExceptions[0] as StorageException;
if (!IgnoreDirectoryCreationError(innnerException))
{
throw;
}
}
}
}
public static void CreateCloudFileDirectoryRecursively(CloudFileDirectory dir)
{
if (null == dir)
{
return;
}
CloudFileDirectory parent = dir.Parent;
// null == parent means dir is root directory,
// we should not call CreateIfNotExists in that case
if (null != parent)
{
CreateCloudFileDirectoryRecursively(parent);
try
{
// create anyway, ignore 409 and 403
dir.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null).GetAwaiter().GetResult();
}
catch (StorageException se)
{
if (!IgnoreDirectoryCreationError(se))
{
throw;
}
}
}
}
private async Task CreateRecursiveIfNotExistsAsync(CloudFileDirectory directory)
{
if (!await directory.ExistsAsync())
{
await CreateRecursiveIfNotExistsAsync(directory.Parent);
await directory.CreateAsync();
}
}
private async Task CreateRecursiveIfNotExists(CloudFileDirectory directory)
{
bool directoryExists = await directory.ExistsAsync();
if (!directoryExists)
{
await CreateRecursiveIfNotExists(directory.Parent);
await directory.CreateAsync();
}
}
public Task CreateDirectoryAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(directory.CreateAsync(options, operationContext, cancellationToken));
}
/**
* Azure Batch AI sample.
* - Create Storage account and Azure file share
* - Upload sample data to Azure file share
* - Create a workspace and experiment
* - Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job
* - Create Microsoft Cognitive Toolkit job to run on the cluster
* - Wait for job to complete
* - Get output files
*/
public static void RunSample(IAzure azure)
{
string saName = SdkContext.RandomResourceName("sa", 10);
string rgName = SdkContext.RandomResourceName("rg", 10);
string workspaceName = SdkContext.RandomResourceName("ws", 10);
string experimentName = SdkContext.RandomResourceName("exp", 10);
string sampleDataPath = Environment.GetEnvironmentVariable("SAMPLE_DATA_PATH");
Region region = Region.USWest2;
string shareName = SdkContext.RandomResourceName("fs", 20);
string clusterName = SdkContext.RandomResourceName("cluster", 15);
string userName = Utilities.CreateUsername();
string sharePath = "mnistcntksample";
try
{
//=============================================================
// Create a new storage account and an Azure file share resource
Utilities.Log("Creating a storage account...");
IStorageAccount storageAccount = azure.StorageAccounts.Define(saName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.Create();
Utilities.Log("Created storage account.");
Utilities.PrintStorageAccount(storageAccount);
StorageAccountKey storageAccountKey = storageAccount.GetKeys().First();
Utilities.Log("Creating Azure File share...");
var cloudFileShare = CloudStorageAccount.Parse($"DefaultEndpointsProtocol=https;AccountName={saName};AccountKey={storageAccountKey.Value};EndpointSuffix=core.windows.net")
.CreateCloudFileClient()
.GetShareReference(shareName);
cloudFileShare.CreateAsync().GetAwaiter().GetResult();
Utilities.Log("Created Azure File share.");
//=============================================================
// Upload sample data to Azure file share
//Get a reference to the root directory for the share.
CloudFileDirectory rootDir = cloudFileShare.GetRootDirectoryReference();
//Get a reference to the sampledir directory
Utilities.Log("Creating directory and uploading data files...");
CloudFileDirectory sampleDir = rootDir.GetDirectoryReference(sharePath);
sampleDir.CreateAsync().GetAwaiter().GetResult();
rootDir.GetFileReference("Train-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Train-28x28_cntk_text.txt").GetAwaiter().GetResult();
rootDir.GetFileReference("Test-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Test-28x28_cntk_text.txt").GetAwaiter().GetResult();
rootDir.GetFileReference("ConvNet_MNIST.py").UploadFromFileAsync(sampleDataPath + "/ConvNet_MNIST.py").GetAwaiter().GetResult();
Utilities.Log("Data files uploaded.");
//=============================================================
// Create a workspace and experiment
IBatchAIWorkspace workspace = azure.BatchAIWorkspaces.Define(workspaceName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.Create();
IBatchAIExperiment experiment = workspace.CreateExperiment(experimentName);
//=============================================================
// Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job
Utilities.Log("Creating Batch AI cluster...");
IBatchAICluster cluster = workspace.Clusters.Define(clusterName)
.WithVMSize(VirtualMachineSizeTypes.StandardNC6.Value)
.WithUserName(userName)
.WithPassword("MyPassword")
.WithAutoScale(0, 2)
.DefineAzureFileShare()
.WithStorageAccountName(saName)
.WithAzureFileUrl(cloudFileShare.Uri.ToString())
.WithRelativeMountPath("azurefileshare")
.WithAccountKey(storageAccountKey.Value)
.Attach()
.Create();
Utilities.Log("Created Batch AI cluster.");
Utilities.Print(cluster);
// =============================================================
// Create Microsoft Cognitive Toolkit job to run on the cluster
Utilities.Log("Creating Batch AI job...");
IBatchAIJob job = experiment.Jobs.Define("myJob")
.WithExistingCluster(cluster)
.WithNodeCount(1)
.WithStdOutErrPathPrefix("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare")
.DefineCognitiveToolkit()
.WithPythonScriptFile("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/ConvNet_MNIST.py")
.WithCommandLineArgs("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare $AZ_BATCHAI_OUTPUT_MODEL")
.Attach()
.WithOutputDirectory("MODEL", "$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/model")
.WithContainerImage("microsoft/cntk:2.1-gpu-python3.5-cuda8.0-cudnn6.0")
.Create();
Utilities.Log("Created Batch AI job.");
Utilities.Print(job);
// =============================================================
// Wait for job results
// Wait for job to start running
Utilities.Log("Waiting for Batch AI job to start running...");
while (ExecutionState.Queued.Equals(job.ExecutionState))
{
SdkContext.DelayProvider.Delay(5000);
job.Refresh();
}
// Wait for job to complete and job output to become available
Utilities.Log("Waiting for Batch AI job to complete...");
while (!(ExecutionState.Succeeded.Equals(job.ExecutionState) || ExecutionState.Failed.Equals(job.ExecutionState)))
{
SdkContext.DelayProvider.Delay(5000);
job.Refresh();
}
// =============================================================
// Get output files
// Print stdout and stderr
foreach (var outputFile in job.ListFiles("stdouterr"))
{
Utilities.Log(Utilities.CheckAddress(outputFile.DownloadUrl));
}
// List model output files
foreach (var outputFile in job.ListFiles("MODEL"))
{
Utilities.Log(outputFile.DownloadUrl);
}
}
finally
{
try
{
Utilities.Log("Deleting Resource Group: " + rgName);
azure.ResourceGroups.BeginDeleteByName(rgName);
Utilities.Log("Deleted Resource Group: " + rgName);
}
catch (Exception)
{
Utilities.Log("Did not create any resources in Azure. No clean up is necessary");
}
}
}
private void CreateCloudFileDestinationDirectory(CloudFileDirectory fileDirectory,
CloudFileNtfsAttributes?fileAttributes,
DateTimeOffset?creationTime,
DateTimeOffset?lastWriteTime,
IDictionary <string, string> metadata,
string portableSDDL,
CancellationToken cancellationToken)
{
bool parentNotExist = false;
if (fileAttributes.HasValue)
{
fileDirectory.Properties.NtfsAttributes = fileAttributes.Value;
fileDirectory.Properties.CreationTime = creationTime;
fileDirectory.Properties.LastWriteTime = lastWriteTime;
}
if (null != metadata)
{
fileDirectory.Metadata.Clear();
foreach (var keyValuePair in metadata)
{
fileDirectory.Metadata.Add(keyValuePair);
}
}
string filePermissionKey = null;
if (!string.IsNullOrEmpty(portableSDDL))
{
if (portableSDDL.Length > Constants.MaxSDDLLengthInProperties)
{
this.baseDirectoryTransfer.SDDLCache.TryGetValue(portableSDDL, out filePermissionKey);
if (null == filePermissionKey)
{
filePermissionKey = fileDirectory.Share.CreateFilePermissionAsync(
portableSDDL,
Transfer_RequestOptions.DefaultFileRequestOptions,
Utils.GenerateOperationContext(null),
cancellationToken).GetAwaiter().GetResult();
this.baseDirectoryTransfer.SDDLCache.TryAddValue(portableSDDL, filePermissionKey);
}
fileDirectory.Properties.FilePermissionKey = filePermissionKey;
}
else
{
fileDirectory.FilePermission = portableSDDL;
}
}
bool needToSetProperties = false;
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
return;
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ParentNotFound", ex.RequestInformation.ErrorCode))
{
parentNotExist = true;
}
else if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
if (parentNotExist)
{
Utils.CreateCloudFileDirectoryRecursively(fileDirectory.Parent);
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
}
if (needToSetProperties)
{
SetAzureFileDirectoryAttributes(
fileDirectory,
fileAttributes,
creationTime,
lastWriteTime,
filePermissionKey,
portableSDDL,
metadata, cancellationToken);
}
}
public static async Task <string> UnZip(
[ActivityTrigger] string name,
//[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
// Get app settings.
var destinationStorage = Environment.GetEnvironmentVariable("DestinationStorageAccount");
var destinationFileShare = Environment.GetEnvironmentVariable("DestinationFileShare");
var sourceStorageConnectionString = Environment.GetEnvironmentVariable("SourceStorageAccount");
//string name = "test.zip";
var localZipFile = SetLocalPath(name);
log.LogInformation($"Blob trigger function Processed blob:{name}");
// Check whether the connection string can be parsed.
CloudStorageAccount storageAccount;
if (CloudStorageAccount.TryParse(sourceStorageConnectionString, out storageAccount))
{
// If the connection string is valid, proceed with operations against Blob
// storage here.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference("archived");
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(name);
await cloudBlockBlob.DownloadToFileAsync(localZipFile, FileMode.Create);
}
else
{
// Otherwise, let the user know that they need to define the environment variable.
log.LogInformation(
"A connection string has not been defined in the system environment variables. " +
"Add an environment variable named 'SourceStorageAccount' with your storage " +
"connection string as a value.");
}
// Parse the connection string for the storage account.
CloudStorageAccount cloudFileStorageAccount = CloudStorageAccount.Parse(destinationStorage);
// Create a CloudFileClient object for credentialed access to Azure Files.
CloudFileClient fileClient = cloudFileStorageAccount.CreateCloudFileClient();
// Get a reference to the file share.
CloudFileShare share = fileClient.GetShareReference(destinationFileShare);
// Get a reference to the root directory for the share.
CloudFileDirectory destinationDirectory = share.GetRootDirectoryReference();
// Create file share if it doesn't exist.
if (!await share.ExistsAsync())
{
await share.CreateAsync();
}
// Set slash character that is used to differentiate the zip entry from file.
char slash = '/';
//var localZipFile = SetLocalPath(name);
try
{
// Filter out only zips.
if (name.Split('.').Last().ToLower() == "zip")
{
// write the zip to the disk
//await myBlob.DownloadToFileAsync(localZipFile, FileMode.Create);
// Opening zip file and specifying encoding for entries in the zip.
using (ZipArchive archive = ZipFile.Open(localZipFile, 0, Encoding.GetEncoding("ISO-8859-1")))
{
foreach (ZipArchiveEntry entry in archive.Entries)
{
/// How to tell if a “ZipArchiveEntry” is directory? - https://stackoverflow.com/questions/40223451/how-to-tell-if-a-ziparchiveentry-is-directory
// Check if th zip archive entry is a folder. FullName property for folders end with a "/" and Name property is empty.
if (slash == entry.FullName[entry.FullName.Length - 1] && 0 == entry.Name.Length)
{
// Create a folder if the zip archive entry is a folder.
log.LogInformation($"Now processing folder '{entry.FullName}'");
CloudFileDirectory EntryDestinationDirectory = destinationDirectory.GetDirectoryReference(entry.FullName);
if (!await EntryDestinationDirectory.ExistsAsync())
{
await EntryDestinationDirectory.CreateAsync();
}
}
// Check if the zip archive entry is a file.
if (slash != entry.FullName.Length - 1 && 0 != entry.Name.Length)
{
log.LogInformation($"Now processing file '{entry.FullName}'");
//// Create buffer that is used to measure the deflated file size
byte[] buf = new byte[1024];
int size = 0;
// Open the entry to measure the size
using (var fileStream = entry.Open())
{
int len;
while ((len = fileStream.Read(buf, 0, buf.Length)) > 0)
{
size += len;
}
}
//var threadCount = 1;
//if (size > 83886080)
//{
// threadCount = 4;
//}
//else
//{
// threadCount = 1;
//}
var requestOptions = new FileRequestOptions()
{
ParallelOperationThreadCount = 8
};
// Open the zip entry for further processing
using (var fileStream = entry.Open())
{
// Open memory stream by specifying the size
using (var fileMemoryStream = new MemoryStream(size + 1024))
//using (var fileMemoryStream = new MemoryStream())
{
fileStream.CopyTo(fileMemoryStream);
fileMemoryStream.Position = 0;
var destinationFile = destinationDirectory.GetFileReference(entry.FullName);
await destinationFile.UploadFromStreamAsync(fileMemoryStream, null, requestOptions, null);
}
}
}
}
}
}
}
catch (Exception ex)
{
log.LogInformation($"Error! Something went wrong: {ex.Message}, {ex.StackTrace}");
}
//log.LogInformation($"cleaning up temp files {localZipFile}");
//CleanUp(localZipFile);
log.LogInformation($"Unzip of '{name}' completed!");
return("OK!");
}
private static void CreateCloudFileDestinationDirectory(CloudFileDirectory fileDirectory,
CloudFileNtfsAttributes?fileAttributes,
DateTimeOffset?creationTime,
DateTimeOffset?lastWriteTime,
IDictionary <string, string> metadata,
CancellationToken cancellationToken)
{
bool parentNotExist = false;
if (fileAttributes.HasValue)
{
fileDirectory.Properties.NtfsAttributes = fileAttributes.Value;
fileDirectory.Properties.CreationTime = creationTime;
fileDirectory.Properties.LastWriteTime = lastWriteTime;
}
if (null != metadata)
{
fileDirectory.Metadata.Clear();
foreach (var keyValuePair in metadata)
{
fileDirectory.Metadata.Add(keyValuePair);
}
}
bool needToSetProperties = false;
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
return;
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ParentNotFound", ex.RequestInformation.ErrorCode))
{
parentNotExist = true;
}
else if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
if (parentNotExist)
{
Utils.CreateCloudFileDirectoryRecursively(fileDirectory.Parent);
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
}
if (needToSetProperties)
{
SetAzureFileDirectoryAttributes(fileDirectory, fileAttributes, creationTime, lastWriteTime, metadata, cancellationToken);
}
}
