/// <inheritdoc /> public IEnumerable <string> GetFiles(string[] path = null) { CloudFileDirectory cloudFileDirectory = CloudFileShare.GetDirectoryReference(path: path); return(cloudFileDirectory.EnumerateDirectory(EFileType.Files, "*")); }
/// <summary> /// Write CloudFileDirectory to output using specified service channel /// </summary> /// <param name="fileDir">The output CloudFileDirectory object</param> /// <param name="channel">IStorageFileManagement channel object</param> internal void WriteCloudFileDirectoryeObject(long taskId, IStorageFileManagement channel, CloudFileDirectory fileDir) { AzureStorageFileDirectory azureFileDir = new AzureStorageFileDirectory(fileDir, channel.StorageContext); OutputStream.WriteObject(taskId, azureFileDir); }
private async Task <CloudFile> BuildCloudFileInstanceFromPathAsync(string defaultFileName, string[] path, bool pathIsDirectory) { CloudFileDirectory baseDirectory = null; bool isPathEmpty = path.Length == 0; switch (this.ParameterSetName) { case LocalConstants.DirectoryParameterSetName: baseDirectory = this.Directory; break; case LocalConstants.ShareNameParameterSetName: NamingUtil.ValidateShareName(this.ShareName, false); baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference(); break; case LocalConstants.ShareParameterSetName: baseDirectory = this.Share.GetRootDirectoryReference(); break; default: throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName)); } if (isPathEmpty) { return(baseDirectory.GetFileReference(defaultFileName)); } var directory = baseDirectory.GetDirectoryReferenceByPath(path); if (pathIsDirectory) { return(directory.GetFileReference(defaultFileName)); } bool directoryExists; try { directoryExists = await this.Channel.DirectoryExistsAsync(directory, this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false); } catch (StorageException e) { if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden) { //Forbidden to check directory existance, might caused by a write only SAS //Don't report error here since should not block upload with write only SAS //If the directory not exist, Error will be reported when upload with DMlib later directoryExists = true; } else { if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest && e.RequestInformation.ExtendedErrorInformation == null) { throw new AzureStorageFileException(ErrorCategory.InvalidArgument, ErrorIdConstants.InvalidResource, Resources.InvalidResource, this); } throw; } } if (directoryExists) { // If the directory exist on the cloud, we treat the path as // to a directory. So we append the default file name after // it and build an instance of CloudFile class. return(directory.GetFileReference(defaultFileName)); } else { // If the directory does not exist, we treat the path as to a // file. So we use the path of the directory to build out a // new instance of CloudFile class. return(baseDirectory.GetFileReferenceByPath(path)); } }
public static void addRecordToAzureFileShare(string myQueueItem, string fileName, ILogger log) { try { string valueLine = string.Empty; string clientHeader = string.Empty; var searcResult = new string[10]; StringBuilder builder = new StringBuilder(); StringBuilder headerBuilder = new StringBuilder(); StringBuilder valueBuilder = new StringBuilder(); string[] arrQueueItem = myQueueItem.Split("||", StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < arrQueueItem.Length; i++) { var arrHeader = arrQueueItem[i]?.Split(":", StringSplitOptions.RemoveEmptyEntries); if (!string.IsNullOrEmpty(arrHeader[0]) && !arrHeader[0].Trim().Equals("CampaignName")) { string header = arrHeader[0].Replace(",", "-"); string value = arrHeader.Length > 1 ? arrHeader[1].Replace(",", "-") : ""; //Replace comma with hyphen in values headerBuilder.Append(header + ","); valueBuilder.Append(value + ","); } } var headerLastIndex = headerBuilder.ToString().LastIndexOf(","); if (headerLastIndex >= 0) { clientHeader = headerBuilder.ToString().Remove(headerLastIndex, 1) + Environment.NewLine; } builder.Append(clientHeader); var valueLastIndex = valueBuilder.ToString().LastIndexOf(","); if (valueLastIndex >= 0) { valueLine = valueBuilder.ToString().Remove(valueLastIndex, 1); } builder.Append(valueLine); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetEnvironmentVariable("AzureWebJobsStorage")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare fileShare = fileClient.GetShareReference(GetEnvironmentVariable("AzureStorageFileShare")); string existingFiletxt = ""; if (fileShare.Exists()) { CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference(); if (rootDirectory.Exists()) { CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(GetEnvironmentVariable("AzureStorageFileDirectory")); if (customDirectory.Exists()) { CloudFile file = customDirectory.GetFileReference(fileName + ".csv"); if (file.Exists()) { existingFiletxt = file.DownloadTextAsync().Result; string existingHeader = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)?[0]; string existingValue = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1]; if (existingHeader + Environment.NewLine == clientHeader) { customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine); } else { existingFiletxt = clientHeader + existingValue; customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine.ToString()); } } else { customDirectory.GetFileReference(fileName + ".csv").UploadText(builder.ToString()); } } } } } catch (Exception ex) { log.LogInformation("Some error occured " + ex); } }
/// <summary> /// Enumerates a specified cloud file directory and returns items based item type and searchPattern. /// </summary> /// <param name="cloudFileDirectory">Cloud file directory to enumerate on</param> /// <param name="fileType">Item type to filter with</param> /// <param name="searchPattern">Search pattern to filter with</param> /// <returns>Enumerable of strings which contains all found names based on file type</returns> public static IEnumerable <string> EnumerateDirectory(this CloudFileDirectory cloudFileDirectory, EFileType fileType, string searchPattern) { #region validation if (cloudFileDirectory == null) { throw new ArgumentNullException(nameof(cloudFileDirectory)); } if (string.IsNullOrEmpty(searchPattern)) { throw new ArgumentNullException(nameof(searchPattern)); } if (!TaskUtilities.ExecuteSync(cloudFileDirectory.ExistsAsync())) { return(new List <string>()); } #endregion var directoryNames = new List <string>(); var fileContinuationToken = new FileContinuationToken(); do { FileResultSegment fileResultSegment = TaskUtilities.ExecuteSync(cloudFileDirectory.ListFilesAndDirectoriesSegmentedAsync(fileContinuationToken)); foreach (IListFileItem listFileItem in fileResultSegment.Results) { string fullListItemPath = string.Empty; switch (fileType) { case EFileType.Files: if (listFileItem is CloudFile cloudFile) { fullListItemPath = cloudFile.StorageUri.PrimaryUri.ToString(); } break; case EFileType.Directories: if (listFileItem is CloudFileDirectory pendingCloudFileDirectory) { fullListItemPath = pendingCloudFileDirectory.StorageUri.PrimaryUri.ToString(); if (fullListItemPath.EndsWith('/')) { fullListItemPath = fullListItemPath.Remove(fullListItemPath.LastIndexOf('/')); } } break; default: throw new ArgumentOutOfRangeException(nameof(fileType), fileType, null); } // replace wildcard for 'multiple letters' with regex-wildcard string regexableSearchPattern = searchPattern.Replace("*", ".*"); // replace wildcard for 'single letter' with regex-wildcard regexableSearchPattern = regexableSearchPattern.Replace("?", ".?"); // set search pattern in 'begin-to-end'-symbols regexableSearchPattern = $"^{regexableSearchPattern}$"; var rgxFileName = new Regex(regexableSearchPattern); if (!string.IsNullOrEmpty(fullListItemPath) && rgxFileName.IsMatch(fullListItemPath)) { directoryNames.Add(fullListItemPath); } } fileContinuationToken = fileResultSegment.ContinuationToken; } while (fileContinuationToken != null); return(directoryNames); }
public async Task <ActionResult> DoFileUploads(HttpPostedFileBase[] files) { // Step 1: Intialize the Azure Configuration form azure settings config var azureconfiguration = ConfigInit.getConfiguration(); // Step 2: Intialize the Azure DataLake Storage var storageAccountConnectionString_File = CloudStorageAccount.Parse(azureconfiguration.AzureStorageConnectionString); CloudFileClient cloudFileClient = storageAccountConnectionString_File.CreateCloudFileClient(); CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(azureconfiguration.AzureFileStorageReference); CloudFileDirectory rootDir = cloudFileShare.GetRootDirectoryReference(); CloudFile fileSas = null; // Step 3: Intailize the API Client HttpClient client = new HttpClient(); client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", azureconfiguration.AzureVisionAPISubscriptionKey); HttpResponseMessage response; // Step 4: Intailize the Cosmos DB var cosmosDBResponse = new CosmosAPIService(); DocumentClient documentClient = new DocumentClient(new Uri(azureconfiguration.AzureCosmosDBConnectionString), azureconfiguration.AzureCosmonDBPrimaryKey1); var createDataBaseResponse = await documentClient.CreateDatabaseIfNotExistsAsync(new Microsoft.Azure.Documents.Database { Id = "CognitiveAPIDemoDataSource" }); var createDataBaseCollectionResponse = await documentClient.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri("CognitiveAPIDemoDataSource"), new DocumentCollection { Id = "PocCollection" }, new RequestOptions { OfferThroughput = 1000 }); List <APIData> objrApiData = new List <APIData>(); for (int i = 0; i < files.Length; i++) { if (files[i].FileName.EndsWith(".jpg") || files[i].FileName.EndsWith(".png")) { bytesFromImage = getbyteFromInputStream(files[i].InputStream, files[i].ContentLength); if (cloudFileShare.Exists()) { fileSas = rootDir.GetFileReference(Path.GetFileName(files[i].FileName)); fileSas.UploadFromStream(new MemoryStream(bytesFromImage)); } // Extracting Image : using (ByteArrayContent content = new ByteArrayContent(bytesFromImage)) { content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); response = await client.PostAsync(azureconfiguration.AzureVisionAPIEndPoint + "?" + "mode=Handwritten", content); } if (response.IsSuccessStatusCode) { string operationLocation = response.Headers.GetValues("Operation-Location").FirstOrDefault(); string contentString; int res = 0; do { System.Threading.Thread.Sleep(1000); response = await client.GetAsync(operationLocation); contentString = await response.Content.ReadAsStringAsync(); ++res; }while (res < 10 && contentString.IndexOf("\"status\":\"Succeeded\"") == -1); if (res == 10 && contentString.IndexOf("\"status\":\"Succeeded\"") == -1) { //return null; } var rootobject = JsonConvert.DeserializeObject <RootObject>(contentString); StringBuilder builder = new StringBuilder(); for (int s = 0; s < rootobject.recognitionResult.lines.Count; s++) { builder.Append(rootobject.recognitionResult.lines[s].text.ToString()); } objrApiData.Add(new APIData() { OCRID = Guid.NewGuid().ToString(), imageData = bytesFromImage, imageText = builder.ToString(), ImageUrl = Path.GetFileName(files[i].FileName),//fileSas.Uri.AbsoluteUri.ToString(), pid = i, sid = "text" + i, Error = "no Error", Remarks = "No Remarks Found" }); } else { string errorString = await response.Content.ReadAsStringAsync(); var errorObject = JsonConvert.DeserializeObject <VisionAPIErrorMessage>(errorString); objrApiData.Add(new APIData() { OCRID = Guid.NewGuid().ToString(), imageData = bytesFromImage, imageText = errorObject.error.message.ToString(), ImageUrl = Path.GetFileName(files[i].FileName),//fileSas.Uri.AbsoluteUri.ToString(), pid = i, sid = "text" + i, Error = errorObject.error.message.ToString(), Remarks = "Image scanning Failed" }); } } } var dbProcessingResponse = await cosmosDBResponse.CreateDocumentCollection(objrApiData, documentClient); TempData["objrApiData"] = objrApiData; return(RedirectToAction("ShowResults")); }
public override void ExecuteCmdlet() { if (this.ShouldProcess(string.Format("Close File Handles for File or FileDirectory on Path: {0}", this.Path != null? this.Path : (this.FileHandle != null? this.FileHandle.Path: null)), "This operation will force the provided file handle(s) closed, which may cause data loss or corruption for active applications/users.", null)) { CloudFileDirectory baseDirectory = null; switch (this.ParameterSetName) { case DirectoryCloseAllParameterSetName: baseDirectory = this.Directory; break; case ShareNameCloseSingleParameterSetName: case ShareNameCloseAllParameterSetName: baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference(); break; case ShareCloseSingleParameterSetName: case ShareCloseAllParameterSetName: baseDirectory = this.Share.GetRootDirectoryReference(); break; case FileCloseAllParameterSetName: // Don't need to set baseDirectory when input is a CloudFile break; default: throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName)); } if (ParameterSetName == ShareNameCloseSingleParameterSetName || ParameterSetName == ShareCloseSingleParameterSetName) { this.Path = FileHandle.Path; } // When not input path/File, the list handle target must be a Dir bool foundAFolder = true; CloudFileDirectory targetDir = baseDirectory; CloudFile targetFile = null; if (this.File != null) { targetFile = this.File; foundAFolder = false; } else { if (!string.IsNullOrEmpty(this.Path)) { string[] subfolders = NamingUtil.ValidatePath(this.Path); targetDir = baseDirectory.GetDirectoryReferenceByPath(subfolders); if (!targetDir.Exists()) { foundAFolder = false; } if (!foundAFolder) { //Get file string[] filePath = NamingUtil.ValidatePath(this.Path, true); targetFile = baseDirectory.GetFileReferenceByPath(filePath); this.Channel.FetchFileAttributesAsync( targetFile, null, this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false); } } } // Recursive only take effect on File Dir if (!foundAFolder && Recursive.IsPresent) { WriteVerbose("The target object of the 'Path' is an Azure File, the parameter '-Recursive' won't take effect."); } //Close handle CloseFileHandleResultSegment closeResult = null; FileContinuationToken continuationToken = null; int numHandlesClosed = 0; do { if (foundAFolder) { if (FileHandle != null) { // close single handle on fileDir if (this.FileHandle.HandleId == null) { throw new System.ArgumentException(string.Format("The HandleId of the FileHandle on path {0} should not be null.", this.FileHandle.Path), "FileHandle"); } closeResult = targetDir.CloseHandleSegmented(this.FileHandle.HandleId.ToString(), continuationToken, Recursive, null, this.RequestOptions, this.OperationContext); } else { // close all handle on fileDir closeResult = targetDir.CloseAllHandlesSegmented(continuationToken, Recursive, null, this.RequestOptions, this.OperationContext); } } else { if (FileHandle != null) { // close single handle on file if (this.FileHandle.HandleId == null) { throw new System.ArgumentException(string.Format("The HandleId of the FileHandle on path {0} should not be null.", this.FileHandle.Path), "FileHandle"); } closeResult = targetFile.CloseHandleSegmented(this.FileHandle.HandleId.ToString(), continuationToken, null, this.RequestOptions, this.OperationContext); } else { // close all handle on file closeResult = targetFile.CloseAllHandlesSegmented(continuationToken, null, this.RequestOptions, this.OperationContext); } } numHandlesClosed += closeResult.NumHandlesClosed; continuationToken = closeResult.ContinuationToken; } while (continuationToken != null && continuationToken.NextMarker != null); if (PassThru) { WriteObject(numHandlesClosed); } } }
static void Main(string[] args) { string shareName = "\\\\unipertest.file.core.windows.net\\SecretDocuments"; string driveLetterAndColon = "z:"; string username = "******"; string password = "******"; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create a CloudFileClient object for credentialed access to File storage. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference("secretdocuments"); // Ensure that the share exists. if (share.Exists()) { // Get a reference to the root directory for the share. CloudFileDirectory rootDir = share.GetRootDirectoryReference(); // Get a reference to the directory we created previously. CloudFileDirectory sampleDir = rootDir.GetDirectoryReference("CustomLogs"); // Ensure that the directory exists. if (sampleDir.Exists()) { // Get a reference to the file we created previously. CloudFile file = sampleDir.GetFileReference("Log1.txt"); // Ensure that the file exists. if (file.Exists()) { // Write the contents of the file to the console window. Console.WriteLine(file.DownloadTextAsync().Result); Console.WriteLine("Successfully connected to file share"); System.Threading.Thread.Sleep(300); } } if (!String.IsNullOrEmpty(driveLetterAndColon)) { // Make sure we aren't using this driveLetter for another mapping WNetCancelConnection2(driveLetterAndColon, 0, true); } NETRESOURCE nr = new NETRESOURCE(); nr.dwType = ResourceType.RESOURCETYPE_DISK; nr.lpRemoteName = shareName; nr.lpLocalName = driveLetterAndColon; int result = WNetAddConnection2(nr, password, username, 0); if (result != 0) { throw new Exception("WNetAddConnection2 failed with error " + result); } } }
public Task <bool> DirectoryExistsAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { return(TaskEx.FromResult(this.availableDirectoryNames.Contains(directory.Name))); }
public JsonResult UploadUserTaxReturn(int taxYear, string subFolder, string userId) { var curUserId = userId == null?User.Identity.GetUserId() : userId; bool uploadedSuccessfully = false; string uploadedURI = ""; using (var db = new WorktripEntities()) { try { var user = db.Users.FirstOrDefault(u => u.Id == curUserId); HttpPostedFileBase uploadedFile = Request.Files.Get(0); var compressedStream = uploadedFile.InputStream; if (uploadedFile.ContentType.StartsWith("image/")) { compressedStream = ResizePictureForBandwidth(uploadedFile); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = rootDir.GetDirectoryReference(user.FirstName + " " + user.LastName + " " + user.PhoneNumber); userDir.CreateIfNotExists(); if (!string.IsNullOrEmpty(subFolder)) { userDir = userDir.GetDirectoryReference(subFolder); userDir.CreateIfNotExists(); } var newFileName = uploadedFile.FileName; var fileExtension = Path.GetExtension(newFileName); CloudFile file = userDir.GetFileReference(newFileName); int fileDuplicateCount = 1; while (file.Exists()) { //generate a file name that doesn't exist yet newFileName = Path.GetFileNameWithoutExtension(newFileName) + "(" + (fileDuplicateCount++) + ")" + fileExtension; file = userDir.GetFileReference(newFileName);; } file.Properties.ContentType = uploadedFile.ContentType; file.UploadFromStream(compressedStream); uploadedURI = file.Uri.ToString(); UserTaxReturn newReturn = new UserTaxReturn() { UserId = curUserId, Date = DateTime.UtcNow, Path = uploadedURI, Year = taxYear }; db.UserTaxReturns.Add(newReturn); db.SaveChanges(); uploadedSuccessfully = true; UserInfoViewModel.UpdateUserActionsLog(curUserId, "uploaded " + taxYear + " " + (fileExtension == ".pdf" ? "tax return" : "tax form(s)")); } catch (Exception e) { //Do some error logging here.. uploadedSuccessfully = false; } } if (uploadedSuccessfully) { return(Json(new { status = 0 })); } else { return(Json(new { status = -1, message = "Error in saving file" })); } }
public void GetDocumentFolders(IDocumentFolderService service, IDocumentFileService fileService, DocumentFolderViewModel dir, bool recursive = false) { if (CancelOperation) { return; } if (dir != null) { CloudFileDirectory dirPath = GetDirectory(dir.Path); if (dirPath != null && dirPath.Exists()) { currentlyIndexedNumber++; IndexingDirectoryChanged?.Invoke(dir.Path, currentlyIndexedNumber); if (dir.Id < 1) { var response = service.Create(dir); if (response.Success) { dir.Id = response?.DocumentFolder?.Id ?? 0; } else { dir.Id = -1; } } if (dir.Id > 0) { dirPath.FetchAttributes(); var subFilesAndDirectories = dirPath.ListFilesAndDirectories(); var subFiles = subFilesAndDirectories.OfType <CloudFile>() .Select(x => new DocumentFileViewModel() { Identifier = Guid.NewGuid(), DocumentFolder = dir, Name = x.Name, Path = x.Uri.LocalPath, Company = new ServiceInterfaces.ViewModels.Common.Companies.CompanyViewModel() { Id = MainWindow.CurrentCompanyId }, CreatedBy = new ServiceInterfaces.ViewModels.Common.Identity.UserViewModel() { Id = MainWindow.CurrentUserId } }) .ToList(); subFiles.ForEach(x => GetDocumentAttributes(x)); var fileResponse = fileService.SubmitList(subFiles); var subDirectories = subFilesAndDirectories.OfType <CloudFileDirectory>() .Select(x => new DocumentFolderViewModel() { Path = x.Uri.LocalPath, Name = x.Name, Identifier = Guid.NewGuid(), ParentFolder = dir, Company = new ServiceInterfaces.ViewModels.Common.Companies.CompanyViewModel() { Id = MainWindow.CurrentCompanyId }, CreatedBy = new ServiceInterfaces.ViewModels.Common.Identity.UserViewModel() { Id = MainWindow.CurrentUserId } }) .ToList(); var response = service.SubmitList(subDirectories); if (response.Success) { dir.SubDirectories = new ObservableCollection <DocumentFolderViewModel>(response?.DocumentFolders ?? new List <DocumentFolderViewModel>()); if (recursive) { foreach (var item in dir.SubDirectories) { if (CancelOperation) { return; } GetDocumentFolders(service, fileService, item, recursive); } } } } } } }
public JsonResult GetSingleUserTaxReturn(string userId, int taxYear) { bool downloadSuccessful = true; var results = new List <Tuple <string, string, byte[]> >(); using (var db = new WorktripEntities()) { try { Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)"); var user = db.Users.FirstOrDefault(u => u.Id == userId); var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear); taxReturn = taxReturn.OrderBy(d => d.Id); var fileUrls = new List <UserTaxReturn>(); if (taxReturn.Count() != 0) { fileUrls.Add(taxReturn.AsEnumerable().Last()); byte[] bytes = new byte[64000]; var parsedFilePaths = new List <Tuple <string, string> >(); foreach (var url in fileUrls) { Match match = filePattern.Match(url.Path); if (match.Success) { var newTuple = new Tuple <string, string>( match.Groups["directory"].Value, match.Groups["filename"].Value ); parsedFilePaths.Add(newTuple); } } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = null; var userDirName = ""; foreach (var parsedPath in parsedFilePaths) { if (userDirName != parsedPath.Item1) { userDir = rootDir.GetDirectoryReference(parsedPath.Item1); if (!userDir.Exists()) { continue; } userDirName = parsedPath.Item1; } var filename = parsedPath.Item2; CloudFile file = userDir.GetFileReference(filename); if (!file.Exists()) { continue; } file.FetchAttributes(); string fileContents = ""; if (file.Properties.ContentType.ToLower() == "application/pdf") { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); bytes = fileStream.ToArray(); fileContents = ConvertStreamToBase64String(fileStream); } else { fileContents = file.DownloadText(); } results.Add( new Tuple <string, string, byte[]>(filename, file.Properties.ContentType, bytes) ); } } } catch (Exception e) { //Do some error logging here.. downloadSuccessful = false; } } if (downloadSuccessful && results.Count > 0) { return(Json(new MyJsonResult { status = 0, fileName = results.ElementAtOrDefault(0).Item1, fileContentType = results.ElementAtOrDefault(0).Item2, fileContents = results.ElementAtOrDefault(0).Item3 })); } else { return(Json(new { status = -1, message = "Error in downloading files" })); } }
private void ListFilesAndAssertResults(Action runCmdletAction, IListFileItem[] listedItems, CloudFileDirectory dir = null) { this.MockChannel.SetsEnumerationResults(dir == null ? string.Empty : dir.Name, listedItems); runCmdletAction(); this.MockCmdRunTime.OutputPipeline.AssertListFileItems(listedItems); }
public async Task <FileStreamResult> Get(int id) { var claimsIdentity = User.Identity as ClaimsIdentity; var userId = Convert.ToInt64(claimsIdentity.Claims.FirstOrDefault(claim => claim.Type == "Id").Value); var user = _multiSourcePlaylistRepository.GetUser(userId); var track = _multiSourcePlaylistRepository.GetTrack(id); byte[] audioArray = new byte[0]; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( _configuration["Production:StorageConnectionString"]); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference(user.FileFolder); CloudFileDirectory userDir = null; // Ensure that the share exists. if (await share.ExistsAsync()) { // Get a reference to the root directory for the share. CloudFileDirectory rootDir = share.GetRootDirectoryReference(); // Get a reference to the directory we created previously. userDir = rootDir.GetDirectoryReference("audio"); // Ensure that the directory exists. if (await userDir.ExistsAsync()) { var audiofile = userDir.GetFileReference(track.Address); if (await audiofile.ExistsAsync()) { await audiofile.FetchAttributesAsync(); audioArray = new byte[audiofile.Properties.Length]; await audiofile.DownloadToByteArrayAsync(audioArray, 0); } } else { return(null); } } else { return(null); } long fSize = audioArray.Length; long startbyte = 0; long endbyte = fSize - 1; int statusCode = 200; var rangeRequest = Request.Headers["Range"].ToString(); if (rangeRequest != "") { string[] range = Request.Headers["Range"].ToString().Split(new char[] { '=', '-' }); startbyte = Convert.ToInt64(range[1]); if (range.Length > 2 && range[2] != "") { endbyte = Convert.ToInt64(range[2]); } if (startbyte != 0 || endbyte != fSize - 1 || range.Length > 2 && range[2] == "") { statusCode = 206; } } long desSize = endbyte - startbyte + 1; Response.StatusCode = statusCode; Response.ContentType = "audio/mp3"; Response.Headers.Add("Content-Accept", Response.ContentType); Response.Headers.Add("Content-Length", desSize.ToString()); Response.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", startbyte, endbyte, fSize)); Response.Headers.Add("Accept-Ranges", "bytes"); Response.Headers.Remove("Cache-Control"); var stream = new MemoryStream(audioArray, (int)startbyte, (int)desSize); return(new FileStreamResult(stream, "audio/mp3") { FileDownloadName = track.Name }); }
private static async Task mainAsync(string[] args) { try { var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); IConfigurationRoot configuration = builder.Build(); var allCitDebtsLines = new string[10000000]; var allSettlementsLines = new string[10000000]; var allPaymentLines = new string[10000000]; while (true) { if (DateTime.Now.AddHours(3).ToShortTimeString() == "12:00 PM") { List <string> fileNames = new List <string>(); List <string[]> lstCloudFilesdata = new List <string[]>(); CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}"); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare fileShare = fileClient.GetShareReference("import"); //looks for a file share in the cloud bool fileShareExists = await fileShare.ExistsAsync(); if (fileShareExists) { List <CloudFile> lstCloudFiles = new List <CloudFile>(); CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference(); //for each file in my fileshare FileContinuationToken token = null; FileResultSegment k = await rootDir.ListFilesAndDirectoriesSegmentedAsync(token); token = k.ContinuationToken; //var context_ = new Db.Data.(); List <string> sl = new List <string>(); foreach (IListFileItem fiile in k.Results) { //if the file exists CloudFile file = (CloudFile)fiile; bool asd = await file.ExistsAsync(); if (asd) { //adds new datasting array sl = await ReadDataAsync(lstCloudFilesdata, file, fileNames); } foreach (string y in sl) { Console.WriteLine("From list new : " + y); } ; IEnumerable <CitizenDepts> o = from eachLine in ( from inner in sl select inner.Split(';') ) select new CitizenDepts { VAT = eachLine[0], FirstName = eachLine[1], LastName = eachLine[2], Email = eachLine[3], Phone = eachLine[4], Address = eachLine[5], County = eachLine[6], BillId = eachLine[7], Bill_description = eachLine[8], Amount = Decimal.Parse(eachLine[9]), DueDate = DateTime.ParseExact(eachLine[10], "yyyyMMdd", CultureInfo.InvariantCulture) }; foreach (var p in o) { Console.WriteLine(p.FirstName + " - " + p.BillId + ", - " + p.DueDate); } ; //string s = context_.Database.ProviderName; // Console.WriteLine(s); /// var all = from c in context_.CitizenDepts select c; //context_.CitizenDepts.RemoveRange(all); /// context_.SaveChanges(); foreach (var p in o) { //Add Student object into Students DBset //if (p.VAT!=null) //context_.Add(p); } //// call SaveChanges method to save student into database //context_.SaveChanges(); } } if (lstCloudFilesdata != null && fileNames != null) { ProccessData(lstCloudFilesdata, fileNames); } } } } catch (Exception ex) { Console.WriteLine(ex.Message); } finally { } }
public Task DeleteDirectoryAsync(CloudFileDirectory directory, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { return(TaskEx.FromResult(true)); }
/// <summary> /// Basic operations to work with Azure Files /// </summary> /// <returns>Task</returns> private static async Task BasicAzureFileOperationsAsync() { const string DemoShare = "hac2019"; const string DemoDirectory = "Upload"; const string FileToUpload = "Info.txt"; // Retrieve storage account information from connection string // How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create a file client for interacting with the file service. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Create a share for organizing files and directories within the storage account. Console.WriteLine("1. Creating file share"); CloudFileShare share = fileClient.GetShareReference(DemoShare); try { await share.CreateIfNotExistsAsync(); } catch (StorageException) { Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample."); Console.WriteLine("Press any key to exit"); //Console.ReadLine(); throw; } // Get a reference to the root directory of the share. CloudFileDirectory root = share.GetRootDirectoryReference(); // Create a directory under the root directory //Console.WriteLine("2. Creating a directory under the root directory"); CloudFileDirectory dir = root.GetDirectoryReference(DemoDirectory); await dir.CreateIfNotExistsAsync(); // Uploading a local file to the directory created above Console.WriteLine("3. Uploading a file to directory"); CloudFile file = dir.GetFileReference(FileToUpload); await file.UploadFromFileAsync(FileToUpload, FileMode.Open); // List all files/directories under the root directory Console.WriteLine("4. List Files/Directories in root directory"); List <IListFileItem> results = new List <IListFileItem>(); FileContinuationToken token = null; do { FileResultSegment resultSegment = await share.GetRootDirectoryReference().ListFilesAndDirectoriesSegmentedAsync(token); results.AddRange(resultSegment.Results); token = resultSegment.ContinuationToken; }while (token != null); // Print all files/directories listed above //foreach (IListFileItem listItem in results) //{ // // listItem type will be CloudFile or CloudFileDirectory // Console.WriteLine("- {0} (type: {1})", listItem.Uri, listItem.GetType()); //} // Download the uploaded file to your file system //Console.WriteLine("5. Download file from {0}", file.Uri.AbsoluteUri); //await file.DownloadToFileAsync(string.Format("./CopyOf{0}", FileToUpload), FileMode.Create); // Clean up after the demo //Console.WriteLine("6. Delete file"); //await file.DeleteAsync(); // When you delete a share it could take several seconds before you can recreate a share with the same // name - hence to enable you to run the demo in quick succession the share is not deleted. If you want // to delete the share uncomment the line of code below. // Console.WriteLine("7. Delete Share"); // await share.DeleteAsync(); }
public AzureServerSideCopyBackupProvider(CloudFileDirectory sourceDirectory, CloudFileDirectory targetDirectory) { this.sourceDirectory = sourceDirectory; this.targetDirectory = targetDirectory; }
public JsonResult GetUserDocuments(string userId, int taxYear, int?skip, int?amount) { bool downloadSuccessful = true; var results = new List <Tuple <string, string, string> >(); using (var db = new WorktripEntities()) { try { Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)"); var user = db.Users.FirstOrDefault(u => u.Id == userId); var miscDocs = db.UserMiscDocs.Where(d => d.UserId == userId && d.Year == taxYear); var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear); miscDocs = miscDocs.OrderBy(d => d.Id); taxReturn = taxReturn.OrderBy(d => d.Id); if (skip.HasValue) { miscDocs = miscDocs.Skip(skip.Value); taxReturn = taxReturn.Skip(skip.Value); } if (amount.HasValue) { miscDocs = miscDocs.Take(amount.Value); taxReturn = taxReturn.Take(amount.Value); } var fileUrls = miscDocs.Select(d => d.Path).ToList(); fileUrls.AddRange(taxReturn.Select(d => d.Path).ToList()); var parsedFilePaths = new List <Tuple <string, string> >(); foreach (var url in fileUrls) { Match match = filePattern.Match(url); if (match.Success) { var newTuple = new Tuple <string, string>( match.Groups["directory"].Value, match.Groups["filename"].Value ); parsedFilePaths.Add(newTuple); } } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = null; var userDirName = ""; foreach (var parsedPath in parsedFilePaths) { if (userDirName != parsedPath.Item1) { userDir = rootDir.GetDirectoryReference(parsedPath.Item1); if (!userDir.Exists()) { continue; } userDirName = parsedPath.Item1; } var filename = parsedPath.Item2; CloudFile file = userDir.GetFileReference(filename); if (!file.Exists()) { continue; } file.FetchAttributes(); string fileContents = ""; if (file.Properties.ContentType != null && file.Properties.ContentType.StartsWith("image/")) { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); fileContents = ConvertImageStreamToBase64String(fileStream); } else if (file.Properties.ContentType.ToLower() == "application/pdf") { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); fileContents = ConvertStreamToBase64String(fileStream); } else { fileContents = file.DownloadText(); } results.Add( new Tuple <string, string, string>(filename, file.Properties.ContentType, fileContents) ); } } catch (Exception e) { //Do some error logging here.. downloadSuccessful = false; } } if (downloadSuccessful) { return(Json(new { status = 0, files = results })); } else { return(Json(new { status = -1, message = "Error in downloading files" })); } }
public Task CreateDirectoryAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { return(directory.CreateAsync(options, operationContext, cancellationToken)); }
public void FileTypesWithStorageUri() { StorageUri endpoint = new StorageUri( new Uri("http://" + AccountName + FileService + EndpointSuffix), new Uri("http://" + AccountName + SecondarySuffix + FileService + EndpointSuffix)); CloudFileClient client = new CloudFileClient(endpoint, new StorageCredentials()); Assert.IsTrue(endpoint.Equals(client.StorageUri)); Assert.IsTrue(endpoint.PrimaryUri.Equals(client.BaseUri)); StorageUri shareUri = new StorageUri( new Uri(endpoint.PrimaryUri + "share"), new Uri(endpoint.SecondaryUri + "share")); CloudFileShare share = client.GetShareReference("share"); Assert.IsTrue(shareUri.Equals(share.StorageUri)); Assert.IsTrue(shareUri.PrimaryUri.Equals(share.Uri)); Assert.IsTrue(endpoint.Equals(share.ServiceClient.StorageUri)); share = new CloudFileShare(shareUri, client.Credentials); Assert.IsTrue(shareUri.Equals(share.StorageUri)); Assert.IsTrue(shareUri.PrimaryUri.Equals(share.Uri)); Assert.IsTrue(endpoint.Equals(share.ServiceClient.StorageUri)); StorageUri directoryUri = new StorageUri( new Uri(shareUri.PrimaryUri + "/directory"), new Uri(shareUri.SecondaryUri + "/directory")); StorageUri subdirectoryUri = new StorageUri( new Uri(directoryUri.PrimaryUri + "/subdirectory"), new Uri(directoryUri.SecondaryUri + "/subdirectory")); CloudFileDirectory directory = share.GetRootDirectoryReference().GetDirectoryReference("directory"); Assert.IsTrue(directoryUri.Equals(directory.StorageUri)); Assert.IsTrue(directoryUri.PrimaryUri.Equals(directory.Uri)); Assert.IsNotNull(directory.Parent); Assert.IsTrue(shareUri.Equals(directory.Share.StorageUri)); Assert.IsTrue(endpoint.Equals(directory.ServiceClient.StorageUri)); CloudFileDirectory subdirectory = directory.GetDirectoryReference("subdirectory"); Assert.IsTrue(subdirectoryUri.Equals(subdirectory.StorageUri)); Assert.IsTrue(subdirectoryUri.PrimaryUri.Equals(subdirectory.Uri)); Assert.IsTrue(directoryUri.Equals(subdirectory.Parent.StorageUri)); Assert.IsTrue(shareUri.Equals(subdirectory.Share.StorageUri)); Assert.IsTrue(endpoint.Equals(subdirectory.ServiceClient.StorageUri)); StorageUri fileUri = new StorageUri( new Uri(subdirectoryUri.PrimaryUri + "/file"), new Uri(subdirectoryUri.SecondaryUri + "/file")); CloudFile file = subdirectory.GetFileReference("file"); Assert.IsTrue(fileUri.Equals(file.StorageUri)); Assert.IsTrue(fileUri.PrimaryUri.Equals(file.Uri)); Assert.IsTrue(subdirectoryUri.Equals(file.Parent.StorageUri)); Assert.IsTrue(shareUri.Equals(file.Share.StorageUri)); Assert.IsTrue(endpoint.Equals(file.ServiceClient.StorageUri)); file = new CloudFile(fileUri, client.Credentials); Assert.IsTrue(fileUri.Equals(file.StorageUri)); Assert.IsTrue(fileUri.PrimaryUri.Equals(file.Uri)); Assert.IsTrue(subdirectoryUri.Equals(file.Parent.StorageUri)); Assert.IsTrue(shareUri.Equals(file.Share.StorageUri)); Assert.IsTrue(endpoint.Equals(file.ServiceClient.StorageUri)); }
public Task <bool> DirectoryExistsAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { return(directory.ExistsAsync(options, operationContext, cancellationToken)); }
/// <summary> /// Recursive delete of folder and all included subfolders or files. /// </summary> /// <param name="cloudFileDirectory">Directory to delete</param> public static void DeleteRecursive(this CloudFileDirectory cloudFileDirectory) { #region validation if (cloudFileDirectory == null) { throw new ArgumentNullException(nameof(cloudFileDirectory)); } #endregion if (!TaskUtilities.ExecuteSync(cloudFileDirectory.ExistsAsync())) { return; } var directoriesToBeDeleted = new List <CloudFileDirectory>(); var continuationToken = new FileContinuationToken(); // get first segment FileResultSegment fileResultSegment; do { fileResultSegment = TaskUtilities.ExecuteSync(cloudFileDirectory.ListFilesAndDirectoriesSegmentedAsync(continuationToken)); // iterate through items foreach (IListFileItem fileListItem in fileResultSegment.Results) { switch (fileListItem) { case CloudFile file: // Delete file directly TaskUtilities.ExecuteSync(file.DeleteAsync()); break; case CloudFileDirectory directory: // Add the current directory directoriesToBeDeleted.Add(directory); // List all sub directories recursively directoriesToBeDeleted.AddRange(directory.ListSubDirectories()); break; } } } while (fileResultSegment.ContinuationToken != null); // Sort directories bottom to top so the most deepest nested directories will be deleted first directoriesToBeDeleted.Sort((aDirectory, bDirectory) => bDirectory.Uri.LocalPath.Length.CompareTo(aDirectory.Uri.LocalPath.Length)); // Delete all found directories foreach (CloudFileDirectory cloudFileDirectoryToBeDeleted in directoriesToBeDeleted) { TaskUtilities.ExecuteSync(cloudFileDirectoryToBeDeleted.DeleteAsync()); } // Delete the parent directory itself in case it´s not the root directory (which is the file share) if (cloudFileDirectory.SnapshotQualifiedUri != cloudFileDirectory.Share.GetRootDirectoryReference().SnapshotQualifiedUri) { TaskUtilities.ExecuteSync(cloudFileDirectory.DeleteAsync()); } }
public Task DeleteDirectoryAsync(CloudFileDirectory directory, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) { return(directory.DeleteAsync(accessCondition, options, operationContext, cancellationToken)); }
public AzureStorageFileSystem(CloudFileDirectory directory) { _directory = directory; }
private IEnumerable <TransferEntry> EnumerateLocationRecursive(CancellationToken cancellationToken) { string fullPrefix = null; if (null != this.baseDirectory) { fullPrefix = Uri.UnescapeDataString(this.baseDirectory.SnapshotQualifiedUri.AbsolutePath); } else { fullPrefix = Uri.UnescapeDataString(this.location.FileDirectory.SnapshotQualifiedUri.AbsolutePath); } // Normalize full prefix to end with slash. if (!string.IsNullOrEmpty(fullPrefix) && !fullPrefix.EndsWith("/", StringComparison.OrdinalIgnoreCase)) { fullPrefix += '/'; } CloudFileDirectory directory = this.location.FileDirectory; Stack <CloudFileDirectory> innerDirList = new Stack <CloudFileDirectory>(); FileContinuationToken continuationToken = null; bool passedContinuationToken = false; if (null == this.listContinuationToken) { passedContinuationToken = true; } do { FileResultSegment resultSegment = null; Utils.CheckCancellation(cancellationToken); ErrorEntry errorEntry = null; try { FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions; resultSegment = directory.ListFilesAndDirectoriesSegmentedAsync( ListFilesSegmentSize, continuationToken, requestOptions, null, cancellationToken).GetAwaiter().GetResult(); } catch (Exception ex) { string errorMessage = string.Format( CultureInfo.CurrentCulture, Resources.FailedToEnumerateDirectory, directory.SnapshotQualifiedUri.AbsoluteUri, string.Empty); TransferException exception = new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex); errorEntry = new ErrorEntry(exception); } if (null != errorEntry) { yield return(errorEntry); yield break; } continuationToken = resultSegment.ContinuationToken; foreach (IListFileItem fileItem in resultSegment.Results) { Utils.CheckCancellation(cancellationToken); if (fileItem is CloudFileDirectory) { CloudFileDirectory cloudDir = fileItem as CloudFileDirectory; if (!passedContinuationToken) { if (string.Equals(cloudDir.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal)) { passedContinuationToken = true; continue; } else { continue; } } string fullPath = Uri.UnescapeDataString(cloudDir.SnapshotQualifiedUri.AbsolutePath); string relativePath = fullPath.Remove(0, fullPrefix.Length); yield return(new AzureFileDirectoryEntry( relativePath, cloudDir, new AzureFileListContinuationToken(cloudDir.Name))); } else if (fileItem is CloudFile) { CloudFile cloudFile = fileItem as CloudFile; if (!passedContinuationToken) { if (string.Equals(cloudFile.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal)) { passedContinuationToken = true; continue; } else { continue; } } string fullPath = Uri.UnescapeDataString(cloudFile.SnapshotQualifiedUri.AbsolutePath); string relativePath = fullPath.Remove(0, fullPrefix.Length); yield return(new AzureFileEntry( relativePath, cloudFile, new AzureFileListContinuationToken(cloudFile.Name))); } } }while (continuationToken != null); }
/** * Azure Batch AI sample. * - Create Storage account and Azure file share * - Upload sample data to Azure file share * - Create a workspace and experiment * - Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job * - Create Microsoft Cognitive Toolkit job to run on the cluster * - Wait for job to complete * - Get output files * * Please note: in order to run this sample, please download and unzip sample package from <a href="https://batchaisamples.blob.core.windows.net/samples/BatchAIQuickStart.zip?st=2017-09-29T18%3A29%3A00Z&se=2099-12-31T08%3A00%3A00Z&sp=rl&sv=2016-05-31&sr=b&sig=hrAZfbZC%2BQ%2FKccFQZ7OC4b%2FXSzCF5Myi4Cj%2BW3sVZDo%3D">here</a> * Export path to the content to $SAMPLE_DATA_PATH. */ public static void RunSample(IAzure azure) { string saName = SdkContext.RandomResourceName("sa", 10); string rgName = SdkContext.RandomResourceName("rg", 10); string workspaceName = SdkContext.RandomResourceName("ws", 10); string experimentName = SdkContext.RandomResourceName("exp", 10); string sampleDataPath = Environment.GetEnvironmentVariable("SAMPLE_DATA_PATH"); Region region = Region.USWest2; string shareName = SdkContext.RandomResourceName("fs", 20); string clusterName = SdkContext.RandomResourceName("cluster", 15); string userName = Utilities.CreateUsername(); string sharePath = "mnistcntksample"; try { //============================================================= // Create a new storage account and an Azure file share resource Utilities.Log("Creating a storage account..."); IStorageAccount storageAccount = azure.StorageAccounts.Define(saName) .WithRegion(region) .WithNewResourceGroup(rgName) .Create(); Utilities.Log("Created storage account."); Utilities.PrintStorageAccount(storageAccount); StorageAccountKey storageAccountKey = storageAccount.GetKeys().First(); Utilities.Log("Creating Azure File share..."); var cloudFileShare = CloudStorageAccount.Parse($"DefaultEndpointsProtocol=https;AccountName={saName};AccountKey={storageAccountKey.Value};EndpointSuffix=core.windows.net") .CreateCloudFileClient() .GetShareReference(shareName); cloudFileShare.CreateAsync().GetAwaiter().GetResult(); Utilities.Log("Created Azure File share."); //============================================================= // Upload sample data to Azure file share //Get a reference to the root directory for the share. CloudFileDirectory rootDir = cloudFileShare.GetRootDirectoryReference(); //Get a reference to the sampledir directory Utilities.Log("Creating directory and uploading data files..."); CloudFileDirectory sampleDir = rootDir.GetDirectoryReference(sharePath); sampleDir.CreateAsync().GetAwaiter().GetResult(); rootDir.GetFileReference("Train-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Train-28x28_cntk_text.txt").GetAwaiter().GetResult(); rootDir.GetFileReference("Test-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Test-28x28_cntk_text.txt").GetAwaiter().GetResult(); rootDir.GetFileReference("ConvNet_MNIST.py").UploadFromFileAsync(sampleDataPath + "/ConvNet_MNIST.py").GetAwaiter().GetResult(); Utilities.Log("Data files uploaded."); //============================================================= // Create a workspace and experiment IBatchAIWorkspace workspace = azure.BatchAIWorkspaces.Define(workspaceName) .WithRegion(region) .WithNewResourceGroup(rgName) .Create(); IBatchAIExperiment experiment = workspace.CreateExperiment(experimentName); //============================================================= // Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job Utilities.Log("Creating Batch AI cluster..."); IBatchAICluster cluster = workspace.Clusters.Define(clusterName) .WithVMSize(VirtualMachineSizeTypes.StandardNC6.Value) .WithUserName(userName) .WithPassword("MyPassword") .WithAutoScale(0, 2) .DefineAzureFileShare() .WithStorageAccountName(saName) .WithAzureFileUrl(cloudFileShare.Uri.ToString()) .WithRelativeMountPath("azurefileshare") .WithAccountKey(storageAccountKey.Value) .Attach() .Create(); Utilities.Log("Created Batch AI cluster."); Utilities.Print(cluster); // ============================================================= // Create Microsoft Cognitive Toolkit job to run on the cluster Utilities.Log("Creating Batch AI job..."); IBatchAIJob job = experiment.Jobs.Define("myJob") .WithExistingCluster(cluster) .WithNodeCount(1) .WithStdOutErrPathPrefix("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare") .DefineCognitiveToolkit() .WithPythonScriptFile("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/ConvNet_MNIST.py") .WithCommandLineArgs("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare $AZ_BATCHAI_OUTPUT_MODEL") .Attach() .WithOutputDirectory("MODEL", "$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/model") .WithContainerImage("microsoft/cntk:2.1-gpu-python3.5-cuda8.0-cudnn6.0") .Create(); Utilities.Log("Created Batch AI job."); Utilities.Print(job); // ============================================================= // Wait for job results // Wait for job to start running Utilities.Log("Waiting for Batch AI job to start running..."); while (ExecutionState.Queued.Equals(job.ExecutionState)) { SdkContext.DelayProvider.Delay(5000); job.Refresh(); } // Wait for job to complete and job output to become available Utilities.Log("Waiting for Batch AI job to complete..."); while (!(ExecutionState.Succeeded.Equals(job.ExecutionState) || ExecutionState.Failed.Equals(job.ExecutionState))) { SdkContext.DelayProvider.Delay(5000); job.Refresh(); } // ============================================================= // Get output files // Print stdout and stderr foreach (var outputFile in job.ListFiles("stdouterr")) { Utilities.Log(Utilities.CheckAddress(outputFile.DownloadUrl)); } // List model output files foreach (var outputFile in job.ListFiles("MODEL")) { Utilities.Log(outputFile.DownloadUrl); } } finally { try { Utilities.Log("Deleting Resource Group: " + rgName); azure.ResourceGroups.BeginDeleteByName(rgName); Utilities.Log("Deleted Resource Group: " + rgName); } catch (Exception) { Utilities.Log("Did not create any resources in Azure. No clean up is necessary"); } } }
/// <summary> /// Initializes a new instance of the <see cref="AzureFileLocation" /> class. /// </summary> /// <param name="location">Azure file directory location.</param> /// <param name="baseDirectory"></param> public AzureFileHierarchyEnumerator(AzureFileDirectoryLocation location, CloudFileDirectory baseDirectory) { this.location = location; this.baseDirectory = baseDirectory; }
public override void ExecuteCmdlet() { CloudFileDirectory baseDirectory; switch (this.ParameterSetName) { case Constants.DirectoryParameterSetName: baseDirectory = this.Directory; break; case Constants.ShareNameParameterSetName: baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference(); break; case Constants.ShareParameterSetName: baseDirectory = this.Share.GetRootDirectoryReference(); break; default: throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName)); } if (string.IsNullOrEmpty(this.Path)) { this.RunTask(async(taskId) => { await this.Channel.EnumerateFilesAndDirectoriesAsync( baseDirectory, item => this.OutputStream.WriteObject(taskId, item), this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false); }); } else { this.RunTask(async(taskId) => { bool foundAFolder = true; string[] subfolders = NamingUtil.ValidatePath(this.Path); CloudFileDirectory targetDir = baseDirectory.GetDirectoryReferenceByPath(subfolders); try { await this.Channel.FetchDirectoryAttributesAsync( targetDir, null, this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false); } catch (StorageException se) { if (null == se.RequestInformation || (int)HttpStatusCode.NotFound != se.RequestInformation.HttpStatusCode) { throw; } foundAFolder = false; } if (foundAFolder) { this.OutputStream.WriteObject(taskId, targetDir); return; } string[] filePath = NamingUtil.ValidatePath(this.Path, true); CloudFile targetFile = baseDirectory.GetFileReferenceByPath(filePath); await this.Channel.FetchFileAttributesAsync( targetFile, null, this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false); this.OutputStream.WriteObject(taskId, targetFile); }); } }
private static void CreateCloudFileDestinationDirectory(CloudFileDirectory fileDirectory, CloudFileNtfsAttributes?fileAttributes, DateTimeOffset?creationTime, DateTimeOffset?lastWriteTime, IDictionary <string, string> metadata, CancellationToken cancellationToken) { bool parentNotExist = false; if (fileAttributes.HasValue) { fileDirectory.Properties.NtfsAttributes = fileAttributes.Value; fileDirectory.Properties.CreationTime = creationTime; fileDirectory.Properties.LastWriteTime = lastWriteTime; } if (null != metadata) { fileDirectory.Metadata.Clear(); foreach (var keyValuePair in metadata) { fileDirectory.Metadata.Add(keyValuePair); } } bool needToSetProperties = false; try { fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult(); return; } catch (StorageException ex) { if (null != ex.RequestInformation) { if (string.Equals("ParentNotFound", ex.RequestInformation.ErrorCode)) { parentNotExist = true; } else if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode)) { needToSetProperties = true; } else { throw; } } else { throw; } } if (parentNotExist) { Utils.CreateCloudFileDirectoryRecursively(fileDirectory.Parent); try { fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult(); } catch (StorageException ex) { if (null != ex.RequestInformation) { if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode)) { needToSetProperties = true; } else { throw; } } else { throw; } } } if (needToSetProperties) { SetAzureFileDirectoryAttributes(fileDirectory, fileAttributes, creationTime, lastWriteTime, metadata, cancellationToken); } }