/// <inheritdoc />
public IEnumerable <string> GetFiles(string[] path = null)
{
CloudFileDirectory cloudFileDirectory = CloudFileShare.GetDirectoryReference(path: path);
return(cloudFileDirectory.EnumerateDirectory(EFileType.Files, "*"));
}
/// <summary>
/// Write CloudFileDirectory to output using specified service channel
/// </summary>
/// <param name="fileDir">The output CloudFileDirectory object</param>
/// <param name="channel">IStorageFileManagement channel object</param>
internal void WriteCloudFileDirectoryeObject(long taskId, IStorageFileManagement channel, CloudFileDirectory fileDir)
{
AzureStorageFileDirectory azureFileDir = new AzureStorageFileDirectory(fileDir, channel.StorageContext);
OutputStream.WriteObject(taskId, azureFileDir);
}
private async Task <CloudFile> BuildCloudFileInstanceFromPathAsync(string defaultFileName, string[] path, bool pathIsDirectory)
{
CloudFileDirectory baseDirectory = null;
bool isPathEmpty = path.Length == 0;
switch (this.ParameterSetName)
{
case LocalConstants.DirectoryParameterSetName:
baseDirectory = this.Directory;
break;
case LocalConstants.ShareNameParameterSetName:
NamingUtil.ValidateShareName(this.ShareName, false);
baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference();
break;
case LocalConstants.ShareParameterSetName:
baseDirectory = this.Share.GetRootDirectoryReference();
break;
default:
throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName));
}
if (isPathEmpty)
{
return(baseDirectory.GetFileReference(defaultFileName));
}
var directory = baseDirectory.GetDirectoryReferenceByPath(path);
if (pathIsDirectory)
{
return(directory.GetFileReference(defaultFileName));
}
bool directoryExists;
try
{
directoryExists = await this.Channel.DirectoryExistsAsync(directory, this.RequestOptions, this.OperationContext, this.CmdletCancellationToken).ConfigureAwait(false);
}
catch (StorageException e)
{
if (e.RequestInformation != null &&
e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden)
{
//Forbidden to check directory existance, might caused by a write only SAS
//Don't report error here since should not block upload with write only SAS
//If the directory not exist, Error will be reported when upload with DMlib later
directoryExists = true;
}
else
{
if (e.RequestInformation != null &&
e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.BadRequest &&
e.RequestInformation.ExtendedErrorInformation == null)
{
throw new AzureStorageFileException(ErrorCategory.InvalidArgument, ErrorIdConstants.InvalidResource, Resources.InvalidResource, this);
}
throw;
}
}
if (directoryExists)
{
// If the directory exist on the cloud, we treat the path as
// to a directory. So we append the default file name after
// it and build an instance of CloudFile class.
return(directory.GetFileReference(defaultFileName));
}
else
{
// If the directory does not exist, we treat the path as to a
// file. So we use the path of the directory to build out a
// new instance of CloudFile class.
return(baseDirectory.GetFileReferenceByPath(path));
}
}
public static void addRecordToAzureFileShare(string myQueueItem, string fileName, ILogger log)
{
try
{
string valueLine = string.Empty;
string clientHeader = string.Empty;
var searcResult = new string[10];
StringBuilder builder = new StringBuilder();
StringBuilder headerBuilder = new StringBuilder();
StringBuilder valueBuilder = new StringBuilder();
string[] arrQueueItem = myQueueItem.Split("||", StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < arrQueueItem.Length; i++)
{
var arrHeader = arrQueueItem[i]?.Split(":", StringSplitOptions.RemoveEmptyEntries);
if (!string.IsNullOrEmpty(arrHeader[0]) && !arrHeader[0].Trim().Equals("CampaignName"))
{
string header = arrHeader[0].Replace(",", "-");
string value = arrHeader.Length > 1 ? arrHeader[1].Replace(",", "-") : ""; //Replace comma with hyphen in values
headerBuilder.Append(header + ",");
valueBuilder.Append(value + ",");
}
}
var headerLastIndex = headerBuilder.ToString().LastIndexOf(",");
if (headerLastIndex >= 0)
{
clientHeader = headerBuilder.ToString().Remove(headerLastIndex, 1) + Environment.NewLine;
}
builder.Append(clientHeader);
var valueLastIndex = valueBuilder.ToString().LastIndexOf(",");
if (valueLastIndex >= 0)
{
valueLine = valueBuilder.ToString().Remove(valueLastIndex, 1);
}
builder.Append(valueLine);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetEnvironmentVariable("AzureWebJobsStorage"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference(GetEnvironmentVariable("AzureStorageFileShare"));
string existingFiletxt = "";
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(GetEnvironmentVariable("AzureStorageFileDirectory"));
if (customDirectory.Exists())
{
CloudFile file = customDirectory.GetFileReference(fileName + ".csv");
if (file.Exists())
{
existingFiletxt = file.DownloadTextAsync().Result;
string existingHeader = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)?[0];
string existingValue = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1];
if (existingHeader + Environment.NewLine == clientHeader)
{
customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine);
}
else
{
existingFiletxt = clientHeader + existingValue;
customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine.ToString());
}
}
else
{
customDirectory.GetFileReference(fileName + ".csv").UploadText(builder.ToString());
}
}
}
}
}
catch (Exception ex)
{
log.LogInformation("Some error occured " + ex);
}
}
/// <summary>
/// Enumerates a specified cloud file directory and returns items based item type and searchPattern.
/// </summary>
/// <param name="cloudFileDirectory">Cloud file directory to enumerate on</param>
/// <param name="fileType">Item type to filter with</param>
/// <param name="searchPattern">Search pattern to filter with</param>
/// <returns>Enumerable of strings which contains all found names based on file type</returns>
public static IEnumerable <string> EnumerateDirectory(this CloudFileDirectory cloudFileDirectory, EFileType fileType, string searchPattern)
{
#region validation
if (cloudFileDirectory == null)
{
throw new ArgumentNullException(nameof(cloudFileDirectory));
}
if (string.IsNullOrEmpty(searchPattern))
{
throw new ArgumentNullException(nameof(searchPattern));
}
if (!TaskUtilities.ExecuteSync(cloudFileDirectory.ExistsAsync()))
{
return(new List <string>());
}
#endregion
var directoryNames = new List <string>();
var fileContinuationToken = new FileContinuationToken();
do
{
FileResultSegment fileResultSegment = TaskUtilities.ExecuteSync(cloudFileDirectory.ListFilesAndDirectoriesSegmentedAsync(fileContinuationToken));
foreach (IListFileItem listFileItem in fileResultSegment.Results)
{
string fullListItemPath = string.Empty;
switch (fileType)
{
case EFileType.Files:
if (listFileItem is CloudFile cloudFile)
{
fullListItemPath = cloudFile.StorageUri.PrimaryUri.ToString();
}
break;
case EFileType.Directories:
if (listFileItem is CloudFileDirectory pendingCloudFileDirectory)
{
fullListItemPath = pendingCloudFileDirectory.StorageUri.PrimaryUri.ToString();
if (fullListItemPath.EndsWith('/'))
{
fullListItemPath = fullListItemPath.Remove(fullListItemPath.LastIndexOf('/'));
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(fileType), fileType, null);
}
// replace wildcard for 'multiple letters' with regex-wildcard
string regexableSearchPattern = searchPattern.Replace("*", ".*");
// replace wildcard for 'single letter' with regex-wildcard
regexableSearchPattern = regexableSearchPattern.Replace("?", ".?");
// set search pattern in 'begin-to-end'-symbols
regexableSearchPattern = $"^{regexableSearchPattern}$";
var rgxFileName = new Regex(regexableSearchPattern);
if (!string.IsNullOrEmpty(fullListItemPath) && rgxFileName.IsMatch(fullListItemPath))
{
directoryNames.Add(fullListItemPath);
}
}
fileContinuationToken = fileResultSegment.ContinuationToken;
} while (fileContinuationToken != null);
return(directoryNames);
}
public async Task <ActionResult> DoFileUploads(HttpPostedFileBase[] files)
{
// Step 1: Intialize the Azure Configuration form azure settings config
var azureconfiguration = ConfigInit.getConfiguration();
// Step 2: Intialize the Azure DataLake Storage
var storageAccountConnectionString_File = CloudStorageAccount.Parse(azureconfiguration.AzureStorageConnectionString);
CloudFileClient cloudFileClient = storageAccountConnectionString_File.CreateCloudFileClient();
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(azureconfiguration.AzureFileStorageReference);
CloudFileDirectory rootDir = cloudFileShare.GetRootDirectoryReference();
CloudFile fileSas = null;
// Step 3: Intailize the API Client
HttpClient client = new HttpClient();
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", azureconfiguration.AzureVisionAPISubscriptionKey);
HttpResponseMessage response;
// Step 4: Intailize the Cosmos DB
var cosmosDBResponse = new CosmosAPIService();
DocumentClient documentClient = new DocumentClient(new Uri(azureconfiguration.AzureCosmosDBConnectionString), azureconfiguration.AzureCosmonDBPrimaryKey1);
var createDataBaseResponse = await documentClient.CreateDatabaseIfNotExistsAsync(new Microsoft.Azure.Documents.Database {
Id = "CognitiveAPIDemoDataSource"
});
var createDataBaseCollectionResponse = await documentClient.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri("CognitiveAPIDemoDataSource"), new DocumentCollection { Id = "PocCollection" }, new RequestOptions { OfferThroughput = 1000 });
List <APIData> objrApiData = new List <APIData>();
for (int i = 0; i < files.Length; i++)
{
if (files[i].FileName.EndsWith(".jpg") || files[i].FileName.EndsWith(".png"))
{
bytesFromImage = getbyteFromInputStream(files[i].InputStream, files[i].ContentLength);
if (cloudFileShare.Exists())
{
fileSas = rootDir.GetFileReference(Path.GetFileName(files[i].FileName));
fileSas.UploadFromStream(new MemoryStream(bytesFromImage));
}
// Extracting Image :
using (ByteArrayContent content = new ByteArrayContent(bytesFromImage))
{
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
response = await client.PostAsync(azureconfiguration.AzureVisionAPIEndPoint + "?" + "mode=Handwritten", content);
}
if (response.IsSuccessStatusCode)
{
string operationLocation = response.Headers.GetValues("Operation-Location").FirstOrDefault();
string contentString;
int res = 0;
do
{
System.Threading.Thread.Sleep(1000);
response = await client.GetAsync(operationLocation);
contentString = await response.Content.ReadAsStringAsync();
++res;
}while (res < 10 && contentString.IndexOf("\"status\":\"Succeeded\"") == -1);
if (res == 10 && contentString.IndexOf("\"status\":\"Succeeded\"") == -1)
{
//return null;
}
var rootobject = JsonConvert.DeserializeObject <RootObject>(contentString);
StringBuilder builder = new StringBuilder();
for (int s = 0; s < rootobject.recognitionResult.lines.Count; s++)
{
builder.Append(rootobject.recognitionResult.lines[s].text.ToString());
}
objrApiData.Add(new APIData()
{
OCRID = Guid.NewGuid().ToString(),
imageData = bytesFromImage,
imageText = builder.ToString(),
ImageUrl = Path.GetFileName(files[i].FileName),//fileSas.Uri.AbsoluteUri.ToString(),
pid = i,
sid = "text" + i,
Error = "no Error",
Remarks = "No Remarks Found"
});
}
else
{
string errorString = await response.Content.ReadAsStringAsync();
var errorObject = JsonConvert.DeserializeObject <VisionAPIErrorMessage>(errorString);
objrApiData.Add(new APIData()
{
OCRID = Guid.NewGuid().ToString(),
imageData = bytesFromImage,
imageText = errorObject.error.message.ToString(),
ImageUrl = Path.GetFileName(files[i].FileName),//fileSas.Uri.AbsoluteUri.ToString(),
pid = i,
sid = "text" + i,
Error = errorObject.error.message.ToString(),
Remarks = "Image scanning Failed"
});
}
}
}
var dbProcessingResponse = await cosmosDBResponse.CreateDocumentCollection(objrApiData, documentClient);
TempData["objrApiData"] = objrApiData;
return(RedirectToAction("ShowResults"));
}
public override void ExecuteCmdlet()
{
if (this.ShouldProcess(string.Format("Close File Handles for File or FileDirectory on Path: {0}", this.Path != null? this.Path : (this.FileHandle != null? this.FileHandle.Path: null)), "This operation will force the provided file handle(s) closed, which may cause data loss or corruption for active applications/users.", null))
{
CloudFileDirectory baseDirectory = null;
switch (this.ParameterSetName)
{
case DirectoryCloseAllParameterSetName:
baseDirectory = this.Directory;
break;
case ShareNameCloseSingleParameterSetName:
case ShareNameCloseAllParameterSetName:
baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference();
break;
case ShareCloseSingleParameterSetName:
case ShareCloseAllParameterSetName:
baseDirectory = this.Share.GetRootDirectoryReference();
break;
case FileCloseAllParameterSetName:
// Don't need to set baseDirectory when input is a CloudFile
break;
default:
throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName));
}
if (ParameterSetName == ShareNameCloseSingleParameterSetName || ParameterSetName == ShareCloseSingleParameterSetName)
{
this.Path = FileHandle.Path;
}
// When not input path/File, the list handle target must be a Dir
bool foundAFolder = true;
CloudFileDirectory targetDir = baseDirectory;
CloudFile targetFile = null;
if (this.File != null)
{
targetFile = this.File;
foundAFolder = false;
}
else
{
if (!string.IsNullOrEmpty(this.Path))
{
string[] subfolders = NamingUtil.ValidatePath(this.Path);
targetDir = baseDirectory.GetDirectoryReferenceByPath(subfolders);
if (!targetDir.Exists())
{
foundAFolder = false;
}
if (!foundAFolder)
{
//Get file
string[] filePath = NamingUtil.ValidatePath(this.Path, true);
targetFile = baseDirectory.GetFileReferenceByPath(filePath);
this.Channel.FetchFileAttributesAsync(
targetFile,
null,
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken).ConfigureAwait(false);
}
}
}
// Recursive only take effect on File Dir
if (!foundAFolder && Recursive.IsPresent)
{
WriteVerbose("The target object of the 'Path' is an Azure File, the parameter '-Recursive' won't take effect.");
}
//Close handle
CloseFileHandleResultSegment closeResult = null;
FileContinuationToken continuationToken = null;
int numHandlesClosed = 0;
do
{
if (foundAFolder)
{
if (FileHandle != null)
{
// close single handle on fileDir
if (this.FileHandle.HandleId == null)
{
throw new System.ArgumentException(string.Format("The HandleId of the FileHandle on path {0} should not be null.", this.FileHandle.Path), "FileHandle");
}
closeResult = targetDir.CloseHandleSegmented(this.FileHandle.HandleId.ToString(), continuationToken, Recursive, null, this.RequestOptions, this.OperationContext);
}
else
{
// close all handle on fileDir
closeResult = targetDir.CloseAllHandlesSegmented(continuationToken, Recursive, null, this.RequestOptions, this.OperationContext);
}
}
else
{
if (FileHandle != null)
{
// close single handle on file
if (this.FileHandle.HandleId == null)
{
throw new System.ArgumentException(string.Format("The HandleId of the FileHandle on path {0} should not be null.", this.FileHandle.Path), "FileHandle");
}
closeResult = targetFile.CloseHandleSegmented(this.FileHandle.HandleId.ToString(), continuationToken, null, this.RequestOptions, this.OperationContext);
}
else
{
// close all handle on file
closeResult = targetFile.CloseAllHandlesSegmented(continuationToken, null, this.RequestOptions, this.OperationContext);
}
}
numHandlesClosed += closeResult.NumHandlesClosed;
continuationToken = closeResult.ContinuationToken;
} while (continuationToken != null && continuationToken.NextMarker != null);
if (PassThru)
{
WriteObject(numHandlesClosed);
}
}
}
static void Main(string[] args)
{
string shareName = "\\\\unipertest.file.core.windows.net\\SecretDocuments";
string driveLetterAndColon = "z:";
string username = "******";
string password = "******";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a CloudFileClient object for credentialed access to File storage.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference("secretdocuments");
// Ensure that the share exists.
if (share.Exists())
{
// Get a reference to the root directory for the share.
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
// Get a reference to the directory we created previously.
CloudFileDirectory sampleDir = rootDir.GetDirectoryReference("CustomLogs");
// Ensure that the directory exists.
if (sampleDir.Exists())
{
// Get a reference to the file we created previously.
CloudFile file = sampleDir.GetFileReference("Log1.txt");
// Ensure that the file exists.
if (file.Exists())
{
// Write the contents of the file to the console window.
Console.WriteLine(file.DownloadTextAsync().Result);
Console.WriteLine("Successfully connected to file share");
System.Threading.Thread.Sleep(300);
}
}
if (!String.IsNullOrEmpty(driveLetterAndColon))
{
// Make sure we aren't using this driveLetter for another mapping
WNetCancelConnection2(driveLetterAndColon, 0, true);
}
NETRESOURCE nr = new NETRESOURCE();
nr.dwType = ResourceType.RESOURCETYPE_DISK;
nr.lpRemoteName = shareName;
nr.lpLocalName = driveLetterAndColon;
int result = WNetAddConnection2(nr, password, username, 0);
if (result != 0)
{
throw new Exception("WNetAddConnection2 failed with error " + result);
}
}
}
public Task <bool> DirectoryExistsAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(TaskEx.FromResult(this.availableDirectoryNames.Contains(directory.Name)));
}
public JsonResult UploadUserTaxReturn(int taxYear, string subFolder, string userId)
{
var curUserId = userId == null?User.Identity.GetUserId() : userId;
bool uploadedSuccessfully = false;
string uploadedURI = "";
using (var db = new WorktripEntities())
{
try
{
var user = db.Users.FirstOrDefault(u => u.Id == curUserId);
HttpPostedFileBase uploadedFile = Request.Files.Get(0);
var compressedStream = uploadedFile.InputStream;
if (uploadedFile.ContentType.StartsWith("image/"))
{
compressedStream = ResizePictureForBandwidth(uploadedFile);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = rootDir.GetDirectoryReference(user.FirstName + " " + user.LastName + " " + user.PhoneNumber);
userDir.CreateIfNotExists();
if (!string.IsNullOrEmpty(subFolder))
{
userDir = userDir.GetDirectoryReference(subFolder);
userDir.CreateIfNotExists();
}
var newFileName = uploadedFile.FileName;
var fileExtension = Path.GetExtension(newFileName);
CloudFile file = userDir.GetFileReference(newFileName);
int fileDuplicateCount = 1;
while (file.Exists())
{
//generate a file name that doesn't exist yet
newFileName = Path.GetFileNameWithoutExtension(newFileName) + "(" + (fileDuplicateCount++) + ")" + fileExtension;
file = userDir.GetFileReference(newFileName);;
}
file.Properties.ContentType = uploadedFile.ContentType;
file.UploadFromStream(compressedStream);
uploadedURI = file.Uri.ToString();
UserTaxReturn newReturn = new UserTaxReturn()
{
UserId = curUserId,
Date = DateTime.UtcNow,
Path = uploadedURI,
Year = taxYear
};
db.UserTaxReturns.Add(newReturn);
db.SaveChanges();
uploadedSuccessfully = true;
UserInfoViewModel.UpdateUserActionsLog(curUserId, "uploaded " + taxYear + " " + (fileExtension == ".pdf" ? "tax return" : "tax form(s)"));
}
catch (Exception e)
{
//Do some error logging here..
uploadedSuccessfully = false;
}
}
if (uploadedSuccessfully)
{
return(Json(new { status = 0 }));
}
else
{
return(Json(new { status = -1, message = "Error in saving file" }));
}
}
public void GetDocumentFolders(IDocumentFolderService service, IDocumentFileService fileService, DocumentFolderViewModel dir, bool recursive = false)
{
if (CancelOperation)
{
return;
}
if (dir != null)
{
CloudFileDirectory dirPath = GetDirectory(dir.Path);
if (dirPath != null && dirPath.Exists())
{
currentlyIndexedNumber++;
IndexingDirectoryChanged?.Invoke(dir.Path, currentlyIndexedNumber);
if (dir.Id < 1)
{
var response = service.Create(dir);
if (response.Success)
{
dir.Id = response?.DocumentFolder?.Id ?? 0;
}
else
{
dir.Id = -1;
}
}
if (dir.Id > 0)
{
dirPath.FetchAttributes();
var subFilesAndDirectories = dirPath.ListFilesAndDirectories();
var subFiles = subFilesAndDirectories.OfType <CloudFile>()
.Select(x => new DocumentFileViewModel()
{
Identifier = Guid.NewGuid(),
DocumentFolder = dir,
Name = x.Name,
Path = x.Uri.LocalPath,
Company = new ServiceInterfaces.ViewModels.Common.Companies.CompanyViewModel()
{
Id = MainWindow.CurrentCompanyId
},
CreatedBy = new ServiceInterfaces.ViewModels.Common.Identity.UserViewModel()
{
Id = MainWindow.CurrentUserId
}
})
.ToList();
subFiles.ForEach(x => GetDocumentAttributes(x));
var fileResponse = fileService.SubmitList(subFiles);
var subDirectories = subFilesAndDirectories.OfType <CloudFileDirectory>()
.Select(x => new DocumentFolderViewModel()
{
Path = x.Uri.LocalPath, Name = x.Name, Identifier = Guid.NewGuid(), ParentFolder = dir,
Company = new ServiceInterfaces.ViewModels.Common.Companies.CompanyViewModel()
{
Id = MainWindow.CurrentCompanyId
},
CreatedBy = new ServiceInterfaces.ViewModels.Common.Identity.UserViewModel()
{
Id = MainWindow.CurrentUserId
}
})
.ToList();
var response = service.SubmitList(subDirectories);
if (response.Success)
{
dir.SubDirectories = new ObservableCollection <DocumentFolderViewModel>(response?.DocumentFolders ?? new List <DocumentFolderViewModel>());
if (recursive)
{
foreach (var item in dir.SubDirectories)
{
if (CancelOperation)
{
return;
}
GetDocumentFolders(service, fileService, item, recursive);
}
}
}
}
}
}
}
public JsonResult GetSingleUserTaxReturn(string userId, int taxYear)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, byte[]> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
taxReturn = taxReturn.OrderBy(d => d.Id);
var fileUrls = new List <UserTaxReturn>();
if (taxReturn.Count() != 0)
{
fileUrls.Add(taxReturn.AsEnumerable().Last());
byte[] bytes = new byte[64000];
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url.Path);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
bytes = fileStream.ToArray();
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, byte[]>(filename, file.Properties.ContentType, bytes)
);
}
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful && results.Count > 0)
{
return(Json(new MyJsonResult
{
status = 0,
fileName = results.ElementAtOrDefault(0).Item1,
fileContentType = results.ElementAtOrDefault(0).Item2,
fileContents = results.ElementAtOrDefault(0).Item3
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
private void ListFilesAndAssertResults(Action runCmdletAction, IListFileItem[] listedItems, CloudFileDirectory dir = null)
{
this.MockChannel.SetsEnumerationResults(dir == null ? string.Empty : dir.Name, listedItems);
runCmdletAction();
this.MockCmdRunTime.OutputPipeline.AssertListFileItems(listedItems);
}
public async Task <FileStreamResult> Get(int id)
{
var claimsIdentity = User.Identity as ClaimsIdentity;
var userId = Convert.ToInt64(claimsIdentity.Claims.FirstOrDefault(claim => claim.Type == "Id").Value);
var user = _multiSourcePlaylistRepository.GetUser(userId);
var track = _multiSourcePlaylistRepository.GetTrack(id);
byte[] audioArray = new byte[0];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
_configuration["Production:StorageConnectionString"]);
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference(user.FileFolder);
CloudFileDirectory userDir = null;
// Ensure that the share exists.
if (await share.ExistsAsync())
{
// Get a reference to the root directory for the share.
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
// Get a reference to the directory we created previously.
userDir = rootDir.GetDirectoryReference("audio");
// Ensure that the directory exists.
if (await userDir.ExistsAsync())
{
var audiofile = userDir.GetFileReference(track.Address);
if (await audiofile.ExistsAsync())
{
await audiofile.FetchAttributesAsync();
audioArray = new byte[audiofile.Properties.Length];
await audiofile.DownloadToByteArrayAsync(audioArray, 0);
}
}
else
{
return(null);
}
}
else
{
return(null);
}
long fSize = audioArray.Length;
long startbyte = 0;
long endbyte = fSize - 1;
int statusCode = 200;
var rangeRequest = Request.Headers["Range"].ToString();
if (rangeRequest != "")
{
string[] range = Request.Headers["Range"].ToString().Split(new char[] { '=', '-' });
startbyte = Convert.ToInt64(range[1]);
if (range.Length > 2 && range[2] != "")
{
endbyte = Convert.ToInt64(range[2]);
}
if (startbyte != 0 || endbyte != fSize - 1 || range.Length > 2 && range[2] == "")
{
statusCode = 206;
}
}
long desSize = endbyte - startbyte + 1;
Response.StatusCode = statusCode;
Response.ContentType = "audio/mp3";
Response.Headers.Add("Content-Accept", Response.ContentType);
Response.Headers.Add("Content-Length", desSize.ToString());
Response.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", startbyte, endbyte, fSize));
Response.Headers.Add("Accept-Ranges", "bytes");
Response.Headers.Remove("Cache-Control");
var stream = new MemoryStream(audioArray, (int)startbyte, (int)desSize);
return(new FileStreamResult(stream, "audio/mp3")
{
FileDownloadName = track.Name
});
}
private static async Task mainAsync(string[] args)
{
try
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
IConfigurationRoot configuration = builder.Build();
var allCitDebtsLines = new string[10000000];
var allSettlementsLines = new string[10000000];
var allPaymentLines = new string[10000000];
while (true)
{
if (DateTime.Now.AddHours(3).ToShortTimeString() == "12:00 PM")
{
List <string> fileNames = new List <string>();
List <string[]> lstCloudFilesdata = new List <string[]>();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}");
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference("import");
//looks for a file share in the cloud
bool fileShareExists = await fileShare.ExistsAsync();
if (fileShareExists)
{
List <CloudFile> lstCloudFiles = new List <CloudFile>();
CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference();
//for each file in my fileshare
FileContinuationToken token = null;
FileResultSegment k = await rootDir.ListFilesAndDirectoriesSegmentedAsync(token);
token = k.ContinuationToken;
//var context_ = new Db.Data.();
List <string> sl = new List <string>();
foreach (IListFileItem fiile in k.Results)
{
//if the file exists
CloudFile file = (CloudFile)fiile;
bool asd = await file.ExistsAsync();
if (asd)
{
//adds new datasting array
sl = await ReadDataAsync(lstCloudFilesdata, file, fileNames);
}
foreach (string y in sl)
{
Console.WriteLine("From list new : " + y);
}
;
IEnumerable <CitizenDepts> o = from eachLine in (
from inner in sl
select inner.Split(';')
)
select new CitizenDepts
{
VAT = eachLine[0],
FirstName = eachLine[1],
LastName = eachLine[2],
Email = eachLine[3],
Phone = eachLine[4],
Address = eachLine[5],
County = eachLine[6],
BillId = eachLine[7],
Bill_description = eachLine[8],
Amount = Decimal.Parse(eachLine[9]),
DueDate = DateTime.ParseExact(eachLine[10],
"yyyyMMdd", CultureInfo.InvariantCulture)
};
foreach (var p in o)
{
Console.WriteLine(p.FirstName + " - " + p.BillId + ", - " + p.DueDate);
}
;
//string s = context_.Database.ProviderName;
// Console.WriteLine(s);
/// var all = from c in context_.CitizenDepts select c;
//context_.CitizenDepts.RemoveRange(all);
/// context_.SaveChanges();
foreach (var p in o)
{
//Add Student object into Students DBset
//if (p.VAT!=null)
//context_.Add(p);
}
//// call SaveChanges method to save student into database
//context_.SaveChanges();
}
}
if (lstCloudFilesdata != null && fileNames != null)
{
ProccessData(lstCloudFilesdata, fileNames);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
}
}
public Task DeleteDirectoryAsync(CloudFileDirectory directory, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(TaskEx.FromResult(true));
}
/// <summary>
/// Basic operations to work with Azure Files
/// </summary>
/// <returns>Task</returns>
private static async Task BasicAzureFileOperationsAsync()
{
const string DemoShare = "hac2019";
const string DemoDirectory = "Upload";
const string FileToUpload = "Info.txt";
// Retrieve storage account information from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a file client for interacting with the file service.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Create a share for organizing files and directories within the storage account.
Console.WriteLine("1. Creating file share");
CloudFileShare share = fileClient.GetShareReference(DemoShare);
try
{
await share.CreateIfNotExistsAsync();
}
catch (StorageException)
{
Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
Console.WriteLine("Press any key to exit");
//Console.ReadLine();
throw;
}
// Get a reference to the root directory of the share.
CloudFileDirectory root = share.GetRootDirectoryReference();
// Create a directory under the root directory
//Console.WriteLine("2. Creating a directory under the root directory");
CloudFileDirectory dir = root.GetDirectoryReference(DemoDirectory);
await dir.CreateIfNotExistsAsync();
// Uploading a local file to the directory created above
Console.WriteLine("3. Uploading a file to directory");
CloudFile file = dir.GetFileReference(FileToUpload);
await file.UploadFromFileAsync(FileToUpload, FileMode.Open);
// List all files/directories under the root directory
Console.WriteLine("4. List Files/Directories in root directory");
List <IListFileItem> results = new List <IListFileItem>();
FileContinuationToken token = null;
do
{
FileResultSegment resultSegment = await share.GetRootDirectoryReference().ListFilesAndDirectoriesSegmentedAsync(token);
results.AddRange(resultSegment.Results);
token = resultSegment.ContinuationToken;
}while (token != null);
// Print all files/directories listed above
//foreach (IListFileItem listItem in results)
//{
// // listItem type will be CloudFile or CloudFileDirectory
// Console.WriteLine("- {0} (type: {1})", listItem.Uri, listItem.GetType());
//}
// Download the uploaded file to your file system
//Console.WriteLine("5. Download file from {0}", file.Uri.AbsoluteUri);
//await file.DownloadToFileAsync(string.Format("./CopyOf{0}", FileToUpload), FileMode.Create);
// Clean up after the demo
//Console.WriteLine("6. Delete file");
//await file.DeleteAsync();
// When you delete a share it could take several seconds before you can recreate a share with the same
// name - hence to enable you to run the demo in quick succession the share is not deleted. If you want
// to delete the share uncomment the line of code below.
// Console.WriteLine("7. Delete Share");
// await share.DeleteAsync();
}
public AzureServerSideCopyBackupProvider(CloudFileDirectory sourceDirectory, CloudFileDirectory targetDirectory)
{
this.sourceDirectory = sourceDirectory;
this.targetDirectory = targetDirectory;
}
public JsonResult GetUserDocuments(string userId, int taxYear, int?skip, int?amount)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, string> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var miscDocs = db.UserMiscDocs.Where(d => d.UserId == userId && d.Year == taxYear);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
miscDocs = miscDocs.OrderBy(d => d.Id);
taxReturn = taxReturn.OrderBy(d => d.Id);
if (skip.HasValue)
{
miscDocs = miscDocs.Skip(skip.Value);
taxReturn = taxReturn.Skip(skip.Value);
}
if (amount.HasValue)
{
miscDocs = miscDocs.Take(amount.Value);
taxReturn = taxReturn.Take(amount.Value);
}
var fileUrls = miscDocs.Select(d => d.Path).ToList();
fileUrls.AddRange(taxReturn.Select(d => d.Path).ToList());
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType != null &&
file.Properties.ContentType.StartsWith("image/"))
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertImageStreamToBase64String(fileStream);
}
else if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, string>(filename, file.Properties.ContentType, fileContents)
);
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful)
{
return(Json(new {
status = 0,
files = results
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
public Task CreateDirectoryAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(directory.CreateAsync(options, operationContext, cancellationToken));
}
public void FileTypesWithStorageUri()
{
StorageUri endpoint = new StorageUri(
new Uri("http://" + AccountName + FileService + EndpointSuffix),
new Uri("http://" + AccountName + SecondarySuffix + FileService + EndpointSuffix));
CloudFileClient client = new CloudFileClient(endpoint, new StorageCredentials());
Assert.IsTrue(endpoint.Equals(client.StorageUri));
Assert.IsTrue(endpoint.PrimaryUri.Equals(client.BaseUri));
StorageUri shareUri = new StorageUri(
new Uri(endpoint.PrimaryUri + "share"),
new Uri(endpoint.SecondaryUri + "share"));
CloudFileShare share = client.GetShareReference("share");
Assert.IsTrue(shareUri.Equals(share.StorageUri));
Assert.IsTrue(shareUri.PrimaryUri.Equals(share.Uri));
Assert.IsTrue(endpoint.Equals(share.ServiceClient.StorageUri));
share = new CloudFileShare(shareUri, client.Credentials);
Assert.IsTrue(shareUri.Equals(share.StorageUri));
Assert.IsTrue(shareUri.PrimaryUri.Equals(share.Uri));
Assert.IsTrue(endpoint.Equals(share.ServiceClient.StorageUri));
StorageUri directoryUri = new StorageUri(
new Uri(shareUri.PrimaryUri + "/directory"),
new Uri(shareUri.SecondaryUri + "/directory"));
StorageUri subdirectoryUri = new StorageUri(
new Uri(directoryUri.PrimaryUri + "/subdirectory"),
new Uri(directoryUri.SecondaryUri + "/subdirectory"));
CloudFileDirectory directory = share.GetRootDirectoryReference().GetDirectoryReference("directory");
Assert.IsTrue(directoryUri.Equals(directory.StorageUri));
Assert.IsTrue(directoryUri.PrimaryUri.Equals(directory.Uri));
Assert.IsNotNull(directory.Parent);
Assert.IsTrue(shareUri.Equals(directory.Share.StorageUri));
Assert.IsTrue(endpoint.Equals(directory.ServiceClient.StorageUri));
CloudFileDirectory subdirectory = directory.GetDirectoryReference("subdirectory");
Assert.IsTrue(subdirectoryUri.Equals(subdirectory.StorageUri));
Assert.IsTrue(subdirectoryUri.PrimaryUri.Equals(subdirectory.Uri));
Assert.IsTrue(directoryUri.Equals(subdirectory.Parent.StorageUri));
Assert.IsTrue(shareUri.Equals(subdirectory.Share.StorageUri));
Assert.IsTrue(endpoint.Equals(subdirectory.ServiceClient.StorageUri));
StorageUri fileUri = new StorageUri(
new Uri(subdirectoryUri.PrimaryUri + "/file"),
new Uri(subdirectoryUri.SecondaryUri + "/file"));
CloudFile file = subdirectory.GetFileReference("file");
Assert.IsTrue(fileUri.Equals(file.StorageUri));
Assert.IsTrue(fileUri.PrimaryUri.Equals(file.Uri));
Assert.IsTrue(subdirectoryUri.Equals(file.Parent.StorageUri));
Assert.IsTrue(shareUri.Equals(file.Share.StorageUri));
Assert.IsTrue(endpoint.Equals(file.ServiceClient.StorageUri));
file = new CloudFile(fileUri, client.Credentials);
Assert.IsTrue(fileUri.Equals(file.StorageUri));
Assert.IsTrue(fileUri.PrimaryUri.Equals(file.Uri));
Assert.IsTrue(subdirectoryUri.Equals(file.Parent.StorageUri));
Assert.IsTrue(shareUri.Equals(file.Share.StorageUri));
Assert.IsTrue(endpoint.Equals(file.ServiceClient.StorageUri));
}
public Task <bool> DirectoryExistsAsync(CloudFileDirectory directory, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(directory.ExistsAsync(options, operationContext, cancellationToken));
}
/// <summary>
/// Recursive delete of folder and all included subfolders or files.
/// </summary>
/// <param name="cloudFileDirectory">Directory to delete</param>
public static void DeleteRecursive(this CloudFileDirectory cloudFileDirectory)
{
#region validation
if (cloudFileDirectory == null)
{
throw new ArgumentNullException(nameof(cloudFileDirectory));
}
#endregion
if (!TaskUtilities.ExecuteSync(cloudFileDirectory.ExistsAsync()))
{
return;
}
var directoriesToBeDeleted = new List <CloudFileDirectory>();
var continuationToken = new FileContinuationToken();
// get first segment
FileResultSegment fileResultSegment;
do
{
fileResultSegment = TaskUtilities.ExecuteSync(cloudFileDirectory.ListFilesAndDirectoriesSegmentedAsync(continuationToken));
// iterate through items
foreach (IListFileItem fileListItem in fileResultSegment.Results)
{
switch (fileListItem)
{
case CloudFile file:
// Delete file directly
TaskUtilities.ExecuteSync(file.DeleteAsync());
break;
case CloudFileDirectory directory:
// Add the current directory
directoriesToBeDeleted.Add(directory);
// List all sub directories recursively
directoriesToBeDeleted.AddRange(directory.ListSubDirectories());
break;
}
}
} while (fileResultSegment.ContinuationToken != null);
// Sort directories bottom to top so the most deepest nested directories will be deleted first
directoriesToBeDeleted.Sort((aDirectory, bDirectory) => bDirectory.Uri.LocalPath.Length.CompareTo(aDirectory.Uri.LocalPath.Length));
// Delete all found directories
foreach (CloudFileDirectory cloudFileDirectoryToBeDeleted in directoriesToBeDeleted)
{
TaskUtilities.ExecuteSync(cloudFileDirectoryToBeDeleted.DeleteAsync());
}
// Delete the parent directory itself in case it´s not the root directory (which is the file share)
if (cloudFileDirectory.SnapshotQualifiedUri != cloudFileDirectory.Share.GetRootDirectoryReference().SnapshotQualifiedUri)
{
TaskUtilities.ExecuteSync(cloudFileDirectory.DeleteAsync());
}
}
public Task DeleteDirectoryAsync(CloudFileDirectory directory, AccessCondition accessCondition, FileRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
return(directory.DeleteAsync(accessCondition, options, operationContext, cancellationToken));
}
public AzureStorageFileSystem(CloudFileDirectory directory)
{
_directory = directory;
}
private IEnumerable <TransferEntry> EnumerateLocationRecursive(CancellationToken cancellationToken)
{
string fullPrefix = null;
if (null != this.baseDirectory)
{
fullPrefix = Uri.UnescapeDataString(this.baseDirectory.SnapshotQualifiedUri.AbsolutePath);
}
else
{
fullPrefix = Uri.UnescapeDataString(this.location.FileDirectory.SnapshotQualifiedUri.AbsolutePath);
}
// Normalize full prefix to end with slash.
if (!string.IsNullOrEmpty(fullPrefix) && !fullPrefix.EndsWith("/", StringComparison.OrdinalIgnoreCase))
{
fullPrefix += '/';
}
CloudFileDirectory directory = this.location.FileDirectory;
Stack <CloudFileDirectory> innerDirList = new Stack <CloudFileDirectory>();
FileContinuationToken continuationToken = null;
bool passedContinuationToken = false;
if (null == this.listContinuationToken)
{
passedContinuationToken = true;
}
do
{
FileResultSegment resultSegment = null;
Utils.CheckCancellation(cancellationToken);
ErrorEntry errorEntry = null;
try
{
FileRequestOptions requestOptions = Transfer_RequestOptions.DefaultFileRequestOptions;
resultSegment = directory.ListFilesAndDirectoriesSegmentedAsync(
ListFilesSegmentSize,
continuationToken,
requestOptions,
null,
cancellationToken).GetAwaiter().GetResult();
}
catch (Exception ex)
{
string errorMessage = string.Format(
CultureInfo.CurrentCulture,
Resources.FailedToEnumerateDirectory,
directory.SnapshotQualifiedUri.AbsoluteUri,
string.Empty);
TransferException exception =
new TransferException(TransferErrorCode.FailToEnumerateDirectory, errorMessage, ex);
errorEntry = new ErrorEntry(exception);
}
if (null != errorEntry)
{
yield return(errorEntry);
yield break;
}
continuationToken = resultSegment.ContinuationToken;
foreach (IListFileItem fileItem in resultSegment.Results)
{
Utils.CheckCancellation(cancellationToken);
if (fileItem is CloudFileDirectory)
{
CloudFileDirectory cloudDir = fileItem as CloudFileDirectory;
if (!passedContinuationToken)
{
if (string.Equals(cloudDir.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal))
{
passedContinuationToken = true;
continue;
}
else
{
continue;
}
}
string fullPath = Uri.UnescapeDataString(cloudDir.SnapshotQualifiedUri.AbsolutePath);
string relativePath = fullPath.Remove(0, fullPrefix.Length);
yield return(new AzureFileDirectoryEntry(
relativePath,
cloudDir,
new AzureFileListContinuationToken(cloudDir.Name)));
}
else if (fileItem is CloudFile)
{
CloudFile cloudFile = fileItem as CloudFile;
if (!passedContinuationToken)
{
if (string.Equals(cloudFile.Name, this.listContinuationToken.FilePath, StringComparison.Ordinal))
{
passedContinuationToken = true;
continue;
}
else
{
continue;
}
}
string fullPath = Uri.UnescapeDataString(cloudFile.SnapshotQualifiedUri.AbsolutePath);
string relativePath = fullPath.Remove(0, fullPrefix.Length);
yield return(new AzureFileEntry(
relativePath,
cloudFile,
new AzureFileListContinuationToken(cloudFile.Name)));
}
}
}while (continuationToken != null);
}
/**
* Azure Batch AI sample.
* - Create Storage account and Azure file share
* - Upload sample data to Azure file share
* - Create a workspace and experiment
* - Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job
* - Create Microsoft Cognitive Toolkit job to run on the cluster
* - Wait for job to complete
* - Get output files
*
* Please note: in order to run this sample, please download and unzip sample package from <a href="https://batchaisamples.blob.core.windows.net/samples/BatchAIQuickStart.zip?st=2017-09-29T18%3A29%3A00Z&se=2099-12-31T08%3A00%3A00Z&sp=rl&sv=2016-05-31&sr=b&sig=hrAZfbZC%2BQ%2FKccFQZ7OC4b%2FXSzCF5Myi4Cj%2BW3sVZDo%3D">here</a>
* Export path to the content to $SAMPLE_DATA_PATH.
*/
public static void RunSample(IAzure azure)
{
string saName = SdkContext.RandomResourceName("sa", 10);
string rgName = SdkContext.RandomResourceName("rg", 10);
string workspaceName = SdkContext.RandomResourceName("ws", 10);
string experimentName = SdkContext.RandomResourceName("exp", 10);
string sampleDataPath = Environment.GetEnvironmentVariable("SAMPLE_DATA_PATH");
Region region = Region.USWest2;
string shareName = SdkContext.RandomResourceName("fs", 20);
string clusterName = SdkContext.RandomResourceName("cluster", 15);
string userName = Utilities.CreateUsername();
string sharePath = "mnistcntksample";
try
{
//=============================================================
// Create a new storage account and an Azure file share resource
Utilities.Log("Creating a storage account...");
IStorageAccount storageAccount = azure.StorageAccounts.Define(saName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.Create();
Utilities.Log("Created storage account.");
Utilities.PrintStorageAccount(storageAccount);
StorageAccountKey storageAccountKey = storageAccount.GetKeys().First();
Utilities.Log("Creating Azure File share...");
var cloudFileShare = CloudStorageAccount.Parse($"DefaultEndpointsProtocol=https;AccountName={saName};AccountKey={storageAccountKey.Value};EndpointSuffix=core.windows.net")
.CreateCloudFileClient()
.GetShareReference(shareName);
cloudFileShare.CreateAsync().GetAwaiter().GetResult();
Utilities.Log("Created Azure File share.");
//=============================================================
// Upload sample data to Azure file share
//Get a reference to the root directory for the share.
CloudFileDirectory rootDir = cloudFileShare.GetRootDirectoryReference();
//Get a reference to the sampledir directory
Utilities.Log("Creating directory and uploading data files...");
CloudFileDirectory sampleDir = rootDir.GetDirectoryReference(sharePath);
sampleDir.CreateAsync().GetAwaiter().GetResult();
rootDir.GetFileReference("Train-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Train-28x28_cntk_text.txt").GetAwaiter().GetResult();
rootDir.GetFileReference("Test-28x28_cntk_text.txt").UploadFromFileAsync(sampleDataPath + "/Test-28x28_cntk_text.txt").GetAwaiter().GetResult();
rootDir.GetFileReference("ConvNet_MNIST.py").UploadFromFileAsync(sampleDataPath + "/ConvNet_MNIST.py").GetAwaiter().GetResult();
Utilities.Log("Data files uploaded.");
//=============================================================
// Create a workspace and experiment
IBatchAIWorkspace workspace = azure.BatchAIWorkspaces.Define(workspaceName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.Create();
IBatchAIExperiment experiment = workspace.CreateExperiment(experimentName);
//=============================================================
// Create Batch AI cluster that uses Azure file share to host the training data and scripts for the learning job
Utilities.Log("Creating Batch AI cluster...");
IBatchAICluster cluster = workspace.Clusters.Define(clusterName)
.WithVMSize(VirtualMachineSizeTypes.StandardNC6.Value)
.WithUserName(userName)
.WithPassword("MyPassword")
.WithAutoScale(0, 2)
.DefineAzureFileShare()
.WithStorageAccountName(saName)
.WithAzureFileUrl(cloudFileShare.Uri.ToString())
.WithRelativeMountPath("azurefileshare")
.WithAccountKey(storageAccountKey.Value)
.Attach()
.Create();
Utilities.Log("Created Batch AI cluster.");
Utilities.Print(cluster);
// =============================================================
// Create Microsoft Cognitive Toolkit job to run on the cluster
Utilities.Log("Creating Batch AI job...");
IBatchAIJob job = experiment.Jobs.Define("myJob")
.WithExistingCluster(cluster)
.WithNodeCount(1)
.WithStdOutErrPathPrefix("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare")
.DefineCognitiveToolkit()
.WithPythonScriptFile("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/ConvNet_MNIST.py")
.WithCommandLineArgs("$AZ_BATCHAI_MOUNT_ROOT/azurefileshare $AZ_BATCHAI_OUTPUT_MODEL")
.Attach()
.WithOutputDirectory("MODEL", "$AZ_BATCHAI_MOUNT_ROOT/azurefileshare/model")
.WithContainerImage("microsoft/cntk:2.1-gpu-python3.5-cuda8.0-cudnn6.0")
.Create();
Utilities.Log("Created Batch AI job.");
Utilities.Print(job);
// =============================================================
// Wait for job results
// Wait for job to start running
Utilities.Log("Waiting for Batch AI job to start running...");
while (ExecutionState.Queued.Equals(job.ExecutionState))
{
SdkContext.DelayProvider.Delay(5000);
job.Refresh();
}
// Wait for job to complete and job output to become available
Utilities.Log("Waiting for Batch AI job to complete...");
while (!(ExecutionState.Succeeded.Equals(job.ExecutionState) || ExecutionState.Failed.Equals(job.ExecutionState)))
{
SdkContext.DelayProvider.Delay(5000);
job.Refresh();
}
// =============================================================
// Get output files
// Print stdout and stderr
foreach (var outputFile in job.ListFiles("stdouterr"))
{
Utilities.Log(Utilities.CheckAddress(outputFile.DownloadUrl));
}
// List model output files
foreach (var outputFile in job.ListFiles("MODEL"))
{
Utilities.Log(outputFile.DownloadUrl);
}
}
finally
{
try
{
Utilities.Log("Deleting Resource Group: " + rgName);
azure.ResourceGroups.BeginDeleteByName(rgName);
Utilities.Log("Deleted Resource Group: " + rgName);
}
catch (Exception)
{
Utilities.Log("Did not create any resources in Azure. No clean up is necessary");
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="AzureFileLocation" /> class.
/// </summary>
/// <param name="location">Azure file directory location.</param>
/// <param name="baseDirectory"></param>
public AzureFileHierarchyEnumerator(AzureFileDirectoryLocation location, CloudFileDirectory baseDirectory)
{
this.location = location;
this.baseDirectory = baseDirectory;
}
public override void ExecuteCmdlet()
{
CloudFileDirectory baseDirectory;
switch (this.ParameterSetName)
{
case Constants.DirectoryParameterSetName:
baseDirectory = this.Directory;
break;
case Constants.ShareNameParameterSetName:
baseDirectory = this.BuildFileShareObjectFromName(this.ShareName).GetRootDirectoryReference();
break;
case Constants.ShareParameterSetName:
baseDirectory = this.Share.GetRootDirectoryReference();
break;
default:
throw new PSArgumentException(string.Format(CultureInfo.InvariantCulture, "Invalid parameter set name: {0}", this.ParameterSetName));
}
if (string.IsNullOrEmpty(this.Path))
{
this.RunTask(async(taskId) =>
{
await this.Channel.EnumerateFilesAndDirectoriesAsync(
baseDirectory,
item => this.OutputStream.WriteObject(taskId, item),
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken).ConfigureAwait(false);
});
}
else
{
this.RunTask(async(taskId) =>
{
bool foundAFolder = true;
string[] subfolders = NamingUtil.ValidatePath(this.Path);
CloudFileDirectory targetDir = baseDirectory.GetDirectoryReferenceByPath(subfolders);
try
{
await this.Channel.FetchDirectoryAttributesAsync(
targetDir,
null,
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken).ConfigureAwait(false);
}
catch (StorageException se)
{
if (null == se.RequestInformation ||
(int)HttpStatusCode.NotFound != se.RequestInformation.HttpStatusCode)
{
throw;
}
foundAFolder = false;
}
if (foundAFolder)
{
this.OutputStream.WriteObject(taskId, targetDir);
return;
}
string[] filePath = NamingUtil.ValidatePath(this.Path, true);
CloudFile targetFile = baseDirectory.GetFileReferenceByPath(filePath);
await this.Channel.FetchFileAttributesAsync(
targetFile,
null,
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken).ConfigureAwait(false);
this.OutputStream.WriteObject(taskId, targetFile);
});
}
}
private static void CreateCloudFileDestinationDirectory(CloudFileDirectory fileDirectory,
CloudFileNtfsAttributes?fileAttributes,
DateTimeOffset?creationTime,
DateTimeOffset?lastWriteTime,
IDictionary <string, string> metadata,
CancellationToken cancellationToken)
{
bool parentNotExist = false;
if (fileAttributes.HasValue)
{
fileDirectory.Properties.NtfsAttributes = fileAttributes.Value;
fileDirectory.Properties.CreationTime = creationTime;
fileDirectory.Properties.LastWriteTime = lastWriteTime;
}
if (null != metadata)
{
fileDirectory.Metadata.Clear();
foreach (var keyValuePair in metadata)
{
fileDirectory.Metadata.Add(keyValuePair);
}
}
bool needToSetProperties = false;
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
return;
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ParentNotFound", ex.RequestInformation.ErrorCode))
{
parentNotExist = true;
}
else if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
if (parentNotExist)
{
Utils.CreateCloudFileDirectoryRecursively(fileDirectory.Parent);
try
{
fileDirectory.CreateAsync(Transfer_RequestOptions.DefaultFileRequestOptions, null, cancellationToken).GetAwaiter().GetResult();
}
catch (StorageException ex)
{
if (null != ex.RequestInformation)
{
if (string.Equals("ResourceAlreadyExists", ex.RequestInformation.ErrorCode))
{
needToSetProperties = true;
}
else
{
throw;
}
}
else
{
throw;
}
}
}
if (needToSetProperties)
{
SetAzureFileDirectoryAttributes(fileDirectory, fileAttributes, creationTime, lastWriteTime, metadata, cancellationToken);
}
}
