public ActionResult CreateFile(ResourceFile model)
{
try
{
model.Extention = model.FileName.Substring(model.FileName.LastIndexOf("."));
model.DateCreated = model.DateModified = DateTime.Now;
var fileName = string.Format("{0}.{1}", model.UniqueId,
model.FileName.Substring(model.FileName.LastIndexOf(".") + 1));// Path.GetFileName(file.FileName);
model.Path = Path.Combine(Server.MapPath("~/App_Data"), fileName);
var student = Academy.Current.GetStudentById(SessionCache.CurrentUser.DbId);
student.Files.Add(model);
Academy.Current.AddOrUpdate(student);
return RedirectToAction("Files");
}
catch (Exception e)
{
return View(model);
}
}
public ResourceFileViewModel(FilesService filesService)
{
this.filesService = filesService;
files = new ObservableCollection<ResourceFile>(filesService.ResourceFiles());
if (files.Count > 0) selectedFile = files.First();
}
/// <summary>
/// Creates a pool with a start task which installs ImageMagick onto each VM.
/// </summary>
private static void CreatePool()
{
using (IPoolManager pm = config.Client.OpenPoolManager())
{
Console.WriteLine("Creating pool: {0}", config.PoolName);
//Create a pool -- note that OSFamily 3 is Windows Server 2012. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(config.PoolName, targetDedicated: 3, vmSize: "small", osFamily: "3");
//Create a start task for the pool (which installs ImageMagick). The start task is run on each
//VM when they first join the pool. In this case it is used as a setup step to place the ImageMagick exe onto the VM.
IStartTask startTask = new StartTask();
startTask.RunElevated = true;
startTask.CommandLine = ImageMagickCmdLine;
//Set up the resource files for the start task (requires the image magick exe)
List<IResourceFile> resFiles = new List<IResourceFile>();
ResourceFile file = new ResourceFile(config.ImageMagickExeSAS, "ImageMagick.exe");
resFiles.Add(file);
startTask.ResourceFiles = resFiles;
startTask.WaitForSuccess = true;
pool.StartTask = startTask;
pool.Commit(); //Commit the pool -- this actually creates it on the Batch service.
}
}
/// <summary>
/// Returns the code reference to resource specified in the parameters
/// </summary>
/// <param name="file">Resource file containing the resource</param>
/// <param name="resourceName">Name of the resource</param>
/// <returns>a piece of code that would reference to the resource provided</returns>
/// <remarks>This method does not verify if resource actually exists</remarks>
public string GetResourceReference(ResourceFile file, string resourceName, Project project)
{
string prefix = "";
if (ExtensibilityMethods.GetProjectType(project) == ProjectType.VB) {
try {
prefix = (string)project.Properties.Item("RootNamespace").Value;
if (!string.IsNullOrEmpty(prefix)) prefix += ".";
} catch { }
}
if (file == null) {
throw new ArgumentNullException("file");
}
if (String.IsNullOrEmpty(resourceName)) {
throw new ArgumentException(Strings.InvalidResourceName, "resourceName");
}
string namespacePrefix = GetNamespacePrefix(file);
if (!file.IsDefaultResXFile()) {
namespacePrefix += Path.GetFileNameWithoutExtension(file.DisplayName).Replace(' ', '_') + ".";
} else {
namespacePrefix += Path.GetFileNameWithoutExtension(file.FileName).Replace(' ', '_') + ".";
}
string reference =namespacePrefix + resourceName.Replace(' ', '_');
return prefix + reference;
}
/// <summary>
/// Constructs a collection of <see cref="ResourceFile"/> objects based on the files specified.
/// </summary>
/// <param name="containerSas">The container SAS which refers to the Azure Storage container which contains the resource file blobs.</param>
/// <param name="dependencies">The files to construct the <see cref="ResourceFile"/> objects with.</param>
/// <returns>A list of resource file objects.</returns>
public static List<ResourceFile> GetResourceFiles(string containerSas, IEnumerable<string> dependencies)
{
List<ResourceFile> resourceFiles = new List<ResourceFile>();
foreach (string dependency in dependencies)
{
ResourceFile resourceFile = new ResourceFile(ConstructBlobSource(containerSas, dependency), dependency);
resourceFiles.Add(resourceFile);
}
return resourceFiles;
}
/// <summary>
/// Returns the code reference to resource specified in the parameters
/// </summary>
/// <param name="file">Resource file containing the resource</param>
/// <param name="resourceName">Name of the resource</param>
/// <returns>a piece of code that would reference to the resource provided</returns>
/// <remarks>This method does not verify if resource actually exists</remarks>
public virtual string GetResourceReference(ResourceFile file, string resourceName, Project project)
{
if (file == null) {
throw new ArgumentNullException("file");
}
if (String.IsNullOrEmpty(resourceName)) {
throw new ArgumentException(Strings.InvalidResourceName, "resourceName");
}
string namespacePrefix = this.GetNamespacePrefix(file);
string reference = namespacePrefix + Path.GetFileNameWithoutExtension(file.DisplayName).Replace(' ', '_') + "."
+ resourceName.Replace(' ', '_');
return reference;
}
public void Init()
{
mockCompiler = new MockPythonCompiler();
compiler = new PythonCompilerTask(mockCompiler);
TaskItem sourceTaskItem = new TaskItem("test.py");
compiler.Sources = new ITaskItem[] {sourceTaskItem};
compiler.TargetType = "Exe";
compiler.OutputAssembly = "test.exe";
resourceTaskItem = new TaskItem(@"C:\Projects\Test\Test.resources");
compiler.Resources = new ITaskItem[] {resourceTaskItem};
compiler.Execute();
if (mockCompiler.ResourceFiles != null && mockCompiler.ResourceFiles.Count > 0) {
resourceFile = mockCompiler.ResourceFiles[0];
}
}
public static List<IResourceFile> GetResourceFiles(string resourceContainerPrefix)
{
string[] dependencies =
{
Constants.ImgProcExeName,
Constants.StorageClientDllName,
Constants.BatchClientDllName
};
var resources = new List<IResourceFile>();
for (int i = 0; i < dependencies.Length; ++i)
{
ResourceFile res = new ResourceFile(ConstructBlobSource(resourceContainerPrefix, dependencies[i]), dependencies[i]);
resources.Add(res);
}
return resources;
}
static void AddTasks(BatchClient client)
{
CloudJob job = client.JobOperations.GetJob("testjob1");
ResourceFile programFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/ProcessTaskData.exe",
"ProcessTaskData.exe"
);
ResourceFile assemblyFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/Microsoft.WindowsAzure.Storage.dll",
"Microsoft.WindowsAzure.Storage.dll"
);
for (int i = 1; i < 4; ++i)
{
string blobName = "taskdata" + i;
string taskName = "mytask" + i;
ResourceFile taskData = new ResourceFile("https://mystorage00.blob.core.windows.net/testcon1/" +
blobName, blobName);
CloudTask task = new CloudTask(
taskName,
"ProcessTaskData.exe https://mystorage00.blob.core.windows.net/testcon1/" +
blobName + " 3");
List<ResourceFile> taskFiles = new List<ResourceFile>();
taskFiles.Add(taskData);
taskFiles.Add(programFile);
taskFiles.Add(assemblyFile);
task.ResourceFiles = taskFiles;
job.AddTask(task);
job.Commit();
job.Refresh();
}
client.Utilities.CreateTaskStateMonitor().WaitAll(job.ListTasks(),
TaskState.Completed, new TimeSpan(0, 30, 0));
Console.WriteLine("The tasks completed successfully.");
foreach (CloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Id + " says:\n" + task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
public void ConfigureTranslationFiles(ProjectDescription projectDescription, string develepmentCulture, List <string> localizationCultures, bool localizable)
{
var project = GetProject(projectDescription);
var translationFolder = GetFolder(project, ProjectTranslationFolder, localizable);
if (translationFolder == null)
{
if (!localizable)
{
return;
}
throw new Exception("GetFolder didn't create folder");
}
//Collect existing files
var existingFiles = new Dictionary <string, ProjectItem>();
foreach (ProjectItem projectItem in translationFolder.ProjectItems)
{
string fileName = Path.GetFileName(projectItem.FileNames[1]);
if (fileName.StartsWith(TranslationFilePrefix, StringComparison.InvariantCultureIgnoreCase) &&
fileName.EndsWith(TranslationFileSuffix, StringComparison.InvariantCultureIgnoreCase))
{
existingFiles.Add(fileName, projectItem);
}
}
//Prepare list of needed files
var neededFiles = new List <string>();
if (localizable)
{
neededFiles.Add(TranslationFilePrefix + TranslationFileSuffix); //Dev culture
foreach (var localizationCulture in localizationCultures)
{
if (localizationCulture == develepmentCulture)
{
continue;
}
neededFiles.Add(String.Format("{0}.{1}{2}", TranslationFilePrefix, localizationCulture, TranslationFileSuffix)); //Dev culture
}
}
//Add missing files
string folderPath = translationFolder.FileNames[1];
for (int i = 0; i < neededFiles.Count; i++)
{
var neededFile = neededFiles[i];
if (existingFiles.ContainsKey(neededFile))
{
CheckTranslationFileProperties(existingFiles[neededFile], (i == 0));
existingFiles.Remove(neededFile);
continue;
}
string fileName = Path.Combine(folderPath, neededFile);
if (!File.Exists(fileName))
{
var resourceFile = new ResourceFile(fileName);
resourceFile.SaveFile();
}
var projectItem = translationFolder.ProjectItems.AddFromFile(fileName);
CheckTranslationFileProperties(projectItem, (i == 0));
}
//Now existingFiles contains files to remove - Remove not needed files
foreach (var existingFileEntry in existingFiles)
{
existingFileEntry.Value.Remove(); //Exclude from project, but leave the file
}
}
public void TestOMJobPrepSchedulingError()
{
string jobId = "TestOMJobPrepSchedulingError-" + CraftTimeString() + "-" + TestUtilities.GetMyName();
void test()
{
using BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
try
{
// create job with prep that triggers prep scheduling error
{
CloudJob unboundJob = client.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = poolFixture.PoolId
});
// add the jobPrep task to the job
{
JobPreparationTask prep = new JobPreparationTask("cmd /c JobPrep Task");
unboundJob.JobPreparationTask = prep;
ResourceFile[] badResFiles = { ResourceFile.FromUrl("https://not.a.domain.invalid/file", "bob.txt") };
prep.ResourceFiles = badResFiles;
prep.WaitForSuccess = true; // be explicit even though this is the default. need JP/ to not run
}
// add the job to the service
unboundJob.Commit();
}
CloudJob boundJob = client.JobOperations.GetJob(jobId);
// add a trivial task to force the JP
client.JobOperations.AddTask(boundJob.Id, new CloudTask("ForceJobPrep", "cmd /c echo TestOMJobPrepSchedulingError"));
// the victim compute node. pool should have size 1.
List <ComputeNode> nodes = client.PoolOperations.ListComputeNodes(poolFixture.PoolId).ToList();
Assert.Single(nodes);
// now we have a job that should be trying to run the JP
// poll for the JP to have been run, and it must have a scheduling error
bool prepNotCompleted = true;
// gotta poll to find out when the jp has been run
while (prepNotCompleted)
{
List <JobPreparationAndReleaseTaskExecutionInformation> jpStatsList =
client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
JobPreparationAndReleaseTaskExecutionInformation jpStatus = jpStatsList.FirstOrDefault();
if (jpStatus == null)
{
Thread.Sleep(2000);
}
else
{
if (JobPreparationTaskState.Completed == jpStatus.JobPreparationTaskExecutionInformation.State)
{
prepNotCompleted = false; // we see a JP has completed
Assert.NotNull(jpStatus.JobPreparationTaskExecutionInformation.FailureInformation);
Assert.Equal(TaskExecutionResult.Failure, jpStatus.JobPreparationTaskExecutionInformation.Result);
// spew the failure
OutputFailureInfo(jpStatus.JobPreparationTaskExecutionInformation.FailureInformation);
}
testOutputHelper.WriteLine("Job Prep is running (waiting for blob dl to timeout)");
}
}
}
finally
{
// cleanup
client.JobOperations.DeleteJob(jobId);
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Returns a preview of the line after the literal is replaced by a reference to the resource
/// </summary>
/// <param name="file">Resource file containing the resource</param>
/// <param name="resourceName">Name of the resource</param>
/// <returns>Full line where the text would be replaced</returns>
public string PreviewChanges(ResourceFile file, string resourceName)
{
Project project = null;
string reference = "";
try {
project = this.StringToExtract.Parent.Document.ProjectItem.ContainingProject;
reference = this.actionObject.GetResourceReference(file, resourceName, project);
reference = this.StringToExtract.GetShortestReference(reference, this.StringToExtract.GetImportedNamespaces());
}
catch (Exception e)
{
Debug.WriteLine(e);
}
return reference;
}
public async Task CacheTagHelper_CanCachePortionsOfViewsPartialViewsAndViewComponents()
{
// Arrange
var assertFile =
"compiler/resources/CacheTagHelper_CanCachePortionsOfViewsPartialViewsAndViewComponents.Assert";
var outputFile1 = assertFile + "1.txt";
var expected1 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile1, sourceFile : false);
var outputFile2 = assertFile + "2.txt";
var expected2 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile2, sourceFile : false);
var outputFile3 = assertFile + "3.txt";
var expected3 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile3, sourceFile : false);
// Act - 1
// Verify that content gets cached based on vary-by-params
var targetUrl = "/catalog?categoryId=1&correlationid=1";
var request = RequestWithLocale(targetUrl, "North");
var response1 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
request = RequestWithLocale(targetUrl, "North");
var response2 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
// Assert - 1
ResourceFile.UpdateOrVerify(_resourcesAssembly, outputFile1, expected1, response1.Trim());
if (!ResourceFile.GenerateBaselines)
{
Assert.Equal(expected1, response2.Trim(), ignoreLineEndingDifferences: true);
}
// Act - 2
// Verify content gets changed in partials when one of the vary by parameters is changed
targetUrl = "/catalog?categoryId=3&correlationid=2";
request = RequestWithLocale(targetUrl, "North");
var response3 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
request = RequestWithLocale(targetUrl, "North");
var response4 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
// Assert - 2
ResourceFile.UpdateOrVerify(_resourcesAssembly, outputFile2, expected2, response3.Trim());
if (!ResourceFile.GenerateBaselines)
{
Assert.Equal(expected2, response4.Trim(), ignoreLineEndingDifferences: true);
}
// Act - 3
// Verify content gets changed in a View Component when the Vary-by-header parameters is changed
targetUrl = "/catalog?categoryId=3&correlationid=3";
request = RequestWithLocale(targetUrl, "East");
var response5 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
request = RequestWithLocale(targetUrl, "East");
var response6 = await(await Client.SendAsync(request)).Content.ReadAsStringAsync();
// Assert - 3
ResourceFile.UpdateOrVerify(_resourcesAssembly, outputFile3, expected3, response5.Trim());
if (!ResourceFile.GenerateBaselines)
{
Assert.Equal(expected3, response6.Trim(), ignoreLineEndingDifferences: true);
}
}
static void AddTasks(BatchClient client, CloudJob cloudJob, string jobId, IEnumerable<IComputeTask> computeTasks)
{
foreach (var computeTask in computeTasks)
{
var definition = computeTask.Definition;
var executable = new ResourceFile($"{definition.StorageUri}/{definition.ExecutableName}", definition.ExecutableName);
var resources = definition.Resources.Select(resource => new ResourceFile($"{definition.StorageUri}/{resource}", resource));
var inputs = computeTask.Inputs.Select(input => new ResourceFile($"{definition.StorageUri}/{input}", input));
var resourceFiles = new List<ResourceFile> { executable };
resourceFiles.AddRange(resources);
resourceFiles.AddRange(inputs);
var task = client.JobOperations.ListTasks(jobId).SingleOrDefault(t => t.Id == computeTask.Id);
if (task == null)
{
task = new CloudTask(computeTask.Id, computeTask.CommandLine)
{
ResourceFiles = resourceFiles
};
cloudJob.AddTask(task);
cloudJob.Commit();
cloudJob.Refresh();
}
}
client.Utilities.CreateTaskStateMonitor().WaitAll(cloudJob.ListTasks(), TaskState.Completed, new TimeSpan(0, 30, 0));
}
// Private Methods (20)
private void addContentTypes()
{
var content = ResourceFile.GetInputFile("SolutionTemplateGenerator.Resources.[Content_Types].xml");
addFileEntry(Path.Combine(_solutionFolder, "[Content_Types].xml"), content);
}
/// <summary>
/// Creates a new task
/// </summary>
/// <param name="parameters">The parameters to use when creating the task</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
CloudTask task = new CloudTask(parameters.TaskName, parameters.CommandLine);
task.RunElevated = parameters.RunElevated;
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List<IEnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = new List<IResourceFile>();
foreach (DictionaryEntry d in parameters.ResourceFiles)
{
ResourceFile file = new ResourceFile(d.Value.ToString(), d.Key.ToString());
task.ResourceFiles.Add(file);
}
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.TaskConstraints != null)
{
task.TaskConstraints = parameters.TaskConstraints.omObject;
}
WriteVerbose(string.Format(Resources.NBT_CreatingTask, parameters.TaskName));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
using (IWorkItemManager wiManager = parameters.Context.BatchOMClient.OpenWorkItemManager())
{
wiManager.AddTask(parameters.WorkItemName, parameters.JobName, task, parameters.AdditionalBehaviors);
}
}
}
public void Deserialize(ITypeSerializer handler, DataCollection data)
{
handler.Deserialize((object)this, data);
this.File = Services.ProjectOperations.CurrentResourceGroup.FindResourceItem(this.relativePath) as ResourceFile;
}
public FilePathData(ResourceFile resourceFile)
{
this.File = resourceFile;
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
public void Bug2342986_StartTaskMissingOnComputeNode()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
CloudPool pool = batchCli.PoolOperations.GetPool(this.poolFixture.PoolId);
this.testOutputHelper.WriteLine("Getting pool");
StartTask poolStartTask = pool.StartTask;
Assert.NotNull(poolStartTask);
Assert.NotNull(poolStartTask.EnvironmentSettings);
IEnumerable <ComputeNode> computeNodes = pool.ListComputeNodes();
Assert.True(computeNodes.Any());
this.testOutputHelper.WriteLine("Checking every compute nodes start task in the pool matches the pools start task");
foreach (ComputeNode computeNode in computeNodes)
{
this.testOutputHelper.WriteLine("Checking start task of compute node: {0}", computeNode.Id);
//Check that the property is correctly set on each compute node
Assert.NotNull(computeNode.StartTask);
Assert.Equal(poolStartTask.CommandLine, computeNode.StartTask.CommandLine);
Assert.Equal(poolStartTask.MaxTaskRetryCount, computeNode.StartTask.MaxTaskRetryCount);
Assert.Equal(AutoUserScope.Pool, poolStartTask.UserIdentity.AutoUser.Scope);
Assert.Equal(AutoUserScope.Pool, computeNode.StartTask.UserIdentity.AutoUser.Scope);
Assert.Equal(poolStartTask.WaitForSuccess, computeNode.StartTask.WaitForSuccess);
if (poolStartTask.EnvironmentSettings != null)
{
Assert.Equal(poolStartTask.EnvironmentSettings.Count, computeNode.StartTask.EnvironmentSettings.Count);
foreach (EnvironmentSetting environmentSetting in poolStartTask.EnvironmentSettings)
{
EnvironmentSetting matchingEnvSetting = computeNode.StartTask.EnvironmentSettings.FirstOrDefault(envSetting => envSetting.Name == environmentSetting.Name);
Assert.NotNull(matchingEnvSetting);
Assert.Equal(environmentSetting.Name, matchingEnvSetting.Name);
Assert.Equal(environmentSetting.Value, matchingEnvSetting.Value);
}
}
if (poolStartTask.ResourceFiles != null)
{
Assert.Equal(poolStartTask.ResourceFiles.Count, computeNode.StartTask.ResourceFiles.Count);
foreach (ResourceFile resourceFile in poolStartTask.ResourceFiles)
{
ResourceFile matchingResourceFile = computeNode.StartTask.ResourceFiles.FirstOrDefault(item => item.HttpUrl == resourceFile.HttpUrl);
Assert.NotNull(matchingResourceFile);
Assert.Equal(resourceFile.HttpUrl, matchingResourceFile.HttpUrl);
Assert.Equal(resourceFile.FilePath, matchingResourceFile.FilePath);
}
}
//Try to set some properties of the compute node's start task and ensure it fails
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.CommandLine = "Test"; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.MaxTaskRetryCount = 5; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.UserIdentity = new UserIdentity("foo"); });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.WaitForSuccess = true; });
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.EnvironmentSettings = new List <EnvironmentSetting>(); });
if (computeNode.StartTask.EnvironmentSettings != null)
{
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.EnvironmentSettings.Add(new EnvironmentSetting("test", "test")); });
}
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.ResourceFiles = new List <ResourceFile>(); });
if (computeNode.StartTask.ResourceFiles != null)
{
TestUtilities.AssertThrows <InvalidOperationException>(() => { computeNode.StartTask.ResourceFiles.Add(ResourceFile.FromUrl("test", "test")); });
}
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public ResourceFile MoveFile(ResourceFile src, ResourceFolder dest)
{
src.Parent = dest;
SaveChanges();
return(src);
}
/// <summary>
/// Creates a new Task
/// </summary>
/// <param name="parameters">The parameters to use when creating the Task</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
if ((string.IsNullOrWhiteSpace(parameters.WorkItemName) || string.IsNullOrWhiteSpace(parameters.JobName)) && parameters.Job == null)
{
throw new ArgumentException(Resources.NBT_NoJobSpecified);
}
if (string.IsNullOrWhiteSpace(parameters.TaskName))
{
throw new ArgumentNullException("TaskName");
}
CloudTask task = new CloudTask(parameters.TaskName, parameters.CommandLine);
task.RunElevated = parameters.RunElevated;
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List <IEnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = new List <IResourceFile>();
foreach (DictionaryEntry d in parameters.ResourceFiles)
{
ResourceFile file = new ResourceFile(d.Value.ToString(), d.Key.ToString());
task.ResourceFiles.Add(file);
}
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.TaskConstraints != null)
{
task.TaskConstraints = parameters.TaskConstraints.omObject;
}
WriteVerbose(string.Format(Resources.NBT_CreatingTask, parameters.TaskName));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
using (IWorkItemManager wiManager = parameters.Context.BatchOMClient.OpenWorkItemManager())
{
wiManager.AddTask(parameters.WorkItemName, parameters.JobName, task, parameters.AdditionalBehaviors);
}
}
}
private static async Task TaskCreation(BatchClient p_batchClient, string p_jobId)
{
Console.WriteLine("Creating the Task");
string taskId = "demotask";
string in_container_name = "input";
string out_container_name = "output";
string l_blobName = "sample.mp4";
string outputfile = "audio.aac";
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
demo_storageAccountName, demo_storageAccountKey);
CloudStorageAccount l_storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient l_blobClient = l_storageAccount.CreateCloudBlobClient();
CloudBlobContainer in_container = l_blobClient.GetContainerReference(in_container_name);
CloudBlobContainer out_container = l_blobClient.GetContainerReference(out_container_name);
SharedAccessBlobPolicy i_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
};
SharedAccessBlobPolicy o_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Write
};
string in_sasToken = in_container.GetSharedAccessSignature(i_sasConstraints);
string in_containerSasUrl = String.Format("{0}{1}", in_container.Uri, in_sasToken);
string out_sasToken = out_container.GetSharedAccessSignature(o_sasConstraints);
string out_containerSasUrl = String.Format("{0}{1}", out_container.Uri, out_sasToken);
ResourceFile inputFile = ResourceFile.FromStorageContainerUrl(in_containerSasUrl);
List <ResourceFile> file = new List <ResourceFile>();
file.Add(inputFile);
string appPath = String.Format("%AZ_BATCH_APP_PACKAGE_{0}#{1}%", demo_packageid, demo_packageversion);
string taskCommandLine = String.Format("cmd /c {0}\\ffmpeg.exe -i {1} -vn -acodec copy audio.aac", appPath, l_blobName);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = file;
// Setting the output file location
List <OutputFile> outputFileList = new List <OutputFile>();
OutputFileBlobContainerDestination outputContainer = new OutputFileBlobContainerDestination(out_containerSasUrl);
OutputFile outputFile = new OutputFile(outputfile,
new OutputFileDestination(outputContainer),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskSuccess));
outputFileList.Add(outputFile);
task.OutputFiles = outputFileList;
await p_batchClient.JobOperations.AddTaskAsync(p_jobId, task);
}
public SimpleAudioObject(ResourceFile resourceFile) : this()
{
this.FileData = resourceFile;
}
private static bool IsLocalizedResourceFile(string resourceFileName)
{
string locale = ResourceFile.GetLocale(resourceFileName);
return(String.IsNullOrEmpty(locale) == false);
}
private void btnFinish_Click(object sender, System.EventArgs e)
{
try
{
foreach (DataRowView row in dataView1)
{
string key = row.Row.ItemArray[0].ToString();
string value = row.Row.ItemArray[1].ToString();
if (!String.IsNullOrEmpty(key) &&
!String.IsNullOrEmpty(value))
{
ResourceFile resFile = new ResourceFile();
resFile.FilePath = key;
resFile.BlobSource = value;
resourceFiles.Add(resFile);
}
}
if (resourceFiles != null &&
resourceFiles.Count > 0)
{
if (task != null)
{
task.ResourceFiles = resourceFiles.ToArray();
}
else if (pool != null)
{
pool.StartTask.ResourceFiles = resourceFiles.ToArray();
}
else if (workitem != null && code.Equals("1"))
{
workitem.JobSpecification.JobManager.ResourceFiles =
resourceFiles.ToArray();
}
else if (workitem != null && code.Equals("2"))
{
workitem.JobExecutionEnvironment.AutoPoolSpecification.Pool.
StartTask.ResourceFiles = resourceFiles.ToArray();
}
else if (starttask != null)
{
starttask.ResourceFiles = resourceFiles.ToArray();
}
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("FilesEnv resource files:\n{0}", ex));
}
try
{
List<EnvironmentSetting> envSettings = new List<EnvironmentSetting>();
foreach (DataRowView row in dataView2)
{
string key = row.Row.ItemArray[0].ToString();
string value = row.Row.ItemArray[1].ToString();
if (!String.IsNullOrEmpty(key) &&
!String.IsNullOrEmpty(value))
{
EnvironmentSetting setting = new EnvironmentSetting();
setting.Name = key;
setting.Value = value;
envSettings.Add(setting);
}
}
if (envSettings.Count > 0)
{
if (task != null)
{
task.EnvironmentSettings = envSettings.ToArray();
}
else if (pool != null)
{
pool.StartTask.EnvironmentSettings = envSettings.ToArray();
}
else if (workitem != null && code.Equals("1"))
{
workitem.JobSpecification.JobManager.EnvironmentSettings = envSettings.ToArray();
}
else if (workitem != null && code.Equals("2"))
{
workitem.JobExecutionEnvironment.AutoPoolSpecification.Pool.
StartTask.EnvironmentSettings = envSettings.ToArray();
}
else if (starttask != null)
{
starttask.EnvironmentSettings = envSettings.ToArray();
}
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("FilesEnv envSettings:\n{0}", ex));
}
try
{
if (task != null)
{
(Sender as BatchTreeViewItem).AddTask(task, TaskName);
(this.Parent as NavigationWindow).Close();
}
else if (pool != null)
{
(Sender as BatchTreeViewItem).AddPool(pool, pool.Name);
(this.Parent as NavigationWindow).Close();
}
else if (workitem != null && code.Equals("1"))
{
CreateWIP5 p5 = new CreateWIP5(this.Sender, workitem);
NavigationService.Navigate(p5);
}
else if (workitem != null && code.Equals("2"))
{
(Sender as BatchTreeViewItem).AddWorkItem(workitem);
(this.Parent as NavigationWindow).Close();
}
else if (starttask != null)
{
//TODO::Certificates and metadata
((Sender as BatchTreeViewItem).Item as PoolViewModel).UpdatePool(starttask, null, null);
(this.Parent as NavigationWindow).Close();
}
}
catch (Exception ex)
{
MessageBox.Show(Utils.ExtractExceptionMessage(ex));
}
}
public IResourceFileReader Create(ResourceFile resourceFile)
{
return(new MockResourceFileReader());
}
/// <summary>
/// Determines the namespace of the provided resource file
/// </summary>
/// <param name="file">Reference to the resource file</param>
/// <returns>Namespace to be used to access the resource file</returns>
protected override string GetNamespacePrefix(ResourceFile file)
{
if (file.CustomToolName.Equals("GlobalResourceProxyGenerator")) {
return "Resources.";
} else {
return base.GetNamespacePrefix(file);
}
}
protected override void OnExpand()
{
int requirement = int.Parse(RuntimeSettings[gCEL.Core.Components.FABRIC.Constants.CAPACITY_SETTING_REQUIREMENT]);
int workers = int.Parse(Settings[SETTING_WORKERS]);
PoolSize = (requirement / workers) + ((requirement % workers) > 0 ? 1 : 0);
if (PoolSize == 0)
{
PoolSize = 1;
}
string reqfile = gCEL.Core.Utils.FileSystem.GetTempPath() + "gcel_request_" + gCEL.Core.Utils.Environment.UniqueIdentifier + ".xml";
//> CREATE
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
ResourceFile req = CreateResourceFile(reqfile);
//BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], Settings[SETTING_BATCH_ACCOUNT_NAME], Settings[SETTING_BATCH_ACCOUNT_KEY]);
BatchTokenCredentials cred = new BatchTokenCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], GetToken());
Client = BatchClient.Open(cred);
//> VM
ImageReference imgref = new ImageReference(Settings[SETTING_BATCH_CUSTOM_IMAGE]);
VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
//> POOL
CreateBatchPool(vmc);
//> JOB
CloudJob job = Client.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
//> TASKS
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 0; i < requirement; i++)
{
string taskId = "gCELTask" + i;
string engine = Settings[SETTING_SYSTEM_FOLDER] + gCEL.Core.Constants.ENGINE_CMD;
string cmd = String.Format("cmd /c {0} command=host component=engine request={1}", engine, req.FilePath);
CloudTask task = new CloudTask(taskId, cmd);
task.ResourceFiles = new List <ResourceFile> {
req
};
tasks.Add(task);
}
Client.JobOperations.AddTask(JobId, tasks);
//>
sw.Stop();
TimeSpan ts = sw.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Logger.Info(Tag + ":POOLCREATED:" + elapsedTime);
}
catch (Exception x)
{
Logger.Error(Tag + ":" + x.Message);
throw;
}
finally
{
gCEL.Core.Utils.FileSystem.SafeDelete(reqfile);
OnContract();
}
Logger.Info(Tag + ":ALLOCATED:" + PoolSize);
}
public void Delete(Guid id)
{
ResourceFile resourceFile = _context.ResourceFiles.First(r => r.Id == id);
_context.Remove(resourceFile);
}
private sealed record StateObject(ResourceFile File, CancellationToken CancellationToken);
/// <summary>
/// Extracts the string to a resource file and replaces it with a reference to the new entry
/// </summary>
/// <param name="file"></param>
/// <param name="resourceName"></param>
public void ExtractStringToResource(ResourceFile file, string resourceName)
{
UndoContext undo = StringToExtract.Parent.Document.DTE.UndoContext;
undo.Open(Strings.ExtractResourceUndoContextName);
try {
Project project = null;
try { project = this.StringToExtract.Parent.Document.ProjectItem.ContainingProject; } catch { }
string reference = this.actionObject.GetResourceReference(file, resourceName, project);
reference = this.StringToExtract.GetShortestReference(reference, this.StringToExtract.GetImportedNamespaces());
this.StringToExtract.Replace(reference);
undo.Close();
} catch {
undo.SetAborted();
throw;
}
}
public ActionResult CreateFile()
{
var newFile = new ResourceFile();
newFile.Init();
return View(newFile);
}
private void btnFinish_Click(object sender, System.EventArgs e)
{
try
{
foreach (DataRowView row in dataView1)
{
string key = row.Row.ItemArray[0].ToString();
string value = row.Row.ItemArray[1].ToString();
if (!String.IsNullOrEmpty(key) &&
!String.IsNullOrEmpty(value))
{
ResourceFile resFile = new ResourceFile();
resFile.FilePath = key;
resFile.BlobSource = value;
resourceFiles.Add(resFile);
}
}
if (resourceFiles != null &&
resourceFiles.Count > 0)
{
if (task != null)
{
task.ResourceFiles = resourceFiles.ToArray();
}
else if (pool != null)
{
pool.StartTask.ResourceFiles = resourceFiles.ToArray();
}
else if (workitem != null && code.Equals("1"))
{
workitem.JobSpecification.JobManager.ResourceFiles =
resourceFiles.ToArray();
}
else if (workitem != null && code.Equals("2"))
{
workitem.JobExecutionEnvironment.AutoPoolSpecification.Pool.
StartTask.ResourceFiles = resourceFiles.ToArray();
}
else if (starttask != null)
{
starttask.ResourceFiles = resourceFiles.ToArray();
}
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("FilesEnv resource files:\n{0}", ex));
}
try
{
List <EnvironmentSetting> envSettings = new List <EnvironmentSetting>();
foreach (DataRowView row in dataView2)
{
string key = row.Row.ItemArray[0].ToString();
string value = row.Row.ItemArray[1].ToString();
if (!String.IsNullOrEmpty(key) &&
!String.IsNullOrEmpty(value))
{
EnvironmentSetting setting = new EnvironmentSetting();
setting.Name = key;
setting.Value = value;
envSettings.Add(setting);
}
}
if (envSettings.Count > 0)
{
if (task != null)
{
task.EnvironmentSettings = envSettings.ToArray();
}
else if (pool != null)
{
pool.StartTask.EnvironmentSettings = envSettings.ToArray();
}
else if (workitem != null && code.Equals("1"))
{
workitem.JobSpecification.JobManager.EnvironmentSettings = envSettings.ToArray();
}
else if (workitem != null && code.Equals("2"))
{
workitem.JobExecutionEnvironment.AutoPoolSpecification.Pool.
StartTask.EnvironmentSettings = envSettings.ToArray();
}
else if (starttask != null)
{
starttask.EnvironmentSettings = envSettings.ToArray();
}
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("FilesEnv envSettings:\n{0}", ex));
}
try
{
if (task != null)
{
(Sender as BatchTreeViewItem).AddTask(task, TaskName);
(this.Parent as NavigationWindow).Close();
}
else if (pool != null)
{
(Sender as BatchTreeViewItem).AddPool(pool, pool.Name);
(this.Parent as NavigationWindow).Close();
}
else if (workitem != null && code.Equals("1"))
{
CreateWIP5 p5 = new CreateWIP5(this.Sender, workitem);
NavigationService.Navigate(p5);
}
else if (workitem != null && code.Equals("2"))
{
(Sender as BatchTreeViewItem).AddWorkItem(workitem);
(this.Parent as NavigationWindow).Close();
}
else if (starttask != null)
{
//TODO::Certificates and metadata
((Sender as BatchTreeViewItem).Item as PoolViewModel).UpdatePool(starttask, null, null);
(this.Parent as NavigationWindow).Close();
}
}
catch (Exception ex)
{
MessageBox.Show(Utils.ExtractExceptionMessage(ex));
}
}
public async Task HtmlGenerationWebSite_GenerateEncodedResults(string action, string antiforgeryPath)
{
// This uses FileVersionProvider which uses Uri.TryCreate - https://github.com/aspnet/External/issues/21
if (TestPlatformHelper.IsMono && (action == "Link" || action == "Script"))
{
return;
}
// Arrange
var server = TestHelper.CreateServer(_app, SiteName, services =>
{
_configureServices(services);
services.AddTransient <IHtmlEncoder, TestHtmlEncoder>();
services.AddTransient <IJavaScriptStringEncoder, TestJavaScriptEncoder>();
services.AddTransient <IUrlEncoder, TestUrlEncoder>();
});
var client = server.CreateClient();
var expectedMediaType = MediaTypeHeaderValue.Parse("text/html; charset=utf-8");
var outputFile = "compiler/resources/HtmlGenerationWebSite.HtmlGeneration_Home." + action + ".Encoded.html";
var expectedContent =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile, sourceFile : false);
// Act
// The host is not important as everything runs in memory and tests are isolated from each other.
var response = await client.GetAsync("http://localhost/HtmlGeneration_Home/" + action);
var responseContent = await response.Content.ReadAsStringAsync();
// Assert
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(expectedMediaType, response.Content.Headers.ContentType);
responseContent = responseContent.Trim();
if (antiforgeryPath == null)
{
#if GENERATE_BASELINES
ResourceFile.UpdateFile(_resourcesAssembly, outputFile, expectedContent, responseContent);
#else
// Mono issue - https://github.com/aspnet/External/issues/19
Assert.Equal(
PlatformNormalizer.NormalizeContent(expectedContent.Trim()),
responseContent,
ignoreLineEndingDifferences: true);
#endif
}
else
{
var forgeryToken = AntiforgeryTestHelper.RetrieveAntiforgeryToken(responseContent, antiforgeryPath);
#if GENERATE_BASELINES
// Reverse usual substitution and insert a format item into the new file content.
responseContent = responseContent.Replace(forgeryToken, "{0}");
ResourceFile.UpdateFile(_resourcesAssembly, outputFile, expectedContent, responseContent);
#else
expectedContent = string.Format(expectedContent, forgeryToken);
// Mono issue - https://github.com/aspnet/External/issues/19
Assert.Equal(
PlatformNormalizer.NormalizeContent(expectedContent.Trim()),
responseContent,
ignoreLineEndingDifferences: true);
#endif
}
}
public void TestOMJobReleaseSchedulingError()
{
string jobId = "TestOMJobReleaseSchedulingError-" + CraftTimeString() + "-" + TestUtilities.GetMyName();
void test()
{
using BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
try
{
// create job schedule with prep that succeeds and release the triggers scheduling error
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = poolFixture.PoolId
};
CloudJob unboundJob = client.JobOperations.CreateJob(jobId, poolInfo);
// add the jobPrep task to the job
{
JobPreparationTask prep = new JobPreparationTask("cmd /c echo the quick job prep jumped over the...");
unboundJob.JobPreparationTask = prep;
prep.WaitForSuccess = false; // we don't really care but why not set this
}
// add a jobRelease task to the job
{
JobReleaseTask relTask = new JobReleaseTask("cmd /c echo Job Release Task");
unboundJob.JobReleaseTask = relTask;
ResourceFile[] badResFiles = { ResourceFile.FromUrl("https://not.a.domain.invalid/file", "bob.txt") };
relTask.ResourceFiles = badResFiles;
relTask.Id = "jobRelease";
}
// add the job to the service
unboundJob.Commit();
}
// add a trivial task to force the JP
client.JobOperations.AddTask(jobId, new CloudTask("ForceJobPrep", "cmd /c echo TestOMJobReleaseSchedulingError"));
// wait for the task to complete
TaskStateMonitor tsm = client.Utilities.CreateTaskStateMonitor();
tsm.WaitAll(
client.JobOperations.ListTasks(jobId),
TaskState.Completed,
TimeSpan.FromMinutes(10),
additionalBehaviors:
new[]
{
// spam/logging interceptor
new Protocol.RequestInterceptor((x) =>
{
testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = client.PoolOperations.ListComputeNodes(poolFixture.PoolId).ToList();
testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach((icn) =>
{
testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
testOutputHelper.WriteLine("");
})
}
);
// ok terminate job to trigger job release
client.JobOperations.TerminateJob(jobId, "BUG: Server will throw 500 if I don't provide reason");
// the victim compute node. pool should have size 1.
List <ComputeNode> computeNodes = client.PoolOperations.ListComputeNodes(poolFixture.PoolId).ToList();
Assert.Single(computeNodes);
// now we have a job that should be trying to run the JP
// poll for the JP to have been run, and it must have a scheduling error
bool releaseNotCompleted = true;
// gotta poll to find out when the jp has been run
while (releaseNotCompleted)
{
List <JobPreparationAndReleaseTaskExecutionInformation> jrStatusList =
client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
JobPreparationAndReleaseTaskExecutionInformation prepAndReleaseStatus = jrStatusList.FirstOrDefault();
if (prepAndReleaseStatus != null && null != prepAndReleaseStatus.JobReleaseTaskExecutionInformation)
{
if (JobReleaseTaskState.Completed == prepAndReleaseStatus.JobReleaseTaskExecutionInformation.State)
{
releaseNotCompleted = false; // we see a JP has been run
// now assert the failure info
Assert.NotNull(prepAndReleaseStatus);
Assert.NotNull(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
Assert.Equal(TaskExecutionResult.Failure, prepAndReleaseStatus.JobReleaseTaskExecutionInformation.Result);
// spew the failure info
OutputFailureInfo(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
}
}
Thread.Sleep(2000);
testOutputHelper.WriteLine("Job Release tasks still running (waiting for blob dl to timeout).");
}
}
finally
{
client.JobOperations.DeleteJob(jobId);
}
}
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
public async Task CacheTagHelper_CanCachePortionsOfViewsPartialViewsAndViewComponents()
{
// Arrange
var assertFile =
"compiler/resources/CacheTagHelper_CanCachePortionsOfViewsPartialViewsAndViewComponents.Assert";
var server = TestHelper.CreateServer(_app, SiteName, _configureServices);
var client = server.CreateClient();
client.BaseAddress = new Uri("http://localhost");
client.DefaultRequestHeaders.Add("Locale", "North");
var outputFile1 = assertFile + "1.txt";
var expected1 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile1, sourceFile : false);
var outputFile2 = assertFile + "2.txt";
var expected2 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile2, sourceFile : false);
var outputFile3 = assertFile + "3.txt";
var expected3 =
await ResourceFile.ReadResourceAsync(_resourcesAssembly, outputFile3, sourceFile : false);
// Act - 1
// Verify that content gets cached based on vary-by-params
var targetUrl = "/catalog?categoryId=1&correlationid=1";
var response1 = await client.GetStringAsync(targetUrl);
var response2 = await client.GetStringAsync(targetUrl);
// Assert - 1
#if GENERATE_BASELINES
ResourceFile.UpdateFile(_resourcesAssembly, outputFile1, expected1, response1.Trim());
#else
Assert.Equal(expected1, response1.Trim(), ignoreLineEndingDifferences: true);
Assert.Equal(expected1, response2.Trim(), ignoreLineEndingDifferences: true);
#endif
// Act - 2
// Verify content gets changed in partials when one of the vary by parameters is changed
targetUrl = "/catalog?categoryId=3&correlationid=2";
var response3 = await client.GetStringAsync(targetUrl);
var response4 = await client.GetStringAsync(targetUrl);
// Assert - 2
#if GENERATE_BASELINES
ResourceFile.UpdateFile(_resourcesAssembly, outputFile2, expected2, response3.Trim());
#else
Assert.Equal(expected2, response3.Trim(), ignoreLineEndingDifferences: true);
Assert.Equal(expected2, response4.Trim(), ignoreLineEndingDifferences: true);
#endif
// Act - 3
// Verify content gets changed in a View Component when the Vary-by-header parameters is changed
client.DefaultRequestHeaders.Remove("Locale");
client.DefaultRequestHeaders.Add("Locale", "East");
targetUrl = "/catalog?categoryId=3&correlationid=3";
var response5 = await client.GetStringAsync(targetUrl);
var response6 = await client.GetStringAsync(targetUrl);
// Assert - 3
#if GENERATE_BASELINES
ResourceFile.UpdateFile(_resourcesAssembly, outputFile3, expected3, response5.Trim());
#else
Assert.Equal(expected3, response5.Trim(), ignoreLineEndingDifferences: true);
Assert.Equal(expected3, response6.Trim(), ignoreLineEndingDifferences: true);
#endif
}
/// <summary>
/// Embeds a single resource into the assembly.
/// </summary>
void AddResource(ModuleBuilder moduleBuilder, ResourceFile resourceFile)
{
string fileName = resourceFile.FileName;
string extension = Path.GetExtension(fileName).ToLowerInvariant();
if (extension == ".resources") {
string fullFileName = Path.GetFileName(fileName);
IResourceWriter resourceWriter = moduleBuilder.DefineResource(fullFileName, resourceFile.Name, ResourceAttributes.Public);
AddResources(resourceWriter, fileName);
} else {
moduleBuilder.DefineManifestResource(resourceFile.Name, new FileStream(fileName, FileMode.Open), ResourceAttributes.Public);
}
}
private void CreateBatchJob(string jobName, int id, string profileUrl, string containerUrl, string nsUrl, int daysDuration, string timeZone, bool uamAsBasal)
{
// Connect to Azure Batch
var credentials = new BatchSharedKeyCredentials(
ConfigurationManager.AppSettings["BatchAccountUrl"],
ConfigurationManager.AppSettings["BatchAccountName"],
ConfigurationManager.AppSettings["BatchAccountKey"]);
using (var batchClient = BatchClient.Open(credentials))
{
var poolId = ConfigurationManager.AppSettings["BatchPoolId"];
// Create the job
var job = batchClient.JobOperations.CreateJob();
job.Id = jobName;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.UsesTaskDependencies = true;
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.Commit();
// Add a task to the job to run Autotune
var commandLine = "/bin/sh -c '" +
"cd \"$AZ_BATCH_TASK_WORKING_DIR\" && " +
"mkdir -p settings && " +
"mv profile.json settings && " +
"cp settings/profile.json settings/pumpprofile.json && " +
"cp settings/profile.json settings/autotune.json && " +
$"TZ='{timeZone}' && " +
"export TZ && " +
$"oref0-autotune --dir=$AZ_BATCH_TASK_WORKING_DIR --ns-host={nsUrl} --start-date={DateTime.Now.AddDays(-daysDuration):yyyy-MM-dd} --end-date={DateTime.Now.AddDays(-1):yyyy-MM-dd} --categorize-uam-as-basal={(uamAsBasal ? "true" : "false")}" +
"'";
var task = new CloudTask("Autotune", commandLine);
// The task needs to use the profile.json file previously added to Azure Storage
task.ResourceFiles = new List <ResourceFile>();
task.ResourceFiles.Add(ResourceFile.FromUrl(profileUrl, "profile.json"));
// Capture the recommendations generated by Autotune into Azure Storage
task.OutputFiles = new List <OutputFile>();
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune_recommendations.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune.*.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
// Ensure the task still completes even if Autotune fails, as we still want to email the
// log file with the error details
task.ExitConditions = new ExitConditions
{
ExitCodes = new List <ExitCodeMapping>(new[]
{
new ExitCodeMapping(1, new ExitOptions {
DependencyAction = DependencyAction.Satisfy
})
})
};
batchClient.JobOperations.AddTask(jobName, task);
// Get the URL for the JobFinished action that the recommendations can be uploaded to to generate the email
var uploadUrl = new Uri(Request.Url, Url.Action("JobFinished", "Results", new { id, key = ConfigurationManager.AppSettings["ResultsCallbackKey"] }));
var uploadCommandLine = $"/bin/sh -c '" +
"cd /usr/src/oref0 && " +
$"wget -O /dev/null -o /dev/null {uploadUrl}\\&commit=$(git rev-parse --short HEAD)" +
"'";
var uploadTask = new CloudTask("Upload", uploadCommandLine);
uploadTask.DependsOn = TaskDependencies.OnId(task.Id);
uploadTask.Constraints = new TaskConstraints(maxTaskRetryCount: 2);
batchClient.JobOperations.AddTask(jobName, uploadTask);
}
}
public ResourceSource(ResourceFile resource, string metadataName)
{
Resource = resource;
MetadataName = metadataName;
}
public byte[] createEMZ(string iniOrElyrText, bool withVideo = false, ResourceFile video = null, ResourceFile audio = null)
{
this.error = null;
string audioFile = (audio != null) ? audio.FileName : this.GetAudioSourceName();
if (audioFile == null)
{
this.error = "Can`t find audio source property!";
return(null);
}
ResourceFile audioResource = (audio != null)
? audio
: this.Resources.Where(r => r.FileName == audioFile).FirstOrDefault();
if (audioResource == null)
{
this.error = "Can`t find resource for audio source property!";
return(null);
}
ResourceFile videoResource = (video != null) ? video : this.GetVideoResource();
if (withVideo && videoResource == null)
{
this.error = "Can`t find or KFN contain more one video resource!";
return(null);
}
FileInfo sourceFile = new FileInfo(audioFile);
string elyrFileName = sourceFile.Name.Substring(0, sourceFile.Name.Length - sourceFile.Extension.Length) + ".elyr";
string elyrText = (Regex.IsMatch(iniOrElyrText, @"[\r\n][Ss]ync[0-9]+=[0-9,]+"))
? this.INIToELYR(iniOrElyrText)
: iniOrElyrText;
if (elyrText == null)
{
if (this.error == null)
{
this.error = "Fail to create ELYR!";
}
return(null);
}
byte[] bom = Encoding.Unicode.GetPreamble();
string elyrHeader = "encore.lg-karaoke.ru ver=02 crc=00000000 \r\n";
byte[] elyr = Encoding.Convert(Encoding.UTF8, Encoding.Unicode, Encoding.UTF8.GetBytes(elyrHeader + elyrText));
Array.Resize(ref bom, bom.Length + elyr.Length);
Array.Copy(elyr, 0, bom, 2, elyr.Length);
using (MemoryStream memStream = new MemoryStream())
{
//int cp = System.Globalization.CultureInfo.CurrentCulture.TextInfo.OEMCodePage;
int cp = 866;
using (ZipArchive archive = new ZipArchive(memStream, ZipArchiveMode.Create, true, Encoding.GetEncoding(cp)))
{
ZipArchiveEntry lyricEntry = archive.CreateEntry(elyrFileName);
using (MemoryStream lyricBody = new MemoryStream(bom))
using (Stream ls = lyricEntry.Open())
{
lyricBody.CopyTo(ls);
}
ZipArchiveEntry audioEntry = archive.CreateEntry(audioResource.FileName);
using (MemoryStream audioBody = new MemoryStream(this.GetDataFromResource(audioResource)))
using (Stream aus = audioEntry.Open())
{
audioBody.CopyTo(aus);
}
if (withVideo)
{
ZipArchiveEntry videoEntry = archive.CreateEntry(videoResource.FileName);
using (MemoryStream videoBody = new MemoryStream(this.GetDataFromResource(videoResource)))
using (Stream aus = videoEntry.Open())
{
videoBody.CopyTo(aus);
}
}
}
return(memStream.ToArray());
}
}
private static void RunDesignTimeTest(
MvcRazorHost host,
string testName,
IEnumerable <LineMapping> expectedLineMappings)
{
var inputFile = "TestFiles/Input/" + testName + ".cshtml";
var outputFile = "TestFiles/Output/DesignTime/" + testName + ".cs";
var expectedCode = ResourceFile.ReadResource(_assembly, outputFile, sourceFile: false);
// Act
GeneratorResults results;
using (var stream = ResourceFile.GetResourceStream(_assembly, inputFile, sourceFile: true))
{
// VS tooling passes in paths in all lower case. We'll mimic this behavior in our tests.
results = host.GenerateCode(inputFile.ToLowerInvariant(), stream);
}
// Assert
Assert.True(results.Success);
Assert.Empty(results.ParserErrors);
#if GENERATE_BASELINES
ResourceFile.UpdateFile(_assembly, outputFile, expectedCode, results.GeneratedCode);
Assert.NotNull(results.DesignTimeLineMappings); // Guard
if (expectedLineMappings == null ||
!Enumerable.SequenceEqual(expectedLineMappings, results.DesignTimeLineMappings))
{
var lineMappings = new StringBuilder();
lineMappings.AppendLine($"// !!! Do not check in. Instead paste content into test method. !!!");
lineMappings.AppendLine();
var indent = " ";
lineMappings.AppendLine($"{ indent }var expectedLineMappings = new[]");
lineMappings.AppendLine($"{ indent }{{");
foreach (var lineMapping in results.DesignTimeLineMappings)
{
var innerIndent = indent + " ";
var documentLocation = lineMapping.DocumentLocation;
var generatedLocation = lineMapping.GeneratedLocation;
lineMappings.AppendLine($"{ innerIndent }{ nameof(BuildLineMapping) }(");
innerIndent += " ";
lineMappings.AppendLine($"{ innerIndent }documentAbsoluteIndex: { documentLocation.AbsoluteIndex },");
lineMappings.AppendLine($"{ innerIndent }documentLineIndex: { documentLocation.LineIndex },");
lineMappings.AppendLine($"{ innerIndent }documentCharacterIndex: { documentLocation.CharacterIndex },");
lineMappings.AppendLine($"{ innerIndent }generatedAbsoluteIndex: { generatedLocation.AbsoluteIndex },");
lineMappings.AppendLine($"{ innerIndent }generatedLineIndex: { generatedLocation.LineIndex },");
lineMappings.AppendLine($"{ innerIndent }generatedCharacterIndex: { generatedLocation.CharacterIndex },");
lineMappings.AppendLine($"{ innerIndent }contentLength: { generatedLocation.ContentLength }),");
}
lineMappings.AppendLine($"{ indent }}};");
var lineMappingFile = Path.ChangeExtension(outputFile, "lineMappings.cs");
ResourceFile.UpdateFile(_assembly, lineMappingFile, previousContent: null, content: lineMappings.ToString());
}
#else
Assert.Equal(expectedCode, results.GeneratedCode, ignoreLineEndingDifferences: true);
Assert.Equal(expectedLineMappings, results.DesignTimeLineMappings);
#endif
}
/// <summary>
/// Returns job preparation and main Batch tasks that represents the given <see cref="TesTask"/>
/// </summary>
/// <param name="task">The <see cref="TesTask"/></param>
/// <returns>Job preparation and main Batch tasks</returns>
private async Task <(JobPreparationTask, CloudTask)> ConvertTesTaskToBatchTaskAsync(TesTask task)
{
JobPreparationTask jobPreparationTask = null;
var cromwellPathPrefixWithoutEndSlash = CromwellPathPrefix.TrimEnd('/');
var taskId = task.Id;
var queryStringsToRemoveFromLocalFilePaths = task.Inputs
.Select(i => i.Path)
.Concat(task.Outputs.Select(o => o.Path))
.Where(p => p != null)
.Select(p => queryStringRegex.Match(p).Groups[1].Value)
.Where(qs => !string.IsNullOrEmpty(qs))
.ToList();
var inputFiles = task.Inputs.Distinct();
var cromwellExecutionDirectoryPath = GetParentPath(task.Inputs.FirstOrDefault(IsCromwellCommandScript)?.Path);
if (cromwellExecutionDirectoryPath == null)
{
throw new Exception($"Could not identify Cromwell execution directory path for task {task.Id}. This TES instance supports Cromwell tasks only.");
}
// TODO: Cromwell bug: Cromwell command write_tsv() generates a file in the execution directory, for example execution/write_tsv_3922310b441805fc43d52f293623efbc.tmp. These are not passed on to TES inputs.
// WORKAROUND: Get the list of files in the execution directory and add them to task inputs.
var executionDirectoryUri = new Uri(await MapLocalPathToSasUrlAsync(cromwellExecutionDirectoryPath, getContainerSas: true));
var blobsInExecutionDirectory = (await azureProxy.ListBlobsAsync(executionDirectoryUri)).Where(b => !b.EndsWith($"/{CromwellScriptFileName}")).Where(b => !b.EndsWith($"/{DownloadFilesScriptFileName}"));
var additionalInputFiles = blobsInExecutionDirectory.Select(b => $"{CromwellPathPrefix}{b}").Select(b => new TesInput {
Content = null, Path = b, Url = b, Type = TesFileType.FILEEnum
});
var filesToDownload = await Task.WhenAll(inputFiles.Union(additionalInputFiles).Select(async f => await GetTesInputFileUrl(f, task.Id, queryStringsToRemoveFromLocalFilePaths)));
foreach (var output in task.Outputs)
{
if (!output.Path.StartsWith(CromwellPathPrefix, StringComparison.OrdinalIgnoreCase))
{
throw new Exception($"Unsupported output path '{output.Path}' for task Id {task.Id}. Must start with {CromwellPathPrefix}");
}
}
var downloadFilesScriptContent = string.Join(" && ", filesToDownload.Select(f => $"blobxfer download --verbose --enable-azure-storage-logger --storage-url '{f.Url}' --local-path '{f.Path}' --rename --no-recursive"));
var downloadFilesScriptPath = $"{cromwellExecutionDirectoryPath}/{DownloadFilesScriptFileName}";
var writableDownloadFilesScriptUrl = new Uri(await MapLocalPathToSasUrlAsync(downloadFilesScriptPath, getContainerSas: true));
var downloadFilesScriptUrl = await MapLocalPathToSasUrlAsync(downloadFilesScriptPath);
await azureProxy.UploadBlobAsync(writableDownloadFilesScriptUrl, downloadFilesScriptContent);
jobPreparationTask = new JobPreparationTask()
{
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)),
ContainerSettings = new TaskContainerSettings(
imageName: "mcr.microsoft.com/blobxfer",
containerRunOptions: $"--rm -v /mnt{cromwellPathPrefixWithoutEndSlash}:{cromwellPathPrefixWithoutEndSlash} --entrypoint /bin/sh"),
ResourceFiles = new List <ResourceFile> {
ResourceFile.FromUrl(downloadFilesScriptUrl, $"/mnt{downloadFilesScriptPath}")
},
CommandLine = $" -c \"/bin/sh {downloadFilesScriptPath}; chmod -R o+rwx {cromwellPathPrefixWithoutEndSlash} \"",
WaitForSuccess = true
};
var executor = task.Executors.First();
// The first command is sh or bash
var command = executor.Command[0] + " -c \"" + string.Join(" && ",
executor.Command.Skip(1)
.Append($"cd {cromwellPathPrefixWithoutEndSlash}")
.Append("for f in $(find -type l -xtype f);do cp --remove-destination $(readlink $f) $f;done;")) + "\"";
var cromwellExecutionsContainerUri = await MapLocalPathToSasUrlAsync(CromwellPathPrefix);
var cloudTask = new CloudTask(taskId, command)
{
// It seems root always downloads, tasks run as root or user depending on Admin/NonAdmin, and upload is always done by non-root user.
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)),
ContainerSettings = new TaskContainerSettings(
imageName: executor.Image,
containerRunOptions: $"--rm -v /mnt{cromwellPathPrefixWithoutEndSlash}:{cromwellPathPrefixWithoutEndSlash} --entrypoint= --workdir /",
registry: (await GetContainerRegistry(executor.Image))),
OutputFiles = task.Outputs
.Select(o => new OutputFile(
"/mnt" + RemoveQueryStringsFromLocalFilePaths(o.Path, queryStringsToRemoveFromLocalFilePaths) + (o.Type == TesFileType.DIRECTORYEnum ? "/**/*" : string.Empty),
new OutputFileDestination(new OutputFileBlobContainerDestination(cromwellExecutionsContainerUri, o.Path.Substring(CromwellPathPrefix.Length))),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)))
.ToList()
};
return(jobPreparationTask, cloudTask);
}
/// <summary>
/// Converts from an array of ITaskItems to a list of
/// ResourceFiles.
/// </summary>
/// <remarks>
/// The resource name is the filename without any preceding
/// path information.
/// </remarks>
IList<ResourceFile> GetResourceFiles(ITaskItem[] taskItems)
{
List<ResourceFile> files = new List<ResourceFile>();
if (taskItems != null) {
foreach (ITaskItem item in taskItems) {
string resourceFileName = GetFullPath(item.ItemSpec);
string resourceName = GetResourceName(item);
ResourceFile resourceFile = new ResourceFile(resourceName, resourceFileName);
files.Add(resourceFile);
}
}
return files;
}
public ResourceSource(ResourceFile resource, string metadataName)
{
Resource = resource;
MetadataName = metadataName;
}
/// <summary>
/// Embeds a single resource into the assembly.
/// </summary>
void AddResource(ModuleBuilder moduleBuilder, ResourceFile resourceFile)
{
string fileName = resourceFile.FileName;
IResourceWriter resourceWriter = moduleBuilder.DefineResource(Path.GetFileName(fileName), resourceFile.Name, ResourceAttributes.Public);
string extension = Path.GetExtension(fileName).ToLowerInvariant();
if (extension == ".resources") {
ResourceReader resourceReader = new ResourceReader(fileName);
using (resourceReader) {
IDictionaryEnumerator enumerator = resourceReader.GetEnumerator();
while (enumerator.MoveNext()) {
string key = enumerator.Key as string;
Stream resourceStream = enumerator.Value as Stream;
if (resourceStream != null) {
BinaryReader reader = new BinaryReader(resourceStream);
MemoryStream stream = new MemoryStream();
byte[] bytes = reader.ReadBytes((int)resourceStream.Length);
stream.Write(bytes, 0, bytes.Length);
resourceWriter.AddResource(key, stream);
} else {
resourceWriter.AddResource(key, enumerator.Value);
}
}
}
} else {
resourceWriter.AddResource(resourceFile.Name, File.ReadAllBytes(fileName));
}
}
public ActionResult EditFile(ResourceFile model)
{
try
{
model.Extention = model.FileName.Substring(model.FileName.LastIndexOf("."));
model.DateModified = DateTime.Now;
var fileName = string.Format("{0}.{1}", model.UniqueId,
model.FileName.Substring(model.FileName.LastIndexOf(".") + 1));// Path.GetFileName(file.FileName);
model.Path = Path.Combine(Server.MapPath("~/App_Data"), fileName);
var teacher = Academy.Current.GetTeacherById(SessionCache.CurrentUser.DbId);
model.SharedBy = teacher.Id.GetUserNameById();
var fileToEdit_IsDeleted = teacher.Files.Remove(model);
//model.Id = lastFile.Id + 1;
teacher.Files.Add(model);
Academy.Current.AddOrUpdate(teacher);
return RedirectToAction("Files");
}
catch (Exception e)
{
return View(model);
}
}
/// <summary>
/// Determines the namespace of the provided resource file
/// </summary>
/// <param name="file">Reference to the resource file</param>
/// <returns>Namespace to be used to access the resource file</returns>
protected virtual string GetNamespacePrefix(ResourceFile file)
{
if (file == null) {
throw new ArgumentNullException("file");
}
string namespacePrefix = file.CustomToolNamespace;
if (String.IsNullOrEmpty(namespacePrefix)) {
namespacePrefix = file.FileNamespace;
}
if (!String.IsNullOrEmpty(namespacePrefix)) {
namespacePrefix += ".";
}
return namespacePrefix;
}
private static TreeNode CreateChild(TreeNode rootNode, ResourceFile entry)
{
var foldername = GetLastPathElement(entry.Path);
TreeNode newChild = new TreeNode(foldername);
newChild.Name = foldername;
newChild.Tag = (string)rootNode.Tag + foldername + "/";
return newChild;
}
/// <summary>
/// Determines the namespace of the provided resource file
/// </summary>
/// <param name="file">Reference to the resource file</param>
/// <returns>Namespace to be used to access the resource file</returns>
protected override string GetNamespacePrefix(ResourceFile file)
{
return "Resources.";
}
private void GetNamespaceFromUri(Uri uri)
{
ResourceFiles.Clear();
var uriString = uri.OriginalString;
var fileNameIndex = uriString.LastIndexOf('/');
var path = uriString.Substring(0, fileNameIndex);
if( path.StartsWith("/"))
{
path = path.Substring(1);
}
var namespaceString = path.Replace('/', '.');
var assembly = Application.Current.GetType().Assembly;
var assemblyName = "Balder.Silverlight.SampleBrowser";
var fullNamespace = string.Format("{0}.{1}", assemblyName, namespaceString);
var resources = assembly.GetManifestResourceNames();
var query = from r in resources
where r.StartsWith(fullNamespace)
select r;
var resource = query.SingleOrDefault();
if( !string.IsNullOrEmpty(resource))
{
var resourceNameWithoutExtension = resource.Replace(".resources", string.Empty);
var resourceManager = new ResourceManager(resourceNameWithoutExtension, assembly);
var resourceSet = resourceManager.GetResourceSet(CultureInfo.InvariantCulture, true, false);
var enumerator = resourceSet.GetEnumerator();
while( enumerator.MoveNext())
{
var resourceName = enumerator.Key.ToString();
var extension = string.Empty;
var fileName = string.Empty;
var extensionIndex = resourceName.IndexOf('_');
if( extensionIndex > 0 )
{
extension = resourceName.Substring(extensionIndex + 1);
fileName = resourceName.Substring(0, extensionIndex);
} else
{
fileName = resourceName;
}
extension = extension.Replace('_', '.');
var file = new ResourceFile
{
Filename = string.IsNullOrEmpty(extension)
? fileName
: string.Format("{0}.{1}", fileName, extension),
Content = enumerator.Value.ToString()
};
ResourceFiles.Add(file);
}
}
if( ResourceFiles.Count > 0 )
{
SelectedFile = ResourceFiles[0];
}
}
/// <summary>Determines the namespace of the provided resource file</summary>
/// <param name="file">Reference to the resource file</param>
/// <returns>Namespace to be used to access the resource file</returns>
protected override string GetNamespacePrefix(ResourceFile file)
{
return("Resources.");
}
public bool Execute(GeneratorExecutionContext context)
{
Lang language;
switch (Language)
{
case LanguageNames.CSharp:
language = Lang.CSharp;
break;
case LanguageNames.VisualBasic:
language = Lang.VisualBasic;
break;
default:
LogError(Lang.CSharp, $"GenerateResxSource doesn't support language: '{Language}'");
return(false);
}
var extension = language switch
{
Lang.CSharp => "cs",
Lang.VisualBasic => "vb",
_ => "cs",
};
OutputTextHintName = ResourceName + $".Designer.{extension}";
if (string.IsNullOrEmpty(ResourceName))
{
LogError(language, "ResourceName not specified");
return(false);
}
var resourceAccessName = RoslynString.IsNullOrEmpty(ResourceClassName) ? ResourceName : ResourceClassName;
SplitName(resourceAccessName, out var namespaceName, out var className);
var classIndent = namespaceName == null ? "" : " ";
var memberIndent = classIndent + " ";
var text = ResourceFile.GetText(context.CancellationToken);
if (text is null)
{
LogError(language, "ResourceFile was null");
return(false);
}
var strings = new StringBuilder();
foreach (var node in XDocument.Parse(text.ToString()).Descendants("data"))
{
var name = node.Attribute("name")?.Value;
if (name == null)
{
LogError(language, "Missing resource name");
return(false);
}
var value = node.Elements("value").FirstOrDefault()?.Value.Trim();
if (value == null)
{
LogError(language, $"Missing resource value: '{name}'");
return(false);
}
if (name.Length == 0)
{
LogError(language, $"Empty resource name");
return(false);
}
var docCommentString = value.Length > maxDocCommentLength?value.Substring(0, maxDocCommentLength) + " ..." : value;
RenderDocComment(language, memberIndent, strings, docCommentString);
var identifier = GetIdentifierFromResourceName(name);
var defaultValue = IncludeDefaultValues ? ", " + CreateStringLiteral(value, language) : string.Empty;
switch (language)
{
case Lang.CSharp:
if (AsConstants)
{
strings.AppendLine($"{memberIndent}internal const string @{identifier} = \"{name}\";");
}
else
{
strings.AppendLine($"{memberIndent}internal static {(IncludeDefaultValues || !Generator.SupportsNullable(context) ? "string" : "string?")} @{identifier} => GetResourceString(\"{name}\"{defaultValue});");
}
if (EmitFormatMethods)
{
var resourceString = new ResourceString(name, value);
if (resourceString.HasArguments)
{
RenderDocComment(language, memberIndent, strings, docCommentString);
RenderFormatMethod(memberIndent, language, strings, resourceString);
}
}
break;
case Lang.VisualBasic:
if (AsConstants)
{
strings.AppendLine($"{memberIndent}Friend Const [{identifier}] As String = \"{name}\"");
}
else
{
strings.AppendLine($"{memberIndent}Friend Shared ReadOnly Property [{identifier}] As String");
strings.AppendLine($"{memberIndent} Get");
strings.AppendLine($"{memberIndent} Return GetResourceString(\"{name}\"{defaultValue})");
strings.AppendLine($"{memberIndent} End Get");
strings.AppendLine($"{memberIndent}End Property");
}
if (EmitFormatMethods)
{
throw new NotImplementedException();
}
break;
default:
throw new InvalidOperationException();
}
}
INamedTypeSymbol?methodImplOptions = context.Compilation.GetOrCreateTypeByMetadataName(typeof(MethodImplOptions).FullName);
var hasAggressiveInlining = methodImplOptions?.MemberNames.Contains(nameof(MethodImplOptions.AggressiveInlining)) ?? false;
var hasNotNullIfNotNull = context.Compilation.GetOrCreateTypeByMetadataName("System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute") is object;
string?getStringMethod;
if (OmitGetResourceString)
{
getStringMethod = null;
}
else
{
switch (language)
{
case Lang.CSharp:
var getResourceStringAttributes = new List <string>();
if (hasAggressiveInlining)
{
getResourceStringAttributes.Add("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]");
}
if (hasNotNullIfNotNull)
{
getResourceStringAttributes.Add("[return: global::System.Diagnostics.CodeAnalysis.NotNullIfNotNull(\"defaultValue\")]");
}
getStringMethod = $@"{memberIndent}internal static global::System.Globalization.CultureInfo{(Generator.SupportsNullable(context) ? "?" : "")} Culture {{ get; set; }}
{string.Join(Environment.NewLine, getResourceStringAttributes.Select(attr => memberIndent + attr))}
{memberIndent}internal static {(Generator.SupportsNullable(context) ? "string?" : "string")} GetResourceString(string resourceKey, {(Generator.SupportsNullable(context) ? "string?" : "string")} defaultValue = null) => ResourceManager.GetString(resourceKey, Culture) ?? defaultValue;";
if (EmitFormatMethods)
{
getStringMethod += $@"
{memberIndent}private static string GetResourceString(string resourceKey, string[]? formatterNames)
{memberIndent}{{
{memberIndent} var value = GetResourceString(resourceKey) ?? """";
{memberIndent} if (formatterNames != null)
{memberIndent} {{
{memberIndent} for (var i = 0; i < formatterNames.Length; i++)
{memberIndent} {{
{memberIndent} value = value.Replace(""{{"" + formatterNames[i] + ""}}"", ""{{"" + i + ""}}"");
{memberIndent} }}
{memberIndent} }}
{memberIndent} return value;
{memberIndent}}}
";
}
break;
case Lang.VisualBasic:
getStringMethod = $@"{memberIndent}Friend Shared Property Culture As Global.System.Globalization.CultureInfo
{memberIndent}<Global.System.Runtime.CompilerServices.MethodImpl(Global.System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)>
{memberIndent}Friend Shared Function GetResourceString(ByVal resourceKey As String, Optional ByVal defaultValue As String = Nothing) As String
{memberIndent} Return ResourceManager.GetString(resourceKey, Culture)
{memberIndent}End Function";
if (EmitFormatMethods)
{
throw new NotImplementedException();
}
break;
default:
throw new InvalidOperationException();
}
}
string?namespaceStart, namespaceEnd;
if (namespaceName == null)
{
namespaceStart = namespaceEnd = null;
}
else
{
switch (language)
{
case Lang.CSharp:
namespaceStart = $@"namespace {namespaceName}{Environment.NewLine}{{";
namespaceEnd = "}";
break;
case Lang.VisualBasic:
namespaceStart = $"Namespace Global.{namespaceName}";
namespaceEnd = "End Namespace";
break;
default:
throw new InvalidOperationException();
}
}
string resourceTypeName;
string?resourceTypeDefinition;
if (string.IsNullOrEmpty(ResourceClassName) || ResourceName == ResourceClassName)
{
// resource name is same as accessor, no need for a second type.
resourceTypeName = className;
resourceTypeDefinition = null;
}
else
{
// resource name differs from the access class, need a type for specifying the resources
// this empty type must remain as it is required by the .NETNative toolchain for locating resources
// once assemblies have been merged into the application
resourceTypeName = ResourceName;
SplitName(resourceTypeName, out var resourceNamespaceName, out var resourceClassName);
var resourceClassIndent = resourceNamespaceName == null ? "" : " ";
switch (language)
{
case Lang.CSharp:
resourceTypeDefinition = $"{resourceClassIndent}internal static class {resourceClassName} {{ }}";
if (resourceNamespaceName != null)
{
resourceTypeDefinition = $@"namespace {resourceNamespaceName}
{{
{resourceTypeDefinition}
}}";
}
break;
case Lang.VisualBasic:
resourceTypeDefinition = $@"{resourceClassIndent}Friend Class {resourceClassName}
{resourceClassIndent}End Class";
if (resourceNamespaceName != null)
{
resourceTypeDefinition = $@"Namespace {resourceNamespaceName}
{resourceTypeDefinition}
End Namespace";
}
break;
default:
throw new InvalidOperationException();
}
}
// The ResourceManager property being initialized lazily is an important optimization that lets .NETNative
// completely remove the ResourceManager class if the disk space saving optimization to strip resources
// (/DisableExceptionMessages) is turned on in the compiler.
string result;
switch (language)
{
case Lang.CSharp:
result = $@"// <auto-generated/>
{(Generator.SupportsNullable(context) ? "#nullable enable" : "")}
using System.Reflection;
{resourceTypeDefinition}
{namespaceStart}
{classIndent}internal static partial class {className}
{classIndent}{{
{memberIndent}private static global::System.Resources.ResourceManager{(Generator.SupportsNullable(context) ? "?" : "")} s_resourceManager;
{memberIndent}internal static global::System.Resources.ResourceManager ResourceManager => s_resourceManager ?? (s_resourceManager = new global::System.Resources.ResourceManager(typeof({resourceTypeName})));
{getStringMethod}
{strings}
{classIndent}}}
{namespaceEnd}
";
break;
case Lang.VisualBasic:
result = $@"' <auto-generated/>
Imports System.Reflection
{resourceTypeDefinition}
{namespaceStart}
{classIndent}Friend Partial Class {className}
{memberIndent}Private Sub New
{memberIndent}End Sub
{memberIndent}
{memberIndent}Private Shared s_resourceManager As Global.System.Resources.ResourceManager
{memberIndent}Friend Shared ReadOnly Property ResourceManager As Global.System.Resources.ResourceManager
{memberIndent} Get
{memberIndent} If s_resourceManager Is Nothing Then
{memberIndent} s_resourceManager = New Global.System.Resources.ResourceManager(GetType({resourceTypeName}))
{memberIndent} End If
{memberIndent} Return s_resourceManager
{memberIndent} End Get
{memberIndent}End Property
{getStringMethod}
{strings}
{classIndent}End Class
{namespaceEnd}
";
break;
default:
throw new InvalidOperationException();
}
OutputText = SourceText.From(result, Encoding.UTF8, SourceHashAlgorithm.Sha256);
return(true);
}
public void AddResourceFile(string name, string fileName, ResourceAttributes attribs)
{
ResourceFile resfile = new ResourceFile();
resfile.Name = name;
resfile.FileName = fileName;
resfile.Attributes = attribs;
resourceFiles.Add(resfile);
}
/// <summary>
/// Creates a new task.
/// </summary>
/// <param name="parameters">The parameters to use when creating the task.</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
CloudTask task = new CloudTask(parameters.TaskId, parameters.CommandLine);
task.DisplayName = parameters.DisplayName;
task.RunElevated = parameters.RunElevated;
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List <EnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = new List <ResourceFile>();
foreach (DictionaryEntry d in parameters.ResourceFiles)
{
ResourceFile file = new ResourceFile(d.Value.ToString(), d.Key.ToString());
task.ResourceFiles.Add(file);
}
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.Constraints != null)
{
task.Constraints = parameters.Constraints.omObject;
}
if (parameters.DependsOn != null)
{
task.DependsOn = parameters.DependsOn;
}
if (parameters.MultiInstanceSettings != null)
{
Utils.Utils.MultiInstanceSettingsSyncCollections(parameters.MultiInstanceSettings);
task.MultiInstanceSettings = parameters.MultiInstanceSettings.omObject;
}
if (parameters.ApplicationPackageReferences != null)
{
task.ApplicationPackageReferences = parameters.ApplicationPackageReferences.ToList().ConvertAll(apr => apr.omObject);
}
if (parameters.ExitConditions != null)
{
Utils.Utils.ExitConditionsSyncCollections(parameters.ExitConditions);
task.ExitConditions = parameters.ExitConditions.omObject;
}
WriteVerbose(string.Format(Resources.CreatingTask, parameters.TaskId));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = parameters.Context.BatchOMClient.JobOperations;
jobOperations.AddTask(parameters.JobId, task, parameters.AdditionalBehaviors);
}
}
/// <summary>
/// Creates a new task.
/// </summary>
/// <param name="parameters">The parameters to use when creating the task.</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
CloudTask task = new CloudTask(parameters.TaskId, parameters.CommandLine);
task.DisplayName = parameters.DisplayName;
task.RunElevated = parameters.RunElevated;
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List<EnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = new List<ResourceFile>();
foreach (DictionaryEntry d in parameters.ResourceFiles)
{
ResourceFile file = new ResourceFile(d.Value.ToString(), d.Key.ToString());
task.ResourceFiles.Add(file);
}
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.Constraints != null)
{
task.Constraints = parameters.Constraints.omObject;
}
WriteVerbose(string.Format(Resources.CreatingTask, parameters.TaskId));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = parameters.Context.BatchOMClient.JobOperations;
jobOperations.AddTask(parameters.JobId, task, parameters.AdditionalBehaviors);
}
}
/// <summary>
/// Returns job preparation and main Batch tasks that represents the given <see cref="TesTask"/>
/// </summary>
/// <param name="task">The <see cref="TesTask"/></param>
/// <returns>Job preparation and main Batch tasks</returns>
private async Task <CloudTask> ConvertTesTaskToBatchTaskAsync(TesTask task)
{
var cromwellPathPrefixWithoutEndSlash = CromwellPathPrefix.TrimEnd('/');
var taskId = task.Id;
var queryStringsToRemoveFromLocalFilePaths = task.Inputs
.Select(i => i.Path)
.Concat(task.Outputs.Select(o => o.Path))
.Where(p => p != null)
.Select(p => queryStringRegex.Match(p).Groups[1].Value)
.Where(qs => !string.IsNullOrEmpty(qs))
.ToList();
var inputFiles = task.Inputs.Distinct();
var cromwellExecutionDirectoryPath = GetParentPath(task.Inputs.FirstOrDefault(IsCromwellCommandScript)?.Path);
if (cromwellExecutionDirectoryPath == null)
{
throw new Exception($"Could not identify Cromwell execution directory path for task {task.Id}. This TES instance supports Cromwell tasks only.");
}
foreach (var output in task.Outputs)
{
if (!output.Path.StartsWith(CromwellPathPrefix, StringComparison.OrdinalIgnoreCase))
{
throw new Exception($"Unsupported output path '{output.Path}' for task Id {task.Id}. Must start with {CromwellPathPrefix}");
}
}
var batchExecutionDirectoryPath = $"{cromwellExecutionDirectoryPath}/{BatchExecutionDirectoryName}";
// TODO: Cromwell bug: Cromwell command write_tsv() generates a file in the execution directory, for example execution/write_tsv_3922310b441805fc43d52f293623efbc.tmp. These are not passed on to TES inputs.
// WORKAROUND: Get the list of files in the execution directory and add them to task inputs.
var executionDirectoryUri = new Uri(await MapLocalPathToSasUrlAsync(cromwellExecutionDirectoryPath, getContainerSas: true));
var blobsInExecutionDirectory = (await azureProxy.ListBlobsAsync(executionDirectoryUri)).Where(b => !b.EndsWith($"/{CromwellScriptFileName}")).Where(b => !b.Contains($"/{BatchExecutionDirectoryName}/"));
var additionalInputFiles = blobsInExecutionDirectory.Select(b => $"{CromwellPathPrefix}{b}").Select(b => new TesInput {
Content = null, Path = b, Url = b, Name = Path.GetFileName(b), Type = TesFileType.FILEEnum
});
var filesToDownload = await Task.WhenAll(inputFiles.Union(additionalInputFiles).Select(async f => await GetTesInputFileUrl(f, task.Id, queryStringsToRemoveFromLocalFilePaths)));
var downloadFilesScriptContent = string.Join(" && ", filesToDownload.Select(f =>
f.Url.Contains(".blob.core.")
? $"blobxfer download --storage-url '{f.Url}' --local-path '{f.Path}' --chunk-size-bytes 104857600 --rename --no-recursive"
: $"mkdir -p {GetParentPath(f.Path)} && wget -O '{f.Path}' '{f.Url}'"));
var downloadFilesScriptPath = $"{batchExecutionDirectoryPath}/{DownloadFilesScriptFileName}";
var writableDownloadFilesScriptUrl = new Uri(await MapLocalPathToSasUrlAsync(downloadFilesScriptPath, getContainerSas: true));
var downloadFilesScriptUrl = await MapLocalPathToSasUrlAsync(downloadFilesScriptPath);
await azureProxy.UploadBlobAsync(writableDownloadFilesScriptUrl, downloadFilesScriptContent);
var filesToUpload = await Task.WhenAll(
task.Outputs.Select(async f =>
new TesOutput {
Path = f.Path, Url = await MapLocalPathToSasUrlAsync(f.Path, getContainerSas: true), Name = f.Name, Type = f.Type
}));
var uploadFilesScriptContent = string.Join(" && ", filesToUpload.Select(f =>
{
// Ignore missing stdout/stderr files. CWL workflows have an issue where if the stdout/stderr are redirected, they are still listed in the TES outputs
// Syntax is: If file doesn't exist, run a noop (":") operator , otherwise run the upload command
var fileExistsCheck = f.Path.EndsWith("/stdout") || f.Path.EndsWith("/stderr") ? $"[ ! -f '{f.Path}' ] && : || " : "";
return($"{fileExistsCheck}blobxfer upload --storage-url '{f.Url}' --local-path '{f.Path}' --one-shot-bytes 104857600 {(f.Type == TesFileType.FILEEnum ? "--rename --no-recursive" : "")}");
}));
var uploadFilesScriptPath = $"{batchExecutionDirectoryPath}/{UploadFilesScriptFileName}";
var writableUploadFilesScriptUrl = new Uri(await MapLocalPathToSasUrlAsync(uploadFilesScriptPath, getContainerSas: true));
var uploadFilesScriptUrl = await MapLocalPathToSasUrlAsync(uploadFilesScriptPath);
await azureProxy.UploadBlobAsync(writableUploadFilesScriptUrl, uploadFilesScriptContent);
var executor = task.Executors.First();
var volumeMountsOption = $"-v /mnt{cromwellPathPrefixWithoutEndSlash}:{cromwellPathPrefixWithoutEndSlash}";
var executorImageIsPublic = (await azureProxy.GetContainerRegistryInfoAsync(executor.Image)) == null;
var taskCommand = $@"
docker pull --quiet {BlobxferImageName} && \
{(executorImageIsPublic ? $"docker pull --quiet {executor.Image} &&" : "")} \
docker run --rm {volumeMountsOption} --entrypoint=/bin/sh {BlobxferImageName} {downloadFilesScriptPath} && \
chmod -R o+rwx /mnt{cromwellPathPrefixWithoutEndSlash} && \
docker run --rm {volumeMountsOption} --entrypoint= --workdir / {executor.Image} {executor.Command[0]} -c '{ string.Join(" && ", executor.Command.Skip(1))}' && \
docker run --rm {volumeMountsOption} --entrypoint=/bin/sh {BlobxferImageName} {uploadFilesScriptPath}
";
var batchExecutionDirectoryUrl = await MapLocalPathToSasUrlAsync($"{batchExecutionDirectoryPath}", getContainerSas : true);
var cloudTask = new CloudTask(taskId, $"/bin/sh -c \"{taskCommand.Trim()}\"")
{
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)),
ResourceFiles = new List <ResourceFile> {
ResourceFile.FromUrl(downloadFilesScriptUrl, $"/mnt{downloadFilesScriptPath}"), ResourceFile.FromUrl(uploadFilesScriptUrl, $"/mnt{uploadFilesScriptPath}")
},
OutputFiles = new List <OutputFile> {
new OutputFile(
"../std*.txt",
new OutputFileDestination(new OutputFileBlobContainerDestination(batchExecutionDirectoryUrl)),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskFailure))
}
};
if (!executorImageIsPublic)
{
// If the executor image is private, and in order to run multiple containers in the main task, the image has to be downloaded via pool ContainerConfiguration.
// This also requires that the main task runs inside a container. So we run the "docker" container that in turn runs other containers.
// If the executor image is public, there is no need for pool ContainerConfiguration and task can run normally, without being wrapped in a docker container.
// Volume mapping for docker.sock below allows the docker client in the container to access host's docker daemon.
var containerRunOptions = $"--rm -v /var/run/docker.sock:/var/run/docker.sock -v /mnt{cromwellPathPrefixWithoutEndSlash}:/mnt{cromwellPathPrefixWithoutEndSlash} ";
cloudTask.ContainerSettings = new TaskContainerSettings(DockerInDockerImageName, containerRunOptions);
}
return(cloudTask);
}
public void SelectFile(ResourceFile file)
{
SelectedFile = file;
}
/// <summary>
/// Determines the namespace of the provided resource file
/// </summary>
/// <param name="file">Reference to the resource file</param>
/// <returns>Namespace to be used to access the resource file</returns>
protected virtual string GetNamespacePrefix(ResourceFile file)
{
string namespacePrefix = String.Empty;
try {
namespacePrefix = file.CustomToolNamespace;
if (String.IsNullOrEmpty(namespacePrefix)) {
if (file.FileNamespace != null) {
namespacePrefix = file.FileNamespace;
} else {
namespacePrefix = file.Item.ContainingProject.Properties.Item("DefaultNamespace").Value.ToString();
}
}
namespacePrefix += ".";
} catch (ArgumentException) {
namespacePrefix = String.Empty;
}
return namespacePrefix;
}
