public void CanCreateNewHiveJob_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(hiveJobFromCommand);
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutJobName()
{
var HiveJobDefinition = new HiveJobCreateParameters {
Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
}
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
AssertQueryDoesNotContainRestrictedCharacters(azureHiveJobDefinition.Query, "Query");
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File,
RunAsFileJob = azureHiveJobDefinition.RunAsFileJob
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
AzureHDInsightJob jobHistoryResult = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
}
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithOutputStorageLocation()
{
var HiveJobDefinition = new HiveJobCreateParameters
{
JobName = "show tables jobDetails",
Query = "show tables",
StatusFolder = "/tablesList"
};
HiveJobDefinition.Defines.Add("map.input.tasks", "1000");
HiveJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.WithParameter(CmdletConstants.Parameters, HiveJobDefinition.Defines)
.WithParameter(CmdletConstants.StatusFolder, HiveJobDefinition.StatusFolder)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
Assert.AreEqual(HiveJobDefinition.StatusFolder, HiveJobFromPowershell.StatusFolder);
foreach (var parameter in HiveJobDefinition.Defines)
{
Assert.IsTrue(
HiveJobFromPowershell.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithFileParameter()
{
var HiveJobDefinition = new HiveJobCreateParameters
{
JobName = "show tables jobDetails",
File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.File, HiveJobDefinition.File)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.File, HiveJobFromPowershell.File);
}
}
private AzureHDInsightHiveJobDefinition UploadFileToStorage(
AzureHDInsightHiveJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
currentConnection.Cluster.DefaultStorageAccount.ArgumentNotNull("DefaultStorageAccount");
WabStorageAccountConfiguration storageAccount = currentConnection.Cluster.DefaultStorageAccount.ToWabStorageAccountConfiguration();
var storageHandler = ServiceLocator.Instance.Locate <IAzureHDInsightStorageHandlerFactory>().Create(storageAccount);
string hiveQueryFilePath = string.Format(
CultureInfo.InvariantCulture,
HiveQueryFileStoragePath,
currentConnection.Cluster.DefaultStorageAccount.StorageAccountName,
currentConnection.Cluster.DefaultStorageAccount.StorageContainerName,
currentConnection.Cluster.HttpUserName,
Guid.NewGuid().ToString("N"));
using (Stream hiveQueryFileStream = this.GetStreamForText(jobDefinition.Query))
{
var hiveQueryFileUri = new Uri(hiveQueryFilePath, UriKind.RelativeOrAbsolute);
storageHandler.UploadFile(hiveQueryFileUri, hiveQueryFileStream);
jobDefinition.Query = string.Empty;
jobDefinition.File = storageHandler.GetStoragePath(hiveQueryFileUri).OriginalString;
}
return(jobDefinition);
}
public void StartJob()
{
// Update HDInsight Management properties for Job.
SetupManagementClientForJobTests();
var cmdlet = new StartAzureHDInsightJobCommand
{
CommandRuntime = commandRuntimeMock.Object,
HDInsightJobClient = hdinsightJobManagementMock.Object,
HDInsightManagementClient = hdinsightManagementMock.Object,
HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
ClusterName = ClusterName
};
var args = new[] { "arg1", "arg2" };
const string query = "show tables;";
const string name = "hivejob";
var hivedef = new AzureHDInsightHiveJobDefinition
{
JobName = name,
Query = query,
};
foreach (var arg in args)
{
hivedef.Arguments.Add(arg);
}
cmdlet.JobDefinition = hivedef;
const string jobid = "jobid_1984120_001";
var jobsub = new JobSubmissionResponse
{
JobSubmissionJsonResponse = new JobSubmissionJsonResponse
{
Id = jobid
},
StatusCode = HttpStatusCode.OK
};
hdinsightJobManagementMock.Setup(
c =>
c.SubmitHiveJob(
It.Is <AzureHDInsightHiveJobDefinition>(
def => def.JobName == name && def.Query == query && def.Arguments.Count == args.Length)))
.Returns(jobsub)
.Verifiable();
var getresponse = new JobGetResponse
{
StatusCode = HttpStatusCode.OK,
JobDetail = new JobDetailRootJsonObject
{
Completed = "false",
User = cmdlet.HttpCredential.UserName,
Id = jobid,
Status = new Status(),
Userargs = new Userargs()
}
};
hdinsightJobManagementMock.Setup(c => c.GetJob(jobsub.JobSubmissionJsonResponse.Id)).Returns(getresponse).Verifiable();
cmdlet.ExecuteCmdlet();
commandRuntimeMock.VerifyAll();
commandRuntimeMock.Verify(
f =>
f.WriteObject(
It.Is <AzureHDInsightJob>(
job => job.Cluster == ClusterName && job.JobId == jobid && job.Completed == "false")));
}
public void CanStartHiveJob()
{
var hiveJobDefinition = new AzureHDInsightHiveJobDefinition { JobName = "pi estimation jobDetails", Query = "show tables;" };
TestJobStart(hiveJobDefinition);
}
public void CanCreateNewHiveJob_WithoutJobName_WithQuery()
{
var hiveJobDefinition = new AzureHDInsightHiveJobDefinition { Query = "show tables" };
AzureHDInsightJob startedJob = TestJobStart(hiveJobDefinition);
Assert.AreEqual(startedJob.Name, "Hive: show tables");
}
public void CanCreateNewHiveJob_WithoutJobName_WithFile()
{
var hiveJobDefinition = new AzureHDInsightHiveJobDefinition
{
File = TestConstants.WabsProtocolSchemeName + "container@hostname/Container1/myqueryfile.hql"
};
AzureHDInsightJob startedJob = TestJobStart(hiveJobDefinition);
Assert.AreEqual(startedJob.Name, "Hive: myqueryfile.hql");
}
private AzureHDInsightHiveJobDefinition UploadFileToStorage(
AzureHDInsightHiveJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
currentConnection.Cluster.DefaultStorageAccount.ArgumentNotNull("DefaultStorageAccount");
WabStorageAccountConfiguration storageAccount = currentConnection.Cluster.DefaultStorageAccount.ToWabStorageAccountConfiguration();
var storageHandler = ServiceLocator.Instance.Locate<IAzureHDInsightStorageHandlerFactory>().Create(storageAccount);
string hiveQueryFilePath = string.Format(
CultureInfo.InvariantCulture,
HiveQueryFileStoragePath,
currentConnection.Cluster.DefaultStorageAccount.StorageAccountName,
currentConnection.Cluster.DefaultStorageAccount.StorageContainerName,
currentConnection.Cluster.HttpUserName,
Guid.NewGuid().ToString("N"));
using (Stream hiveQueryFileStream = this.GetStreamForText(jobDefinition.Query))
{
var hiveQueryFileUri = new Uri(hiveQueryFilePath, UriKind.RelativeOrAbsolute);
storageHandler.UploadFile(hiveQueryFileUri, hiveQueryFileStream);
jobDefinition.Query = string.Empty;
jobDefinition.File = storageHandler.GetStoragePath(hiveQueryFileUri).OriginalString;
}
return jobDefinition;
}
/// <inheritdoc />
protected override void EndProcessing()
{
AzureHDInsightClusterConnection currentConnection = this.AssertValidConnection();
this.hiveJobDefinitionCommand.EndProcessing().Wait();
AzureHDInsightHiveJobDefinition hiveJob = this.hiveJobDefinitionCommand.Output.Last();
this.command.JobDefinition = hiveJob;
this.command.Output.CollectionChanged += this.OutputItemAdded;
this.command.Connection = currentConnection;
try
{
this.command.Logger = this.Logger;
this.command.CurrentSubscription = this.GetCurrentSubscription(string.Empty, null);
Task task = this.command.EndProcessing();
CancellationToken token = this.command.CancellationToken;
while (!task.IsCompleted)
{
this.WriteDebugLog();
task.Wait(1000, token);
if (this.command.JobDetailsStatus.IsNotNull())
{
string msg = string.Format(CultureInfo.CurrentCulture, "Waiting for jobDetails : {0}", this.command.JobId);
var record = new ProgressRecord(
0, msg, this.command.JobDetailsStatus.StatusCode.ToString() + " : " + this.command.JobDetailsStatus.PercentComplete);
this.WriteProgress(record);
}
while (this.queue.Count > 0)
{
lock (this.queue)
{
this.WriteObject(this.queue.Dequeue(), true);
}
}
}
if (task.IsFaulted)
{
throw new AggregateException(task.Exception);
}
this.WriteDebugLog();
}
catch (Exception ex)
{
Type type = ex.GetType();
this.Logger.Log(Severity.Error, Verbosity.Normal, this.FormatException(ex));
this.WriteDebugLog();
if (type == typeof(AggregateException) || type == typeof(TargetInvocationException) || type == typeof(TaskCanceledException))
{
ex.Rethrow();
}
else
{
throw;
}
}
this.WriteDebugLog();
while (this.queue.Count > 0)
{
lock (this.queue)
{
this.WriteObject(this.queue.Dequeue(), true);
}
}
}
