public void ICanCallThe_Connect_ClusterHDInsightClusterCmdlet_WithDebug()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var logWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = logWriter;
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.WithParameter(CmdletConstants.Debug, null)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
Assert.IsNotNull(currentCluster);
string expectedLogMessage = "Getting hdinsight clusters for subscriptionid : ";
ValidateGetCluster(currentCluster.Cluster);
Assert.IsTrue(logWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
BufferingLogWriterFactory.Reset();
}
}
public void CanConnectToValidClustersMoreThanOnce()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IHDInsightCertificateCredential creds = GetValidCredentials();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
IEnumerable <AzureHDInsightCluster> clusters = results.Results.ToEnumerable <AzureHDInsightCluster>();
foreach (AzureHDInsightCluster cluster in clusters)
{
IUseAzureHDInsightClusterCommand connectCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateUseCluster();
connectCommand.CurrentSubscription = GetCurrentSubscription();
connectCommand.Name = cluster.Name;
connectCommand.EndProcessing();
Assert.AreEqual(1, connectCommand.Output.Count);
AzureHDInsightClusterConnection currentCluster = connectCommand.Output.First();
Assert.IsNotNull(currentCluster);
ValidateGetCluster(cluster, currentCluster.Cluster);
}
}
}
public void CanCallTheExecHiveCommand_DoesNotUploadFile()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
AzureHDInsightClusterConnection connection = ConnectToCluster(runspace, creds);
var storageHandlerSimulator = new AzureHDInsightStorageHandlerSimulator();
AzureHDInsightStorageHandlerSimulatorFactory.Instance = storageHandlerSimulator;
IInvokeHiveCommand execHiveCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateInvokeHive();
execHiveCommand.JobDefinition = new AzureHDInsightHiveJobDefinition {
File = "query.hql", JobName = "show tables"
};
execHiveCommand.CurrentSubscription = GetCurrentSubscription();
execHiveCommand.Connection = connection;
execHiveCommand.EndProcessing();
string outputContent = execHiveCommand.Output.Last();
Assert.AreEqual(5, execHiveCommand.Output.Count());
Assert.AreEqual("hivesampletable", outputContent);
Assert.IsNull(storageHandlerSimulator.UploadedStream);
Assert.IsNull(storageHandlerSimulator.Path);
}
}
public void ICanCallThe_Connect_ClusterHDInsightClusterCmdlet_MoreThanOnce()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IHDInsightCertificateCredential credentials = GetValidCredentials();
IPipelineResult clusterResults =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
IEnumerable <AzureHDInsightCluster> clusters = clusterResults.Results.ToEnumerable <AzureHDInsightCluster>();
foreach (AzureHDInsightCluster cluster in clusters)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
Assert.IsNotNull(currentCluster);
ValidateGetCluster(cluster, currentCluster.Cluster);
}
}
}
public async Task EndProcessing()
{
this.Connection.ArgumentNotNull("Connection");
this.JobDefinition.ArgumentNotNull("HiveJob");
this.Output.Clear();
AzureHDInsightClusterConnection currentConnection = this.Connection;
this.WriteOutput("Submitting Hive query..");
AzureHDInsightHiveJobDefinition hiveJobDefinition = this.JobDefinition;
if (hiveJobDefinition.Query.IsNotNullOrEmpty() && hiveJobDefinition.File.IsNullOrEmpty())
{
hiveJobDefinition = this.UploadFileToStorage(this.JobDefinition, currentConnection);
}
var startedJob = await this.StartJob(hiveJobDefinition, currentConnection);
this.JobId = startedJob.JobId;
this.WriteOutput("Started Hive query with jobDetails Id : {0}", startedJob.JobId);
var completedJob = await this.WaitForCompletion(startedJob, currentConnection);
if (completedJob.ExitCode == Success)
{
await this.WriteJobSuccess(completedJob, currentConnection);
}
else
{
await this.WriteJobFailure(completedJob, currentConnection);
}
}
private static AzureHDInsightClusterConnection ConnectToCluster(IRunspace runspace, IHDInsightCertificateCredential creds)
{
IUseAzureHDInsightClusterCommand connectCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateUseCluster();
connectCommand.CurrentSubscription = GetCurrentSubscription();
connectCommand.Name = TestCredentials.WellKnownCluster.DnsName;
connectCommand.EndProcessing();
Assert.AreEqual(1, connectCommand.Output.Count);
AzureHDInsightClusterConnection currentCluster = connectCommand.Output.First();
ValidateGetCluster(currentCluster.Cluster);
return(currentCluster);
}
private static void ConnectToCluster(IRunspace runspace, IHDInsightCertificateCredential creds)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
ValidateGetCluster(currentCluster.Cluster);
}
public void CanConnectToValidCluster()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
IUseAzureHDInsightClusterCommand connectCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateUseCluster();
connectCommand.CurrentSubscription = GetCurrentSubscription();
connectCommand.Name = TestCredentials.WellKnownCluster.DnsName;
connectCommand.EndProcessing();
Assert.AreEqual(1, connectCommand.Output.Count);
AzureHDInsightClusterConnection currentCluster = connectCommand.Output.First();
Assert.IsNotNull(currentCluster);
ValidateGetCluster(currentCluster.Cluster);
}
protected virtual async Task <AzureHDInsightJob> StartJob(
AzureHDInsightJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
jobDefinition.ArgumentNotNull("jobDefinition");
currentConnection.ArgumentNotNull("currentCluster");
this.ValidateNotCanceled();
var startJobCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateStartJob();
this.SetClient(startJobCommand);
startJobCommand.Cluster = currentConnection.Cluster.Name;
startJobCommand.Logger = this.Logger;
startJobCommand.CurrentSubscription = this.CurrentSubscription;
startJobCommand.JobDefinition = jobDefinition;
this.ValidateNotCanceled();
await startJobCommand.EndProcessing();
return(startJobCommand.Output.Last());
}
public static AzureHDInsightClusterConnection AssertValidConnection(this PSCmdlet cmdlet)
{
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(cmdlet.SessionState);
AzureHDInsightClusterConnection currentConnection = sessionManager.GetCurrentCluster();
if (currentConnection == null)
{
cmdlet.ThrowTerminatingError(
new ErrorRecord(
new NotSupportedException("Please connect to a valid Azure HDInsight cluster before calling this cmdlet."),
"1024",
ErrorCategory.ConnectionError,
cmdlet));
}
return(currentConnection);
}
protected virtual async Task <AzureHDInsightJob> WaitForCompletion(
AzureHDInsightJob startedJob, AzureHDInsightClusterConnection currentConnection)
{
startedJob.ArgumentNotNull("startedJob");
currentConnection.ArgumentNotNull("currentConnection");
this.ValidateNotCanceled();
var waitJobCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateWaitJobs();
waitJobCommand.JobStatusEvent += this.ClientOnJobStatus;
this.SetClient(waitJobCommand);
waitJobCommand.Job = startedJob;
waitJobCommand.Logger = this.Logger;
waitJobCommand.WaitTimeoutInSeconds = WaitAnHourInSeconds;
waitJobCommand.CurrentSubscription = this.CurrentSubscription;
this.ValidateNotCanceled();
await waitJobCommand.ProcessRecord();
return(waitJobCommand.Output.Last());
}
public void CanCallTheExecHiveCommandWithAFailedQuery()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
AzureHDInsightClusterConnection connection = ConnectToCluster(runspace, creds);
IInvokeHiveCommand execHiveCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateInvokeHive();
execHiveCommand.CurrentSubscription = GetCurrentSubscription();
execHiveCommand.JobDefinition = new AzureHDInsightHiveJobDefinition {
Query = "show tableaus", JobName = "Fail_this_job"
};
execHiveCommand.Connection = connection;
execHiveCommand.EndProcessing();
string outputContent = execHiveCommand.Output.Last();
Assert.AreEqual(5, execHiveCommand.Output.Count());
Assert.AreEqual(AzureHDInsightJobSubmissionClientSimulator.JobFailed, outputContent);
}
}
/// <summary>
/// Finishes the execution of the cmdlet by listing the clusters.
/// </summary>
protected override void EndProcessing()
{
this.WriteWarning(string.Format(AzureHdInsightPowerShellConstants.AsmWarning, "Use-AzHDInsightCluster"));
try
{
this.command.Logger = this.Logger;
this.command.CurrentSubscription = this.GetCurrentSubscription(this.Subscription, this.Certificate);
Task task = this.command.EndProcessing();
CancellationToken token = this.command.CancellationToken;
while (!task.IsCompleted)
{
this.WriteDebugLog();
task.Wait(1000, token);
}
if (task.IsFaulted)
{
throw new AggregateException(task.Exception);
}
AzureHDInsightClusterConnection connection = this.command.Output.FirstOrDefault();
var sessionManager = ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(this.SessionState);
sessionManager.SetCurrentCluster(connection);
this.WriteObject(string.Format(CultureInfo.InvariantCulture, "Successfully connected to cluster {0}", this.Name));
this.WriteDebugLog();
}
catch (Exception ex)
{
Type type = ex.GetType();
this.Logger.Log(Severity.Error, Verbosity.Normal, this.FormatException(ex));
this.WriteDebugLog();
if (type == typeof(AggregateException) || type == typeof(TargetInvocationException) || type == typeof(TaskCanceledException))
{
ex.Rethrow();
}
else
{
throw;
}
}
this.WriteDebugLog();
}
protected virtual async Task WriteJobSuccess(AzureHDInsightJob completedJob, AzureHDInsightClusterConnection currentConnection)
{
completedJob.ArgumentNotNull("completedJob");
currentConnection.ArgumentNotNull("currentConnection");
this.WriteOutput("Hive query completed Successfully");
this.WriteOutput(Environment.NewLine);
this.ValidateNotCanceled();
var getJobOutputCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobOutput();
this.SetClient(getJobOutputCommand);
getJobOutputCommand.JobId = completedJob.JobId;
getJobOutputCommand.Logger = this.Logger;
getJobOutputCommand.CurrentSubscription = this.CurrentSubscription;
getJobOutputCommand.Cluster = currentConnection.Cluster.Name;
this.ValidateNotCanceled();
await getJobOutputCommand.EndProcessing();
var outputStream = getJobOutputCommand.Output.First();
string content = new StreamReader(outputStream).ReadToEnd();
this.WriteOutput(content);
}
public void CanCallTheExecHiveCommand_UploadsFile()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
AzureHDInsightClusterConnection connection = ConnectToCluster(runspace, creds);
var storageHandlerSimulator = new AzureHDInsightStorageHandlerSimulator();
AzureHDInsightStorageHandlerSimulatorFactory.Instance = storageHandlerSimulator;
IInvokeHiveCommand execHiveCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateInvokeHive();
execHiveCommand.JobDefinition = new AzureHDInsightHiveJobDefinition {
Query = "show tables", JobName = "show tables"
};
execHiveCommand.CurrentSubscription = GetCurrentSubscription();
execHiveCommand.Connection = connection;
execHiveCommand.EndProcessing();
string destinationPath = string.Format(
CultureInfo.InvariantCulture,
"http://{0}/{1}/user/{2}/",
connection.Cluster.DefaultStorageAccount.StorageAccountName,
connection.Cluster.DefaultStorageAccount.StorageContainerName,
connection.Cluster.HttpUserName);
string outputContent = execHiveCommand.Output.Last();
Assert.IsNotNull(storageHandlerSimulator.UploadedStream);
storageHandlerSimulator.UploadedStream.Seek(0, SeekOrigin.Begin);
string contents = new StreamReader(storageHandlerSimulator.UploadedStream).ReadToEnd();
Assert.AreEqual("show tables", contents);
Assert.IsFalse(string.IsNullOrEmpty(storageHandlerSimulator.Path.OriginalString));
Assert.IsTrue(storageHandlerSimulator.Path.OriginalString.StartsWith(destinationPath, StringComparison.OrdinalIgnoreCase));
Assert.AreEqual(5, execHiveCommand.Output.Count());
Assert.AreEqual("hivesampletable", outputContent);
}
}
private AzureHDInsightHiveJobDefinition UploadFileToStorage(
AzureHDInsightHiveJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
currentConnection.Cluster.DefaultStorageAccount.ArgumentNotNull("DefaultStorageAccount");
WabStorageAccountConfiguration storageAccount = currentConnection.Cluster.DefaultStorageAccount.ToWabStorageAccountConfiguration();
var storageHandler = ServiceLocator.Instance.Locate <IAzureHDInsightStorageHandlerFactory>().Create(storageAccount);
string hiveQueryFilePath = string.Format(
CultureInfo.InvariantCulture,
HiveQueryFileStoragePath,
currentConnection.Cluster.DefaultStorageAccount.StorageAccountName,
currentConnection.Cluster.DefaultStorageAccount.StorageContainerName,
currentConnection.Cluster.HttpUserName,
Guid.NewGuid().ToString("N"));
using (Stream hiveQueryFileStream = this.GetStreamForText(jobDefinition.Query))
{
var hiveQueryFileUri = new Uri(hiveQueryFilePath, UriKind.RelativeOrAbsolute);
storageHandler.UploadFile(hiveQueryFileUri, hiveQueryFileStream);
jobDefinition.Query = string.Empty;
jobDefinition.File = storageHandler.GetStoragePath(hiveQueryFileUri).OriginalString;
}
return(jobDefinition);
}
protected virtual async Task<AzureHDInsightJob> WaitForCompletion(
AzureHDInsightJob startedJob, AzureHDInsightClusterConnection currentConnection)
{
startedJob.ArgumentNotNull("startedJob");
currentConnection.ArgumentNotNull("currentConnection");
this.ValidateNotCanceled();
var waitJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateWaitJobs();
waitJobCommand.JobStatusEvent += this.ClientOnJobStatus;
this.SetClient(waitJobCommand);
waitJobCommand.Job = startedJob;
waitJobCommand.Logger = this.Logger;
waitJobCommand.WaitTimeoutInSeconds = WaitAnHourInSeconds;
waitJobCommand.CurrentSubscription = this.CurrentSubscription;
this.ValidateNotCanceled();
await waitJobCommand.ProcessRecord();
return waitJobCommand.Output.Last();
}
/// <inheritdoc />
protected override void EndProcessing()
{
AzureHDInsightClusterConnection currentConnection = this.AssertValidConnection();
this.hiveJobDefinitionCommand.EndProcessing().Wait();
AzureHDInsightHiveJobDefinition hiveJob = this.hiveJobDefinitionCommand.Output.Last();
this.command.JobDefinition = hiveJob;
this.command.Output.CollectionChanged += this.OutputItemAdded;
this.command.Connection = currentConnection;
try
{
this.command.Logger = this.Logger;
this.command.CurrentSubscription = this.GetCurrentSubscription(string.Empty, null);
Task task = this.command.EndProcessing();
CancellationToken token = this.command.CancellationToken;
while (!task.IsCompleted)
{
this.WriteDebugLog();
task.Wait(1000, token);
if (this.command.JobDetailsStatus.IsNotNull())
{
string msg = string.Format(CultureInfo.CurrentCulture, "Waiting for jobDetails : {0}", this.command.JobId);
var record = new ProgressRecord(
0, msg, this.command.JobDetailsStatus.StatusCode.ToString() + " : " + this.command.JobDetailsStatus.PercentComplete);
this.WriteProgress(record);
}
while (this.queue.Count > 0)
{
lock (this.queue)
{
this.WriteObject(this.queue.Dequeue(), true);
}
}
}
if (task.IsFaulted)
{
throw new AggregateException(task.Exception);
}
this.WriteDebugLog();
}
catch (Exception ex)
{
Type type = ex.GetType();
this.Logger.Log(Severity.Error, Verbosity.Normal, this.FormatException(ex));
this.WriteDebugLog();
if (type == typeof(AggregateException) || type == typeof(TargetInvocationException) || type == typeof(TaskCanceledException))
{
ex.Rethrow();
}
else
{
throw;
}
}
this.WriteDebugLog();
while (this.queue.Count > 0)
{
lock (this.queue)
{
this.WriteObject(this.queue.Dequeue(), true);
}
}
}
protected virtual async Task<AzureHDInsightJob> StartJob(
AzureHDInsightJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
jobDefinition.ArgumentNotNull("jobDefinition");
currentConnection.ArgumentNotNull("currentCluster");
this.ValidateNotCanceled();
var startJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateStartJob();
this.SetClient(startJobCommand);
startJobCommand.Cluster = currentConnection.Cluster.Name;
startJobCommand.Logger = this.Logger;
startJobCommand.CurrentSubscription = this.CurrentSubscription;
startJobCommand.JobDefinition = jobDefinition;
this.ValidateNotCanceled();
await startJobCommand.EndProcessing();
return startJobCommand.Output.Last();
}
public void SetCurrentCluster(AzureHDInsightClusterConnection cluster)
{
this.sessionState.PSVariable.Set(CurrentClusterVariableName, cluster);
}
private AzureHDInsightHiveJobDefinition UploadFileToStorage(
AzureHDInsightHiveJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection)
{
currentConnection.Cluster.DefaultStorageAccount.ArgumentNotNull("DefaultStorageAccount");
WabStorageAccountConfiguration storageAccount = currentConnection.Cluster.DefaultStorageAccount.ToWabStorageAccountConfiguration();
var storageHandler = ServiceLocator.Instance.Locate<IAzureHDInsightStorageHandlerFactory>().Create(storageAccount);
string hiveQueryFilePath = string.Format(
CultureInfo.InvariantCulture,
HiveQueryFileStoragePath,
currentConnection.Cluster.DefaultStorageAccount.StorageAccountName,
currentConnection.Cluster.DefaultStorageAccount.StorageContainerName,
currentConnection.Cluster.HttpUserName,
Guid.NewGuid().ToString("N"));
using (Stream hiveQueryFileStream = this.GetStreamForText(jobDefinition.Query))
{
var hiveQueryFileUri = new Uri(hiveQueryFilePath, UriKind.RelativeOrAbsolute);
storageHandler.UploadFile(hiveQueryFileUri, hiveQueryFileStream);
jobDefinition.Query = string.Empty;
jobDefinition.File = storageHandler.GetStoragePath(hiveQueryFileUri).OriginalString;
}
return jobDefinition;
}
public void SetCurrentCluster(AzureHDInsightClusterConnection cluster)
{
this.currentCluster = cluster;
}
public void SetCurrentCluster(AzureHDInsightClusterConnection cluster)
{
this.currentCluster = cluster;
}
protected virtual async Task WriteJobSuccess(AzureHDInsightJob completedJob, AzureHDInsightClusterConnection currentConnection)
{
completedJob.ArgumentNotNull("completedJob");
currentConnection.ArgumentNotNull("currentConnection");
this.WriteOutput("Hive query completed Successfully");
this.WriteOutput(Environment.NewLine);
this.ValidateNotCanceled();
var getJobOutputCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateGetJobOutput();
this.SetClient(getJobOutputCommand);
getJobOutputCommand.JobId = completedJob.JobId;
getJobOutputCommand.Logger = this.Logger;
getJobOutputCommand.CurrentSubscription = this.CurrentSubscription;
getJobOutputCommand.Cluster = currentConnection.Cluster.Name;
this.ValidateNotCanceled();
await getJobOutputCommand.EndProcessing();
var outputStream = getJobOutputCommand.Output.First();
string content = new StreamReader(outputStream).ReadToEnd();
this.WriteOutput(content);
}
public void SetCurrentCluster(AzureHDInsightClusterConnection cluster)
{
this.sessionState.PSVariable.Set(CurrentClusterVariableName, cluster);
}
