/// <summary>
/// Initializes a new instance of the PowerShell ISE color formatter
/// with the specified runtime.
/// </summary>
/// <param name="runspace">
/// The runtime to use for formatting instructions.
/// </param>
public PowerShellIseColorFormatter(IRunspace runspace)
{
if (runspace == null)
{
throw new ArgumentNullException("runspace");
}
_runspace = runspace;
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveHiveConfig()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var hiveConfig = new Hashtable();
hiveConfig.Add("hadoop.logfiles.size", "12345");
var hiveServiceConfig = new AzureHDInsightHiveConfiguration
{
Configuration = hiveConfig,
AdditionalLibraries =
new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
}
};
var hiveConfig2 = new Hashtable();
hiveConfig2.Add("hadoop.logfiles.size2", "12345");
var hiveServiceConfig2 = new AzureHDInsightHiveConfiguration
{
Configuration = hiveConfig2,
AdditionalLibraries =
new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
}
};
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, new AzureHDInsightConfig())
.WithParameter(CmdletConstants.HiveConfig, hiveServiceConfig)
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.HiveConfig, hiveServiceConfig2)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First();
ValidateConfigurationOptions(hiveConfig, config.HiveConfiguration.ConfigurationCollection);
Assert.IsNotNull(config.HiveConfiguration.AdditionalLibraries);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Container, hiveServiceConfig2.AdditionalLibraries.StorageContainerName);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Key, hiveServiceConfig2.AdditionalLibraries.StorageAccountKey);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Name, hiveServiceConfig2.AdditionalLibraries.StorageAccountName);
}
}
internal static void RevokeRdpAccessToCluster(
IHDInsightCertificateCredential connectionCredentials, AzureHDInsightCluster cluster, IRunspace runspace)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.RevokeAzureHDInsightRdpAccess)
.WithParameter(CmdletConstants.Location, cluster.Location)
.WithParameter(CmdletConstants.Name, cluster.Name)
.Invoke();
AzureHDInsightCluster accessRevokedCluster = GetCluster(connectionCredentials, cluster.Name, runspace);
Assert.NotNull(accessRevokedCluster);
Assert.True(string.IsNullOrEmpty(accessRevokedCluster.RdpUserName));
}
public void Start(IRunspace runspaceClient, string scriptId, string scriptName, IScriptExecutionStoreProvider scriptExecutionWriter)
{
int maxGetLastScriptFailures = 3;
int lastScriptFailures = 0;
Task.Run(() => {
IScriptExecutionResult scriptExecutionResult = null;
do
{
try {
scriptExecutionResult = runspaceClient.GetScript(scriptId);
} catch (Exception exc) {
lastScriptFailures++;
_logger.Log(LogLevel.Error, exc.ToString());
}
scriptExecutionWriter.WriteScriptExecution(new NamedScriptExecution(scriptName, scriptExecutionResult));
scriptExecutionWriter.WriteScriptExecutionOutput(new ScriptExecutionOutput(scriptExecutionResult));
scriptExecutionWriter.WriteScriptExecutionDataStreams(new ScriptExecutionDataStreams(scriptExecutionResult?.Streams));
Thread.Sleep(500);
} while (lastScriptFailures < maxGetLastScriptFailures && scriptExecutionResult.State == ScriptState.Running);
if (lastScriptFailures >= maxGetLastScriptFailures)
{
// Retrieval of last script failed which mean script has been lost because
// bad communication with the runspace
// 1. Read the last persisted result
// 2. Update Script ExecutionState to Error
// 3. Write script state
var lastPersistedScript = scriptExecutionWriter.ReadScriptExecution();
var updatedScriptExecution = new NamedScriptExecution {
Name = lastPersistedScript?.Name,
Id = lastPersistedScript?.Id,
StarTime = lastPersistedScript?.StarTime,
EndTime = DateTime.Now,
OutputObjectsFormat = lastPersistedScript?.OutputObjectsFormat ?? OutputObjectsFormat.Text,
State = ScriptState.Error,
Reason = APIGatewayResources.PollingScriptExecutionPersister_ScriptFailed_RunspaceDisappeared
};
scriptExecutionWriter.WriteScriptExecution(updatedScriptExecution);
}
scriptExecutionWriter.Flush();
ScriptResultPersisted?.Invoke(this, new ScriptResultStoredEventArgs {
ScriptId = scriptId
});
});
}
public void CanCallTheGetHDInsightPropertiesCmdletWithoutCertificate()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightProperties)
.Invoke();
HDInsightGetPropertiesCommandTests.ValidateCapabilities(results.Results.ToEnumerable <AzureHDInsightCapabilities>());
}
}
public virtual void ICanCallThe_Start_HDInsightJobsCmdlet()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
RunJobInPowershell(runspace, mapReduceJobDefinition);
}
}
public void ICanCreateAClusterUsingPowerShell()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results = runspace.NewPipeline().AddCommand(CmdletConstants.NewAzureHDInsightCluster)
// Ensure that the subscription Id can be accepted as a string as well as a guid.
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(
CmdletConstants.DefaultStorageAccountName,
TestCredentials.Environments[0].DefaultStorageAccount.Name)
.WithParameter(
CmdletConstants.DefaultStorageAccountKey,
TestCredentials.Environments[0].DefaultStorageAccount.Key)
.WithParameter(
CmdletConstants.DefaultStorageContainerName,
TestCredentials.Environments[0].DefaultStorageAccount.Container)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(0, getCommand.Output.Count);
}
}
private static void ConnectToCluster(IRunspace runspace, IHDInsightCertificateCredential creds)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
ValidateGetCluster(currentCluster.Cluster);
}
public void StartStoringScriptExecution(
string userId,
IRunspace runspaceClient,
string scriptId,
string scriptName)
{
var fileStoreProvider = _scriptExecutionWriterFactory.Create(_logger, _rootFolder, userId, scriptId, _fileSystem);
var pollingScriptExecutionPersister = _scriptExecutionPersisterFactory.Create(_logger);
pollingScriptExecutionPersister.ScriptResultPersisted += PollingScriptExecutionPersister_ScriptResultPersisted;
AddScriptStorageControllers(scriptId, fileStoreProvider, pollingScriptExecutionPersister);
pollingScriptExecutionPersister.Start(runspaceClient, scriptId, scriptName, fileStoreProvider);
}
public void SetUpMultipleScriptsForDifferentUsers()
{
var loggerFactoryMock = new Mock <ILoggerFactory>();
ILogger logger = new Mock <ILogger>().Object;
loggerFactoryMock.Setup(m => m.CreateLogger(typeof(ScriptExecutionFileStorage).FullName)).Returns(logger);
_fileSystem = new MockFileSystem();
var runspaceMock = new Mock <IRunspace>();
_scriptExecutionFileStorage = new ScriptExecutionFileStorage(
loggerFactoryMock.Object,
new ScriptExecutionStorageSettings {
ServiceScriptStorageDir = _rootPath
},
_fileSystem,
new ScriptExecutionFileStoreProviderFactory(),
new PollingScriptExecutionPersisterFactory());
var script1ResultMock = new Mock <IScriptExecutionResult>();
script1ResultMock.SetupGet(m => m.Id).Returns(_scriptId1);
script1ResultMock.Setup(m => m.State).Returns(ScriptState.Success);
script1ResultMock.Setup(m => m.OutputObjectCollection).Returns(new OutputObjectCollection());
runspaceMock.Setup(m => m.GetScript(_scriptId1)).Returns(() => script1ResultMock.Object);
var script2ResultMock = new Mock <IScriptExecutionResult>();
script2ResultMock.SetupGet(m => m.Id).Returns(_scriptId2);
script2ResultMock.Setup(m => m.State).Returns(ScriptState.Success);
script2ResultMock.Setup(m => m.OutputObjectCollection).Returns(new OutputObjectCollection {
FormattedTextPresentation = _script2Output
});
script2ResultMock.Setup(m => m.Streams).Returns(new DataStreams {
Information = _script2InfoStream
});
runspaceMock.Setup(m => m.GetScript(_scriptId2)).Returns(() => script2ResultMock.Object);
var script3ResultMock = new Mock <IScriptExecutionResult>();
script3ResultMock.SetupGet(m => m.Id).Returns(_scriptId3);
script3ResultMock.Setup(m => m.State).Returns(ScriptState.Success);
script3ResultMock.Setup(m => m.OutputObjectCollection).Returns(new OutputObjectCollection());
runspaceMock.Setup(m => m.GetScript(_scriptId3)).Returns(() => script3ResultMock.Object);
_runspace = runspaceMock.Object;
}
public void ICanCallThe_Get_ClusterHDInsightClusterCmdlet()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.Invoke();
IEnumerable <AzureHDInsightCluster> clusters = results.Results.ToEnumerable <AzureHDInsightCluster>();
AzureHDInsightCluster wellKnownCluster = clusters.FirstOrDefault(cluster => cluster.Name == TestCredentials.WellKnownCluster.DnsName);
Assert.IsNotNull(wellKnownCluster);
}
}
public void ICanCallThe_Get_ClusterHDInsightClusterCmdlet_WithADnsName()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightCluster cluster = results.Results.ToEnumerable <AzureHDInsightCluster>().Last();
ValidateGetCluster(cluster);
}
}
public void ICanCallThe_Get_HDInsightJobsCmdlet_WithNonExistantJobId()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
string jobId = Guid.NewGuid().ToString();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword))
.WithParameter(CmdletConstants.Id, jobId)
.Invoke();
Assert.AreEqual(results.Results.Count, 0);
}
}
public void ICanCallThe_Start_HDInsightJobsCmdlet_WithDebug()
{
var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
{
JobName = "pi estimation jobDetails",
ClassName = "pi",
JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
string expectedLogMessage = string.Format(CultureInfo.InvariantCulture, "Starting jobDetails '{0}'.", mapReduceJobDefinition.JobName);
RunJobInPowershell(
runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster(), true, expectedLogMessage);
}
}
public void ICanCreateAClusterWithThreeOrMoreAsvAccountsUsingPowerShell()
{
string dnsName = this.GetRandomClusterName();
var storageAccounts = GetWellKnownStorageAccounts().ToList();
Assert.IsTrue(storageAccounts.Count >= 3);
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, storageAccounts[0].Name)
.WithParameter(CmdletConstants.StorageAccountKey, storageAccounts[0].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, storageAccounts[1].Name)
.WithParameter(CmdletConstants.StorageAccountKey, storageAccounts[1].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, storageAccounts[2].Name)
.WithParameter(CmdletConstants.StorageAccountKey, storageAccounts[2].Key)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
AzureHDInsightCluster clusterFromPowershell = results.Results.ToEnumerable <AzureHDInsightCluster>().First();
List <AzureHDInsightStorageAccount> storageAccountsFromPowershell = clusterFromPowershell.StorageAccounts.ToList();
Assert.AreEqual(3, storageAccountsFromPowershell.Count);
foreach (WabStorageAccountConfiguration storageAccount in storageAccounts)
{
AzureHDInsightStorageAccount additionalStorageAccountFromOutput =
storageAccountsFromPowershell.FirstOrDefault(acc => acc.StorageAccountName == storageAccount.Name);
Assert.IsNotNull(additionalStorageAccountFromOutput, "Storage account " + storageAccount.Name + " was not found");
Assert.AreEqual(storageAccount.Key, additionalStorageAccountFromOutput.StorageAccountKey);
}
}
}
public void ICannotCallThe_New_HDInsightPigJobDefinitionCmdlet_WithoutFileOrQueryParameter()
{
try
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
runspace.NewPipeline().AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition).Invoke();
Assert.Fail("test failed.");
}
}
catch (CmdletInvocationException invokeException)
{
var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
Assert.IsNotNull(psArgumentException);
Assert.AreEqual("Either File or Query should be specified for Pig jobs.", psArgumentException.Message);
}
}
public void ICanCallThe_Get_HDInsightJobsCmdlet()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword))
.Invoke();
IEnumerable <AzureHDInsightJob> jobHistory = results.Results.ToEnumerable <AzureHDInsightJob>();
JobList expectedJobHistory = HDInsightGetJobsCommandTests.GetJobHistory(testCluster.ConnectionUrl);
Assert.AreEqual(expectedJobHistory.Jobs.Count, jobHistory.Count(), "Should have {0} jobs.", expectedJobHistory.Jobs.Count);
}
}
public void ICanCallThe_Get_HDInsightJobsCmdlet_WithNonExistantJobId()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
string jobId = Guid.NewGuid().ToString();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJobOutput)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.WithParameter(CmdletConstants.Id, jobId)
.WithParameter(CmdletConstants.StdErr, null)
.Invoke();
Assert.IsTrue(results.Results.ToEnumerable <string>().All(string.IsNullOrEmpty));
}
}
public void CanCallTheGetHDInsightPropertiesCmdletWithDebugSwitch()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightProperties)
.WithParameter(CmdletConstants.Debug, null)
.Invoke();
List <KeyValuePair <string, string> > capabilities =
results.Results.ToEnumerable <IEnumerable <KeyValuePair <string, string> > >().SelectMany(ver => ver.ToList()).ToList();
Assert.IsNotNull(capabilities);
Assert.IsTrue(capabilities.Count > 0);
}
}
internal static AzureHDInsightCluster GetClusterWithHttpAccessDisabled(IRunspace runspace)
{
IHDInsightCertificateCredential creds = GetValidCredentials();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.Invoke();
List<AzureHDInsightCluster> testClusters = results.Results.ToEnumerable<AzureHDInsightCluster>().ToList();
AzureHDInsightCluster testCluster = testClusters.FirstOrDefault(cluster => cluster.HttpUserName.IsNullOrEmpty());
if (testCluster == null)
{
testCluster = testClusters.Last();
RevokeHttpAccessToCluster(creds, testCluster, runspace);
}
return testCluster;
}
public virtual void WaitForInvalidJobDoesNotThrow()
{
var jobDetails = new JobDetails {
JobId = Guid.NewGuid().ToString()
};
var invalidJob = new AzureHDInsightJob(jobDetails, TestCredentials.WellKnownCluster.DnsName);
// IHadoopClientExtensions.GetPollingInterval = () => 0;
ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
runspace.NewPipeline()
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.Job, invalidJob)
.Invoke();
}
}
internal static AzureHDInsightCluster GetClusterWithHttpAccessDisabled(IRunspace runspace)
{
IHDInsightCertificateCredential creds = GetValidCredentials();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.Invoke();
List <AzureHDInsightCluster> testClusters = results.Results.ToEnumerable <AzureHDInsightCluster>().ToList();
AzureHDInsightCluster testCluster = testClusters.FirstOrDefault(cluster => cluster.HttpUserName.IsNullOrEmpty());
if (testCluster == null)
{
testCluster = testClusters.Last();
RevokeHttpAccessToCluster(creds, testCluster, runspace);
}
return(testCluster);
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutJobName()
{
var HiveJobDefinition = new HiveJobCreateParameters {
Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
}
}
public void ICanCallThe_Get_HDInsightJobsCmdletWithJobId()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword))
.Invoke();
var jobDetail = results.Results.ElementAt(0).ImmediateBaseObject as AzureHDInsightJobBase;
Assert.IsNotNull(jobDetail);
AzureHDInsightJob getJobDetailObj = GetJobWithID(runspace, jobDetail.JobId, testCluster);
Assert.IsNotNull(getJobDetailObj);
Assert.AreEqual(jobDetail.JobId, getJobDetailObj.JobId);
}
}
public virtual void NewPigJob_StartJob_GetJob()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
AzureHDInsightJob pigJobfromHistory = RunJobAndGetWithId(runspace, pigJobFromPowershell);
Assert.AreEqual(pigJobfromHistory.Query, pigJobDefinition.Query, "Failed to retrieve query for executed pig jobDetails");
}
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_MapReduceConfig()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var coreConfig = new Hashtable();
coreConfig.Add("hadoop.logfiles.size", "12345");
var mapRedConfig = new AzureHDInsightMapReduceConfiguration {
Configuration = coreConfig
};
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, new AzureHDInsightConfig())
.WithParameter(CmdletConstants.MapReduceConfig, mapRedConfig)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First();
ValidateConfigurationOptions(coreConfig, config.MapReduceConfiguration.ConfigurationCollection);
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithFileParameter()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.File, sqoopJobDefinition.File)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.File, SqoopJobFromPowershell.File);
}
}
public virtual void ICanCallThe_NewPigJob_Then_Start_HDInsightJobsCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load table from 'A'"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
RunJobInPowershell(runspace, pigJobFromPowershell);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
Command = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command);
}
}
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithParameters()
{
var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
{
JobName = "pi estimation jobDetails",
Input = TestConstants.WabsProtocolSchemeName + "input",
Output = TestConstants.WabsProtocolSchemeName + "input",
Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
};
streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000");
streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
.WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
.WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
.WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
.WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
.WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
.WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
.WithParameter(CmdletConstants.Parameters, streamingMapReduceJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First();
AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
foreach (var parameter in streamingMapReduceJobDefinition.Defines)
{
Assert.IsTrue(
streamingMapReduceJobFromPowershell.Defines.Any(
arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
internal static AzureHDInsightJob RunJobInPowershell(
IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IEnumerable <AzureHDInsightJob> jobCreationCmdletResults = results.Results.ToEnumerable <AzureHDInsightJob>();
AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
return(jobCreationResults);
}
public void ICanCallThe_Connect_ClusterHDInsightClusterCmdlet()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate <IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
Assert.IsNotNull(currentCluster);
ValidateGetCluster(currentCluster.Cluster);
}
}
internal static AzureHDInsightJob GetJobWithID(IRunspace runspace, string jobId, ClusterDetails cluster)
{
IPipelineResult getJobDetailResults =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.Id, jobId)
.Invoke();
return getJobDetailResults.Results.ToEnumerable<AzureHDInsightJob>().FirstOrDefault();
}
private static void RunConfigOptionstest(
IRunspace runspace, string configOptionName, Hashtable expected, Func<AzureHDInsightConfig, ConfigValuesCollection> configPropertyAccessor)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, new AzureHDInsightConfig())
.WithParameter(configOptionName, expected)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable<AzureHDInsightConfig>().First();
ValidateConfigurationOptions(expected, configPropertyAccessor(config));
}
protected IRunspace GetPowerShellRunspace(string location)
{
if (this.runspace.IsNull())
{
// string loc = typeof(GetAzureHDInsightClusterCmdlet).Assembly.Location;
this.runspace = Help.SafeCreate(() => RunspaceAbstraction.Create());
this.runspace.NewPipeline().AddCommand("Import-Module").WithParameter("Name", location).Invoke();
}
return this.runspace;
}
private static void ConnectToCluster(IRunspace runspace, IHDInsightCertificateCredential creds)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.UseAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, TestCredentials.WellKnownCluster.DnsName)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IAzureHDInsightConnectionSessionManager sessionManager =
ServiceLocator.Instance.Locate<IAzureHDInsightConnectionSessionManagerFactory>().Create(null);
AzureHDInsightClusterConnection currentCluster = sessionManager.GetCurrentCluster();
ValidateGetCluster(currentCluster.Cluster);
}
internal static AzureHDInsightJob RunJobInPowershell(
IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster, bool debug, string expectedLogMessage)
{
IPipelineResult result = null;
if (debug)
{
var logWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = logWriter;
result =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.WithParameter(CmdletConstants.Debug, null)
.Invoke();
Assert.IsTrue(logWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
BufferingLogWriterFactory.Reset();
}
else
{
result =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.Invoke();
}
Assert.AreEqual(1, result.Results.Count);
IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = result.Results.ToEnumerable<AzureHDInsightJob>();
AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
return jobCreationResults;
}
internal static AzureHDInsightJob RunJobInPowershell(
IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = results.Results.ToEnumerable<AzureHDInsightJob>();
AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
return jobCreationResults;
}
internal static AzureHDInsightJob RunJobInPowershell(IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition)
{
return RunJobInPowershell(runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster());
}
private static AzureHDInsightClusterConnection ConnectToCluster(IRunspace runspace, IHDInsightCertificateCredential creds)
{
IUseAzureHDInsightClusterCommand connectCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateUseCluster();
connectCommand.CurrentSubscription = GetCurrentSubscription();
connectCommand.Name = TestCredentials.WellKnownCluster.DnsName;
connectCommand.EndProcessing();
Assert.AreEqual(1, connectCommand.Output.Count);
AzureHDInsightClusterConnection currentCluster = connectCommand.Output.First();
ValidateGetCluster(currentCluster.Cluster);
return currentCluster;
}
internal static AzureHDInsightCluster GetCluster(
IHDInsightCertificateCredential connectionCredentials, string clusterName, IRunspace runspace)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, clusterName)
.Invoke();
List<AzureHDInsightCluster> clusters = results.Results.ToEnumerable<AzureHDInsightCluster>().ToList();
return clusters.FirstOrDefault();
}
