public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
{
if (hiveJobCreateParameters.Query.IsNullOrEmpty())
{
hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
if (hiveJobCreateParameters.File.Contains("://") &&
!hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
}
}
else
{
this.PrepareQueryJob(hiveJobCreateParameters);
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails
{
Name = hiveJobCreateParameters.JobName,
Query = hiveJobCreateParameters.Query,
StatusDirectory = hiveJobCreateParameters.StatusFolder
},
hiveJobCreateParameters.JobName);
return(TaskEx2.FromResult(retval));
}
public Task <ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
LastCreateRequest = clusterCreateParameters;
var clusterDetails = new ClusterDetails();
if (clusterCreateParameters.EnsureHighAvailability)
{
clusterDetails.ClusterSizeInNodes = clusterCreateParameters.ClusterSizeInNodes + 2;
}
clusterDetails.Name = clusterCreateParameters.Name;
clusterDetails.HttpPassword = clusterCreateParameters.Password;
clusterDetails.HttpUserName = clusterCreateParameters.UserName;
clusterDetails.Version = clusterCreateParameters.Version;
clusterDetails.Location = clusterCreateParameters.Location;
clusterDetails.State = ClusterState.Running;
clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts;
clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration(
clusterCreateParameters.DefaultStorageAccountName,
clusterCreateParameters.DefaultStorageAccountKey,
clusterCreateParameters.DefaultStorageContainer);
Clusters.Add(new SimulatorClusterContainer {
Cluster = clusterDetails
});
return(TaskEx2.FromResult(clusterDetails));
}
public Task<Collection<HDInsightVersion>> ListAvailableVersionsAsync()
{
var versions = new Collection<HDInsightVersion>();
versions.Add(new HDInsightVersion { Version = "1.2", VersionStatus = VersionStatus.Obsolete });
versions.Add(new HDInsightVersion { Version = "1.5", VersionStatus = VersionStatus.Obsolete });
versions.Add(new HDInsightVersion { Version = "1.6", VersionStatus = VersionStatus.Compatible });
versions.Add(new HDInsightVersion { Version = "2.1", VersionStatus = VersionStatus.Compatible });
return TaskEx2.FromResult(versions);
}
public Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
{
if (streamingMapReduceJobCreateParameters == null)
{
throw new ArgumentNullException("streamingMapReduceJobCreateParameters");
}
return
(TaskEx2.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName)));
}
public Task <JobDetails> StopJobAsync(string jobId)
{
JobDetails jobToStop = this.GetJob(jobId);
lock (this.cluster.JobDeletionQueue)
{
this.cluster.JobDeletionQueue.Add(jobId, DateTime.Now.AddSeconds(0.5));
}
return(TaskEx2.FromResult(jobToStop));
}
public Task <Stream> GetJobTaskLogSummaryAsync(string jobId)
{
Stream resultStream = new MemoryStream();
if (this.taskLogs.ContainsKey(jobId))
{
resultStream = GetStream(this.taskLogs[jobId]);
}
return(TaskEx2.FromResult(resultStream));
}
public Task <Stream> GetJobOutputAsync(string jobId)
{
Stream resultStream = new MemoryStream();
if (this.jobOutput.ContainsKey(jobId))
{
resultStream = GetStream(this.jobOutput[jobId]);
}
return(TaskEx2.FromResult(resultStream));
}
public Task <JobCreationResults> CreateMapReduceJobAsync(MapReduceJobCreateParameters mapReduceJobCreateParameters)
{
if (mapReduceJobCreateParameters.JobName == "1456577")
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "{ \"error\": \"File /example/files/WordCount.jar does not exist.\"}");
}
mapReduceJobCreateParameters.JarFile.ArgumentNotNullOrEmpty("JarFile");
mapReduceJobCreateParameters.ClassName.ArgumentNotNullOrEmpty("ClassName");
return(TaskEx2.FromResult(this.CreateJobSuccessResult(mapReduceJobCreateParameters, mapReduceJobCreateParameters.JobName)));
}
public Task<ICollection<ClusterDetails>> ListClustersAsync()
{
this.LogMessage("Getting hdinsight clusters for subscriptionid : {0}", this.credentials.SubscriptionId.ToString());
ICollection<ClusterDetails> resultClusters = new Collection<ClusterDetails>(Clusters.Select(c => c.Cluster).ToList());
foreach (ClusterDetails cluster in resultClusters)
{
cluster.SubscriptionId = this.credentials.SubscriptionId;
}
return TaskEx2.FromResult(resultClusters);
}
public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters)
{
if (sqoopJobCreateParameters == null)
{
throw new ArgumentNullException("sqoopJobCreateParameters");
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails {
Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder
}, string.Empty);
return(TaskEx2.FromResult(retval));
}
public Task <JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters)
{
if (pigJobCreateParameters == null)
{
throw new ArgumentNullException("pigJobCreateParameters");
}
this.PrepareQueryJob(pigJobCreateParameters);
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails {
Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder
}, string.Empty);
return(TaskEx2.FromResult(retval));
}
public Task DownloadJobTaskLogsAsync(string jobId, string targetDirectory)
{
if (this.taskLogAttempts.ContainsKey(jobId))
{
if (!Directory.Exists(targetDirectory))
{
Directory.CreateDirectory(targetDirectory);
}
foreach (string file in this.taskLogAttempts[jobId])
{
File.Create(Path.Combine(targetDirectory, file));
}
}
else
{
throw new InvalidOperationException("Job with id " + jobId + " does not exist.");
}
return(TaskEx2.FromResult(""));
}
public Task<IEnumerable<KeyValuePair<string, string>>> ListResourceProviderPropertiesAsync()
{
var resourceProviderProperties = new List<KeyValuePair<string, string>>();
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_FEATURE_CUSTOM_ACTIONS_V2", "Custom Actions V2"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_EAST_US", "East US"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_EAST_US_2", "East US 2"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_NORTH_EUROPE", "North Europe"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.2", "1.2.0.0.LargeSKU-amd64-134231"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_2.1", "2.1.0.0.LargeSKU-amd64-134231"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.5", "1.5.0.0.LargeSKU-amd64-134231"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.6", "1.6.0.0.LargeSKU-amd64-134231"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_Count", "1"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_CoresUsed", "168"));
resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_MaxCoresAllowed", "170"));
return TaskEx2.FromResult(resourceProviderProperties.AsEnumerable());
}
public Task <JobDetails> GetJobAsync(string jobId)
{
this.LogMessage("Getting jobDetails '{0}'.", jobId);
lock (this.cluster.JobQueue)
{
if (this.cluster.JobQueue.ContainsKey(jobId))
{
JobList jobHistory = this.ListJobs();
JobDetails jobHistoryItem = jobHistory.Jobs.FirstOrDefault(job => job.JobId == jobId);
if (jobHistoryItem != null)
{
if (jobHistoryItem.Name.IsNotNull() && jobHistoryItem.Name.IndexOf("timeout", StringComparison.OrdinalIgnoreCase) > -1)
{
Thread.Sleep(50);
}
return(TaskEx2.FromResult(jobHistoryItem));
}
}
}
return(TaskEx2.FromResult <JobDetails>(null));
}
public Task <JobList> ListJobsAsync()
{
this.LogMessage("Listing jobs");
var jobDetailList = new JobList();
var changedJobs = new List <JobDetails>();
lock (this.cluster.JobQueue)
{
foreach (JobDetails jobHistoryItem in this.cluster.JobQueue.Values)
{
JobDetails changedJob = this.ChangeJobState(jobHistoryItem);
jobDetailList.Jobs.Add(changedJob);
changedJobs.Add(changedJob);
}
foreach (JobDetails changedJob in changedJobs)
{
this.cluster.JobQueue[changedJob.JobId] = changedJob;
}
}
return(TaskEx2.FromResult(jobDetailList));
}
public Task <Collection <string> > ListAvailableLocationsAsync()
{
return(TaskEx2.FromResult(new Collection <string> {
"East US", "East US 2", "West US", "North Europe"
}));
}
