public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters) { if (hiveJobCreateParameters.Query.IsNullOrEmpty()) { hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File"); if (hiveJobCreateParameters.File.Contains("://") && !hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File); } } else { this.PrepareQueryJob(hiveJobCreateParameters); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Name = hiveJobCreateParameters.JobName, Query = hiveJobCreateParameters.Query, StatusDirectory = hiveJobCreateParameters.StatusFolder }, hiveJobCreateParameters.JobName); return(TaskEx2.FromResult(retval)); }
public Task <ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters) { this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location); LastCreateRequest = clusterCreateParameters; var clusterDetails = new ClusterDetails(); if (clusterCreateParameters.EnsureHighAvailability) { clusterDetails.ClusterSizeInNodes = clusterCreateParameters.ClusterSizeInNodes + 2; } clusterDetails.Name = clusterCreateParameters.Name; clusterDetails.HttpPassword = clusterCreateParameters.Password; clusterDetails.HttpUserName = clusterCreateParameters.UserName; clusterDetails.Version = clusterCreateParameters.Version; clusterDetails.Location = clusterCreateParameters.Location; clusterDetails.State = ClusterState.Running; clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts; clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration( clusterCreateParameters.DefaultStorageAccountName, clusterCreateParameters.DefaultStorageAccountKey, clusterCreateParameters.DefaultStorageContainer); Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails }); return(TaskEx2.FromResult(clusterDetails)); }
public Task<Collection<HDInsightVersion>> ListAvailableVersionsAsync() { var versions = new Collection<HDInsightVersion>(); versions.Add(new HDInsightVersion { Version = "1.2", VersionStatus = VersionStatus.Obsolete }); versions.Add(new HDInsightVersion { Version = "1.5", VersionStatus = VersionStatus.Obsolete }); versions.Add(new HDInsightVersion { Version = "1.6", VersionStatus = VersionStatus.Compatible }); versions.Add(new HDInsightVersion { Version = "2.1", VersionStatus = VersionStatus.Compatible }); return TaskEx2.FromResult(versions); }
public Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { if (streamingMapReduceJobCreateParameters == null) { throw new ArgumentNullException("streamingMapReduceJobCreateParameters"); } return (TaskEx2.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName))); }
public Task <JobDetails> StopJobAsync(string jobId) { JobDetails jobToStop = this.GetJob(jobId); lock (this.cluster.JobDeletionQueue) { this.cluster.JobDeletionQueue.Add(jobId, DateTime.Now.AddSeconds(0.5)); } return(TaskEx2.FromResult(jobToStop)); }
public Task <Stream> GetJobTaskLogSummaryAsync(string jobId) { Stream resultStream = new MemoryStream(); if (this.taskLogs.ContainsKey(jobId)) { resultStream = GetStream(this.taskLogs[jobId]); } return(TaskEx2.FromResult(resultStream)); }
public Task <Stream> GetJobOutputAsync(string jobId) { Stream resultStream = new MemoryStream(); if (this.jobOutput.ContainsKey(jobId)) { resultStream = GetStream(this.jobOutput[jobId]); } return(TaskEx2.FromResult(resultStream)); }
public Task <JobCreationResults> CreateMapReduceJobAsync(MapReduceJobCreateParameters mapReduceJobCreateParameters) { if (mapReduceJobCreateParameters.JobName == "1456577") { throw new HttpLayerException(HttpStatusCode.BadRequest, "{ \"error\": \"File /example/files/WordCount.jar does not exist.\"}"); } mapReduceJobCreateParameters.JarFile.ArgumentNotNullOrEmpty("JarFile"); mapReduceJobCreateParameters.ClassName.ArgumentNotNullOrEmpty("ClassName"); return(TaskEx2.FromResult(this.CreateJobSuccessResult(mapReduceJobCreateParameters, mapReduceJobCreateParameters.JobName))); }
public Task<ICollection<ClusterDetails>> ListClustersAsync() { this.LogMessage("Getting hdinsight clusters for subscriptionid : {0}", this.credentials.SubscriptionId.ToString()); ICollection<ClusterDetails> resultClusters = new Collection<ClusterDetails>(Clusters.Select(c => c.Cluster).ToList()); foreach (ClusterDetails cluster in resultClusters) { cluster.SubscriptionId = this.credentials.SubscriptionId; } return TaskEx2.FromResult(resultClusters); }
public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters) { if (sqoopJobCreateParameters == null) { throw new ArgumentNullException("sqoopJobCreateParameters"); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder }, string.Empty); return(TaskEx2.FromResult(retval)); }
public Task <JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters) { if (pigJobCreateParameters == null) { throw new ArgumentNullException("pigJobCreateParameters"); } this.PrepareQueryJob(pigJobCreateParameters); JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty); return(TaskEx2.FromResult(retval)); }
public Task DownloadJobTaskLogsAsync(string jobId, string targetDirectory) { if (this.taskLogAttempts.ContainsKey(jobId)) { if (!Directory.Exists(targetDirectory)) { Directory.CreateDirectory(targetDirectory); } foreach (string file in this.taskLogAttempts[jobId]) { File.Create(Path.Combine(targetDirectory, file)); } } else { throw new InvalidOperationException("Job with id " + jobId + " does not exist."); } return(TaskEx2.FromResult("")); }
public Task<IEnumerable<KeyValuePair<string, string>>> ListResourceProviderPropertiesAsync() { var resourceProviderProperties = new List<KeyValuePair<string, string>>(); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_FEATURE_CUSTOM_ACTIONS_V2", "Custom Actions V2")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_EAST_US", "East US")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_EAST_US_2", "East US 2")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_REGION_NORTH_EUROPE", "North Europe")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.2", "1.2.0.0.LargeSKU-amd64-134231")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_2.1", "2.1.0.0.LargeSKU-amd64-134231")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.5", "1.5.0.0.LargeSKU-amd64-134231")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CAPABILITY_VERSION_1.6", "1.6.0.0.LargeSKU-amd64-134231")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_Count", "1")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_CoresUsed", "168")); resourceProviderProperties.Add(new KeyValuePair<string, string>("CONTAINERS_MaxCoresAllowed", "170")); return TaskEx2.FromResult(resourceProviderProperties.AsEnumerable()); }
public Task <JobDetails> GetJobAsync(string jobId) { this.LogMessage("Getting jobDetails '{0}'.", jobId); lock (this.cluster.JobQueue) { if (this.cluster.JobQueue.ContainsKey(jobId)) { JobList jobHistory = this.ListJobs(); JobDetails jobHistoryItem = jobHistory.Jobs.FirstOrDefault(job => job.JobId == jobId); if (jobHistoryItem != null) { if (jobHistoryItem.Name.IsNotNull() && jobHistoryItem.Name.IndexOf("timeout", StringComparison.OrdinalIgnoreCase) > -1) { Thread.Sleep(50); } return(TaskEx2.FromResult(jobHistoryItem)); } } } return(TaskEx2.FromResult <JobDetails>(null)); }
public Task <JobList> ListJobsAsync() { this.LogMessage("Listing jobs"); var jobDetailList = new JobList(); var changedJobs = new List <JobDetails>(); lock (this.cluster.JobQueue) { foreach (JobDetails jobHistoryItem in this.cluster.JobQueue.Values) { JobDetails changedJob = this.ChangeJobState(jobHistoryItem); jobDetailList.Jobs.Add(changedJob); changedJobs.Add(changedJob); } foreach (JobDetails changedJob in changedJobs) { this.cluster.JobQueue[changedJob.JobId] = changedJob; } } return(TaskEx2.FromResult(jobDetailList)); }
public Task <Collection <string> > ListAvailableLocationsAsync() { return(TaskEx2.FromResult(new Collection <string> { "East US", "East US 2", "West US", "North Europe" })); }