/// <summary>
/// List jobs.
/// </summary>
/// <param name="resourceGroupName">Name of the resource group.</param>
/// <param name="jobCollectionName">job collection name.</param>
/// <param name="jobName">Name of the job.</param>
/// <param name="jobState">State of the job.</param>
/// <returns>List of job definition.</returns>
public IList <PSSchedulerJobDefinition> ListJobs(string resourceGroupName, string jobCollectionName, string jobName = "", string jobState = "")
{
if (string.IsNullOrWhiteSpace(resourceGroupName))
{
throw new PSManagement.PSArgumentNullException(paramName: "ResourceGroupName");
}
if (string.IsNullOrWhiteSpace(jobCollectionName))
{
throw new PSManagement.PSArgumentNullException(paramName: "JobCollectionName");
}
JobState?jobStateEnum = jobState.GetValueOrDefaultEnum <JobState?>(defaultValue: null);
IList <JobDefinition> listOfJobs;
if (!string.IsNullOrWhiteSpace(jobName))
{
JobDefinition job = this.GetJob(resourceGroupName, jobCollectionName, jobName);
listOfJobs = new List <JobDefinition>()
{
job
};
}
else
{
listOfJobs = this.ListJobs(resourceGroupName, jobCollectionName, jobStateEnum);
}
return(Converter.ConvertJobDefinitionListToPSList(listOfJobs));
}
/// <summary>
/// Initializes a new instance of the CloudJob class.
/// </summary>
/// <param name="id">A string that uniquely identifies the Job within
/// the Account.</param>
/// <param name="displayName">The display name for the Job.</param>
/// <param name="usesTaskDependencies">Whether Tasks in the Job can
/// define dependencies on each other. The default is false.</param>
/// <param name="url">The URL of the Job.</param>
/// <param name="eTag">The ETag of the Job.</param>
/// <param name="lastModified">The last modified time of the
/// Job.</param>
/// <param name="creationTime">The creation time of the Job.</param>
/// <param name="state">The current state of the Job.</param>
/// <param name="stateTransitionTime">The time at which the Job entered
/// its current state.</param>
/// <param name="previousState">The previous state of the Job.</param>
/// <param name="previousStateTransitionTime">The time at which the Job
/// entered its previous state.</param>
/// <param name="priority">The priority of the Job.</param>
/// <param name="maxParallelTasks">The maximum number of tasks that can
/// be executed in parallel for the job.</param>
/// <param name="constraints">The execution constraints for the
/// Job.</param>
/// <param name="jobManagerTask">Details of a Job Manager Task to be
/// launched when the Job is started.</param>
/// <param name="jobPreparationTask">The Job Preparation Task.</param>
/// <param name="jobReleaseTask">The Job Release Task.</param>
/// <param name="commonEnvironmentSettings">The list of common
/// environment variable settings. These environment variables are set
/// for all Tasks in the Job (including the Job Manager, Job
/// Preparation and Job Release Tasks).</param>
/// <param name="poolInfo">The Pool settings associated with the
/// Job.</param>
/// <param name="onAllTasksComplete">The action the Batch service
/// should take when all Tasks in the Job are in the completed
/// state.</param>
/// <param name="onTaskFailure">The action the Batch service should
/// take when any Task in the Job fails.</param>
/// <param name="networkConfiguration">The network configuration for
/// the Job.</param>
/// <param name="metadata">A list of name-value pairs associated with
/// the Job as metadata.</param>
/// <param name="executionInfo">The execution information for the
/// Job.</param>
/// <param name="stats">Resource usage statistics for the entire
/// lifetime of the Job.</param>
public CloudJob(string id = default(string), string displayName = default(string), bool?usesTaskDependencies = default(bool?), string url = default(string), string eTag = default(string), System.DateTime?lastModified = default(System.DateTime?), System.DateTime?creationTime = default(System.DateTime?), JobState?state = default(JobState?), System.DateTime?stateTransitionTime = default(System.DateTime?), JobState?previousState = default(JobState?), System.DateTime?previousStateTransitionTime = default(System.DateTime?), int?priority = default(int?), int?maxParallelTasks = default(int?), JobConstraints constraints = default(JobConstraints), JobManagerTask jobManagerTask = default(JobManagerTask), JobPreparationTask jobPreparationTask = default(JobPreparationTask), JobReleaseTask jobReleaseTask = default(JobReleaseTask), IList <EnvironmentSetting> commonEnvironmentSettings = default(IList <EnvironmentSetting>), PoolInformation poolInfo = default(PoolInformation), OnAllTasksComplete?onAllTasksComplete = default(OnAllTasksComplete?), OnTaskFailure?onTaskFailure = default(OnTaskFailure?), JobNetworkConfiguration networkConfiguration = default(JobNetworkConfiguration), IList <MetadataItem> metadata = default(IList <MetadataItem>), JobExecutionInformation executionInfo = default(JobExecutionInformation), JobStatistics stats = default(JobStatistics))
{
Id = id;
DisplayName = displayName;
UsesTaskDependencies = usesTaskDependencies;
Url = url;
ETag = eTag;
LastModified = lastModified;
CreationTime = creationTime;
State = state;
StateTransitionTime = stateTransitionTime;
PreviousState = previousState;
PreviousStateTransitionTime = previousStateTransitionTime;
Priority = priority;
MaxParallelTasks = maxParallelTasks;
Constraints = constraints;
JobManagerTask = jobManagerTask;
JobPreparationTask = jobPreparationTask;
JobReleaseTask = jobReleaseTask;
CommonEnvironmentSettings = commonEnvironmentSettings;
PoolInfo = poolInfo;
OnAllTasksComplete = onAllTasksComplete;
OnTaskFailure = onTaskFailure;
NetworkConfiguration = networkConfiguration;
Metadata = metadata;
ExecutionInfo = executionInfo;
Stats = stats;
CustomInit();
}
internal CloudJob(string id, string displayName, bool?usesTaskDependencies, string url, string eTag, DateTimeOffset?lastModified, DateTimeOffset?creationTime, JobState?state, DateTimeOffset?stateTransitionTime, JobState?previousState, DateTimeOffset?previousStateTransitionTime, int?priority, JobConstraints constraints, JobManagerTask jobManagerTask, JobPreparationTask jobPreparationTask, JobReleaseTask jobReleaseTask, IReadOnlyList <EnvironmentSetting> commonEnvironmentSettings, PoolInformation poolInfo, OnAllTasksComplete?onAllTasksComplete, OnTaskFailure?onTaskFailure, JobNetworkConfiguration networkConfiguration, IReadOnlyList <MetadataItem> metadata, JobExecutionInformation executionInfo, JobStatistics stats)
{
Id = id;
DisplayName = displayName;
UsesTaskDependencies = usesTaskDependencies;
Url = url;
ETag = eTag;
LastModified = lastModified;
CreationTime = creationTime;
State = state;
StateTransitionTime = stateTransitionTime;
PreviousState = previousState;
PreviousStateTransitionTime = previousStateTransitionTime;
Priority = priority;
Constraints = constraints;
JobManagerTask = jobManagerTask;
JobPreparationTask = jobPreparationTask;
JobReleaseTask = jobReleaseTask;
CommonEnvironmentSettings = commonEnvironmentSettings;
PoolInfo = poolInfo;
OnAllTasksComplete = onAllTasksComplete;
OnTaskFailure = onTaskFailure;
NetworkConfiguration = networkConfiguration;
Metadata = metadata;
ExecutionInfo = executionInfo;
Stats = stats;
}
public static bool IsCancelled(this JobState?state)
{
if (!state.HasValue)
{
return(false);
}
return(_cancelledStates.Contains(state.Value));
}
public static bool IsSuccess(this JobState?state)
{
if (!state.HasValue)
{
return(false);
}
return(_finishedSuccessStates.Contains(state.Value));
}
public static bool IsComplete(this JobState?state)
{
if (!state.HasValue)
{
return(false);
}
return(_finishedStates.Contains(state.Value));
}
public List <PeriodicJob> GetPeriodicJobs(JobState?jobState = null)
{
return(jobState switch
{
JobState.Stoped => GetStoppedPeriodicJobs(),
JobState.Running => GetRunningPeriodicJobs(),
_ => GetAllPeriodicJobs(),
});
/// <summary>
/// Initializes a new instance of the JobProperties class.
/// </summary>
public JobProperties(DateTime?startTime = default(DateTime?), JobAction action = default(JobAction), JobRecurrence recurrence = default(JobRecurrence), JobState?state = default(JobState?), JobStatus status = default(JobStatus))
{
StartTime = startTime;
Action = action;
Recurrence = recurrence;
State = state;
Status = status;
}
/// <summary>
/// Initializes a new instance of the JobPropertiesInner class.
/// </summary>
/// <param name="startTime">Gets or sets the job start time.</param>
/// <param name="action">Gets or sets the job action.</param>
/// <param name="recurrence">Gets or sets the job recurrence.</param>
/// <param name="state">Gets or set the job state. Possible values
/// include: 'Enabled', 'Disabled', 'Faulted', 'Completed'</param>
/// <param name="status">Gets the job status.</param>
public JobPropertiesInner(System.DateTime?startTime = default(System.DateTime?), JobAction action = default(JobAction), JobRecurrence recurrence = default(JobRecurrence), JobState?state = default(JobState?), JobStatus status = default(JobStatus))
{
StartTime = startTime;
Action = action;
Recurrence = recurrence;
State = state;
Status = status;
CustomInit();
}
public Task <bool> UpdateState(string tenantId, string jobId, JobState?expectedState, JobState newState)
{
_jobs.TryGetValue(tenantId, out var value);
var job = value?[jobId];
if (job == null)
{
return(Task.FromResult(false));
}
job.Status.State = newState;
return(Task.FromResult(true));
}
private async Task UpdateJobStatisticsAsync(long?sourceJobId = null, JobState?sourceJobState = null)
{
JobState currentJobState;
if (sourceJobId.HasValue && sourceJobState.HasValue)
{
currentJobState = sourceJobState.Value;
}
else
{
currentJobState = State.CurrentJobState;
}
await UpdateJobStatisticsImplAsync(State.JobId, currentJobState, sourceJobId);
}
public async Task <bool> UpdateState(string tenantId, string jobId, JobState?expectedState, JobState newState)
{
var update = Builders <JobData> .Update
.Set(jd => jd.Status.State, newState)
.Set(jd => jd.Status.StateTime, DateTime.UtcNow);
if (expectedState.HasValue)
{
return((await Mongo.Jobs.UpdateOneAsync(jd =>
jd.TenantId == tenantId &&
jd.JobId == jobId &&
jd.Status.State == expectedState.Value,
update)).ModifiedCount > 0);
}
return((await Mongo.Jobs.UpdateOneAsync(jd => jd.TenantId == tenantId && jd.JobId == jobId, update))
.ModifiedCount > 0);
}
/// <summary>
/// Initializes a new instance of the JobInformation class.
/// </summary>
/// <param name="name">the friendly name of the job.</param>
/// <param name="type">the job type of the current job (Hive or USql).
/// Possible values include: 'USql', 'Hive'</param>
/// <param name="properties">the job specific properties.</param>
/// <param name="jobId">the job's unique identifier (a GUID).</param>
/// <param name="submitter">the user or account that submitted the
/// job.</param>
/// <param name="errorMessage">the error message details for the job,
/// if the job failed.</param>
/// <param name="degreeOfParallelism">the degree of parallelism used
/// for this job. This must be greater than 0.</param>
/// <param name="priority">the priority value for the current job.
/// Lower numbers have a higher priority. By default, a job has a
/// priority of 1000. This must be greater than 0.</param>
/// <param name="submitTime">the time the job was submitted to the
/// service.</param>
/// <param name="startTime">the start time of the job.</param>
/// <param name="endTime">the completion time of the job.</param>
/// <param name="state">the job state. When the job is in the Ended
/// state, refer to Result and ErrorMessage for details. Possible
/// values include: 'Accepted', 'Compiling', 'Ended', 'New',
/// 'Queued', 'Running', 'Scheduling', 'Starting', 'Paused',
/// 'WaitingForCapacity'</param>
/// <param name="result">the result of job execution or the current
/// result of the running job. Possible values include: 'None',
/// 'Succeeded', 'Cancelled', 'Failed'</param>
/// <param name="logFolder">the log folder path to use in the
/// following format:
/// adl://<accountName>.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/.</param>
/// <param name="logFilePatterns">the list of log file name patterns
/// to find in the logFolder. '*' is the only matching character
/// allowed. Example format: jobExecution*.log or *mylog*.txt</param>
/// <param name="stateAuditRecords">the job state audit records,
/// indicating when various operations have been performed on this
/// job.</param>
public JobInformation(string name, JobType type, JobProperties properties, Guid?jobId = default(Guid?), string submitter = default(string), IList <JobErrorDetails> errorMessage = default(IList <JobErrorDetails>), int?degreeOfParallelism = default(int?), int?priority = default(int?), DateTimeOffset?submitTime = default(DateTimeOffset?), DateTimeOffset?startTime = default(DateTimeOffset?), DateTimeOffset?endTime = default(DateTimeOffset?), JobState?state = default(JobState?), JobResult?result = default(JobResult?), string logFolder = default(string), IList <string> logFilePatterns = default(IList <string>), IList <JobStateAuditRecord> stateAuditRecords = default(IList <JobStateAuditRecord>))
{
JobId = jobId;
Name = name;
Type = type;
Submitter = submitter;
ErrorMessage = errorMessage;
DegreeOfParallelism = degreeOfParallelism;
Priority = priority;
SubmitTime = submitTime;
StartTime = startTime;
EndTime = endTime;
State = state;
Result = result;
LogFolder = logFolder;
LogFilePatterns = logFilePatterns;
StateAuditRecords = stateAuditRecords;
Properties = properties;
}
public override void NavigatedTo(object parameter = null)
{
try
{
base.NavigatedTo(parameter);
Jobs = new ObservableCollection <Job>();
_selectedJobState = null;
_selectedDocumentType = null;
_windowsServiceClient.JobChangedAction = JobChanged;
LoadJobs();
}
catch (Exception e)
{
_loggingService?.WriteError(e);
}
}
/// <summary>
/// Initializes a new instance of the JobInformationBasic class.
/// </summary>
/// <param name="name">The friendly name of the job.</param>
/// <param name="type">The job type of the current job (Hive, USql, or
/// Scope (for internal use only)). Possible values include: 'USql',
/// 'Hive', 'Scope'</param>
/// <param name="jobId">The job's unique identifier (a GUID).</param>
/// <param name="submitter">The user or account that submitted the
/// job.</param>
/// <param name="degreeOfParallelism">The degree of parallelism used
/// for this job.</param>
/// <param name="degreeOfParallelismPercent">the degree of parallelism
/// in percentage used for this job.</param>
/// <param name="priority">The priority value for the current job.
/// Lower numbers have a higher priority. By default, a job has a
/// priority of 1000. This must be greater than 0.</param>
/// <param name="submitTime">The time the job was submitted to the
/// service.</param>
/// <param name="startTime">The start time of the job.</param>
/// <param name="endTime">The completion time of the job.</param>
/// <param name="state">The job state. When the job is in the Ended
/// state, refer to Result and ErrorMessage for details. Possible
/// values include: 'Accepted', 'Compiling', 'Ended', 'New', 'Queued',
/// 'Running', 'Scheduling', 'Starting', 'Paused',
/// 'WaitingForCapacity', 'Yielded', 'Finalizing'</param>
/// <param name="result">The result of job execution or the current
/// result of the running job. Possible values include: 'None',
/// 'Succeeded', 'Cancelled', 'Failed'</param>
/// <param name="logFolder">The log folder path to use in the following
/// format:
/// adl://<accountName>.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/.</param>
/// <param name="logFilePatterns">The list of log file name patterns to
/// find in the logFolder. '*' is the only matching character allowed.
/// Example format: jobExecution*.log or *mylog*.txt</param>
/// <param name="related">The recurring job relationship information
/// properties.</param>
/// <param name="tags">The key-value pairs used to add additional
/// metadata to the job information. (Only for use internally with
/// Scope job type.)</param>
/// <param name="hierarchyQueueNode">the name of hierarchy queue node
/// this job is assigned to, Null if job has not been assigned yet or
/// the account doesn't have hierarchy queue.</param>
public JobInformationBasic(string name, JobType type, System.Guid?jobId = default(System.Guid?), string submitter = default(string), int?degreeOfParallelism = default(int?), double?degreeOfParallelismPercent = default(double?), int?priority = default(int?), System.DateTimeOffset?submitTime = default(System.DateTimeOffset?), System.DateTimeOffset?startTime = default(System.DateTimeOffset?), System.DateTimeOffset?endTime = default(System.DateTimeOffset?), JobState?state = default(JobState?), JobResult?result = default(JobResult?), string logFolder = default(string), IList <string> logFilePatterns = default(IList <string>), JobRelationshipProperties related = default(JobRelationshipProperties), IDictionary <string, string> tags = default(IDictionary <string, string>), string hierarchyQueueNode = default(string))
{
JobId = jobId;
Name = name;
Type = type;
Submitter = submitter;
DegreeOfParallelism = degreeOfParallelism;
DegreeOfParallelismPercent = degreeOfParallelismPercent;
Priority = priority;
SubmitTime = submitTime;
StartTime = startTime;
EndTime = endTime;
State = state;
Result = result;
LogFolder = logFolder;
LogFilePatterns = logFilePatterns;
Related = related;
Tags = tags;
HierarchyQueueNode = hierarchyQueueNode;
CustomInit();
}
/// <summary>
/// Initializes a new instance of the JobStateFilter class.
/// </summary>
public JobStateFilter(JobState?state = default(JobState?))
{
State = state;
}
private Job(string name, Input <string> id, JobState?state = null, CustomResourceOptions?options = null)
: base("gcp:cloudscheduler/job:Job", name, state, MakeResourceOptions(options, id))
{
}
/// <summary>
/// Initializes a new instance of the JobStateFilterInner class.
/// </summary>
/// <param name="state">Gets or sets the job state. Possible values
/// include: 'Enabled', 'Disabled', 'Faulted', 'Completed'</param>
public JobStateFilterInner(JobState?state = default(JobState?))
{
State = state;
CustomInit();
}
public Task <bool> UpdateState(string tenantId, string jobId, JobState?expectedState, JobState newState)
{
throw new NotImplementedException();
}
/// <summary>
/// Get an existing Job resource's state with the given name, ID, and optional extra
/// properties used to qualify the lookup.
/// </summary>
///
/// <param name="name">The unique name of the resulting resource.</param>
/// <param name="id">The unique provider ID of the resource to lookup.</param>
/// <param name="state">Any extra arguments used during the lookup.</param>
/// <param name="options">A bag of options that control this resource's behavior</param>
public static Job Get(string name, Input <string> id, JobState?state = null, CustomResourceOptions?options = null)
{
return(new Job(name, id, state, options));
}
/// <summary>
/// Initializes a new instance of the JobInformation class.
/// </summary>
/// <param name="name">The friendly name of the job.</param>
/// <param name="type">The job type of the current job (Hive, USql, or
/// Scope (for internal use only)). Possible values include: 'USql',
/// 'Hive', 'Scope'</param>
/// <param name="properties">The job specific properties.</param>
/// <param name="jobId">The job's unique identifier (a GUID).</param>
/// <param name="submitter">The user or account that submitted the
/// job.</param>
/// <param name="degreeOfParallelism">The degree of parallelism used
/// for this job.</param>
/// <param name="degreeOfParallelismPercent">the degree of parallelism
/// in percentage used for this job.</param>
/// <param name="priority">The priority value for the current job.
/// Lower numbers have a higher priority. By default, a job has a
/// priority of 1000. This must be greater than 0.</param>
/// <param name="submitTime">The time the job was submitted to the
/// service.</param>
/// <param name="startTime">The start time of the job.</param>
/// <param name="endTime">The completion time of the job.</param>
/// <param name="state">The job state. When the job is in the Ended
/// state, refer to Result and ErrorMessage for details. Possible
/// values include: 'Accepted', 'Compiling', 'Ended', 'New', 'Queued',
/// 'Running', 'Scheduling', 'Starting', 'Paused',
/// 'WaitingForCapacity', 'Yielded', 'Finalizing'</param>
/// <param name="result">The result of job execution or the current
/// result of the running job. Possible values include: 'None',
/// 'Succeeded', 'Cancelled', 'Failed'</param>
/// <param name="logFolder">The log folder path to use in the following
/// format:
/// adl://<accountName>.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/.</param>
/// <param name="logFilePatterns">The list of log file name patterns to
/// find in the logFolder. '*' is the only matching character allowed.
/// Example format: jobExecution*.log or *mylog*.txt</param>
/// <param name="related">The recurring job relationship information
/// properties.</param>
/// <param name="tags">The key-value pairs used to add additional
/// metadata to the job information. (Only for use internally with
/// Scope job type.)</param>
/// <param name="hierarchyQueueNode">the name of hierarchy queue node
/// this job is assigned to, Null if job has not been assigned yet or
/// the account doesn't have hierarchy queue.</param>
/// <param name="errorMessage">The error message details for the job,
/// if the job failed.</param>
/// <param name="stateAuditRecords">The job state audit records,
/// indicating when various operations have been performed on this
/// job.</param>
public JobInformation(string name, JobType type, JobProperties properties, System.Guid?jobId = default(System.Guid?), string submitter = default(string), int?degreeOfParallelism = default(int?), double?degreeOfParallelismPercent = default(double?), int?priority = default(int?), System.DateTimeOffset?submitTime = default(System.DateTimeOffset?), System.DateTimeOffset?startTime = default(System.DateTimeOffset?), System.DateTimeOffset?endTime = default(System.DateTimeOffset?), JobState?state = default(JobState?), JobResult?result = default(JobResult?), string logFolder = default(string), IList <string> logFilePatterns = default(IList <string>), JobRelationshipProperties related = default(JobRelationshipProperties), IDictionary <string, string> tags = default(IDictionary <string, string>), string hierarchyQueueNode = default(string), IList <JobErrorDetails> errorMessage = default(IList <JobErrorDetails>), IList <JobStateAuditRecord> stateAuditRecords = default(IList <JobStateAuditRecord>))
: base(name, type, jobId, submitter, degreeOfParallelism, degreeOfParallelismPercent, priority, submitTime, startTime, endTime, state, result, logFolder, logFilePatterns, related, tags, hierarchyQueueNode)
{
ErrorMessage = errorMessage;
StateAuditRecords = stateAuditRecords;
Properties = properties;
CustomInit();
}
/// <summary>
/// List jobs.
/// </summary>
/// <param name="resourceGroupName">Name of the resource group.</param>
/// <param name="jobCollectionName">job collection name.</param>
/// <param name="jobName">Name of the job.</param>
/// <param name="jobState">State of the job.</param>
/// <returns>List of Job definition.</returns>
internal IList <JobDefinition> ListJobs(string resourceGroupName, string jobCollectionName, JobState?jobState)
{
var listOfJobs = new List <JobDefinition>();
var oDataQuery = new ODataQuery <JobStateFilter>();
if (jobState != null)
{
Expression <Func <JobStateFilter, bool> > jobFilterExpression = (jobStateFilter) => jobStateFilter.State == jobState;
oDataQuery.SetFilter(jobFilterExpression);
}
IPage <JobDefinition> jobsPage = this.SchedulerManagementClient.Jobs.List(resourceGroupName, jobCollectionName, oDataQuery);
listOfJobs.AddRange(jobsPage);
while (!string.IsNullOrWhiteSpace(jobsPage.NextPageLink))
{
jobsPage = this.SchedulerManagementClient.Jobs.ListNext(jobsPage.NextPageLink);
listOfJobs.AddRange(jobsPage);
}
return(listOfJobs);
}
public static void Submit()
{
Log("Start submission process.");
state = null;
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings.batchEndpoint, Settings.batchAccount, Settings.batchKey);
using (BatchClient client = BatchClient.Open(cred)) // <-- connect to the cluster
{
#region job submission
string jobname = prefix + Environment.GetEnvironmentVariable("USERNAME") + "_" + DateTime.Now.ToString("yyyyMMdd-HHmmss");
PoolInformation pool = new PoolInformation();
pool.PoolId = Settings.poolname;
CloudJob job = client.JobOperations.CreateJob(jobname, pool); // <-- create a workitem that runs on pool "trdemo"
Log("Submitting...");
job.Commit();
jobName = jobname;
Log(string.Format("Job {0} created.", jobname));
job = client.JobOperations.GetJob(jobname);
Log("Analyzing input blobs...");
string inputcontainersas = StorageHelper.GetContainerSAS(Settings.inputContainer);
string outputcontainersas = StorageHelper.GetContainerSAS(Settings.outputContainer);
foreach (string blob in StorageHelper.ListBlobs(Settings.inputContainer))
{
string filename = System.IO.Path.GetFileName((new Uri(blob)).LocalPath);
string taskname = "task_" + System.IO.Path.GetFileNameWithoutExtension(filename);
// prepare the command line
string cli;
cli = ". robocopy.exe ${env:WATASK_TVM_ROOT_DIR}\\shared\\ . *.*;";
cli += "ffmpeg.exe -i {0} -vf 'movie=microsoft.png [watermark]; [in][watermark] overlay=10:main_h-overlay_h-10 [out]' {0}.output.avi;".Replace("{0}", filename);
cli += "azcopy.exe . {0} *.output.avi /destsas:'{1}' /y".Replace("{0}", Settings.outputContainer).Replace("{1}", outputcontainersas);
cli = string.Format("powershell -command \"{0}\"", cli);
// prepare task object
CloudTask task = new CloudTask(taskname, cli);
task.ResourceFiles = new List<ResourceFile>();
task.ResourceFiles.Add(new ResourceFile(blob + inputcontainersas, filename));
job.AddTask(task); // <-- add Task
}
#endregion job submission
ThreadPool.QueueUserWorkItem((x) => { Monitor(); });
client.Utilities.CreateTaskStateMonitor().WaitAll(client.JobOperations.ListTasks(jobname), TaskState.Completed, new TimeSpan(0, 60, 0));
client.JobOperations.GetJob(jobname).Terminate();
}
}
private Job(string name, Input <string> id, JobState?state = null, CustomResourceOptions?options = null)
: base("azure:batch/job:Job", name, state, MakeResourceOptions(options, id))
{
}
public static void Monitor()
{
if (!string.IsNullOrEmpty(jobName))
{
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings.batchEndpoint, Settings.batchAccount, Settings.batchKey);
using (BatchClient client = BatchClient.Open(cred)) // <-- connect to the cluster
{
{
do
{
int completed = 0;
int all = 0;
CloudJob job = client.JobOperations.GetJob(jobName);
if(state != job.State)
Log("job state is " + job.State);
state = job.State;
var tasks = client.JobOperations.ListTasks(jobName).ToList<CloudTask>();
completed = tasks.Where(t => t.State == TaskState.Completed).Count();
all = tasks.Count();
BatchServiceClient.completed = completed;
BatchServiceClient.all = all;
Thread.Sleep(1000);
} while (state != JobState.Completed);
}
}
}
}
internal static string ToSerializedValue(this JobState?value) => value == null ? null : ((JobState)value).ToSerializedValue();
private Job(string name, Input <string> id, JobState?state = null, CustomResourceOptions?options = null)
: base("databricks:index/job:Job", name, state, MakeResourceOptions(options, id))
{
}