private InternalTask GetNextTask()
{
lock (this) {
var boundary = this._nextChunkBoundary.Invoke();
var t = new InternalTask(boundary);
this._internalTasks.Add(t);
return(t);
}
}
private void CompleteTask(object handler, Resource resource, object result)
{
var item = new InternalTask(resource);
if (this.tasks.TryGetValue(item, out var onCompleteActions) == true)
{
onCompleteActions.Invoke(result);
this.tasks.Remove(item);
}
}
public T.Task <List <InternalTask> > GenerateTasksAsync(Job job, CancellationToken token)
{
return(T.Task.FromResult(Enumerable.Range(1, job.TargetNodes.Length).Select(id =>
{
var t = InternalTask.CreateFrom(job);
t.CustomizedData = t.Node = job.TargetNodes[id - 1];
t.Id = id;
t.MaximumRuntimeSeconds = job.DefaultTaskMaximumRuntimeSeconds;
return t;
}).ToList()));
}
public T.Task <DispatchResult> DispatchAsync(Job job, CancellationToken token)
{
return(T.Task.FromResult(new DispatchResult()
{
ModifiedJob = job, Tasks = Enumerable.Range(1, job.TargetNodes.Length).Select(id =>
{
var t = InternalTask.CreateFrom(job);
t.CustomizedData = t.Node = job.TargetNodes[id - 1];
t.Id = id;
t.MaximumRuntimeSeconds = job.MaximumRuntimeSeconds;
return t;
}).ToList()
}));
}
private void Timer_Job(object obj)
{
Stop();
lock (lockObject)
{
if (followingTask != null)
{
queue.Enqueue(followingTask.Task);
followingTask = null;
}
}
if (!isDisposed)
{
PlanTimer();
}
}
private bool RequestLoad <T, TClosure>(object handler, TClosure closure, Resource resource, System.Action <T, TClosure> onComplete) where T : class
{
var item = new InternalTask(resource);
if (this.tasks.TryGetValue(item, out var onCompleteActions) == true)
{
onCompleteActions += (obj) => onComplete.Invoke((T)obj, closure);
this.tasks[item] = onCompleteActions;
return(true);
}
else
{
onCompleteActions += (obj) => onComplete.Invoke((T)obj, closure);
this.tasks.Add(item, onCompleteActions);
}
return(false);
}
private void PlanTimer()
{
lock (lockObject)
{
followingTask = null;
DateTime now = DateTime.Now;
InternalTask next = null;
TimeSpan addDay = TimeSpan.FromDays(0);
while (next == null && registeredTasksAscSorted.Count > 0)
{
var potentiallyNext = availableTasks.First?.Value;
if (potentiallyNext != null)
{
availableTasks.RemoveFirst();
if (potentiallyNext.Time.Add(addDay) > now.TimeOfDay)
{
next = potentiallyNext;
}
}
else
{
availableTasks = new LinkedList <InternalTask>(registeredTasksAscSorted);
addDay = addDay.Add(TimeSpan.FromDays(1));
}
}
if (next != null)
{
var waitTime = next.Time.Add(addDay) - now.TimeOfDay;
if (!isDisposed)
{
followingTask = next;
timer.Change(waitTime, waitTime);
}
}
}
}
public override bool Write(GH_IWriter writer)
{
InternalTask.Write(writer);
return(base.Write(writer));
}
private void SetColor(Color color)
{
var newTask = InternalTask.ChangeColor(color);
SetTask(newTask);
}
public override async T.Task ProcessAsync(Job job, JobEventMessage message, CancellationToken token)
{
var jobTable = this.Utilities.GetJobsTable();
if (job.State != JobState.Queued)
{
this.Logger.Error("The job {0} state {1} is not queued.", job.Id, job.State);
return;
}
var tasks = await this.JobTypeHandler.GenerateTasksAsync(job, token);
if (tasks == null)
{
this.Logger.Error("The job {0} script doesn't generate any tasks", job.Id);
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = $"The job {job.Id} script doesn't generate any tasks.",
Source = EventSource.Job,
Type = EventType.Alert,
});
j.State = JobState.Failed;
j.TaskCount = 0;
}, token, this.Logger);
return;
}
var allParentIds = new HashSet <int>(tasks.SelectMany(t => t.ParentIds ?? new List <int>()));
var endingIds = tasks.Where(t => !allParentIds.Contains(t.Id)).Select(t => t.Id).ToList();
var startTask = InternalTask.CreateFrom(job);
startTask.Id = 0;
startTask.CustomizedData = InternalTask.StartTaskMark;
tasks.ForEach(t =>
{
if (t.ParentIds == null || t.ParentIds.Count == 0)
{
t.ParentIds = new List <int>()
{
startTask.Id
}
}
;
t.ChildIds?.Clear();
});
var endTask = InternalTask.CreateFrom(job);
endTask.Id = int.MaxValue;
endTask.CustomizedData = InternalTask.EndTaskMark;
endTask.ParentIds = endingIds;
this.Logger.Information("Job {0} task {1} has {2} parent ids, {3}", job.Id, endTask.Id, endTask.ParentIds.Count, string.Join(",", endTask.ParentIds));
tasks.Add(startTask);
tasks.Add(endTask);
var(success, msg) = this.FillData(tasks, job);
if (!success)
{
this.Logger.Error(msg);
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
j.State = JobState.Failed;
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = msg,
Source = EventSource.Job,
Type = EventType.Alert
});
}, token, this.Logger);
return;
}
const int MaxChildIds = 1000;
this.Logger.Information("Job {0} Converting {1} Tasks to Instances.", job.Id, tasks.Count);
var taskInstances = tasks.Select(it =>
{
string zipppedParentIds = Compress.GZip(string.Join(",", it.ParentIds ?? new List <int>()));
var childIds = it.ChildIds;
childIds = childIds ?? new List <int>();
childIds = childIds.Count > MaxChildIds ? null : childIds;
return(new Task()
{
ChildIds = childIds,
ZippedParentIds = zipppedParentIds,
CommandLine = it.CommandLine,
CustomizedData = it.CustomizedData,
Id = it.Id,
JobId = it.JobId,
JobType = it.JobType,
Node = it.Node,
RequeueCount = it.RequeueCount,
State = string.Equals(it.CustomizedData, Task.StartTaskMark, StringComparison.OrdinalIgnoreCase) ? TaskState.Finished : TaskState.Queued,
MaximumRuntimeSeconds = it.MaximumRuntimeSeconds,
});
}).ToList();
var childIdsContent = tasks
.Where(it => (it.ChildIds?.Count ?? 0) > MaxChildIds)
.Select(it => new
{
it.Id,
it.JobId,
it.RequeueCount,
it.ChildIds,
})
.ToList();
this.Logger.Information("Job {0} Converting {1} Tasks to TaskStartInfo.", job.Id, tasks.Count);
var taskInfos = tasks.Select(it => new TaskStartInfo()
{
Id = it.Id,
JobId = it.JobId,
JobType = it.JobType,
NodeName = it.Node,
Password = it.Password,
PrivateKey = it.PrivateKey,
PublicKey = it.PublicKey,
UserName = it.UserName,
StartInfo = new ProcessStartInfo(it.CommandLine, it.WorkingDirectory, null, null, null, it.EnvironmentVariables, null, it.RequeueCount),
}).ToList();
this.Logger.Information("Job {0} Inserting {1} Tasks to Table.", job.Id, tasks.Count);
var jobPartitionKey = this.Utilities.GetJobPartitionKey(job.Type, job.Id);
await jobTable.InsertOrReplaceBatchAsync(token, taskInstances.Select(t => new JsonTableEntity(
jobPartitionKey,
this.Utilities.GetTaskKey(job.Id, t.Id, job.RequeueCount),
t)).ToArray());
if (childIdsContent.Select(cid => cid.Id).Distinct().Count() != childIdsContent.Count)
{
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
j.State = JobState.Failed;
(j.Events ?? (j.Events = new List <Event>())).Add(new Event()
{
Content = $"Duplicate task ids found.",
Source = EventSource.Job,
Type = EventType.Alert,
});
}, token, this.Logger);
return;
}
this.Logger.Information("Job {0} Uploading {1} Tasks child ids content to blob.", job.Id, childIdsContent.Count);
await T.Task.WhenAll(childIdsContent.Select(async childIds =>
{
var taskKey = this.Utilities.GetTaskKey(childIds.JobId, childIds.Id, childIds.RequeueCount);
var childIdsBlob = await this.Utilities.CreateOrReplaceTaskChildrenBlobAsync(taskKey, token);
var jsonContent = JsonConvert.SerializeObject(childIds.ChildIds);
await childIdsBlob.UploadTextAsync(jsonContent, Encoding.UTF8, null, null, null, token);
}));
this.Logger.Information("Job {0} Inserting {1} TaskInfo to Table.", job.Id, taskInfos.Count);
await jobTable.InsertOrReplaceBatchAsync(token, taskInfos.Select(t => new JsonTableEntity(
jobPartitionKey,
this.Utilities.GetTaskInfoKey(job.Id, t.Id, job.RequeueCount),
t)).ToArray());
this.Logger.Information("Job {0} updating job status.", job.Id);
JobState state = JobState.Queued;
await this.Utilities.UpdateJobAsync(job.Type, job.Id, j =>
{
state = j.State = (j.State == JobState.Queued ? JobState.Running : j.State);
j.TaskCount = taskInstances.Count - 2;
}, token, this.Logger);
if (state == JobState.Running)
{
this.Logger.Information("Job {0} Starting the job", job.Id);
async T.Task addFirstTask()
{
var taskCompletionQueue = await this.Utilities.GetOrCreateJobTaskCompletionQueueAsync(job.Id, token);
await taskCompletionQueue.AddMessageAsync(new CloudQueueMessage(
JsonConvert.SerializeObject(new TaskCompletionMessage()
{
JobId = job.Id, Id = 0, JobType = job.Type, RequeueCount = job.RequeueCount, ChildIds = startTask.ChildIds
})),
null, null, null, null, token);
};
async T.Task addRunningJob()
{
var runningJobQueue = this.Utilities.GetRunningJobQueue();
await runningJobQueue.AddMessageAsync(new CloudQueueMessage(
JsonConvert.SerializeObject(new RunningJobMessage()
{
JobId = job.Id, JobType = job.Type, RequeueCount = job.RequeueCount
})),
null, null, null, null, token);
};
await T.Task.WhenAll(addFirstTask(), addRunningJob());
}
}
public IEnumerator LoadAsync <T, TClosure>(object handler, TClosure closure, Resource resource, System.Action <T, TClosure> onComplete) where T : class
{
if (this.RequestLoad(handler, closure, resource, onComplete) == true)
{
// Waiting for loading then break
var item = new InternalTask(resource);
while (this.tasks.ContainsKey(item) == true)
{
yield return(null);
}
yield break;
}
switch (resource.type)
{
case Resource.Type.Manual: {
this.CompleteTask(handler, resource, default);
break;
}
case Resource.Type.Direct: {
if (resource.directRef is T direct)
{
this.AddObject(handler, direct, resource);
this.CompleteTask(handler, resource, direct);
yield break;
}
this.CompleteTask(handler, resource, default);
break;
}
case Resource.Type.Addressables: {
if (resource.objectType == Resource.ObjectType.Component)
{
//Debug.Log("Loading: " + resource.guid);
var op = UnityEngine.AddressableAssets.Addressables.LoadAssetAsync <GameObject>(resource.guid);
System.Action cancellationTask = () => { if (op.IsValid() == true)
{
UnityEngine.AddressableAssets.Addressables.Release(op);
}
};
this.LoadBegin(handler, cancellationTask);
while (op.IsDone == false)
{
yield return(null);
}
if (op.IsValid() == false)
{
//this.CompleteTask(handler, resource, default);
}
else
{
if (op.Status == UnityEngine.ResourceManagement.AsyncOperations.AsyncOperationStatus.Succeeded)
{
var asset = op.Result;
if (asset == null)
{
this.CompleteTask(handler, resource, default);
}
else
{
this.AddObject(handler, asset, resource);
this.CompleteTask(handler, resource, asset.GetComponent <T>());
}
}
}
this.LoadEnd(handler, cancellationTask);
}
else
{
UnityEngine.ResourceManagement.AsyncOperations.AsyncOperationHandle <T> op;
if (string.IsNullOrEmpty(resource.subObjectName) == false)
{
op = UnityEngine.AddressableAssets.Addressables.LoadAssetAsync <T>(resource.guid + "[" + resource.subObjectName + "]");
}
else
{
op = UnityEngine.AddressableAssets.Addressables.LoadAssetAsync <T>(resource.guid);
}
System.Action cancellationTask = () => { if (op.IsValid() == true)
{
UnityEngine.AddressableAssets.Addressables.Release(op);
}
};
this.LoadBegin(handler, cancellationTask);
while (op.IsDone == false)
{
yield return(null);
}
if (op.IsValid() == false)
{
//this.CompleteTask(handler, resource, default);
}
else
{
if (op.Status == UnityEngine.ResourceManagement.AsyncOperations.AsyncOperationStatus.Succeeded)
{
var asset = op.Result;
if (asset == null)
{
this.CompleteTask(handler, resource, default);
}
else
{
this.AddObject(handler, asset, resource);
this.CompleteTask(handler, resource, asset);
}
}
}
this.LoadEnd(handler, cancellationTask);
}
break;
}
}
}
