/// <summary>
/// 更新最后一次作业记录
/// </summary>
/// <param name="newJob">New job.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public bool UpdateLastJobRecord(JobBase newJob, CancellationToken cancellationToken = default(CancellationToken))
{
// TODO:可能在这里中断，需要搞个类似事务的机制，比如先写一个新记录标记，然后写作业记录，然后更新作业配置，最后删除记录文件
// 如果创建新纪录的时候，新纪录标记存在，则检查作业记录和作业配置是否是最新的
var newJobKey = GetJobRecordFullKey(newJob.Cluster.Name, newJob.Name, newJob.Id);
var newJobKV = ConsulKV.Create(newJobKey);
newJobKV.Value = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newJob));
bool createResult = ConsulKV.CAS(newJobKV, cancellationToken);
LogWriter.Write(string.Format("create job record kv result: {0},{1},{2}", newJob.Name, newJob.Id, createResult));
if (!createResult)
{
return(false);
}
KVPair jobConfigKV = null;
int doTimes = 0;
do
{
doTimes++;
LogWriter.Write(string.Format("update the last job record id of the job config in config center: {0}", doTimes), Log.LogLevel.Debug);
var jobConfigKey = GetJobConfigFullKey(newJob.Cluster.Name, newJob.Name);
jobConfigKV = ConsulKV.Get(jobConfigKey, cancellationToken);
if (jobConfigKV == null)
{
LogWriter.Write(string.Format("the job config not exists in config center: {0}", newJob.Name));
return(false);
}
var jobConfig = JobConfig.CreateInstance(Encoding.UTF8.GetString(jobConfigKV.Value));
jobConfig.LastRecordId = newJob.Id;
jobConfig.LastRecordCreateTime = DateTime.Now;
jobConfig.ModifyIndex = jobConfigKV.ModifyIndex;
var jobConfigJson = JsonConvert.SerializeObject(jobConfig);
jobConfigKV.Value = Encoding.UTF8.GetBytes(jobConfigJson);
}while (!ConsulKV.CAS(jobConfigKV, cancellationToken));
LogWriter.Write(string.Format("the last job record id of the job configs has updated: {0}", newJob.Name));
return(true);
}
/// <summary>
/// 更新作业任务计划
/// </summary>
/// <returns>The job task plan.</returns>
/// <param name="job">Job.</param>
public JobBase UpdateJobTaskPlan(JobBase job, CancellationToken cancellationToken = default(CancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
JobBase consulJobWrapper = null;
KVPair jobRecordKV;
int updateTimes = 0;
do
{
updateTimes++;
Log.LogWriter.Write("UpdateJobTaskPlan Execute Times:" + updateTimes, Log.LogLevel.Info);
if (updateTimes > 1)
{
Thread.Sleep(200);
}
var jobRecordKey = GetJobRecordFullKey(job.Cluster.Name, job.Name, job.Id);
jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken);
var jobRecordJson = Encoding.UTF8.GetString(jobRecordKV.Value);
var jobRecord = JsonConvert.DeserializeObject <JobWrapper>(jobRecordJson);
if (jobRecord == null)
{
Log.LogWriter.Write(string.Format("正在更新作业任务计划，但是作业已不存在: {0}", jobRecordKey), Log.LogLevel.Error);
break;
}
jobRecord.ModifyIndex = jobRecordKV.ModifyIndex;
jobRecord.TaskPlan = job.TaskPlan;
if (jobRecord.Status == EnumJobRecordStatus.TaskCompleted)
{
jobRecord.Status = EnumJobRecordStatus.TaskExecuting;
}
// 将作业信息更新到本地
consulJobWrapper = jobRecord;
jobRecordJson = JsonConvert.SerializeObject(jobRecord);
jobRecordKV.Value = Encoding.UTF8.GetBytes(jobRecordJson);
} while (!ConsulKV.CAS(jobRecordKV, cancellationToken)); // 可能同时更新作业记录配置，所以这里用CAS
return(consulJobWrapper);
}
/// <summary>
/// Cancels the job record.
/// </summary>
/// <returns><c>true</c>, if job record was canceled, <c>false</c> otherwise.</returns>
/// <param name="jobRecord">Job record.</param>
public bool CancelJobRecord(JobBase jobRecord)
{
var jobRecordKey = GetJobRecordFullKey(jobRecord.Cluster.Name, jobRecord.Name, jobRecord.Id);
var jobRecordKV = ConsulKV.Get(jobRecordKey);
if (jobRecordKV == null || jobRecordKV.Value == null)
{
LogWriter.Write(string.Format("配置中心作业记录丢失：{0}", jobRecord.BusinessId), Log.LogLevel.Warn);
return(true);
}
if (jobRecordKV.ModifyIndex != jobRecord.ModifyIndex)
{
return(false);
}
jobRecordKV.Value = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(jobRecord));
return(ConsulKV.CAS(jobRecordKV));
}
/// <summary>
/// Gets the job record.
/// </summary>
/// <returns>The job record.</returns>
/// <param name="jobName">Job name.</param>
/// <param name="jobId">Job identifier.</param>
/// <param name="cluster">Cluster.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public JobBase GetJobRecord(string jobName, string jobId, Cluster cluster, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(jobName) || string.IsNullOrWhiteSpace(jobId))
{
return(null);
}
var jobRecordKey = GetJobRecordFullKey(cluster.Name, jobName, jobId);
var jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken);
if (jobRecordKV == null || jobRecordKV.Value == null)
{
LogWriter.Write(string.Format("配置中心作业记录丢失[{0}->{1}]", jobName, jobId), Log.LogLevel.Error);
return(null);
}
var jobRecord = JobBase.Deserialize(Encoding.UTF8.GetString(jobRecordKV.Value), cluster);
jobRecord.ModifyIndex = jobRecordKV.ModifyIndex;
return(jobRecord);
}
/// <summary>
/// 尝试更新作业实例状态
/// </summary>
/// <returns>The job status.</returns>
/// <param name="job">Job.</param>
/// <param name="status">Status.</param>
/// <param name="errCode">错误代码：0无错误 1不能设置为这个状态 2无需重复设置状态 3作业不存在</param>
/// <param name="latestJob">更新后最新的作业信息</param>
/// <param name="cancellationToken">Cancellation token.</param>
public bool TryUpdateJobStatus(JobBase job, EnumJobRecordStatus status, out int errCode, out JobBase latestJob, CancellationToken cancellationToken = default(CancellationToken))
{
errCode = 0;
latestJob = null;
cancellationToken.ThrowIfCancellationRequested();
var jobRecordKey = GetJobRecordFullKey(job.Cluster.Name, job.Name, job.Id);
KVPair jobRecordKV;
int updateTimes = 0;
do
{
updateTimes++;
Log.LogWriter.Write(string.Format("UpdateJobStatus Execute Times:{0},{1}", jobRecordKey, updateTimes), Log.LogLevel.Debug);
if (updateTimes > 1)
{
Thread.Sleep(200);
}
jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken);
if (jobRecordKV == null)
{
Log.LogWriter.Write(string.Format("the job missing: {0}", jobRecordKey), Log.LogLevel.Error);
errCode = 3;
return(false);
}
var jobRecordJson = Encoding.UTF8.GetString(jobRecordKV.Value);
Log.LogWriter.Write("UpdateJobStatus Get Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace);
JobWrapper jobRecord = JsonConvert.DeserializeObject <JobWrapper>(jobRecordJson);
// 状态没有改变
if (jobRecord.Status == status)
{
LogWriter.Write(string.Format("the job status is already in {0}", status));
errCode = 2;
return(false);
}
// 取消状态只能更新为 已取消或者取消失败
if (jobRecord.Status == EnumJobRecordStatus.Canceling &&
status != EnumJobRecordStatus.Canceled && status != EnumJobRecordStatus.CancelFailed)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
// 计划制定完毕 不能更新为 计划指定中
if (jobRecord.Status == EnumJobRecordStatus.PlanMaked && status == EnumJobRecordStatus.PlanMaking)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
// 任务合并完毕 不能更新为 任务合并中
if (jobRecord.Status == EnumJobRecordStatus.TaskMerged && status == EnumJobRecordStatus.TaskMerging)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
// 任务执行完成、任务执行失败、任务已同步、任务已合并 不能更新为 任务正在执行
if ((jobRecord.Status == EnumJobRecordStatus.TaskCompleted ||
jobRecord.Status == EnumJobRecordStatus.TaskExecutingFailed ||
jobRecord.Status == EnumJobRecordStatus.TaskSynced ||
jobRecord.Status == EnumJobRecordStatus.TaskMerged) &&
status == EnumJobRecordStatus.TaskExecuting)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
// 如果是任务计划还没有制定，则设置任务计划为null
if (status == EnumJobRecordStatus.Pending)
{
jobRecord.TaskPlan = null;
jobRecord.StartTime = DateTime.MinValue;
jobRecord.JobSplitStartTime = DateTime.MinValue;
jobRecord.CollectTaskResultStartTime = DateTime.MinValue;
}
// 如果是任务计划开始制定，则附上开始时间
if (status == EnumJobRecordStatus.PlanMaking)
{
jobRecord.StartTime = DateTime.Now;
jobRecord.JobSplitStartTime = DateTime.Now;
}
// 如果是任务计划制定完毕，则附上计划
if (status == EnumJobRecordStatus.PlanMaked)
{
jobRecord.TaskPlan = job.TaskPlan;
jobRecord.JobSplitEndTime = DateTime.Now;
}
// 如果是任务结果合并，则附上开始时间
if (status == EnumJobRecordStatus.TaskMerging)
{
jobRecord.CollectTaskResultStartTime = DateTime.Now;
}
// 如果是任务合并完毕，则附上结束时间
if (status == EnumJobRecordStatus.TaskMerged)
{
jobRecord.FinishedTime = DateTime.Now;
jobRecord.CollectTaskResultEndTime = DateTime.Now;
}
jobRecord.ModifyIndex = jobRecordKV.ModifyIndex;
jobRecord.Status = status;
latestJob = jobRecord;
jobRecordJson = JsonConvert.SerializeObject(jobRecord);
Log.LogWriter.Write("UpdateJobStatus CAS Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace);
jobRecordKV.Value = Encoding.UTF8.GetBytes(jobRecordJson);
} while (!ConsulKV.CAS(jobRecordKV, cancellationToken));
return(true);
}
/// <summary>
/// 尝试更新任务状态
/// </summary>
/// <returns><c>true</c>, if update task status was tryed, <c>false</c> otherwise.</returns>
/// <param name="task">Task.</param>
/// <param name="status">Status.</param>
/// <param name="errCode">错误代码：0无错误 1不能设置为这个状态 2无需重复设置状态 3作业或任务不存在</param>
/// <param name="latestJob">Latest job.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public bool TryUpdateTaskStatus(JobTask task, EnumTaskStatus status, out int errCode, out JobBase latestJob, CancellationToken cancellationToken = default(CancellationToken))
{
errCode = 0;
latestJob = null;
cancellationToken.ThrowIfCancellationRequested();
var jobRecordKey = GetJobRecordFullKey(task.Job.Cluster.Name, task.Job.Name, task.Job.Id);
KVPair jobRecordKV;
int updateIndex = 0;
do
{
updateIndex++;
Log.LogWriter.Write(string.Format("UpdateTaskStatus Execute Times: {0},{1}", jobRecordKey + ":" + task.Id, updateIndex), Log.LogLevel.Debug);
if (updateIndex > 1)
{
Thread.Sleep(200);
}
jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken);
if (jobRecordKV == null)
{
errCode = 3;
Log.LogWriter.Write(string.Format("the job missing: {0}", jobRecordKey), Log.LogLevel.Error);
return(false);
}
var jobRecordJson = Encoding.UTF8.GetString(jobRecordKV.Value);
Log.LogWriter.Write("UpdateTaskStatus Get Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace);
var jobRecord = JobBase.Deserialize(jobRecordJson, task.Job.Cluster);
jobRecord.ModifyIndex = jobRecordKV.ModifyIndex;
// 从作业任务计划中查找出任务：Synced和SyncFailed是Manager更改的状态，其它情况下是任务所属的Worker来更改
var consulTask = jobRecord.TaskPlan.Where(d => d.Key == task.Job.Cluster.CurrentMember.Id).SelectMany(d => d.Value.Where(t => t.Id == task.Id)).FirstOrDefault();
if (status == EnumTaskStatus.Synced || status == EnumTaskStatus.SyncFailed)
{
consulTask = jobRecord.TaskPlan.SelectMany(d => d.Value.Where(t => t.Id == task.Id)).FirstOrDefault();
}
if (consulTask == null)
{
errCode = 3;
Log.LogWriter.Write(string.Format("the job task missing: {0}", jobRecordKey), Log.LogLevel.Error);
return(false);
}
// 取消状态只能更新为 已取消或者取消失败
if (consulTask.Status == EnumTaskStatus.Canceling &&
status != EnumTaskStatus.Canceled && status != EnumTaskStatus.CancelFailed)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
if ((consulTask.Status == EnumTaskStatus.Completed ||
consulTask.Status == EnumTaskStatus.Synced) &&
status == EnumTaskStatus.Executing)
{
LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status));
errCode = 1;
return(false);
}
if (status == EnumTaskStatus.Executing)
{
consulTask.StartTime = DateTime.Now;
}
if (status == EnumTaskStatus.Completed)
{
consulTask.FinishedTime = DateTime.Now;
}
consulTask.Status = status;
jobRecord.ModifyIndex = jobRecordKV.ModifyIndex;
latestJob = jobRecord;
jobRecordJson = JsonConvert.SerializeObject(jobRecord);
Log.LogWriter.Write("UpdateTaskStatus CAS Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace);
jobRecordKV.Value = Encoding.UTF8.GetBytes(jobRecordJson);
} while (!ConsulKV.CAS(jobRecordKV, cancellationToken));
return(true);
}