static void Main(string[] args) { var paras = ResolveArguments(args); // 分割作业为不同的任务 if (paras.ContainsKey("-d")) { var jobSpacePath = paras["-jp"]; var jobConfigPath = Path.Combine(jobSpacePath, "job.json"); var jobConfigJson = File.ReadAllText(jobConfigPath, Encoding.UTF8); var jobWrapper = JobBase.Deserialize(jobConfigJson, null); var demoJob = CreateJobInstance(jobWrapper.JobClassName, jobWrapper); demoJob.GenerateTasks(); } // 处理任务 if (paras.ContainsKey("-p")) { if (!paras.ContainsKey("-t")) { Console.Write("缺少任务Id"); return; } var taskId = paras["-t"]; var jobSpacePath = paras["-jp"]; // 读取作业配置,创建当前作业的实例 var jobConfigPath = Path.Combine(jobSpacePath, "job.json"); var jobConfigJson = File.ReadAllText(jobConfigPath, Encoding.UTF8); var jobWrapper = JobBase.Deserialize(jobConfigJson, null); var demoJob = CreateJobInstance(jobWrapper.JobClassName, jobWrapper); // 读取任务配置,创建当前任务的实例 var taskConfigPath = Path.Combine(jobSpacePath, "tasks", taskId, "task.json"); var task = JobTask.CreateInstance(taskConfigPath); task.Job = jobWrapper; task.LoadRequirement(); demoJob.PerformTask(task); } // 合并任务 if (paras.ContainsKey("-m")) { // 读取作业配置,创建当前作业的实例 var jobSpacePath = paras["-jp"]; var jobConfigPath = Path.Combine(jobSpacePath, "job.json"); var jobConfigJson = File.ReadAllText(jobConfigPath, Encoding.UTF8); var jobWrapper = JobBase.Deserialize(jobConfigJson, null); var demoJob = CreateJobInstance(jobWrapper.JobClassName, jobWrapper); demoJob.CollectTaskResults(); } }
/// <summary> /// Gets the job record. /// </summary> /// <returns>The job record.</returns> /// <param name="jobName">Job name.</param> /// <param name="jobId">Job identifier.</param> /// <param name="cluster">Cluster.</param> /// <param name="cancellationToken">Cancellation token.</param> public JobBase GetJobRecord(string jobName, string jobId, Cluster cluster, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(jobName) || string.IsNullOrWhiteSpace(jobId)) { return(null); } var jobRecordKey = GetJobRecordFullKey(cluster.Name, jobName, jobId); var jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken); if (jobRecordKV == null || jobRecordKV.Value == null) { LogWriter.Write(string.Format("配置中心作业记录丢失[{0}->{1}]", jobName, jobId), Log.LogLevel.Error); return(null); } var jobRecord = JobBase.Deserialize(Encoding.UTF8.GetString(jobRecordKV.Value), cluster); jobRecord.ModifyIndex = jobRecordKV.ModifyIndex; return(jobRecord); }
/// <summary> /// 获取所有作业记录 /// </summary> /// <param name="clusterName"></param> /// <param name="job"></param> /// <returns></returns> private List <JobBase> GetSpecifiedDayJobRecords(string clusterName, string job, DateTime date) { string year = date.ToString("yyyy"); string month = date.ToString("MM"); string day = date.ToString("dd"); List <JobBase> jobRecordList = new List <JobBase>(); var jobRecordKeyPrefix = string.Format("Swift/{0}/Jobs/{1}/Records/{2}/{3}/{4}", clusterName, job, year, month, day); var jobRecordKeys = ConsulKV.Keys(jobRecordKeyPrefix); if (jobRecordKeys != null && jobRecordKeys.Length > 0) { var orderedKeys = jobRecordKeys.OrderByDescending(d => d); foreach (var recordKey in orderedKeys) { var jobRecordKV = ConsulKV.Get(recordKey); var jobRecord = JobBase.Deserialize(Encoding.UTF8.GetString(jobRecordKV.Value), new Cluster(clusterName, string.Empty)); jobRecordList.Add(jobRecord); } } return(jobRecordList); }
/// <summary> /// 尝试更新任务状态 /// </summary> /// <returns><c>true</c>, if update task status was tryed, <c>false</c> otherwise.</returns> /// <param name="task">Task.</param> /// <param name="status">Status.</param> /// <param name="errCode">错误代码:0无错误 1不能设置为这个状态 2无需重复设置状态 3作业或任务不存在</param> /// <param name="latestJob">Latest job.</param> /// <param name="cancellationToken">Cancellation token.</param> public bool TryUpdateTaskStatus(JobTask task, EnumTaskStatus status, out int errCode, out JobBase latestJob, CancellationToken cancellationToken = default(CancellationToken)) { errCode = 0; latestJob = null; cancellationToken.ThrowIfCancellationRequested(); var jobRecordKey = GetJobRecordFullKey(task.Job.Cluster.Name, task.Job.Name, task.Job.Id); KVPair jobRecordKV; int updateIndex = 0; do { updateIndex++; Log.LogWriter.Write(string.Format("UpdateTaskStatus Execute Times: {0},{1}", jobRecordKey + ":" + task.Id, updateIndex), Log.LogLevel.Debug); if (updateIndex > 1) { Thread.Sleep(200); } jobRecordKV = ConsulKV.Get(jobRecordKey, cancellationToken); if (jobRecordKV == null) { errCode = 3; Log.LogWriter.Write(string.Format("the job missing: {0}", jobRecordKey), Log.LogLevel.Error); return(false); } var jobRecordJson = Encoding.UTF8.GetString(jobRecordKV.Value); Log.LogWriter.Write("UpdateTaskStatus Get Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace); var jobRecord = JobBase.Deserialize(jobRecordJson, task.Job.Cluster); jobRecord.ModifyIndex = jobRecordKV.ModifyIndex; // 从作业任务计划中查找出任务:Synced和SyncFailed是Manager更改的状态,其它情况下是任务所属的Worker来更改 var consulTask = jobRecord.TaskPlan.Where(d => d.Key == task.Job.Cluster.CurrentMember.Id).SelectMany(d => d.Value.Where(t => t.Id == task.Id)).FirstOrDefault(); if (status == EnumTaskStatus.Synced || status == EnumTaskStatus.SyncFailed) { consulTask = jobRecord.TaskPlan.SelectMany(d => d.Value.Where(t => t.Id == task.Id)).FirstOrDefault(); } if (consulTask == null) { errCode = 3; Log.LogWriter.Write(string.Format("the job task missing: {0}", jobRecordKey), Log.LogLevel.Error); return(false); } // 取消状态只能更新为 已取消或者取消失败 if (consulTask.Status == EnumTaskStatus.Canceling && status != EnumTaskStatus.Canceled && status != EnumTaskStatus.CancelFailed) { LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status)); errCode = 1; return(false); } if ((consulTask.Status == EnumTaskStatus.Completed || consulTask.Status == EnumTaskStatus.Synced) && status == EnumTaskStatus.Executing) { LogWriter.Write(string.Format("{0} can not change to {1}", jobRecord.Status, status)); errCode = 1; return(false); } if (status == EnumTaskStatus.Executing) { consulTask.StartTime = DateTime.Now; } if (status == EnumTaskStatus.Completed) { consulTask.FinishedTime = DateTime.Now; } consulTask.Status = status; jobRecord.ModifyIndex = jobRecordKV.ModifyIndex; latestJob = jobRecord; jobRecordJson = JsonConvert.SerializeObject(jobRecord); Log.LogWriter.Write("UpdateTaskStatus CAS Value[" + jobRecordKV.ModifyIndex + "]" + jobRecordJson, Log.LogLevel.Trace); jobRecordKV.Value = Encoding.UTF8.GetBytes(jobRecordJson); } while (!ConsulKV.CAS(jobRecordKV, cancellationToken)); return(true); }