private void HandleJobTestLoad(string[] args) { var jobFileName = args.Length <= 0 ? "job-generated.json" : args[0]; var job = GridJob.Deserialize(File.ReadAllText(jobFileName)); foreach (var task in job.JobTasks) { task.ParentJob = job; } GridIo.CreateDirectoriesForJob(job); foreach (var file in job.JobFiles) { if (file.Direction == EGridJobFileDirection.WorkerInput) { var localPath = GridIo.ResolveFilePath(job, null, file); file.Bytes = File.ReadAllBytes(localPath); file.UpdateCheckSum(); } } if (Server.IsAnyJobActive()) { Logger.Warn($"Server has already executing job {Server.GetActiveJob()}, new job will be added to the queue"); } Logger.Info($"Queue new job {job}"); Server.RunQueueJob(job); }
public override void FinishJob(GridJob job, ILog logger) { var count = 0; foreach (var task in job.JobTasks) { if (task.OutputFile?.Bytes != null) { try { count += int.Parse(Encoding.UTF8.GetString(task.OutputFile.Bytes)); } catch { ; } } } logger.Info($"Total count is '{count}'"); File.WriteAllText(GridIo.ResolveFilePathShared(job, job.JobOutFile.FileName), $"{count}"); }
public override GridJob GenerateJob(string jobName, params string[] args) { if (args.Length < 1) { throw new Exception("Not enough arguments"); } var job = new GridJob { Name = jobName }; var moduleFile = args[0]; var textFile = args[1]; var outputFile = args[2]; var searchString = args[3]; var jobModuleFile = LoadJobFile(moduleFile, EGridJobFileDirection.WorkerInput, EGridJobFileShare.SharedBetweenTasks); job.JobFiles.Add(jobModuleFile); var jobInputTextFile = LoadJobFile(textFile, EGridJobFileDirection.WorkerInput, EGridJobFileShare.SharedBetweenTasks); job.JobFiles.Add(jobInputTextFile); var jobOutputFile = CreateJobTaskOutFile(outputFile); job.JobFiles.Add(jobOutputFile); job.JobOutFile = CreateJobTotalOutFile(outputFile); uint lastTaskId = 0; using (var fs = new FileStream(textFile, FileMode.Open, FileAccess.Read)) { using (var br = new BinaryReader(fs)) { while (fs.Position < fs.Length) { if (fs.Length - fs.Position < BlockSize) { job.JobTasks.Add(CreateTask( jobModuleFile, jobInputTextFile, outputFile, fs.Position, fs.Position + fs.Length, searchString, lastTaskId)); return(job); } var startPos = fs.Position; var endPos = startPos + BlockSize; if (fs.Length >= BlockSize + searchString.Length + 1) { fs.Seek(BlockSize - (searchString.Length + 1), SeekOrigin.Current); var middleBuffer = br.ReadBytes((searchString.Length + 1) * 2); var middleStr = Encoding.UTF8.GetString(middleBuffer); for (var i = 0; i < middleStr.Length; i++) { if (SearchSubstringAt(middleStr, searchString, i) && i > searchString.Length) { endPos = startPos + BlockSize + i + searchString.Length; break; } } } else { endPos = fs.Length; } job.JobTasks.Add(CreateTask( jobModuleFile, jobInputTextFile, outputFile, startPos, endPos, searchString, lastTaskId++)); fs.Seek(endPos, SeekOrigin.Begin); } } } return(job); }