/// <summary> /// Deletes other parts of this job as failed, preventing the worker from taking them for calculation /// </summary> private void DeletePendingParts() { lock (jobsDataContext) { jobsDataContext.ExecuteCommand("DELETE FROM Job WHERE Hash='{0}' AND Status={1}", Hash, (int)JobOrPartState.Pending); } }
/// <summary> /// Creates the Jobs table in the database if it doesn't exist or waits until it is created /// </summary> /// <param name="sqlConnString">a DB to check</param> /// <param name="isInitiator">if true the table will be created, otherwise the method is blocked until the table is created by someone else</param> public static void InitializeJobTable(string sqlConnString, bool isInitiator) { var context = new JobsDBDataContext(sqlConnString); bool jobsSchemaExists = false; while (!jobsSchemaExists) { try { JobManagerTrace.TraceInfo("Connected to jobs database. {0} job(s) are pending.", (from j in context.Jobs where j.Status == (byte)JobOrPartState.Pending select j.Hash).Count()); jobsSchemaExists = true; } catch (SqlException) { if (!isInitiator) { JobManagerTrace.TraceInfo("Jobs database doesn't contain expected schema. Waiting for the frontend (role index 0) to initialize the schema. Rechecking JobsDB in couple of seconds"); Thread.Sleep(TimeSpan.FromSeconds(10)); } else { JobManagerTrace.TraceInfo("Jobs database doesn't contain expected schema. Deploying the schema"); StringBuilder sqlText = new StringBuilder(); using (System.IO.StreamReader reader = new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream("Microsoft.Research.Science.FetchClimate2.Jobs.sql"))) { while (!reader.EndOfStream) { var str = reader.ReadLine(); if (!str.Contains("GO")) { sqlText.AppendLine(str); } } } context.ExecuteCommand(sqlText.ToString()); JobManagerTrace.TraceInfo("Schema for JobsDB successfully deployed"); } } } }
public JobStatus Submit(IFetchRequest request, string hash, double jobRegistrationPermitedTime, int minPointPerPartition, int maxPointPerPartition, int totalWorkers) { var jobStatus = GetStatus(hash, false); int retries = 0; while (true) { if (jobStatus == null) { JobManagerTrace.TraceVerbose("{0}: Request is new (not in jobs table)", hash); // Try to create blob try { if (CreateBlobWithJob(request, hash, jobRegistrationPermitedTime)) // Successfully created { DateTime currentUTC = DateTime.UtcNow; int partsCount = GetPartitionsCount(request, minPointPerPartition, maxPointPerPartition, GetFreeWorkersNo(totalWorkers)); var partsToInsert = Enumerable.Range(0, partsCount).Select(partNumber => new Job() { Hash = hash, PartNo = partNumber, PartsCount = partsCount, Status = (byte)JobOrPartState.Pending, Priority = 0, SubmitTime = currentUTC, Touchtime = currentUTC }).ToArray(); try { jobsDataContext.Jobs.InsertAllOnSubmit(partsToInsert); jobsDataContext.SubmitChanges(System.Data.Linq.ConflictMode.FailOnFirstConflict); } catch (Exception ex) { JobManagerTrace.TraceError("{0}:Exception during insertion of job records into the job table. Reverting job registration (deleting job blob, all job records) {1}", hash, ex.ToString()); DiscardDataContextPendingChanges(); jobsDataContext.ExecuteCommand("delete from Job where hash='{0}'", hash); //order is important. deleting records before blob! DeleteJobBlob(hash); throw; //to be caught by retrying code } for (int i = 1; i <= partsCount; ++i) { JobManagerTrace.TraceVerbose("{0}:{1}:{2}:Record created in the jobs table", hash, i, partsCount); } return(GetStatus(hash)); } else // Blob exists { jobStatus = JobStatus.GetPendingStatus(hash, Int32.MaxValue); // Unknown queue length JobManagerTrace.TraceVerbose("{0}: Blob for request is created recently by someone else. Returning {1} status.", hash, jobStatus); return(jobStatus); } } catch (Exception e) { JobManagerTrace.TraceError("{0}:Request submitting error: {1}", hash, e.ToString()); if (retries++ > 3) { JobManagerTrace.TraceError("{0}:3 retries failed for request. Reporting failed state to the client", hash); return(JobStatus.GetFailedStatus(e.ToString())); } } } else { JobManagerTrace.TraceVerbose("{0}: Request is already submitted. Current state is {1}", hash, jobStatus); return(jobStatus); } } }