/// <summary>
/// Deletes other parts of this job as failed, preventing the worker from taking them for calculation
/// </summary>
private void DeletePendingParts()
{
lock (jobsDataContext)
{
jobsDataContext.ExecuteCommand("DELETE FROM Job WHERE Hash='{0}' AND Status={1}",
Hash,
(int)JobOrPartState.Pending);
}
}
/// <summary>
/// Creates the Jobs table in the database if it doesn't exist or waits until it is created
/// </summary>
/// <param name="sqlConnString">a DB to check</param>
/// <param name="isInitiator">if true the table will be created, otherwise the method is blocked until the table is created by someone else</param>
public static void InitializeJobTable(string sqlConnString, bool isInitiator)
{
var context = new JobsDBDataContext(sqlConnString);
bool jobsSchemaExists = false;
while (!jobsSchemaExists)
{
try
{
JobManagerTrace.TraceInfo("Connected to jobs database. {0} job(s) are pending.",
(from j in context.Jobs where j.Status == (byte)JobOrPartState.Pending select j.Hash).Count());
jobsSchemaExists = true;
}
catch (SqlException)
{
if (!isInitiator)
{
JobManagerTrace.TraceInfo("Jobs database doesn't contain expected schema. Waiting for the frontend (role index 0) to initialize the schema. Rechecking JobsDB in couple of seconds");
Thread.Sleep(TimeSpan.FromSeconds(10));
}
else
{
JobManagerTrace.TraceInfo("Jobs database doesn't contain expected schema. Deploying the schema");
StringBuilder sqlText = new StringBuilder();
using (System.IO.StreamReader reader = new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream("Microsoft.Research.Science.FetchClimate2.Jobs.sql")))
{
while (!reader.EndOfStream)
{
var str = reader.ReadLine();
if (!str.Contains("GO"))
{
sqlText.AppendLine(str);
}
}
}
context.ExecuteCommand(sqlText.ToString());
JobManagerTrace.TraceInfo("Schema for JobsDB successfully deployed");
}
}
}
}
public JobStatus Submit(IFetchRequest request, string hash, double jobRegistrationPermitedTime, int minPointPerPartition, int maxPointPerPartition, int totalWorkers)
{
var jobStatus = GetStatus(hash, false);
int retries = 0;
while (true)
{
if (jobStatus == null)
{
JobManagerTrace.TraceVerbose("{0}: Request is new (not in jobs table)", hash);
// Try to create blob
try
{
if (CreateBlobWithJob(request, hash, jobRegistrationPermitedTime)) // Successfully created
{
DateTime currentUTC = DateTime.UtcNow;
int partsCount = GetPartitionsCount(request, minPointPerPartition, maxPointPerPartition, GetFreeWorkersNo(totalWorkers));
var partsToInsert = Enumerable.Range(0, partsCount).Select(partNumber => new Job()
{
Hash = hash, PartNo = partNumber, PartsCount = partsCount, Status = (byte)JobOrPartState.Pending, Priority = 0, SubmitTime = currentUTC, Touchtime = currentUTC
}).ToArray();
try
{
jobsDataContext.Jobs.InsertAllOnSubmit(partsToInsert);
jobsDataContext.SubmitChanges(System.Data.Linq.ConflictMode.FailOnFirstConflict);
}
catch (Exception ex)
{
JobManagerTrace.TraceError("{0}:Exception during insertion of job records into the job table. Reverting job registration (deleting job blob, all job records) {1}", hash, ex.ToString());
DiscardDataContextPendingChanges();
jobsDataContext.ExecuteCommand("delete from Job where hash='{0}'", hash); //order is important. deleting records before blob!
DeleteJobBlob(hash);
throw; //to be caught by retrying code
}
for (int i = 1; i <= partsCount; ++i)
{
JobManagerTrace.TraceVerbose("{0}:{1}:{2}:Record created in the jobs table", hash, i, partsCount);
}
return(GetStatus(hash));
}
else // Blob exists
{
jobStatus = JobStatus.GetPendingStatus(hash, Int32.MaxValue); // Unknown queue length
JobManagerTrace.TraceVerbose("{0}: Blob for request is created recently by someone else. Returning {1} status.", hash, jobStatus);
return(jobStatus);
}
}
catch (Exception e)
{
JobManagerTrace.TraceError("{0}:Request submitting error: {1}", hash, e.ToString());
if (retries++ > 3)
{
JobManagerTrace.TraceError("{0}:3 retries failed for request. Reporting failed state to the client", hash);
return(JobStatus.GetFailedStatus(e.ToString()));
}
}
}
else
{
JobManagerTrace.TraceVerbose("{0}: Request is already submitted. Current state is {1}", hash, jobStatus);
return(jobStatus);
}
}
}
