private void InternalStartJobs()
{
this.StatusUpdate?.Invoke("Starting jobs, queue ready");
for (int i = 0; i < this.Context.Execution.ThreadCount; i++)
{
string jobId = Guid.NewGuid().ToString("D");
System.Net.Http.HttpClient client = this.CreateClient(jobId);
JobThreadData threadData = new JobThreadData()
{
JobId = jobId,
Context = this.Context,
Client = client,
Records = this.RecordSupplier,
RecordComplete = this.RecordComplete,
ThreadExiting = this.ThreadCompleted
};
// Tag the start
this.ExecutionData.Add(threadData.JobId, new JobExecutionTiming()
{
Start = DateTime.Now
});
// Create and start the thread
System.Threading.Thread newThread = new System.Threading.Thread(Job.ScoreRecord);
newThread.Start(threadData);
this.Threads.Add(newThread);
this.StatusUpdate?.Invoke("Starting job : " + jobId);
}
}
public static async void ScoreRecord(object data)
{
if (data is JobThreadData)
{
JobThreadData threadData = data as JobThreadData;
List <ThroughputMessage> eventMessages = new List <ThroughputMessage>();
int recordsProcessed = 0;
// Gets a list of messages appropriate to the number of threads that are
// being run for this test.
Dictionary <int, string> rvp = threadData.Records.GetNextBatch();
String optionalError = String.Empty;
try
{
// Consecutive errors. This is when an attempt to send fails
// the limit and an error is returned here. If we get three
// of these in a row, kill the thread.
List <String> consecutiveErrorCodes = new List <String>();
foreach (KeyValuePair <int, string> record in rvp)
{
recordsProcessed++;
// Full Execution Time
DateTime fullStart = DateTime.Now;
ScoringExecutionSummary jobExecutionSummary = new ScoringExecutionSummary();
jobExecutionSummary.PayloadSize = record.Value.Length;
// Get new time for AI call
RequestResult returnValue = await EndpointRequest.MakeRequest(
threadData.Client,
record.Value,
threadData.Context.Execution.RetryCount);
// Record timing for request
jobExecutionSummary.AITime = returnValue.ResponseTime;
jobExecutionSummary.Response = returnValue.Response;
jobExecutionSummary.State = returnValue.State;
jobExecutionSummary.Attempts = returnValue.Attempts;
jobExecutionSummary.ExecutionTime = (DateTime.Now - fullStart);
// Let the caller know about the call, good or bad
threadData.RecordComplete?.Invoke(threadData.JobId, record.Key, jobExecutionSummary);
// Record this send data for the event hub
RecordEventHubRecords(threadData.Context.Execution.ClientName, eventMessages, returnValue);
if (returnValue.State == false)
{
consecutiveErrorCodes.Add(returnValue.ResponseCode.ToString());
if (consecutiveErrorCodes.Count > 3)
{
String errorText =
String.Format("Too many consecutive errors ; {0}",
String.Join(" | ", consecutiveErrorCodes));
EventHubUtility.ProcessOneOff(threadData.Context.HubConfiguration,
threadData.Context.Execution.ClientName,
2,
errorText);
// Get this thread to terminate and report
throw new Exception(errorText);
}
}
else
{
consecutiveErrorCodes.Clear();
}
}
}
catch (Exception ex)
{
String exception = ex.Message;
Exception tmpEx = ex.InnerException;
while (tmpEx != null)
{
exception = String.Format("{0}{1}{2}", exception, Environment.NewLine, tmpEx.Message);
tmpEx = tmpEx.InnerException;
}
optionalError = String.Format("Exception after processing {0} records, terminating run {1}{2}",
recordsProcessed,
Environment.NewLine,
exception);
}
// Upload everything to event hub
EventHubUtility.ProcessMessages(threadData.Context.HubConfiguration, eventMessages);
// Notify that job is done
threadData.ThreadExiting?.Invoke(threadData.JobId, recordsProcessed, optionalError);
}
}