private void RecordComplete(string jobId, int recordId, ScoringExecutionSummary executionTime)
{
lock (JobManager.ThreadLock)
{
this.RecordCompleted?.Invoke(jobId, recordId, executionTime);
}
}
private void ManagerRecordCompleted(string jobId, int recordId, ScoringExecutionSummary runTime)
{
this.Statistics.AITimes.Add(runTime.AITime.TotalSeconds);
this.Statistics.FailureCount += (runTime.State == false ? 1 : 0);
// If the Event Hub is not configured, save the data locally
if (!this.Context.HubConfiguration.IsValid)
{
String output = String.Format("{0},{1},{2},{3},{4},{5},{6}",
recordId,
runTime.PayloadSize.ToString(),
(runTime.State == false ? "0" : "1"),
runTime.Attempts,
runTime.Response,
runTime.ExecutionTime.TotalSeconds.ToString(),
runTime.AITime.TotalSeconds.ToString());
TimingLog.RecordTiming(this.Context, output);
}
}
public static async void ScoreRecord(object data)
{
if (data is JobThreadData)
{
JobThreadData threadData = data as JobThreadData;
List <ThroughputMessage> eventMessages = new List <ThroughputMessage>();
int recordsProcessed = 0;
// Gets a list of messages appropriate to the number of threads that are
// being run for this test.
Dictionary <int, string> rvp = threadData.Records.GetNextBatch();
String optionalError = String.Empty;
try
{
// Consecutive errors. This is when an attempt to send fails
// the limit and an error is returned here. If we get three
// of these in a row, kill the thread.
List <String> consecutiveErrorCodes = new List <String>();
foreach (KeyValuePair <int, string> record in rvp)
{
recordsProcessed++;
// Full Execution Time
DateTime fullStart = DateTime.Now;
ScoringExecutionSummary jobExecutionSummary = new ScoringExecutionSummary();
jobExecutionSummary.PayloadSize = record.Value.Length;
// Get new time for AI call
RequestResult returnValue = await EndpointRequest.MakeRequest(
threadData.Client,
record.Value,
threadData.Context.Execution.RetryCount);
// Record timing for request
jobExecutionSummary.AITime = returnValue.ResponseTime;
jobExecutionSummary.Response = returnValue.Response;
jobExecutionSummary.State = returnValue.State;
jobExecutionSummary.Attempts = returnValue.Attempts;
jobExecutionSummary.ExecutionTime = (DateTime.Now - fullStart);
// Let the caller know about the call, good or bad
threadData.RecordComplete?.Invoke(threadData.JobId, record.Key, jobExecutionSummary);
// Record this send data for the event hub
RecordEventHubRecords(threadData.Context.Execution.ClientName, eventMessages, returnValue);
if (returnValue.State == false)
{
consecutiveErrorCodes.Add(returnValue.ResponseCode.ToString());
if (consecutiveErrorCodes.Count > 3)
{
String errorText =
String.Format("Too many consecutive errors ; {0}",
String.Join(" | ", consecutiveErrorCodes));
EventHubUtility.ProcessOneOff(threadData.Context.HubConfiguration,
threadData.Context.Execution.ClientName,
2,
errorText);
// Get this thread to terminate and report
throw new Exception(errorText);
}
}
else
{
consecutiveErrorCodes.Clear();
}
}
}
catch (Exception ex)
{
String exception = ex.Message;
Exception tmpEx = ex.InnerException;
while (tmpEx != null)
{
exception = String.Format("{0}{1}{2}", exception, Environment.NewLine, tmpEx.Message);
tmpEx = tmpEx.InnerException;
}
optionalError = String.Format("Exception after processing {0} records, terminating run {1}{2}",
recordsProcessed,
Environment.NewLine,
exception);
}
// Upload everything to event hub
EventHubUtility.ProcessMessages(threadData.Context.HubConfiguration, eventMessages);
// Notify that job is done
threadData.ThreadExiting?.Invoke(threadData.JobId, recordsProcessed, optionalError);
}
}
