public static async void ScoreRecord(object data)
{
if (data is JobThreadData)
{
JobThreadData threadData = data as JobThreadData;
List <ThroughputMessage> eventMessages = new List <ThroughputMessage>();
int recordsProcessed = 0;
// Gets a list of messages appropriate to the number of threads that are
// being run for this test.
Dictionary <int, string> rvp = threadData.Records.GetNextBatch();
String optionalError = String.Empty;
try
{
// Consecutive errors. This is when an attempt to send fails
// the limit and an error is returned here. If we get three
// of these in a row, kill the thread.
List <String> consecutiveErrorCodes = new List <String>();
foreach (KeyValuePair <int, string> record in rvp)
{
recordsProcessed++;
// Full Execution Time
DateTime fullStart = DateTime.Now;
ScoringExecutionSummary jobExecutionSummary = new ScoringExecutionSummary();
jobExecutionSummary.PayloadSize = record.Value.Length;
// Get new time for AI call
RequestResult returnValue = await EndpointRequest.MakeRequest(
threadData.Client,
record.Value,
threadData.Context.Execution.RetryCount);
// Record timing for request
jobExecutionSummary.AITime = returnValue.ResponseTime;
jobExecutionSummary.Response = returnValue.Response;
jobExecutionSummary.State = returnValue.State;
jobExecutionSummary.Attempts = returnValue.Attempts;
jobExecutionSummary.ExecutionTime = (DateTime.Now - fullStart);
// Let the caller know about the call, good or bad
threadData.RecordComplete?.Invoke(threadData.JobId, record.Key, jobExecutionSummary);
// Record this send data for the event hub
RecordEventHubRecords(threadData.Context.Execution.ClientName, eventMessages, returnValue);
if (returnValue.State == false)
{
consecutiveErrorCodes.Add(returnValue.ResponseCode.ToString());
if (consecutiveErrorCodes.Count > 3)
{
String errorText =
String.Format("Too many consecutive errors ; {0}",
String.Join(" | ", consecutiveErrorCodes));
EventHubUtility.ProcessOneOff(threadData.Context.HubConfiguration,
threadData.Context.Execution.ClientName,
2,
errorText);
// Get this thread to terminate and report
throw new Exception(errorText);
}
}
else
{
consecutiveErrorCodes.Clear();
}
}
}
catch (Exception ex)
{
String exception = ex.Message;
Exception tmpEx = ex.InnerException;
while (tmpEx != null)
{
exception = String.Format("{0}{1}{2}", exception, Environment.NewLine, tmpEx.Message);
tmpEx = tmpEx.InnerException;
}
optionalError = String.Format("Exception after processing {0} records, terminating run {1}{2}",
recordsProcessed,
Environment.NewLine,
exception);
}
// Upload everything to event hub
EventHubUtility.ProcessMessages(threadData.Context.HubConfiguration, eventMessages);
// Notify that job is done
threadData.ThreadExiting?.Invoke(threadData.JobId, recordsProcessed, optionalError);
}
}
private void ManagerAllThreadsCompleted()
{
this.Manager.AllThreadsCompleted -= ManagerAllThreadsCompleted;
this.Manager.RecordCompleted -= ManagerRecordCompleted;
JobStatistics stats = new JobStatistics(this.Manager.ExecutionData.Values);
bool hasErrors = (this.Statistics.FailureCount > 0);
HistoryLog.RecordHistory(
this.Context,
String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
this.Statistics.ExecutionCount,
this.Context.Execution.TestCountPerThreadStep,
this.Context.Execution.ThreadCount,
this.Statistics.RecordsInRun,
this.Statistics.FailureCount,
this.Statistics.AverageAITime(),
stats.TotalExecutionTime,
(this.Statistics.RecordsInRun / stats.TotalExecutionTime),
stats.MaxProcessedRecords,
stats.MinProcessedRecords)
);
this.StatusUpdate?.Invoke(String.Format("Finished job with {0} threads", this.Context.Execution.ThreadCount));
// Reset AI times and failure counts for next run
this.Statistics.AITimes.Clear();
this.Statistics.FailureCount = 0;
if (this.Context.Execution.AutoScaling)
{
if (++this.Statistics.ExecutionCount < this.Context.Execution.TestCountPerThreadStep)
{
String errorText = String.Format("INTERNAL - Scaling up but no more threads left -> {0}.", this.Context.Execution.ThreadCount);
if (!hasErrors)
{
this.ErrorTrackingAutoScale[SUCCESS] = this.Context.Execution.ThreadCount;
if (this.ErrorTrackingAutoScale[FAILURE] != -1)
{
errorText = String.Format("INTERNAL - Previous errors detected, not moving up thread count.");
}
else if (this.Context.Execution.ThreadCount < this.Context.Execution.MaxThreadCount)
{
this.Context.Execution.ThreadCount += this.Context.Execution.AutoScaleIncrement;
errorText = String.Format("INTERNAL - Scaling up thread count");
ClientScalingLog.RecordScalingChange(this.Context, this.Context.Execution.ThreadCount);
}
}
else // Something is wrong, scale back
{
this.StatusUpdate?.Invoke("Errors detected, scaling back");
this.ErrorTrackingAutoScale[FAILURE] = this.Context.Execution.ThreadCount;
errorText = String.Format("INTERNAL - Scaled back to a single thread already");
if (this.Context.Execution.ThreadCount > 1)
{
this.Context.Execution.ThreadCount -= 1;
errorText = String.Format("INTERNAL - Scaling back thread count with errors");
}
ClientScalingLog.RecordScalingChange(this.Context, this.Context.Execution.ThreadCount);
}
EventHubUtility.ProcessOneOff(this.Context.HubConfiguration,
this.Context.Execution.ClientName,
2,
errorText);
this.StartExecution();
}
else
{
// Let caller know we're done.
this.AllThreadsCompleted?.Invoke();
}
}
else
{
if (++this.Statistics.ExecutionCount < this.Context.Execution.TestCountPerThreadStep)
{
this.StartExecution();
}
else
{
// Increase thread count until max
if (this.Context.Execution.ThreadStep > 0 &&
this.Context.Execution.ThreadCount + this.Context.Execution.ThreadStep <= this.Context.Execution.MaxThreadCount)
{
this.Statistics.ExecutionCount = 0;
this.Context.Execution.ThreadCount += this.Context.Execution.ThreadStep;
this.StartExecution();
}
else
{
this.AllThreadsCompleted?.Invoke();
}
}
}
}