private IEnumerable<PipelineData> Stage1_Deserialize(PipelineData data)
{
try
{
using (var jsonReader = new JsonTextReader(new StringReader(data.JSON)))
{
//jsonReader.FloatParser = Util.ReadDoubleString;
// jsonReader.ArrayPool = pool;
VowpalWabbitJsonSerializer vwJsonSerializer = null;
try
{
vwJsonSerializer = new VowpalWabbitJsonSerializer(this.trainer.VowpalWabbit, this.trainer.ReferenceResolver);
vwJsonSerializer.RegisterExtension((state, property) =>
{
if (property.Equals("_eventid", StringComparison.OrdinalIgnoreCase))
{
if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.String)
throw new VowpalWabbitJsonException(state.Reader, "Expected string");
data.EventId = (string)state.Reader.Value;
return true;
}
else if (property.Equals("_timestamp", StringComparison.OrdinalIgnoreCase))
{
if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.Date)
throw new VowpalWabbitJsonException(state.Reader, "Expected date");
data.Timestamp = (DateTime)state.Reader.Value;
}
return false;
});
data.Example = vwJsonSerializer.ParseAndCreate(jsonReader);
if (data.Example == null)
{
// unable to create example due to missing data
// will be trigger later
vwJsonSerializer.UserContext = data.Example;
// make sure the serialize is not deallocated
vwJsonSerializer = null;
}
}
finally
{
if (vwJsonSerializer != null)
vwJsonSerializer.Dispose();
}
performanceCounters.Stage1_JSON_DeserializePerSec.Increment();
// delayed
if (data.Example == null)
{
this.performanceCounters.Feature_Requests_Pending.Increment();
yield break;
}
}
}
catch (Exception ex)
{
this.telemetry.TrackException(ex, new Dictionary<string, string> { { "JSON", data.JSON } });
this.performanceCounters.Stage2_Faulty_Examples_Total.Increment();
this.performanceCounters.Stage2_Faulty_ExamplesPerSec.Increment();
yield break;
}
yield return data;
}
public TrainerResult Learn(PipelineData example)
{
try
{
if (this.settings.EnableExampleTracing)
this.telemetry.TrackTrace(
"Example",
SeverityLevel.Verbose,
new Dictionary<string, string>
{
{ "ID", example.EventId },
{ "VW", example.Example.VowpalWabbitString },
{ "JSON", example.JSON }
});
var label = example.Example.Labels
.OfType<ContextualBanditLabel>()
.FirstOrDefault(l => l.Probability != 0f || l.Cost != 0);
if (label == null)
this.telemetry.TrackTrace($"Unable to find valid label for event '{example.EventId}'", SeverityLevel.Warning);
var progressivePrediction = example.Example.Learn(VowpalWabbitPredictionType.ActionScore, this.vw);
//if (this.vwAllReduce != null)
//{
// this.vwAllReduce.Post(vw =>
// {
// var actions = example.Example.Learn(VowpalWabbitPredictionType.Multilabel, vw);
// PerformanceCounters.Instance.ExamplesLearnedTotal.Increment();
// PerformanceCounters.Instance.ExamplesLearnedSec.Increment();
// PerformanceCounters.Instance.FeaturesLearnedSec.IncrementBy((long)example.Example.NumberOfFeatures);
// example.Example.Dispose();
// });
//}
// record event id for reproducibility
this.trackbackList.Add(example.EventId);
this.perfCounters.Stage2_Learn_Total.Increment();
this.perfCounters.Stage2_Learn_ExamplesPerSec.Increment();
this.perfCounters.Stage2_Learn_FeaturesPerSec.IncrementBy((long)example.Example.NumberOfFeatures);
// measure latency
const int TimeSpanTicksPerMillisecond = 10000;
var latency = DateTime.UtcNow - example.Timestamp;
var performanceCounterTicks =
latency.Ticks * Stopwatch.Frequency / TimeSpanTicksPerMillisecond;
this.perfCounters.AverageExampleLatency.IncrementBy(performanceCounterTicks);
this.perfCounters.AverageExampleLatencyBase.Increment();
// update partition state
if (example.PartitionKey != null && example.PartitionKey != null)
{
this.state.Partitions[example.PartitionKey] = example.Offset;
// this.state.PartitionsDateTime[eventHubExample.PartitionKey] = eventHubExample.Offset;
}
return new TrainerResult(example.Actions, example.Probabilities)
{
Label = label,
ProgressivePrediction = progressivePrediction,
PartitionKey = example.PartitionKey,
Latency = latency,
ProbabilityOfDrop = example.ProbabilityOfDrop
};
}
catch (Exception ex)
{
this.telemetry.TrackException(ex);
this.perfCounters.Stage2_Faulty_ExamplesPerSec.Increment();
this.perfCounters.Stage2_Faulty_Examples_Total.Increment();
return null;
}
finally
{
if (example.Example != null)
example.Example.Dispose();
}
}
internal async Task Stage0_Split(PartitionContext context, IEnumerable<EventData> messages)
{
foreach (EventData eventData in messages)
{
try
{
using (var eventStream = eventData.GetBodyStream())
{
using (var sr = new StreamReader(eventStream, Encoding.UTF8))
{
string line;
while ((line = await sr.ReadLineAsync()) != null)
{
var data = new PipelineData
{
JSON = line,
PartitionKey = context.Lease.PartitionId,
Offset = eventData.Offset
};
// TODO: ArrayBuffer to avoid string allocation...
// also just send char ref + offset + length
if (!await this.deserializeBlock.SendAsync(data))
this.telemetry.TrackTrace("Failed to enqueue data");
}
this.performanceCounters.Stage0_IncomingBytesPerSec.IncrementBy(eventStream.Position);
this.performanceCounters.Stage0_Batches_Size.IncrementBy(eventStream.Position);
this.performanceCounters.Stage0_Batches_SizeBase.Increment();
}
}
this.performanceCounters.Stage0_BatchesPerSec.Increment();
this.performanceCounters.Stage0_Batches_Total.Increment();
}
catch (Exception ex)
{
this.telemetry.TrackException(ex);
}
}
}