private void DelayedExampleCallback(VowpalWabbitJsonSerializer serializer) { try { this.perfCounters.Feature_Requests_Pending.IncrementBy(-1); var data = (PipelineData)serializer.UserContext; data.Example = serializer.CreateExamples(); // fire and forget // must not block to avoid dead lock this.trainProcessorFactory.LearnBlock .SendAsync(data) .ContinueWith(async ret => { if (!await ret) { this.telemetry.TrackTrace("Unable to enqueue delayed examples", SeverityLevel.Error); // since we couldn't enqueue, need to dispose here data.Example.Dispose(); } }); } catch (Exception e) { this.telemetry.TrackException(e); } finally { serializer.Dispose(); } }
public void TestJsonDictReverse() { var vec = new float[] { 1, 2, 3 }; var jsonResolver = new RefResolve(); var settings = new JsonSerializerSettings { ReferenceResolverProvider = () => jsonResolver }; var ctx1 = new Context(vec, 1, settings); var ctx2 = new Context(vec, 2, settings); VowpalWabbitJsonSerializer delayedSerializer = null; using (var validator = new VowpalWabbitExampleJsonValidator()) using (var vw = new VowpalWabbit(new VowpalWabbitSettings { EnableStringExampleGeneration = true })) using (var resolver = new VowpalWabbitJsonReferenceResolver(serializer => delayedSerializer = serializer)) { var serializer2 = new VowpalWabbitJsonSerializer(vw, resolver); var example2 = serializer2.ParseAndCreate(ctx2.JSON); // incomplete data Assert.IsNull(example2); // triggers example2 completion using (var serializer1 = new VowpalWabbitJsonSerializer(vw, resolver)) using (var example1 = serializer1.ParseAndCreate(ctx1.JSON)) { validator.Validate("| Id:1 :1 :2 :3", example1); } Assert.IsNotNull(delayedSerializer); using (var delayedExample2 = delayedSerializer.CreateExamples()) { validator.Validate("| Id:2 :1 :2 :3", delayedExample2); } delayedSerializer.Dispose(); } }
private IEnumerable <PipelineData> Stage1_Deserialize(PipelineData data) { try { using (var jsonReader = new JsonTextReader(new StringReader(data.JSON))) { //jsonReader.FloatParser = Util.ReadDoubleString; // jsonReader.ArrayPool = pool; VowpalWabbitJsonSerializer vwJsonSerializer = null; try { vwJsonSerializer = new VowpalWabbitJsonSerializer(this.trainer.VowpalWabbit, this.trainer.ReferenceResolver); vwJsonSerializer.RegisterExtension((state, property) => { if (TryExtractProperty(state, property, "_eventid", JsonToken.String, reader => data.EventId = (string)reader.Value)) { return(true); } else if (TryExtractProperty(state, property, "_timestamp", JsonToken.Date, reader => data.Timestamp = (DateTime)reader.Value)) { return(true); } else if (TryExtractProperty(state, property, "_ProbabilityOfDrop", JsonToken.Float, reader => data.ProbabilityOfDrop = (float)(reader.Value ?? 0f))) { return(true); } else if (TryExtractArrayProperty <float>(state, property, "_p", arr => data.Probabilities = arr)) { return(true); } else if (TryExtractArrayProperty <int>(state, property, "_a", arr => data.Actions = arr)) { return(true); } return(false); }); data.Example = vwJsonSerializer.ParseAndCreate(jsonReader); if (data.Probabilities == null) { throw new ArgumentNullException("Missing probabilities (_p)"); } if (data.Actions == null) { throw new ArgumentNullException("Missing actions (_a)"); } if (data.Example == null) { // unable to create example due to missing data // will be trigger later vwJsonSerializer.UserContext = data.Example; // make sure the serialize is not deallocated vwJsonSerializer = null; } } finally { if (vwJsonSerializer != null) { vwJsonSerializer.Dispose(); } } performanceCounters.Stage1_JSON_DeserializePerSec.Increment(); // delayed if (data.Example == null) { this.performanceCounters.Feature_Requests_Pending.Increment(); yield break; } } } catch (Exception ex) { this.telemetry.TrackException(ex, new Dictionary <string, string> { { "JSON", data.JSON } }); this.performanceCounters.Stage2_Faulty_Examples_Total.Increment(); this.performanceCounters.Stage2_Faulty_ExamplesPerSec.Increment(); yield break; } yield return(data); }
public void TestJsonDictThreading() { var jsonResolver = new RefResolve(); var settings = new JsonSerializerSettings { ReferenceResolverProvider = () => jsonResolver }; var rnd = new Random(123); var examples = new List <Context>(); var id = 0; // different reference objects for (int i = 0; i < 10; i++) { var data = Enumerable.Range(1, 5).Select(_ => (float)rnd.Next(10)).ToArray(); // referencing the same data for (int j = 0; j < 5; j++) { examples.Add(new Context(data, id++, settings)); } } for (int i = 0; i < 4; i++) { Permute(examples, rnd); for (int maxDegreeOfParallelism = 1; maxDegreeOfParallelism < 4; maxDegreeOfParallelism++) { var examplesFound = 0; using (var vw = new VowpalWabbit(new VowpalWabbitSettings { EnableStringExampleGeneration = true, EnableThreadSafeExamplePooling = true })) using (var resolver = new VowpalWabbitJsonReferenceResolver(serializer => { using (var example = serializer.CreateExamples()) { ValidateExample(example, (Context)serializer.UserContext); } serializer.Dispose(); Interlocked.Increment(ref examplesFound); })) { Parallel.ForEach( Partitioner.Create(0, examples.Count), new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, range => { for (int j = range.Item1; j < range.Item2; j++) { var ctx = examples[j]; var serializer = new VowpalWabbitJsonSerializer(vw, resolver) { UserContext = ctx }; var example = serializer.ParseAndCreate(ctx.JSON); // example not ready yet if (example == null) { continue; } ValidateExample(example, ctx); example.Dispose(); serializer.Dispose(); Interlocked.Increment(ref examplesFound); } }); } Assert.AreEqual(examples.Count, examplesFound); } } }
private IEnumerable <PipelineData> Stage1_Deserialize(PipelineData data) { try { using (var jsonReader = new JsonTextReader(new StringReader(data.JSON))) { //jsonReader.FloatParser = Util.ReadDoubleString; // jsonReader.ArrayPool = pool; VowpalWabbitJsonSerializer vwJsonSerializer = null; try { vwJsonSerializer = new VowpalWabbitJsonSerializer(this.trainer.VowpalWabbit, this.trainer.ReferenceResolver); vwJsonSerializer.RegisterExtension((state, property) => { if (property.Equals("_eventid", StringComparison.OrdinalIgnoreCase)) { if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.String) { throw new VowpalWabbitJsonException(state.Reader, "Expected string"); } data.EventId = (string)state.Reader.Value; return(true); } else if (property.Equals("_timestamp", StringComparison.OrdinalIgnoreCase)) { if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.Date) { throw new VowpalWabbitJsonException(state.Reader, "Expected date"); } data.Timestamp = (DateTime)state.Reader.Value; } return(false); }); data.Example = vwJsonSerializer.ParseAndCreate(jsonReader); if (data.Example == null) { // unable to create example due to missing data // will be trigger later vwJsonSerializer.UserContext = data.Example; // make sure the serialize is not deallocated vwJsonSerializer = null; } } finally { if (vwJsonSerializer != null) { vwJsonSerializer.Dispose(); } } performanceCounters.Stage1_JSON_DeserializePerSec.Increment(); // delayed if (data.Example == null) { this.performanceCounters.Feature_Requests_Pending.Increment(); yield break; } } } catch (Exception ex) { this.telemetry.TrackException(ex, new Dictionary <string, string> { { "JSON", data.JSON } }); this.performanceCounters.Stage2_Faulty_Examples_Total.Increment(); this.performanceCounters.Stage2_Faulty_ExamplesPerSec.Increment(); yield break; } yield return(data); }