public ParseAndCreate ( |
||
reader | The example to parse. | |
label | ILabel |
/// Optional label, taking precedence over "_label" property found in |
index | int | Optional index of example the given label should be applied for multi-line examples. |
return | VowpalWabbitExampleCollection |
public async Task<HttpResponseMessage> Post() { var header = this.Request.Headers.SingleOrDefault(x => x.Key == "Authorization"); if (header.Value == null) throw new UnauthorizedAccessException("AuthorizationToken missing"); var userToken = header.Value.First(); if (string.IsNullOrWhiteSpace(userToken) || userToken != ConfigurationManager.AppSettings["UserToken"]) return Request.CreateResponse(HttpStatusCode.Unauthorized); if (this.metaData == null || lastDownload + TimeSpan.FromMinutes(1) < DateTime.Now) { var url = ConfigurationManager.AppSettings["DecisionServiceSettingsUrl"]; this.metaData = ApplicationMetadataUtil.DownloadMetadata<ApplicationClientMetadata>(url); lastDownload = DateTime.Now; } using (var vw = new VowpalWabbit(new VowpalWabbitSettings(metaData.TrainArguments) { EnableStringExampleGeneration = true, EnableStringFloatCompact = true })) using (var serializer = new VowpalWabbitJsonSerializer(vw)) using (var example = serializer.ParseAndCreate(new JsonTextReader(new StreamReader(await Request.Content.ReadAsStreamAsync())))) { return Request.CreateResponse(HttpStatusCode.OK, example.VowpalWabbitString); } }
public void Validate(string line, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null) { using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw)) using (var jsonExample = jsonSerializer.ParseAndCreate(json, label)) { this.Validate(line, jsonExample, labelComparator, label); } }
public void TestJsonFeatureExtraction() { string json = "{\"ns1\":{\"location\":\"New York\", \"f2\":3.4}}"; using (var vw = new VowpalWabbit("-b 3 --noconstant")) using (var serializer = new VowpalWabbitJsonSerializer(vw)) using (var result = serializer.ParseAndCreate(json)) { var singleExample = result as VowpalWabbitSingleLineExampleCollection; Assert.IsNotNull(singleExample); if (singleExample != null) { foreach (var ns in singleExample.Example) { Console.WriteLine(ns.Index); foreach (var feature in ns) { Console.WriteLine("{0}:{1}", feature.WeightIndex, feature.X); } } var ns1 = singleExample.Example.ToArray(); Assert.AreEqual(1, ns1.Length); Assert.AreEqual((byte)'n', ns1[0].Index); CollectionAssert.AreEqual( new[] { new VowpalWabbitFeature(1, 12), new VowpalWabbitFeature(3.4f, 28) }, ns1[0].ToArray()); } // for documentation purpose only var multiExample = result as VowpalWabbitMultiLineExampleCollection; Assert.IsNull(multiExample); if (multiExample != null) { foreach (var example in multiExample.Examples) { foreach (var ns in example) { Console.WriteLine(ns.Index); foreach (var feature in ns) { Console.WriteLine("{0}:{1}", feature.WeightIndex, feature.X); } } } } } }
public void Validate(string line, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null) { using (var strExample = this.vw.ParseLine(line)) using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw)) using (var jsonExample = (VowpalWabbitSingleLineExampleCollection)jsonSerializer.ParseAndCreate(json, label)) using (var strJsonExample = this.vw.ParseLine(jsonExample.Example.VowpalWabbitString)) { var diff = strExample.Diff(this.vw, jsonExample.Example, labelComparator); Assert.IsNull(diff, diff + " generated string: '" + jsonExample.Example.VowpalWabbitString + "'"); diff = strExample.Diff(this.vw, strJsonExample, labelComparator); Assert.IsNull(diff, diff); } }
public void Validate(string[] lines, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null, int? index = null) { VowpalWabbitExample[] strExamples = new VowpalWabbitExample[lines.Count()]; try { for (int i = 0; i < lines.Length; i++) strExamples[i] = this.vw.ParseLine(lines[i]); using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw)) using (var jsonExample = (VowpalWabbitMultiLineExampleCollection)jsonSerializer.ParseAndCreate(json, label, index)) { var jsonExamples = new List<VowpalWabbitExample>(); if (jsonExample.SharedExample != null) jsonExamples.Add(jsonExample.SharedExample); jsonExamples.AddRange(jsonExample.Examples); Assert.AreEqual(strExamples.Length, jsonExamples.Count); for (int i = 0; i < strExamples.Length; i++) { using (var strJsonExample = this.vw.ParseLine(jsonExamples[i].VowpalWabbitString)) { var diff = strExamples[i].Diff(this.vw, jsonExamples[i], labelComparator); Assert.IsNull(diff, diff + " generated string: '" + jsonExamples[i].VowpalWabbitString + "'"); diff = strExamples[i].Diff(this.vw, strJsonExample, labelComparator); Assert.IsNull(diff, diff); } } } } finally { foreach (var ex in strExamples) if (ex != null) ex.Dispose(); } }
private IEnumerable<PipelineData> Stage1_Deserialize(PipelineData data) { try { using (var jsonReader = new JsonTextReader(new StringReader(data.JSON))) { //jsonReader.FloatParser = Util.ReadDoubleString; // jsonReader.ArrayPool = pool; VowpalWabbitJsonSerializer vwJsonSerializer = null; try { vwJsonSerializer = new VowpalWabbitJsonSerializer(this.trainer.VowpalWabbit, this.trainer.ReferenceResolver); vwJsonSerializer.RegisterExtension((state, property) => { if (property.Equals("_eventid", StringComparison.OrdinalIgnoreCase)) { if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.String) throw new VowpalWabbitJsonException(state.Reader, "Expected string"); data.EventId = (string)state.Reader.Value; return true; } else if (property.Equals("_timestamp", StringComparison.OrdinalIgnoreCase)) { if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.Date) throw new VowpalWabbitJsonException(state.Reader, "Expected date"); data.Timestamp = (DateTime)state.Reader.Value; } return false; }); data.Example = vwJsonSerializer.ParseAndCreate(jsonReader); if (data.Example == null) { // unable to create example due to missing data // will be trigger later vwJsonSerializer.UserContext = data.Example; // make sure the serialize is not deallocated vwJsonSerializer = null; } } finally { if (vwJsonSerializer != null) vwJsonSerializer.Dispose(); } performanceCounters.Stage1_JSON_DeserializePerSec.Increment(); // delayed if (data.Example == null) { this.performanceCounters.Feature_Requests_Pending.Increment(); yield break; } } } catch (Exception ex) { this.telemetry.TrackException(ex, new Dictionary<string, string> { { "JSON", data.JSON } }); this.performanceCounters.Stage2_Faulty_Examples_Total.Increment(); this.performanceCounters.Stage2_Faulty_ExamplesPerSec.Increment(); yield break; } yield return data; }