public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, CustomClass value) { Assert.IsNotNull(context); Assert.IsNotNull(ns); Assert.IsNotNull(feature); Assert.IsNotNull(value); Assert.AreEqual(5, value.X); value.HasVisited = true; }
public void Marshal(VowpalWabbitMarshalContext ctx, Namespace ns, Feature feature) { var jsonSerializer = new JsonSerializer(); using (var jsonBuilder = new VowpalWabbitJsonBuilder(ctx.VW, VowpalWabbitDefaultMarshaller.Instance, jsonSerializer)) { // serialize from object to JSON var sb = new StringBuilder(); using (var writer = new JsonTextWriter(new StringWriter(sb))) { this.jsonConverter.WriteJson(writer, this.value, jsonSerializer); } // marshal from JSON to VW using (var reader = new JsonTextReader(new StringReader(sb.ToString()))) { jsonBuilder.Parse(reader, ctx, new Namespace(ctx.VW, feature.Name)); } } }
/// <summary> /// Parses the example. /// </summary> /// <param name="reader">The example to parse.</param> /// <param name="label"> /// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>. /// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label. /// </param> public void Parse(JsonReader reader, ILabel label = null) { // avoid parameter passing for the sake of non-reentrantness this.reader = reader; if (label != null) this.defaultMarshaller.MarshalLabel(this.Context, label); // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(reader.Path, "Expected start object"); Namespace defaultNamespace = new Namespace(this.vw); using (this.DefaultNamespaceContext.NamespaceBuilder = this.DefaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace)) { while (reader.Read()) { switch (reader.TokenType) { case JsonToken.PropertyName: var propertyName = (string)reader.Value; if (propertyName.StartsWith(FeatureIgnorePrefix)) { // special fields switch (propertyName) { case "_label": // passed in label has precedence if (label == null) this.ParseLabel(); else reader.Skip(); break; //case "_shared": // break; //case "_multi": // break; default: reader.Skip(); break; } } else { if (!reader.Read()) throw new VowpalWabbitJsonException(reader.Path, "Unexpected end"); if (reader.TokenType == JsonToken.StartObject) this.ParseNamespaceAndFeatures(this.Context, propertyName); else this.ParseFeature(this.DefaultNamespaceContext, defaultNamespace, propertyName); } break; case JsonToken.EndObject: goto done; } } done: // append default namespaces features if we found some if (this.DefaultNamespaceContext.StringExample != null && this.DefaultNamespaceContext.StringExample.Length > 0) { this.Context.StringExample.AppendFormat(CultureInfo.InvariantCulture, " | {0}", this.DefaultNamespaceContext.StringExample); } } }
/// <summary> /// Triggers parsing at the current state of the <see cref="Reader"/> using the default namespace. /// </summary> public void Parse() { using (var context = new VowpalWabbitMarshalContext(this.VW, this.JsonBuilder.DefaultNamespaceContext.ExampleBuilder)) { var ns = new Namespace(this.VW); this.Parse(context, ns); } }
/// <summary> /// Triggers parsing at the current state of the <see cref="Reader"/> using the given <paramref name="namespaceContext"/>. /// </summary> /// <param name="namespaceContext">The namespace the JSON should be marshalled into.</param> /// <param name="ns">The namespace the JSON should be marshalled into.</param> public void Parse(VowpalWabbitMarshalContext namespaceContext, Namespace ns) { this.JsonBuilder.Parse(this.Path, namespaceContext, ns); }
/// <summary> /// Parses { "feature1":1, "feature2":true, .... } /// </summary> private void ParseNamespaceAndFeatures(List<VowpalWabbitJsonParseContext> path, string namespaceValue) { VowpalWabbitJsonParseContext localContext = null; try { var ns = new Namespace(this.vw, namespaceValue); localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder), JsonProperty = namespaceValue }; path.Add(localContext); var propertyConfiguration = this.vw.Settings.PropertyConfiguration; featureCount += this.defaultMarshaller.MarshalNamespace(localContext.Context, ns, () => this.ParseProperties(path)); path.RemoveAt(path.Count - 1); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = localContext.Context.ToString(); if (str.Length > 0) this.namespaceStrings.Add(str); } } finally { if (localContext != null && localContext.Context != null) localContext.Context.Dispose(); } }
// re-entering from extension internal void Parse(List<VowpalWabbitJsonParseContext> path, VowpalWabbitMarshalContext namespaceContext, Namespace ns) { this.featureCount = this.defaultMarshaller.MarshalNamespace(namespaceContext, ns, () => this.ParseProperties(path)) + this.featureCount; }
public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List<VowpalWabbitJsonExtension> extensions = null) { this.reader = reader; this.extensions = extensions; // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(this.reader, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); // re-direct default namespace to the one passed var saveDefaultNamespaceContext = this.DefaultNamespaceContext; try { using (this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder)) { VowpalWabbitJsonParseContext localContext = null; try { // setup current namespace localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder), JsonProperty = ns.Name }; { this.defaultMarshaller.MarshalNamespace( localContext.Context, ns, () => this.ParseProperties(new List<VowpalWabbitJsonParseContext> { localContext })); // append string features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { context.StringExample .Append(localContext.Context.StringExample) .Append(string.Join(" ", this.namespaceStrings)); } } } finally { if (localContext != null && localContext.Context != null) { localContext.Context.Dispose(); localContext.Context = null; } } } } finally { this.DefaultNamespaceContext = saveDefaultNamespaceContext; } }
/// <summary> /// Expects that actual feature value. /// </summary> private void ParseFeature(VowpalWabbitMarshalContext context, Namespace ns, string featureName) { switch (reader.TokenType) { case JsonToken.Float: { var feature = new PreHashedFeature(this.vw, ns, featureName); this.defaultMarshaller.MarshalFeature(context, ns, feature, (double)reader.Value); } break; case JsonToken.Integer: { var feature = new PreHashedFeature(this.vw, ns, featureName); this.defaultMarshaller.MarshalFeature(context, ns, feature, (long)reader.Value); } break; case JsonToken.String: { var feature = new Feature(featureName); this.defaultMarshaller.MarshalFeatureStringEscape(context, ns, feature, (string)reader.Value); } break; case JsonToken.Boolean: { var feature = new PreHashedFeature(this.vw, ns, featureName); this.defaultMarshaller.MarshalFeature(context, ns, feature, (bool)reader.Value); } break; case JsonToken.Comment: case JsonToken.Null: // probably best to ignore? break; case JsonToken.StartArray: this.ParseFeatureArray(context, ns); break; default: throw new VowpalWabbitJsonException(reader.Path, "Unexpected token " + reader.TokenType + " while deserializing primitive feature"); } }
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName) { // special fields switch (propertyName) { case VowpalWabbitConstants.LabelProperty: // passed in label has precedence if (label == null) this.ParseLabel(); else reader.Skip(); break; case VowpalWabbitConstants.TextProperty: // parse text segment feature this.defaultMarshaller.MarshalFeatureStringSplit( context, ns, new Feature(propertyName), reader.ReadAsString()); break; default: // forward to handler if (specialPropertyAction == null || !specialPropertyAction(propertyName)) reader.Skip(); // if not handled, skip it break; } }
/// <summary> /// Parses the example. /// </summary> /// <param name="reader">The example to parse.</param> /// <param name="label"> /// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>. /// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label. /// </param> /// <param name="specialPropertyAction">Action to be executed when sepcial properties are discovered.</param> /// <returns>The VowpalWabbit native example.</returns> public void Parse(JsonReader reader, ILabel label = null, Func<string, bool> specialPropertyAction = null) { // avoid parameter passing for the sake of non-reentrantness this.reader = reader; this.label = label; this.specialPropertyAction = specialPropertyAction; if (label != null) this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, label); // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(this.reader, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); var ns = new Namespace(this.vw); var path = new List<LocalContext> { new LocalContext { Namespace = ns, Context = this.DefaultNamespaceContext, JsonProperty = string.Empty } }; this.defaultMarshaller.MarshalNamespace(this.DefaultNamespaceContext, ns, () => this.ParseProperties(path)); if (this.labelObject != null) { var propertyName = ((JProperty)this.labelObject.First).Name; Type labelType; if (!labelPropertyMapping.TryGetValue(propertyName, out labelType)) throw new VowpalWabbitJsonException(this.reader, "The first property ('" + propertyName + "') must match to a property of a VowpalWabbit label type."); var labelObj = (ILabel)this.labelObject.ToObject(labelType); if (this.foundMulti) this.Label = labelObj; else this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, labelObj); } }
/// <summary> /// Parses { "feature1":1, "feature2":true, .... } /// </summary> private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue) { var ns = new Namespace(this.vw, namespaceValue); var propertyConfiguration = context.VW.Settings.PropertyConfiguration; this.defaultMarshaller.MarshalNamespace(context, ns, () => { while (reader.Read()) { switch (reader.TokenType) { case JsonToken.PropertyName: var propertyName = (string)reader.Value; if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName)) { this.ParseSpecialProperty(context, ns, propertyName); continue; } if (!reader.Read()) throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace"); this.ParseFeature(context, ns, propertyName); break; case JsonToken.EndObject: return; } } }); }
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName) { var propertyConfiguration = context.VW.Settings.PropertyConfiguration; // special fields if (propertyName == propertyConfiguration.LabelProperty) { // passed in label has precedence if (label == null) this.ParseLabel(); else reader.Skip(); } else if (propertyName == propertyConfiguration.TextProperty) { // parse text segment feature this.defaultMarshaller.MarshalFeatureStringSplit( context, ns, new Feature(propertyName), reader.ReadAsString()); } else { // forward to handler if (specialPropertyAction == null || !specialPropertyAction(propertyName)) reader.Skip(); // if not handled, skip it } }
/// <summary> /// Parses the example. /// </summary> /// <param name="reader">The example to parse.</param> /// <param name="label"> /// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>. /// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label. /// </param> /// <param name="specialPropertyAction">Action to be executed when sepcial properties are discovered.</param> /// <returns>The VowpalWabbit native example.</returns> public void Parse(JsonReader reader, ILabel label = null, Func<string, bool> specialPropertyAction = null) { // avoid parameter passing for the sake of non-reentrantness this.reader = reader; this.label = label; this.specialPropertyAction = specialPropertyAction; if (label != null) this.defaultMarshaller.MarshalLabel(this.Context, label); // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(reader.Path, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); Namespace defaultNamespace = new Namespace(this.vw); using (this.DefaultNamespaceContext.NamespaceBuilder = this.DefaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace)) { var propertyConfiguration = this.vw.Settings.PropertyConfiguration; while (reader.Read()) { switch (reader.TokenType) { case JsonToken.PropertyName: var propertyName = (string)reader.Value; if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName)) this.ParseSpecialProperty(this.DefaultNamespaceContext, defaultNamespace, propertyName); else { if (!reader.Read()) throw new VowpalWabbitJsonException(reader.Path, "Unexpected end"); if (reader.TokenType == JsonToken.StartObject) this.ParseNamespaceAndFeatures(this.Context, propertyName); else this.ParseFeature(this.DefaultNamespaceContext, defaultNamespace, propertyName); } break; case JsonToken.EndObject: goto done; } } done: // append default namespaces features if we found some if (this.DefaultNamespaceContext.StringExample != null && this.DefaultNamespaceContext.StringExample.Length > 0) { this.Context.StringExample.AppendFormat(CultureInfo.InvariantCulture, " | {0}", this.DefaultNamespaceContext.StringExample); } } }
/// <summary> /// Expects: "1,2.2,3]" (excluding the leading [) /// </summary> private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns) { ulong index = 0; while (reader.Read()) { switch (reader.TokenType) { case JsonToken.Integer: MarshalFloatFeature(context, ns, index, (float)(long)reader.Value); break; case JsonToken.Float: MarshalFloatFeature(context, ns, index, (float)(double)reader.Value); break; case JsonToken.EndArray: return; default: throw new VowpalWabbitJsonException(reader.Path, "Unxpected token " + reader.TokenType + " while deserializing dense feature array"); } index++; } }
private static void MarshalFloatFeature(VowpalWabbitMarshalContext context, Namespace ns, ulong index, float value) { context.NamespaceBuilder.AddFeature(ns.NamespaceHash + index, value); if (context.StringExample != null) { context.AppendStringExample( false, " {0}:" + (context.VW.Settings.EnableStringFloatCompact ? "{1}" : "{1:E20}"), index, value); } }
/// <summary> /// Parses the example. /// </summary> /// <param name="reader">The example to parse.</param> /// <param name="label"> /// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>. /// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label. /// </param> /// <param name="extensions">Action to be executed when special properties are discovered.</param> /// <returns>The VowpalWabbit native example.</returns> public void Parse(JsonReader reader, ILabel label = null, List<VowpalWabbitJsonExtension> extensions = null) { this.featureCount = 0; this.labelObject = null; this.foundMulti = false; // avoid parameter passing for the sake of non-reentrantness this.reader = reader; this.label = label; this.extensions = extensions; if (label != null) this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, label); // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(this.reader, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); var ns = new Namespace(this.vw); var path = new List<VowpalWabbitJsonParseContext> { new VowpalWabbitJsonParseContext { Namespace = ns, Context = this.DefaultNamespaceContext, JsonProperty = string.Empty } }; this.extensionState.Reader = reader; this.extensionState.Path = path; // TODO: duplicate namespace recursion to enable async // featureCount might be modified inside ParseProperties... this.featureCount = this.defaultMarshaller.MarshalNamespace(this.DefaultNamespaceContext, ns, () => this.ParseProperties(path)) + this.featureCount; if (this.labelObject != null) { var propertyName = ((JProperty)this.labelObject.First).Name; Type labelType; if (!labelPropertyMapping.TryGetValue(propertyName.ToLowerInvariant(), out labelType)) throw new VowpalWabbitJsonException(this.reader, "The first property ('" + propertyName + "') must match to a property of a VowpalWabbit label type."); var labelObj = (ILabel)this.labelObject.ToObject(labelType); if (this.foundMulti) this.Label = labelObj; else this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, labelObj); } }
/// <summary> /// Parses { "feature1":1, "feature2":true, .... } /// </summary> private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue) { var ns = new Namespace(this.vw, namespaceValue); this.defaultMarshaller.MarshalNamespace(context, ns, () => { while (reader.Read()) { switch (reader.TokenType) { case JsonToken.PropertyName: var featureName = (string)reader.Value; if (!reader.Read()) throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace"); this.ParseFeature(context, ns, featureName); break; case JsonToken.EndObject: return; } } }); }
/// <summary> /// Parses the example. /// </summary> /// <param name="reader">The example to parse.</param> /// <param name="label"> /// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>. /// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label. /// </param> /// <returns>The VowpalWabbit native example.</returns> public VowpalWabbitExample Parse(JsonReader reader, ILabel label = null) { // avoid parameter passing for the sake of non-reentrantness this.reader = reader; using (VowpalWabbitMarshalContext context = new VowpalWabbitMarshalContext(this.vw)) using (VowpalWabbitMarshalContext defaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder)) { if (label != null) this.defaultMarshaller.MarshalLabel(context, label); if (!reader.Read() || reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(reader.Path, "Expected start object"); Namespace defaultNamespace = new Namespace(this.vw); using (defaultNamespaceContext.NamespaceBuilder = defaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace)) { while (reader.Read()) { switch (reader.TokenType) { case JsonToken.PropertyName: var propertyName = (string)reader.Value; if (propertyName.StartsWith(FeatureIgnorePrefix)) { // special fields switch (propertyName) { case "_label": // passed in label has precedence if (label == null) this.ParseLabel(context); else reader.Skip(); break; //case "_shared": // break; //case "_multi": // break; default: reader.Skip(); break; } } else { if (!reader.Read()) throw new VowpalWabbitJsonException(reader.Path, "Unexpected end"); if (reader.TokenType == JsonToken.StartObject) this.ParseNamespaceAndFeatures(context, propertyName); else this.ParseFeature(defaultNamespaceContext, defaultNamespace, propertyName); } break; } } // append default namespaces features if we found some if (defaultNamespaceContext.StringExample != null && defaultNamespaceContext.StringExample.Length > 0) { context.StringExample.AppendFormat(CultureInfo.InvariantCulture, "| {0}", defaultNamespaceContext.StringExample); } } var vwExample = context.ExampleBuilder.CreateExample(); if (this.vw.Settings.EnableStringExampleGeneration) vwExample.VowpalWabbitString = context.StringExample.ToString(); return vwExample; } }