private void WrapInNamespace(List <VowpalWabbitJsonParseContext> path, string namespaceValue, Action <VowpalWabbitJsonParseContext> action) { VowpalWabbitJsonParseContext parseContext = null; VowpalWabbitMarshalContext marshalContext = null; try { var ns = new Namespace(this.vw, namespaceValue); marshalContext = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder); parseContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = marshalContext, JsonProperty = namespaceValue }; // the namespace is only added on dispose, to be able to check if at least a single feature has been added marshalContext.NamespaceBuilder = marshalContext.ExampleBuilder.AddNamespace(ns.FeatureGroup); var position = 0; var stringExample = marshalContext.StringExample; if (marshalContext.StringExample != null) { position = stringExample.Append(ns.NamespaceString).Length; } path.Add(parseContext); action(parseContext); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = marshalContext.ToString(); if (str.Length > 0) { this.namespaceStrings.Add(str); } } this.featureCount += (int)marshalContext.NamespaceBuilder.FeatureCount; } finally { path.RemoveAt(path.Count - 1); if (marshalContext.NamespaceBuilder != null) { marshalContext.NamespaceBuilder.Dispose(); marshalContext.NamespaceBuilder = null; } if (parseContext != null && parseContext.Context != null) { parseContext.Context.Dispose(); parseContext.Context = null; } } }
/// <summary> /// Parses { "feature1":1, "feature2":true, .... } /// </summary> private void ParseNamespaceAndFeatures(List <VowpalWabbitJsonParseContext> path, string namespaceValue) { VowpalWabbitJsonParseContext localContext = null; try { var ns = new Namespace(this.vw, namespaceValue); localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder), JsonProperty = namespaceValue }; path.Add(localContext); var propertyConfiguration = this.vw.Settings.PropertyConfiguration; featureCount += this.defaultMarshaller.MarshalNamespace(localContext.Context, ns, () => this.ParseProperties(path)); path.RemoveAt(path.Count - 1); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = localContext.Context.ToString(); if (str.Length > 0) { this.namespaceStrings.Add(str); } } } finally { if (localContext != null && localContext.Context != null) { localContext.Context.Dispose(); } } }
private void ParseSpecialProperty(VowpalWabbitJsonParseContext context, string propertyName) { var propertyConfiguration = this.vw.Settings.PropertyConfiguration; // special fields if (propertyName.Equals(propertyConfiguration.LabelProperty, StringComparison.OrdinalIgnoreCase)) { // passed in label has precedence if (label == null) { this.ParseLabel(); } else { reader.Skip(); } } else if (propertyName.Equals(propertyConfiguration.TextProperty, StringComparison.OrdinalIgnoreCase)) { // parse text segment feature this.defaultMarshaller.MarshalFeatureStringSplit( context.Context, context.Namespace, new Feature(propertyName), reader.ReadAsString()); } else if (propertyName.Equals(propertyConfiguration.LabelIndexProperty, StringComparison.OrdinalIgnoreCase)) { if (!this.reader.Read()) { throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); } // skip if (this.reader.TokenType == JsonToken.Null) { return; } this.LabelIndex = (int)(long)this.reader.Value; } else if (propertyName.StartsWith(propertyConfiguration.LabelPropertyPrefix, StringComparison.OrdinalIgnoreCase)) { if (!this.reader.Read()) { throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); } // skip if (this.reader.TokenType == JsonToken.Null) { return; } if (this.labelObject == null) { this.labelObject = new JObject(); } var targetPropertyName = propertyName.Substring(propertyConfiguration.LabelPropertyPrefix.Length); this.labelObject.Add(targetPropertyName, new JValue(this.reader.Value)); } else { if (propertyName.Equals(propertyConfiguration.MultiProperty, StringComparison.Ordinal)) { this.foundMulti = true; } // forward to handler foreach (var extension in this.extensions) { if (extension(this.extensionState, propertyName)) { return; } } // if not handled, skip it reader.Skip(); } }
/// <summary> /// Parse VW JSON /// </summary> public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List <VowpalWabbitJsonExtension> extensions = null) { this.reader = reader; this.extensions = extensions; // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) { return; } if (reader.TokenType != JsonToken.StartObject) { throw new VowpalWabbitJsonException(this.reader, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); } // re-direct default namespace to the one passed var saveDefaultNamespaceContext = this.DefaultNamespaceContext; try { using (this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder)) { VowpalWabbitJsonParseContext localContext = null; try { // setup current namespace localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder), JsonProperty = ns.Name }; { this.defaultMarshaller.MarshalNamespace( localContext.Context, ns, () => this.ParseProperties(new List <VowpalWabbitJsonParseContext> { localContext })); // append string features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { context.StringExample .Append(localContext.Context.StringExample) .Append(string.Join(" ", this.namespaceStrings)); } } } finally { if (localContext != null && localContext.Context != null) { localContext.Context.Dispose(); localContext.Context = null; } } } } finally { this.DefaultNamespaceContext = saveDefaultNamespaceContext; } }
private void ParseSpecialProperty(VowpalWabbitJsonParseContext context, string propertyName) { var propertyConfiguration = this.vw.Settings.PropertyConfiguration; // special fields if (propertyName.Equals(propertyConfiguration.LabelProperty, StringComparison.OrdinalIgnoreCase)) { // passed in label has precedence if (label == null) this.ParseLabel(); else reader.Skip(); } else if (propertyName.Equals(propertyConfiguration.TextProperty, StringComparison.OrdinalIgnoreCase)) { // parse text segment feature this.defaultMarshaller.MarshalFeatureStringSplit( context.Context, context.Namespace, new Feature(propertyName), reader.ReadAsString()); } else if (propertyName.Equals(propertyConfiguration.LabelIndexProperty, StringComparison.OrdinalIgnoreCase)) { if (!this.reader.Read()) throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); // skip if (this.reader.TokenType == JsonToken.Null) return; this.LabelIndex = (int)(long)this.reader.Value; } else if (propertyName.StartsWith(propertyConfiguration.LabelPropertyPrefix, StringComparison.OrdinalIgnoreCase)) { if (!this.reader.Read()) throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); // skip if (this.reader.TokenType == JsonToken.Null) return; if (this.labelObject == null) this.labelObject = new JObject(); var targetPropertyName = propertyName.Substring(propertyConfiguration.LabelPropertyPrefix.Length); this.labelObject.Add(targetPropertyName, new JValue(this.reader.Value)); } else { if (propertyName.Equals(propertyConfiguration.MultiProperty, StringComparison.Ordinal)) this.foundMulti = true; // forward to handler foreach (var extension in this.extensions) if (extension(this.extensionState, propertyName)) return; // if not handled, skip it reader.Skip(); } }
/// <summary> /// Parses { "feature1":1, "feature2":true, .... } /// </summary> private void ParseNamespaceAndFeatures(List<VowpalWabbitJsonParseContext> path, string namespaceValue) { VowpalWabbitJsonParseContext localContext = null; try { var ns = new Namespace(this.vw, namespaceValue); localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder), JsonProperty = namespaceValue }; path.Add(localContext); var propertyConfiguration = this.vw.Settings.PropertyConfiguration; featureCount += this.defaultMarshaller.MarshalNamespace(localContext.Context, ns, () => this.ParseProperties(path)); path.RemoveAt(path.Count - 1); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = localContext.Context.ToString(); if (str.Length > 0) this.namespaceStrings.Add(str); } } finally { if (localContext != null && localContext.Context != null) localContext.Context.Dispose(); } }
public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List<VowpalWabbitJsonExtension> extensions = null) { this.reader = reader; this.extensions = extensions; // handle the case when the reader is already positioned at JsonToken.StartObject if (reader.TokenType == JsonToken.None && !reader.Read()) return; if (reader.TokenType != JsonToken.StartObject) throw new VowpalWabbitJsonException(this.reader, string.Format("Expected start object. Found '{0}' and value '{1}'", reader.TokenType, reader.Value)); // re-direct default namespace to the one passed var saveDefaultNamespaceContext = this.DefaultNamespaceContext; try { using (this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder)) { VowpalWabbitJsonParseContext localContext = null; try { // setup current namespace localContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder), JsonProperty = ns.Name }; { this.defaultMarshaller.MarshalNamespace( localContext.Context, ns, () => this.ParseProperties(new List<VowpalWabbitJsonParseContext> { localContext })); // append string features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { context.StringExample .Append(localContext.Context.StringExample) .Append(string.Join(" ", this.namespaceStrings)); } } } finally { if (localContext != null && localContext.Context != null) { localContext.Context.Dispose(); localContext.Context = null; } } } } finally { this.DefaultNamespaceContext = saveDefaultNamespaceContext; } }