public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, CustomClass value)
{
Assert.IsNotNull(context);
Assert.IsNotNull(ns);
Assert.IsNotNull(feature);
Assert.IsNotNull(value);
Assert.AreEqual(5, value.X);
value.HasVisited = true;
}
public void Marshal(VowpalWabbitMarshalContext ctx, Namespace ns, Feature feature)
{
var jsonSerializer = new JsonSerializer();
using (var jsonBuilder = new VowpalWabbitJsonBuilder(ctx.VW, VowpalWabbitDefaultMarshaller.Instance, jsonSerializer))
{
// serialize from object to JSON
var sb = new StringBuilder();
using (var writer = new JsonTextWriter(new StringWriter(sb)))
{
this.jsonConverter.WriteJson(writer, this.value, jsonSerializer);
}
// marshal from JSON to VW
using (var reader = new JsonTextReader(new StringReader(sb.ToString())))
{
jsonBuilder.Parse(reader, ctx, new Namespace(ctx.VW, feature.Name));
}
}
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
public void Parse(JsonReader reader, ILabel label = null)
{
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
if (label != null)
this.defaultMarshaller.MarshalLabel(this.Context, label);
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(reader.Path, "Expected start object");
Namespace defaultNamespace = new Namespace(this.vw);
using (this.DefaultNamespaceContext.NamespaceBuilder = this.DefaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace))
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(FeatureIgnorePrefix))
{
// special fields
switch (propertyName)
{
case "_label":
// passed in label has precedence
if (label == null)
this.ParseLabel();
else
reader.Skip();
break;
//case "_shared":
// break;
//case "_multi":
// break;
default:
reader.Skip();
break;
}
}
else
{
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end");
if (reader.TokenType == JsonToken.StartObject)
this.ParseNamespaceAndFeatures(this.Context, propertyName);
else
this.ParseFeature(this.DefaultNamespaceContext, defaultNamespace, propertyName);
}
break;
case JsonToken.EndObject:
goto done;
}
}
done:
// append default namespaces features if we found some
if (this.DefaultNamespaceContext.StringExample != null && this.DefaultNamespaceContext.StringExample.Length > 0)
{
this.Context.StringExample.AppendFormat(CultureInfo.InvariantCulture,
" | {0}", this.DefaultNamespaceContext.StringExample);
}
}
}
/// <summary>
/// Triggers parsing at the current state of the <see cref="Reader"/> using the default namespace.
/// </summary>
public void Parse()
{
using (var context = new VowpalWabbitMarshalContext(this.VW, this.JsonBuilder.DefaultNamespaceContext.ExampleBuilder))
{
var ns = new Namespace(this.VW);
this.Parse(context, ns);
}
}
/// <summary>
/// Triggers parsing at the current state of the <see cref="Reader"/> using the given <paramref name="namespaceContext"/>.
/// </summary>
/// <param name="namespaceContext">The namespace the JSON should be marshalled into.</param>
/// <param name="ns">The namespace the JSON should be marshalled into.</param>
public void Parse(VowpalWabbitMarshalContext namespaceContext, Namespace ns)
{
this.JsonBuilder.Parse(this.Path, namespaceContext, ns);
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(List<VowpalWabbitJsonParseContext> path, string namespaceValue)
{
VowpalWabbitJsonParseContext localContext = null;
try
{
var ns = new Namespace(this.vw, namespaceValue);
localContext = new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder),
JsonProperty = namespaceValue
};
path.Add(localContext);
var propertyConfiguration = this.vw.Settings.PropertyConfiguration;
featureCount += this.defaultMarshaller.MarshalNamespace(localContext.Context, ns, () => this.ParseProperties(path));
path.RemoveAt(path.Count - 1);
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = localContext.Context.ToString();
if (str.Length > 0)
this.namespaceStrings.Add(str);
}
}
finally
{
if (localContext != null && localContext.Context != null)
localContext.Context.Dispose();
}
}
// re-entering from extension
internal void Parse(List<VowpalWabbitJsonParseContext> path, VowpalWabbitMarshalContext namespaceContext, Namespace ns)
{
this.featureCount = this.defaultMarshaller.MarshalNamespace(namespaceContext, ns, () => this.ParseProperties(path)) + this.featureCount;
}
public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List<VowpalWabbitJsonExtension> extensions = null)
{
this.reader = reader;
this.extensions = extensions;
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(this.reader,
string.Format("Expected start object. Found '{0}' and value '{1}'",
reader.TokenType, reader.Value));
// re-direct default namespace to the one passed
var saveDefaultNamespaceContext = this.DefaultNamespaceContext;
try
{
using (this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder))
{
VowpalWabbitJsonParseContext localContext = null;
try
{
// setup current namespace
localContext = new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder),
JsonProperty = ns.Name
};
{
this.defaultMarshaller.MarshalNamespace(
localContext.Context,
ns,
() => this.ParseProperties(new List<VowpalWabbitJsonParseContext> { localContext }));
// append string features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
context.StringExample
.Append(localContext.Context.StringExample)
.Append(string.Join(" ", this.namespaceStrings));
}
}
}
finally
{
if (localContext != null && localContext.Context != null)
{
localContext.Context.Dispose();
localContext.Context = null;
}
}
}
}
finally
{
this.DefaultNamespaceContext = saveDefaultNamespaceContext;
}
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(VowpalWabbitMarshalContext context, Namespace ns, string featureName)
{
switch (reader.TokenType)
{
case JsonToken.Float:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (double)reader.Value);
}
break;
case JsonToken.Integer:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (long)reader.Value);
}
break;
case JsonToken.String:
{
var feature = new Feature(featureName);
this.defaultMarshaller.MarshalFeatureStringEscape(context, ns, feature, (string)reader.Value);
}
break;
case JsonToken.Boolean:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (bool)reader.Value);
}
break;
case JsonToken.Comment:
case JsonToken.Null:
// probably best to ignore?
break;
case JsonToken.StartArray:
this.ParseFeatureArray(context, ns);
break;
default:
throw new VowpalWabbitJsonException(reader.Path, "Unexpected token " + reader.TokenType + " while deserializing primitive feature");
}
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
// special fields
switch (propertyName)
{
case VowpalWabbitConstants.LabelProperty:
// passed in label has precedence
if (label == null)
this.ParseLabel();
else
reader.Skip();
break;
case VowpalWabbitConstants.TextProperty:
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
break;
default:
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
reader.Skip(); // if not handled, skip it
break;
}
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
/// <param name="specialPropertyAction">Action to be executed when sepcial properties are discovered.</param>
/// <returns>The VowpalWabbit native example.</returns>
public void Parse(JsonReader reader, ILabel label = null, Func<string, bool> specialPropertyAction = null)
{
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
this.label = label;
this.specialPropertyAction = specialPropertyAction;
if (label != null)
this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, label);
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(this.reader,
string.Format("Expected start object. Found '{0}' and value '{1}'",
reader.TokenType, reader.Value));
var ns = new Namespace(this.vw);
var path = new List<LocalContext>
{
new LocalContext
{
Namespace = ns,
Context = this.DefaultNamespaceContext,
JsonProperty = string.Empty
}
};
this.defaultMarshaller.MarshalNamespace(this.DefaultNamespaceContext, ns, () => this.ParseProperties(path));
if (this.labelObject != null)
{
var propertyName = ((JProperty)this.labelObject.First).Name;
Type labelType;
if (!labelPropertyMapping.TryGetValue(propertyName, out labelType))
throw new VowpalWabbitJsonException(this.reader, "The first property ('" + propertyName + "') must match to a property of a VowpalWabbit label type.");
var labelObj = (ILabel)this.labelObject.ToObject(labelType);
if (this.foundMulti)
this.Label = labelObj;
else
this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, labelObj);
}
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue)
{
var ns = new Namespace(this.vw, namespaceValue);
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
this.defaultMarshaller.MarshalNamespace(context, ns, () =>
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName))
{
this.ParseSpecialProperty(context, ns, propertyName);
continue;
}
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace");
this.ParseFeature(context, ns, propertyName);
break;
case JsonToken.EndObject:
return;
}
}
});
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
// special fields
if (propertyName == propertyConfiguration.LabelProperty)
{
// passed in label has precedence
if (label == null)
this.ParseLabel();
else
reader.Skip();
}
else if (propertyName == propertyConfiguration.TextProperty)
{
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
}
else
{
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
reader.Skip(); // if not handled, skip it
}
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
/// <param name="specialPropertyAction">Action to be executed when sepcial properties are discovered.</param>
/// <returns>The VowpalWabbit native example.</returns>
public void Parse(JsonReader reader, ILabel label = null, Func<string, bool> specialPropertyAction = null)
{
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
this.label = label;
this.specialPropertyAction = specialPropertyAction;
if (label != null)
this.defaultMarshaller.MarshalLabel(this.Context, label);
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(reader.Path,
string.Format("Expected start object. Found '{0}' and value '{1}'",
reader.TokenType, reader.Value));
Namespace defaultNamespace = new Namespace(this.vw);
using (this.DefaultNamespaceContext.NamespaceBuilder = this.DefaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace))
{
var propertyConfiguration = this.vw.Settings.PropertyConfiguration;
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName))
this.ParseSpecialProperty(this.DefaultNamespaceContext, defaultNamespace, propertyName);
else
{
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end");
if (reader.TokenType == JsonToken.StartObject)
this.ParseNamespaceAndFeatures(this.Context, propertyName);
else
this.ParseFeature(this.DefaultNamespaceContext, defaultNamespace, propertyName);
}
break;
case JsonToken.EndObject:
goto done;
}
}
done:
// append default namespaces features if we found some
if (this.DefaultNamespaceContext.StringExample != null && this.DefaultNamespaceContext.StringExample.Length > 0)
{
this.Context.StringExample.AppendFormat(CultureInfo.InvariantCulture,
" | {0}", this.DefaultNamespaceContext.StringExample);
}
}
}
/// <summary>
/// Expects: "1,2.2,3]" (excluding the leading [)
/// </summary>
private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns)
{
ulong index = 0;
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.Integer:
MarshalFloatFeature(context, ns, index, (float)(long)reader.Value);
break;
case JsonToken.Float:
MarshalFloatFeature(context, ns, index, (float)(double)reader.Value);
break;
case JsonToken.EndArray:
return;
default:
throw new VowpalWabbitJsonException(reader.Path, "Unxpected token " + reader.TokenType + " while deserializing dense feature array");
}
index++;
}
}
private static void MarshalFloatFeature(VowpalWabbitMarshalContext context, Namespace ns, ulong index, float value)
{
context.NamespaceBuilder.AddFeature(ns.NamespaceHash + index, value);
if (context.StringExample != null)
{
context.AppendStringExample(
false,
" {0}:" + (context.VW.Settings.EnableStringFloatCompact ? "{1}" : "{1:E20}"),
index,
value);
}
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
/// <param name="extensions">Action to be executed when special properties are discovered.</param>
/// <returns>The VowpalWabbit native example.</returns>
public void Parse(JsonReader reader, ILabel label = null, List<VowpalWabbitJsonExtension> extensions = null)
{
this.featureCount = 0;
this.labelObject = null;
this.foundMulti = false;
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
this.label = label;
this.extensions = extensions;
if (label != null)
this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, label);
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(this.reader,
string.Format("Expected start object. Found '{0}' and value '{1}'",
reader.TokenType, reader.Value));
var ns = new Namespace(this.vw);
var path = new List<VowpalWabbitJsonParseContext>
{
new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = this.DefaultNamespaceContext,
JsonProperty = string.Empty
}
};
this.extensionState.Reader = reader;
this.extensionState.Path = path;
// TODO: duplicate namespace recursion to enable async
// featureCount might be modified inside ParseProperties...
this.featureCount = this.defaultMarshaller.MarshalNamespace(this.DefaultNamespaceContext, ns, () => this.ParseProperties(path)) + this.featureCount;
if (this.labelObject != null)
{
var propertyName = ((JProperty)this.labelObject.First).Name;
Type labelType;
if (!labelPropertyMapping.TryGetValue(propertyName.ToLowerInvariant(), out labelType))
throw new VowpalWabbitJsonException(this.reader, "The first property ('" + propertyName + "') must match to a property of a VowpalWabbit label type.");
var labelObj = (ILabel)this.labelObject.ToObject(labelType);
if (this.foundMulti)
this.Label = labelObj;
else
this.defaultMarshaller.MarshalLabel(this.DefaultNamespaceContext, labelObj);
}
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue)
{
var ns = new Namespace(this.vw, namespaceValue);
this.defaultMarshaller.MarshalNamespace(context, ns, () =>
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var featureName = (string)reader.Value;
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace");
this.ParseFeature(context, ns, featureName);
break;
case JsonToken.EndObject:
return;
}
}
});
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
/// <returns>The VowpalWabbit native example.</returns>
public VowpalWabbitExample Parse(JsonReader reader, ILabel label = null)
{
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
using (VowpalWabbitMarshalContext context = new VowpalWabbitMarshalContext(this.vw))
using (VowpalWabbitMarshalContext defaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder))
{
if (label != null)
this.defaultMarshaller.MarshalLabel(context, label);
if (!reader.Read() || reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(reader.Path, "Expected start object");
Namespace defaultNamespace = new Namespace(this.vw);
using (defaultNamespaceContext.NamespaceBuilder = defaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace))
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(FeatureIgnorePrefix))
{
// special fields
switch (propertyName)
{
case "_label":
// passed in label has precedence
if (label == null)
this.ParseLabel(context);
else
reader.Skip();
break;
//case "_shared":
// break;
//case "_multi":
// break;
default:
reader.Skip();
break;
}
}
else
{
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end");
if (reader.TokenType == JsonToken.StartObject)
this.ParseNamespaceAndFeatures(context, propertyName);
else
this.ParseFeature(defaultNamespaceContext, defaultNamespace, propertyName);
}
break;
}
}
// append default namespaces features if we found some
if (defaultNamespaceContext.StringExample != null && defaultNamespaceContext.StringExample.Length > 0)
{
context.StringExample.AppendFormat(CultureInfo.InvariantCulture,
"| {0}", defaultNamespaceContext.StringExample);
}
}
var vwExample = context.ExampleBuilder.CreateExample();
if (this.vw.Settings.EnableStringExampleGeneration)
vwExample.VowpalWabbitString = context.StringExample.ToString();
return vwExample;
}
}