/// <summary>
/// Marshals JSON string into VW example.
/// </summary>
public void Marshal(VowpalWabbitMarshalContext ctx, Namespace ns, Feature feature)
{
if (this.value == null)
{
return;
}
var jsonSerializer = new JsonSerializer();
using (var jsonBuilder = new VowpalWabbitJsonBuilder(ctx.VW, VowpalWabbitDefaultMarshaller.Instance, jsonSerializer))
{
// serialize from object to JSON
var sb = new StringBuilder();
using (var writer = new JsonTextWriter(new StringWriter(sb)))
{
this.jsonConverter.WriteJson(writer, this.value, jsonSerializer);
}
// marshal from JSON to VW
using (var reader = new JsonTextReader(new StringReader(sb.ToString())))
{
jsonBuilder.Parse(reader, ctx, new Namespace(ctx.VW, feature.Name));
}
}
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue)
{
var ns = new Namespace(this.vw, namespaceValue);
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
this.defaultMarshaller.MarshalNamespace(context, ns, () =>
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName))
{
this.ParseSpecialProperty(context, ns, propertyName);
continue;
}
if (!reader.Read())
{
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace");
}
this.ParseFeature(context, ns, propertyName);
break;
case JsonToken.EndObject:
return;
}
}
});
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
// special fields
switch (propertyName)
{
case VowpalWabbitConstants.LabelProperty:
// passed in label has precedence
if (label == null)
{
this.ParseLabel();
}
else
{
reader.Skip();
}
break;
case VowpalWabbitConstants.TextProperty:
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
break;
default:
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
{
reader.Skip(); // if not handled, skip it
}
break;
}
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
// special fields
if (propertyName == propertyConfiguration.LabelProperty)
{
// passed in label has precedence
if (label == null)
{
this.ParseLabel();
}
else
{
reader.Skip();
}
}
else if (propertyName == propertyConfiguration.TextProperty)
{
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
}
else
{
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
{
reader.Skip(); // if not handled, skip it
}
}
}
private void WrapInNamespace(List <VowpalWabbitJsonParseContext> path, string namespaceValue, Action <VowpalWabbitJsonParseContext> action)
{
VowpalWabbitJsonParseContext parseContext = null;
VowpalWabbitMarshalContext marshalContext = null;
try
{
var ns = new Namespace(this.vw, namespaceValue);
marshalContext = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder);
parseContext = new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = marshalContext,
JsonProperty = namespaceValue
};
// the namespace is only added on dispose, to be able to check if at least a single feature has been added
marshalContext.NamespaceBuilder = marshalContext.ExampleBuilder.AddNamespace(ns.FeatureGroup);
var position = 0;
var stringExample = marshalContext.StringExample;
if (marshalContext.StringExample != null)
{
position = stringExample.Append(ns.NamespaceString).Length;
}
path.Add(parseContext);
action(parseContext);
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = marshalContext.ToString();
if (str.Length > 0)
{
this.namespaceStrings.Add(str);
}
}
this.featureCount += (int)marshalContext.NamespaceBuilder.FeatureCount;
}
finally
{
path.RemoveAt(path.Count - 1);
if (marshalContext.NamespaceBuilder != null)
{
marshalContext.NamespaceBuilder.Dispose();
marshalContext.NamespaceBuilder = null;
}
if (parseContext != null && parseContext.Context != null)
{
parseContext.Context.Dispose();
parseContext.Context = null;
}
}
}
/// <summary>
/// Marshals the supplied string into VW native space, by splitting the word by white space.
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
/// <example><paramref name="value"/> is "New York". Result is "New York", corresponding to 2 featuers in VW native space.</example>
public void MarshalFeatureStringSplit(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var words = value.Split((char[])null, StringSplitOptions.RemoveEmptyEntries);
foreach (var s in words)
{
var featureHash = context.VW.HashFeature(s, ns.NamespaceHash);
context.NamespaceBuilder.AddFeature(featureHash, 1f);
}
if (context.StringExample == null)
{
return;
}
foreach (var s in words)
{
context.AppendStringExample(feature.Dictify, " {0}", s);
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, IEnumerable <string> value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
if (value == null)
{
return;
}
foreach (var item in value)
{
context.NamespaceBuilder.AddFeature(context.VW.HashFeature(item.Replace(' ', '_'), ns.NamespaceHash), 1f);
}
if (context.StringExample == null)
{
return;
}
foreach (var item in value)
{
context.AppendStringExample(feature.Dictify, " {0}", item);
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="featureVisits"></param>
public void MarshalNamespace(VowpalWabbitMarshalContext context, Namespace ns, Action featureVisits)
{
try
{
// the namespace is only added on dispose, to be able to check if at least a single feature has been added
context.NamespaceBuilder = context.ExampleBuilder.AddNamespace(ns.FeatureGroup);
var position = 0;
var stringExample = context.StringExample;
if (context.StringExample != null)
{
position = stringExample.Append(ns.NamespaceString).Length;
}
featureVisits();
if (context.StringExample != null)
{
if (position == stringExample.Length)
{
// no features added, remove namespace
stringExample.Length = position - ns.NamespaceString.Length;
}
}
}
finally
{
if (context.NamespaceBuilder != null)
{
context.NamespaceBuilder.Dispose();
context.NamespaceBuilder = null;
}
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, IDictionary value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
if (value == null)
{
return;
}
foreach (DictionaryEntry item in value)
{
context.NamespaceBuilder.AddFeature(
context.VW.HashFeature(Convert.ToString(item.Key), ns.NamespaceHash),
Convert.ToSingle(item.Value, CultureInfo.InvariantCulture));
}
if (context.StringExample == null)
{
return;
}
foreach (DictionaryEntry item in value)
{
context.AppendStringExample(
feature.Dictify,
" {0}:" + (context.VW.Settings.EnableStringFloatCompact ? "{1}" : "{1:E20}"),
Convert.ToString(item.Key),
Convert.ToSingle(item.Value, CultureInfo.InvariantCulture));
}
}
/// <summary>
/// Marshals JSON string into VW example.
/// </summary>
public void Marshal(VowpalWabbitMarshalContext ctx, Namespace ns, Feature feature)
{
if (this.value == null)
{
return;
}
try
{
var jsonSerializer = new JsonSerializer();
using (var jsonBuilder = new VowpalWabbitJsonBuilder(ctx.VW, VowpalWabbitDefaultMarshaller.Instance, jsonSerializer))
{
// marshal from JSON to VW
foreach (var json in jsonConverter.JsonFragments(this.value))
{
if (json == null)
{
continue;
}
using (var reader = new JsonTextReader(new StringReader(json)))
{
jsonBuilder.Parse(reader, ctx, new Namespace(ctx.VW, feature.Name));
}
}
}
}
catch (Exception e)
{
throw new VowpalWabbitSerializationException("Optimized marshalling failed", e, ns, feature);
}
}
/// <summary>
/// Expects: "1,2.2,3]" (excluding the leading [)
/// </summary>
private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns)
{
ulong index = 0;
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.Integer:
MarshalFloatFeature(context, ns, index, (float)(long)reader.Value);
break;
case JsonToken.Float:
MarshalFloatFeature(context, ns, index, (float)(double)reader.Value);
break;
case JsonToken.EndArray:
return;
default:
throw new VowpalWabbitJsonException(this.reader, "Unxpected token " + reader.TokenType + " while deserializing dense feature array");
}
index++;
}
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(VowpalWabbitMarshalContext context, Namespace ns, string featureName)
{
switch (reader.TokenType)
{
case JsonToken.Float:
VowpalWabbitMarshalActions.Marshal(this.defaultMarshaller, context, ns, featureName, (double)reader.Value);
break;
case JsonToken.Integer:
VowpalWabbitMarshalActions.Marshal(this.defaultMarshaller, context, ns, featureName, (long)reader.Value);
break;
case JsonToken.String:
VowpalWabbitMarshalActions.Marshal(this.defaultMarshaller, context, ns, featureName, (string)reader.Value);
break;
case JsonToken.Boolean:
VowpalWabbitMarshalActions.Marshal(this.defaultMarshaller, context, ns, featureName, (bool)reader.Value);
break;
case JsonToken.Comment:
case JsonToken.Null:
// probably best to ignore?
break;
case JsonToken.StartArray:
this.ParseFeatureArray(context, ns);
break;
default:
throw new VowpalWabbitJsonException(this.reader, "Unexpected token " + reader.TokenType + " while deserializing primitive feature");
}
}
/// <summary>
/// Creates the managed example representation.
/// </summary>
/// <returns>Returns the managed example.</returns>
public VowpalWabbitExample CreateExample()
{
try
{
var vwExample = this.DefaultNamespaceContext.ExampleBuilder.CreateExample();
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = this.DefaultNamespaceContext.ToString();
if (str.Length > 0)
{
this.namespaceStrings.Insert(0, str);
}
vwExample.VowpalWabbitString = string.Join(" ", this.namespaceStrings);
}
return(vwExample);
}
finally
{
// useful for tracking down bugs
// this.DefaultNamespaceContextStackTrace = "Create Example" + Environment.StackTrace;
this.DefaultNamespaceContext.Dispose();
this.DefaultNamespaceContext = null;
}
}
/// <summary>
/// Triggers parsing at the current state of the <see cref="Reader"/> using the default namespace.
/// </summary>
public void Parse()
{
using (var context = new VowpalWabbitMarshalContext(this.VW, this.JsonBuilder.DefaultNamespaceContext.ExampleBuilder))
{
var ns = new Namespace(this.VW);
this.Parse(context, ns);
}
}
/// <summary>
/// Serializes the given <paramref name="example"/> to VW string format.
/// </summary>
/// <param name="example">The example to serialize.</param>
/// <param name="label">The label to serialize.</param>
/// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param>
/// <param name="dictionary">Dictionary used for dictify operation.</param>
/// <param name="fastDictionary">Dictionary used for dictify operation.</param>
/// <returns>The resulting VW string.</returns>
public string SerializeToString(TExample example, ILabel label = null, int?index = null, Dictionary <string, string> dictionary = null, Dictionary <object, string> fastDictionary = null)
{
Contract.Requires(example != null);
using (var context = new VowpalWabbitMarshalContext(vw, dictionary, fastDictionary))
{
this.serializerFunc(context, example, label);
return(context.ToString());
}
}
/// <summary>
/// Marshals an enum value into native VW.
/// </summary>
/// <typeparam name="T">The enum type.</typeparam>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
/// <example>Gender = Male yields "GenderMale" in VW native string format.</example>
public void MarshalEnumFeature <T>(VowpalWabbitMarshalContext context, Namespace ns, EnumerizedFeature <T> feature, T value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
context.NamespaceBuilder.AddFeature(feature.FeatureHash(value), 1f);
context.AppendStringExample(feature.Dictify, " {0}{1}", feature.Name, value);
}
public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, CustomClass value)
{
Assert.IsNotNull(context);
Assert.IsNotNull(ns);
Assert.IsNotNull(feature);
Assert.IsNotNull(value);
Assert.AreEqual(5, value.X);
value.HasVisited = true;
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="label"></param>
public void MarshalLabel(VowpalWabbitMarshalContext context, string label)
{
if (label == null)
{
return;
}
context.ExampleBuilder.ApplyLabel(new StringLabel(label));
context.StringLabel = label;
}
/// <summary>
/// Marshals any type into native VW, by constructing a 1-hot encoding using <see cref="object.ToString()"/>.
/// </summary>
/// <typeparam name="T">The type to be enumerized.</typeparam>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
/// <example><typeparamref name="T"/> is <see cref="System.Int32"/>, actual value '25' and <see cref="Feature.Name"/> is 'Age'.
/// The result is equivalent to 'Age25'.
/// </example>
public void MarshalEnumerizeFeature <T>(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, T value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
var stringValue = feature.Name + value.ToString();
context.NamespaceBuilder.AddFeature(context.VW.HashFeature(stringValue, ns.NamespaceHash), 1f);
context.AppendStringExample(feature.Dictify, " {0}", stringValue);
}
/// <summary>
/// Marshals the given value <paramref name="value"/> into native VW by
///
/// <list type="number">
/// <item><description>string concatenating the feature name and the value.</description></item>
/// <item><description>Hashing of the resulting string.</description></item>
/// </list>
///
/// e.g. feature name = "Age", value = 25 yields "Age25:1" in VW native string format.
/// </summary>
/// <typeparam name="T">The value type.</typeparam>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
/// <remarks>This is a fallback method if no other types match.</remarks>
public void MarshalFeature <T>(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, T value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
var featureString = feature.Name + Convert.ToString(value);
var featureHash = context.VW.HashFeature(featureString, ns.NamespaceHash);
context.NamespaceBuilder.AddFeature(featureHash, 1f);
context.AppendStringExample(feature.Dictify, " {0}", featureString);
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (this.DefaultNamespaceContext != null)
{
// useful for tracking down bugs
// this.DefaultNamespaceContextStackTrace = "Dispose" + Environment.StackTrace;
this.DefaultNamespaceContext.Dispose();
this.DefaultNamespaceContext = null;
}
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, IVowpalWabbitSerializable value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
if (value == null)
{
return;
}
value.Marshal(context, ns, feature);
}
private static void MarshalFloatFeature(VowpalWabbitMarshalContext context, Namespace ns, ulong index, float value)
{
context.NamespaceBuilder.AddFeature(ns.NamespaceHash + index, value);
if (context.StringExample != null)
{
context.AppendStringExample(
false,
" {0}:" + (context.VW.Settings.EnableStringFloatCompact ? "{1}" : "{1:E20}"),
index,
value);
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="label"></param>
public void MarshalLabel(VowpalWabbitMarshalContext context, ILabel label)
{
if (label == null)
{
return;
}
var labelString = label.ToVowpalWabbitFormat();
context.ExampleBuilder.ParseLabel(labelString);
// prefix with label
context.AppendStringExample(false, "{0}", labelString);
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="label"></param>
public void MarshalLabel(VowpalWabbitMarshalContext context, string label)
{
if (label == null)
{
return;
}
context.ExampleBuilder.ParseLabel(label);
// prefix with label
if (context.StringExample != null)
{
context.StringExample.Insert(0, label);
}
}
/// <summary>
/// Marshals a boolean value into native VW.
///
/// e.g. loggedIn = true yields "loggedIn" in VW native string format.
/// e.g. loggedIn = false yields an empty string.
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, PreHashedFeature feature, bool value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
if (!value)
{
return;
}
context.NamespaceBuilder.AddFeature(feature.FeatureHash, 1f);
context.AppendStringExample(feature.Dictify, " {0}", feature.Name);
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="label"></param>
public void MarshalLabel(VowpalWabbitMarshalContext context, ILabel label)
{
if (label == null)
{
return;
}
context.ExampleBuilder.ApplyLabel(label);
// prefix with label
if (context.StringExample != null)
{
context.StringLabel = label.ToString();
}
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (this.DefaultNamespaceContext != null)
{
this.DefaultNamespaceContext.Dispose();
this.DefaultNamespaceContext = null;
}
if (this.Context != null)
{
this.Context.Dispose();
this.Context = null;
}
}
}
/// <summary>
///
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="label"></param>
public void MarshalLabel(VowpalWabbitMarshalContext context, ILabel label)
{
if (label == null)
{
return;
}
var labelString = label.ToVowpalWabbitFormat();
context.ExampleBuilder.ParseLabel(labelString);
// prefix with label
if (context.StringExample != null)
{
context.StringExample.Insert(0, labelString);
}
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(VowpalWabbitMarshalContext context, Namespace ns, string featureName)
{
switch (reader.TokenType)
{
case JsonToken.Float:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (double)reader.Value);
}
break;
case JsonToken.Integer:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (long)reader.Value);
}
break;
case JsonToken.String:
{
var feature = new Feature(featureName);
this.defaultMarshaller.MarshalFeatureStringEscapeAndIncludeName(context, ns, feature, (string)reader.Value);
}
break;
case JsonToken.Boolean:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (bool)reader.Value);
}
break;
case JsonToken.Comment:
case JsonToken.Null:
// probably best to ignore?
break;
case JsonToken.StartArray:
this.ParseFeatureArray(context, ns);
break;
default:
throw new VowpalWabbitJsonException(reader.Path, "Unexpected token " + reader.TokenType + " while deserializing primitive feature");
}
}
public void Marshal(VowpalWabbitMarshalContext ctx, Namespace ns, Feature feature)
{
var jsonSerializer = new JsonSerializer();
using (var jsonBuilder = new VowpalWabbitJsonBuilder(ctx.VW, VowpalWabbitDefaultMarshaller.Instance, jsonSerializer))
{
// serialize from object to JSON
var sb = new StringBuilder();
using (var writer = new JsonTextWriter(new StringWriter(sb)))
{
this.jsonConverter.WriteJson(writer, this.value, jsonSerializer);
}
// marshal from JSON to VW
using (var reader = new JsonTextReader(new StringReader(sb.ToString())))
{
jsonBuilder.Parse(reader, ctx, new Namespace(ctx.VW, feature.Name));
}
}
}
/// <summary>
/// Marshals the supplied string into VW native space. Spaces are escaped using '_'. Includes the <see cref="Feature.Name"/> in the 1-hot encoded feature.
/// </summary>
/// <param name="context">The marshalling context.</param>
/// <param name="ns">The namespace description.</param>
/// <param name="feature">The feature description.</param>
/// <param name="value">The actual feature value.</param>
/// <example><paramref name="value"/> is "New York". <paramref name="feature"/> Name is "Location". Result is "LocationNew_York".</example>
public void MarshalFeatureStringEscapeAndIncludeName(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, string value)
{
Contract.Requires(context != null);
Contract.Requires(ns != null);
Contract.Requires(feature != null);
if (string.IsNullOrWhiteSpace(value))
{
return;
}
// safe escape spaces
value = feature.Name + value.Replace(' ', '_');
var featureHash = context.VW.HashFeature(value, ns.NamespaceHash);
context.NamespaceBuilder.AddFeature(featureHash, 1f);
context.AppendStringExample(feature.Dictify, " {0}", value);
}
private void ParseLabel(VowpalWabbitMarshalContext context)
{
// peak the first property name
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end");
switch (reader.TokenType)
{
case JsonToken.StartObject:
{
// parse complex object
if (!reader.Read() || reader.TokenType != JsonToken.PropertyName)
throw new VowpalWabbitJsonException(reader.Path, "Expected at least a single property to determine the label object");
var propertyName = (string)reader.Value;
var prefixReader = new PrefixedJsonReader(this.reader,
Tuple.Create(JsonToken.StartObject, (object)null),
Tuple.Create(JsonToken.PropertyName, (object)propertyName));
Type labelType;
if (!labelPropertyMapping.TryGetValue(propertyName, out labelType))
throw new VowpalWabbitJsonException(reader.Path, "The first property ('" + propertyName + "') must match to a property of a VowpalWabbit label type.");
var label = (ILabel)jsonSerializer.Deserialize(prefixReader, labelType);
this.defaultMarshaller.MarshalLabel(context, label);
}
break;
case JsonToken.Integer:
case JsonToken.Float:
case JsonToken.String:
{
// pass label directly to VW
var labelString = reader.Value.ToString();
context.ExampleBuilder.ParseLabel(labelString);
// prefix with label
context.AppendStringExample(false, "{0}", labelString);
}
break;
default:
throw new VowpalWabbitJsonException(reader.Path, "Expected label object");
}
}
/// <summary>
/// Parses the example.
/// </summary>
/// <param name="reader">The example to parse.</param>
/// <param name="label">
/// Optional label, taking precedence over "_label" property found in <paramref name="reader"/>.
/// If null, <paramref name="reader"/> will be inspected and the "_label" property used as label.
/// </param>
/// <returns>The VowpalWabbit native example.</returns>
public VowpalWabbitExample Parse(JsonReader reader, ILabel label = null)
{
// avoid parameter passing for the sake of non-reentrantness
this.reader = reader;
using (VowpalWabbitMarshalContext context = new VowpalWabbitMarshalContext(this.vw))
using (VowpalWabbitMarshalContext defaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder))
{
if (label != null)
this.defaultMarshaller.MarshalLabel(context, label);
if (!reader.Read() || reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(reader.Path, "Expected start object");
Namespace defaultNamespace = new Namespace(this.vw);
using (defaultNamespaceContext.NamespaceBuilder = defaultNamespaceContext.ExampleBuilder.AddNamespace(VowpalWabbitConstants.DefaultNamespace))
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(FeatureIgnorePrefix))
{
// special fields
switch (propertyName)
{
case "_label":
// passed in label has precedence
if (label == null)
this.ParseLabel(context);
else
reader.Skip();
break;
//case "_shared":
// break;
//case "_multi":
// break;
default:
reader.Skip();
break;
}
}
else
{
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end");
if (reader.TokenType == JsonToken.StartObject)
this.ParseNamespaceAndFeatures(context, propertyName);
else
this.ParseFeature(defaultNamespaceContext, defaultNamespace, propertyName);
}
break;
}
}
// append default namespaces features if we found some
if (defaultNamespaceContext.StringExample != null && defaultNamespaceContext.StringExample.Length > 0)
{
context.StringExample.AppendFormat(CultureInfo.InvariantCulture,
"| {0}", defaultNamespaceContext.StringExample);
}
}
var vwExample = context.ExampleBuilder.CreateExample();
if (this.vw.Settings.EnableStringExampleGeneration)
vwExample.VowpalWabbitString = context.StringExample.ToString();
return vwExample;
}
}
internal async Task TestAllReduceInternal()
{
var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList();
var stringSerializerCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfShared>)
VowpalWabbitSerializerFactory.CreateSerializer<CbAdfShared>(new VowpalWabbitSettings { EnableStringExampleGeneration = true });
var stringSerializerAdfCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfAction>)
VowpalWabbitSerializerFactory.CreateSerializer<CbAdfAction>(new VowpalWabbitSettings { EnableStringExampleGeneration = true });
var stringData = new List<List<string>>();
VowpalWabbitPerformanceStatistics statsExpected;
using (var spanningTree = new SpanningTreeClr())
{
spanningTree.Start();
using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true }))
using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true } ))
{
var stringSerializer = stringSerializerCompiler.Func(vw1);
var stringSerializerAdf = stringSerializerAdfCompiler.Func(vw1);
// serialize
foreach (var d in data)
{
var block = new List<string>();
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializer(context, d.Item1, SharedLabel.Instance);
block.Add(context.ToString());
}
block.AddRange(d.Item2.Select((a, i) =>
{
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null);
return context.ToString();
}
}));
stringData.Add(block);
}
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500))));
vw1.SaveModel("expected.1.model");
vw2.SaveModel("expected.2.model");
statsExpected = vw1.PerformanceStatistics;
}
}
// skip header
var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList();
var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList();
var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy")
{
ParallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = 2
},
ExampleCountPerRun = 2000,
ExampleDistribution = VowpalWabbitExampleDistribution.RoundRobin
};
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
var vwManaged = vw.Create<CbAdfShared, CbAdfAction>();
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))),
Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.managed.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
}
/// <summary>
/// Triggers parsing at the current state of the <see cref="Reader"/> using the default namespace.
/// </summary>
public void Parse()
{
using (var context = new VowpalWabbitMarshalContext(this.VW, this.JsonBuilder.DefaultNamespaceContext.ExampleBuilder))
{
var ns = new Namespace(this.VW);
this.Parse(context, ns);
}
}
/// <summary>
/// Triggers parsing at the current state of the <see cref="Reader"/> using the given <paramref name="namespaceContext"/>.
/// </summary>
/// <param name="namespaceContext">The namespace the JSON should be marshalled into.</param>
/// <param name="ns">The namespace the JSON should be marshalled into.</param>
public void Parse(VowpalWabbitMarshalContext namespaceContext, Namespace ns)
{
this.JsonBuilder.Parse(this.Path, namespaceContext, ns);
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(List<VowpalWabbitJsonParseContext> path, string featureName)
{
switch (featureName)
{
case "$id":
{
if (this.referenceResolver == null)
return;
var id = (string)reader.Value;
if (!reader.Read() ||
reader.TokenType != JsonToken.PropertyName ||
(string)reader.Value != "$values")
throw new VowpalWabbitJsonException(this.reader, "Expecting '$values' property");
// read $values
if (!reader.Read())
throw new VowpalWabbitJsonException(this.reader, "Unexpected end");
// create re-useable marshalling call
var marshalAction = this.ParseFeatureReUsable();
// keep action for re-use
this.referenceResolver.AddReference(id, marshalAction);
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var context = path[path.Count - 2];
marshalAction.Marshal(this.defaultMarshaller, context.Context, context.Namespace, featureName);
}
return;
case "$ref":
{
if (this.referenceResolver == null)
return;
var id = (string)reader.Value;
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var ns = path[path.Count - 2].Namespace;
this.referenceResolver.Resolve(
this.serializer,
id,
marshalAction =>
{
// setup fresh context
using (var context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder))
{
this.featureCount += this.defaultMarshaller.MarshalNamespace(
context,
ns,
() => marshalAction.Marshal(this.defaultMarshaller, context, ns, featureName));
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = context.ToString();
if (str.Length > 0)
this.namespaceStrings.Add(str);
}
}
});
}
return;
}
var localContext = path.Last();
this.ParseFeature(localContext.Context, localContext.Namespace, featureName);
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(VowpalWabbitMarshalContext context, Namespace ns, string featureName)
{
switch (reader.TokenType)
{
case JsonToken.Float:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (double)reader.Value);
}
break;
case JsonToken.Integer:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (long)reader.Value);
}
break;
case JsonToken.String:
{
var feature = new Feature(featureName);
this.defaultMarshaller.MarshalFeatureStringEscape(context, ns, feature, (string)reader.Value);
}
break;
case JsonToken.Boolean:
{
var feature = new PreHashedFeature(this.vw, ns, featureName);
this.defaultMarshaller.MarshalFeature(context, ns, feature, (bool)reader.Value);
}
break;
case JsonToken.Comment:
case JsonToken.Null:
// probably best to ignore?
break;
case JsonToken.StartArray:
this.ParseFeatureArray(context, ns);
break;
default:
throw new VowpalWabbitJsonException(reader.Path, "Unexpected token " + reader.TokenType + " while deserializing primitive feature");
}
}
/// <summary>
/// Expects: "1,2.2,3]" (excluding the leading [)
/// </summary>
private void ParseFeatureArray(VowpalWabbitMarshalContext context, Namespace ns)
{
ulong index = 0;
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.Integer:
MarshalFloatFeature(context, ns, index, (float)(long)reader.Value);
break;
case JsonToken.Float:
MarshalFloatFeature(context, ns, index, (float)(double)reader.Value);
break;
case JsonToken.EndArray:
return;
default:
throw new VowpalWabbitJsonException(reader.Path, "Unxpected token " + reader.TokenType + " while deserializing dense feature array");
}
index++;
}
}
/// <summary>
/// Creates the managed example representation.
/// </summary>
/// <returns>Returns the managed example.</returns>
public VowpalWabbitExample CreateExample()
{
try
{
var vwExample = this.Context.ExampleBuilder.CreateExample();
if (this.vw.Settings.EnableStringExampleGeneration)
vwExample.VowpalWabbitString = this.Context.StringExample.ToString();
return vwExample;
}
finally
{
this.DefaultNamespaceContext.Dispose();
this.DefaultNamespaceContext = null;
this.Context.Dispose();
this.Context = null;
}
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
// special fields
switch (propertyName)
{
case VowpalWabbitConstants.LabelProperty:
// passed in label has precedence
if (label == null)
this.ParseLabel();
else
reader.Skip();
break;
case VowpalWabbitConstants.TextProperty:
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
break;
default:
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
reader.Skip(); // if not handled, skip it
break;
}
}
// re-entering from extension
internal void Parse(List<VowpalWabbitJsonParseContext> path, VowpalWabbitMarshalContext namespaceContext, Namespace ns)
{
this.featureCount = this.defaultMarshaller.MarshalNamespace(namespaceContext, ns, () => this.ParseProperties(path)) + this.featureCount;
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (this.DefaultNamespaceContext != null)
{
this.DefaultNamespaceContext.Dispose();
this.DefaultNamespaceContext = null;
}
if (this.Context != null)
{
this.Context.Dispose();
this.Context = null;
}
}
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue)
{
var ns = new Namespace(this.vw, namespaceValue);
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
this.defaultMarshaller.MarshalNamespace(context, ns, () =>
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var propertyName = (string)reader.Value;
if (propertyName.StartsWith(propertyConfiguration.FeatureIgnorePrefix) || propertyConfiguration.IsSpecialProperty(propertyName))
{
this.ParseSpecialProperty(context, ns, propertyName);
continue;
}
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace");
this.ParseFeature(context, ns, propertyName);
break;
case JsonToken.EndObject:
return;
}
}
});
}
private static void MarshalFloatFeature(VowpalWabbitMarshalContext context, Namespace ns, ulong index, float value)
{
context.NamespaceBuilder.AddFeature(ns.NamespaceHash + index, value);
if (context.StringExample != null)
{
context.AppendStringExample(
false,
" {0}:" + (context.VW.Settings.EnableStringFloatCompact ? "{1}" : "{1:E20}"),
index,
value);
}
}
private void ParseSpecialProperty(VowpalWabbitMarshalContext context, Namespace ns, string propertyName)
{
var propertyConfiguration = context.VW.Settings.PropertyConfiguration;
// special fields
if (propertyName == propertyConfiguration.LabelProperty)
{
// passed in label has precedence
if (label == null)
this.ParseLabel();
else
reader.Skip();
}
else if (propertyName == propertyConfiguration.TextProperty)
{
// parse text segment feature
this.defaultMarshaller.MarshalFeatureStringSplit(
context,
ns,
new Feature(propertyName),
reader.ReadAsString());
}
else
{
// forward to handler
if (specialPropertyAction == null || !specialPropertyAction(propertyName))
reader.Skip(); // if not handled, skip it
}
}
/// <summary>
/// Parses { "feature1":1, "feature2":true, .... }
/// </summary>
private void ParseNamespaceAndFeatures(VowpalWabbitMarshalContext context, string namespaceValue)
{
var ns = new Namespace(this.vw, namespaceValue);
this.defaultMarshaller.MarshalNamespace(context, ns, () =>
{
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.PropertyName:
var featureName = (string)reader.Value;
if (!reader.Read())
throw new VowpalWabbitJsonException(reader.Path, "Unexpected end while parsing namespace");
this.ParseFeature(context, ns, featureName);
break;
case JsonToken.EndObject:
return;
}
}
});
}
public void Parse(JsonReader reader, VowpalWabbitMarshalContext context, Namespace ns, List<VowpalWabbitJsonExtension> extensions = null)
{
this.reader = reader;
this.extensions = extensions;
// handle the case when the reader is already positioned at JsonToken.StartObject
if (reader.TokenType == JsonToken.None && !reader.Read())
return;
if (reader.TokenType != JsonToken.StartObject)
throw new VowpalWabbitJsonException(this.reader,
string.Format("Expected start object. Found '{0}' and value '{1}'",
reader.TokenType, reader.Value));
// re-direct default namespace to the one passed
var saveDefaultNamespaceContext = this.DefaultNamespaceContext;
try
{
using (this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder))
{
VowpalWabbitJsonParseContext localContext = null;
try
{
// setup current namespace
localContext = new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = new VowpalWabbitMarshalContext(this.vw, context.ExampleBuilder),
JsonProperty = ns.Name
};
{
this.defaultMarshaller.MarshalNamespace(
localContext.Context,
ns,
() => this.ParseProperties(new List<VowpalWabbitJsonParseContext> { localContext }));
// append string features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
context.StringExample
.Append(localContext.Context.StringExample)
.Append(string.Join(" ", this.namespaceStrings));
}
}
}
finally
{
if (localContext != null && localContext.Context != null)
{
localContext.Context.Dispose();
localContext.Context = null;
}
}
}
}
finally
{
this.DefaultNamespaceContext = saveDefaultNamespaceContext;
}
}