private void WrapInNamespace(List <VowpalWabbitJsonParseContext> path, string namespaceValue, Action <VowpalWabbitJsonParseContext> action)
{
VowpalWabbitJsonParseContext parseContext = null;
VowpalWabbitMarshalContext marshalContext = null;
try
{
var ns = new Namespace(this.vw, namespaceValue);
marshalContext = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder);
parseContext = new VowpalWabbitJsonParseContext
{
Namespace = ns,
Context = marshalContext,
JsonProperty = namespaceValue
};
// the namespace is only added on dispose, to be able to check if at least a single feature has been added
marshalContext.NamespaceBuilder = marshalContext.ExampleBuilder.AddNamespace(ns.FeatureGroup);
var position = 0;
var stringExample = marshalContext.StringExample;
if (marshalContext.StringExample != null)
{
position = stringExample.Append(ns.NamespaceString).Length;
}
path.Add(parseContext);
action(parseContext);
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = marshalContext.ToString();
if (str.Length > 0)
{
this.namespaceStrings.Add(str);
}
}
this.featureCount += (int)marshalContext.NamespaceBuilder.FeatureCount;
}
finally
{
path.RemoveAt(path.Count - 1);
if (marshalContext.NamespaceBuilder != null)
{
marshalContext.NamespaceBuilder.Dispose();
marshalContext.NamespaceBuilder = null;
}
if (parseContext != null && parseContext.Context != null)
{
parseContext.Context.Dispose();
parseContext.Context = null;
}
}
}
/// <summary>
/// Serializes the given <paramref name="example"/> to VW string format.
/// </summary>
/// <param name="example">The example to serialize.</param>
/// <param name="label">The label to serialize.</param>
/// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param>
/// <param name="dictionary">Dictionary used for dictify operation.</param>
/// <param name="fastDictionary">Dictionary used for dictify operation.</param>
/// <returns>The resulting VW string.</returns>
public string SerializeToString(TExample example, ILabel label = null, int?index = null, Dictionary <string, string> dictionary = null, Dictionary <object, string> fastDictionary = null)
{
Contract.Requires(example != null);
using (var context = new VowpalWabbitMarshalContext(vw, dictionary, fastDictionary))
{
this.serializerFunc(context, example, label);
return(context.ToString());
}
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(List <VowpalWabbitJsonParseContext> path, string featureName)
{
switch (featureName)
{
case "$id":
{
if (this.referenceResolver == null)
{
return;
}
var id = (string)reader.Value;
if (!reader.Read() ||
reader.TokenType != JsonToken.PropertyName ||
(string)reader.Value != "$values")
{
throw new VowpalWabbitJsonException(this.reader, "Expecting '$values' property");
}
// read $values
if (!reader.Read())
{
throw new VowpalWabbitJsonException(this.reader, "Unexpected end");
}
// create re-useable marshalling call
var marshalAction = this.ParseFeatureReUsable();
// keep action for re-use
this.referenceResolver.AddReference(id, marshalAction);
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var context = path[path.Count - 2];
marshalAction.Marshal(this.defaultMarshaller, context.Context, context.Namespace, featureName);
}
return;
case "$ref":
{
if (this.referenceResolver == null || this.serializer == null)
{
return;
}
var id = (string)reader.Value;
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var ns = path[path.Count - 2].Namespace;
this.referenceResolver.Resolve(
this.serializer,
id,
marshalAction =>
{
// setup fresh context
using (var context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder))
{
this.featureCount += this.defaultMarshaller.MarshalNamespace(
context,
ns,
() => marshalAction.Marshal(this.defaultMarshaller, context, ns, featureName));
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = context.ToString();
if (str.Length > 0)
{
this.namespaceStrings.Add(str);
}
}
}
});
}
return;
}
var localContext = path.Last();
this.ParseFeature(localContext.Context, localContext.Namespace, featureName);
}
/// <summary>
/// Expects that actual feature value.
/// </summary>
private void ParseFeature(List<VowpalWabbitJsonParseContext> path, string featureName)
{
switch (featureName)
{
case "$id":
{
if (this.referenceResolver == null)
return;
var id = (string)reader.Value;
if (!reader.Read() ||
reader.TokenType != JsonToken.PropertyName ||
(string)reader.Value != "$values")
throw new VowpalWabbitJsonException(this.reader, "Expecting '$values' property");
// read $values
if (!reader.Read())
throw new VowpalWabbitJsonException(this.reader, "Unexpected end");
// create re-useable marshalling call
var marshalAction = this.ParseFeatureReUsable();
// keep action for re-use
this.referenceResolver.AddReference(id, marshalAction);
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var context = path[path.Count - 2];
marshalAction.Marshal(this.defaultMarshaller, context.Context, context.Namespace, featureName);
}
return;
case "$ref":
{
if (this.referenceResolver == null)
return;
var id = (string)reader.Value;
// go up 2 levels to find actual namespace, the last one is actually the property we want to serialize
featureName = path.Last().JsonProperty;
var ns = path[path.Count - 2].Namespace;
this.referenceResolver.Resolve(
this.serializer,
id,
marshalAction =>
{
// setup fresh context
using (var context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder))
{
this.featureCount += this.defaultMarshaller.MarshalNamespace(
context,
ns,
() => marshalAction.Marshal(this.defaultMarshaller, context, ns, featureName));
// append default namespaces features if we found some
if (this.vw.Settings.EnableStringExampleGeneration)
{
var str = context.ToString();
if (str.Length > 0)
this.namespaceStrings.Add(str);
}
}
});
}
return;
}
var localContext = path.Last();
this.ParseFeature(localContext.Context, localContext.Namespace, featureName);
}
/// <summary>
/// Serialize the example.
/// </summary>
/// <param name="example">The example to serialize.</param>
/// <param name="label">The label to be serialized.</param>
/// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param>
/// <returns>The serialized example.</returns>
/// <remarks>If <typeparamref name="TExample"/> is annotated using the Cachable attribute, examples are returned from cache.</remarks>
public VowpalWabbitExample Serialize(TExample example, ILabel label = null, int?index = null)
{
Contract.Requires(example != null);
Contract.Requires(index == null);
if (this.exampleCache == null || label != null)
{
using (var context = new VowpalWabbitMarshalContext(vw))
{
this.serializerFunc(context, example, label);
var vwExample = context.ExampleBuilder.CreateExample();
if (this.EnableStringExampleGeneration)
{
vwExample.VowpalWabbitString = context.ToString();
}
return(vwExample);
}
}
CacheEntry result;
if (this.exampleCache.TryGetValue(example, out result))
{
result.LastRecentUse = DateTime.UtcNow;
#if DEBUG
if (result.InUse)
{
throw new ArgumentException("Cached example already in use.");
}
#endif
}
else
{
VowpalWabbitExample nativeExample = null;
try
{
using (var context = new VowpalWabbitMarshalContext(this))
{
this.serializerFunc(context, example, label);
nativeExample = context.ExampleBuilder.CreateExample();
}
result = new CacheEntry
{
Example = nativeExample,
LastRecentUse = DateTime.UtcNow
};
this.exampleCache.Add(example, result);
#if DEBUG
this.reverseLookup.Add(result.Example, result);
#endif
}
catch (Exception e)
{
if (nativeExample != null)
{
nativeExample.Dispose();
}
throw e;
}
}
#if DEBUG
result.InUse = true;
#endif
// TODO: support Label != null here and update cached example using new label
return(result.Example);
}
internal async Task TestAllReduceInternal()
{
var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList();
var stringSerializerCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfShared>)
VowpalWabbitSerializerFactory.CreateSerializer<CbAdfShared>(new VowpalWabbitSettings { EnableStringExampleGeneration = true });
var stringSerializerAdfCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfAction>)
VowpalWabbitSerializerFactory.CreateSerializer<CbAdfAction>(new VowpalWabbitSettings { EnableStringExampleGeneration = true });
var stringData = new List<List<string>>();
VowpalWabbitPerformanceStatistics statsExpected;
using (var spanningTree = new SpanningTreeClr())
{
spanningTree.Start();
using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true }))
using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true } ))
{
var stringSerializer = stringSerializerCompiler.Func(vw1);
var stringSerializerAdf = stringSerializerAdfCompiler.Func(vw1);
// serialize
foreach (var d in data)
{
var block = new List<string>();
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializer(context, d.Item1, SharedLabel.Instance);
block.Add(context.ToString());
}
block.AddRange(d.Item2.Select((a, i) =>
{
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null);
return context.ToString();
}
}));
stringData.Add(block);
}
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500))));
vw1.SaveModel("expected.1.model");
vw2.SaveModel("expected.2.model");
statsExpected = vw1.PerformanceStatistics;
}
}
// skip header
var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList();
var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList();
var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy")
{
ParallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = 2
},
ExampleCountPerRun = 2000,
ExampleDistribution = VowpalWabbitExampleDistribution.RoundRobin
};
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
var vwManaged = vw.Create<CbAdfShared, CbAdfAction>();
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))),
Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.managed.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
}