public VowpalWabbit(VowpalWabbitModel model, VowpalWabbitSerializerSettings settings = null)
: base(model)
{
var visitor = new VowpalWabbitInterfaceVisitor(this);
this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings);
}
public void Visit <T>(IFeature <T> feature)
{
Contract.Requires(feature != null);
// can't specify constraints to narrow for enums
var valueType = typeof(T);
if (valueType.IsEnum)
{
this.builder.AppendFormat(
CultureInfo.InvariantCulture,
" {0}{1}",
feature.Name,
Enum.GetName(valueType, feature.Value));
}
else if (VowpalWabbitSerializerFactory.IsValidDenseFeatureValueElementType(typeof(T)))
{
this.builder.AppendFormat(
CultureInfo.InvariantCulture,
" {0}:{1}",
feature.Name,
feature.Value);
}
else
{
this.builder.AppendFormat(
CultureInfo.InvariantCulture,
" {0}{1}",
feature.Name,
feature.Value);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
/// </summary>
/// <param name="vw">The native instance to wrap.</param>
/// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
public VowpalWabbit(VowpalWabbit vw)
{
if (vw == null)
{
throw new ArgumentNullException("vw");
}
Contract.Ensures(this.serializer != null);
Contract.EndContractBlock();
this.vw = vw;
this.compiledSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings);
if (this.compiledSerializer == null)
{
throw new ArgumentException("No features found for " + typeof(TExample));
}
this.serializer = this.compiledSerializer.Create(vw);
// have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results.
this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer;
// have a 3rd member to avoid cast everytime...
this.singleLineSerializer = this.serializer as VowpalWabbitSingleExampleSerializer <TExample>;
}
public void TestCustomFeaturizer()
{
var context = new MyContext()
{
Feature = new CustomClass()
{
X = 5
}
};
using (var vw = new VowpalWabbit(""))
{
var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings {
CustomFeaturizer = new List <Type> {
typeof(CustomFeaturizer)
}
})
.Create(vw);
var example = serializer.Serialize(context);
Assert.IsNotNull(example);
example.Dispose();
}
Assert.IsTrue(context.Feature.HasVisited);
}
public void TestCustomFeaturizerOverideMethod()
{
var context = new MyContext()
{
Feature = new CustomClass()
{
X = 5
}
};
using (var vw = new VowpalWabbit(""))
{
var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings
{
Schema = new Schema
{
Features = new List <FeatureExpression>
{
new FeatureExpression(typeof(CustomClass), "Feature",
// TODO: looks a bit awkward for an API. The compiler needs to know what property to access to copy the value into the Feature<T> object
valueExpression => Expression.Property(valueExpression, (PropertyInfo)ReflectionHelper.GetInfo((MyContext m) => m.Feature)),
overrideSerializeMethod: (MethodInfo)ReflectionHelper.GetInfo((CustomFeaturizer c) => c.MarshalFeature(null, null, null, null)))
}
}
}).Create(vw);
var example = serializer.Serialize(context);
Assert.IsNotNull(example);
example.Dispose();
}
Assert.IsTrue(context.Feature.HasVisited);
}
public VowpalWabbit(string arguments, VowpalWabbitSerializerSettings settings = null)
: base(arguments)
{
var visitor = new VowpalWabbitInterfaceVisitor(this);
this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings);
}
internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings)
{
this.vw = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true));
this.serializer = this.vw.Serializer.Func(this.vw.Native);
this.vwNative = new VowpalWabbit <TExample>(settings);
this.serializerNative = this.vwNative.Serializer.Func(this.vwNative.Native);
this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native);
}
/// <summary>
/// Constructor using a memory stream.
/// </summary>
/// <param name="vwModelStream">The VW model memory stream.</param>
public VWRanker(Stream vwModelStream = null, ITypeInspector typeInspector = null, bool developmentMode = false)
: base(vwModelStream, typeInspector, developmentMode)
{
this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TContext>(new VowpalWabbitSettings
{
TypeInspector = this.typeInspector,
EnableStringExampleGeneration = this.developmentMode,
EnableStringFloatCompact = this.developmentMode
}) as IVowpalWabbitMultiExampleSerializerCompiler <TContext>;
}
public void TestNumADFs()
{
var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings(featureDiscovery: VowpalWabbitFeatureDiscovery.Json))
as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>;
Assert.IsNotNull(jsonDirectSerializer);
Assert.AreEqual(3,
jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext {
Multi = new MyADF[3]
}));
}
internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager)
{
Contract.Requires(manager != null);
Contract.Ensures(this.serializers != null);
this.manager = manager;
// create a serializer for each instance - maintaining separate example caches
this.serializers = Enumerable
.Range(0, manager.Settings.ParallelOptions.MaxDegreeOfParallelism)
.Select(_ => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings))
.ToArray();
}
public void TestNumADFs()
{
var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings {
TypeInspector = JsonTypeInspector.Default
})
as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>;
Assert.IsNotNull(jsonDirectSerializer);
Assert.AreEqual(3,
jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext {
Multi = new MyADF[3]
}));
}
internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager)
{
Contract.Requires(manager != null);
Contract.Ensures(this.serializers != null);
this.manager = manager;
// create a serializer for each instance - maintaining separate example caches
var serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings);
this.serializers = this.manager.VowpalWabbits
.Select(vw => serializer.Create(vw))
.ToArray();
}
/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
/// </summary>
/// <param name="vw">The native instance to wrap.</param>
/// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
public VowpalWabbit(VowpalWabbit vw)
{
if (vw == null)
{
throw new ArgumentNullException("vw");
}
Contract.Ensures(this.serializer != null);
Contract.EndContractBlock();
this.vw = vw;
this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings);
// have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results.
this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer;
}
private void Initialize(VowpalWabbitSerializerSettings settings)
{
var visitor = new VowpalWabbitInterfaceVisitor(this);
this.actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(visitor, settings);
if (this.actionDependentFeatureSerializer == null)
{
throw new ArgumentException(typeof(TActionDependentFeature) + " must have a least a single [Feature] defined.");
}
using (var exBuilder = new VowpalWabbitExampleBuilder(this))
{
this.emptyExample = exBuilder.CreateExample();
}
}
private VowpalWabbitExampleCollection SerializeTyped <T>(T example, ILabel label, int?index)
{
IDisposable serializer;
if (!this.serializers.TryGetValue(typeof(T), out serializer))
{
var serializerCompiler = VowpalWabbitSerializerFactory.CreateSerializer <T>(this.vw.Settings);
if (serializerCompiler == null)
{
throw new ArgumentException("No feature discovered for type: " + typeof(T));
}
serializer = serializerCompiler.Create(this.vw);
this.serializers.Add(typeof(T), serializer);
}
return(((IVowpalWabbitSerializer <T>)serializer).Serialize(example, label, index));
}
public void TestDictify()
{
using (var vw = new VowpalWabbit(new VowpalWabbitSettings(enableStringExampleGeneration: true, enableStringFloatCompact: true)))
using (var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyDictifyContext>(vw.Settings).Create(vw))
{
var dictionary = new Dictionary <string, string>();
var ctx = new MyDictifyContext
{
A = 5,
B = new[] { 1f, 2f, 3f }
};
var str = serializer.SerializeToString(ctx, dictionary: dictionary);
Assert.AreEqual(" | A:5 d0", str);
Assert.AreEqual(1, dictionary.Count);
Assert.IsTrue(dictionary.ContainsKey(" 0:1 1:2 2:3"));
Assert.AreEqual("d0", dictionary[" 0:1 1:2 2:3"]);
}
}
internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings)
{
this.vw = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true));
var compiler = this.vw.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>;
if (compiler != null)
{
this.serializer = compiler.Func(this.vw.Native);
}
this.vwNative = new VowpalWabbit <TExample>(settings);
compiler = this.vwNative.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>;
if (compiler != null)
{
this.serializerNative = compiler.Func(this.vwNative.Native);
}
this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native);
}
private VowpalWabbitSerializer <object> GetOrCreateSerializer(Type type)
{
VowpalWabbitSerializer <object> serializer;
if (!this.serializers.TryGetValue(type, out serializer))
{
var allFeatures = AnnotationInspector.ExtractFeatures(type, (_, __) => true);
foreach (var feature in allFeatures)
{
// inject type cast to the actual type (always works)
// needed since the serializer is generated for "type", not for "object"
feature.ValueExpressionFactory = expr => feature.ValueExpressionFactory(Expression.Convert(expr, type));
}
serializer = VowpalWabbitSerializerFactory
.CreateSerializer <object>(this.vw.Settings.ShallowCopy(allFeatures: allFeatures))
.Create(this.vw);
this.serializers.Add(type, serializer);
}
return(serializer);
}
/// <summary>
/// Initializes a new instance.
/// </summary>
/// <param name="settings">The list of settings to be used.</param>
public VowpalWabbitSweep(List <VowpalWabbitSettings> settings)
{
if (settings == null || settings.Count == 0)
{
throw new ArgumentException("settings");
}
Contract.EndContractBlock();
// TODO: check that the sweeps are not across incompatible options.
this.settings = settings;
this.vws = settings.Select(setting => new VowpalWabbit(setting)).ToArray();
var diffs = this.vws.Skip(1).Select(vw => vw.AreFeaturesCompatible(this.vws[0])).Where(e => e != null).ToList();
if (diffs.Count > 0)
{
throw new ArgumentException("Feature settings are not compatible for sweeping: " + string.Join(",", diffs));
}
this.serializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings).Create(vw)).ToArray();
this.actionDependentFeatureSerializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(vw.Settings).Create(vw)).ToArray();
}
internal async Task TestAllReduceInternal()
{
var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList();
var stringSerializerCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfShared>(new VowpalWabbitSettings(enableStringExampleGeneration: true));
var stringSerializerAdfCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfAction>(new VowpalWabbitSettings(enableStringExampleGeneration: true));
var stringData = new List <List <string> >();
VowpalWabbitPerformanceStatistics statsExpected;
using (var spanningTree = new SpanningTreeClr())
{
spanningTree.Start();
using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
{
var stringSerializer = stringSerializerCompiled.Func(vw1);
var stringSerializerAdf = stringSerializerAdfCompiled.Func(vw1);
// serialize
foreach (var d in data)
{
var block = new List <string>();
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializer(context, d.Item1, SharedLabel.Instance);
block.Add(context.StringExample.ToString());
}
block.AddRange(d.Item2.Select((a, i) =>
{
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null);
return(context.StringExample.ToString());
}
}));
stringData.Add(block);
}
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500))));
vw1.SaveModel("expected.1.model");
vw2.SaveModel("expected.2.model");
statsExpected = vw1.PerformanceStatistics;
}
}
// skip header
var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList();
var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList();
var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy",
parallelOptions: new ParallelOptions
{
MaxDegreeOfParallelism = 2
},
exampleCountPerRun: 2000,
exampleDistribution: VowpalWabbitExampleDistribution.RoundRobin);
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
var vwManaged = vw.Create <CbAdfShared, CbAdfAction>();
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))),
Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.managed.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
}
/// <summary>
/// Serializes the specifed example to VW native string format.
/// </summary>
/// <typeparam name="TExample">The user example type.</typeparam>
/// <typeparam name="TActionDependentFeature">The user action dependent feature type.</typeparam>
/// <param name="vw">The VW instance.</param>
/// <param name="example">The shared example.</param>
/// <param name="actionDependentFeatures">The action dependent features.</param>
/// <param name="index">The optional index of the label example.</param>
/// <param name="label">The optional label.</param>
/// <param name="serializer">The example serializer.</param>
/// <param name="actionDependentFeatureSerializer">The action dependent feature serializer.</param>
/// <param name="dictionary">Dictionary used for dictify operation.</param>
/// <param name="fastDictionary">Dictionary used for dictify operation.</param>
/// <returns>The string serialized example.</returns>
public static string SerializeToString <TExample, TActionDependentFeature>(
VowpalWabbit vw,
TExample example,
IReadOnlyCollection <TActionDependentFeature> actionDependentFeatures,
int?index = null,
ILabel label = null,
VowpalWabbitSerializer <TExample> serializer = null,
VowpalWabbitSerializer <TActionDependentFeature> actionDependentFeatureSerializer = null,
Dictionary <string, string> dictionary = null,
Dictionary <object, string> fastDictionary = null)
{
if (vw == null)
{
throw new ArgumentNullException("vw");
}
if (serializer == null)
{
serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw);
}
else if (!serializer.EnableStringExampleGeneration)
{
throw new ArgumentException("Serializer must be compiled using EnableStringExampleGeneration = true");
}
if (actionDependentFeatureSerializer == null)
{
actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw);
}
else if (!actionDependentFeatureSerializer.EnableStringExampleGeneration)
{
throw new ArgumentException("Action dependent serializer must be compiled using EnableStringExampleGeneration = true");
}
var stringExample = new StringBuilder();
var sharedExample = serializer.SerializeToString(example, SharedLabel.Instance, dictionary, fastDictionary);
// check if we have shared features
if (!string.IsNullOrWhiteSpace(sharedExample))
{
stringExample.AppendLine(sharedExample);
}
var i = 0;
foreach (var actionDependentFeature in actionDependentFeatures)
{
var adfExample = actionDependentFeatureSerializer.SerializeToString(actionDependentFeature,
index != null && i == index ? label : null, dictionary, fastDictionary);
if (!string.IsNullOrWhiteSpace(adfExample))
{
stringExample.AppendLine(adfExample);
}
i++;
}
return(stringExample.ToString());
}
/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
/// </summary>
/// <param name="vw">The native instance to wrap.</param>
/// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
public VowpalWabbit(VowpalWabbit vw) : this(vw, VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings))
{
}