public void MarshalFeature(VowpalWabbitMarshalContext context, Namespace ns, Feature feature, CustomClass value)
{
Assert.IsNotNull(context);
Assert.IsNotNull(ns);
Assert.IsNotNull(feature);
Assert.IsNotNull(value);
Assert.AreEqual(5, value.X);
value.HasVisited = true;
}
public void Validate(string line, TExample example, ILabel label = null)
{
IVowpalWabbitLabelComparator comparator;
if (label == null || label == SharedLabel.Instance)
{
comparator = null;
}
else if (label is SimpleLabel)
{
comparator = VowpalWabbitLabelComparator.Simple;
}
else if (label is ContextualBanditLabel)
{
comparator = VowpalWabbitLabelComparator.ContextualBandit;
}
else
{
throw new ArgumentException("Label type not supported: " + label.GetType());
}
using (var context = new VowpalWabbitMarshalContext(this.vw.Native))
using (var contextNative = new VowpalWabbitMarshalContext(this.vwNative.Native))
{
// validate string serializer
this.serializer(context, example, label);
this.serializerNative(contextNative, example, label);
// natively parsed string example compared against:
// (1) natively build example
// (2) string serialized & natively parsed string example
using (var strExample = this.vw.Native.ParseLine(line))
using (var strConvertedExample = this.vw.Native.ParseLine(context.StringExample.ToString()))
using (var nativeExample = contextNative.ExampleBuilder.CreateExample())
using (var nativeExampleWithString = this.factorySerializer.Serialize(example, label))
{
var diff = strExample.Diff(this.vw.Native, strConvertedExample, comparator);
Assert.IsNull(diff, diff + " generated string: '" + context.StringExample + "'");
diff = strExample.Diff(this.vw.Native, nativeExample, comparator);
Assert.IsNull(diff, diff);
if (!strExample.IsNewLine)
{
Assert.IsFalse(string.IsNullOrEmpty(nativeExampleWithString.VowpalWabbitString));
Assert.IsFalse(string.IsNullOrEmpty(this.factorySerializer.SerializeToString(example, label)));
}
}
}
}
internal async Task TestAllReduceInternal()
{
var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList();
var stringSerializerCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfShared>(new VowpalWabbitSettings(enableStringExampleGeneration: true));
var stringSerializerAdfCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfAction>(new VowpalWabbitSettings(enableStringExampleGeneration: true));
var stringData = new List <List <string> >();
VowpalWabbitPerformanceStatistics statsExpected;
using (var spanningTree = new SpanningTreeClr())
{
spanningTree.Start();
using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
{
var stringSerializer = stringSerializerCompiled.Func(vw1);
var stringSerializerAdf = stringSerializerAdfCompiled.Func(vw1);
// serialize
foreach (var d in data)
{
var block = new List <string>();
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializer(context, d.Item1, SharedLabel.Instance);
block.Add(context.StringExample.ToString());
}
block.AddRange(d.Item2.Select((a, i) =>
{
using (var context = new VowpalWabbitMarshalContext(vw1))
{
stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null);
return(context.StringExample.ToString());
}
}));
stringData.Add(block);
}
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500))));
vw1.SaveModel("expected.1.model");
vw2.SaveModel("expected.2.model");
statsExpected = vw1.PerformanceStatistics;
}
}
// skip header
var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList();
var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList();
var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy",
parallelOptions: new ParallelOptions
{
MaxDegreeOfParallelism = 2
},
exampleCountPerRun: 2000,
exampleDistribution: VowpalWabbitExampleDistribution.RoundRobin);
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))),
Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
using (var vw = new VowpalWabbitThreadedLearning(settings))
{
var vwManaged = vw.Create <CbAdfShared, CbAdfAction>();
await Task.WhenAll(
Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))),
Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500))));
// important to enqueue the request before Complete() is called
var statsTask = vw.PerformanceStatistics;
var modelSave = vw.SaveModel("actual.managed.model");
await vw.Complete();
var statsActual = await statsTask;
VWTestHelper.AssertEqual(statsExpected, statsActual);
await modelSave;
// skip header
var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList();
CollectionAssert.AreEqual(expected1Model, actualModel);
CollectionAssert.AreEqual(expected2Model, actualModel);
}
}
