/// <summary> /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class. /// </summary> /// <param name="vw">The native instance to wrap.</param> /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks> public VowpalWabbit(VowpalWabbit vw) { if (vw == null) { throw new ArgumentNullException("vw"); } Contract.Ensures(this.serializer != null); Contract.EndContractBlock(); this.vw = vw; this.compiledSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings); if (this.compiledSerializer == null) { throw new ArgumentException("No features found for " + typeof(TExample)); } this.serializer = this.compiledSerializer.Create(vw); // have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results. this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer; // have a 3rd member to avoid cast everytime... this.singleLineSerializer = this.serializer as VowpalWabbitSingleExampleSerializer <TExample>; }
public VowpalWabbit(VowpalWabbitModel model, VowpalWabbitSerializerSettings settings = null) : base(model) { var visitor = new VowpalWabbitInterfaceVisitor(this); this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings); }
public void TestCustomFeaturizer() { var context = new MyContext() { Feature = new CustomClass() { X = 5 } }; using (var vw = new VowpalWabbit("")) { var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings { CustomFeaturizer = new List <Type> { typeof(CustomFeaturizer) } }) .Create(vw); var example = serializer.Serialize(context); Assert.IsNotNull(example); example.Dispose(); } Assert.IsTrue(context.Feature.HasVisited); }
public void TestCustomFeaturizerOverideMethod() { var context = new MyContext() { Feature = new CustomClass() { X = 5 } }; using (var vw = new VowpalWabbit("")) { var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings { Schema = new Schema { Features = new List <FeatureExpression> { new FeatureExpression(typeof(CustomClass), "Feature", // TODO: looks a bit awkward for an API. The compiler needs to know what property to access to copy the value into the Feature<T> object valueExpression => Expression.Property(valueExpression, (PropertyInfo)ReflectionHelper.GetInfo((MyContext m) => m.Feature)), overrideSerializeMethod: (MethodInfo)ReflectionHelper.GetInfo((CustomFeaturizer c) => c.MarshalFeature(null, null, null, null))) } } }).Create(vw); var example = serializer.Serialize(context); Assert.IsNotNull(example); example.Dispose(); } Assert.IsTrue(context.Feature.HasVisited); }
public VowpalWabbit(string arguments, VowpalWabbitSerializerSettings settings = null) : base(arguments) { var visitor = new VowpalWabbitInterfaceVisitor(this); this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings); }
internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings) { this.vw = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)); this.serializer = this.vw.Serializer.Func(this.vw.Native); this.vwNative = new VowpalWabbit <TExample>(settings); this.serializerNative = this.vwNative.Serializer.Func(this.vwNative.Native); this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native); }
/// <summary> /// Constructor using a memory stream. /// </summary> /// <param name="vwModelStream">The VW model memory stream.</param> public VWRanker(Stream vwModelStream = null, ITypeInspector typeInspector = null, bool developmentMode = false) : base(vwModelStream, typeInspector, developmentMode) { this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TContext>(new VowpalWabbitSettings { TypeInspector = this.typeInspector, EnableStringExampleGeneration = this.developmentMode, EnableStringFloatCompact = this.developmentMode }) as IVowpalWabbitMultiExampleSerializerCompiler <TContext>; }
public void TestNumADFs() { var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings(featureDiscovery: VowpalWabbitFeatureDiscovery.Json)) as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>; Assert.IsNotNull(jsonDirectSerializer); Assert.AreEqual(3, jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext { Multi = new MyADF[3] })); }
internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager) { Contract.Requires(manager != null); Contract.Ensures(this.serializers != null); this.manager = manager; // create a serializer for each instance - maintaining separate example caches this.serializers = Enumerable .Range(0, manager.Settings.ParallelOptions.MaxDegreeOfParallelism) .Select(_ => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings)) .ToArray(); }
public void TestNumADFs() { var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings { TypeInspector = JsonTypeInspector.Default }) as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>; Assert.IsNotNull(jsonDirectSerializer); Assert.AreEqual(3, jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext { Multi = new MyADF[3] })); }
internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager) { Contract.Requires(manager != null); Contract.Ensures(this.serializers != null); this.manager = manager; // create a serializer for each instance - maintaining separate example caches var serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings); this.serializers = this.manager.VowpalWabbits .Select(vw => serializer.Create(vw)) .ToArray(); }
/// <summary> /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class. /// </summary> /// <param name="vw">The native instance to wrap.</param> /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks> public VowpalWabbit(VowpalWabbit vw) { if (vw == null) { throw new ArgumentNullException("vw"); } Contract.Ensures(this.serializer != null); Contract.EndContractBlock(); this.vw = vw; this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings); // have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results. this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer; }
private void Initialize(VowpalWabbitSerializerSettings settings) { var visitor = new VowpalWabbitInterfaceVisitor(this); this.actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(visitor, settings); if (this.actionDependentFeatureSerializer == null) { throw new ArgumentException(typeof(TActionDependentFeature) + " must have a least a single [Feature] defined."); } using (var exBuilder = new VowpalWabbitExampleBuilder(this)) { this.emptyExample = exBuilder.CreateExample(); } }
private VowpalWabbitExampleCollection SerializeTyped <T>(T example, ILabel label, int?index) { IDisposable serializer; if (!this.serializers.TryGetValue(typeof(T), out serializer)) { var serializerCompiler = VowpalWabbitSerializerFactory.CreateSerializer <T>(this.vw.Settings); if (serializerCompiler == null) { throw new ArgumentException("No feature discovered for type: " + typeof(T)); } serializer = serializerCompiler.Create(this.vw); this.serializers.Add(typeof(T), serializer); } return(((IVowpalWabbitSerializer <T>)serializer).Serialize(example, label, index)); }
internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings) { this.vw = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)); var compiler = this.vw.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>; if (compiler != null) { this.serializer = compiler.Func(this.vw.Native); } this.vwNative = new VowpalWabbit <TExample>(settings); compiler = this.vwNative.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>; if (compiler != null) { this.serializerNative = compiler.Func(this.vwNative.Native); } this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native); }
public void TestDictify() { using (var vw = new VowpalWabbit(new VowpalWabbitSettings(enableStringExampleGeneration: true, enableStringFloatCompact: true))) using (var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyDictifyContext>(vw.Settings).Create(vw)) { var dictionary = new Dictionary <string, string>(); var ctx = new MyDictifyContext { A = 5, B = new[] { 1f, 2f, 3f } }; var str = serializer.SerializeToString(ctx, dictionary: dictionary); Assert.AreEqual(" | A:5 d0", str); Assert.AreEqual(1, dictionary.Count); Assert.IsTrue(dictionary.ContainsKey(" 0:1 1:2 2:3")); Assert.AreEqual("d0", dictionary[" 0:1 1:2 2:3"]); } }
private VowpalWabbitSerializer <object> GetOrCreateSerializer(Type type) { VowpalWabbitSerializer <object> serializer; if (!this.serializers.TryGetValue(type, out serializer)) { var allFeatures = AnnotationInspector.ExtractFeatures(type, (_, __) => true); foreach (var feature in allFeatures) { // inject type cast to the actual type (always works) // needed since the serializer is generated for "type", not for "object" feature.ValueExpressionFactory = expr => feature.ValueExpressionFactory(Expression.Convert(expr, type)); } serializer = VowpalWabbitSerializerFactory .CreateSerializer <object>(this.vw.Settings.ShallowCopy(allFeatures: allFeatures)) .Create(this.vw); this.serializers.Add(type, serializer); } return(serializer); }
/// <summary> /// Initializes a new instance. /// </summary> /// <param name="settings">The list of settings to be used.</param> public VowpalWabbitSweep(List <VowpalWabbitSettings> settings) { if (settings == null || settings.Count == 0) { throw new ArgumentException("settings"); } Contract.EndContractBlock(); // TODO: check that the sweeps are not across incompatible options. this.settings = settings; this.vws = settings.Select(setting => new VowpalWabbit(setting)).ToArray(); var diffs = this.vws.Skip(1).Select(vw => vw.AreFeaturesCompatible(this.vws[0])).Where(e => e != null).ToList(); if (diffs.Count > 0) { throw new ArgumentException("Feature settings are not compatible for sweeping: " + string.Join(",", diffs)); } this.serializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings).Create(vw)).ToArray(); this.actionDependentFeatureSerializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(vw.Settings).Create(vw)).ToArray(); }
internal async Task TestAllReduceInternal() { var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList(); var stringSerializerCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfShared>(new VowpalWabbitSettings(enableStringExampleGeneration: true)); var stringSerializerAdfCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfAction>(new VowpalWabbitSettings(enableStringExampleGeneration: true)); var stringData = new List <List <string> >(); VowpalWabbitPerformanceStatistics statsExpected; using (var spanningTree = new SpanningTreeClr()) { spanningTree.Start(); using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true))) using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true))) { var stringSerializer = stringSerializerCompiled.Func(vw1); var stringSerializerAdf = stringSerializerAdfCompiled.Func(vw1); // serialize foreach (var d in data) { var block = new List <string>(); using (var context = new VowpalWabbitMarshalContext(vw1)) { stringSerializer(context, d.Item1, SharedLabel.Instance); block.Add(context.StringExample.ToString()); } block.AddRange(d.Item2.Select((a, i) => { using (var context = new VowpalWabbitMarshalContext(vw1)) { stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null); return(context.StringExample.ToString()); } })); stringData.Add(block); } await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))), Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500)))); vw1.SaveModel("expected.1.model"); vw2.SaveModel("expected.2.model"); statsExpected = vw1.PerformanceStatistics; } } // skip header var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList(); var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList(); var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy", parallelOptions: new ParallelOptions { MaxDegreeOfParallelism = 2 }, exampleCountPerRun: 2000, exampleDistribution: VowpalWabbitExampleDistribution.RoundRobin); using (var vw = new VowpalWabbitThreadedLearning(settings)) { await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))), Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500)))); // important to enqueue the request before Complete() is called var statsTask = vw.PerformanceStatistics; var modelSave = vw.SaveModel("actual.model"); await vw.Complete(); var statsActual = await statsTask; VWTestHelper.AssertEqual(statsExpected, statsActual); await modelSave; // skip header var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList(); CollectionAssert.AreEqual(expected1Model, actualModel); CollectionAssert.AreEqual(expected2Model, actualModel); } using (var vw = new VowpalWabbitThreadedLearning(settings)) { var vwManaged = vw.Create <CbAdfShared, CbAdfAction>(); await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))), Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500)))); // important to enqueue the request before Complete() is called var statsTask = vw.PerformanceStatistics; var modelSave = vw.SaveModel("actual.managed.model"); await vw.Complete(); var statsActual = await statsTask; VWTestHelper.AssertEqual(statsExpected, statsActual); await modelSave; // skip header var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList(); CollectionAssert.AreEqual(expected1Model, actualModel); CollectionAssert.AreEqual(expected2Model, actualModel); } }
/// <summary> /// Serializes the specifed example to VW native string format. /// </summary> /// <typeparam name="TExample">The user example type.</typeparam> /// <typeparam name="TActionDependentFeature">The user action dependent feature type.</typeparam> /// <param name="vw">The VW instance.</param> /// <param name="example">The shared example.</param> /// <param name="actionDependentFeatures">The action dependent features.</param> /// <param name="index">The optional index of the label example.</param> /// <param name="label">The optional label.</param> /// <param name="serializer">The example serializer.</param> /// <param name="actionDependentFeatureSerializer">The action dependent feature serializer.</param> /// <param name="dictionary">Dictionary used for dictify operation.</param> /// <param name="fastDictionary">Dictionary used for dictify operation.</param> /// <returns>The string serialized example.</returns> public static string SerializeToString <TExample, TActionDependentFeature>( VowpalWabbit vw, TExample example, IReadOnlyCollection <TActionDependentFeature> actionDependentFeatures, int?index = null, ILabel label = null, VowpalWabbitSerializer <TExample> serializer = null, VowpalWabbitSerializer <TActionDependentFeature> actionDependentFeatureSerializer = null, Dictionary <string, string> dictionary = null, Dictionary <object, string> fastDictionary = null) { if (vw == null) { throw new ArgumentNullException("vw"); } if (serializer == null) { serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw); } else if (!serializer.EnableStringExampleGeneration) { throw new ArgumentException("Serializer must be compiled using EnableStringExampleGeneration = true"); } if (actionDependentFeatureSerializer == null) { actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw); } else if (!actionDependentFeatureSerializer.EnableStringExampleGeneration) { throw new ArgumentException("Action dependent serializer must be compiled using EnableStringExampleGeneration = true"); } var stringExample = new StringBuilder(); var sharedExample = serializer.SerializeToString(example, SharedLabel.Instance, dictionary, fastDictionary); // check if we have shared features if (!string.IsNullOrWhiteSpace(sharedExample)) { stringExample.AppendLine(sharedExample); } var i = 0; foreach (var actionDependentFeature in actionDependentFeatures) { var adfExample = actionDependentFeatureSerializer.SerializeToString(actionDependentFeature, index != null && i == index ? label : null, dictionary, fastDictionary); if (!string.IsNullOrWhiteSpace(adfExample)) { stringExample.AppendLine(adfExample); } i++; } return(stringExample.ToString()); }
/// <summary> /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class. /// </summary> /// <param name="vw">The native instance to wrap.</param> /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks> public VowpalWabbit(VowpalWabbit vw) : this(vw, VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings)) { }