Esempio n. 1
0
        public VowpalWabbit(VowpalWabbitModel model, VowpalWabbitSerializerSettings settings = null)
            : base(model)
        {
            var visitor = new VowpalWabbitInterfaceVisitor(this);

            this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings);
        }
Esempio n. 2
0
        public void Visit <T>(IFeature <T> feature)
        {
            Contract.Requires(feature != null);

            // can't specify constraints to narrow for enums
            var valueType = typeof(T);

            if (valueType.IsEnum)
            {
                this.builder.AppendFormat(
                    CultureInfo.InvariantCulture,
                    " {0}{1}",
                    feature.Name,
                    Enum.GetName(valueType, feature.Value));
            }
            else if (VowpalWabbitSerializerFactory.IsValidDenseFeatureValueElementType(typeof(T)))
            {
                this.builder.AppendFormat(
                    CultureInfo.InvariantCulture,
                    " {0}:{1}",
                    feature.Name,
                    feature.Value);
            }
            else
            {
                this.builder.AppendFormat(
                    CultureInfo.InvariantCulture,
                    " {0}{1}",
                    feature.Name,
                    feature.Value);
            }
        }
Esempio n. 3
0
        /// <summary>
        /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
        /// </summary>
        /// <param name="vw">The native instance to wrap.</param>
        /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
        public VowpalWabbit(VowpalWabbit vw)
        {
            if (vw == null)
            {
                throw new ArgumentNullException("vw");
            }
            Contract.Ensures(this.serializer != null);
            Contract.EndContractBlock();

            this.vw = vw;
            this.compiledSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings);

            if (this.compiledSerializer == null)
            {
                throw new ArgumentException("No features found for " + typeof(TExample));
            }

            this.serializer = this.compiledSerializer.Create(vw);

            // have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results.
            this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer;

            // have a 3rd member to avoid cast everytime...
            this.singleLineSerializer = this.serializer as VowpalWabbitSingleExampleSerializer <TExample>;
        }
        public void TestCustomFeaturizer()
        {
            var context = new MyContext()
            {
                Feature = new CustomClass()
                {
                    X = 5
                }
            };

            using (var vw = new VowpalWabbit(""))
            {
                var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings {
                    CustomFeaturizer = new List <Type> {
                        typeof(CustomFeaturizer)
                    }
                })
                                 .Create(vw);

                var example = serializer.Serialize(context);

                Assert.IsNotNull(example);

                example.Dispose();
            }

            Assert.IsTrue(context.Feature.HasVisited);
        }
        public void TestCustomFeaturizerOverideMethod()
        {
            var context = new MyContext()
            {
                Feature = new CustomClass()
                {
                    X = 5
                }
            };

            using (var vw = new VowpalWabbit(""))
            {
                var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings
                {
                    Schema = new Schema
                    {
                        Features = new List <FeatureExpression>
                        {
                            new FeatureExpression(typeof(CustomClass), "Feature",
                                                  // TODO: looks a bit awkward for an API. The compiler needs to know what property to access to copy the value into the Feature<T> object
                                                  valueExpression => Expression.Property(valueExpression, (PropertyInfo)ReflectionHelper.GetInfo((MyContext m) => m.Feature)),
                                                  overrideSerializeMethod: (MethodInfo)ReflectionHelper.GetInfo((CustomFeaturizer c) => c.MarshalFeature(null, null, null, null)))
                        }
                    }
                }).Create(vw);
                var example = serializer.Serialize(context);

                Assert.IsNotNull(example);

                example.Dispose();
            }

            Assert.IsTrue(context.Feature.HasVisited);
        }
Esempio n. 6
0
        public VowpalWabbit(string arguments, VowpalWabbitSerializerSettings settings = null)
            : base(arguments)
        {
            var visitor = new VowpalWabbitInterfaceVisitor(this);

            this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(visitor, settings);
        }
Esempio n. 7
0
        internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings)
        {
            this.vw         = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true));
            this.serializer = this.vw.Serializer.Func(this.vw.Native);

            this.vwNative         = new VowpalWabbit <TExample>(settings);
            this.serializerNative = this.vwNative.Serializer.Func(this.vwNative.Native);

            this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native);
        }
Esempio n. 8
0
 /// <summary>
 /// Constructor using a memory stream.
 /// </summary>
 /// <param name="vwModelStream">The VW model memory stream.</param>
 public VWRanker(Stream vwModelStream = null, ITypeInspector typeInspector = null, bool developmentMode = false)
     : base(vwModelStream, typeInspector, developmentMode)
 {
     this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TContext>(new VowpalWabbitSettings
     {
         TypeInspector = this.typeInspector,
         EnableStringExampleGeneration = this.developmentMode,
         EnableStringFloatCompact      = this.developmentMode
     }) as IVowpalWabbitMultiExampleSerializerCompiler <TContext>;
 }
Esempio n. 9
0
        public void TestNumADFs()
        {
            var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings(featureDiscovery: VowpalWabbitFeatureDiscovery.Json))
                                       as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>;

            Assert.IsNotNull(jsonDirectSerializer);
            Assert.AreEqual(3,
                            jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext {
                Multi = new MyADF[3]
            }));
        }
Esempio n. 10
0
        internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager)
        {
            Contract.Requires(manager != null);
            Contract.Ensures(this.serializers != null);

            this.manager = manager;

            // create a serializer for each instance - maintaining separate example caches
            this.serializers = Enumerable
                               .Range(0, manager.Settings.ParallelOptions.MaxDegreeOfParallelism)
                               .Select(_ => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings))
                               .ToArray();
        }
Esempio n. 11
0
        public void TestNumADFs()
        {
            var jsonDirectSerializer = VowpalWabbitSerializerFactory.CreateSerializer <MyContext>(new VowpalWabbitSettings {
                TypeInspector = JsonTypeInspector.Default
            })
                                       as IVowpalWabbitMultiExampleSerializerCompiler <MyContext>;

            Assert.IsNotNull(jsonDirectSerializer);
            Assert.AreEqual(3,
                            jsonDirectSerializer.GetNumberOfActionDependentExamples(new MyContext {
                Multi = new MyADF[3]
            }));
        }
Esempio n. 12
0
        internal VowpalWabbitAsync(VowpalWabbitThreadedLearning manager)
        {
            Contract.Requires(manager != null);
            Contract.Ensures(this.serializers != null);

            this.manager = manager;

            // create a serializer for each instance - maintaining separate example caches
            var serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(manager.Settings);

            this.serializers = this.manager.VowpalWabbits
                               .Select(vw => serializer.Create(vw))
                               .ToArray();
        }
Esempio n. 13
0
        /// <summary>
        /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
        /// </summary>
        /// <param name="vw">The native instance to wrap.</param>
        /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
        public VowpalWabbit(VowpalWabbit vw)
        {
            if (vw == null)
            {
                throw new ArgumentNullException("vw");
            }
            Contract.Ensures(this.serializer != null);
            Contract.EndContractBlock();

            this.vw         = vw;
            this.serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings);

            // have a 2nd member to throw NullReferenceException in release instead of silently producing wrong results.
            this.learnSerializer = this.serializer.CachesExamples ? null : this.serializer;
        }
Esempio n. 14
0
        private void Initialize(VowpalWabbitSerializerSettings settings)
        {
            var visitor = new VowpalWabbitInterfaceVisitor(this);

            this.actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(visitor, settings);

            if (this.actionDependentFeatureSerializer == null)
            {
                throw new ArgumentException(typeof(TActionDependentFeature) + " must have a least a single [Feature] defined.");
            }

            using (var exBuilder = new VowpalWabbitExampleBuilder(this))
            {
                this.emptyExample = exBuilder.CreateExample();
            }
        }
Esempio n. 15
0
        private VowpalWabbitExampleCollection SerializeTyped <T>(T example, ILabel label, int?index)
        {
            IDisposable serializer;

            if (!this.serializers.TryGetValue(typeof(T), out serializer))
            {
                var serializerCompiler = VowpalWabbitSerializerFactory.CreateSerializer <T>(this.vw.Settings);

                if (serializerCompiler == null)
                {
                    throw new ArgumentException("No feature discovered for type: " + typeof(T));
                }

                serializer = serializerCompiler.Create(this.vw);

                this.serializers.Add(typeof(T), serializer);
            }

            return(((IVowpalWabbitSerializer <T>)serializer).Serialize(example, label, index));
        }
Esempio n. 16
0
        public void TestDictify()
        {
            using (var vw = new VowpalWabbit(new VowpalWabbitSettings(enableStringExampleGeneration: true, enableStringFloatCompact: true)))
                using (var serializer = VowpalWabbitSerializerFactory.CreateSerializer <MyDictifyContext>(vw.Settings).Create(vw))
                {
                    var dictionary = new Dictionary <string, string>();
                    var ctx        = new MyDictifyContext
                    {
                        A = 5,
                        B = new[] { 1f, 2f, 3f }
                    };

                    var str = serializer.SerializeToString(ctx, dictionary: dictionary);

                    Assert.AreEqual(" |  A:5 d0", str);

                    Assert.AreEqual(1, dictionary.Count);
                    Assert.IsTrue(dictionary.ContainsKey(" 0:1 1:2 2:3"));
                    Assert.AreEqual("d0", dictionary[" 0:1 1:2 2:3"]);
                }
        }
        internal VowpalWabbitExampleValidator(VowpalWabbitSettings settings)
        {
            this.vw = new VowpalWabbit <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true));

            var compiler = this.vw.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>;

            if (compiler != null)
            {
                this.serializer = compiler.Func(this.vw.Native);
            }

            this.vwNative = new VowpalWabbit <TExample>(settings);

            compiler = this.vwNative.Serializer as VowpalWabbitSingleExampleSerializerCompiler <TExample>;
            if (compiler != null)
            {
                this.serializerNative = compiler.Func(this.vwNative.Native);
            }

            this.factorySerializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(settings.ShallowCopy(enableStringExampleGeneration: true)).Create(this.vw.Native);
        }
Esempio n. 18
0
        private VowpalWabbitSerializer <object> GetOrCreateSerializer(Type type)
        {
            VowpalWabbitSerializer <object> serializer;

            if (!this.serializers.TryGetValue(type, out serializer))
            {
                var allFeatures = AnnotationInspector.ExtractFeatures(type, (_, __) => true);
                foreach (var feature in allFeatures)
                {
                    // inject type cast to the actual type (always works)
                    // needed since the serializer is generated for "type", not for "object"
                    feature.ValueExpressionFactory = expr => feature.ValueExpressionFactory(Expression.Convert(expr, type));
                }

                serializer = VowpalWabbitSerializerFactory
                             .CreateSerializer <object>(this.vw.Settings.ShallowCopy(allFeatures: allFeatures))
                             .Create(this.vw);

                this.serializers.Add(type, serializer);
            }

            return(serializer);
        }
        /// <summary>
        /// Initializes a new instance.
        /// </summary>
        /// <param name="settings">The list of settings to be used.</param>
        public VowpalWabbitSweep(List <VowpalWabbitSettings> settings)
        {
            if (settings == null || settings.Count == 0)
            {
                throw new ArgumentException("settings");
            }

            Contract.EndContractBlock();

            // TODO: check that the sweeps are not across incompatible options.
            this.settings = settings;
            this.vws      = settings.Select(setting => new VowpalWabbit(setting)).ToArray();

            var diffs = this.vws.Skip(1).Select(vw => vw.AreFeaturesCompatible(this.vws[0])).Where(e => e != null).ToList();

            if (diffs.Count > 0)
            {
                throw new ArgumentException("Feature settings are not compatible for sweeping: " + string.Join(",", diffs));
            }

            this.serializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings).Create(vw)).ToArray();

            this.actionDependentFeatureSerializers = this.vws.Select(vw => VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(vw.Settings).Create(vw)).ToArray();
        }
Esempio n. 20
0
        internal async Task TestAllReduceInternal()
        {
            var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList();

            var stringSerializerCompiled    = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfShared>(new VowpalWabbitSettings(enableStringExampleGeneration: true));
            var stringSerializerAdfCompiled = VowpalWabbitSerializerFactory.CreateSerializer <CbAdfAction>(new VowpalWabbitSettings(enableStringExampleGeneration: true));

            var stringData = new List <List <string> >();

            VowpalWabbitPerformanceStatistics statsExpected;

            using (var spanningTree = new SpanningTreeClr())
            {
                spanningTree.Start();

                using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
                    using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy", enableStringExampleGeneration: true)))
                    {
                        var stringSerializer    = stringSerializerCompiled.Func(vw1);
                        var stringSerializerAdf = stringSerializerAdfCompiled.Func(vw1);

                        // serialize
                        foreach (var d in data)
                        {
                            var block = new List <string>();

                            using (var context = new VowpalWabbitMarshalContext(vw1))
                            {
                                stringSerializer(context, d.Item1, SharedLabel.Instance);
                                block.Add(context.StringExample.ToString());
                            }

                            block.AddRange(d.Item2.Select((a, i) =>
                            {
                                using (var context = new VowpalWabbitMarshalContext(vw1))
                                {
                                    stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null);
                                    return(context.StringExample.ToString());
                                }
                            }));

                            stringData.Add(block);
                        }

                        await Task.WhenAll(
                            Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))),
                            Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500))));

                        vw1.SaveModel("expected.1.model");
                        vw2.SaveModel("expected.2.model");

                        statsExpected = vw1.PerformanceStatistics;
                    }
            }

            // skip header
            var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList();
            var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList();

            var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy",
                                                    parallelOptions: new ParallelOptions
            {
                MaxDegreeOfParallelism = 2
            },
                                                    exampleCountPerRun: 2000,
                                                    exampleDistribution: VowpalWabbitExampleDistribution.RoundRobin);

            using (var vw = new VowpalWabbitThreadedLearning(settings))
            {
                await Task.WhenAll(
                    Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))),
                    Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500))));

                // important to enqueue the request before Complete() is called
                var statsTask = vw.PerformanceStatistics;
                var modelSave = vw.SaveModel("actual.model");

                await vw.Complete();

                var statsActual = await statsTask;
                VWTestHelper.AssertEqual(statsExpected, statsActual);

                await modelSave;

                // skip header
                var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList();

                CollectionAssert.AreEqual(expected1Model, actualModel);
                CollectionAssert.AreEqual(expected2Model, actualModel);
            }

            using (var vw = new VowpalWabbitThreadedLearning(settings))
            {
                var vwManaged = vw.Create <CbAdfShared, CbAdfAction>();

                await Task.WhenAll(
                    Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))),
                    Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500))));

                // important to enqueue the request before Complete() is called
                var statsTask = vw.PerformanceStatistics;
                var modelSave = vw.SaveModel("actual.managed.model");

                await vw.Complete();

                var statsActual = await statsTask;
                VWTestHelper.AssertEqual(statsExpected, statsActual);

                await modelSave;

                // skip header
                var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList();

                CollectionAssert.AreEqual(expected1Model, actualModel);
                CollectionAssert.AreEqual(expected2Model, actualModel);
            }
        }
        /// <summary>
        /// Serializes the specifed example to VW native string format.
        /// </summary>
        /// <typeparam name="TExample">The user example type.</typeparam>
        /// <typeparam name="TActionDependentFeature">The user action dependent feature type.</typeparam>
        /// <param name="vw">The VW instance.</param>
        /// <param name="example">The shared example.</param>
        /// <param name="actionDependentFeatures">The action dependent features.</param>
        /// <param name="index">The optional index of the label example.</param>
        /// <param name="label">The optional label.</param>
        /// <param name="serializer">The example serializer.</param>
        /// <param name="actionDependentFeatureSerializer">The action dependent feature serializer.</param>
        /// <param name="dictionary">Dictionary used for dictify operation.</param>
        /// <param name="fastDictionary">Dictionary used for dictify operation.</param>
        /// <returns>The string serialized example.</returns>
        public static string SerializeToString <TExample, TActionDependentFeature>(
            VowpalWabbit vw,
            TExample example,
            IReadOnlyCollection <TActionDependentFeature> actionDependentFeatures,
            int?index    = null,
            ILabel label = null,
            VowpalWabbitSerializer <TExample> serializer = null,
            VowpalWabbitSerializer <TActionDependentFeature> actionDependentFeatureSerializer = null,
            Dictionary <string, string> dictionary     = null,
            Dictionary <object, string> fastDictionary = null)
        {
            if (vw == null)
            {
                throw new ArgumentNullException("vw");
            }

            if (serializer == null)
            {
                serializer = VowpalWabbitSerializerFactory.CreateSerializer <TExample>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw);
            }
            else if (!serializer.EnableStringExampleGeneration)
            {
                throw new ArgumentException("Serializer must be compiled using EnableStringExampleGeneration = true");
            }

            if (actionDependentFeatureSerializer == null)
            {
                actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(new VowpalWabbitSettings(enableStringExampleGeneration: true)).Create(vw);
            }
            else if (!actionDependentFeatureSerializer.EnableStringExampleGeneration)
            {
                throw new ArgumentException("Action dependent serializer must be compiled using EnableStringExampleGeneration = true");
            }

            var stringExample = new StringBuilder();

            var sharedExample = serializer.SerializeToString(example, SharedLabel.Instance, dictionary, fastDictionary);

            // check if we have shared features
            if (!string.IsNullOrWhiteSpace(sharedExample))
            {
                stringExample.AppendLine(sharedExample);
            }

            var i = 0;

            foreach (var actionDependentFeature in actionDependentFeatures)
            {
                var adfExample = actionDependentFeatureSerializer.SerializeToString(actionDependentFeature,
                                                                                    index != null && i == index ? label : null, dictionary, fastDictionary);

                if (!string.IsNullOrWhiteSpace(adfExample))
                {
                    stringExample.AppendLine(adfExample);
                }

                i++;
            }

            return(stringExample.ToString());
        }
Esempio n. 22
0
 /// <summary>
 /// Initializes a new instance of the <see cref="VowpalWabbit{TExample}"/> class.
 /// </summary>
 /// <param name="vw">The native instance to wrap.</param>
 /// <remarks>This instance takes ownership of <paramref name="vw"/> instance and disposes it.</remarks>
 public VowpalWabbit(VowpalWabbit vw) : this(vw, VowpalWabbitSerializerFactory.CreateSerializer <TExample>(vw.Settings))
 {
 }