private void WrapInNamespace(List <VowpalWabbitJsonParseContext> path, string namespaceValue, Action <VowpalWabbitJsonParseContext> action) { VowpalWabbitJsonParseContext parseContext = null; VowpalWabbitMarshalContext marshalContext = null; try { var ns = new Namespace(this.vw, namespaceValue); marshalContext = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder); parseContext = new VowpalWabbitJsonParseContext { Namespace = ns, Context = marshalContext, JsonProperty = namespaceValue }; // the namespace is only added on dispose, to be able to check if at least a single feature has been added marshalContext.NamespaceBuilder = marshalContext.ExampleBuilder.AddNamespace(ns.FeatureGroup); var position = 0; var stringExample = marshalContext.StringExample; if (marshalContext.StringExample != null) { position = stringExample.Append(ns.NamespaceString).Length; } path.Add(parseContext); action(parseContext); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = marshalContext.ToString(); if (str.Length > 0) { this.namespaceStrings.Add(str); } } this.featureCount += (int)marshalContext.NamespaceBuilder.FeatureCount; } finally { path.RemoveAt(path.Count - 1); if (marshalContext.NamespaceBuilder != null) { marshalContext.NamespaceBuilder.Dispose(); marshalContext.NamespaceBuilder = null; } if (parseContext != null && parseContext.Context != null) { parseContext.Context.Dispose(); parseContext.Context = null; } } }
/// <summary> /// Serializes the given <paramref name="example"/> to VW string format. /// </summary> /// <param name="example">The example to serialize.</param> /// <param name="label">The label to serialize.</param> /// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param> /// <param name="dictionary">Dictionary used for dictify operation.</param> /// <param name="fastDictionary">Dictionary used for dictify operation.</param> /// <returns>The resulting VW string.</returns> public string SerializeToString(TExample example, ILabel label = null, int?index = null, Dictionary <string, string> dictionary = null, Dictionary <object, string> fastDictionary = null) { Contract.Requires(example != null); using (var context = new VowpalWabbitMarshalContext(vw, dictionary, fastDictionary)) { this.serializerFunc(context, example, label); return(context.ToString()); } }
/// <summary> /// Expects that actual feature value. /// </summary> private void ParseFeature(List <VowpalWabbitJsonParseContext> path, string featureName) { switch (featureName) { case "$id": { if (this.referenceResolver == null) { return; } var id = (string)reader.Value; if (!reader.Read() || reader.TokenType != JsonToken.PropertyName || (string)reader.Value != "$values") { throw new VowpalWabbitJsonException(this.reader, "Expecting '$values' property"); } // read $values if (!reader.Read()) { throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); } // create re-useable marshalling call var marshalAction = this.ParseFeatureReUsable(); // keep action for re-use this.referenceResolver.AddReference(id, marshalAction); // go up 2 levels to find actual namespace, the last one is actually the property we want to serialize featureName = path.Last().JsonProperty; var context = path[path.Count - 2]; marshalAction.Marshal(this.defaultMarshaller, context.Context, context.Namespace, featureName); } return; case "$ref": { if (this.referenceResolver == null || this.serializer == null) { return; } var id = (string)reader.Value; // go up 2 levels to find actual namespace, the last one is actually the property we want to serialize featureName = path.Last().JsonProperty; var ns = path[path.Count - 2].Namespace; this.referenceResolver.Resolve( this.serializer, id, marshalAction => { // setup fresh context using (var context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder)) { this.featureCount += this.defaultMarshaller.MarshalNamespace( context, ns, () => marshalAction.Marshal(this.defaultMarshaller, context, ns, featureName)); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = context.ToString(); if (str.Length > 0) { this.namespaceStrings.Add(str); } } } }); } return; } var localContext = path.Last(); this.ParseFeature(localContext.Context, localContext.Namespace, featureName); }
/// <summary> /// Expects that actual feature value. /// </summary> private void ParseFeature(List<VowpalWabbitJsonParseContext> path, string featureName) { switch (featureName) { case "$id": { if (this.referenceResolver == null) return; var id = (string)reader.Value; if (!reader.Read() || reader.TokenType != JsonToken.PropertyName || (string)reader.Value != "$values") throw new VowpalWabbitJsonException(this.reader, "Expecting '$values' property"); // read $values if (!reader.Read()) throw new VowpalWabbitJsonException(this.reader, "Unexpected end"); // create re-useable marshalling call var marshalAction = this.ParseFeatureReUsable(); // keep action for re-use this.referenceResolver.AddReference(id, marshalAction); // go up 2 levels to find actual namespace, the last one is actually the property we want to serialize featureName = path.Last().JsonProperty; var context = path[path.Count - 2]; marshalAction.Marshal(this.defaultMarshaller, context.Context, context.Namespace, featureName); } return; case "$ref": { if (this.referenceResolver == null) return; var id = (string)reader.Value; // go up 2 levels to find actual namespace, the last one is actually the property we want to serialize featureName = path.Last().JsonProperty; var ns = path[path.Count - 2].Namespace; this.referenceResolver.Resolve( this.serializer, id, marshalAction => { // setup fresh context using (var context = new VowpalWabbitMarshalContext(this.vw, this.DefaultNamespaceContext.ExampleBuilder)) { this.featureCount += this.defaultMarshaller.MarshalNamespace( context, ns, () => marshalAction.Marshal(this.defaultMarshaller, context, ns, featureName)); // append default namespaces features if we found some if (this.vw.Settings.EnableStringExampleGeneration) { var str = context.ToString(); if (str.Length > 0) this.namespaceStrings.Add(str); } } }); } return; } var localContext = path.Last(); this.ParseFeature(localContext.Context, localContext.Namespace, featureName); }
/// <summary> /// Serialize the example. /// </summary> /// <param name="example">The example to serialize.</param> /// <param name="label">The label to be serialized.</param> /// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param> /// <returns>The serialized example.</returns> /// <remarks>If <typeparamref name="TExample"/> is annotated using the Cachable attribute, examples are returned from cache.</remarks> public VowpalWabbitExample Serialize(TExample example, ILabel label = null, int?index = null) { Contract.Requires(example != null); Contract.Requires(index == null); if (this.exampleCache == null || label != null) { using (var context = new VowpalWabbitMarshalContext(vw)) { this.serializerFunc(context, example, label); var vwExample = context.ExampleBuilder.CreateExample(); if (this.EnableStringExampleGeneration) { vwExample.VowpalWabbitString = context.ToString(); } return(vwExample); } } CacheEntry result; if (this.exampleCache.TryGetValue(example, out result)) { result.LastRecentUse = DateTime.UtcNow; #if DEBUG if (result.InUse) { throw new ArgumentException("Cached example already in use."); } #endif } else { VowpalWabbitExample nativeExample = null; try { using (var context = new VowpalWabbitMarshalContext(this)) { this.serializerFunc(context, example, label); nativeExample = context.ExampleBuilder.CreateExample(); } result = new CacheEntry { Example = nativeExample, LastRecentUse = DateTime.UtcNow }; this.exampleCache.Add(example, result); #if DEBUG this.reverseLookup.Add(result.Example, result); #endif } catch (Exception e) { if (nativeExample != null) { nativeExample.Dispose(); } throw e; } } #if DEBUG result.InUse = true; #endif // TODO: support Label != null here and update cached example using new label return(result.Example); }
internal async Task TestAllReduceInternal() { var data = Enumerable.Range(1, 1000).Select(_ => Generator.GenerateShared(10)).ToList(); var stringSerializerCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfShared>) VowpalWabbitSerializerFactory.CreateSerializer<CbAdfShared>(new VowpalWabbitSettings { EnableStringExampleGeneration = true }); var stringSerializerAdfCompiler = (VowpalWabbitSingleExampleSerializerCompiler<CbAdfAction>) VowpalWabbitSerializerFactory.CreateSerializer<CbAdfAction>(new VowpalWabbitSettings { EnableStringExampleGeneration = true }); var stringData = new List<List<string>>(); VowpalWabbitPerformanceStatistics statsExpected; using (var spanningTree = new SpanningTreeClr()) { spanningTree.Start(); using (var vw1 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 1 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true })) using (var vw2 = new VowpalWabbit(new VowpalWabbitSettings(@"--total 2 --node 0 --unique_id 0 --span_server localhost --cb_adf --rank_all --interact xy") { EnableStringExampleGeneration = true } )) { var stringSerializer = stringSerializerCompiler.Func(vw1); var stringSerializerAdf = stringSerializerAdfCompiler.Func(vw1); // serialize foreach (var d in data) { var block = new List<string>(); using (var context = new VowpalWabbitMarshalContext(vw1)) { stringSerializer(context, d.Item1, SharedLabel.Instance); block.Add(context.ToString()); } block.AddRange(d.Item2.Select((a, i) => { using (var context = new VowpalWabbitMarshalContext(vw1)) { stringSerializerAdf(context, a, i == d.Item3.Action ? d.Item3 : null); return context.ToString(); } })); stringData.Add(block); } await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vw1, stringData.Take(500))), Task.Factory.StartNew(() => Ingest(vw2, stringData.Skip(500)))); vw1.SaveModel("expected.1.model"); vw2.SaveModel("expected.2.model"); statsExpected = vw1.PerformanceStatistics; } } // skip header var expected1Model = File.ReadAllBytes("expected.1.model").Skip(0x15).ToList(); var expected2Model = File.ReadAllBytes("expected.2.model").Skip(0x15).ToList(); var settings = new VowpalWabbitSettings("--cb_adf --rank_all --interact xy") { ParallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 2 }, ExampleCountPerRun = 2000, ExampleDistribution = VowpalWabbitExampleDistribution.RoundRobin }; using (var vw = new VowpalWabbitThreadedLearning(settings)) { await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vw, stringData.Take(500))), Task.Factory.StartNew(() => Ingest(vw, stringData.Skip(500)))); // important to enqueue the request before Complete() is called var statsTask = vw.PerformanceStatistics; var modelSave = vw.SaveModel("actual.model"); await vw.Complete(); var statsActual = await statsTask; VWTestHelper.AssertEqual(statsExpected, statsActual); await modelSave; // skip header var actualModel = File.ReadAllBytes("actual.model").Skip(0x15).ToList(); CollectionAssert.AreEqual(expected1Model, actualModel); CollectionAssert.AreEqual(expected2Model, actualModel); } using (var vw = new VowpalWabbitThreadedLearning(settings)) { var vwManaged = vw.Create<CbAdfShared, CbAdfAction>(); await Task.WhenAll( Task.Factory.StartNew(() => Ingest(vwManaged, data.Take(500))), Task.Factory.StartNew(() => Ingest(vwManaged, data.Skip(500)))); // important to enqueue the request before Complete() is called var statsTask = vw.PerformanceStatistics; var modelSave = vw.SaveModel("actual.managed.model"); await vw.Complete(); var statsActual = await statsTask; VWTestHelper.AssertEqual(statsExpected, statsActual); await modelSave; // skip header var actualModel = File.ReadAllBytes("actual.managed.model").Skip(0x15).ToList(); CollectionAssert.AreEqual(expected1Model, actualModel); CollectionAssert.AreEqual(expected2Model, actualModel); } }