public async Task<HttpResponseMessage> Post()
{
var header = this.Request.Headers.SingleOrDefault(x => x.Key == "Authorization");
if (header.Value == null)
throw new UnauthorizedAccessException("AuthorizationToken missing");
var userToken = header.Value.First();
if (string.IsNullOrWhiteSpace(userToken) || userToken != ConfigurationManager.AppSettings["UserToken"])
return Request.CreateResponse(HttpStatusCode.Unauthorized);
if (this.metaData == null || lastDownload + TimeSpan.FromMinutes(1) < DateTime.Now)
{
var url = ConfigurationManager.AppSettings["DecisionServiceSettingsUrl"];
this.metaData = ApplicationMetadataUtil.DownloadMetadata<ApplicationClientMetadata>(url);
lastDownload = DateTime.Now;
}
using (var vw = new VowpalWabbit(new VowpalWabbitSettings(metaData.TrainArguments)
{
EnableStringExampleGeneration = true,
EnableStringFloatCompact = true
}))
using (var serializer = new VowpalWabbitJsonSerializer(vw))
using (var example = serializer.ParseAndCreate(new JsonTextReader(new StreamReader(await Request.Content.ReadAsStreamAsync()))))
{
return Request.CreateResponse(HttpStatusCode.OK, example.VowpalWabbitString);
}
}
/// <summary>
/// Initializes a new instance of <see cref="VowpalWabbitJsonBuilder"/>.
/// </summary>
public VowpalWabbitJsonBuilder(VowpalWabbitJsonSerializer serializer, IVowpalWabbitExamplePool vwPool, VowpalWabbitDefaultMarshaller defaultMarshaller, JsonSerializer jsonSerializer, int multiIndex = -1)
{
Contract.Requires(serializer != null);
Contract.Requires(vw != null);
Contract.Requires(defaultMarshaller != null);
Contract.Requires(jsonSerializer != null);
this.extensionState = new VowpalWabbitJsonParseState
{
JsonBuilder = this,
VW = vwPool.Native,
MultiIndex = multiIndex
};
this.namespaceStrings = new List <string>();
this.foundMulti = false;
if (serializer != null)
{
this.referenceResolver = serializer.ReferenceResolver;
}
this.serializer = serializer;
this.vw = vwPool.Native;
this.defaultMarshaller = defaultMarshaller;
this.jsonSerializer = jsonSerializer;
this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw);
}
public void Validate(string line, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null)
{
using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw))
using (var jsonExample = jsonSerializer.ParseAndCreate(json, label))
{
this.Validate(line, jsonExample, labelComparator, label);
}
}
public void TestJsonFeatureExtraction()
{
string json = "{\"ns1\":{\"location\":\"New York\", \"f2\":3.4}}";
using (var vw = new VowpalWabbit("-b 3 --noconstant"))
using (var serializer = new VowpalWabbitJsonSerializer(vw))
using (var result = serializer.ParseAndCreate(json))
{
var singleExample = result as VowpalWabbitSingleLineExampleCollection;
Assert.IsNotNull(singleExample);
if (singleExample != null)
{
foreach (var ns in singleExample.Example)
{
Console.WriteLine(ns.Index);
foreach (var feature in ns)
{
Console.WriteLine("{0}:{1}", feature.WeightIndex, feature.X);
}
}
var ns1 = singleExample.Example.ToArray();
Assert.AreEqual(1, ns1.Length);
Assert.AreEqual((byte)'n', ns1[0].Index);
CollectionAssert.AreEqual(
new[] {
new VowpalWabbitFeature(1, 12),
new VowpalWabbitFeature(3.4f, 28)
},
ns1[0].ToArray());
}
// for documentation purpose only
var multiExample = result as VowpalWabbitMultiLineExampleCollection;
Assert.IsNull(multiExample);
if (multiExample != null)
{
foreach (var example in multiExample.Examples)
{
foreach (var ns in example)
{
Console.WriteLine(ns.Index);
foreach (var feature in ns)
{
Console.WriteLine("{0}:{1}", feature.WeightIndex, feature.X);
}
}
}
}
}
}
internal void Resolve(VowpalWabbitJsonSerializer serializer, string id, Action <IVowpalWabbitMarshalAction> resolveAction)
{
IVowpalWabbitMarshalAction marshal;
lock (this.lockObject)
{
marshal = (IVowpalWabbitMarshalAction)this.cache.Get(id);
if (marshal == null)
{
// not found, register for delayed completion
var requests = (List <IncompleteReferenceRequest>) this.cacheRequests.Get(id);
if (requests == null)
{
var policy = this.cacheRequestItemPolicyFactory(id);
// dispatch to original handler too
var removeHandler = policy.RemovedCallback;
if (removeHandler == null)
{
policy.RemovedCallback = this.CacheEntryRemovedCallback;
}
else
{
policy.RemovedCallback = args => { removeHandler(args); this.CacheEntryRemovedCallback(args); }
};
requests = new List <IncompleteReferenceRequest>();
this.cacheRequests.Add(
new CacheItem(id, requests),
policy);
}
requests.Add(
new IncompleteReferenceRequest
{
Serializer = serializer,
Marshal = resolveAction
});
this.numberOfOpenRequests++;
serializer.IncreaseUnresolved();
return;
}
}
// avoid extensive locking
resolveAction(marshal);
}
public void Validate(string line, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null)
{
using (var strExample = this.vw.ParseLine(line))
using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw))
using (var jsonExample = (VowpalWabbitSingleLineExampleCollection)jsonSerializer.ParseAndCreate(json, label))
using (var strJsonExample = this.vw.ParseLine(jsonExample.Example.VowpalWabbitString))
{
var diff = strExample.Diff(this.vw, jsonExample.Example, labelComparator);
Assert.IsNull(diff, diff + " generated string: '" + jsonExample.Example.VowpalWabbitString + "'");
diff = strExample.Diff(this.vw, strJsonExample, labelComparator);
Assert.IsNull(diff, diff);
}
}
/// <summary>
/// Initializes a new instance of <see cref="VowpalWabbitJsonBuilder"/>.
/// </summary>
public VowpalWabbitJsonBuilder(VowpalWabbitJsonSerializer serializer, IVowpalWabbitExamplePool vwPool, VowpalWabbitDefaultMarshaller defaultMarshaller, JsonSerializer jsonSerializer)
{
Contract.Requires(serializer != null);
Contract.Requires(vw != null);
Contract.Requires(defaultMarshaller != null);
Contract.Requires(jsonSerializer != null);
this.namespaceStrings = new List <string>();
this.referenceResolver = serializer.ReferenceResolver;
this.serializer = serializer;
this.vw = vwPool.Native;
this.defaultMarshaller = defaultMarshaller;
this.jsonSerializer = jsonSerializer;
this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw);
}
/// <summary>
/// Initializes a new instance of <see cref="VowpalWabbitJsonBuilder"/>.
/// </summary>
public VowpalWabbitJsonBuilder(VowpalWabbitJsonSerializer serializer, IVowpalWabbitExamplePool vwPool, VowpalWabbitDefaultMarshaller defaultMarshaller, JsonSerializer jsonSerializer)
{
Contract.Requires(serializer != null);
Contract.Requires(vw != null);
Contract.Requires(defaultMarshaller != null);
Contract.Requires(jsonSerializer != null);
this.namespaceStrings = new List<string>();
this.foundMulti = false;
this.referenceResolver = serializer.ReferenceResolver;
this.serializer = serializer;
this.vw = vwPool.Native;
this.defaultMarshaller = defaultMarshaller;
this.jsonSerializer = jsonSerializer;
this.DefaultNamespaceContext = new VowpalWabbitMarshalContext(this.vw);
}
public void Validate(string[] lines, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null, int? index = null, VowpalWabbitJsonExtension extension = null)
{
VowpalWabbitExample[] strExamples = new VowpalWabbitExample[lines.Count()];
try
{
for (int i = 0; i < lines.Length; i++)
strExamples[i] = this.vw.ParseLine(lines[i]);
using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw))
{
if (extension != null)
jsonSerializer.RegisterExtension(extension);
using (var jsonExample = (VowpalWabbitMultiLineExampleCollection)jsonSerializer.ParseAndCreate(json, label, index))
{
var jsonExamples = new List<VowpalWabbitExample>();
if (jsonExample.SharedExample != null)
jsonExamples.Add(jsonExample.SharedExample);
jsonExamples.AddRange(jsonExample.Examples);
Assert.AreEqual(strExamples.Length, jsonExamples.Count);
for (int i = 0; i < strExamples.Length; i++)
{
using (var strJsonExample = this.vw.ParseLine(jsonExamples[i].VowpalWabbitString))
{
var diff = strExamples[i].Diff(this.vw, jsonExamples[i], labelComparator);
Assert.IsNull(diff, diff + " generated string: '" + jsonExamples[i].VowpalWabbitString + "'");
diff = strExamples[i].Diff(this.vw, strJsonExample, labelComparator);
Assert.IsNull(diff, diff);
}
}
}
}
}
finally
{
foreach (var ex in strExamples)
if (ex != null)
ex.Dispose();
}
}
public void TestJsonLabelExtraction()
{
using (var vw = new VowpalWabbit("--cb_adf --rank_all"))
{
using (var jsonSerializer = new VowpalWabbitJsonSerializer(vw))
{
string eventId = null;
jsonSerializer.RegisterExtension((state, property) =>
{
Assert.AreEqual(property, "_eventid");
Assert.IsTrue(state.Reader.Read());
eventId = (string)state.Reader.Value;
return true;
});
jsonSerializer.Parse("{\"_eventid\":\"abc123\",\"a\":1,\"_label_cost\":-1,\"_label_probability\":0.3}");
Assert.AreEqual("abc123", eventId);
using (var examples = jsonSerializer.CreateExamples())
{
var single = examples as VowpalWabbitSingleLineExampleCollection;
Assert.IsNotNull(single);
var label = single.Example.Label as ContextualBanditLabel;
Assert.IsNotNull(label);
Assert.AreEqual(-1, label.Cost);
Assert.AreEqual(0.3, label.Probability, 0.0001);
}
}
using (var jsonSerializer = new VowpalWabbitJsonSerializer(vw))
{
jsonSerializer.Parse("{\"_multi\":[{\"_text\":\"w1 w2\", \"a\":{\"x\":1}}, {\"_text\":\"w2 w3\"}], \"_labelindex\":1, \"_label_cost\":-1, \"_label_probability\":0.3}");
using (var examples = jsonSerializer.CreateExamples())
{
var multi = examples as VowpalWabbitMultiLineExampleCollection;
Assert.IsNotNull(multi);
Assert.AreEqual(2, multi.Examples.Length);
var label = multi.Examples[0].Label as ContextualBanditLabel;
Assert.AreEqual(0, label.Cost);
Assert.AreEqual(0, label.Probability);
label = multi.Examples[1].Label as ContextualBanditLabel;
Assert.IsNotNull(label);
Assert.AreEqual(-1, label.Cost);
Assert.AreEqual(0.3, label.Probability, 0.0001);
}
}
}
}
internal VowpalWabbitExampleJsonValidator(VowpalWabbitSettings settings)
{
this.vw = new VowpalWabbit(settings.ShallowCopy(enableStringExampleGeneration: true));
this.jsonSerializer = new VowpalWabbitJsonSerializer(this.vw);
}
private void DelayedExampleCallback(VowpalWabbitJsonSerializer serializer)
{
try
{
this.perfCounters.Feature_Requests_Pending.IncrementBy(-1);
var data = (PipelineData)serializer.UserContext;
data.Example = serializer.CreateExamples();
// fire and forget
// must not block to avoid dead lock
this.trainProcessorFactory.LearnBlock
.SendAsync(data)
.ContinueWith(async ret =>
{
if (!await ret)
{
this.telemetry.TrackTrace("Unable to enqueue delayed examples", SeverityLevel.Error);
// since we couldn't enqueue, need to dispose here
data.Example.Dispose();
}
});
}
catch (Exception e)
{
this.telemetry.TrackException(e);
}
finally
{
serializer.Dispose();
}
}
internal void Resolve(VowpalWabbitJsonSerializer serializer, string id, Action<IVowpalWabbitMarshalAction> resolveAction)
{
IVowpalWabbitMarshalAction marshal;
lock (this.lockObject)
{
marshal = (IVowpalWabbitMarshalAction)this.cache.Get(id);
if (marshal == null)
{
// not found, register for delayed completion
var requests = (List<IncompleteReferenceRequest>)this.cacheRequests.Get(id);
if (requests == null)
{
var policy = this.cacheRequestItemPolicyFactory(id);
// dispatch to original handler too
var removeHandler = policy.RemovedCallback;
if (removeHandler == null)
policy.RemovedCallback = this.CacheEntryRemovedCallback;
else
policy.RemovedCallback = args => { removeHandler(args); this.CacheEntryRemovedCallback(args); };
requests = new List<IncompleteReferenceRequest>();
this.cacheRequests.Add(
new CacheItem(id, requests),
policy);
}
requests.Add(
new IncompleteReferenceRequest
{
Serializer = serializer,
Marshal = resolveAction
});
this.numberOfOpenRequests++;
serializer.IncreaseUnresolved();
return;
}
}
// avoid extensive locking
resolveAction(marshal);
}
private IEnumerable<PipelineData> Stage1_Deserialize(PipelineData data)
{
try
{
using (var jsonReader = new JsonTextReader(new StringReader(data.JSON)))
{
//jsonReader.FloatParser = Util.ReadDoubleString;
// jsonReader.ArrayPool = pool;
VowpalWabbitJsonSerializer vwJsonSerializer = null;
try
{
vwJsonSerializer = new VowpalWabbitJsonSerializer(this.trainer.VowpalWabbit, this.trainer.ReferenceResolver);
vwJsonSerializer.RegisterExtension((state, property) =>
{
if (property.Equals("_eventid", StringComparison.OrdinalIgnoreCase))
{
if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.String)
throw new VowpalWabbitJsonException(state.Reader, "Expected string");
data.EventId = (string)state.Reader.Value;
return true;
}
else if (property.Equals("_timestamp", StringComparison.OrdinalIgnoreCase))
{
if (!state.Reader.Read() && state.Reader.TokenType != JsonToken.Date)
throw new VowpalWabbitJsonException(state.Reader, "Expected date");
data.Timestamp = (DateTime)state.Reader.Value;
}
return false;
});
data.Example = vwJsonSerializer.ParseAndCreate(jsonReader);
if (data.Example == null)
{
// unable to create example due to missing data
// will be trigger later
vwJsonSerializer.UserContext = data.Example;
// make sure the serialize is not deallocated
vwJsonSerializer = null;
}
}
finally
{
if (vwJsonSerializer != null)
vwJsonSerializer.Dispose();
}
performanceCounters.Stage1_JSON_DeserializePerSec.Increment();
// delayed
if (data.Example == null)
{
this.performanceCounters.Feature_Requests_Pending.Increment();
yield break;
}
}
}
catch (Exception ex)
{
this.telemetry.TrackException(ex, new Dictionary<string, string> { { "JSON", data.JSON } });
this.performanceCounters.Stage2_Faulty_Examples_Total.Increment();
this.performanceCounters.Stage2_Faulty_ExamplesPerSec.Increment();
yield break;
}
yield return data;
}
