/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbitSingleLineExampleCollection"/> class.
/// </summary>
public VowpalWabbitSingleLineExampleCollection(VowpalWabbit vw, VowpalWabbitExample example)
: base(vw)
{
Contract.Requires(example != null);
this.Example = example;
}
public void ReturnExampleToPool(VowpalWabbitExample example)
{
lock (this.poolLock)
{
this.pool.ReturnExampleToPool(example);
}
}
public void ReturnExampleToPool(VowpalWabbitExample example)
{
lock (this.poolLock)
{
this.pool.ReturnExampleToPool(example);
}
}
public void Validate(string[] lines, List <VowpalWabbitExample> examples, IVowpalWabbitLabelComparator labelComparator = null)
{
VowpalWabbitExample[] strExamples = new VowpalWabbitExample[lines.Count()];
try
{
for (int i = 0; i < lines.Length; i++)
{
strExamples[i] = this.vw.ParseLine(lines[i]);
}
for (int i = 0; i < strExamples.Length; i++)
{
var diff = strExamples[i].Diff(this.vw, examples[i], labelComparator);
Assert.IsNull(diff, diff + " generated string: '" + strExamples[i].VowpalWabbitString + "'");
}
}
finally
{
foreach (var ex in strExamples)
{
if (ex != null)
{
ex.Dispose();
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbitMultiLineExampleCollection"/> class.
/// </summary>
public VowpalWabbitMultiLineExampleCollection(VowpalWabbit vw, VowpalWabbitExample shared, VowpalWabbitExample[] examples)
: base(vw)
{
Contract.Requires(examples != null);
this.SharedExample = shared;
this.Examples = examples;
}
public void Validate(string[] lines, JsonReader jsonReader, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null, int?index = null, VowpalWabbitJsonExtension extension = null)
{
VowpalWabbitExample[] strExamples = new VowpalWabbitExample[lines.Count()];
try
{
for (int i = 0; i < lines.Length; i++)
{
strExamples[i] = this.vw.ParseLine(lines[i]);
}
using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw))
{
if (extension != null)
{
jsonSerializer.RegisterExtension(extension);
// extension are not supported with native JSON parsing
}
using (var jsonExample = (VowpalWabbitMultiLineExampleCollection)jsonSerializer.ParseAndCreate(jsonReader, label, index))
{
var jsonExamples = new List <VowpalWabbitExample>();
if (jsonExample.SharedExample != null)
{
jsonExamples.Add(jsonExample.SharedExample);
}
jsonExamples.AddRange(jsonExample.Examples);
Assert.AreEqual(strExamples.Length, jsonExamples.Count);
for (int i = 0; i < strExamples.Length; i++)
{
using (var strJsonExample = this.vw.ParseLine(jsonExamples[i].VowpalWabbitString))
{
var diff = strExamples[i].Diff(this.vw, jsonExamples[i], labelComparator);
Assert.IsNull(diff, diff + " generated string: '" + jsonExamples[i].VowpalWabbitString + "'");
diff = strExamples[i].Diff(this.vw, strJsonExample, labelComparator);
Assert.IsNull(diff, diff);
}
}
}
}
}
finally
{
foreach (var ex in strExamples)
{
if (ex != null)
{
ex.Dispose();
}
}
}
}
internal static void Validate(string line, VowpalWabbitExample ex, IVowpalWabbitLabelComparator comparator, string args = null)
{
using (var vw = new VowpalWabbit(args))
using (var strExample = vw.ParseLine(line))
{
var diff = strExample.Diff(vw, ex, comparator);
Assert.IsNull(diff, diff + " generated string: '" + ex.VowpalWabbitString + "'");
}
}
internal static void Validate(string line, VowpalWabbitExample ex, IVowpalWabbitLabelComparator comparator, string args = null)
{
using (var vw = new VowpalWabbit(args))
using (var strExample = vw.ParseLine(line))
{
var diff = strExample.Diff(vw, ex, comparator);
Assert.IsNull(diff, diff + " generated string: '" + ex.VowpalWabbitString + "'");
}
}
public VowpalWabbitMultiLineExampleCollection CreateExample(VowpalWabbit vw)
{
VowpalWabbitDecisionServiceInteractionHeader header;
var examples = vw.ParseDecisionServiceJson(this.exampleBuffer, 0, this.length, true, out header);
var adf = new VowpalWabbitExample[examples.Count - 1];
examples.CopyTo(1, adf, 0, examples.Count - 1);
return(new VowpalWabbitMultiLineExampleCollection(vw, examples[0], adf));
}
private void Initialize(VowpalWabbitSerializerSettings settings)
{
var visitor = new VowpalWabbitInterfaceVisitor(this);
this.actionDependentFeatureSerializer = VowpalWabbitSerializerFactory.CreateSerializer <TActionDependentFeature>(visitor, settings);
if (this.actionDependentFeatureSerializer == null)
{
throw new ArgumentException(typeof(TActionDependentFeature) + " must have a least a single [Feature] defined.");
}
using (var exBuilder = new VowpalWabbitExampleBuilder(this))
{
this.emptyExample = exBuilder.CreateExample();
}
}
/// <summary>
/// Calls learn or predict for the set of examples. Does required filtering of potential new line examples.
/// </summary>
private TPrediction Execute <TPrediction>(VowpalWabbit vw, Action <List <VowpalWabbitExample> > predictOrLearn, IVowpalWabbitPredictionFactory <TPrediction> predictionFactory = null)
{
Contract.Requires(predictOrLearn != null);
// firstExample will contain prediction result
VowpalWabbitExample firstExample = null;
VowpalWabbitExample empty = null;
try
{
var ecCol = new List <VowpalWabbitExample>();
if (this.SharedExample != null && !this.SharedExample.IsNewLine)
{
firstExample = this.SharedExample;
ecCol.Add(firstExample);
}
foreach (var ex in this.Examples)
{
if (!ex.IsNewLine)
{
ecCol.Add(ex);
if (firstExample == null)
{
firstExample = ex;
}
}
}
// signal end-of-block
empty = vw.GetOrCreateNativeExample();
empty.MakeEmpty(vw);
predictOrLearn(ecCol);
return(predictionFactory != null?firstExample.GetPrediction(vw, predictionFactory) : default(TPrediction));
}
finally
{
if (empty != null)
{
empty.Dispose();
}
}
}
/// <summary>
/// Accepts an example back into this pool.
/// </summary>
/// <param name="example">The example to be returned.</param>
public void ReturnExampleToPool(VowpalWabbitExample example)
{
if (this.exampleCache == null)
{
throw new ObjectDisposedException("VowpalWabbitSerializer");
}
#if DEBUG
CacheEntry cacheEntry;
if (!this.reverseLookup.TryGetValue(example, out cacheEntry))
{
throw new ArgumentException("Example is not found in pool");
}
if (!cacheEntry.InUse)
{
throw new ArgumentException("Unused example returned");
}
cacheEntry.InUse = false;
#endif
// if we reach the cache boundary, dispose the oldest example
if (this.exampleCache.Count > this.vw.Settings.MaxExampleCacheSize)
{
var enumerator = this.exampleCache.GetEnumerator();
// this.settings.MaxExampleCacheSize is >= 1
enumerator.MoveNext();
var min = enumerator.Current;
while (enumerator.MoveNext())
{
if (min.Value.LastRecentUse > enumerator.Current.Value.LastRecentUse)
{
min = enumerator.Current;
}
}
#if DEBUG
this.reverseLookup.Remove(min.Value.Example);
#endif
this.exampleCache.Remove(min.Key);
min.Value.Example.InnerExample.Dispose();
}
}
/// <summary>
/// Cleanup.
/// </summary>
/// <param name="isDiposing">See IDiposable pattern.</param>
protected override void Dispose(bool isDiposing)
{
if (isDiposing)
{
if (this.actionDependentFeatureSerializer != null)
{
this.actionDependentFeatureSerializer.Dispose();
this.actionDependentFeatureSerializer = null;
}
if (this.emptyExample != null)
{
this.emptyExample.Dispose();
this.emptyExample = null;
}
}
base.Dispose(isDiposing);
}
public void Validate(string[] lines, string json, IVowpalWabbitLabelComparator labelComparator = null, ILabel label = null, int? index = null, VowpalWabbitJsonExtension extension = null)
{
VowpalWabbitExample[] strExamples = new VowpalWabbitExample[lines.Count()];
try
{
for (int i = 0; i < lines.Length; i++)
strExamples[i] = this.vw.ParseLine(lines[i]);
using (var jsonSerializer = new VowpalWabbitJsonSerializer(this.vw))
{
if (extension != null)
jsonSerializer.RegisterExtension(extension);
using (var jsonExample = (VowpalWabbitMultiLineExampleCollection)jsonSerializer.ParseAndCreate(json, label, index))
{
var jsonExamples = new List<VowpalWabbitExample>();
if (jsonExample.SharedExample != null)
jsonExamples.Add(jsonExample.SharedExample);
jsonExamples.AddRange(jsonExample.Examples);
Assert.AreEqual(strExamples.Length, jsonExamples.Count);
for (int i = 0; i < strExamples.Length; i++)
{
using (var strJsonExample = this.vw.ParseLine(jsonExamples[i].VowpalWabbitString))
{
var diff = strExamples[i].Diff(this.vw, jsonExamples[i], labelComparator);
Assert.IsNull(diff, diff + " generated string: '" + jsonExamples[i].VowpalWabbitString + "'");
diff = strExamples[i].Diff(this.vw, strJsonExample, labelComparator);
Assert.IsNull(diff, diff);
}
}
}
}
}
finally
{
foreach (var ex in strExamples)
if (ex != null)
ex.Dispose();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="VowpalWabbitMultiLineExampleCollection"/> class.
/// </summary>
public VowpalWabbitMultiLineExampleCollection(VowpalWabbit vw, VowpalWabbitExample shared, VowpalWabbitExample[] examples)
: base(vw)
{
Contract.Requires(examples != null);
this.SharedExample = shared;
this.Examples = examples;
if (shared != null)
{
numberOfFeatures += shared.NumberOfFeatures;
}
foreach (var e in examples)
{
numberOfFeatures += e.NumberOfFeatures;
}
}
public VowpalWabbitExampleCollection Serialize(TExample example, Interfaces.ILabel label = null, int?index = null)
{
VowpalWabbitExample shared = null;
var adf = new List <VowpalWabbitExample>();
try
{
if (this.sharedSerializer != null)
{
shared = this.sharedSerializer.Serialize(example, SharedLabel.Instance);
}
var adfCollection = this.adfAccessor(example);
if (adfCollection != null)
{
var i = 0;
foreach (var adfExample in adfCollection)
{
adf.Add(this.adfSerializer.Serialize(adfExample, index != null && i == index ? label : null));
i++;
}
}
return(new VowpalWabbitMultiLineExampleCollection(this.vw, shared, adf.ToArray()));
}
catch (Exception)
{
if (shared != null)
{
shared.Dispose();
}
foreach (var adfExample in adf)
{
adfExample.Dispose();
}
throw;
}
}
/// <summary>
/// Creates the examples ready for learning or prediction.
/// </summary>
public VowpalWabbitExampleCollection CreateExamplesInternal()
{
try
{
if (this.ExampleBuilders == null)
{
return(new VowpalWabbitSingleLineExampleCollection(this.vwPool.Native, this.ExampleBuilder.CreateExample()));
}
else
{
// making sure we don't leak memory
VowpalWabbitExample sharedExample = null;
var examples = new VowpalWabbitExample[this.ExampleBuilders.Count];
try
{
// mark shared example as shared
this.defaultMarshaller.MarshalLabel(this.ExampleBuilder.DefaultNamespaceContext, SharedLabel.Instance);
sharedExample = this.ExampleBuilder.CreateExample();
for (int i = 0; i < this.ExampleBuilders.Count; i++)
{
examples[i] = this.ExampleBuilders[i].CreateExample();
}
return(new VowpalWabbitMultiLineExampleCollection(this.vwPool.Native, sharedExample, examples));
}
catch (Exception)
{
if (sharedExample != null)
{
sharedExample.Dispose();
}
foreach (var e in examples)
{
if (e != null)
{
e.Dispose();
}
}
throw;
}
}
}
finally
{
this.ExampleBuilder.Dispose();
this.ExampleBuilder = null;
if (this.ExampleBuilders != null)
{
foreach (var eb in this.ExampleBuilders)
{
eb.Dispose();
}
this.ExampleBuilders = null;
}
}
}
/// <summary>
/// Simplify learning of examples with action dependent features.
/// </summary>
/// <typeparam name="TExample">User example type.</typeparam>
/// <typeparam name="TActionDependentFeature">Action dependent feature type.</typeparam>
/// <param name="vw">The VowpalWabbit instances.</param>
/// <param name="serializer">The example serializer.</param>
/// <param name="actionDependentFeatureSerializer">The action dependent feature serializer.</param>
/// <param name="example">The example.</param>
/// <param name="actionDependentFeatures">The action dependent features.</param>
/// <param name="predictOrLearn">An action executed once the set of valid examples is determined. </param>
/// <param name="index">The optional index of the action dependent feature this label belongs too.</param>
/// <param name="label">The optional label to be used for learning or evaluation.</param>
public static void Execute <TExample, TActionDependentFeature>(
VowpalWabbit vw,
VowpalWabbitSingleExampleSerializer <TExample> serializer,
VowpalWabbitSingleExampleSerializer <TActionDependentFeature> actionDependentFeatureSerializer,
TExample example,
IReadOnlyCollection <TActionDependentFeature> actionDependentFeatures,
LearnOrPredictAction <TActionDependentFeature> predictOrLearn,
int?index = null,
ILabel label = null)
{
Contract.Requires(vw != null);
Contract.Requires(actionDependentFeatureSerializer != null);
Contract.Requires(example != null);
Contract.Requires(actionDependentFeatures != null);
var examples = new List <VowpalWabbitExample>(actionDependentFeatures.Count + 1);
var validExamples = new List <VowpalWabbitExample>(actionDependentFeatures.Count + 1);
var validActionDependentFeatures = new List <ActionDependentFeature <TActionDependentFeature> >(actionDependentFeatures.Count + 1);
var emptyActionDependentFeatures = new List <ActionDependentFeature <TActionDependentFeature> >(actionDependentFeatures.Count + 1);
VowpalWabbitExample emptyExample = null;
try
{
// contains prediction results
if (serializer != null)
{
var sharedExample = serializer.Serialize(example, SharedLabel.Instance);
// check if we have shared features
if (sharedExample != null)
{
examples.Add(sharedExample);
if (!sharedExample.IsNewLine)
{
validExamples.Add(sharedExample);
}
}
}
var i = 0;
foreach (var actionDependentFeature in actionDependentFeatures)
{
var adfExample = actionDependentFeatureSerializer.Serialize(actionDependentFeature,
index != null && i == index ? label : null);
Contract.Assert(adfExample != null);
examples.Add(adfExample);
if (!adfExample.IsNewLine)
{
validExamples.Add(adfExample);
validActionDependentFeatures.Add(new ActionDependentFeature <TActionDependentFeature>(i, actionDependentFeature));
}
else
{
emptyActionDependentFeatures.Add(new ActionDependentFeature <TActionDependentFeature>(i, actionDependentFeature));
}
i++;
}
if (validActionDependentFeatures.Count == 0)
{
return;
}
// signal we're finished using an empty example
emptyExample = vw.GetOrCreateNativeExample();
emptyExample.MakeEmpty(vw);
predictOrLearn(validExamples, validActionDependentFeatures, emptyActionDependentFeatures);
}
finally
{
if (emptyExample != null)
{
emptyExample.Dispose();
}
// dispose examples
// Note: must not dispose examples before final example
// as the learning algorithm (such as cbf) keeps a reference
// to the example
foreach (var e in examples)
{
e.Dispose();
}
}
}
internal VowpalWabbitDebugExample(VowpalWabbitExample example, string vwString) :
base(example.Owner, example)
{
this.VowpalWabbitString = vwString;
}
/// <summary>
/// Creates the examples ready for learning or prediction.
/// </summary>
public VowpalWabbitExampleCollection CreateExamplesInternal()
{
try
{
if (this.ExampleBuilders == null)
{
return new VowpalWabbitSingleLineExampleCollection(this.vwPool.Native, this.ExampleBuilder.CreateExample());
}
else
{
// making sure we don't leak memory
VowpalWabbitExample sharedExample = null;
var examples = new VowpalWabbitExample[this.ExampleBuilders.Count];
try
{
// mark shared example as shared
VowpalWabbitDefaultMarshaller.Instance.MarshalLabel(this.ExampleBuilder.DefaultNamespaceContext, SharedLabel.Instance);
sharedExample = this.ExampleBuilder.CreateExample();
for (int i = 0; i < this.ExampleBuilders.Count; i++)
examples[i] = this.ExampleBuilders[i].CreateExample();
return new VowpalWabbitMultiLineExampleCollection(this.vwPool.Native, sharedExample, examples);
}
catch (Exception)
{
if (sharedExample != null)
sharedExample.Dispose();
foreach (var e in examples)
if (e != null)
e.Dispose();
throw;
}
}
}
finally
{
this.ExampleBuilder.Dispose();
this.ExampleBuilder = null;
if (this.ExampleBuilders != null)
{
foreach (var eb in this.ExampleBuilders)
eb.Dispose();
this.ExampleBuilders = null;
}
}
}
/// <summary>
/// Serialize the example.
/// </summary>
/// <param name="example">The example to serialize.</param>
/// <param name="label">The label to be serialized.</param>
/// <param name="index">The optional index of the example, the <paramref name="label"/> should be attributed to.</param>
/// <returns>The serialized example.</returns>
/// <remarks>If <typeparamref name="TExample"/> is annotated using the Cachable attribute, examples are returned from cache.</remarks>
public VowpalWabbitExample Serialize(TExample example, ILabel label = null, int?index = null)
{
Contract.Requires(example != null);
Contract.Requires(index == null);
if (this.exampleCache == null || label != null)
{
using (var context = new VowpalWabbitMarshalContext(vw))
{
this.serializerFunc(context, example, label);
var vwExample = context.ExampleBuilder.CreateExample();
if (this.EnableStringExampleGeneration)
{
vwExample.VowpalWabbitString = context.ToString();
}
return(vwExample);
}
}
CacheEntry result;
if (this.exampleCache.TryGetValue(example, out result))
{
result.LastRecentUse = DateTime.UtcNow;
#if DEBUG
if (result.InUse)
{
throw new ArgumentException("Cached example already in use.");
}
#endif
}
else
{
VowpalWabbitExample nativeExample = null;
try
{
using (var context = new VowpalWabbitMarshalContext(this))
{
this.serializerFunc(context, example, label);
nativeExample = context.ExampleBuilder.CreateExample();
}
result = new CacheEntry
{
Example = nativeExample,
LastRecentUse = DateTime.UtcNow
};
this.exampleCache.Add(example, result);
#if DEBUG
this.reverseLookup.Add(result.Example, result);
#endif
}
catch (Exception e)
{
if (nativeExample != null)
{
nativeExample.Dispose();
}
throw e;
}
}
#if DEBUG
result.InUse = true;
#endif
// TODO: support Label != null here and update cached example using new label
return(result.Example);
}
internal VowpalWabbitDebugExample(VowpalWabbitExample example, string vwString)
: base(example.Owner, example)
{
this.VowpalWabbitString = vwString;
}
