/// <summary>
/// Evaluates the samples with a given number of partitions.
/// </summary>
/// <param name="samples">The samples to train and test.</param>
/// <param name="partitions">The number of folds.</param>
public void Evaluate(IObjectStream <NameSample> samples, int partitions)
{
// Note: The name samples need to be grouped on a document basis.
var partitioner = new CrossValidationPartitioner <DocumentSample>(new NameToDocumentSampleStream(samples),
partitions);
while (partitioner.HasNext)
{
var trainingSampleStream = partitioner.Next();
var model = NameFinderME.Train(
languageCode,
type,
new DocumentToNameSampleStream(trainingSampleStream),
parameters,
factory);
// do testing
var evaluator = new TokenNameFinderEvaluator(new NameFinderME(model), listeners);
evaluator.Evaluate(new DocumentToNameSampleStream(trainingSampleStream.GetTestSampleStream()));
FMeasure.MergeInto(evaluator.FMeasure);
}
}
/// <summary>
/// Processes the specified sample stream.
/// </summary>
/// <param name="sampleStream">The sample stream.</param>
/// <returns>The computed f-measure of the sample stream.</returns>
protected override FMeasure <Span> Process(CrossValidationPartitioner <Parse> .TrainingSampleStream sampleStream)
{
ParserModel model;
switch (parserType)
{
case ParserType.Chunking:
model = Chunking.Parser.Train(languageCode, sampleStream, headRules, parameters);
break;
case ParserType.TreeInsert:
model = TreeInsert.Parser.Train(languageCode, sampleStream, headRules, parameters);
break;
default:
throw new InvalidOperationException("Unexpected parser type.");
}
var evaluator = new ParserEvaluator(ParserFactory.Create(model), monitors);
evaluator.Evaluate(sampleStream);
return(evaluator.FMeasure);
}
/// <summary>
/// Evaluates the samples with a given number of partitions.
/// </summary>
/// <param name="samples">The samples to train and test.</param>
/// <param name="partitions">The number of folds.</param>
public void Evaluate(IObjectStream <T> samples, int partitions)
{
var partitioner = new CrossValidationPartitioner <T>(samples, partitions);
while (partitioner.HasNext)
{
var ps = partitioner.Next();
var fm = Process(ps);
FMeasure.MergeInto(fm);
}
}
/// <summary>
/// Evaluates the specified chunk samples.
/// </summary>
/// <param name="samples">The chunk samples to be evaluated.</param>
/// <param name="partitions">The partitions (folds).</param>
public void Evaluate(IObjectStream<ChunkSample> samples, int partitions) {
var partitioner = new CrossValidationPartitioner<ChunkSample>(samples, partitions);
while (partitioner.HasNext) {
var trainingSampleStream = partitioner.Next();
var model = ChunkerME.Train(languageCode, trainingSampleStream, parameters, chunkerFactory);
var evaluator = new ChunkerEvaluator(new ChunkerME(model), listeners);
evaluator.Evaluate(trainingSampleStream.GetTestSampleStream());
FMeasure.MergeInto(evaluator.FMeasure);
}
}
/// <summary>
/// Evaluates the specified chunk samples.
/// </summary>
/// <param name="samples">The chunk samples to be evaluated.</param>
/// <param name="partitions">The partitions (folds).</param>
public void Evaluate(IObjectStream <ChunkSample> samples, int partitions)
{
var partitioner = new CrossValidationPartitioner <ChunkSample>(samples, partitions);
while (partitioner.HasNext)
{
var trainingSampleStream = partitioner.Next();
var model = ChunkerME.Train(languageCode, trainingSampleStream, parameters, chunkerFactory);
var evaluator = new ChunkerEvaluator(new ChunkerME(model), listeners);
evaluator.Evaluate(trainingSampleStream.GetTestSampleStream());
FMeasure.MergeInto(evaluator.FMeasure);
}
}
public void TestFailSafety()
{
var partitioner = new CrossValidationPartitioner <String>(new[] {
"01", "02", "03", "04"
}, 4);
// Test that iterator from previous partition fails
// if it is accessed
var firstTraining = partitioner.Next();
Assert.AreEqual("02", firstTraining.Read());
var secondTraining = partitioner.Next();
try {
firstTraining.Read();
Assert.Fail();
} catch (Exception) {}
try {
firstTraining.GetTestSampleStream();
Assert.Fail();
} catch (Exception) {}
// Test that training iterator fails if there is a test iterator
secondTraining.GetTestSampleStream();
try {
secondTraining.Read();
Assert.Fail();
} catch (Exception) {}
// Test that test iterator from previous partition fails
// if there is a new partition
var thirdTraining = partitioner.Next();
var thirdTest = thirdTraining.GetTestSampleStream();
Assert.True(partitioner.HasNext);
partitioner.Next();
try {
thirdTest.Read();
Assert.Fail();
} catch (Exception) {}
}
public void TestEmptyDataSet()
{
var partitioner = new CrossValidationPartitioner <string>(new string[] {}, 2);
Assert.True(partitioner.HasNext);
Assert.Null(partitioner.Next().Read());
Assert.True(partitioner.HasNext);
Assert.Null(partitioner.Next().Read());
Assert.False(partitioner.HasNext);
try {
partitioner.Next();
Assert.Fail("Ups, hasn't thrown one!");
} catch (Exception) {
// expected
}
}
public void Test3FoldCv()
{
var partitioner = new CrossValidationPartitioner <string>(new[] {
"01", "02", "03", "04", "05", "06", "07", "08", "09", "10"
}, 3);
// first partition
Assert.True(partitioner.HasNext);
var firstTraining = partitioner.Next();
Assert.AreEqual("02", firstTraining.Read());
Assert.AreEqual("03", firstTraining.Read());
Assert.AreEqual("05", firstTraining.Read());
Assert.AreEqual("06", firstTraining.Read());
Assert.AreEqual("08", firstTraining.Read());
Assert.AreEqual("09", firstTraining.Read());
Assert.Null(firstTraining.Read());
var firstTest = firstTraining.GetTestSampleStream();
Assert.AreEqual("01", firstTest.Read());
Assert.AreEqual("04", firstTest.Read());
Assert.AreEqual("07", firstTest.Read());
Assert.AreEqual("10", firstTest.Read());
Assert.Null(firstTest.Read());
// second partition
Assert.True(partitioner.HasNext);
var secondTraining = partitioner.Next();
Assert.AreEqual("01", secondTraining.Read());
Assert.AreEqual("03", secondTraining.Read());
Assert.AreEqual("04", secondTraining.Read());
Assert.AreEqual("06", secondTraining.Read());
Assert.AreEqual("07", secondTraining.Read());
Assert.AreEqual("09", secondTraining.Read());
Assert.AreEqual("10", secondTraining.Read());
Assert.Null(secondTraining.Read());
var secondTest = secondTraining.GetTestSampleStream();
Assert.AreEqual("02", secondTest.Read());
Assert.AreEqual("05", secondTest.Read());
Assert.AreEqual("08", secondTest.Read());
Assert.Null(secondTest.Read());
// third partition
Assert.True(partitioner.HasNext);
var thirdTraining = partitioner.Next();
Assert.AreEqual("01", thirdTraining.Read());
Assert.AreEqual("02", thirdTraining.Read());
Assert.AreEqual("04", thirdTraining.Read());
Assert.AreEqual("05", thirdTraining.Read());
Assert.AreEqual("07", thirdTraining.Read());
Assert.AreEqual("08", thirdTraining.Read());
Assert.AreEqual("10", thirdTraining.Read());
Assert.Null(thirdTraining.Read());
var thirdTest = thirdTraining.GetTestSampleStream();
Assert.AreEqual("03", thirdTest.Read());
Assert.AreEqual("06", thirdTest.Read());
Assert.AreEqual("09", thirdTest.Read());
Assert.Null(thirdTest.Read());
Assert.False(partitioner.HasNext);
}
public void ToStringTest()
{
var value = new CrossValidationPartitioner <string>(new string[] {}, 10).ToString();
Assert.AreEqual("At partition 1 of 10.", value);
}
public void TestEmptyDataSet() {
var partitioner = new CrossValidationPartitioner<string>(new string[] {}, 2);
Assert.True(partitioner.HasNext);
Assert.Null(partitioner.Next().Read());
Assert.True(partitioner.HasNext);
Assert.Null(partitioner.Next().Read());
Assert.False(partitioner.HasNext);
try {
partitioner.Next();
Assert.Fail("Ups, hasn't thrown one!");
} catch (Exception) {
// expected
}
}
public void Test3FoldCV() {
var partitioner = new CrossValidationPartitioner<string>(new[] {
"01", "02", "03", "04", "05", "06", "07", "08", "09", "10"
}, 3);
// first partition
Assert.True(partitioner.HasNext);
var firstTraining = partitioner.Next();
Assert.AreEqual("02", firstTraining.Read());
Assert.AreEqual("03", firstTraining.Read());
Assert.AreEqual("05", firstTraining.Read());
Assert.AreEqual("06", firstTraining.Read());
Assert.AreEqual("08", firstTraining.Read());
Assert.AreEqual("09", firstTraining.Read());
Assert.Null(firstTraining.Read());
var firstTest = firstTraining.GetTestSampleStream();
Assert.AreEqual("01", firstTest.Read());
Assert.AreEqual("04", firstTest.Read());
Assert.AreEqual("07", firstTest.Read());
Assert.AreEqual("10", firstTest.Read());
Assert.Null(firstTest.Read());
// second partition
Assert.True(partitioner.HasNext);
var secondTraining = partitioner.Next();
Assert.AreEqual("01", secondTraining.Read());
Assert.AreEqual("03", secondTraining.Read());
Assert.AreEqual("04", secondTraining.Read());
Assert.AreEqual("06", secondTraining.Read());
Assert.AreEqual("07", secondTraining.Read());
Assert.AreEqual("09", secondTraining.Read());
Assert.AreEqual("10", secondTraining.Read());
Assert.Null(secondTraining.Read());
var secondTest = secondTraining.GetTestSampleStream();
Assert.AreEqual("02", secondTest.Read());
Assert.AreEqual("05", secondTest.Read());
Assert.AreEqual("08", secondTest.Read());
Assert.Null(secondTest.Read());
// third partition
Assert.True(partitioner.HasNext);
var thirdTraining = partitioner.Next();
Assert.AreEqual("01", thirdTraining.Read());
Assert.AreEqual("02", thirdTraining.Read());
Assert.AreEqual("04", thirdTraining.Read());
Assert.AreEqual("05", thirdTraining.Read());
Assert.AreEqual("07", thirdTraining.Read());
Assert.AreEqual("08", thirdTraining.Read());
Assert.AreEqual("10", thirdTraining.Read());
Assert.Null(thirdTraining.Read());
var thirdTest = thirdTraining.GetTestSampleStream();
Assert.AreEqual("03", thirdTest.Read());
Assert.AreEqual("06", thirdTest.Read());
Assert.AreEqual("09", thirdTest.Read());
Assert.Null(thirdTest.Read());
Assert.False(partitioner.HasNext);
}
public void ToStringTest() {
var value = new CrossValidationPartitioner<string>(new string[] {}, 10).ToString();
Assert.AreEqual("At partition 1 of 10.", value);
}
public void TestFailSafety() {
var partitioner = new CrossValidationPartitioner<String>(new[] {
"01", "02", "03", "04"
}, 4);
// Test that iterator from previous partition fails
// if it is accessed
var firstTraining = partitioner.Next();
Assert.AreEqual("02", firstTraining.Read());
var secondTraining = partitioner.Next();
try {
firstTraining.Read();
Assert.Fail();
} catch (Exception) {}
try {
firstTraining.GetTestSampleStream();
Assert.Fail();
} catch (Exception) {}
// Test that training iterator fails if there is a test iterator
secondTraining.GetTestSampleStream();
try {
secondTraining.Read();
Assert.Fail();
} catch (Exception) {}
// Test that test iterator from previous partition fails
// if there is a new partition
var thirdTraining = partitioner.Next();
var thirdTest = thirdTraining.GetTestSampleStream();
Assert.True(partitioner.HasNext);
partitioner.Next();
try {
thirdTest.Read();
Assert.Fail();
} catch (Exception) {}
}
/// <summary> /// Processes the specified sample stream. /// </summary> /// <param name="sampleStream">The sample stream.</param> /// <returns>The computed f-mesure of the sample stream.</returns> protected abstract FMeasure <P> Process(CrossValidationPartitioner <T> .TrainingSampleStream sampleStream);
