private static void BuildPieceHashes(IDictionary <string, int> pieceHash, ModificationData data, bool ignoreWhitespace, bool ignoreCase, IChunker chunker)
{
var pieces = string.IsNullOrEmpty(data.RawData)
? emptyStringArray
: chunker.Chunk(data.RawData);
data.Pieces = pieces;
data.HashedPieces = new int[pieces.Length];
data.Modifications = new bool[pieces.Length];
for (int i = 0; i < pieces.Length; i++)
{
string piece = pieces[i];
if (ignoreWhitespace)
{
piece = piece.Trim();
}
if (ignoreCase)
{
piece = piece.ToUpperInvariant();
}
if (pieceHash.ContainsKey(piece))
{
data.HashedPieces[i] = pieceHash[piece];
}
else
{
data.HashedPieces[i] = pieceHash.Count;
pieceHash[piece] = pieceHash.Count;
}
}
}
/// <summary>
/// Evaluates the given reference <see cref="ChunkSample"/> object.
/// The implementation has to update the score after every invocation.
/// </summary>
/// <param name="reference">The reference sample.</param>
/// <returns>The predicted sample</returns>
protected override ChunkSample ProcessSample(ChunkSample reference)
{
var preds = chunker.Chunk(reference.Sentence.ToArray(), reference.Tags.ToArray());
var result = new ChunkSample(reference.Sentence.ToArray(), reference.Tags.ToArray(), preds);
FMeasure.UpdateScores(reference.GetPhrasesAsSpanList(), result.GetPhrasesAsSpanList());
return(result);
}
public void TestChunkAsArray()
{
var sPreds = sChunker.Chunk(ChunkerMETest.toks1, ChunkerMETest.tags1);
var jPreds = jChunker.chunk(ChunkerMETest.toks1, ChunkerMETest.tags1);
Assert.AreEqual(sPreds.Length, jPreds.Length);
for (var i = 0; i < jPreds.Length; i++)
{
Assert.AreEqual(jPreds[i], sPreds[i]);
}
}
public void TestChunkAsArray()
{
var preds = chunker.Chunk(toks1, tags1);
Assert.True(expect1.SequenceEqual(preds));
}