private static void BuildPieceHashes(IDictionary <string, int> pieceHash, ModificationData data, bool ignoreWhitespace, bool ignoreCase, IChunker chunker) { var pieces = string.IsNullOrEmpty(data.RawData) ? emptyStringArray : chunker.Chunk(data.RawData); data.Pieces = pieces; data.HashedPieces = new int[pieces.Length]; data.Modifications = new bool[pieces.Length]; for (int i = 0; i < pieces.Length; i++) { string piece = pieces[i]; if (ignoreWhitespace) { piece = piece.Trim(); } if (ignoreCase) { piece = piece.ToUpperInvariant(); } if (pieceHash.ContainsKey(piece)) { data.HashedPieces[i] = pieceHash[piece]; } else { data.HashedPieces[i] = pieceHash.Count; pieceHash[piece] = pieceHash.Count; } } }
/// <summary> /// Evaluates the given reference <see cref="ChunkSample"/> object. /// The implementation has to update the score after every invocation. /// </summary> /// <param name="reference">The reference sample.</param> /// <returns>The predicted sample</returns> protected override ChunkSample ProcessSample(ChunkSample reference) { var preds = chunker.Chunk(reference.Sentence.ToArray(), reference.Tags.ToArray()); var result = new ChunkSample(reference.Sentence.ToArray(), reference.Tags.ToArray(), preds); FMeasure.UpdateScores(reference.GetPhrasesAsSpanList(), result.GetPhrasesAsSpanList()); return(result); }
public void TestChunkAsArray() { var sPreds = sChunker.Chunk(ChunkerMETest.toks1, ChunkerMETest.tags1); var jPreds = jChunker.chunk(ChunkerMETest.toks1, ChunkerMETest.tags1); Assert.AreEqual(sPreds.Length, jPreds.Length); for (var i = 0; i < jPreds.Length; i++) { Assert.AreEqual(jPreds[i], sPreds[i]); } }
public void TestChunkAsArray() { var preds = chunker.Chunk(toks1, tags1); Assert.True(expect1.SequenceEqual(preds)); }