public void Tokenize(string text, string textTokensExpected)
{
// Arrange
var textPreprocessor = new TextPreprocessor(new PorterStemmer());
// Act
var tokens = textPreprocessor.Tokenize(text);
// Assert
Assert.AreEqual(textTokensExpected, string.Join(" ", tokens));
}
public async Task FindDuplicates(string text)
{
// Arrange
var stemmer = new PorterStemmer();
var textPreprocessor = new TextPreprocessor(stemmer);
var document = new Document
{
Id = 3,
Tokens = textPreprocessor.Tokenize(text)
};
var tfIdfSimilarityScoring = await BuildService(textPreprocessor);
// Act
var scores = await tfIdfSimilarityScoring.GetSimilarityScoresAsync(document);
// Assert
Assert.IsTrue(scores.Any(s => s.Score > 0.5d));
}