private void ExecuteTokenization(TokenizationAssignment assignment)
{
if (assignment == null)
{
throw new ArgumentNullException(nameof(assignment));
}
if (!File.Exists(assignment.Filepath))
{
throw new InvalidOperationException("Cannot tokenize a file that does not exist: " + assignment.Filepath);
}
Debug.WriteLine($"Node {Id} - Running tokenization on {assignment.Filepath}");
Document tokenizedDocument;
using (var fileStream = File.OpenRead(assignment.Filepath)) {
var filename = Path.GetFileName(assignment.Filepath);
tokenizedDocument = documentTokenizer.TokenizeStream(fileStream, filename);
}
messageSink.PostTokenizedDocument(tokenizedDocument);
}
