private void ExecuteTokenization(TokenizationAssignment assignment) { if (assignment == null) { throw new ArgumentNullException(nameof(assignment)); } if (!File.Exists(assignment.Filepath)) { throw new InvalidOperationException("Cannot tokenize a file that does not exist: " + assignment.Filepath); } Debug.WriteLine($"Node {Id} - Running tokenization on {assignment.Filepath}"); Document tokenizedDocument; using (var fileStream = File.OpenRead(assignment.Filepath)) { var filename = Path.GetFileName(assignment.Filepath); tokenizedDocument = documentTokenizer.TokenizeStream(fileStream, filename); } messageSink.PostTokenizedDocument(tokenizedDocument); }