C# (CSharp) TokenFilter.IsTokenValid Examples

Example #1

File: TextNormalizer.cs Project: san4oz/Warden

        public string Normalize(string text)
        {
            var words = ExtractWords(RemoveNoizyCharacters(text));

            var processedWords = words
                                 .Where(word => Filter.IsTokenValid(word))
                                 .Select(word => ProcessWronglyEncodedSymbols(word)).ToArray();

            return(string.Join(";", processedWords));
        }