public void DetailedCheck() { var result = WanaKana.Tokenize("5romaji here...!?漢字ひらがなカタ カナ4「SHIO」。! لنذهب"); var expectedResult = new Token[] { new Token(tokenType: TokenTypes.EnglishNumber, content: "5"), new Token(tokenType: TokenTypes.English, content: "romaji"), new Token(tokenType: TokenTypes.Space, content: " "), new Token(tokenType: TokenTypes.English, content: "here"), new Token(tokenType: TokenTypes.EnglishPunctuation, content: "...!?"), new Token(tokenType: TokenTypes.Kanji, content: "漢字"), new Token(tokenType: TokenTypes.Hiragana, content: "ひらがな"), new Token(tokenType: TokenTypes.Katakana, content: "カタ"), new Token(tokenType: TokenTypes.Space, content: " "), new Token(tokenType: TokenTypes.Katakana, content: "カナ"), new Token(tokenType: TokenTypes.JapaneseNumber, content: "4"), new Token(tokenType: TokenTypes.JapanesePunctuation, content: "「"), new Token(tokenType: TokenTypes.Japanese, content: "SHIO"), new Token(tokenType: TokenTypes.JapanesePunctuation, content: "」。!"), new Token(tokenType: TokenTypes.Space, content: " "), new Token(tokenType: TokenTypes.Other, content: "لنذهب"), }; Assert.Equal(expectedResult, result); }
public void BasicTests(string input, string expectedResult) { var result = WanaKana.Tokenize(input); var split = expectedResult.Split(','); Assert.Equal(split, result.Select(t => t.Content)); }
public void CompactTrue() { var result = WanaKana.Tokenize("5romaji here...!?漢字ひらがなカタ カナ4「SHIO」。! لنذهب", true); var expectedResult = new string[] { "5", "romaji here", "...!?", "漢字ひらがなカタ カナ", "4「", "SHIO", "」。!", " ", "لنذهب", }; Assert.Equal(expectedResult, result.Select(t => t.Content)); }
public void CompactDetailedCheck() { var result = WanaKana.Tokenize("5romaji here...!?漢字ひらがなカタ カナ4「SHIO」。! لنذهب", true); var expectedResult = new Token[] { new Token(tokenType: TokenTypes.Other, content: "5"), new Token(tokenType: TokenTypes.English, content: "romaji here"), new Token(tokenType: TokenTypes.Other, content: "...!?"), new Token(tokenType: TokenTypes.Japanese, content: "漢字ひらがなカタ カナ"), new Token(tokenType: TokenTypes.Other, content: "4「"), new Token(tokenType: TokenTypes.Japanese, content: "SHIO"), new Token(tokenType: TokenTypes.Other, content: "」。!"), new Token(tokenType: TokenTypes.English, content: " "), new Token(tokenType: TokenTypes.Other, content: "لنذهب"), }; Assert.Equal(expectedResult, result); }
public void HandlesMixedInput() { var result = WanaKana.Tokenize("5romaji here...!?漢字ひらがなカタ カナ4「SHIO」。!"); var expectedResult = new string[] { "5", "romaji", " ", "here", "...!?", "漢字", "ひらがな", "カタ", " ", "カナ", "4", "「", "SHIO", "」。!", }; Assert.Equal(expectedResult, result.Select(t => t.Content)); }
public void EmptyInput() { Assert.Empty(WanaKana.Tokenize(string.Empty)); }