public void WordPieceTokenizeWithCutomCocabTest(string input, string[] expected) { var vocabText = @"[UNK] [CLS] [SEP] want ##want ##ed wa un runn ##ing"; var table = Bert.LoadVocabularies(vocabText); Assert.True(table.ContainsKey("[UNK]")); Assert.True(table.ContainsKey("want")); Assert.True(table.ContainsKey("##want")); ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, table)); }
public void WordPieceTokenizeTest(string input, string[] expected) { ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, vocabularyTable)); }