Esempi in C# (CSharp) per TensorFlowLite BertTokenizer.WordPieceTokenize

Esempio n. 1

        public void WordPieceTokenizeWithCutomCocabTest(string input, string[] expected)
        {
            var vocabText = @"[UNK]
[CLS]
[SEP]
want
##want
##ed
wa
un
runn
##ing";
            var table     = Bert.LoadVocabularies(vocabText);

            Assert.True(table.ContainsKey("[UNK]"));
            Assert.True(table.ContainsKey("want"));
            Assert.True(table.ContainsKey("##want"));
            ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, table));
        }

Esempio n. 2

 public void WordPieceTokenizeTest(string input, string[] expected)
 {
     ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, vocabularyTable));
 }