public void SetUp() { if (vocabularyTable != null) { return; } var vocabText = AssetDatabase.LoadAssetAtPath <TextAsset>("Assets/Samples/Bert/vocab.txt"); vocabularyTable = Bert.LoadVocabularies(vocabText.text); // Debug.Log("Vocab Loaded"); }
public void SetUp() { if (bert != null) { return; } string path = System.IO.Path.Combine(Application.streamingAssetsPath, "mobilebert_float.tflite"); var vocabText = AssetDatabase.LoadAssetAtPath <TextAsset>("Assets/Samples/Bert/vocab.txt"); bert = new Bert(path, vocabText.text); }
public void WordPieceTokenizeWithCutomCocabTest(string input, string[] expected) { var vocabText = @"[UNK] [CLS] [SEP] want ##want ##ed wa un runn ##ing"; var table = Bert.LoadVocabularies(vocabText); Assert.True(table.ContainsKey("[UNK]")); Assert.True(table.ContainsKey("want")); Assert.True(table.ContainsKey("##want")); ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, table)); }