public void SetUp()
{
if (vocabularyTable != null)
{
return;
}
var vocabText = AssetDatabase.LoadAssetAtPath <TextAsset>("Assets/Samples/Bert/vocab.txt");
vocabularyTable = Bert.LoadVocabularies(vocabText.text);
// Debug.Log("Vocab Loaded");
}
public void SetUp()
{
if (bert != null)
{
return;
}
string path = System.IO.Path.Combine(Application.streamingAssetsPath, "mobilebert_float.tflite");
var vocabText = AssetDatabase.LoadAssetAtPath <TextAsset>("Assets/Samples/Bert/vocab.txt");
bert = new Bert(path, vocabText.text);
}
public void WordPieceTokenizeWithCutomCocabTest(string input, string[] expected)
{
var vocabText = @"[UNK]
[CLS]
[SEP]
want
##want
##ed
wa
un
runn
##ing";
var table = Bert.LoadVocabularies(vocabText);
Assert.True(table.ContainsKey("[UNK]"));
Assert.True(table.ContainsKey("want"));
Assert.True(table.ContainsKey("##want"));
ArrayEqual(expected, BertTokenizer.WordPieceTokenize(input, table));
}
