public static void Setup(TestContext context)
{
using (var charDefOutput = new MemoryStream())
using (var unkDicOutput = new MemoryStream())
using (var charDefResource = GetResource("char.def"))
using (var unkDefResource = GetResource("unk.def"))
{
var charDefCompiler = new CharacterDefinitionsCompiler(charDefOutput);
charDefCompiler.ReadCharacterDefinition(charDefResource, Encoding.GetEncoding("euc-jp"));
charDefCompiler.Compile();
var categoryMap = charDefCompiler.MakeCharacterCategoryMap();
var unkDefCompiler = new UnknownDictionaryCompiler(unkDicOutput, categoryMap);
unkDefCompiler.ReadUnknownDefinition(unkDefResource, Encoding.GetEncoding("euc-jp"));
unkDefCompiler.Compile();
charDefOutput.Seek(0, SeekOrigin.Begin);
unkDicOutput.Seek(0, SeekOrigin.Begin);
var definitions = IntArrayIO.ReadSparseArray2D(charDefOutput);
var mappings = IntArrayIO.ReadSparseArray2D(charDefOutput);
var symbols = StringArrayIO.ReadArray(charDefOutput);
CharacterDefinitions = new CharacterDefinitions(definitions, mappings, symbols);
Costs = IntArrayIO.ReadArray2D(unkDicOutput);
References = IntArrayIO.ReadArray2D(unkDicOutput);
Features = StringArrayIO.ReadArray2D(unkDicOutput);
UnknownDictionary = new UnknownDictionary(CharacterDefinitions, References, Costs, Features);
}
}
public void SetUp()
{
if (Directory.Exists(TestUtils.CompiledPath))
{
Directory.Delete(TestUtils.CompiledPath, true);
}
Directory.CreateDirectory(TestUtils.CompiledPath);
using (var outputStream = File.Create(charDef))
{
CharacterDefinitionsCompiler compiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance);
string assetFileName = @"./Core/Resource/char.def";
using (var defStream = File.OpenRead(assetFileName))
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
compiler.ReadCharacterDefinition(defStream, "euc-jp");
categoryIdMap = Invert(compiler.MakeCharacterCategoryMap());
compiler.Compile(outputStream);
}
}
using (var input = File.OpenRead(charDef))
using (BinaryReader reader = new BinaryReader(input))
{
int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader);
int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader);
string[] symbols = StringArrayIO.ReadArray(reader);
characterDefinition = new CharacterDefinitions(definitions, mappings, symbols);
}
}
public CharDef(String loadPath, CharacterDefinitions _defID)
{
Reset();
Path = loadPath;
Read();
DefID = (int)_defID;
}
public CharDef(String loadPath, CharacterDefinitions _defID)
{
Reset();
Path = loadPath;
Read();
DefID = (int)_defID;
}
public void attackPlayer()
{
// Check the player is not dead anymore.
CharacterDefinitions cdScript = player.GetComponent <CharacterDefinitions>();
if (cdScript.stats.status == CharacterDefinitions.CharacterStats.StatusTypes.dead)
{
isAttacking = false;
}
else
{
// Get player pos.
Vector2 playerPos = player.transform.position;
if (playerPos.x < this.transform.position.x)
{
lookLeft();
}
else
{
lookRight();
}
lookAt(playerPos);
attackPoint = playerPos;
ChasePoint();
}
}
public override void LoadDictionaries()
{
Penalties = new List <int>();
Penalties.Add(kanjiPenaltyLengthTreshold);
Penalties.Add(kanjiPenalty);
Penalties.Add(otherPenaltyLengthThreshold);
Penalties.Add(otherPenalty);
try
{
Fst = FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
if (IsSplitOnNakaguro)
{
CharacterDefinitions.SetCategories('・', new string[] { "SYMBOL" });
}
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries: " + ouch.Message);
}
}
protected internal virtual void LoadDictionaries()
{
DoubleArrayTrie = DoubleArrayTrie.NewInstance(Resolver);
ConnectionCosts = ConnectionCosts.NewInstance(Resolver);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(Resolver);
CharacterDefinitions = CharacterDefinitions.NewInstance(Resolver);
UnknownDictionary = UnknownDictionary.NewInstance(Resolver, CharacterDefinitions, TotalFeatures);
InsertedDictionary = new InsertedDictionary(TotalFeatures);
}
public void AddItem(Vector2 loc, CharacterDefinitions charDef)
{
for (int i = 0; i < bucketItem.Length; i++)
{
if (bucketItem[i] == null)
{
bucketItem[i] = new BucketItem(loc, charDef);
return;
}
}
}
public void AddItem(Vector2 loc, CharacterDefinitions charDef)
{
for (int i = 0; i < bucketItem.Length; i++)
{
if (bucketItem[i] == null)
{
bucketItem[i] = new BucketItem(loc, charDef);
return;
}
}
}
int ProcessUnknownWord(int category, int i, ViterbiLattice lattice, int unknownWordEndIndex, int startIndex, string suffix, bool found)
{
var unknownWordLength = 0;
var definition = CharacterDefinitions.LookupDefinition(category);
if (definition[CharacterDefinitions.Invoke] == 1 || found == false)
{
if (definition[CharacterDefinitions.Group] == 0)
{
unknownWordLength = 1;
}
else
{
unknownWordLength = 1;
for (var j = 1; j < suffix.Length; j++)
{
var c = suffix[j];
var categories = CharacterDefinitions.LookupCategories(c);
if (categories == null)
{
break;
}
if (i < categories.Length && category == categories[i])
{
unknownWordLength++;
}
else
{
break;
}
}
}
}
if (unknownWordLength > 0)
{
var unkWord = suffix.Substring(0, unknownWordLength);
var wordIds = UnknownDictionary.LookupWordIds(category); // characters in input text are supposed to be the same
foreach (var wordId in wordIds)
{
var node = new ViterbiNode(wordId, unkWord, UnknownDictionary, startIndex, ViterbiNode.NodeType.Unknown);
lattice.AddNode(node, startIndex + 1, startIndex + 1 + unknownWordLength);
}
unknownWordEndIndex = startIndex + unknownWordLength;
}
return(unknownWordEndIndex);
}
public void SetUp()
{
SortedDictionary <string, int> categoryMap;
using (var outputStream = File.Create(charDef))
{
CharacterDefinitionsCompiler charDefCompiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance);
string assetFileName = @"./Core/Resource/char.def";
using (var defStream = File.OpenRead(assetFileName))
{
charDefCompiler.ReadCharacterDefinition(defStream, "euc-jp");
charDefCompiler.Compile(outputStream);
}
categoryMap = charDefCompiler.MakeCharacterCategoryMap();
}
var unkDefFile = TestUtils.CompiledPath + Path.DirectorySeparatorChar + "kuromoji-unkdef-.bin";
using (var outputStream = File.Create(unkDefFile))
{
UnknownDictionaryCompiler unkDefCompiler = new UnknownDictionaryCompiler(categoryMap);
string assetFileName = @"./Core/Resource/unk.def";
using (var defStream = File.OpenRead(assetFileName))
{
unkDefCompiler.ReadUnknownDefinition(defStream, "euc-jp");
unkDefCompiler.Compile(outputStream);
}
}
using (var charDefInput = File.OpenRead(charDef))
using (var reader = new BinaryReader(charDefInput))
{
int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader);
int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader);
string[] symbols = StringArrayIO.ReadArray(reader);
characterDefinitions = new CharacterDefinitions(definitions, mappings, symbols);
}
using (var unkDefInput = File.OpenRead(unkDefFile))
using (var reader = new BinaryReader(unkDefInput))
{
costs = IntegerArrayIO.ReadArray2D(reader);
references = IntegerArrayIO.ReadArray2D(reader);
features = StringArrayIO.ReadArray2D(reader);
unknownDictionary = new UnknownDictionary(characterDefinitions, references, costs, features);
}
}
public virtual void LoadDictionaries()
{
try
{
Fst = FST.FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries.", ouch);
}
}
// Start is called before the first frame update
void Start()
{
rb = GetComponent <Rigidbody2D>();
col = GetComponent <Collider2D>();
animator = this.GetComponentInChildren <Animator>();
runningSpeed = speed + 4f;
selfCollider = GetComponent <Collider2D>();
applyLayerMaskToChildren();
// Set up hit delegate
OnEnemyHit oehScript = GetComponent <OnEnemyHit>();
oehScript.executionDelegate = OnEnemyHitEffect;
// Definitions
defScript = GetComponent <CharacterDefinitions>();
stats = defScript.stats;
heroCharacterScript = GetComponent <Character>();
gameController = GameObject.FindGameObjectWithTag("GameController");
}
void AssertCharacterCategories(CharacterDefinitions characterDefinition, char c, params string[] categories)
{
var categoryIds = characterDefinition.LookupCategories(c);
if (categoryIds == null)
{
categories.IsNull();
return;
}
categoryIds.Length.Is(categories.Length);
foreach (var categoryId in categoryIds)
{
var category = CategoryIdMap[categoryId];
categories.Contains(category).IsTrue();
}
}
/**
* Constructor
*
* @param fst FST with surface forms
* @param dictionary token info dictionary
* @param unknownDictionary unknown word dictionary
* @param userDictionary user dictionary
* @param mode tokenization {@link Mode mode}
*/
public ViterbiBuilder(FST.FST fst,
TokenInfoDictionary dictionary,
UnknownDictionary unknownDictionary,
UserDictionary userDictionary,
Mode mode)
{
this.fst = fst;
this.dictionary = dictionary;
this.unknownDictionary = unknownDictionary;
this.userDictionary = userDictionary;
this.useUserDictionary = (userDictionary != null);
if (mode == Mode.SEARCH || mode == Mode.EXTENDED)
{
searchMode = true;
}
this.characterDefinitions = unknownDictionary.GetCharacterDefinition();
}
public void AssertCharacterCategories(CharacterDefinitions characterDefinition, char c, params string[] categories)
{
int[] categoryIds = characterDefinition.LookupCategories(c);
if (categoryIds == null)
{
Assert.IsNull(categories);
return;
}
Assert.AreEqual(categories.Length, categoryIds.Length);
List <string> categoryList = new List <string>(categories);
foreach (int categoryId in categoryIds)
{
string category = categoryIdMap[categoryId];
Assert.IsTrue(categoryList.Contains(category));
}
}
public void Setup()
{
CharDefFileName = Path.GetTempFileName();
using (var fs = new FileStream(CharDefFileName, FileMode.Create, FileAccess.ReadWrite))
using (var resource = GetResource("char.def"))
{
var compiler = new CharacterDefinitionsCompiler(fs);
compiler.ReadCharacterDefinition(resource, Encoding.GetEncoding("euc-jp"));
CategoryIdMap = Invert(compiler.MakeCharacterCategoryMap());
compiler.Compile();
}
using (var fs = new FileStream(CharDefFileName, FileMode.Open, FileAccess.Read))
{
var definitions = IntArrayIO.ReadSparseArray2D(fs);
var mappings = IntArrayIO.ReadSparseArray2D(fs);
var symbols = StringArrayIO.ReadArray(fs);
CharacterDefinition = new CharacterDefinitions(definitions, mappings, symbols);
}
}
/// <summary>
/// Build lattice from input text
/// </summary>
/// <param name="text">source text for the lattice</param>
/// <returns>built lattice, not null</returns>
public ViterbiLattice Build(string text)
{
var textLength = text.Length;
var lattice = new ViterbiLattice(textLength + 2);
lattice.AddBos();
var unknownWordEndIndex = -1; // index of the last character of unknown word
for (var startIndex = 0; startIndex < textLength; startIndex++)
{
// If no token ends where current token starts, skip this index
if (lattice.TokenEndsWhereCurrentTokenStarts(startIndex))
{
var suffix = text.Substring(startIndex);
var found = ProcessIndex(lattice, startIndex, suffix);
// In the case of normal mode, it doesn't process unknown word greedily.
if (SearchMode || unknownWordEndIndex <= startIndex)
{
int[] categories = CharacterDefinitions.LookupCategories(suffix[0]);
for (int i = 0; i < categories.Length; i++)
{
int category = categories[i];
unknownWordEndIndex = ProcessUnknownWord(category, i, lattice, unknownWordEndIndex, startIndex, suffix, found);
}
}
}
}
if (UseUserDictionary)
{
ProcessUserDictionary(text, lattice);
}
lattice.AddEos();
return(lattice);
}
public void TestCostsAndFeatures()
{
var categories = CharacterDefinitions.LookupCategories('一');
// KANJI & KANJINUMERIC
categories.Length.Is(2);
categories.SequenceEqual(new int[] { 5, 6 }).IsTrue();
// KANJI entries
UnknownDictionary.LookupWordIds(categories[0]).SequenceEqual(new int[] { 2, 3, 4, 5, 6, 7 }).IsTrue();
// KANJI feature variety
UnknownDictionary.GetAllFeaturesArray(2).SequenceEqual(new string[] { "名詞", "一般", "*", "*", "*", "*", "*" }).IsTrue();
UnknownDictionary.GetAllFeaturesArray(3).SequenceEqual(new string[] { "名詞", "サ変接続", "*", "*", "*", "*", "*" }).IsTrue();
UnknownDictionary.GetAllFeaturesArray(4).SequenceEqual(new string[] { "名詞", "固有名詞", "地域", "一般", "*", "*", "*" }).IsTrue();
UnknownDictionary.GetAllFeaturesArray(5).SequenceEqual(new string[] { "名詞", "固有名詞", "組織", "*", "*", "*", "*" }).IsTrue();
UnknownDictionary.GetAllFeaturesArray(6).SequenceEqual(new string[] { "名詞", "固有名詞", "人名", "一般", "*", "*", "*" }).IsTrue();
UnknownDictionary.GetAllFeaturesArray(6).SequenceEqual(new string[] { "名詞", "固有名詞", "人名", "一般", "*", "*", "*" }).IsTrue();
// KANJINUMERIC entry
UnknownDictionary.LookupWordIds(categories[1]).SequenceEqual(new int[] { 29 }).IsTrue();
// KANJINUMERIC costs
UnknownDictionary.GetLeftId(29).Is(1295);
UnknownDictionary.GetRightId(29).Is(1295);
UnknownDictionary.GetWordCost(29).Is(27473);
// KANJINUMERIC features
UnknownDictionary.GetAllFeaturesArray(29).SequenceEqual(new string[] { "名詞", "数", "*", "*", "*", "*", "*" }).IsTrue();
}
public BucketItem(Vector2 loc, CharacterDefinitions charDef)
{
Location = loc;
CharDef = charDef;
}
public BucketItem(Vector2 loc, CharacterDefinitions charDef)
{
Location = loc;
CharDef = charDef;
}
