public unsafe Token[] GetToken(DoubleArrayResultPair n)
{
Token[] dist = new Token[0xFF & n.Value];
int tokenPos = n.Value >> 8;
this.tokens.ReadArray <Token>(tokenPos * sizeof(Token), dist, 0, dist.Length);
return(dist);
}
public unsafe Token[] GetToken(DoubleArrayResultPair n)
{
Token[] dist = new Token[0xFF & n.Value];
int tokenPos = n.Value >> 8;
Array.Copy(this.tokens, tokenPos, dist, 0, dist.Length);
return(dist);
}
public unsafe DoubleArrayResultPair ExactMatchSearch(char *key, int len, int nodePos = 0)
{
//if (this.encoding == Encoding.Unicode)
// return this.da.ExactMatchSearch((byte*)key, len, nodePos);
//エンコード
int maxByteCount = this.encoding.GetMaxByteCount(len);
byte *bytes = stackalloc byte[maxByteCount];
int bytesLen = this.encoding.GetBytes(key, len, bytes, maxByteCount);
DoubleArrayResultPair result = this.da.ExactMatchSearch(bytes, bytesLen, nodePos);
//文字数をデコードしたものに変換
result.Length = this.encoding.GetCharCount(bytes, result.Length);
return(result);
}
public unsafe int CommonPrefixSearch(byte *key, DoubleArrayResultPair *result, int resultLen, int len, int nodePos = 0)
{
int b = this.ReadBase(nodePos);
int num = 0;
int n;
Unit p;
for (int i = 0; i < len; i++)
{
this.ReadUnit(b, out p);
n = p.Base;
if (b == p.Check && n < 0)
{
if (num < resultLen)
{
result[num] = new DoubleArrayResultPair(-n - 1, i);
}
num++;
}
this.ReadUnit(b + key[i] + 1, out p);
if (b == p.Check)
{
b = p.Base;
}
else
{
return(num);
}
}
this.ReadUnit(b, out p);
n = p.Base;
if (b == p.Check && n < 0)
{
if (num < resultLen)
{
result[num] = new DoubleArrayResultPair(-n - 1, len);
}
num++;
}
return(num);
}
public void Open(MeCabParam param)
{
var dictFactory = param.UseMemoryMappedFile
? new Func <IMeCabDictionary>(() => new MeCabDictionaryMMF())
: new Func <IMeCabDictionary>(() => new MeCabDictionary());
this.unkDic = dictFactory();
this.dic = new IMeCabDictionary[param.UserDic.Length + 1];
string prefix = param.DicDir;
this.property.Open(prefix);
this.unkDic.Open(Path.Combine(prefix, UnkDicFile));
if (this.unkDic.Type != DictionaryType.Unk)
{
throw new MeCabInvalidFileException("not a unk dictionary", this.unkDic.FileName);
}
IMeCabDictionary sysDic = dictFactory();
sysDic.Open(Path.Combine(prefix, SysDicFile));
if (sysDic.Type != DictionaryType.Sys)
{
throw new MeCabInvalidFileException("not a system dictionary", sysDic.FileName);
}
this.dic[0] = sysDic;
for (int i = 0; i < param.UserDic.Length; i++)
{
IMeCabDictionary d = dictFactory();
d.Open(Path.Combine(prefix, param.UserDic[i]));
if (d.Type != DictionaryType.Usr)
{
throw new MeCabInvalidFileException("not a user dictionary", d.FileName);
}
if (!sysDic.IsCompatible(d))
{
throw new MeCabInvalidFileException("incompatible dictionary", d.FileName);
}
this.dic[i + 1] = d;
}
this.unkTokens = new Token[this.property.Size][];
for (int i = 0; i < this.unkTokens.Length; i++)
{
string key = this.property.Name(i);
DoubleArrayResultPair n = this.unkDic.ExactMatchSearch(key);
if (n.Value == -1)
{
throw new MeCabInvalidFileException("cannot find UNK category: " + key, this.unkDic.FileName);
}
this.unkTokens[i] = this.unkDic.GetToken(n);
}
this.space = this.property.GetCharInfo(' ');
this.bosFeature = param.BosFeature;
this.unkFeature = param.UnkFeature;
this.maxGroupingSize = param.MaxGroupingSize;
if (this.maxGroupingSize <= 0)
{
this.maxGroupingSize = DefaltMaxGroupingSize;
}
}
