public Token[] GetToken(DoubleArray.ResultPair n)
{
Token[] array = new Token[this.GetTokenSize(n)];
int sourceIndex = n.Value >> 8;
Array.Copy(this.tokens, sourceIndex, array, 0, array.Length);
return(array);
}
public unsafe Token[] GetToken(DoubleArray.ResultPair n)
{
Token[] dist = new Token[0xFF & n.Value];
int tokenPos = n.Value >> 8;
Array.Copy(this.tokens, tokenPos, dist, 0, dist.Length);
return(dist);
}
public void Open(MeCabParam param)
{
this.dic = new MeCabDictionary[param.UserDic.Length + 1];
this.property.Open();
this.unkDic.Open(Properties.Resources.unk);
if (this.unkDic.Type != DictionaryType.Unk)
{
throw new MeCabInvalidFileException("not a unk dictionary", "unk.resource");
}
MeCabDictionary sysDic = new MeCabDictionary();
sysDic.Open(Properties.Resources.sys);
if (sysDic.Type != DictionaryType.Sys)
{
throw new MeCabInvalidFileException("not a system dictionary", "sys.resource");
}
this.dic[0] = sysDic;
/*
* for (int i = 0; i < param.UserDic.Length; i++)
* {
* MeCabDictionary d = new MeCabDictionary();
* d.Open(Path.Combine(prefix, param.UserDic[i]));
* if (d.Type != DictionaryType.Usr)
* throw new MeCabInvalidFileException("not a user dictionary", d.FileName);
* if (!sysDic.IsCompatible(d))
* throw new MeCabInvalidFileException("incompatible dictionary", d.FileName);
* this.dic[i + 1] = d;
* }
*/
this.unkTokens = new Token[this.property.Size][];
for (int i = 0; i < this.unkTokens.Length; i++)
{
string key = this.property.Name(i);
DoubleArray.ResultPair n = this.unkDic.ExactMatchSearch(key);
if (n.Value == -1)
{
throw new MeCabInvalidFileException("cannot find UNK category: " + key, "unk.resource");
}
this.unkTokens[i] = this.unkDic.GetToken(n);
}
this.space = this.property.GetCharInfo(' ');
this.bosFeature = param.BosFeature;
this.unkFeature = param.UnkFeature;
this.maxGroupingSize = param.MaxGroupingSize;
if (this.maxGroupingSize <= 0)
{
this.maxGroupingSize = DefaltMaxGroupingSize;
}
}
public unsafe DoubleArray.ResultPair ExactMatchSearch(char *key, int len, int nodePos = 0)
{
int maxByteCount = this.encoding.GetMaxByteCount(len);
byte *ptr = stackalloc byte[(int)(uint)maxByteCount];
int bytes = this.encoding.GetBytes(key, len, ptr, maxByteCount);
DoubleArray.ResultPair result = this.da.ExactMatchSearch(ptr, bytes, nodePos);
result.Length = this.encoding.GetCharCount(ptr, result.Length);
return(result);
}
public unsafe Token[] GetToken(DoubleArray.ResultPair n)
{
Token[] dist = new Token[0xFF & n.Value];
int tokenPos = n.Value >> 8;
#if MMF_DIC
this.tokens.ReadArray <Token>(tokenPos * sizeof(Token), dist, 0, dist.Length);
#else
Array.Copy(this.tokens, tokenPos, dist, 0, dist.Length);
#endif
return(dist);
}
public void Open(MeCabParam param)
{
this.dic = new MeCabDictionary[param.UserDic.Length + 1];
string dicDir = param.DicDir;
this.property.Open(dicDir);
this.unkDic.Open(Path.Combine(dicDir, "unk.dic"));
if (this.unkDic.Type != DictionaryType.Unk)
{
throw new MeCabInvalidFileException("not a unk dictionary", this.unkDic.FileName);
}
MeCabDictionary meCabDictionary = new MeCabDictionary();
meCabDictionary.Open(Path.Combine(dicDir, "sys.dic"));
if (meCabDictionary.Type != 0)
{
throw new MeCabInvalidFileException("not a system dictionary", meCabDictionary.FileName);
}
this.dic[0] = meCabDictionary;
for (int i = 0; i < param.UserDic.Length; i++)
{
MeCabDictionary meCabDictionary2 = new MeCabDictionary();
meCabDictionary2.Open(Path.Combine(dicDir, param.UserDic[i]));
if (meCabDictionary2.Type != DictionaryType.Usr)
{
throw new MeCabInvalidFileException("not a user dictionary", meCabDictionary2.FileName);
}
if (!meCabDictionary.IsCompatible(meCabDictionary2))
{
throw new MeCabInvalidFileException("incompatible dictionary", meCabDictionary2.FileName);
}
this.dic[i + 1] = meCabDictionary2;
}
this.unkTokens = new Token[this.property.Size][];
for (int j = 0; j < this.unkTokens.Length; j++)
{
string text = this.property.Name(j);
DoubleArray.ResultPair n = this.unkDic.ExactMatchSearch(text);
if (n.Value == -1)
{
throw new MeCabInvalidFileException("cannot find UNK category: " + text, this.unkDic.FileName);
}
this.unkTokens[j] = this.unkDic.GetToken(n);
}
this.space = this.property.GetCharInfo(' ');
this.bosFeature = param.BosFeature;
this.unkFeature = param.UnkFeature;
this.maxGroupingSize = param.MaxGroupingSize;
if (this.maxGroupingSize <= 0)
{
this.maxGroupingSize = 24;
}
}
public unsafe DoubleArray.ResultPair ExactMatchSearch(char *key, int len, int nodePos = 0)
{
//エンコード
int maxByteCount = this.encoding.GetMaxByteCount(len);
byte *bytes = stackalloc byte[maxByteCount];
int bytesLen = this.encoding.GetBytes(key, len, bytes, maxByteCount);
DoubleArray.ResultPair result = this.da.ExactMatchSearch(bytes, bytesLen, nodePos);
//文字数をデコードしたものに変換
result.Length = this.encoding.GetCharCount(bytes, result.Length);
return(result);
}
public int GetTokenSize(DoubleArray.ResultPair n)
{
return(0xFF & n.Value);
}
