public unsafe Token[] GetToken(DoubleArrayResultPair n) { Token[] dist = new Token[0xFF & n.Value]; int tokenPos = n.Value >> 8; this.tokens.ReadArray <Token>(tokenPos * sizeof(Token), dist, 0, dist.Length); return(dist); }
public unsafe Token[] GetToken(DoubleArrayResultPair n) { Token[] dist = new Token[0xFF & n.Value]; int tokenPos = n.Value >> 8; Array.Copy(this.tokens, tokenPos, dist, 0, dist.Length); return(dist); }
public unsafe DoubleArrayResultPair ExactMatchSearch(char *key, int len, int nodePos = 0) { //if (this.encoding == Encoding.Unicode) // return this.da.ExactMatchSearch((byte*)key, len, nodePos); //エンコード int maxByteCount = this.encoding.GetMaxByteCount(len); byte *bytes = stackalloc byte[maxByteCount]; int bytesLen = this.encoding.GetBytes(key, len, bytes, maxByteCount); DoubleArrayResultPair result = this.da.ExactMatchSearch(bytes, bytesLen, nodePos); //文字数をデコードしたものに変換 result.Length = this.encoding.GetCharCount(bytes, result.Length); return(result); }
public unsafe int CommonPrefixSearch(byte *key, DoubleArrayResultPair *result, int resultLen, int len, int nodePos = 0) { int b = this.ReadBase(nodePos); int num = 0; int n; Unit p; for (int i = 0; i < len; i++) { this.ReadUnit(b, out p); n = p.Base; if (b == p.Check && n < 0) { if (num < resultLen) { result[num] = new DoubleArrayResultPair(-n - 1, i); } num++; } this.ReadUnit(b + key[i] + 1, out p); if (b == p.Check) { b = p.Base; } else { return(num); } } this.ReadUnit(b, out p); n = p.Base; if (b == p.Check && n < 0) { if (num < resultLen) { result[num] = new DoubleArrayResultPair(-n - 1, len); } num++; } return(num); }
public void Open(MeCabParam param) { var dictFactory = param.UseMemoryMappedFile ? new Func <IMeCabDictionary>(() => new MeCabDictionaryMMF()) : new Func <IMeCabDictionary>(() => new MeCabDictionary()); this.unkDic = dictFactory(); this.dic = new IMeCabDictionary[param.UserDic.Length + 1]; string prefix = param.DicDir; this.property.Open(prefix); this.unkDic.Open(Path.Combine(prefix, UnkDicFile)); if (this.unkDic.Type != DictionaryType.Unk) { throw new MeCabInvalidFileException("not a unk dictionary", this.unkDic.FileName); } IMeCabDictionary sysDic = dictFactory(); sysDic.Open(Path.Combine(prefix, SysDicFile)); if (sysDic.Type != DictionaryType.Sys) { throw new MeCabInvalidFileException("not a system dictionary", sysDic.FileName); } this.dic[0] = sysDic; for (int i = 0; i < param.UserDic.Length; i++) { IMeCabDictionary d = dictFactory(); d.Open(Path.Combine(prefix, param.UserDic[i])); if (d.Type != DictionaryType.Usr) { throw new MeCabInvalidFileException("not a user dictionary", d.FileName); } if (!sysDic.IsCompatible(d)) { throw new MeCabInvalidFileException("incompatible dictionary", d.FileName); } this.dic[i + 1] = d; } this.unkTokens = new Token[this.property.Size][]; for (int i = 0; i < this.unkTokens.Length; i++) { string key = this.property.Name(i); DoubleArrayResultPair n = this.unkDic.ExactMatchSearch(key); if (n.Value == -1) { throw new MeCabInvalidFileException("cannot find UNK category: " + key, this.unkDic.FileName); } this.unkTokens[i] = this.unkDic.GetToken(n); } this.space = this.property.GetCharInfo(' '); this.bosFeature = param.BosFeature; this.unkFeature = param.UnkFeature; this.maxGroupingSize = param.MaxGroupingSize; if (this.maxGroupingSize <= 0) { this.maxGroupingSize = DefaltMaxGroupingSize; } }