void GrowTokenBuffer()
{
var newLength = TokensLength * 2;
Tokens = NativeArrayUtility.Resize(Tokens, TokensLength, newLength, 4, Label);
TokensLength = newLength;
}
private unsafe void AddUnknown(ref TNode resultNode, CharInfo cInfo,
char *begin, char *begin2, char *begin3,
Func <TNode> nodeAllocator)
{
var tokens = this.unkTokens[cInfo.DefaultType];
fixed(Token *fpTokens = tokens)
{
Token *pTokens = fpTokens;
var length = (int)(begin3 - begin2);
var rLength = (int)(begin3 - begin);
var surface = new string(begin2, 0, length);
for (int i = 0; i < tokens.Length; i++)
{
var newNode = nodeAllocator();
newNode.Surface = surface;
newNode.Length = length;
newNode.RLength = rLength;
newNode.LCAttr = pTokens->LcAttr;
newNode.RCAttr = pTokens->RcAttr;
newNode.PosId = pTokens->PosId;
newNode.WCost = pTokens->WCost;
newNode.PFeature = this.unkDic.GetFeature(pTokens->Feature);
newNode.Encoding = this.Encoding;
newNode.CharType = cInfo.DefaultType;
newNode.Stat = MeCabNodeStat.Unk;
newNode.BNext = resultNode;
resultNode = newNode;
pTokens++;
}
}
}
public ParseBuffer(Token *queue, Int64 *stack)
{
mQueue = queue;
mStack = stack;
mQueueIndex = -1;
mStackIndex = -1;
}
public unsafe void Open(string fileName)
{
this.FileName = fileName;
var sourceFileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read);
try
{
this.mmf = MemoryMappedFile.CreateFromFile(sourceFileStream, null, 0L, MemoryMappedFileAccess.Read, HandleInheritability.None, false);
}
catch (Exception)
{
sourceFileStream.Dispose();
throw;
}
this.mmva = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read);
byte *ptr = null;
this.mmva.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
using (var stream = this.mmf.CreateViewStream(0L, 0L, MemoryMappedFileAccess.Read))
using (var reader = new BinaryReader(stream))
{
uint magic = reader.ReadUInt32();
if (this.mmva.Capacity < (magic ^ DictionaryMagicID))
{
throw new InvalidDataException($"dictionary file is broken. {fileName}");
}
this.Version = reader.ReadUInt32();
if (this.Version != DicVersion)
{
throw new InvalidDataException($"incompatible version dictionaly. {fileName}");
}
this.Type = (DictionaryType)reader.ReadUInt32();
this.LexSize = reader.ReadUInt32();
this.LSize = reader.ReadUInt32();
this.RSize = reader.ReadUInt32();
uint dSize = reader.ReadUInt32();
uint tSize = reader.ReadUInt32();
uint fSize = reader.ReadUInt32();
reader.ReadUInt32(); //dummy
string charSet = StrUtils.GetString(reader.ReadBytes(32), Encoding.ASCII);
this.encoding = StrUtils.GetEncoding(charSet);
ptr += stream.Position;
this.da.Open(ptr, (int)dSize);
ptr += dSize;
this.tokens = (Token *)ptr;
ptr += tSize;
this.features = ptr;
}
}
public unsafe void Open(string fileName)
{
this.FileName = fileName;
uint *uintPtr = (uint *)this.mmfLoader.Invoke(fileName);
uint magic = *uintPtr++;
if (this.mmfLoader.FileSize != (magic ^ DictionaryMagicID))
{
throw new InvalidDataException($"dictionary file is broken. {fileName}");
}
this.Version = *uintPtr++;
if (this.Version != DicVersion)
{
throw new InvalidDataException($"incompatible version dictionaly. {fileName}");
}
this.Type = (DictionaryType)(*uintPtr++);
this.LexSize = *uintPtr++;
this.LSize = *uintPtr++;
this.RSize = *uintPtr++;
uint dSize = *uintPtr++;
uint tSize = *uintPtr++;
uint fSize = *uintPtr++;
uintPtr++; // dummy
byte *bytePtr = (byte *)uintPtr;
var encName = StrUtils.GetString(bytePtr, Encoding.ASCII);
this.Encoding = encName.GetEncodingOrNull()
?? throw new Exception($"not supported encoding dictionary. {encName} {fileName}");
bytePtr += 32;
this.da.Open(bytePtr, (int)dSize);
bytePtr += dSize;
this.tokens = (Token *)bytePtr;
bytePtr += tSize;
this.features = bytePtr;
}
/// <summary>
/// Returns true if the given token is an object key.
/// </summary>
static bool IsObjectKey(Token *tokens, int index)
{
var token = tokens[index];
if (token.Type != TokenType.String && token.Type != TokenType.Primitive)
{
return(false);
}
if (token.Parent == -1)
{
return(false);
}
var parent = tokens[token.Parent];
return(parent.Type == TokenType.Object);
}
public unsafe void Open(BinaryReader reader)
{
uint magic = reader.ReadUInt32();
//CanSeekの時のみストリーム長のチェック
if (reader.BaseStream.CanSeek && reader.BaseStream.Length != (magic ^ DictionaryMagicID))
{
throw new InvalidDataException($"dictionary file is broken. {this.FileName}");
}
this.Version = reader.ReadUInt32();
if (this.Version != DicVersion)
{
throw new InvalidDataException($"incompatible version dictionaly. {this.FileName}");
}
this.Type = (DictionaryType)reader.ReadUInt32();
this.LexSize = reader.ReadUInt32();
this.LSize = reader.ReadUInt32();
this.RSize = reader.ReadUInt32();
uint dSize = reader.ReadUInt32();
uint tSize = reader.ReadUInt32();
uint fSize = reader.ReadUInt32();
reader.ReadUInt32(); //dummy
string charSet = StrUtils.GetString(reader.ReadBytes(32), Encoding.ASCII);
this.encoding = StrUtils.GetEncoding(charSet);
this.da.Open(reader, (int)dSize);
this.tokens = new Token[tSize / sizeof(Token)];
for (int i = 0; i < this.tokens.Length; i++)
{
this.tokens[i] = Token.Create(reader);
}
this.features = reader.ReadBytes((int)fSize);
}