public MeCabNode Next()
{
while (this.agenda.Count != 0)
{
QueueElement top = this.agenda.Pop();
MeCabNode rNode = top.Node;
if (rNode.Stat == MeCabNodeStat.Bos)
{
for (QueueElement n = top; n.Next != null; n = n.Next)
{
n.Node.Next = n.Next.Node; // change next & prev
n.Next.Node.Prev = n.Node;
}
return(rNode);
}
for (MeCabPath path = rNode.LPath; path != null; path = path.LNext)
{
QueueElement n = new QueueElement()
{
Node = path.LNode,
Gx = path.Cost + top.Gx,
Fx = path.LNode.Cost + path.Cost + top.Gx,
Next = top
};
this.agenda.Push(n);
}
}
return(null);
}
public MeCabNode Next()
{
while (this.agenda.Count != 0)
{
QueueElement queueElement = this.agenda.Pop();
MeCabNode node = queueElement.Node;
if (node.Stat == MeCabNodeStat.Bos)
{
QueueElement queueElement2 = queueElement;
while (queueElement2.Next != null)
{
queueElement2.Node.Next = queueElement2.Next.Node;
queueElement2.Next.Node.Prev = queueElement2.Node;
queueElement2 = queueElement2.Next;
}
return(node);
}
for (MeCabPath meCabPath = node.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
QueueElement queueElement3 = new QueueElement();
queueElement3.Node = meCabPath.LNode;
queueElement3.Gx = meCabPath.Cost + queueElement.Gx;
queueElement3.Fx = meCabPath.LNode.Cost + meCabPath.Cost + queueElement.Gx;
queueElement3.Next = queueElement;
QueueElement item = queueElement3;
this.agenda.Push(item);
}
}
return(null);
}
private MeCabNode BuildAllLattice(ThreadData work)
{
if (this.BuildBestLattice(work) == null)
{
return(null);
}
MeCabNode prev = work.BosNode;
for (int pos = 0; pos < work.BeginNodeList.Length; pos++)
{
for (MeCabNode node = work.BeginNodeList[pos]; node != null; node = node.BNext)
{
prev.Next = node;
node.Prev = prev;
prev = node;
for (MeCabPath path = node.LPath; path != null; path = path.LNext)
{
path.Prob = (float)(path.LNode.Alpha
- this.theta * path.Cost
+ path.RNode.Beta - work.Z);
}
}
}
return(work.BosNode);
}
private void CalcAlpha(MeCabNode n, double beta)
{
n.Alpha = 0f;
for (MeCabPath meCabPath = n.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
n.Alpha = (float)Utils.LogSumExp((double)n.Alpha, (0.0 - beta) * (double)meCabPath.Cost + (double)meCabPath.LNode.Alpha, meCabPath == n.LPath);
}
}
private void CalcBeta(MeCabNode n, double beta)
{
n.Beta = 0f;
for (MeCabPath meCabPath = n.RPath; meCabPath != null; meCabPath = meCabPath.RNext)
{
n.Beta = (float)Utils.LogSumExp((double)n.Beta, (0.0 - beta) * (double)meCabPath.Cost + (double)meCabPath.RNode.Beta, meCabPath == n.RPath);
}
}
private void CalcBeta(MeCabNode n, double beta)
{
n.Beta = 0f;
for (MeCabPath path = n.RPath; path != null; path = path.RNext)
{
n.Beta = (float)Utils.LogSumExp(n.Beta,
-beta * path.Cost + path.RNode.Beta,
path == n.RPath);
}
}
private void CalcAlpha(MeCabNode n, double beta)
{
n.Alpha = 0f;
for (MeCabPath path = n.LPath; path != null; path = path.LNext)
{
n.Alpha = (float)Utils.LogSumExp(n.Alpha,
-beta * path.Cost + path.LNode.Alpha,
path == n.LPath);
}
}
public void WriteDump(StringBuilder os, MeCabNode bosNode)
{
for (MeCabNode node = bosNode; node != null; node = node.Next)
{
#if NeedId
os.Append(node.Id).Append(" ");
#endif
if (node.Stat == MeCabNodeStat.Bos)
{
os.Append("BOS");
}
else if (node.Stat == MeCabNodeStat.Eos)
{
os.Append("EOS");
}
else
{
os.Append(node.Surface);
}
os.Append(" ").Append(node.Feature);
os.Append(" ").Append(node.BPos);
os.Append(" ").Append(node.EPos);
os.Append(" ").Append(node.RCAttr);
os.Append(" ").Append(node.LCAttr);
os.Append(" ").Append(node.PosId);
os.Append(" ").Append(node.CharType);
os.Append(" ").Append((int)node.Stat);
os.Append(" ").Append(node.IsBest ? "1" : "0");
os.Append(" ").Append(node.Alpha.ToString(FloatFormat));
os.Append(" ").Append(node.Beta.ToString(FloatFormat));
os.Append(" ").Append(node.Prob.ToString(FloatFormat));
os.Append(" ").Append(node.Cost);
for (MeCabPath path = node.LPath; path != null; path = path.LNext)
{
#if NeedId
os.Append(" ").Append(path.LNode.Id);
#endif
os.Append(" ");
os.Append(":").Append(path.Cost);
os.Append(":").Append(path.Prob.ToString(FloatFormat));
}
os.AppendLine();
}
}
private void Connect(int pos, TNode rNode, TNode[] endNodeList, bool withAllPath)
{
for (; rNode != null; rNode = rNode.BNext)
{
long bestCost = long.MaxValue;
TNode bestNode = null;
for (var lNode = endNodeList[pos]; lNode != null; lNode = lNode.ENext)
{
int localCost = this.connector.Cost(lNode, rNode);
long cost = Math.Min(lNode.Cost + localCost, long.MaxValue - int.MaxValue); // オーバーフロー対策のため最大値を制限
if (cost < bestCost)
{
bestNode = lNode;
bestCost = cost;
}
if (withAllPath)
{
var path = new MeCabPath <TNode>()
{
Cost = localCost,
RNode = rNode,
LNode = lNode,
LNext = rNode.LPath,
RNext = lNode.RPath
};
rNode.LPath = path;
lNode.RPath = path;
}
}
rNode.Prev = bestNode;
rNode.Cost = bestCost;
rNode.BPos = pos;
rNode.EPos = pos + rNode.RLength;
if (rNode.Stat != MeCabNodeStat.Eos)
{
rNode.ENext = endNodeList[rNode.EPos];
endNodeList[rNode.EPos] = rNode;
}
}
}
private void Connect(int pos, TNode rNode, TNode[] endNodeList, bool withAllPath)
{
for (; rNode != null; rNode = rNode.BNext)
{
long bestCost = int.MaxValue; // 2147483647
TNode bestNode = null;
for (var lNode = endNodeList[pos]; lNode != null; lNode = lNode.ENext)
{
int localCost = this.connector.Cost(lNode, rNode);
long cost = lNode.Cost + localCost;
if (cost < bestCost)
{
bestNode = lNode;
bestCost = cost;
}
if (withAllPath)
{
var path = new MeCabPath <TNode>()
{
Cost = localCost,
RNode = rNode,
LNode = lNode,
LNext = rNode.LPath,
RNext = lNode.RPath
};
rNode.LPath = path;
lNode.RPath = path;
}
}
rNode.Prev = bestNode ?? throw new ArgumentException("too long sentence.");
rNode.Cost = bestCost;
rNode.BPos = pos;
rNode.EPos = pos + rNode.RLength;
if (rNode.RLength != 0)
{
rNode.ENext = endNodeList[rNode.EPos];
endNodeList[rNode.EPos] = rNode;
}
}
}
private void ConnectWithAllPath(int pos, MeCabNode rNode, ThreadData work)
{
for (; rNode != null; rNode = rNode.BNext)
{
long bestCost = int.MaxValue; // 2147483647
MeCabNode bestNode = null;
for (MeCabNode lNode = work.EndNodeList[pos]; lNode != null; lNode = lNode.ENext)
{
int lCost = this.connector.Cost(lNode, rNode); // local cost
long cost = lNode.Cost + lCost;
if (cost < bestCost)
{
bestNode = lNode;
bestCost = cost;
}
MeCabPath path = new MeCabPath()
{
Cost = lCost,
RNode = rNode,
LNode = lNode,
LNext = rNode.LPath,
RNext = lNode.RPath
};
rNode.LPath = path;
lNode.RPath = path;
}
if (bestNode == null)
{
throw new ArgumentException("too long sentence.");
}
rNode.Prev = bestNode;
rNode.Next = null;
rNode.Cost = bestCost;
int x = rNode.RLength + pos;
rNode.ENext = work.EndNodeList[x];
work.EndNodeList[x] = rNode;
}
}
private void ConnectWithAllPath(int pos, MeCabNode rNode, ThreadData work)
{
while (true)
{
if (rNode != null)
{
long num = 2147483647L;
MeCabNode meCabNode = null;
for (MeCabNode meCabNode2 = work.EndNodeList[pos]; meCabNode2 != null; meCabNode2 = meCabNode2.ENext)
{
int num2 = this.connector.Cost(meCabNode2, rNode);
long num3 = meCabNode2.Cost + num2;
if (num3 < num)
{
meCabNode = meCabNode2;
num = num3;
}
MeCabPath meCabPath = new MeCabPath();
meCabPath.Cost = num2;
meCabPath.RNode = rNode;
meCabPath.LNode = meCabNode2;
meCabPath.LNext = rNode.LPath;
meCabPath.RNext = meCabNode2.RPath;
MeCabPath meCabPath4 = meCabNode2.RPath = (rNode.LPath = meCabPath);
}
if (meCabNode != null)
{
rNode.Prev = meCabNode;
rNode.Next = null;
rNode.Cost = num;
int num4 = rNode.RLength + pos;
rNode.ENext = work.EndNodeList[num4];
work.EndNodeList[num4] = rNode;
rNode = rNode.BNext;
continue;
}
break;
}
return;
}
throw new ArgumentException("too long sentence.");
}
public void WriteEM(StringBuilder os, MeCabNode bosNode)
{
const float MinProb = 0.0001f;
for (MeCabNode node = bosNode; node != null; node = node.Next)
{
if (node.Prob >= MinProb)
{
os.Append("U\t");
if (node.Stat == MeCabNodeStat.Bos)
{
os.Append("BOS");
}
else if (node.Stat == MeCabNodeStat.Eos)
{
os.Append("EOS");
}
else
{
os.Append(node.Surface);
}
os.Append("\t").Append(node.Feature);
os.Append("\t").Append(node.Prob.ToString(FloatFormat));
os.AppendLine();
}
for (MeCabPath path = node.LPath; path != null; path = path.LNext)
{
if (path.Prob >= MinProb)
{
os.Append("B\t").Append(path.LNode.Feature);
os.Append("\t").Append(node.Feature);
os.Append("\t").Append(path.Prob.ToString(FloatFormat));
os.AppendLine();
}
}
}
os.AppendLine("EOS");
}
public void WriteDump(StringBuilder os, MeCabNode bosNode)
{
for (MeCabNode meCabNode = bosNode; meCabNode != null; meCabNode = meCabNode.Next)
{
if (meCabNode.Stat == MeCabNodeStat.Bos)
{
os.Append("BOS");
}
else if (meCabNode.Stat == MeCabNodeStat.Eos)
{
os.Append("EOS");
}
else
{
os.Append(meCabNode.Surface);
}
os.Append(" ").Append(meCabNode.Feature);
os.Append(" ").Append(meCabNode.BPos);
os.Append(" ").Append(meCabNode.EPos);
os.Append(" ").Append(meCabNode.RCAttr);
os.Append(" ").Append(meCabNode.LCAttr);
os.Append(" ").Append(meCabNode.PosId);
os.Append(" ").Append(meCabNode.CharType);
os.Append(" ").Append((int)meCabNode.Stat);
os.Append(" ").Append(meCabNode.IsBest ? "1" : "0");
os.Append(" ").Append(meCabNode.Alpha.ToString("f6"));
os.Append(" ").Append(meCabNode.Beta.ToString("f6"));
os.Append(" ").Append(meCabNode.Prob.ToString("f6"));
os.Append(" ").Append(meCabNode.Cost);
for (MeCabPath meCabPath = meCabNode.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
os.Append(" ");
os.Append(":").Append(meCabPath.Cost);
os.Append(":").Append(meCabPath.Prob.ToString("f6"));
}
os.AppendLine();
}
}
private MeCabNode BuildAllLattice(ThreadData work)
{
if (this.BuildBestLattice(work) == null)
{
return(null);
}
MeCabNode meCabNode = work.BosNode;
for (int i = 0; i < work.BeginNodeList.Length; i++)
{
for (MeCabNode meCabNode2 = work.BeginNodeList[i]; meCabNode2 != null; meCabNode2 = meCabNode2.BNext)
{
meCabNode.Next = meCabNode2;
meCabNode2.Prev = meCabNode;
meCabNode = meCabNode2;
for (MeCabPath meCabPath = meCabNode2.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
meCabPath.Prob = meCabPath.LNode.Alpha - this.theta * (float)meCabPath.Cost + meCabPath.RNode.Beta - work.Z;
}
}
}
return(work.BosNode);
}
public void WriteEM(StringBuilder os, MeCabNode bosNode)
{
for (MeCabNode meCabNode = bosNode; meCabNode != null; meCabNode = meCabNode.Next)
{
if (meCabNode.Prob >= 0.0001f)
{
os.Append("U\t");
if (meCabNode.Stat == MeCabNodeStat.Bos)
{
os.Append("BOS");
}
else if (meCabNode.Stat == MeCabNodeStat.Eos)
{
os.Append("EOS");
}
else
{
os.Append(meCabNode.Surface);
}
os.Append("\t").Append(meCabNode.Feature);
os.Append("\t").Append(meCabNode.Prob.ToString("f6"));
os.AppendLine();
}
for (MeCabPath meCabPath = meCabNode.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
if (meCabPath.Prob >= 0.0001f)
{
os.Append("B\t").Append(meCabPath.LNode.Feature);
os.Append("\t").Append(meCabNode.Feature);
os.Append("\t").Append(meCabPath.Prob.ToString("f6"));
os.AppendLine();
}
}
}
os.AppendLine("EOS");
}
public unsafe void WriteNode(StringBuilder os, char *p, string sentence, MeCabNode node)
{
while (*p != 0)
{
char c = *p;
if (c != '%')
{
os.Append(*p);
}
else
{
switch (*(++p))
{
default:
os.Append("unkonwn meta char ").Append(*p);
break;
case 'S':
os.Append(sentence);
break;
case 'L':
os.Append(sentence.Length);
break;
case 'm':
os.Append(node.Surface);
break;
case 'M':
os.Append(sentence, node.BPos - node.RLength + node.Length, node.RLength);
break;
case 'h':
os.Append(node.PosId);
break;
case '%':
os.Append('%');
break;
case 'c':
os.Append(node.WCost);
break;
case 'H':
os.Append(node.Feature);
break;
case 't':
os.Append(node.CharType);
break;
case 's':
os.Append(node.Stat);
break;
case 'P':
os.Append(node.Prob);
break;
case 'p':
switch (*(++p))
{
default:
throw new ArgumentException("[iseSCwcnblLh] is required after %p");
case 'i':
throw new ArgumentException("%pi is not supported");
case 'S':
os.Append(sentence, node.BPos, node.RLength - node.Length);
break;
case 's':
os.Append(node.BPos);
break;
case 'e':
os.Append(node.EPos);
break;
case 'C':
os.Append(node.Cost - node.Prev.Cost - node.WCost);
break;
case 'w':
os.Append(node.WCost);
break;
case 'c':
os.Append(node.Cost);
break;
case 'n':
os.Append(node.Cost - node.Prev.Cost);
break;
case 'b':
os.Append((char)(node.IsBest ? 42 : 32));
break;
case 'P':
os.Append(node.Prob);
break;
case 'A':
os.Append(node.Alpha);
break;
case 'B':
os.Append(node.Beta);
break;
case 'l':
os.Append(node.Length);
break;
case 'L':
os.Append(node.RLength);
break;
case 'h':
switch (*(++p))
{
default:
throw new ArgumentException("lr is required after %ph");
case 'l':
os.Append(node.LCAttr);
break;
case 'r':
os.Append(node.RCAttr);
break;
}
break;
case 'p':
{
char c2 = *(++p);
char c3 = *(++p);
if (c3 == '\\')
{
c3 = this.GetEscapedChar(*(++p));
}
if (node.LPath == null)
{
throw new InvalidOperationException("no path information, use -l option");
}
for (MeCabPath meCabPath = node.LPath; meCabPath != null; meCabPath = meCabPath.LNext)
{
if (meCabPath != node.LPath)
{
os.Append(c3);
}
switch (c2)
{
case 'i':
os.Append(meCabPath.LNode.PosId);
break;
case 'c':
os.Append(meCabPath.Cost);
break;
case 'P':
os.Append(meCabPath.Prob);
break;
default:
throw new ArgumentException("[icP] is required after %pp");
}
}
break;
}
}
break;
case 'F':
case 'f':
{
char value = '\t';
if (*p == 'F')
{
value = ((*(++p) != '\\') ? (*p) : this.GetEscapedChar(*(++p)));
}
if (*(++p) != '[')
{
throw new ArgumentException("cannot find '['");
}
string[] array = node.Feature.Split(',');
int num = 0;
while (true)
{
if (char.IsDigit(*(++p)))
{
num = num * 10 + (*p - 48);
}
else
{
if (num >= array.Length)
{
throw new ArgumentException("given index is out of range");
}
os.Append(array[num]);
if (*(++p) != ',')
{
break;
}
os.Append(value);
num = 0;
}
}
if (*p == ']')
{
break;
}
throw new ArgumentException("cannot find ']'");
}
}
}
p++;
}
}
private void ConnectWithAllPath(int pos, MeCabNode rNode, ThreadData work)
{
for (; rNode != null; rNode = rNode.BNext)
{
long bestCost = int.MaxValue; // 2147483647
MeCabNode bestNode = null;
for (MeCabNode lNode = work.EndNodeList[pos]; lNode != null; lNode = lNode.ENext)
{
int lCost = this.connector.Cost(lNode, rNode); // local cost
long cost = lNode.Cost + lCost;
if (cost < bestCost)
{
bestNode = lNode;
bestCost = cost;
}
MeCabPath path = new MeCabPath()
{
Cost = lCost,
RNode = rNode,
LNode = lNode,
LNext = rNode.LPath,
RNext = lNode.RPath
};
rNode.LPath = path;
lNode.RPath = path;
}
if (bestNode == null) throw new ArgumentException("too long sentence.");
rNode.Prev = bestNode;
rNode.Next = null;
rNode.Cost = bestCost;
int x = rNode.RLength + pos;
rNode.ENext = work.EndNodeList[x];
work.EndNodeList[x] = rNode;
}
}
public unsafe void WriteNode(StringBuilder os, char *p, string sentence, MeCabNode node)
{
for (; *p != 0x0; p++)
{
switch (*p)
{
default: os.Append(*p); break;
case '%':
switch (*++p)
{
default: os.Append("unkonwn meta char ").Append(*p); break;
case 'S': os.Append(sentence); break;
case 'L': os.Append(sentence.Length); break;
case 'm': os.Append(node.Surface); break;
case 'M': os.Append(sentence, (node.BPos - node.RLength + node.Length), node.RLength); break;
case 'h': os.Append(node.PosId); break;
case '%': os.Append('%'); break;
case 'c': os.Append(node.WCost); break;
case 'H': os.Append(node.Feature); break;
case 't': os.Append(node.CharType); break;
case 's': os.Append(node.Stat); break;
case 'P': os.Append(node.Prob); break;
case 'p':
switch (*++p)
{
default: throw new ArgumentException("[iseSCwcnblLh] is required after %p");
#if NeedId
case 'i': os.Append(node.Id); break;
#else
case 'i': throw new ArgumentException("%pi is not supported");
#endif
case 'S': os.Append(sentence, node.BPos, (node.RLength - node.Length)); break;
case 's': os.Append(node.BPos); break;
case 'e': os.Append(node.EPos); break;
case 'C': os.Append(node.Cost - node.Prev.Cost - node.WCost); break;
case 'w': os.Append(node.WCost); break;
case 'c': os.Append(node.Cost); break;
case 'n': os.Append(node.Cost - node.Prev.Cost); break;
case 'b': os.Append(node.IsBest ? '*' : ' '); break;
case 'P': os.Append(node.Prob); break;
case 'A': os.Append(node.Alpha); break;
case 'B': os.Append(node.Beta); break;
case 'l': os.Append(node.Length); break;
case 'L': os.Append(node.RLength); break;
case 'h':
switch (*++p)
{
default: throw new ArgumentException("lr is required after %ph");
case 'l': os.Append(node.LCAttr); break;
case 'r': os.Append(node.RCAttr); break;
}
; break;
case 'p':
char mode = *++p;
char sep = *++p;
if (sep == '\\')
{
sep = this.GetEscapedChar(*++p);
}
if (node.LPath == null)
{
throw new InvalidOperationException("no path information, use -l option");
}
for (MeCabPath path = node.LPath; path != null; path = path.LNext)
{
if (path != node.LPath)
{
os.Append(sep);
}
switch (mode)
{
case 'i': os.Append(path.LNode.PosId); break;
case 'c': os.Append(path.Cost); break;
case 'P': os.Append(path.Prob); break;
default: throw new ArgumentException("[icP] is required after %pp");
}
}
break;
}
break;
case 'f':
case 'F':
char separator = '\t';
if (*p == 'F')
{
if (*++p == '\\')
{
separator = this.GetEscapedChar(*++p);
}
else
{
separator = *p;
}
}
if (*++p != '[')
{
throw new ArgumentException("cannot find '['");
}
string[] features = node.Feature.Split(',');
int n = 0;
while (true)
{
if (char.IsDigit(*++p))
{
n = n * 10 + (*p - '0'); continue;
}
if (n >= features.Length)
{
throw new ArgumentException("given index is out of range");
}
os.Append(features[n]);
if (*++p == ',')
{
os.Append(separator); n = 0; continue;
}
if (*p == ']')
{
break;
}
throw new ArgumentException("cannot find ']'");
}
break;
}
break;
}
}
}