/// <summary>Adds dependencies to list depList.</summary>
/// <remarks>
/// Adds dependencies to list depList. These are in terms of the original
/// tag set not the reduced (projected) tag set.
/// </remarks>
protected internal static MLEDependencyGrammar.EndHead TreeToDependencyHelper(Tree tree, IList <IntDependency> depList, int loc, IIndex <string> wordIndex, IIndex <string> tagIndex)
{
// try {
// PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.out,"GB18030"),true);
// tree.pennPrint(pw);
// }
// catch (UnsupportedEncodingException e) {}
if (tree.IsLeaf() || tree.IsPreTerminal())
{
MLEDependencyGrammar.EndHead tempEndHead = new MLEDependencyGrammar.EndHead();
tempEndHead.head = loc;
tempEndHead.end = loc + 1;
return(tempEndHead);
}
Tree[] kids = tree.Children();
if (kids.Length == 1)
{
return(TreeToDependencyHelper(kids[0], depList, loc, wordIndex, tagIndex));
}
MLEDependencyGrammar.EndHead tempEndHead_1 = TreeToDependencyHelper(kids[0], depList, loc, wordIndex, tagIndex);
int lHead = tempEndHead_1.head;
int split = tempEndHead_1.end;
tempEndHead_1 = TreeToDependencyHelper(kids[1], depList, tempEndHead_1.end, wordIndex, tagIndex);
int end = tempEndHead_1.end;
int rHead = tempEndHead_1.head;
string hTag = ((IHasTag)tree.Label()).Tag();
string lTag = ((IHasTag)kids[0].Label()).Tag();
string rTag = ((IHasTag)kids[1].Label()).Tag();
string hWord = ((IHasWord)tree.Label()).Word();
string lWord = ((IHasWord)kids[0].Label()).Word();
string rWord = ((IHasWord)kids[1].Label()).Word();
bool leftHeaded = hWord.Equals(lWord);
string aTag = (leftHeaded ? rTag : lTag);
string aWord = (leftHeaded ? rWord : lWord);
int hT = tagIndex.IndexOf(hTag);
int aT = tagIndex.IndexOf(aTag);
int hW = (wordIndex.Contains(hWord) ? wordIndex.IndexOf(hWord) : wordIndex.IndexOf(LexiconConstants.UnknownWord));
int aW = (wordIndex.Contains(aWord) ? wordIndex.IndexOf(aWord) : wordIndex.IndexOf(LexiconConstants.UnknownWord));
int head = (leftHeaded ? lHead : rHead);
int arg = (leftHeaded ? rHead : lHead);
IntDependency dependency = new IntDependency(hW, hT, aW, aT, leftHeaded, (leftHeaded ? split - head - 1 : head - split));
depList.Add(dependency);
IntDependency stopL = new IntDependency(aW, aT, IntTaggedWord.StopWordInt, IntTaggedWord.StopTagInt, false, (leftHeaded ? arg - split : arg - loc));
depList.Add(stopL);
IntDependency stopR = new IntDependency(aW, aT, IntTaggedWord.StopWordInt, IntTaggedWord.StopTagInt, true, (leftHeaded ? end - arg - 1 : split - arg - 1));
depList.Add(stopR);
//System.out.println("Adding: "+dependency+" at "+tree.label());
tempEndHead_1.head = head;
return(tempEndHead_1);
}
/// <summary>The dependency arg is still in the full tag space.</summary>
/// <param name="dependency">An opbserved dependency</param>
/// <param name="count">The weight of the dependency</param>
protected internal virtual void ExpandDependency(IntDependency dependency, double count)
{
//if (Test.prunePunc && pruneTW(dependency.arg))
// return;
if (dependency.head == null || dependency.arg == null)
{
return;
}
if (dependency.arg.word != IntTaggedWord.StopWordInt)
{
ExpandArg(dependency, ValenceBin(dependency.distance), count);
}
ExpandStop(dependency, DistanceBin(dependency.distance), count, true);
}
/// <summary>
/// This is a custom interner that simultaneously creates and interns
/// an IntDependency.
/// </summary>
/// <returns>An interned IntDependency</returns>
protected internal virtual IntDependency Intern(IntTaggedWord headTW, IntTaggedWord argTW, bool leftHeaded, short dist)
{
IDictionary <IntDependency, IntDependency> map = expandDependencyMap;
IntDependency internTempDependency = new IntDependency(itwInterner.Intern(headTW), itwInterner.Intern(argTW), leftHeaded, dist);
IntDependency returnDependency = internTempDependency;
if (map != null)
{
returnDependency = map[internTempDependency];
if (returnDependency == null)
{
map[internTempDependency] = internTempDependency;
returnDependency = internTempDependency;
}
}
return(returnDependency);
}
/// <summary>Add this dependency with the given count to the grammar.</summary>
/// <remarks>
/// Add this dependency with the given count to the grammar.
/// This is the main entry point of MLEDependencyGrammarExtractor.
/// This is a dependency represented in the full tag space.
/// </remarks>
public virtual void AddRule(IntDependency dependency, double count)
{
if (!directional)
{
dependency = new IntDependency(dependency.head, dependency.arg, false, dependency.distance);
}
// coreDependencies.incrementCount(dependency, count);
/*new IntDependency(dependency.head.word,
* dependency.head.tag,
* dependency.arg.word,
* dependency.arg.tag,
* dependency.leftHeaded,
* dependency.distance), count);
*/
ExpandDependency(dependency, count);
}
/// <summary>
/// Populates data in this DependencyGrammar from the character stream
/// given by the Reader r.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public override void ReadData(BufferedReader @in)
{
string Left = "left";
int lineNum = 1;
// all lines have one rule per line
bool doingStop = false;
for (string line = @in.ReadLine(); line != null && line.Length > 0; line = @in.ReadLine())
{
try
{
if (line.Equals("BEGIN_STOP"))
{
doingStop = true;
continue;
}
string[] fields = StringUtils.SplitOnCharWithQuoting(line, ' ', '\"', '\\');
// split on spaces, quote with doublequote, and escape with backslash
// System.out.println("fields:\n" + fields[0] + "\n" + fields[1] + "\n" + fields[2] + "\n" + fields[3] + "\n" + fields[4] + "\n" + fields[5]);
short distance = (short)System.Convert.ToInt32(fields[4]);
IntTaggedWord tempHead = new IntTaggedWord(fields[0], '/', wordIndex, tagIndex);
IntTaggedWord tempArg = new IntTaggedWord(fields[2], '/', wordIndex, tagIndex);
IntDependency tempDependency = new IntDependency(tempHead, tempArg, fields[3].Equals(Left), distance);
double count = double.Parse(fields[5]);
if (doingStop)
{
ExpandStop(tempDependency, distance, count, false);
}
else
{
ExpandArg(tempDependency, distance, count);
}
}
catch (Exception e)
{
IOException ioe = new IOException("Error on line " + lineNum + ": " + line);
ioe.InitCause(e);
throw ioe;
}
// System.out.println("read line " + lineNum + ": " + line);
lineNum++;
}
}
private void ExpandStop(IntDependency dependency, short distBinDist, double count, bool wildForStop)
{
IntTaggedWord headT = GetCachedITW(dependency.head.tag);
IntTaggedWord head = new IntTaggedWord(dependency.head.word, TagBin(dependency.head.tag));
//dependency.head;
IntTaggedWord arg = new IntTaggedWord(dependency.arg.word, TagBin(dependency.arg.tag));
//dependency.arg;
bool leftHeaded = dependency.leftHeaded;
if (arg.word == IntTaggedWord.StopWordInt)
{
stopCounter.IncrementCount(Intern(head, arg, leftHeaded, distBinDist), count);
stopCounter.IncrementCount(Intern(headT, arg, leftHeaded, distBinDist), count);
}
if (wildForStop || arg.word != IntTaggedWord.StopWordInt)
{
stopCounter.IncrementCount(Intern(head, wildTW, leftHeaded, distBinDist), count);
stopCounter.IncrementCount(Intern(headT, wildTW, leftHeaded, distBinDist), count);
}
}
/// <summary>
/// Return the probability (as a real number between 0 and 1) of stopping
/// rather than generating another argument at this position.
/// </summary>
/// <param name="dependency">
/// The dependency used as the basis for stopping on.
/// Tags are assumed to be in the TagProjection space.
/// </param>
/// <returns>The probability of generating this stop probability</returns>
protected internal virtual double GetStopProb(IntDependency dependency)
{
short binDistance = DistanceBin(dependency.distance);
IntTaggedWord unknownHead = new IntTaggedWord(-1, dependency.head.tag);
IntTaggedWord anyHead = new IntTaggedWord(IntTaggedWord.AnyWordInt, dependency.head.tag);
IntDependency temp = new IntDependency(dependency.head, stopTW, dependency.leftHeaded, binDistance);
double c_stop_hTWds = stopCounter.GetCount(temp);
temp = new IntDependency(unknownHead, stopTW, dependency.leftHeaded, binDistance);
double c_stop_hTds = stopCounter.GetCount(temp);
temp = new IntDependency(dependency.head, wildTW, dependency.leftHeaded, binDistance);
double c_hTWds = stopCounter.GetCount(temp);
temp = new IntDependency(anyHead, wildTW, dependency.leftHeaded, binDistance);
double c_hTds = stopCounter.GetCount(temp);
double p_stop_hTds = (c_hTds > 0.0 ? c_stop_hTds / c_hTds : 1.0);
double pb_stop_hTWds = (c_stop_hTWds + smooth_stop * p_stop_hTds) / (c_hTWds + smooth_stop);
return(pb_stop_hTWds);
}
/// <summary>Collect counts for a non-STOP dependent.</summary>
/// <remarks>
/// Collect counts for a non-STOP dependent.
/// The dependency arg is still in the full tag space.
/// </remarks>
/// <param name="dependency">A non-stop dependency</param>
/// <param name="valBinDist">A binned distance</param>
/// <param name="count">The weight with which to add this dependency</param>
private void ExpandArg(IntDependency dependency, short valBinDist, double count)
{
IntTaggedWord headT = GetCachedITW(dependency.head.tag);
IntTaggedWord argT = GetCachedITW(dependency.arg.tag);
IntTaggedWord head = new IntTaggedWord(dependency.head.word, TagBin(dependency.head.tag));
//dependency.head;
IntTaggedWord arg = new IntTaggedWord(dependency.arg.word, TagBin(dependency.arg.tag));
//dependency.arg;
bool leftHeaded = dependency.leftHeaded;
// argCounter stores stuff in both the original and the reduced tag space???
argCounter.IncrementCount(Intern(head, arg, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headT, arg, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(head, argT, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headT, argT, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(head, wildTW, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headT, wildTW, leftHeaded, valBinDist), count);
// the WILD head stats are always directionless and not useDistance!
argCounter.IncrementCount(Intern(wildTW, arg, false, (short)-1), count);
argCounter.IncrementCount(Intern(wildTW, argT, false, (short)-1), count);
if (useSmoothTagProjection)
{
// added stuff to do more smoothing. CDM Jan 2007
IntTaggedWord headP = new IntTaggedWord(dependency.head.word, TagProject(dependency.head.tag));
IntTaggedWord headTP = new IntTaggedWord(IntTaggedWord.AnyWordInt, TagProject(dependency.head.tag));
IntTaggedWord argP = new IntTaggedWord(dependency.arg.word, TagProject(dependency.arg.tag));
IntTaggedWord argTP = new IntTaggedWord(IntTaggedWord.AnyWordInt, TagProject(dependency.arg.tag));
argCounter.IncrementCount(Intern(headP, argP, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headTP, argP, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headP, argTP, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headTP, argTP, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headP, wildTW, leftHeaded, valBinDist), count);
argCounter.IncrementCount(Intern(headTP, wildTW, leftHeaded, valBinDist), count);
// the WILD head stats are always directionless and not useDistance!
argCounter.IncrementCount(Intern(wildTW, argP, false, (short)-1), count);
argCounter.IncrementCount(Intern(wildTW, argTP, false, (short)-1), count);
argCounter.IncrementCount(Intern(wildTW, new IntTaggedWord(dependency.head.word, IntTaggedWord.AnyTagInt), false, (short)-1), count);
}
numWordTokens++;
}
public override double ScoreTB(IntDependency dependency)
{
//return op.testOptions.depWeight * Math.log(probSimilarWordAvg(dependency));
return(op.testOptions.depWeight * Math.Log(ProbTBwithSimWords(dependency)));
}
private double ProbSimilarWordAvg(IntDependency dep)
{
double regProb = ProbTB(dep);
statsCounter.IncrementCount("total");
IList <Triple <int, string, double> > sim2arg = simArgMap[new Pair <int, string>(dep.arg.word, StringBasicCategory(dep.arg.tag))];
IList <Triple <int, string, double> > sim2head = simHeadMap[new Pair <int, string>(dep.head.word, StringBasicCategory(dep.head.tag))];
if (sim2head == null && sim2arg == null)
{
return(regProb);
}
double sumScores = 0;
double sumWeights = 0;
if (sim2head == null)
{
statsCounter.IncrementCount("aSim");
foreach (Triple <int, string, double> simArg in sim2arg)
{
//double weight = 1 - simArg.third;
double weight = Math.Exp(-50 * simArg.third);
for (int tag = 0; tag < numT; tag++)
{
if (!StringBasicCategory(tag).Equals(simArg.second))
{
continue;
}
IntTaggedWord tempArg = new IntTaggedWord(simArg.first, tag);
IntDependency tempDep = new IntDependency(dep.head, tempArg, dep.leftHeaded, dep.distance);
double probArg = Math.Exp(lex.Score(tempArg, 0, wordIndex.Get(tempArg.word), null));
if (probArg == 0.0)
{
continue;
}
sumScores += ProbTB(tempDep) * weight / probArg;
sumWeights += weight;
}
}
}
else
{
if (sim2arg == null)
{
statsCounter.IncrementCount("hSim");
foreach (Triple <int, string, double> simHead in sim2head)
{
//double weight = 1 - simHead.third;
double weight = Math.Exp(-50 * simHead.third);
for (int tag = 0; tag < numT; tag++)
{
if (!StringBasicCategory(tag).Equals(simHead.second))
{
continue;
}
IntTaggedWord tempHead = new IntTaggedWord(simHead.first, tag);
IntDependency tempDep = new IntDependency(tempHead, dep.arg, dep.leftHeaded, dep.distance);
sumScores += ProbTB(tempDep) * weight;
sumWeights += weight;
}
}
}
else
{
statsCounter.IncrementCount("hSim");
statsCounter.IncrementCount("aSim");
statsCounter.IncrementCount("aSim&hSim");
foreach (Triple <int, string, double> simArg in sim2arg)
{
for (int aTag = 0; aTag < numT; aTag++)
{
if (!StringBasicCategory(aTag).Equals(simArg.second))
{
continue;
}
IntTaggedWord tempArg = new IntTaggedWord(simArg.first, aTag);
double probArg = Math.Exp(lex.Score(tempArg, 0, wordIndex.Get(tempArg.word), null));
if (probArg == 0.0)
{
continue;
}
foreach (Triple <int, string, double> simHead in sim2head)
{
for (int hTag = 0; hTag < numT; hTag++)
{
if (!StringBasicCategory(hTag).Equals(simHead.second))
{
continue;
}
IntTaggedWord tempHead = new IntTaggedWord(simHead.first, aTag);
IntDependency tempDep = new IntDependency(tempHead, tempArg, dep.leftHeaded, dep.distance);
//double weight = (1-simHead.third) * (1-simArg.third);
double weight = Math.Exp(-50 * simHead.third) * Math.Exp(-50 * simArg.third);
sumScores += ProbTB(tempDep) * weight / probArg;
sumWeights += weight;
}
}
}
}
}
}
IntDependency temp = new IntDependency(dep.head, wildTW, dep.leftHeaded, dep.distance);
double countHead = argCounter.GetCount(temp);
double simProb;
if (sim2arg == null)
{
simProb = sumScores / sumWeights;
}
else
{
double probArg = Math.Exp(lex.Score(dep.arg, 0, wordIndex.Get(dep.arg.word), null));
simProb = probArg * sumScores / sumWeights;
}
if (simProb == 0)
{
statsCounter.IncrementCount("simProbZero");
}
if (regProb == 0)
{
// log.info("zero reg prob");
statsCounter.IncrementCount("regProbZero");
}
double smoothProb = (countHead * regProb + simSmooth * simProb) / (countHead + simSmooth);
if (smoothProb == 0)
{
// log.info("zero smooth prob");
statsCounter.IncrementCount("smoothProbZero");
}
return(smoothProb);
}
/*
** An alternative kind of smoothing.
** The first one is "probSimilarWordAvg" implemented by Galen
** This one is trying to modify "probTB" in MLEDependencyGrammar using the simWords list we have
** -pichuan
*/
private double ProbTBwithSimWords(IntDependency dependency)
{
bool leftHeaded = dependency.leftHeaded && directional;
IntTaggedWord unknownHead = new IntTaggedWord(-1, dependency.head.tag);
IntTaggedWord unknownArg = new IntTaggedWord(-1, dependency.arg.tag);
short distance = dependency.distance;
// int hW = dependency.head.word;
// int aW = dependency.arg.word;
IntTaggedWord aTW = dependency.arg;
// IntTaggedWord hTW = dependency.head;
double pb_stop_hTWds = GetStopProb(dependency);
bool isRoot = RootTW(dependency.head);
if (dependency.arg.word == -2)
{
// did we generate stop?
if (isRoot)
{
return(0.0);
}
return(pb_stop_hTWds);
}
double pb_go_hTWds = 1.0 - pb_stop_hTWds;
if (isRoot)
{
pb_go_hTWds = 1.0;
}
// generate the argument
int valenceBinDistance = ValenceBin(distance);
// KEY:
// c_ count of
// p_ MLE prob of
// pb_ MAP prob of
// a arg
// h head
// T tag
// W word
// d direction
// ds distance
IntDependency temp = new IntDependency(dependency.head, dependency.arg, leftHeaded, valenceBinDistance);
double c_aTW_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(dependency.head, unknownArg, leftHeaded, valenceBinDistance);
double c_aT_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(dependency.head, wildTW, leftHeaded, valenceBinDistance);
double c_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(unknownHead, dependency.arg, leftHeaded, valenceBinDistance);
double c_aTW_hTd = argCounter.GetCount(temp);
temp = new IntDependency(unknownHead, unknownArg, leftHeaded, valenceBinDistance);
double c_aT_hTd = argCounter.GetCount(temp);
temp = new IntDependency(unknownHead, wildTW, leftHeaded, valenceBinDistance);
double c_hTd = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, dependency.arg, false, -1);
double c_aTW = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, unknownArg, false, -1);
double c_aT = argCounter.GetCount(temp);
// do the magic
double p_aTW_hTd = (c_hTd > 0.0 ? c_aTW_hTd / c_hTd : 0.0);
double p_aT_hTd = (c_hTd > 0.0 ? c_aT_hTd / c_hTd : 0.0);
double p_aTW_aT = (c_aTW > 0.0 ? c_aTW / c_aT : 1.0);
double pb_aTW_hTWd;
// = (c_aTW_hTWd + smooth_aTW_hTWd * p_aTW_hTd) / (c_hTWd + smooth_aTW_hTWd);
double pb_aT_hTWd = (c_aT_hTWd + smooth_aT_hTWd * p_aT_hTd) / (c_hTWd + smooth_aT_hTWd);
double score;
// = (interp * pb_aTW_hTWd + (1.0 - interp) * p_aTW_aT * pb_aT_hTWd) * pb_go_hTWds;
/* smooth by simWords -pichuan */
IList <Triple <int, string, double> > sim2arg = simArgMap[new Pair <int, string>(dependency.arg.word, StringBasicCategory(dependency.arg.tag))];
IList <Triple <int, string, double> > sim2head = simHeadMap[new Pair <int, string>(dependency.head.word, StringBasicCategory(dependency.head.tag))];
IList <int> simArg = new List <int>();
IList <int> simHead = new List <int>();
if (sim2arg != null)
{
foreach (Triple <int, string, double> t in sim2arg)
{
simArg.Add(t.first);
}
}
if (sim2head != null)
{
foreach (Triple <int, string, double> t in sim2head)
{
simHead.Add(t.first);
}
}
double cSim_aTW_hTd = 0;
double cSim_hTd = 0;
foreach (int h in simHead)
{
IntTaggedWord hWord = new IntTaggedWord(h, dependency.head.tag);
temp = new IntDependency(hWord, dependency.arg, dependency.leftHeaded, dependency.distance);
cSim_aTW_hTd += argCounter.GetCount(temp);
temp = new IntDependency(hWord, wildTW, dependency.leftHeaded, dependency.distance);
cSim_hTd += argCounter.GetCount(temp);
}
double pSim_aTW_hTd = (cSim_hTd > 0.0 ? cSim_aTW_hTd / cSim_hTd : 0.0);
// P(Wa,Ta|Th)
//if (simHead.size() > 0 && cSim_hTd == 0.0) {
if (pSim_aTW_hTd > 0.0)
{
//System.out.println("# simHead("+dependency.head.word+"-"+wordNumberer.object(dependency.head.word)+") =\t"+cSim_hTd);
System.Console.Out.WriteLine(dependency + "\t" + pSim_aTW_hTd);
}
//System.out.println(wordNumberer);
//pb_aTW_hTWd = (c_aTW_hTWd + smooth_aTW_hTWd * pSim_aTW_hTd + smooth_aTW_hTWd * p_aTW_hTd) / (c_hTWd + smooth_aTW_hTWd + smooth_aTW_hTWd);
//if (pSim_aTW_hTd > 0.0) {
double smoothSim_aTW_hTWd = 17.7;
double smooth_aTW_hTWd = 17.7 * 2;
//smooth_aTW_hTWd = smooth_aTW_hTWd*2;
pb_aTW_hTWd = (c_aTW_hTWd + smoothSim_aTW_hTWd * pSim_aTW_hTd + smooth_aTW_hTWd * p_aTW_hTd) / (c_hTWd + smoothSim_aTW_hTWd + smooth_aTW_hTWd);
System.Console.Out.WriteLine(dependency);
System.Console.Out.WriteLine(c_aTW_hTWd + " + " + smoothSim_aTW_hTWd + " * " + pSim_aTW_hTd + " + " + smooth_aTW_hTWd + " * " + p_aTW_hTd);
System.Console.Out.WriteLine("-------------------------------- = " + pb_aTW_hTWd);
System.Console.Out.WriteLine(c_hTWd + " + " + smoothSim_aTW_hTWd + " + " + smooth_aTW_hTWd);
System.Console.Out.WriteLine();
//}
//pb_aT_hTWd = (c_aT_hTWd + smooth_aT_hTWd * p_aT_hTd) / (c_hTWd + smooth_aT_hTWd);
score = (interp * pb_aTW_hTWd + (1.0 - interp) * p_aTW_aT * pb_aT_hTWd) * pb_go_hTWds;
if (op.testOptions.prunePunc && PruneTW(aTW))
{
return(1.0);
}
if (double.IsNaN(score))
{
score = 0.0;
}
//if (op.testOptions.rightBonus && ! dependency.leftHeaded)
// score -= 0.2;
if (score < MinProbability)
{
score = 0.0;
}
return(score);
}
public abstract double ScoreTB(IntDependency arg1);
// this method tag bins
public virtual double ScoreTB(int headWord, int headTag, int argWord, int argTag, bool leftHeaded, int dist)
{
IntDependency tempDependency = new IntDependency(headWord, headTag, argWord, argTag, leftHeaded, dist);
return(ScoreTB(tempDependency));
}
public virtual double Score(IntDependency dependency)
{
return(ScoreTB(dependency.head.word, TagBin(dependency.head.tag), dependency.arg.word, TagBin(dependency.arg.tag), dependency.leftHeaded, dependency.distance));
}
public virtual double CountHistory(IntDependency dependency)
{
IntDependency temp = new IntDependency(dependency.head.word, TagBin(dependency.head.tag), wildTW.word, wildTW.tag, dependency.leftHeaded, ValenceBin(dependency.distance));
return(argCounter.GetCount(temp));
}
/// <summary>
/// Tune the smoothing and interpolation parameters of the dependency
/// grammar based on a tuning treebank.
/// </summary>
/// <param name="trees">A Collection of Trees for setting parameters</param>
public override void Tune(ICollection <Tree> trees)
{
IList <IntDependency> deps = new List <IntDependency>();
foreach (Tree tree in trees)
{
Sharpen.Collections.AddAll(deps, TreeToDependencyList(tree, wordIndex, tagIndex));
}
double bestScore = double.NegativeInfinity;
double bestSmooth_stop = 0.0;
double bestSmooth_aTW_hTWd = 0.0;
double bestSmooth_aT_hTWd = 0.0;
double bestInterp = 0.0;
log.Info("Tuning smooth_stop...");
for (smooth_stop = 1.0 / 100.0; smooth_stop < 100.0; smooth_stop *= 1.25)
{
double totalScore = 0.0;
foreach (IntDependency dep in deps)
{
if (!RootTW(dep.head))
{
double stopProb = GetStopProb(dep);
if (!dep.arg.Equals(stopTW))
{
stopProb = 1.0 - stopProb;
}
if (stopProb > 0.0)
{
totalScore += Math.Log(stopProb);
}
}
}
if (totalScore > bestScore)
{
bestScore = totalScore;
bestSmooth_stop = smooth_stop;
}
}
smooth_stop = bestSmooth_stop;
log.Info("Tuning selected smooth_stop: " + smooth_stop);
for (IEnumerator <IntDependency> iter = deps.GetEnumerator(); iter.MoveNext();)
{
IntDependency dep = iter.Current;
if (dep.arg.Equals(stopTW))
{
iter.Remove();
}
}
log.Info("Tuning other parameters...");
if (!useSmoothTagProjection)
{
bestScore = double.NegativeInfinity;
for (smooth_aTW_hTWd = 0.5; smooth_aTW_hTWd < 100.0; smooth_aTW_hTWd *= 1.25)
{
log.Info(".");
for (smooth_aT_hTWd = 0.5; smooth_aT_hTWd < 100.0; smooth_aT_hTWd *= 1.25)
{
for (interp = 0.02; interp < 1.0; interp += 0.02)
{
double totalScore = 0.0;
foreach (IntDependency dep in deps)
{
double score = Score(dep);
if (score > double.NegativeInfinity)
{
totalScore += score;
}
}
if (totalScore > bestScore)
{
bestScore = totalScore;
bestInterp = interp;
bestSmooth_aTW_hTWd = smooth_aTW_hTWd;
bestSmooth_aT_hTWd = smooth_aT_hTWd;
log.Info("Current best interp: " + interp + " with score " + totalScore);
}
}
}
}
smooth_aTW_hTWd = bestSmooth_aTW_hTWd;
smooth_aT_hTWd = bestSmooth_aT_hTWd;
interp = bestInterp;
}
else
{
// for useSmoothTagProjection
double bestSmooth_aTW_aT = 0.0;
double bestSmooth_aTW_hTd = 0.0;
double bestSmooth_aT_hTd = 0.0;
bestScore = double.NegativeInfinity;
for (smooth_aTW_hTWd = 1.125; smooth_aTW_hTWd < 100.0; smooth_aTW_hTWd *= 1.5)
{
log.Info("#");
for (smooth_aT_hTWd = 1.125; smooth_aT_hTWd < 100.0; smooth_aT_hTWd *= 1.5)
{
log.Info(":");
for (smooth_aTW_aT = 1.125; smooth_aTW_aT < 200.0; smooth_aTW_aT *= 1.5)
{
log.Info(".");
for (smooth_aTW_hTd = 1.125; smooth_aTW_hTd < 100.0; smooth_aTW_hTd *= 1.5)
{
for (smooth_aT_hTd = 1.125; smooth_aT_hTd < 100.0; smooth_aT_hTd *= 1.5)
{
for (interp = 0.2; interp <= 0.8; interp += 0.02)
{
double totalScore = 0.0;
foreach (IntDependency dep in deps)
{
double score = Score(dep);
if (score > double.NegativeInfinity)
{
totalScore += score;
}
}
if (totalScore > bestScore)
{
bestScore = totalScore;
bestInterp = interp;
bestSmooth_aTW_hTWd = smooth_aTW_hTWd;
bestSmooth_aT_hTWd = smooth_aT_hTWd;
bestSmooth_aTW_aT = smooth_aTW_aT;
bestSmooth_aTW_hTd = smooth_aTW_hTd;
bestSmooth_aT_hTd = smooth_aT_hTd;
log.Info("Current best interp: " + interp + " with score " + totalScore);
}
}
}
}
}
}
log.Info();
}
smooth_aTW_hTWd = bestSmooth_aTW_hTWd;
smooth_aT_hTWd = bestSmooth_aT_hTWd;
smooth_aTW_aT = bestSmooth_aTW_aT;
smooth_aTW_hTd = bestSmooth_aTW_hTd;
smooth_aT_hTd = bestSmooth_aT_hTd;
interp = bestInterp;
}
log.Info("\nTuning selected smooth_aTW_hTWd: " + smooth_aTW_hTWd + " smooth_aT_hTWd: " + smooth_aT_hTWd + " interp: " + interp + " smooth_aTW_aT: " + smooth_aTW_aT + " smooth_aTW_hTd: " + smooth_aTW_hTd + " smooth_aT_hTd: " + smooth_aT_hTd);
}
/// <summary>
/// Calculate the probability of a dependency as a real probability between
/// 0 and 1 inclusive.
/// </summary>
/// <param name="dependency">
/// The dependency for which the probability is to be
/// calculated. The tags in this dependency are in the reduced
/// TagProjection space.
/// </param>
/// <returns>The probability of the dependency</returns>
protected internal virtual double ProbTB(IntDependency dependency)
{
// System.out.println("tagIndex: " + tagIndex);
bool leftHeaded = dependency.leftHeaded && directional;
int hW = dependency.head.word;
int aW = dependency.arg.word;
short hT = dependency.head.tag;
short aT = dependency.arg.tag;
IntTaggedWord aTW = dependency.arg;
IntTaggedWord hTW = dependency.head;
bool isRoot = RootTW(dependency.head);
double pb_stop_hTWds;
if (isRoot)
{
pb_stop_hTWds = 0.0;
}
else
{
pb_stop_hTWds = GetStopProb(dependency);
}
if (dependency.arg.word == IntTaggedWord.StopWordInt)
{
// did we generate stop?
return(pb_stop_hTWds);
}
double pb_go_hTWds = 1.0 - pb_stop_hTWds;
// generate the argument
short binDistance = ValenceBin(dependency.distance);
// KEY:
// c_ count of (read as joint count of first and second)
// p_ MLE prob of (or MAP if useSmoothTagProjection)
// pb_ MAP prob of (read as prob of first given second thing)
// a arg
// h head
// T tag
// PT projected tag
// W word
// d direction
// ds distance (implicit: there when direction is mentioned!)
IntTaggedWord anyHead = new IntTaggedWord(IntTaggedWord.AnyWordInt, dependency.head.tag);
IntTaggedWord anyArg = new IntTaggedWord(IntTaggedWord.AnyWordInt, dependency.arg.tag);
IntTaggedWord anyTagArg = new IntTaggedWord(dependency.arg.word, IntTaggedWord.AnyTagInt);
IntDependency temp = new IntDependency(dependency.head, dependency.arg, leftHeaded, binDistance);
double c_aTW_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(dependency.head, anyArg, leftHeaded, binDistance);
double c_aT_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(dependency.head, wildTW, leftHeaded, binDistance);
double c_hTWd = argCounter.GetCount(temp);
temp = new IntDependency(anyHead, dependency.arg, leftHeaded, binDistance);
double c_aTW_hTd = argCounter.GetCount(temp);
temp = new IntDependency(anyHead, anyArg, leftHeaded, binDistance);
double c_aT_hTd = argCounter.GetCount(temp);
temp = new IntDependency(anyHead, wildTW, leftHeaded, binDistance);
double c_hTd = argCounter.GetCount(temp);
// for smooth tag projection
short aPT = short.MinValue;
double c_aPTW_hPTd = double.NaN;
double c_aPT_hPTd = double.NaN;
double c_hPTd = double.NaN;
double c_aPTW_aPT = double.NaN;
double c_aPT = double.NaN;
if (useSmoothTagProjection)
{
aPT = TagProject(dependency.arg.tag);
short hPT = TagProject(dependency.head.tag);
IntTaggedWord projectedArg = new IntTaggedWord(dependency.arg.word, aPT);
IntTaggedWord projectedAnyHead = new IntTaggedWord(IntTaggedWord.AnyWordInt, hPT);
IntTaggedWord projectedAnyArg = new IntTaggedWord(IntTaggedWord.AnyWordInt, aPT);
temp = new IntDependency(projectedAnyHead, projectedArg, leftHeaded, binDistance);
c_aPTW_hPTd = argCounter.GetCount(temp);
temp = new IntDependency(projectedAnyHead, projectedAnyArg, leftHeaded, binDistance);
c_aPT_hPTd = argCounter.GetCount(temp);
temp = new IntDependency(projectedAnyHead, wildTW, leftHeaded, binDistance);
c_hPTd = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, projectedArg, false, IntDependency.AnyDistanceInt);
c_aPTW_aPT = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, projectedAnyArg, false, IntDependency.AnyDistanceInt);
c_aPT = argCounter.GetCount(temp);
}
// wild head is always directionless and no use distance
temp = new IntDependency(wildTW, dependency.arg, false, IntDependency.AnyDistanceInt);
double c_aTW = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, anyArg, false, IntDependency.AnyDistanceInt);
double c_aT = argCounter.GetCount(temp);
temp = new IntDependency(wildTW, anyTagArg, false, IntDependency.AnyDistanceInt);
double c_aW = argCounter.GetCount(temp);
// do the Bayesian magic
// MLE probs
double p_aTW_hTd;
double p_aT_hTd;
double p_aTW_aT;
double p_aW;
double p_aPTW_aPT;
double p_aPTW_hPTd;
double p_aPT_hPTd;
// backoffs either mle or themselves bayesian smoothed depending on useSmoothTagProjection
if (useSmoothTagProjection)
{
if (useUnigramWordSmoothing)
{
p_aW = c_aW > 0.0 ? (c_aW / numWordTokens) : 1.0;
// NEED this 1.0 for unknown words!!!
p_aPTW_aPT = (c_aPTW_aPT + smooth_aPTW_aPT * p_aW) / (c_aPT + smooth_aPTW_aPT);
}
else
{
p_aPTW_aPT = c_aPTW_aPT > 0.0 ? (c_aPTW_aPT / c_aPT) : 1.0;
}
// NEED this 1.0 for unknown words!!!
p_aTW_aT = (c_aTW + smooth_aTW_aT * p_aPTW_aPT) / (c_aT + smooth_aTW_aT);
p_aPTW_hPTd = c_hPTd > 0.0 ? (c_aPTW_hPTd / c_hPTd) : 0.0;
p_aTW_hTd = (c_aTW_hTd + smooth_aTW_hTd * p_aPTW_hPTd) / (c_hTd + smooth_aTW_hTd);
p_aPT_hPTd = c_hPTd > 0.0 ? (c_aPT_hPTd / c_hPTd) : 0.0;
p_aT_hTd = (c_aT_hTd + smooth_aT_hTd * p_aPT_hPTd) / (c_hTd + smooth_aT_hTd);
}
else
{
// here word generation isn't smoothed - can't get previously unseen word with tag. Ugh.
if (op.testOptions.useLexiconToScoreDependencyPwGt)
{
// We don't know the position. Now -1 means average over 0 and 1.
p_aTW_aT = dependency.leftHeaded ? Math.Exp(lex.Score(dependency.arg, 1, wordIndex.Get(dependency.arg.word), null)) : Math.Exp(lex.Score(dependency.arg, -1, wordIndex.Get(dependency.arg.word), null));
}
else
{
// double oldScore = c_aTW > 0.0 ? (c_aTW / c_aT) : 1.0;
// if (oldScore == 1.0) {
// log.info("#### arg=" + dependency.arg + " score=" + p_aTW_aT +
// " oldScore=" + oldScore + " c_aTW=" + c_aTW + " c_aW=" + c_aW);
// }
p_aTW_aT = c_aTW > 0.0 ? (c_aTW / c_aT) : 1.0;
}
p_aTW_hTd = c_hTd > 0.0 ? (c_aTW_hTd / c_hTd) : 0.0;
p_aT_hTd = c_hTd > 0.0 ? (c_aT_hTd / c_hTd) : 0.0;
}
double pb_aTW_hTWd = (c_aTW_hTWd + smooth_aTW_hTWd * p_aTW_hTd) / (c_hTWd + smooth_aTW_hTWd);
double pb_aT_hTWd = (c_aT_hTWd + smooth_aT_hTWd * p_aT_hTd) / (c_hTWd + smooth_aT_hTWd);
double score = (interp * pb_aTW_hTWd + (1.0 - interp) * p_aTW_aT * pb_aT_hTWd) * pb_go_hTWds;
if (op.testOptions.prunePunc && PruneTW(aTW))
{
return(1.0);
}
if (double.IsNaN(score))
{
score = 0.0;
}
//if (op.testOptions.rightBonus && ! dependency.leftHeaded)
// score -= 0.2;
if (score < MinProbability)
{
score = 0.0;
}
return(score);
}
/// <summary>Score a tag binned dependency.</summary>
public override double ScoreTB(IntDependency dependency)
{
return(op.testOptions.depWeight * Math.Log(ProbTB(dependency)));
}