private Parser(IMaxentModel buildModel, IMaxentModel checkModel, IPOSTagger tagger, IChunker chunker,
AbstractHeadRules headRules, int beamSize, double advancePercentage) :
base(tagger, chunker, headRules, beamSize, advancePercentage)
{
this.buildModel = buildModel;
this.checkModel = checkModel;
bProbs = new double[buildModel.GetNumOutcomes()];
cProbs = new double[checkModel.GetNumOutcomes()];
buildContextGenerator = new BuildContextGenerator();
checkContextGenerator = new CheckContextGenerator();
startTypeMap = new Dictionary <string, string>();
contTypeMap = new Dictionary <string, string>();
for (int boi = 0, bon = buildModel.GetNumOutcomes(); boi < bon; boi++)
{
var outcome = buildModel.GetOutcome(boi);
if (outcome.StartsWith(START))
{
startTypeMap[outcome] = outcome.Substring(START.Length);
}
else if (outcome.StartsWith(CONT))
{
contTypeMap[outcome] = outcome.Substring(CONT.Length);
}
}
topStartIndex = buildModel.GetIndex(TOP_START);
completeIndex = checkModel.GetIndex(COMPLETE);
incompleteIndex = checkModel.GetIndex(INCOMPLETE);
}
private bool IsModelValid(IMaxentModel model)
{
var outcomes = new string[model.GetNumOutcomes()];
for (var i = 0; i < model.GetNumOutcomes(); i++)
{
outcomes[i] = model.GetOutcome(i);
}
return(Factory.CreateSequenceCodec().AreOutcomesCompatible(outcomes));
}
/// <summary>
/// Checks if the expected outcomes are all contained as outcomes in the given model.
/// </summary>
/// <param name="model">The model.</param>
/// <param name="expectedOutcomes">The expected outcomes.</param>
/// <returns><c>true</c> if all expected outcomes are the only outcomes of the model;<c>false</c> otherwise.</returns>
public static bool ValidateOutcomes(IMaxentModel model, params string[] expectedOutcomes) {
if (model.GetNumOutcomes() == expectedOutcomes.Length) {
var count = model.GetNumOutcomes();
for (int i = 0; i < count; i++) {
if (!expectedOutcomes.Contains(model.GetOutcome(i))) {
return false;
}
}
} else {
return false;
}
return true;
}
private Parser(
IMaxentModel buildModel,
IMaxentModel attachModel,
IMaxentModel checkModel,
IPOSTagger tagger,
IChunker chunker,
AbstractHeadRules headRules,
int beamSize,
double advancePercentage) : base(tagger, chunker, headRules, beamSize, advancePercentage)
{
this.buildModel = buildModel;
this.attachModel = attachModel;
this.checkModel = checkModel;
buildContextGenerator = new BuildContextGenerator();
attachContextGenerator = new AttachContextGenerator(punctSet);
checkContextGenerator = new CheckContextGenerator(punctSet);
bProbs = new double[buildModel.GetNumOutcomes()];
aProbs = new double[attachModel.GetNumOutcomes()];
cProbs = new double[checkModel.GetNumOutcomes()];
doneIndex = buildModel.GetIndex(DONE);
sisterAttachIndex = attachModel.GetIndex(ATTACH_SISTER);
daughterAttachIndex = attachModel.GetIndex(ATTACH_DAUGHTER);
// nonAttachIndex = attachModel.GetIndex(NON_ATTACH);
attachments = new[] { daughterAttachIndex, sisterAttachIndex };
completeIndex = checkModel.GetIndex(COMPLETE);
}
/// <summary>
/// Gets all possible outcomes.
/// </summary>
/// <returns>all possible outcomes.</returns>
public string[] GetOutcomes()
{
var outcomes = new string[model.GetNumOutcomes()];
for (var i = 0; i < outcomes.Length; i++)
{
outcomes[i] = model.GetOutcome(i);
}
return(outcomes);
}
/// <summary>
/// Checks if the expected outcomes are all contained as outcomes in the given model.
/// </summary>
/// <param name="model">The model.</param>
/// <param name="expectedOutcomes">The expected outcomes.</param>
/// <returns><c>true</c> if all expected outcomes are the only outcomes of the model;<c>false</c> otherwise.</returns>
public static bool ValidateOutcomes(IMaxentModel model, params string[] expectedOutcomes)
{
if (model.GetNumOutcomes() == expectedOutcomes.Length)
{
var count = model.GetNumOutcomes();
for (int i = 0; i < count; i++)
{
if (!expectedOutcomes.Contains(model.GetOutcome(i)))
{
return(false);
}
}
}
else
{
return(false);
}
return(true);
}
/// <summary>
/// Creates new search object with the specified cache size.
/// </summary>
/// <param name="size">The size of the beam (k).</param>
/// <param name="model">The model for assigning probabilities to the sequence outcomes.</param>
/// <param name="cacheSize">Size of the cache.</param>
public BeamSearch(int size, IMaxentModel model, int cacheSize)
{
this.size = size;
this.model = model;
if (cacheSize > 0)
{
contextsCache = new Cache(cacheSize);
}
probs = new double[model.GetNumOutcomes()];
}
/// <summary>
/// Creates new search object.
/// </summary>
/// <param name="size">The size of the beam (k).</param>
/// <param name="cg">The context generator for the model.</param>
/// <param name="model">The model for assigning probabilities to the sequence outcomes.</param>
/// <param name="validator">The sequence validator.</param>
/// <param name="cacheSize">Size of the cache.</param>
public BeamSearch(int size, IBeamSearchContextGenerator <T> cg, IMaxentModel model, ISequenceValidator <T> validator, int cacheSize)
{
this.cg = cg;
this.size = size;
this.model = model;
this.validator = validator;
if (cacheSize > 0)
{
contextsCache = new Cache(cacheSize);
}
probs = new double[model.GetNumOutcomes()];
}
private Parser(IMaxentModel buildModel, IMaxentModel checkModel, IPOSTagger tagger, IChunker chunker,
AbstractHeadRules headRules, int beamSize, double advancePercentage) :
base(tagger, chunker, headRules, beamSize, advancePercentage) {
this.buildModel = buildModel;
this.checkModel = checkModel;
bProbs = new double[buildModel.GetNumOutcomes()];
cProbs = new double[checkModel.GetNumOutcomes()];
buildContextGenerator = new BuildContextGenerator();
checkContextGenerator = new CheckContextGenerator();
startTypeMap = new Dictionary<string, string>();
contTypeMap = new Dictionary<string, string>();
for (int boi = 0, bon = buildModel.GetNumOutcomes(); boi < bon; boi++) {
var outcome = buildModel.GetOutcome(boi);
if (outcome.StartsWith(START)) {
startTypeMap[outcome] = outcome.Substring(START.Length);
} else if (outcome.StartsWith(CONT)) {
contTypeMap[outcome] = outcome.Substring(CONT.Length);
}
}
topStartIndex = buildModel.GetIndex(TOP_START);
completeIndex = checkModel.GetIndex(COMPLETE);
incompleteIndex = checkModel.GetIndex(INCOMPLETE);
}
private Parser(
IMaxentModel buildModel,
IMaxentModel attachModel,
IMaxentModel checkModel,
IPOSTagger tagger,
IChunker chunker,
AbstractHeadRules headRules,
int beamSize,
double advancePercentage) : base(tagger, chunker, headRules, beamSize, advancePercentage) {
this.buildModel = buildModel;
this.attachModel = attachModel;
this.checkModel = checkModel;
buildContextGenerator = new BuildContextGenerator();
attachContextGenerator = new AttachContextGenerator(punctSet);
checkContextGenerator = new CheckContextGenerator(punctSet);
bProbs = new double[buildModel.GetNumOutcomes()];
aProbs = new double[attachModel.GetNumOutcomes()];
cProbs = new double[checkModel.GetNumOutcomes()];
doneIndex = buildModel.GetIndex(DONE);
sisterAttachIndex = attachModel.GetIndex(ATTACH_SISTER);
daughterAttachIndex = attachModel.GetIndex(ATTACH_DAUGHTER);
// nonAttachIndex = attachModel.GetIndex(NON_ATTACH);
attachments = new[] {daughterAttachIndex, sisterAttachIndex};
completeIndex = checkModel.GetIndex(COMPLETE);
}
/// <summary>
/// Advances the specified parse and returns the an array advanced parses whose probability accounts for
/// more than the specified amount of probability mass.
/// </summary>
/// <param name="p">The parse to advance.</param>
/// <param name="probMass">The amount of probability mass that should be accounted for by the advanced parses.</param>
protected override Parse[] AdvanceParses(Parse p, double probMass)
{
var q = 1 - probMass;
/** The closest previous node which has been labeled as a start node. */
Parse lastStartNode = null;
/** The index of the closest previous node which has been labeled as a start node. */
var lastStartIndex = -1;
/** The type of the closest previous node which has been labeled as a start node. */
string lastStartType = null;
/** The index of the node which will be labeled in this iteration of advancing the parse. */
int advanceNodeIndex;
/** The node which will be labeled in this iteration of advancing the parse. */
Parse advanceNode = null;
var originalChildren = p.Children;
var children = CollapsePunctuation(originalChildren, punctSet);
var numNodes = children.Length;
if (numNodes == 0)
{
return(null);
}
//determines which node needs to be labeled and prior labels.
for (advanceNodeIndex = 0; advanceNodeIndex < numNodes; advanceNodeIndex++)
{
advanceNode = children[advanceNodeIndex];
if (advanceNode.Label == null)
{
break;
}
if (startTypeMap.ContainsKey(advanceNode.Label))
{
lastStartType = startTypeMap[advanceNode.Label];
lastStartNode = advanceNode;
lastStartIndex = advanceNodeIndex;
//System.err.println("lastStart "+i+" "+lastStart.label+" "+lastStart.prob);
}
}
var originalAdvanceIndex = MapParseIndex(advanceNodeIndex, children, originalChildren);
var newParsesList = new List <Parse>(buildModel.GetNumOutcomes());
//call build
buildModel.Eval(buildContextGenerator.GetContext(children, advanceNodeIndex), bProbs);
var bProbSum = 0d;
while (bProbSum < probMass)
{
// The largest un-advanced labeling.
var max = 0;
for (var pi = 1; pi < bProbs.Length; pi++)
{
//for each build outcome
if (bProbs[pi] > bProbs[max])
{
max = pi;
}
}
if (bProbs[max].Equals(0d))
{
break;
}
var bProb = bProbs[max];
bProbs[max] = 0; //zero out so new max can be found
bProbSum += bProb;
var tag = buildModel.GetOutcome(max);
//System.out.println("trying "+tag+" "+bprobSum+" lst="+lst);
if (max == topStartIndex)
{
// can't have top until complete
continue;
}
//System.err.println(i+" "+tag+" "+bprob);
if (startTypeMap.ContainsKey(tag))
{
//update last start
lastStartIndex = advanceNodeIndex;
lastStartNode = advanceNode;
lastStartType = startTypeMap[tag];
}
else if (contTypeMap.ContainsKey(tag))
{
if (lastStartNode == null || !lastStartType.Equals(contTypeMap[tag]))
{
continue; //Cont must match previous start or continue
}
}
var newParse1 = (Parse)p.Clone(); //clone parse
if (createDerivationString)
{
newParse1.Derivation.Append(max).Append("-");
}
newParse1.SetChild(originalAdvanceIndex, tag); //replace constituent being labeled to create new derivation
newParse1.AddProbability(Math.Log(bProb));
//check
//String[] context = checkContextGenerator.getContext(newParse1.getChildren(), lastStartType, lastStartIndex, advanceNodeIndex);
checkModel.Eval(
checkContextGenerator.GetContext(
CollapsePunctuation(newParse1.Children, punctSet),
lastStartType,
lastStartIndex,
advanceNodeIndex),
cProbs);
//System.out.println("check "+lastStartType+" "+cprobs[completeIndex]+" "+cprobs[incompleteIndex]+" "+tag+" "+java.util.Arrays.asList(context));
if (cProbs[completeIndex] > q)
{
//make sure a reduce is likely
var newParse2 = (Parse)newParse1.Clone();
if (createDerivationString)
{
newParse2.Derivation.Append(1).Append(".");
}
newParse2.AddProbability(Math.Log(cProbs[completeIndex]));
var cons = new Parse[advanceNodeIndex - lastStartIndex + 1];
var flat = true;
if (lastStartNode == null)
{
throw new InvalidOperationException("lastStartNode is null.");
}
//first
cons[0] = lastStartNode;
flat &= cons[0].IsPosTag;
//last
cons[advanceNodeIndex - lastStartIndex] = advanceNode;
flat &= cons[advanceNodeIndex - lastStartIndex].IsPosTag;
//middle
for (var ci = 1; ci < advanceNodeIndex - lastStartIndex; ci++)
{
cons[ci] = children[ci + lastStartIndex];
flat &= cons[ci].IsPosTag;
}
if (!flat)
{
//flat chunks are done by chunker
if (lastStartIndex == 0 && advanceNodeIndex == numNodes - 1)
{
//check for top node to include end and beginning punctuation
//System.err.println("ParserME.advanceParses: reducing entire span: "+new Span(lastStartNode.getSpan().getStart(), advanceNode.getSpan().getEnd())+" "+lastStartType+" "+java.util.Arrays.asList(children));
newParse2.Insert(new Parse(p.Text, p.Span, lastStartType, cProbs[1],
headRules.GetHead(cons, lastStartType)));
}
else
{
newParse2.Insert(new Parse(p.Text, new Span(lastStartNode.Span.Start, advanceNode.Span.End),
lastStartType, cProbs[1], headRules.GetHead(cons, lastStartType)));
}
newParsesList.Add(newParse2);
}
}
if (cProbs[incompleteIndex] > q)
{
//make sure a shift is likely
if (createDerivationString)
{
newParse1.Derivation.Append(0).Append(".");
}
if (advanceNodeIndex != numNodes - 1)
{
//can't shift last element
newParse1.AddProbability(Math.Log(cProbs[incompleteIndex]));
newParsesList.Add(newParse1);
}
}
}
return(newParsesList.ToArray());
}
private bool IsModelValid(IMaxentModel model) {
var outcomes = new string[model.GetNumOutcomes()];
for (var i = 0; i < model.GetNumOutcomes(); i++) {
outcomes[i] = model.GetOutcome(i);
}
return Factory.CreateSequenceCodec().AreOutcomesCompatible(outcomes);
}