public virtual Tree GetBestParse <_T0>(IList <_T0> sentence)
where _T0 : IHasWord
{
ScoredObject <Tree> scoredParse = GetBestScoredParse(sentence);
return((scoredParse != null) ? scoredParse.Object() : null);
}
private void ClearFocusedObjectIfOld()
{
if (!_focusedObject.IsRecentlyHit())
{
_focusedObject = ScoredObject.Empty();
}
}
public FocusedObject GetFocusedObject(
IEnumerable <global::Tobii.Framework.GazePoint> lastGazePoints, Camera camera)
{
var gazePoints = new List <global::Tobii.Framework.GazePoint>();
/*Note: Do not use LINQ here - too inefficient to be called every update.*/
foreach (var gazePoint in lastGazePoints)
{
if (gazePoint.IsValid)
{
gazePoints.Add(gazePoint);
}
}
foreach (var gazePoint in gazePoints)
{
var objectsInGaze = FindObjectsInGaze(gazePoint.Screen, camera);
UpdateFocusConfidenceScore(objectsInGaze);
}
var focusChallenger = FindFocusChallenger();
if (focusChallenger.GetScore() >
_focusedObject.GetScore() + Threshold)
{
_focusedObject = focusChallenger;
}
return(FocusedGameObject);
}
public void RemoveObject(GameObject gameObject)
{
_scoredObjects.Remove(gameObject.GetInstanceID());
if (_focusedObject.GameObject.GetInstanceID() ==
gameObject.GetInstanceID())
{
_focusedObject = ScoredObject.Empty();
}
}
protected virtual bool RowCompare(ScoredObject oldObj, ScoredObject newObj)
{
var info = new PredicateInfo();
var factory = new PredicateFactory();
var predicateInfo = BuildPredicateInfo(oldObj.Score);
var predicate = factory.Instantiate(predicateInfo);
return(predicate.Execute(newObj.Score));
}
protected virtual void SetUp()
{
beam = new Beam <ScoredObject <string> >(2, ScoredComparator.AscendingComparator);
object1 = new ScoredObject <string>("1", 1.0);
object2 = new ScoredObject <string>("2", 2.0);
object3 = new ScoredObject <string>("3", 3.0);
object0 = new ScoredObject <string>("0", 0.0);
beam.Add(object1);
beam.Add(object2);
beam.Add(object3);
beam.Add(object0);
}
protected virtual bool RowCompare <T>(ScoredObject oldObj, ScoredObject newObj)
{
var factory = new PredicateFactory();
var predicateArgs = new ReferencePredicateArgs()
{
ColumnType = ColumnType,
ComparerType = GetComparerType(),
Reference = new LiteralScalarResolver <T>(oldObj.Score)
};
var predicate = factory.Instantiate(predicateArgs);
return(predicate.Execute(newObj.Score));
}
/// <summary>Return the k best parses of the sentence most recently parsed.</summary>
/// <remarks>
/// Return the k best parses of the sentence most recently parsed.
/// NB: The dependency parser does not implement a k-best method
/// and the factored parser's method seems to be broken and therefore
/// this method always returns a list of size 1 if either of these
/// two parsers was used.
/// </remarks>
/// <returns>A list of scored trees</returns>
/// <exception cref="Edu.Stanford.Nlp.Parser.Common.NoSuchParseException">
/// If no previously successfully parsed
/// sentence
/// </exception>
public virtual IList <ScoredObject <Tree> > GetKBestParses(int k)
{
if (parseSkipped)
{
return(null);
}
if (bparser != null && parseSucceeded)
{
//The getKGoodParses seems to be broken, so just return the best parse
Tree binaryTree = bparser.GetBestParse();
Tree tree = debinarizer.TransformTree(binaryTree);
if (op.nodePrune)
{
NodePruner np = new NodePruner(pparser, debinarizer);
tree = np.Prune(tree);
}
tree = subcategoryStripper.TransformTree(tree);
RestoreOriginalWords(tree);
double score = dparser.GetBestScore();
ScoredObject <Tree> so = new ScoredObject <Tree>(tree, score);
IList <ScoredObject <Tree> > trees = new List <ScoredObject <Tree> >(1);
trees.Add(so);
return(trees);
}
else
{
if (pparser != null && pparser.HasParse() && fallbackToPCFG)
{
return(this.GetKBestPCFGParses(k));
}
else
{
if (dparser != null && dparser.HasParse())
{
// && fallbackToDG
// The dependency parser doesn't support k-best parse extraction, so just
// return the best parse
Tree tree = this.GetBestDependencyParse(true);
double score = dparser.GetBestScore();
ScoredObject <Tree> so = new ScoredObject <Tree>(tree, score);
IList <ScoredObject <Tree> > trees = new List <ScoredObject <Tree> >(1);
trees.Add(so);
return(trees);
}
else
{
throw new NoSuchParseException();
}
}
}
}
protected virtual bool RowCompare(ScoredObject oldObj, ScoredObject newObj)
{
switch (ColumnType)
{
case ColumnType.Text: return(RowCompare <string>(oldObj, newObj));
case ColumnType.Numeric: return(RowCompare <decimal>(oldObj, newObj));
case ColumnType.DateTime: return(RowCompare <DateTime>(oldObj, newObj));
case ColumnType.Boolean: return(RowCompare <bool>(oldObj, newObj));
default: throw new ArgumentOutOfRangeException();
}
}
public FocusedObject GetFocusedObject(
IEnumerable <global::Tobii.Framework.GazePoint> lastGazePoints, Camera camera)
{
var objectsInGaze = FindObjectsInGaze(lastGazePoints, camera);
UpdateFocusConfidenceScore(objectsInGaze);
var focusChallenger = FindFocusChallenger();
if (focusChallenger.GetScore()
> _focusedObject.GetScore() + Threshold)
{
_focusedObject = focusChallenger;
}
return(FocusedGameObject);
}
protected override void InsertRow(ScoredObject newObj, ref IList <ScoredObject> list)
{
var i = 0;
var isObjAdded = false;
while (!isObjAdded)
{
if (list.Count == i || RowCompare(list[i], newObj))
{
list.Insert(i, newObj);
isObjAdded = true;
}
i++;
}
if (list.Count > TableLength)
{
list.RemoveAt(TableLength);
}
}
private ScoredObject FindFocusChallenger()
{
var topFocusChallenger = ScoredObject.Empty();
var topScore = 0.0f;
foreach (var key in _scoredObjects.Keys)
{
var scoredObject = _scoredObjects[key];
var score = scoredObject.GetScore(
Time.unscaledTime - LoseGazeDwellTime,
Time.unscaledTime - GainGazeDwellTime);
if (score > topScore)
{
topScore = score;
topFocusChallenger = scoredObject;
}
}
return(topFocusChallenger);
}
/// <summary>Parse a Sentence.</summary>
/// <remarks>
/// Parse a Sentence. It is assumed that when this is called, the pparser
/// has already been called to parse the sentence.
/// </remarks>
/// <param name="words">The list of words to parse.</param>
/// <returns>true iff it could be parsed</returns>
public virtual bool Parse <_T0>(IList <_T0> words)
where _T0 : IHasWord
{
nGoodTrees.Clear();
int numParsesToConsider = numToFind * op.testOptions.fastFactoredCandidateMultiplier + op.testOptions.fastFactoredCandidateAddend;
if (pparser.HasParse())
{
IList <ScoredObject <Tree> > pcfgBest = pparser.GetKBestParses(numParsesToConsider);
Beam <ScoredObject <Tree> > goodParses = new Beam <ScoredObject <Tree> >(numToFind);
foreach (ScoredObject <Tree> candidate in pcfgBest)
{
if (Thread.Interrupted())
{
throw new RuntimeInterruptedException();
}
double depScore = DepScoreTree(candidate.Object());
ScoredObject <Tree> x = new ScoredObject <Tree>(candidate.Object(), candidate.Score() + depScore);
goodParses.Add(x);
}
nGoodTrees = goodParses.AsSortedList();
}
return(!nGoodTrees.IsEmpty());
}
public void Reset()
{
_scoredObjects.Clear();
_focusedObject = ScoredObject.Empty();
}
/// <summary>TODO: return more if this used a beam</summary>
public virtual IList <ScoredObject <Tree> > GetKBestPCFGParses(int kbestPCFG)
{
ScoredObject <Tree> parse = new ScoredObject <Tree>(debinarized, finalState.score);
return(Java.Util.Collections.SingletonList(parse));
}
private IList <ScoredObject <Tree> > GetNext()
{
try
{
string line;
int parsesExpected = 0;
int sentenceId = lastSentenceId;
ScoredObject <Tree> curParse = null;
double score = null;
IList <ScoredObject <Tree> > curParses = null;
while ((line = br.ReadLine()) != null)
{
line = line.Trim();
if (!line.IsEmpty())
{
if (parsesExpected == 0)
{
// Finished processing parses
string[] fields = wsDelimiter.Split(line, 2);
parsesExpected = System.Convert.ToInt32(fields[0]);
sentenceId = System.Convert.ToInt32(fields[1]);
if (expectConsecutiveSentenceIds)
{
if (sentenceId != lastSentenceId + 1)
{
if (lastSentenceId < sentenceId)
{
StringBuilder sb = new StringBuilder("Missing sentences");
for (int i = lastSentenceId + 1; i < sentenceId; i++)
{
sb.Append(' ').Append(i);
}
logger.Warning(sb.ToString());
}
else
{
logger.Warning("sentenceIds are not increasing (last=" + lastSentenceId + ", curr=" + sentenceId + ")");
}
}
}
lastSentenceId = sentenceId;
curParses = new List <ScoredObject <Tree> >(parsesExpected);
}
else
{
if (score == null)
{
// read score
score = double.ParseDouble(wsDelimiter.Split(line, 2)[0]);
}
else
{
// Reading a parse
curParse = new ScoredObject <Tree>(Edu.Stanford.Nlp.Trees.Trees.ReadTree(line), score);
curParses.Add(curParse);
curParse = null;
score = null;
parsesExpected--;
if (parsesExpected == 0)
{
return(curParses);
}
}
}
}
}
}
catch (IOException ex)
{
throw new Exception(ex);
}
return(null);
}
protected abstract void InsertRow(ScoredObject score, ref IList <ScoredObject> subset);
private Pair <int, int> TrainTree(int index, IList <Tree> binarizedTrees, IList <IList <ITransition> > transitionLists, IList <PerceptronModel.Update> updates, Oracle oracle)
{
int numCorrect = 0;
int numWrong = 0;
Tree tree = binarizedTrees[index];
ReorderingOracle reorderer = null;
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderOracle || op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderBeam)
{
reorderer = new ReorderingOracle(op);
}
// TODO. This training method seems to be working in that it
// trains models just like the gold and early termination methods do.
// However, it causes the feature space to go crazy. Presumably
// leaving out features with low weights or low frequencies would
// significantly help with that. Otherwise, not sure how to keep
// it under control.
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.Oracle)
{
State state = ShiftReduceParser.InitialStateFromGoldTagTree(tree);
while (!state.IsFinished())
{
IList <string> features = featureFactory.Featurize(state);
ScoredObject <int> prediction = FindHighestScoringTransition(state, features, true);
if (prediction == null)
{
throw new AssertionError("Did not find a legal transition");
}
int predictedNum = prediction.Object();
ITransition predicted = transitionIndex.Get(predictedNum);
OracleTransition gold = oracle.GoldTransition(index, state);
if (gold.IsCorrect(predicted))
{
numCorrect++;
if (gold.transition != null && !gold.transition.Equals(predicted))
{
int transitionNum = transitionIndex.IndexOf(gold.transition);
if (transitionNum < 0)
{
// TODO: do we want to add unary transitions which are
// only possible when the parser has gone off the rails?
continue;
}
updates.Add(new PerceptronModel.Update(features, transitionNum, -1, learningRate));
}
}
else
{
numWrong++;
int transitionNum = -1;
if (gold.transition != null)
{
transitionNum = transitionIndex.IndexOf(gold.transition);
}
// TODO: this can theoretically result in a -1 gold
// transition if the transition exists, but is a
// CompoundUnaryTransition which only exists because the
// parser is wrong. Do we want to add those transitions?
updates.Add(new PerceptronModel.Update(features, transitionNum, predictedNum, learningRate));
}
state = predicted.Apply(state);
}
}
else
{
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.Beam || op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderBeam)
{
if (op.TrainOptions().beamSize <= 0)
{
throw new ArgumentException("Illegal beam size " + op.TrainOptions().beamSize);
}
IList <ITransition> transitions = Generics.NewLinkedList(transitionLists[index]);
PriorityQueue <State> agenda = new PriorityQueue <State>(op.TrainOptions().beamSize + 1, ScoredComparator.AscendingComparator);
State goldState = ShiftReduceParser.InitialStateFromGoldTagTree(tree);
agenda.Add(goldState);
// int transitionCount = 0;
while (transitions.Count > 0)
{
ITransition goldTransition = transitions[0];
ITransition highestScoringTransitionFromGoldState = null;
double highestScoreFromGoldState = 0.0;
PriorityQueue <State> newAgenda = new PriorityQueue <State>(op.TrainOptions().beamSize + 1, ScoredComparator.AscendingComparator);
State highestScoringState = null;
State highestCurrentState = null;
foreach (State currentState in agenda)
{
bool isGoldState = (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderBeam && goldState.AreTransitionsEqual(currentState));
IList <string> features = featureFactory.Featurize(currentState);
ICollection <ScoredObject <int> > stateTransitions = FindHighestScoringTransitions(currentState, features, true, op.TrainOptions().beamSize, null);
foreach (ScoredObject <int> transition in stateTransitions)
{
State newState = transitionIndex.Get(transition.Object()).Apply(currentState, transition.Score());
newAgenda.Add(newState);
if (newAgenda.Count > op.TrainOptions().beamSize)
{
newAgenda.Poll();
}
if (highestScoringState == null || highestScoringState.Score() < newState.Score())
{
highestScoringState = newState;
highestCurrentState = currentState;
}
if (isGoldState && (highestScoringTransitionFromGoldState == null || transition.Score() > highestScoreFromGoldState))
{
highestScoringTransitionFromGoldState = transitionIndex.Get(transition.Object());
highestScoreFromGoldState = transition.Score();
}
}
}
// This can happen if the REORDER_BEAM method backs itself
// into a corner, such as transitioning to something that
// can't have a FinalizeTransition applied. This doesn't
// happen for the BEAM method because in that case the correct
// state (eg one with ROOT) isn't on the agenda so it stops.
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderBeam && highestScoringTransitionFromGoldState == null)
{
break;
}
State newGoldState = goldTransition.Apply(goldState, 0.0);
// if highest scoring state used the correct transition, no training
// otherwise, down the last transition, up the correct
if (!newGoldState.AreTransitionsEqual(highestScoringState))
{
++numWrong;
IList <string> goldFeatures = featureFactory.Featurize(goldState);
int lastTransition = transitionIndex.IndexOf(highestScoringState.transitions.Peek());
updates.Add(new PerceptronModel.Update(featureFactory.Featurize(highestCurrentState), -1, lastTransition, learningRate));
updates.Add(new PerceptronModel.Update(goldFeatures, transitionIndex.IndexOf(goldTransition), -1, learningRate));
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.Beam)
{
// If the correct state has fallen off the agenda, break
if (!ShiftReduceUtils.FindStateOnAgenda(newAgenda, newGoldState))
{
break;
}
else
{
transitions.Remove(0);
}
}
else
{
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderBeam)
{
if (!ShiftReduceUtils.FindStateOnAgenda(newAgenda, newGoldState))
{
if (!reorderer.Reorder(goldState, highestScoringTransitionFromGoldState, transitions))
{
break;
}
newGoldState = highestScoringTransitionFromGoldState.Apply(goldState);
if (!ShiftReduceUtils.FindStateOnAgenda(newAgenda, newGoldState))
{
break;
}
}
else
{
transitions.Remove(0);
}
}
}
}
else
{
++numCorrect;
transitions.Remove(0);
}
goldState = newGoldState;
agenda = newAgenda;
}
}
else
{
if (op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.ReorderOracle || op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod.EarlyTermination || op.TrainOptions().trainingMethod == ShiftReduceTrainOptions.TrainingMethod
.Gold)
{
State state = ShiftReduceParser.InitialStateFromGoldTagTree(tree);
IList <ITransition> transitions = transitionLists[index];
transitions = Generics.NewLinkedList(transitions);
bool keepGoing = true;
while (transitions.Count > 0 && keepGoing)
{
ITransition transition = transitions[0];
int transitionNum = transitionIndex.IndexOf(transition);
IList <string> features = featureFactory.Featurize(state);
int predictedNum = FindHighestScoringTransition(state, features, false).Object();
ITransition predicted = transitionIndex.Get(predictedNum);
if (transitionNum == predictedNum)
{
transitions.Remove(0);
state = transition.Apply(state);
numCorrect++;
}
else
{
numWrong++;
// TODO: allow weighted features, weighted training, etc
updates.Add(new PerceptronModel.Update(features, transitionNum, predictedNum, learningRate));
switch (op.TrainOptions().trainingMethod)
{
case ShiftReduceTrainOptions.TrainingMethod.EarlyTermination:
{
keepGoing = false;
break;
}
case ShiftReduceTrainOptions.TrainingMethod.Gold:
{
transitions.Remove(0);
state = transition.Apply(state);
break;
}
case ShiftReduceTrainOptions.TrainingMethod.ReorderOracle:
{
keepGoing = reorderer.Reorder(state, predicted, transitions);
if (keepGoing)
{
state = predicted.Apply(state);
}
break;
}
default:
{
throw new ArgumentException("Unexpected method " + op.TrainOptions().trainingMethod);
}
}
}
}
}
}
}
return(Pair.MakePair(numCorrect, numWrong));
}
