// Records the number of times word/tag pair was seen in training data.
// Counts of each tag (stored as a Label) on unknown words.
// tag (Label) --> signature --> count
public override void InitializeTraining(Options op, ILexicon lex, IIndex <string> wordIndex, IIndex <string> tagIndex, double totalTrees)
{
base.InitializeTraining(op, lex, wordIndex, tagIndex, totalTrees);
seenCounter = new ClassicCounter <IntTaggedWord>();
unSeenCounter = new ClassicCounter <IntTaggedWord>();
tagHash = Generics.NewHashMap();
tc = new ClassicCounter <ILabel>();
c = Generics.NewHashMap();
seenEnd = Generics.NewHashSet();
useEnd = (op.lexOptions.unknownSuffixSize > 0 && op.lexOptions.useUnknownWordSignatures > 0);
useFirstCap = op.lexOptions.useUnknownWordSignatures > 0;
useGT = (op.lexOptions.useUnknownWordSignatures == 0);
useFirst = false;
if (useFirst)
{
log.Info("Including first letter for unknown words.");
}
if (useFirstCap)
{
log.Info("Including whether first letter is capitalized for unknown words");
}
if (useEnd)
{
log.Info("Classing unknown word as the average of their equivalents by identity of last " + op.lexOptions.unknownSuffixSize + " letters.");
}
if (useGT)
{
log.Info("Using Good-Turing smoothing for unknown words.");
}
this.indexToStartUnkCounting = (totalTrees * op.trainOptions.fractionBeforeUnseenCounting);
this.unknownGTTrainer = (useGT) ? new UnknownGTTrainer() : null;
this.model = BuildUWM();
}
// Records the number of times word/tag pair was seen in training data.
// c has a map from tags as Label to a Counter from word
// signatures to Strings; it is used to collect counts that will
// initialize the probabilities in tagHash
// tc record the marginal counts for each tag as an unknown. It
// should be the same as c's totalCount ??
public override void InitializeTraining(Options op, ILexicon lex, IIndex <string> wordIndex, IIndex <string> tagIndex, double totalTrees)
{
base.InitializeTraining(op, lex, wordIndex, tagIndex, totalTrees);
bool useGoodTuringUnknownWordModel = ChineseTreebankParserParams.DefaultUseGoodTurningUnknownWordModel;
useFirst = true;
useGT = (op.lexOptions.useUnknownWordSignatures == 0);
if (lex is ChineseLexicon)
{
useGoodTuringUnknownWordModel = ((ChineseLexicon)lex).useGoodTuringUnknownWordModel;
}
else
{
if (op.tlpParams is ChineseTreebankParserParams)
{
useGoodTuringUnknownWordModel = ((ChineseTreebankParserParams)op.tlpParams).useGoodTuringUnknownWordModel;
}
}
if (useGoodTuringUnknownWordModel)
{
this.useGT = true;
this.useFirst = false;
}
this.useUnicodeType = op.lexOptions.useUnicodeType;
if (useFirst)
{
log.Info("ChineseUWM: treating unknown word as the average of their equivalents by first-character identity. useUnicodeType: " + useUnicodeType);
}
if (useGT)
{
log.Info("ChineseUWM: using Good-Turing smoothing for unknown words.");
}
this.c = Generics.NewHashMap();
this.tc = new ClassicCounter <ILabel>();
this.unSeenCounter = new ClassicCounter <IntTaggedWord>();
this.seenCounter = new ClassicCounter <IntTaggedWord>();
this.seenFirst = Generics.NewHashSet();
this.tagHash = Generics.NewHashMap();
this.indexToStartUnkCounting = (totalTrees * op.trainOptions.fractionBeforeUnseenCounting);
this.unknownGTTrainer = (useGT) ? new UnknownGTTrainer() : null;
IDictionary <string, float> unknownGT = null;
if (useGT)
{
unknownGT = unknownGTTrainer.unknownGT;
}
this.model = new ChineseUnknownWordModel(op, lex, wordIndex, tagIndex, unSeenCounter, tagHash, unknownGT, useGT, seenFirst);
}