private void HashHistories()
{
int fAll = maxentTagger.extractors.Size() + maxentTagger.extractorsRare.Size();
int fGeneral = maxentTagger.extractors.Size();
log.Info("Hashing histories ...");
for (int x = 0; x < xSize; x++)
{
History h = tHistories.GetHistory(x);
// It's the 2010s now and it doesn't take so long to featurize....
// if (x > 0 && x % 10000 == 0) {
// System.err.printf("%d ",x);
// if (x % 100000 == 0) { log.info(); }
// }
int fSize = (maxentTagger.IsRare(ExtractorFrames.cWord.Extract(h)) ? fAll : fGeneral);
for (int i = 0; i < fSize; i++)
{
tFeature.AddPrev(i, h);
}
}
// for x
// now for the populated ones
// log.info();
log.Info("Hashed " + xSize + " histories.");
log.Info("Hashing populated histories ...");
for (int x_1 = 0; x_1 < xSize; x_1++)
{
History h = tHistories.GetHistory(x_1);
// It's the 2010s now and it doesn't take so long to featurize....
// if (x > 0 && x % 10000 == 0) {
// log.info(x + " ");
// if (x % 100000 == 0) { log.info(); }
// }
int fSize = (maxentTagger.IsRare(ExtractorFrames.cWord.Extract(h)) ? fAll : fGeneral);
for (int i = 0; i < fSize; i++)
{
tFeature.Add(i, h, x_1);
}
}
// write this to check whether to add
// for x
// log.info();
log.Info("Hashed populated histories.");
}
// This precomputes scores of local features (localScores).
protected internal virtual double[] GetHistories(string[] tags, History h)
{
bool rare = maxentTagger.IsRare(ExtractorFrames.cWord.Extract(h));
Extractors ex = maxentTagger.extractors;
Extractors exR = maxentTagger.extractorsRare;
string w = pairs.GetWord(h.current);
double[] lS;
double[] lcS;
lS = localScores[w];
if (lS == null)
{
lS = GetHistories(tags, h, ex.local, rare ? exR.local : null);
localScores[w] = lS;
}
else
{
if (lS.Length != tags.Length)
{
// This case can occur when a word was given a specific forced
// tag, and then later it shows up without the forced tag.
// TODO: if a word is given a forced tag, we should always get
// its features rather than use the cache, just in case the tag
// given is not the same tag as before
lS = GetHistories(tags, h, ex.local, rare ? exR.local : null);
if (tags.Length > 1)
{
localScores[w] = lS;
}
}
}
if ((lcS = localContextScores[h.current]) == null)
{
lcS = GetHistories(tags, h, ex.localContext, rare ? exR.localContext : null);
localContextScores[h.current] = lcS;
ArrayMath.PairwiseAddInPlace(lcS, lS);
}
double[] totalS = GetHistories(tags, h, ex.dynamic, rare ? exR.dynamic : null);
ArrayMath.PairwiseAddInPlace(totalS, lcS);
return(totalS);
}
