/*
* /// (non-Javadoc)
*
* /// @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
*/
public override void NewProperties(PropertySheet ps)
{
_logMath = LogMath.GetLogMath();
_acousticModel = (AcousticModel)ps.GetComponent(PropAcousticModel);
_unitManager = (UnitManager)ps.GetComponent(PropUnitManager);
LanguageModel = (LanguageModel)ps.GetComponent(PropLanguageModel);
Dictionary = (IDictionary)ps.GetComponent(PropDictionary);
//fullWordHistories = ps.getBoolean(PROP_FULL_WORD_HISTORIES);
_wantUnigramSmear = ps.GetBoolean(PropWantUnigramSmear);
_logWordInsertionProbability = _logMath.LinearToLog(ps.GetDouble(PropWordInsertionProbability));
_logSilenceInsertionProbability = _logMath.LinearToLog(ps.GetDouble(PropSilenceInsertionProbability));
_logFillerInsertionProbability = _logMath.LinearToLog(ps.GetDouble(PropFillerInsertionProbability));
_logUnitInsertionProbability = _logMath.LinearToLog(ps.GetDouble(PropUnitInsertionProbability));
LanguageWeight = ps.GetFloat(PropLanguageWeight);
AddFillerWords = (ps.GetBoolean(PropAddFillerWords));
GenerateUnitStates = (ps.GetBoolean(PropGenerateUnitStates));
_unigramSmearWeight = ps.GetFloat(PropUnigramSmearWeight);
_maxArcCacheSize = ps.GetInt(PropCacheSize);
CacheEnabled = _maxArcCacheSize > 0;
if (CacheEnabled)
{
ArcCache = new LRUCache <LexTreeState, ISearchStateArc[]>(_maxArcCacheSize);
}
}
public LexTreeLinguist(AcousticModel acousticModel, UnitManager unitManager,
LanguageModel languageModel, IDictionary dictionary, Boolean fullWordHistories, Boolean wantUnigramSmear,
double wordInsertionProbability, double silenceInsertionProbability, double fillerInsertionProbability,
double unitInsertionProbability, float languageWeight, Boolean addFillerWords, Boolean generateUnitStates,
float unigramSmearWeight, int maxArcCacheSize)
{
_acousticModel = acousticModel;
_logMath = LogMath.GetLogMath();
_unitManager = unitManager;
LanguageModel = languageModel;
Dictionary = dictionary;
//this.fullWordHistories = fullWordHistories;
_wantUnigramSmear = wantUnigramSmear;
_logWordInsertionProbability = _logMath.LinearToLog(wordInsertionProbability);
_logSilenceInsertionProbability = _logMath.LinearToLog(silenceInsertionProbability);
_logFillerInsertionProbability = _logMath.LinearToLog(fillerInsertionProbability);
_logUnitInsertionProbability = _logMath.LinearToLog(unitInsertionProbability);
LanguageWeight = languageWeight;
AddFillerWords = addFillerWords;
GenerateUnitStates = generateUnitStates;
_unigramSmearWeight = unigramSmearWeight;
_maxArcCacheSize = maxArcCacheSize;
CacheEnabled = maxArcCacheSize > 0;
if (CacheEnabled)
{
ArcCache = new LRUCache <LexTreeState, ISearchStateArc[]>(maxArcCacheSize);
}
}
public virtual void NewProperties(PropertySheet ps)
{
LogMath = LogMath.GetLogMath();
AbsoluteBeamWidth = ps.GetInt(PropAbsoluteBeamWidth);
double relativeBeamWidth = ps.GetDouble(PropRelativeBeamWidth);
LogRelativeBeamWidth = LogMath.LinearToLog(relativeBeamWidth);
}
public override void NewProperties(PropertySheet ps)
{
logMath = LogMath.GetLogMath();
location = ConfigurationManagerUtils.GetResource(PropLocation, ps);
ngramLogFile = ps.GetString(PROP_QUERY_LOG_FILE);
MaxDepth = ps.GetInt(LanguageModel.PropMaxDepth);
ngramCacheSize = ps.GetInt(PROP_NGRAM_CACHE_SIZE);
clearCacheAfterUtterance = ps.GetBoolean(PROP_CLEAR_CACHES_AFTER_UTTERANCE);
dictionary = (IDictionary)ps.GetComponent(PropDictionary);
applyLanguageWeightAndWip = ps.GetBoolean(PROP_APPLY_LANGUAGE_WEIGHT_AND_WIP);
languageWeight = ps.GetFloat(PROP_LANGUAGE_WEIGHT);
logWip = logMath.LinearToLog(ps.GetDouble(PROP_WORD_INSERTION_PROBABILITY));
unigramWeight = ps.GetFloat(PropUnigramWeight);
}
/// <summary>
/// Apply the unigram weight to the set of unigrams
/// </summary>
private void ApplyUnigramWeight()
{
var logUnigramWeight = _logMath.LinearToLog(_unigramWeight);
var logNotUnigramWeight = _logMath.LinearToLog(1.0f - _unigramWeight);
var logUniform = _logMath.LinearToLog(1.0f / (_numberNGrams[0]));
var logWip = _logMath.LinearToLog(_wip);
var p2 = logUniform + logNotUnigramWeight;
for (var i = 0; i < _numberNGrams[0]; i++)
{
var unigram = Unigrams[i];
var p1 = unigram.LogProbability;
if (i != _startWordID)
{
p1 += logUnigramWeight;
p1 = _logMath.AddAsLinear(p1, p2);
}
if (_applyLanguageWeightAndWip)
{
p1 = p1 * _languageWeight + logWip;
unigram.SetLogBackoff(unigram.LogBackoff * _languageWeight);
}
if (unigram.WordID == 2520)
{
this.LogInfo("TEST");
}
unigram.SetLogProbability(p1);
}
}
protected GrammarNode CreateGrammar()
{
ExtendedStreamTokenizer tok = null;
if (File.Exists(_path))
{
tok = new ExtendedStreamTokenizer(_path, true);
}
else
{
tok = new ExtendedStreamTokenizer((StreamReader)null, true);
}
GrammarNode initialNode = CreateGrammarNode("<sil>");
GrammarNode branchNode = CreateGrammarNode(false);
GrammarNode finalNode = CreateGrammarNode("<sil>");
finalNode.SetFinalNode(true);
List <GrammarNode> wordGrammarNodes = new List <GrammarNode>();
while (!tok.IsEOF())
{
string word;
while ((word = tok.GetString()) != null)
{
wordGrammarNodes.Add(CreateGrammarNode(word));
}
}
// now connect all the GrammarNodes together
initialNode.Add(branchNode, LogMath.LogOne);
if (wordGrammarNodes.Count != 0)
{
float branchScore = _logMath.LinearToLog(1.0 / wordGrammarNodes.Count);
foreach (GrammarNode wordNode in wordGrammarNodes)
{
branchNode.Add(wordNode, branchScore);
wordNode.Add(finalNode, LogMath.LogOne);
if (_isLooping)
{
wordNode.Add(branchNode, LogMath.LogOne);
}
}
}
return(initialNode);
}
public NgramTrieModel(string format, URL location, string ngramLogFile,
int maxNGramCacheSize, bool clearCacheAfterUtterance,
int maxDepth, IDictionary dictionary,
bool applyLanguageWeightAndWip, float languageWeight,
double wip, float unigramWeight)
{
this.format = format;
this.location = location;
this.ngramLogFile = ngramLogFile;
this.ngramCacheSize = maxNGramCacheSize;
this.clearCacheAfterUtterance = clearCacheAfterUtterance;
this.MaxDepth = maxDepth;
logMath = LogMath.GetLogMath();
this.dictionary = dictionary;
this.applyLanguageWeightAndWip = applyLanguageWeightAndWip;
this.languageWeight = languageWeight;
this.logWip = logMath.LinearToLog(wip);
this.unigramWeight = unigramWeight;
}
public override void NewProperties(PropertySheet ps)
{
base.NewProperties(ps);
_logMath = LogMath.GetLogMath();
Linguist = (Linguist.Linguist)ps.GetComponent(PropLinguist);
Pruner = (IPruner)ps.GetComponent(PropPruner);
Scorer = (IAcousticScorer)ps.GetComponent(PropScorer);
_activeListManager = (ActiveListManager)ps.GetComponent(PropActiveListManager);
_showTokenCount = ps.GetBoolean(PropShowTokenCount);
_growSkipInterval = ps.GetInt(PropGrowSkipInterval);
_checkStateOrder = ps.GetBoolean(PropCheckStateOrder);
_maxLatticeEdges = ps.GetInt(PropMaxLatticeEdges);
_acousticLookaheadFrames = ps.GetFloat(PropAcousticLookaheadFrames);
_relativeBeamWidth = _logMath.LinearToLog(ps.GetDouble(PropRelativeBeamWidth));
}
public WordPruningBreadthFirstSearchManager(Linguist.Linguist linguist, IPruner pruner,
IAcousticScorer scorer, ActiveListManager activeListManager,
Boolean showTokenCount, double relativeWordBeamWidth,
int growSkipInterval,
Boolean checkStateOrder, Boolean buildWordLattice,
int maxLatticeEdges, float acousticLookaheadFrames,
Boolean keepAllTokens)
{
_logMath = LogMath.GetLogMath();
Linguist = linguist;
Pruner = pruner;
Scorer = scorer;
_activeListManager = activeListManager;
_showTokenCount = showTokenCount;
_growSkipInterval = growSkipInterval;
_checkStateOrder = checkStateOrder;
BuildWordLattice = buildWordLattice;
_maxLatticeEdges = maxLatticeEdges;
_acousticLookaheadFrames = acousticLookaheadFrames;
KeepAllTokens = keepAllTokens;
_relativeBeamWidth = _logMath.LinearToLog(relativeWordBeamWidth);
}
/// <summary>
/// Loads the language model from the given location.
/// </summary>
/// <param name="location">The URL location of the model.</param>
/// <param name="unigramWeightValue">The unigram weight.</param>
/// <param name="dictionaryValue">The dictionary.</param>
private void Load(URL location, float unigramWeightValue, IDictionary dictionaryValue)
{
string line;
float logUnigramWeight = _logMath.LinearToLog(unigramWeightValue);
float inverseLogUnigramWeight = _logMath
.LinearToLog(1.0 - unigramWeightValue);
Open(location);
// look for beginning of data
ReadUntil("\\data\\");
// look for ngram statements
List <int> ngramList = new List <int>();
while ((line = ReadLine()) != null)
{
if (line.StartsWith("ngram"))
{
StringTokenizer st = new StringTokenizer(line, " \t\n\r\f=");
if (st.countTokens() != 3)
{
Corrupt("corrupt ngram field " + line + ' '
+ st.countTokens());
}
st.nextToken();
int index = int.Parse(st.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
int count = int.Parse(st.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
ngramList.Insert(index - 1, count);
MaxDepth = Math.Max(index, MaxDepth);
}
else if (line.Equals("\\1-grams:"))
{
break;
}
}
int numUnigrams = ngramList[0] - 1;
// -log(x) = log(1/x)
float logUniformProbability = -_logMath.LinearToLog(numUnigrams);
for (int index = 0; index < ngramList.Count; index++)
{
int ngram = index + 1;
int ngramCount = ngramList[index];
for (int i = 0; i < ngramCount; i++)
{
StringTokenizer tok = new StringTokenizer(ReadLine());
int tokenCount = tok.countTokens();
if (tokenCount != ngram + 1 && tokenCount != ngram + 2)
{
Corrupt("Bad format");
}
float log10Prob = float.Parse(tok.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
float log10Backoff = 0.0f;
// construct the WordSequence for this N-Gram
List <Word> wordList = new List <Word>(MaxDepth);
for (int j = 0; j < ngram; j++)
{
string word = tok.nextToken();
_vocabulary.Add(word);
Word wordObject = dictionaryValue.GetWord(word);
if (wordObject == null)
{
wordObject = Word.Unknown;
}
wordList.Add(wordObject);
}
WordSequence wordSequence = new WordSequence(wordList);
if (tok.hasMoreTokens())
{
log10Backoff = float.Parse(tok.nextToken(), CultureInfo.InvariantCulture.NumberFormat);
}
float logProb = _logMath.Log10ToLog(log10Prob);
float logBackoff = _logMath.Log10ToLog(log10Backoff);
// Apply unigram weights if this is a unigram probability
if (ngram == 1)
{
float p1 = logProb + logUnigramWeight;
float p2 = logUniformProbability + inverseLogUnigramWeight;
logProb = _logMath.AddAsLinear(p1, p2);
// System.out
// .println("p1 " + p1 + " p2 " + p2 + " luw "
// + logUnigramWeight + " iluw "
// + inverseLogUnigramWeight + " lup "
// + logUniformProbability + " logprog "
// + logProb);
}
Put(wordSequence, logProb, logBackoff);
}
if (index < ngramList.Count - 1)
{
string next = "\\" + (ngram + 1) + "-grams:";
ReadUntil(next);
}
}
ReadUntil("\\end\\");
Close();
}
/// <summary>
/// Initializes a new instance of the <see cref="ActiveListFactory"/> class.
/// </summary>
/// <param name="absoluteBeamWidth">Width of the absolute beam.</param>
/// <param name="relativeBeamWidth">Width of the relative beam.</param>
protected ActiveListFactory(int absoluteBeamWidth, double relativeBeamWidth)
{
LogMath = LogMath.GetLogMath();
AbsoluteBeamWidth = absoluteBeamWidth;
LogRelativeBeamWidth = LogMath.LinearToLog(relativeBeamWidth);
}
public override void Allocate()
{
_vocabulary.Clear();
_logProbs.Clear();
_logBackoffs.Clear();
HashMap <WordSequence, Integer> unigrams = new HashMap <WordSequence, Integer>();
HashMap <WordSequence, Integer> bigrams = new HashMap <WordSequence, Integer>();
HashMap <WordSequence, Integer> trigrams = new HashMap <WordSequence, Integer>();
int wordCount = 0;
foreach (string sentence in _sentences)
{
string[] textWords = sentence.Split("\\s+");
var words = new List <Word>();
words.Add(_dictionary.GetSentenceStartWord());
foreach (String wordString in textWords)
{
if (wordString.Length == 0)
{
continue;
}
_vocabulary.Add(wordString);
Word word = _dictionary.GetWord(wordString);
if (word == null)
{
words.Add(Word.Unknown);
}
else
{
words.Add(word);
}
}
words.Add(_dictionary.GetSentenceEndWord());
if (words.Count > 0)
{
AddSequence(unigrams, new WordSequence(words[0]));
wordCount++;
}
if (words.Count > 1)
{
wordCount++;
AddSequence(unigrams, new WordSequence(words[1]));
AddSequence(bigrams, new WordSequence(words[0], words[1]));
}
for (int i = 2; i < words.Count; ++i)
{
wordCount++;
AddSequence(unigrams, new WordSequence(words[i]));
AddSequence(bigrams, new WordSequence(words[i - 1], words[i]));
AddSequence(trigrams, new WordSequence(words[i - 2], words[i - 1], words[i]));
}
}
float discount = .5f;
float deflate = 1 - discount;
var uniprobs = new HashMap <WordSequence, Float>();
foreach (var e in unigrams)
{
uniprobs.Put(e.Key, e.Value * deflate / wordCount);
}
LogMath lmath = LogMath.GetLogMath();
float logUnigramWeight = lmath.LinearToLog(_unigramWeight);
float invLogUnigramWeight = lmath.LinearToLog(1 - _unigramWeight);
float logUniformProb = -lmath.LinearToLog(uniprobs.Count);
var sorted1Grams = new SortedSet <WordSequence>(unigrams.Keys);
var iter = new SortedSet <WordSequence>(bigrams.KeySet()).GetEnumerator();
WordSequence ws = iter.MoveNext() ? iter.Current : null;
foreach (WordSequence unigram in sorted1Grams)
{
float p = lmath.LinearToLog(uniprobs.Get(unigram));
p += logUnigramWeight;
p = lmath.AddAsLinear(p, logUniformProb + invLogUnigramWeight);
_logProbs.Put(unigram, p);
float sum = 0f;
while (ws != null)
{
int cmp = ws.GetOldest().CompareTo(unigram);
if (cmp > 0)
{
break;
}
if (cmp == 0)
{
sum += uniprobs.Get(ws.GetNewest());
}
ws = iter.MoveNext() ? iter.Current : null;
}
_logBackoffs.Put(unigram, lmath.LinearToLog(discount / (1 - sum)));
}
var biprobs = new HashMap <WordSequence, Float>();
foreach (var entry in bigrams)
{
int unigramCount = unigrams.Get(entry.Key.GetOldest());
biprobs.Put(entry.Key, entry.Value * deflate / unigramCount);
}
var sorted2Grams = new SortedSet <WordSequence>(bigrams.KeySet());
iter = new SortedSet <WordSequence>(trigrams.KeySet()).GetEnumerator();
ws = iter.MoveNext() ? iter.Current : null;
foreach (WordSequence biword in sorted2Grams)
{
_logProbs.Put(biword, lmath.LinearToLog(biprobs.Get(biword)));
float sum = 0f;
while (ws != null)
{
int cmp = ws.GetOldest().CompareTo(biword);
if (cmp > 0)
{
break;
}
if (cmp == 0)
{
sum += biprobs.Get(ws.GetNewest());
}
ws = iter.MoveNext() ? iter.Current : null;
}
_logBackoffs.Put(biword, lmath.LinearToLog(discount / (1 - sum)));
}
foreach (var e in trigrams)
{
float p = e.Value * deflate;
p /= bigrams.Get(e.Key.GetOldest());
_logProbs.Put(e.Key, lmath.LinearToLog(p));
}
}