public void Run_3ObservationsAnd2StatesAndRainySunnyModel_PathCount3()
{
var algo = new Viterbi(false);
var observations = new List <IObservation>
{
new Observation(new double[] { 0 }, "walk"),
new Observation(new double[] { 1 }, "shop"),
new Observation(new double[] { 2 }, "clean")
};
var startDistribution = new [] { 0.6, 0.4 };
var states = new List <IState> {
new State(0, "Rainy"), new State(1, "Sunny")
};
var tpm = new double[][] { new[] { 0.7, 0.3 }, new[] { 0.4, 0.6 } };
var distributions = new List <IDistribution> {
new RainyDistribution(), new SunnyDistribution()
};
var path = algo.Run(observations, states, startDistribution, tpm, distributions);
Assert.AreEqual(path.Count, 3);
Assert.AreEqual("Sunny", path[0].Description);
Assert.AreEqual("Rainy", path[1].Description);
Assert.AreEqual("Rainy", path[2].Description);
}
public void Run_Normalized5ObservationsAnd3StatesAndHealthySickModel_PathCount5()
{
var algo = new Viterbi(true);
var observations = new List <IObservation>
{
new Observation(new double[] { 0 }, "high"),
new Observation(new double[] { 1 }, "average"),
new Observation(new double[] { 1 }, "average"),
new Observation(new double[] { 2 }, "low"),
new Observation(new double[] { 2 }, "low")
};
var startDistribution = new[] { 1 / 3d, 1 / 3d, 1 / 3d };
var states = new List <IState> {
new State(0, "Healthy"), new State(1, "OK"), new State(2, "Sick")
};
var tpm = new double[][] { new[] { 0.4, 0.3, 0.3 }, new[] { 0.2, 0.6, 0.2 }, new[] { 0, 0.4, 0.6 } };
var distributions = new List <IDistribution> {
new HealthyDistribution(), new OkDistribution(), new SickDistribution()
};
var path = algo.Run(observations, states, startDistribution, tpm, distributions);
Assert.AreEqual(path.Count, 5);
Assert.AreEqual("Healthy", path[0].Description);
Assert.AreEqual("OK", path[1].Description);
Assert.AreEqual("OK", path[2].Description);
Assert.AreEqual("Sick", path[3].Description);
Assert.AreEqual("Sick", path[4].Description);
}
public void Run_3ObservationsAnd2StatesAnd123Model_PathCount3()
{
var algo = new Viterbi(false);
var observations = new List <IObservation>
{
new Observation(new double[] { 0 }, "Red"),
new Observation(new double[] { 1 }, "Blue"),
new Observation(new double[] { 0 }, "Red")
};
var startDistribution = new [] { 1 / 3d, 1 / 3d, 1 / 3d };
var states = new List <IState> {
new State(0, "One"), new State(1, "Two"), new State(2, "Three")
};
var tpm = new double[][] { new[] { 0.3, 0.6, 0.1 }, new[] { 0.5, 0.2, 0.3 }, new[] { 0.4, 0.1, 0.5 } };
var distributions = new List <IDistribution>
{
new FirstDistribution(), new SecondDistribution(), new ThirdDistribution()
};
var path = algo.Run(observations, states, startDistribution, tpm, distributions);
Assert.AreEqual(path.Count, 3);
Assert.AreEqual("One", path[0].Description);
Assert.AreEqual("Two", path[1].Description);
Assert.AreEqual("One", path[2].Description);
}
public void HealthyFeverTest()
{
double[,] A =
{
{ 0.7, 0.3 },
{ 0.4, 0.6 }
};
double[,] B =
{
{ 0.5, 0.4, 0.1 },
{ 0.1, 0.3, 0.6 }
};
double[] pi =
{
0.6, 0.4
};
int[] observation = { 0, 1, 2 };
string[] Q = { "Healthy", "Fever" };
int[] V = { 0, 1, 2 };
var T = observation.Length; // length of the observation sequence
var N = Q.Length; // number of states in the model (H, C)
var M = V.Length; // number of observation model (Small, Medium, Large)
var expectedResult = new string[] { "Healthy", "Healthy", "Fever" };
var resultFromViterbi = Viterbi.GetResult(T, N, M, Q, V, A, B, pi, observation);
CollectionAssert.AreEqual(expectedResult, resultFromViterbi);
}
private static void DemoDiscreteHamming()
{
var config = CodeConfig.Size7_6d_4f;
var input = 15003.GetBools(14)
.Concat(Enumerable.Repeat(false, 6));
//var input = "11101010011011".ParseBools();
var encoder = new Encoder(config, terminateCode: false);
var output = encoder.Encode(input);
var viterbi = Viterbi.CreateWithHammingDistance(config);
var restored = viterbi.Solve(output);
Console.WriteLine("Configuration: " + config);
Console.WriteLine();
Console.WriteLine("Input: " + input.Format());
Console.WriteLine("Encoded: " + output.Format());
Console.WriteLine("Restored: " + restored.Message.Format());
Console.Write($"Score: {restored.BestEndScore:F2}");
//if (restored.TerminationStateScore.HasValue)
Console.Write($" / {restored.TerminationStateScore:F2}");
Console.WriteLine();
Console.WriteLine();
}
public void Run_4ObservationsAnd2StatesAndSTModel_PathCount4()
{
var observations = new List <IObservation>
{
new Observation(new double[] { 0 }, "A"),
new Observation(new double[] { 1 }, "B"),
new Observation(new double[] { 1 }, "B"),
new Observation(new double[] { 0 }, "A")
};
var startDistribution = new[] { 0.85, 0.15 };
var states = new List <IState> {
new State(0, "s"), new State(1, "t")
};
var tpm = new double[2][];
tpm[0] = new[] { 0.3, 0.7 };
tpm[1] = new[] { 0.1, 0.9 };
var distributions = new List <IDistribution> {
new HealthyDistribution(), new OkDistribution(), new SickDistribution()
};
var algo = new Viterbi(false);
var path = algo.Run(observations, states, startDistribution, tpm, distributions);
Assert.AreEqual(path.Count, 4);
Assert.AreEqual("s", path[0].Description);
Assert.AreEqual("t", path[1].Description);
Assert.AreEqual("t", path[2].Description);
Assert.AreEqual("t", path[3].Description);
}
/// <summary>
/// Creates an instance.
/// </summary>
/// <param name="tagger">A tagger that this instnace peeps in.</param>
/// <remarks>
/// It is usually easier to use <see cref="Hack.GetDictionaries{TNode}(MeCabTaggerBase{TNode})"/> than this constructor.
/// </remarks>
public DictionaryBundle(MeCabTaggerBase <TNode> tagger)
{
NodeAllocator = Hack.GetFieldValue(tagger, "nodeAllocator") as Func <TNode>;
Viterbi = Hack.GetFieldValue(tagger, "viterbi") as Viterbi <TNode>;
Connector = Hack.GetFieldValue(Viterbi, "connector") as Connector <TNode>;
Tokenizer = Hack.GetFieldValue(Viterbi, "tokenizer") as Tokenizer <TNode>;
Dictionaries = Hack.GetFieldValue(Tokenizer, "dic") as MeCabDictionary[];
}
public void Run_100LengthObservations2MixtureDistribution_PathCount100()
{
var algo = new Viterbi(false);
var path = algo.Run(_observations, _states, _startDistribution, _tpm, _mixtures);
Assert.AreEqual(path.Count, 100);
}
private double[] PredictNextValue <TDistribution>(IHiddenMarkovModel <TDistribution> model, double[][] trainingSet) where TDistribution : IDistribution
{
var alg = new Viterbi(model.Normalized);
var mpp = alg.Run(Helper.Convert(trainingSet), model.GetStates(), model.Pi, model.TransitionProbabilityMatrix, model.GetEmissions());
var emission = model.Emission[mpp[trainingSet.Length - 1].Index];
var prediction = CalculatePredictionValue(emission, trainingSet);
return(prediction);
}
private ViterbiResult RunViterbi(FilterResult filterResult, string fileSha1)
{
// Load any user-defined state machines.
List <UserState> userStates;
try
{
userStates = Loader.LoadUserStates(MainForm.Program.UserStatesEnabled,
MainForm.Program.UserStatesPath);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "User-Defined States");
return(null);
}
ViterbiResult viterbiResultFields = null;
try
{
write("Running Viterbi on fields");
DateTime dt = DateTime.Now;
#if _GENERALPARSE
Viterbi viterbi = new Viterbi(RunType.GeneralParse, false);
viterbiResultFields = viterbi.Run(filterResult.UnfilteredBlocks, this.filePath);
#else
#if SKIPREALWORK
viterbiResultFields = new ViterbiResult();
viterbiResultFields.Fields = new List <ViterbiField>();
#else
if (filterResult.UnfilteredBlocks.Count > 0)
{
ThreadedViterbi tv = new ThreadedViterbi(filterResult.UnfilteredBlocks, RunType.GeneralParse, userStates, this.filePath, this.fileSha1);
viterbiResultFields = tv.RunThreadedViterbi();
TimeSpan ts = DateTime.Now.Subtract(dt);
write("Time elapsed for Viterbi fields: {0}", ts.ToString("c"));
write("Field count: {0}", viterbiResultFields.Fields.Count);
}
#endif
#endif
filterResult.UnfilteredBlocks.Clear(); // Allow gc to clean things up
}
catch (ThreadAbortException)
{
return(null);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "Viterbi Fields");
return(null);
}
return(viterbiResultFields);
}
/// <summary>
/// Create a new instance.
/// </summary>
/// <param name="lnbIndex">Zero based index of the LNB the channel should be bound to.</param>
/// <param name="uFrequency">Frequency.</param>
/// <param name="eInversion">Spectrum inversion.</param>
/// <param name="uSymbolRate">Symbol rate.</param>
/// <param name="ePower">Polarisation selection.</param>
/// <param name="eViterbi">[Don't know]</param>
/// <exception cref="ArgumentOutOfRangeException">The LNB index is negative or greater than 3.</exception>
public SatelliteChannel(int lnbIndex, uint uFrequency, SpectrumInversion eInversion, uint uSymbolRate, PowerMode ePower, Viterbi eViterbi, bool DVBS2)
: base(uFrequency, eInversion)
{
// Verify
if ( (lnbIndex < 0) || (lnbIndex > 3) ) throw new ArgumentOutOfRangeException("lnbIndex", lnbIndex, "Must not be less than zero or greater than 3");
// Remember
SymbolRate = uSymbolRate;
S2Modulation = DVBS2;
LNBIndex = lnbIndex;
Viterbi = eViterbi;
Power = ePower;
}
public void WorkbookExamples(TestCase testCase)
{
var decoder = Viterbi.CreateWithHammingDistance(testCase.Config);
var res = decoder.Solve(testCase.Input.ParseBools());
var expected = testCase.Expected.ParseBools();
output.WriteLine("Result: " + res.Message.Format());
output.WriteLine("Expected: " + expected.Format());
res.Message.ShouldBe(expected);
}
public void Setup()
{
var config = CodeConfig.Size7_6d_4f;
var input =
15003.GetBools(14)
.Concat(Enumerable.Repeat(false, 6));
var encoder = new Encoder(config, terminateCode: true);
encoded = encoder.Encode(input);
viterbi = Viterbi.CreateWithHammingDistance(config);
}
public void TestCalculateHits()
{
int[] traceBackPath = new int[10];
traceBackPath[0] = 1;
traceBackPath[1] = 1;
traceBackPath[2] = 1;
traceBackPath[3] = 1;
traceBackPath[4] = 1;
traceBackPath[5] = 0;
traceBackPath[6] = 0;
traceBackPath[7] = 0;
traceBackPath[8] = 1;
traceBackPath[9] = 1;
var result = Viterbi.CalculateHits(traceBackPath);
}
public void TestExample(Func <IEnumerable <bool>, IEnumerable <double>, double> calcScore, ScoreMethod scoreMethod, double falseValue = 0.0)
{
var input = "11 10 00 01 01 11 00"
.ParseBools()
.Select(b => b?1.0: falseValue);
var expected = "1 0 1 1 0 0 0";
var config = CodeConfig.Size3_7_5;
var decoder = new Viterbi <double>(config.EnumerateTransitions(), calcScore, new ViterbiConfig()
{
InitialState = State.Zero(config.NoOfStateRegisters), ScoreMethod = scoreMethod
});
var res = decoder.Solve(input);
res.Message.ShouldBe(expected.ParseBools());
}
public void Run_Normalized9ObservationsAnd2StatesAndHLModel_PathCount9()
{
var algo = new Viterbi(true);
var observations = new List <IObservation>
{
new Observation(new double[] { 2 }, "G"),
new Observation(new double[] { 2 }, "G"),
new Observation(new double[] { 1 }, "C"),
new Observation(new double[] { 0 }, "A"),
new Observation(new double[] { 1 }, "C"),
new Observation(new double[] { 3 }, "T"),
new Observation(new double[] { 2 }, "G"),
new Observation(new double[] { 0 }, "A"),
new Observation(new double[] { 0 }, "A")
};
var startDistribution = new[] { 0.5, 0.5 };
var states = new List <IState> {
new State(0, "H"), new State(1, "L")
};
var tpm = new double[][] { new[] { 0.5, 0.5 }, new[] { 0.4, 0.6 } };
var distributions = new List <IDistribution> {
new HDistribution(), new LDistribution()
};
var path = algo.Run(observations, states, startDistribution, tpm, distributions);
Assert.AreEqual(path.Count, 9);
Assert.AreEqual("H", path[0].Description);
Assert.AreEqual("H", path[1].Description);
Assert.AreEqual("H", path[2].Description);
Assert.AreEqual("L", path[3].Description);
Assert.AreEqual("L", path[4].Description);
Assert.AreEqual("L", path[5].Description);
Assert.AreEqual("L", path[6].Description);
Assert.AreEqual("L", path[7].Description);
Assert.AreEqual("L", path[8].Description);
}
/// <summary>
/// The start simulation_ click.
/// </summary>
/// <param name="sender">
/// The sender.
/// </param>
/// <param name="e">
/// The e.
/// </param>
private void StartSimulation(object sender, RoutedEventArgs e)
{
double swicthToUnfairChance = (double)this.ChangeToSwitchToUnfair.Value;
double switchToFairChance = (double)this.ChanceToSwitchToFair.Value;
int numberOfRolls = (int)this.NumberOfRolls.Value;
var casinoSimulation = new Casino(this.DiceSides.ToList(), switchToFairChance, swicthToUnfairChance);
this.steps = casinoSimulation.Simulate(numberOfRolls);
var alfaBeta = new AlfaBeta(this.steps, switchToFairChance, swicthToUnfairChance, this.DiceSides.ToList());
var alfaBetaResult = alfaBeta.Calculate();
double alfaPass = 0;
for (int i = 0; i < alfaBetaResult.Length; ++i)
{
if (alfaBetaResult[i] == this.steps[i].DiceState)
{
++alfaPass;
}
}
var viterbi = new Viterbi(this.steps, switchToFairChance, swicthToUnfairChance, this.DiceSides.ToList());
var viterbiResult = viterbi.Calculate();
double viterbiPass = 0;
for (int i = 0; i < viterbiResult.Length; ++i)
{
if (viterbiResult[i] == this.steps[i].DiceState)
{
++viterbiPass;
}
}
int a = 0;
}
private static void DemoContinuousSymmetricScore()
{
var config = CodeConfig.Size7_6d_4f;
var input = 15003.GetBools(14)
.Concat(Enumerable.Repeat(false, 6));
var encoder = new Encoder(config, terminateCode: false);
var encoded = encoder.Encode(input);
var transmitted =
encoded
.Select(e => e ? 255.0 : 0);
var decoder = new Viterbi <double>(
config.EnumerateTransitions(),
SymmetricScore.Range_0_255.CalculateScore,
new ViterbiConfig()
{
InitialState = State.Zero(config.NoOfStateRegisters),
ScoreMethod = ScoreMethod.Maximize,
TerminationState = State.Zero(config.NoOfStateRegisters)
});
var restored = decoder.Solve(transmitted);
Console.WriteLine("Configuration: " + config);
Console.WriteLine();
Console.WriteLine("Input: " + input.Format());
Console.WriteLine("Encoded: " + encoded.Format());
Console.WriteLine("Restored: " + restored.Message.Format());
Console.Write($"Score: {restored.BestEndScore:F2}");
//if (restored.TerminationStateScore.HasValue)
Console.Write($" / {restored.TerminationStateScore:F2}");
Console.WriteLine();
Console.WriteLine();
}
/// <summary> /// 维特比算法求解最优标签 /// </summary> /// <param name="roleTagList"></param> /// <returns></returns> public static List <NT> ViterbiExCompute(List <TagFreqItem <NT> > roleTagList) => Viterbi.Compute(roleTagList, OrgDictionary.transformMatrixDictionary);
/**
* 维特比算法求解最优标签
*
* @param roleTagList
* @return
*/
public static LinkedList <Corpus.Tag.NT> viterbiExCompute(LinkedList <EnumItem <Corpus.Tag.NT> > roleTagList)
{
return(Viterbi.computeEnum(roleTagList, OrganizationDictionary.transformMatrixDictionary));
}
/**
* 简化的"维特比算法"求解最优标签
* @param roleTagList
* @return
*/
public static List <Corpus.Tag.NR> viterbiComputeSimply(LinkedList <EnumItem <Corpus.Tag.NR> > roleTagList)
{
return(Viterbi.computeEnumSimply(roleTagList, PersonDictionary.transformMatrixDictionary));
}
/// <summary>
/// Method for evaluating model
/// </summary>
/// <param name="stateToIntMap">State to int map</param>
/// <param name="obsToIntMap">Observation to int map</param>
/// <param name="stateToObservation">Map of observations found in states where state is the key</param>
/// <param name="stateNgramProbs">State N-grams probabilities</param>
/// <param name="obsNgramProbs">Observation N-grams probabilities </param>
/// <param name="order">Max N-gram order</param>
/// <param name="testFile">Test file containing test sentences</param>
public static void testModel(Mapper stateToIntMap, Mapper obsToIntMap,
SortedDictionary <int, SortedDictionary <int, int> > stateToObservation,
List <Dictionary <List <int>, double[]> > stateNgramProbs,
List <Dictionary <List <int>, double[]> > obsNgramProbs, int order, String testFile)
{
//initialize Ngrams HMM
HMM stateHMM = new HMM(stateNgramProbs);
HMM obsHMM = new HMM(obsNgramProbs);
//initialize viterbi
List <List <String>[]> sentences = getTestSentences(testFile);
Viterbi viterbi = new Viterbi(stateToIntMap, obsToIntMap, stateToObservation, stateHMM, order);
int errors = 0;
double totalStateProb = 0;
int totalStateWords = 0;
double totalObsProb = 0;
int totalObsWords = 0;
double totalObsProbOOV = 0;
int totalObsWordsOOV = 0;
int numOfSkippedWords = 0;
int numOfSkippedSentences = 0;
int numOfSkippedOOV = 0;
bool skip;
//read test file and perform calculations
for (int i = 0; i < sentences.Count; i++)
{
List <String> obs = sentences[i][0];
List <String> states = sentences[i][1];
List <String> retStates = new List <String>(viterbi.process(obs.ToArray()));
//calculating state Ngrams perplexity
List <int> stack = new List <int> {
-1
};
double sentProb = 0;
for (int j = 0; j < retStates.Count; j++)
{
int token = stateToIntMap.getValue(retStates[j]);
stack.Add(token);
double value = stateHMM.getStateToStateProb(stack);
sentProb += value;
while (true)
{
if (stateHMM.contains(stack))
{
break;
}
if (stack.Count > 1)
{
stack.RemoveAt(0);
}
else
{
break;
}
}
if (stack.Count == order)
{
stack.RemoveAt(0);
}
}
totalStateWords += states.Count;
totalStateProb += sentProb;
//calculating obs Ngrams perplexity
stack = new List <int> {
-1
};
sentProb = 0;
int skippedWords = 0;
skip = false;
for (int j = 0; j < obs.Count; j++)
{
try {
int token = obsToIntMap.getValue(obs[j]);
if (token == -1)
{
throw new Exception();
}
stack.Add(token);
double value = obsHMM.getStateToStateProb(stack);
sentProb += value;
while (true)
{
if (obsHMM.contains(stack))
{
break;
}
if (stack.Count > 1)
{
stack.RemoveAt(0);
}
else
{
break;
}
}
if (stack.Count == order)
{
stack.RemoveAt(0);
}
}
catch (Exception) {
skip = true;
skippedWords++;
}
}
if (!skip)
{
totalObsProb += sentProb;
totalObsWords += obs.Count;
}
else
{
numOfSkippedWords += obs.Count;
numOfSkippedSentences++;
totalObsProbOOV += sentProb;
numOfSkippedOOV += skippedWords;
totalObsWordsOOV += (obs.Count - skippedWords);
}
//calculating WER
for (int j = 0; j < retStates.Count; j++)
{
if (!states[j].Equals(retStates[j]))
{
errors++;
}
}
}
//show results
double exp = totalStateProb / totalStateWords;
double perplexity = Math.Pow(2, -exp);
Console.WriteLine("\nSTATE N-GRAMS RESULTS:");
Console.WriteLine("Number of state sentences: " + sentences.Count);
Console.WriteLine("Number of state words: " + totalStateWords);
Console.WriteLine("Total states log2 probability: " + totalStateProb);
Console.WriteLine("State perplexity: " + perplexity);
Console.WriteLine("WER: " + (errors / (double)totalStateWords * 100) + "%");
exp = totalObsProbOOV / totalObsWordsOOV;
perplexity = Math.Pow(2, -exp);
Console.WriteLine("\nOBS N-GRAMS RESULTS:");
Console.WriteLine("Number of observation sentences: " + sentences.Count);
Console.WriteLine("Number of observations in vocabulary: " + totalObsWordsOOV);
Console.WriteLine("Number of observations out of vocabulary: " + numOfSkippedOOV);
Console.WriteLine("Total observation log2 probability with OOV skipped: " + totalObsProbOOV);
Console.WriteLine("Observation perplexity with OOV skipped: " + perplexity);
exp = totalObsProb / totalObsWords;
perplexity = Math.Pow(2, -exp);
Console.WriteLine("\nOOV SENTENCES SKIPPED:");
Console.WriteLine("Number of observation sentences: " + (sentences.Count - numOfSkippedSentences));
Console.WriteLine("Number of observations: " + totalObsWords);
Console.WriteLine("Number of skipped sentences: " + numOfSkippedSentences);
Console.WriteLine("Number of skipped observations: " + numOfSkippedWords);
Console.WriteLine("Total observation log2 probability: " + totalObsProb);
Console.WriteLine("Observation perplexity: " + perplexity);
}
/// <summary>
/// Method for evaluating sentence
/// </summary>
/// <param name="stateToIntMap">State to int map</param>
/// <param name="obsToIntMap">Observation to int map</param>
/// <param name="stateToObservation">Map of observations found in states where state is the key</param>
/// <param name="stateNgramProbs">State N-grams probabilities</param>
/// <param name="obsNgramProbs">Observation N-grams probabilities</param>
/// <param name="order">Max N-gram order</param>
/// <param name="sentence">Test sentence</param>
public static void testSentence(Mapper stateToIntMap, Mapper obsToIntMap,
SortedDictionary <int, SortedDictionary <int, int> > stateToObservation,
List <Dictionary <List <int>, double[]> > stateNgramProbs,
List <Dictionary <List <int>, double[]> > obsNgramProbs, int order, String sentence)
{
//initialize Ngrams HMM
HMM stateHMM = new HMM(stateNgramProbs);
HMM obsHMM = new HMM(obsNgramProbs);
//initialize viterbi
Viterbi viterbi = new Viterbi(stateToIntMap, obsToIntMap, stateToObservation, stateHMM, order);
String[] split = Regex.Split(sentence, "\\s+");
List <String> retStates = new List <String>(viterbi.process(split.ToArray()));
//calculating state Ngrams perplexity
List <int> stack = new List <int> {
-1
};
double sentStateProb = 0;
String stateSequence = "";
for (int j = 0; j < retStates.Count; j++)
{
int token = stateToIntMap.getValue(retStates[j]);
stack.Add(token);
double value = stateHMM.getStateToStateProb(stack);
sentStateProb += value;
while (true)
{
if (stateHMM.contains(stack))
{
break;
}
stack.RemoveAt(0);
}
if (stack.Count == order)
{
stack.RemoveAt(0);
}
stateSequence += retStates[j] + " ";
}
//calculating obs Ngrams perplexity
stack = new List <int> {
-1
};
double sentObsProb = 0;
int skippedWords = 0;
for (int j = 0; j < split.Length; j++)
{
try {
int token = obsToIntMap.getValue(split[j]);
if (token == -1)
{
throw new Exception();
}
stack.Add(token);
double value = obsHMM.getStateToStateProb(stack);
sentObsProb += value;
while (true)
{
if (obsHMM.contains(stack))
{
break;
}
if (stack.Count > 1)
{
stack.RemoveAt(0);
}
else
{
break;
}
}
if (stack.Count == order)
{
stack.RemoveAt(0);
}
}
catch (Exception) {
skippedWords++;
}
}
Console.WriteLine("\nSENTENCE EVALUATION RESULTS");
Console.WriteLine("Sentence: " + sentence);
Console.WriteLine("Number of out of vocabulary words in sentence: " + skippedWords);
Console.WriteLine("Most probable state sequence: " + stateSequence);
Console.WriteLine("State sequence probability: " + Math.Pow(2, sentStateProb));
Console.WriteLine("Sentence probability with out of vocabulary words skipped: " + Math.Pow(2, sentObsProb));
}
public SatelliteChannel(int lnbIndex, uint uFrequency, SpectrumInversion eInversion, uint uSymbolRate, PowerMode ePower, Viterbi eViterbi)
: this(lnbIndex, uFrequency, eInversion, uSymbolRate, ePower, eViterbi, false)
{
}
public static List <NS> ViterbiExCompute(List <TagFreqItem <NS> > roleTagList) => Viterbi.Compute(roleTagList, PlaceDictionary.trans_tr_dict);
public static List <NR> ViterbiComputeSimply(List <TagFreqItem <NR> > tags) => Viterbi.ComputeSimply(tags, ChsPersonNameDict.transformMatrixDictionary);
/// <summary> /// 词性标注 /// </summary> /// <param name="list"></param> public static void SpeechTaggint(List <Vertex> list) => Viterbi.Compute(list, CoreDictTransfromMatrixDictionary.transformMatrixDictionary);
public override List <Term> SegSentence(char[] sentence)
{
var list = new List <Term>();
if (sentence.Length == 0)
{
return(list);
}
var convertedChars = CharTable.Convert(sentence);
var table = new Table();
table.v = AtomSeg2Table(convertedChars);
_crfModel.Tag(table);
int offset = 0;
for (int i = 0; i < table.Size; offset += table.v[i][1].Length, i++)
{
var line = table.v[i];
switch (line[2][0])
{
case 'B':
int begin = offset;
while (table.v[i][2][0] != 'E') // 寻找结束标签'E'
{
offset += table.v[i][1].Length;
i++;
if (i == table.Size)
{
break; // 达到最后一个字符
}
}
// 退出while循环
if (i == table.Size) // 肯定是由while loop的break退出的，offset已经包含了最后一格词的长度
{
list.Add(new Term(new string(sentence, begin, offset - begin), Nature.none));
}
else // 由while loop正常退出，当前词标注为'E',offset尚未包含这个词的长度
{
list.Add(new Term(new string(sentence, begin, offset - begin + table.v[i][1].Length), Nature.none));
}
break;
default: // 理论来说，只可能是标注为'S'，所以单独成词
list.Add(new Term(new string(sentence, offset, table.v[i][1].Length), Nature.none));
break;
}
}
if (config.natureTagging)
{
var vertices = ToVertexList(list, true);
Viterbi.Compute(vertices, CoreDictTransfromMatrixDictionary.transformMatrixDictionary);
for (int i = 0; i < list.Count; i++)
{
var term = list[i];
if (term.nature == Nature.none)
{
term.nature = vertices[i + 1].GuessNature(); // vertices[i+1] -> 附加了辅助起始节点
}
}
}
if (config.useCustomDict)
{
var vertices = ToVertexList(list, false); //? 会不会覆盖上面的词性标注值
CombineByCustomDict(vertices);
list = ToTermList(vertices, config.offset);
}
return(list);
}
