public ViterbiForGlobalLinearModel(WeightVector weightVector, Tags tags) { WeightVector = weightVector; Tags = tags; Pi = new List<Dictionary<string, double>>(); Bp = new List<Dictionary<string, string>>(); }
public ForwardBackwordAlgo(List<string> inputSentence, WeightVector wc, List<string> tagList) { _inputSentence = inputSentence; _wc = wc; _tagList = tagList; _alphaDictionary = new Dictionary<int, Dictionary<string, double>>(); _betaDictionary = new Dictionary<int, Dictionary<string, double>>(); _uDictionary = new Dictionary<int, Dictionary<string, double>>(); UabDictionary = new Dictionary<int, Dictionary<string, double>>(); Z = 0; _weightedFeaturesum = new WeightedFeatureSum(wc, inputSentence, true); cList = new List<double>(_inputSentence.Count); dList = new List<double>(_inputSentence.Count); _useScaling = true; _useLog = false; _twoGramsList = new string[4]; var ngramTags = new Tags(_tagList); int index = 0; foreach (var ngram in ngramTags.GetNGramTags(2)) { if (index >= _twoGramsList.Length) { Array.Resize(ref _twoGramsList, index + 1); } string[] split = ngram.Split(new[] { ':' }); _twoGramsList[index] = split[0] + "@#" + split[1]; index++; } }
public ComputeGradient(List<List<string>> inputSentence, List<List<string>> tagsList, List<string> tagList, double lambda, double learningParam, FeatureCache cache, WriteModel logger) { Logger = logger; _inputSentence = inputSentence; _outputTagsList = tagsList; _tagList = tagList; _lambda = lambda; _learningParam = learningParam; _cache = cache; forwardBackwordAlgos = new List<ForwardBackwordAlgo>(); _weightVector = null; _twoGramsList = new string[4]; _twoGramPair = new KeyValuePair<string, string>[4]; var ngramTags = new Tags(_tagList); int index = 0; foreach (var ngram in ngramTags.GetNGramTags(2)) { if (index >= _twoGramsList.Length) { Array.Resize(ref _twoGramsList, index+1); Array.Resize(ref _twoGramPair, index + 1); } string[] split = ngram.Split(new[] { ':' }); _twoGramsList[index] = split[0] +"@#"+ split[1]; _twoGramPair[index] = new KeyValuePair<string, string>(split[0], split[1]); index++; } }
public void Setup(bool debug) { var readModel = new ReadModel(InputModelFile); var temp = new ReadModel(string.Concat(InputModelFile, ".featuresToK")); _weightVector = new WeightVector(temp.GetFeatureToKdDictionary()); foreach (var pair in readModel.ModelIterator()) { _weightVector.Add(pair); } _tags = new Tags(_tagList); _viterbiForGlobalLinearModel = new ViterbiForGlobalLinearModel(_weightVector, _tags); // read input file in a class and per line iterator. var inputData = new ReadInputData(InputTestFile); var writeModel = new WriteModel(_outputTestFile); foreach (var line in inputData.GetSentence()) { List<string> debugList; var outputTags = _viterbiForGlobalLinearModel.Decode(line, debug, out debugList); if (debug) { writeModel.WriteDataWithTagDebug(line, outputTags, debugList); } else { writeModel.WriteDataWithTag(line, outputTags); } } writeModel.Flush(); }
public ThreadInfoObject(ComputeGradient cg, int start, int end, WeightVector wc, ManualResetEvent resetEvent) { Gradient = cg; Start = start; End = end; NewWeightVector = wc; ResetEvent = resetEvent; }
public ComputeGradient(List<List<string>> inputSentence, List<List<string>> tagsList, List<string> tagList, double lambda, FeatureCache cache) { _inputSentence = inputSentence; _outputTagsList = tagsList; _tagList = tagList; _lambda = lambda; _cache = cache; forwardBackwordAlgos = new List<ForwardBackwordAlgo>(); _weightVector = null; }
public void ReMappingFromWeightVector(WeightVector weightVector, bool normalize = true) { var newDictKtoF = new Dictionary<int, string>(); var newDictFtoK = new Dictionary<string, int>(); //var weightDict = new Dictionary<int, double>(); //int k = 0; //var elements = from element in weightVector.WeightArray // orderby element descending // where element > 1 // select element; //var sortedDictionary = from pair in weightVector.WDictionary // where Math.Abs(pair.Value) > 1 // orderby Math.Abs(pair.Value) descending // select pair; const double limit = 0; var newWeights = new double[weightVector.FeatureCount]; int featureCount = 0; Array.Clear(newWeights, 0, newWeights.Length); double max = 0; for (int i = 0; i < weightVector.FeatureCount; i++) { if (max < Math.Abs(weightVector.WeightArray[i])) { max = Math.Abs(weightVector.WeightArray[i]); } if (Math.Abs(weightVector.WeightArray[i]) > limit) { newWeights[featureCount] = weightVector.WeightArray[i]; var feature = DictKToFeatures[i]; newDictFtoK[feature] = featureCount; newDictKtoF[featureCount] = feature; featureCount++; } } //if (normalize) //{ // for (int i = 0; i < featureCount; i++) // { // newWeights[i] /= max; // } //} weightVector.WeightArray = newWeights; weightVector.FeatureCount = featureCount; DictFeaturesToK = weightVector.FeatureKDictionary = newDictFtoK; DictKToFeatures = newDictKtoF; if (normalize) { weightVector.AvgNormalize(); } }
public Perceptron(string inputFile, string outputFile, List<string> tagList) { _inputFile = inputFile; _outputFile = outputFile; var tags = new Tags(tagList); MapFeatures = new MapFeaturesToK(inputFile, string.Concat(outputFile, ".featuresToK"), tagList); MapFeatures.StartMapping(); WeightVector = new WeightVector(MapFeatures.DictFeaturesToK, MapFeatures.FeatureCount); _viterbiForGlobalLinearModel = new ViterbiForGlobalLinearModel(WeightVector, tags); InputSentences = new List<List<string>>(); TagsList = new List<List<string>>(); ReadInputs(); }
public ForwardBackwordAlgo(List<string> inputSentence, WeightVector wc, List<string> tagList) { _inputSentence = inputSentence; _wc = wc; _tagList = tagList; _tags = new Tags(tagList); _alphaDictionary = new Dictionary<int, Dictionary<string, double>>(); _betaDictionary = new Dictionary<int, Dictionary<string, double>>(); _uDictionary = new Dictionary<int, Dictionary<string, double>>(); UabDictionary = new Dictionary<int, Dictionary<string, double>>(); Z = 0; _weightedFeaturesum = new WeightedFeatureSum(wc, inputSentence, true); }
public Perceptron(List<string> inputFiles, string outputFile, List<string> tagList, bool useAvg = false) { _outputFile = outputFile; _useAvg = useAvg; var tags = new Tags(tagList); MapFeatures = new MapFeaturesToK(string.Concat(outputFile, ".featuresToK"), tagList); MapFeatures.StartMapping(inputFiles); WeightVector = new WeightVector(MapFeatures.DictFeaturesToK, MapFeatures.FeatureCount); AvgWeightVector = new WeightVector(MapFeatures.DictFeaturesToK, MapFeatures.FeatureCount); _viterbiForGlobalLinearModel = new ViterbiForGlobalLinearModel(WeightVector, tags); InputSentences = new List<List<string>>(); TagsList = new List<List<string>>(); //ReadInputs(); }
public void Init() { var readModel = new ReadModel(InputModelFile + ".preceptron"); var temp = new ReadModel(string.Concat(InputModelFile, ".featuresToK")); var dict = temp.GetFeatureToKdDictionary(); _weightVector = new WeightVector(dict, dict.Count); foreach (var pair in readModel.ModelIterator()) { _weightVector.Add(pair); } _tags = new Tags(_tagList); ViterbiForGLM = new ViterbiForGlobalLinearModel(_weightVector, _tags); }
public void ComputeRange(int start, int end, WeightVector newWeightVector, int threadIndex = 0) { for (var k = start; k < end; k++) { if (k % 100 == 0) { Console.WriteLine(DateTime.Now + "threadIndex: " + threadIndex + " running iteration for k " + k); } var wk = Compute(k); if (double.IsNaN(wk) || double.IsInfinity(wk)) { Logger.WriteLine("k: "+ k + "wk is infiity of nana"+ wk); Logger.Flush(false); } newWeightVector.SetKey(k, wk); } }
public WeightVector RunIterations(WeightVector weightVector, int iterationCount) { for (int iter = 0; iter < iterationCount; iter++) { Console.WriteLine(DateTime.Now + " running iteration " + iter); var newWeightVector = new WeightVector(weightVector.FeatureKDictionary); SetForwardBackwordAlgo(weightVector); //for (var k = 0; k < weightVector.FeatureKDictionary.Count; k++) for (var k = weightVector.FeatureKDictionary.Count-1; k >= 0; k--) { if (k%100 == 0) { Console.WriteLine(DateTime.Now + " running iteration for k " + k); } var wk = Compute(k, weightVector); wk = weightVector.Get(k) + _lambda*wk; newWeightVector.SetKey(k, wk); } weightVector = newWeightVector; } _weightVector = weightVector; return weightVector; }
public void ReMappingFromWeightVector(WeightVector weightVector) { var newDictKtoF = new Dictionary<int, string>(); var newDictFtoK = new Dictionary<string, int>(); var weightDict = new Dictionary<int, double>(); int k = 0; var sortedDictionary = from pair in weightVector.WDictionary where Math.Abs(pair.Value) > 2 orderby Math.Abs(pair.Value) descending select pair; foreach (var weight in sortedDictionary) { var feature = DictKToFeatures[weight.Key]; newDictFtoK[feature] = k; newDictKtoF[k] = feature; weightDict[k] = weight.Value; k++; } weightVector.WDictionary = weightDict; DictFeaturesToK = weightVector.FeatureKDictionary = newDictFtoK; DictKToFeatures = newDictKtoF; }
public void ReMapFeatureToK(bool normalize = true) { MapFeatures.ReMappingFromWeightVector(_useAvg ? AvgWeightVector : WeightVector, normalize); if (_useAvg) { WeightVector = AvgWeightVector; } }
public double ComputeFunctionValue(WeightVector weightVector) { double outputDouble = 0; for (var lineIndex = 0; lineIndex < _inputSentence.Count; lineIndex++) { var outputTags = _outputTagsList[lineIndex]; double initOutputDouble = 0; for (var k = 0; k < weightVector.FeatureCount; k++) { initOutputDouble += GetAllFeatureKFromCacheWithWeights(outputTags, k, lineIndex, weightVector); } initOutputDouble -= Math.Log(forwardBackwordAlgos[lineIndex].Z); outputDouble += initOutputDouble; } Console.WriteLine(DateTime.Now+": new function value is: "+outputDouble); return -outputDouble; }
public void ComputeGradientMultiThread(WeightVector weightVector, double[] gradient, int threadCount) { if (threadCount > 1) { //var newWeightVector = weightVector.DeepCopy(); var doneEvents = new ManualResetEvent[threadCount]; var partition = weightVector.FeatureCount / threadCount; for (int threadIndex = 0; threadIndex < threadCount; threadIndex++) { var start = threadIndex*partition; var end = start + partition; end = end > weightVector.FeatureCount ? weightVector.FeatureCount : end; doneEvents[threadIndex] = new ManualResetEvent(false); var info = new ThreadInfoObject(this, start, end, weightVector, doneEvents[threadIndex], gradient); ThreadPool.QueueUserWorkItem(info.StartLBFGGradientComputing, threadIndex); } WaitHandle.WaitAll(doneEvents); } ComputeGradientValues(weightVector, gradient, 0, weightVector.FeatureCount); }
public void ComputeGradientValues(WeightVector weightVector, double[] gradient, int startIndex, int endIndex ) { for (var k = startIndex; k < endIndex; k++) { double outputDouble = 0; var kstring = "@#" + k.ToString(CultureInfo.InvariantCulture); for (var lineIndex = 0; lineIndex < _inputSentence.Count; lineIndex++) { var outputTags = _outputTagsList[lineIndex]; double initOutputDouble = 0; initOutputDouble += GetAllFeatureKFromCache(outputTags, k, lineIndex); initOutputDouble -= CalculateGradient(outputTags, k, lineIndex, kstring); outputDouble += initOutputDouble; } gradient[k] = outputDouble - (_lambda * weightVector.Get(k)); } }
public WeightedFeatureSum(WeightVector weightVector, List<string> sentence, bool crf = false) { _sentence = sentence; _crf = crf; WeightVector = weightVector; }
private void SetForwardBackwordAlgo(WeightVector weightVector) { if (_inputSentence.Count != _outputTagsList.Count) { throw new Exception("counts dont match " + _inputSentence.Count + "with " + _outputTagsList.Count); } int counter = 0; forwardBackwordAlgos.Clear(); foreach (var sentence in _inputSentence) { var outputTags = _outputTagsList[counter]; if (sentence.Count != outputTags.Count) { throw new Exception("counts dont match " + sentence.Count + "with " + outputTags.Count); } forwardBackwordAlgos.Add(new ForwardBackwordAlgo(sentence, weightVector, outputTags)); counter++; } }
public double GetAllFeatureKFromCacheWithWeights(List<string> tags, int k, int lineIndex, WeightVector weightVector) { double sum = 0; for (var pos = 0; pos < tags.Count; pos++) { var prevTag = "*"; if (pos > 0) { prevTag = tags[pos - 1]; } if (_cache.Contains(prevTag, tags[pos], k, pos, lineIndex)) { //var val = Math.Exp(_weightVector.Get(k)); sum += (_weightVector.Get(k)); } } return sum; }
private double Compute(int k, WeightVector weightVector) { double output = 0; //double secondTerm = 0; int lineIndex = 0; //var weightedFeaturesum = new WeightedFeatureSum(weightVector, null, true); if (_inputSentence.Count != _outputTagsList.Count) { throw new Exception("counts dont match "+ _inputSentence.Count + "with "+ _outputTagsList.Count); } var ngramTags = new Tags(_tagList); // first term. foreach (var sentence in _inputSentence) { var outputTags = _outputTagsList[lineIndex]; if (sentence.Count != outputTags.Count) { throw new Exception("compute counts dont match " + sentence.Count + "with " + outputTags.Count); } output += CalculateGradient(outputTags, k, ngramTags, lineIndex); //output += weightedFeaturesum.GetAllFeatureK(outputTags, k, sentence); //// second term. //for (var j = 0; j < outputTags.Count; j++) //{ // double sum = 0; // foreach (var ngramTag in ngramTags.GetNGramTags(2)) // { // string[] split = ngramTag.Split(new[] {':'}); // sum += (forwardBackwordAlgos[i].GetQ(j, split[0], split[1]) * // weightedFeaturesum.GetFeatureK(split[0], split[1], j, k, sentence)); // } // secondTerm += sum; //} lineIndex++; } output = output - (_lambda*weightVector.Get(k)); return output; }
public WeightVector RunIterations(WeightVector weightVector, int iterationCount, int threadCount = 1) { _weightVector = weightVector; for (var iter = 0; iter < iterationCount; iter++) { Console.WriteLine(DateTime.Now + " running iteration " + iter); var newWeightVector = _weightVector.DeepCopy(); SetForwardBackwordAlgo(newWeightVector); if (threadCount > 1) { var doneEvents = new ManualResetEvent[threadCount]; var partition = newWeightVector.FeatureCount / threadCount; for (int threadIndex = 0; threadIndex < threadCount; threadIndex++) { var start = threadIndex*partition; var end = start + partition; end = end > newWeightVector.FeatureCount ? newWeightVector.FeatureCount : end; doneEvents[threadIndex] = new ManualResetEvent(false); var info = new ThreadInfoObject(this, start, end, newWeightVector, doneEvents[threadIndex], null); ThreadPool.QueueUserWorkItem(info.StartGradientComputing, threadIndex); } WaitHandle.WaitAll(doneEvents); } else { ComputeRange(0, _weightVector.FeatureCount, newWeightVector); } _weightVector = newWeightVector; if (iter + 1 < iterationCount) { _weightVector.AvgNormalize(); } } _weightVector.AvgNormalize(); return _weightVector; }
public void AddWeightVector(WeightVector weightVector) { for (int i = 0; i < weightVector.WeightArray.Length; i++) { WeightArray[i] += weightVector.WeightArray[i]; } }
public void RunLBFGAlgo(WeightVector weightVector) { double epsg = 0.0000000001; double epsf = 0; double epsx = 0; int maxits = 50; alglib.minlbfgsstate state; alglib.minlbfgsreport rep; _weightVector = weightVector; this.FeatureKDictionary = weightVector.FeatureKDictionary; alglib.minlbfgscreate(5, weightVector.WeightArray, out state); alglib.minlbfgssetcond(state, epsg, epsf, epsx, maxits); alglib.minlbfgsoptimize(state, GetFunctionValueAndGradient, null, this); double[] output; alglib.minlbfgsresults(state, out output, out rep); weightVector.WeightArray = output; System.Console.WriteLine(DateTime.Now+": terminationtype {0}", rep.terminationtype); // EXPECTED: 4 //System.Console.WriteLine("{0}", alglib.ap.format(x, 2)); // EXPECTED: [-3,3] //System.Console.ReadLine(); }
private void SetForwardBackwordAlgo(WeightVector weightVector) { if (_inputSentence.Count != _outputTagsList.Count) { throw new Exception("counts dont match " + _inputSentence.Count + "with " + _outputTagsList.Count); } int counter = 0; forwardBackwordAlgos.Clear(); foreach (var sentence in _inputSentence) { if (counter % 100 == 0) Console.WriteLine(DateTime.Now + "running fw/backword iteration: "+counter); var algo = new ForwardBackwordAlgo(sentence, weightVector, _tagList); algo.Run(); forwardBackwordAlgos.Add(algo); counter++; } }