static void TrainingTest(List<string> tags)
{
//const string modelFile = "../../data/gene.key.model";
//const string input = "../../data/gene.key";
const string modelFile = "../../data/training/tag.model.trial1";
const string input = "../../data/training/NYT_19980403_parsed.key";
string LoggerFile = "../../Logs/Log_"+DateTime.Now.ToFileTime()+".txt";
const int threadCount = 1;
var perceptron = new Perceptron(input, modelFile, tags);
perceptron.Train();
perceptron.ReMapFeatureToK();
//perceptron.Dump();
perceptron.MapFeatures.Dump();
perceptron.ReadInputs();
var featureCache = new FeatureCache(perceptron.InputSentences, tags,
perceptron.MapFeatures.DictFeaturesToK);
featureCache.CreateCache();
var logger = new WriteModel(LoggerFile);
var gradient = new ComputeGradient(perceptron.InputSentences, perceptron.TagsList,
tags, .1, featureCache, logger);
//perceptron.WeightVector.ResetAllToZero();
gradient.RunIterations(perceptron.WeightVector, 10, threadCount);
gradient.Dump(modelFile, perceptron.MapFeatures.DictKToFeatures);
}
static void TrainingTest(List<string> tags)
{
//const string modelFile = "../../data/gene.key.model";
//const string input = "../../data/gene.key";
var inputFiles = new[]
{
"../../data/training/NYT_19980403_parsed.key",
"../../data/training/APW_19980314_parsed.key",
"../../data/training/APW_19980424_parsed.key",
"../../data/training/APW_19980429_parsed.key",
"../../data/training/NYT_19980315_parsed.key",
"../../data/training/NYT_19980407_parsed.key",
"../../data/travelTraining/InputToCRF1.key",
"../../data/travelTraining/InputToCRF2.key",
"../../data/travelTraining/InputToCRF3.key",
"../../data/travelTraining/InputToCRF4.key",
"../../data/travelTraining/InputToCRF5.key",
"../../data/travelTraining/InputToCRF6.key",
"../../data/travelTraining/InputToCRF7.key",
"../../data/travelTraining/InputToCRF8.key",
"../../data/travelTraining/InputToCRF9.key",
"../../data/travelTraining/InputToCRF10.key",
//"../../data/travelTraining/InputToCRF11.key",
//"../../data/travelTraining/InputToCRF12.key",
};
const string modelFile = "../../data/tag.model.withoutsecondPassImpFor407";
const string input = "../../data/training/NYT_19980403_parsed.key";
string LoggerFile = "../../Logs/Log_"+DateTime.Now.ToFileTime()+".txt";
const int threadCount = 8;
var perceptron = new Perceptron(inputFiles.ToList(), modelFile, tags, false);
perceptron.Train(inputFiles);
perceptron.ReMapFeatureToK(true);
perceptron.Dump();
perceptron.MapFeatures.Dump();
//ComputeGradient gradient = null;
//var logger = new WriteModel(LoggerFile);
//for (int i = 0; i < inputFiles.Length; i++)
//{
// Console.WriteLine("running for input["+i+"]: "+ input[i]);
// perceptron.ReadInputs(inputFiles[i]);
// var featureCache = new FeatureCache(perceptron.InputSentences, tags,
// perceptron.MapFeatures.DictFeaturesToK);
// featureCache.CreateCache();
// const double lambda = 0;
// const double learningParam = .1;
// gradient = new ComputeGradient(perceptron.InputSentences, perceptron.TagsList,
// tags, lambda, learningParam, featureCache, logger);
// //perceptron.WeightVector.ResetAllToZero();
// //gradient.RunIterations(perceptron.WeightVector, 10, threadCount);
// gradient.RunLBFGAlgo(perceptron.WeightVector);
//}
//if (gradient != null)
//{
// gradient.Dump(modelFile, perceptron.MapFeatures.DictKToFeatures);
//}
}
static void TrainingTest(List<string> tags)
{
//const string modelFile = "../../data/gene.key.model";
//const string input = "../../data/gene.key";
const string modelFile = "../../data/training/tag.model.trial1";
const string input = "../../data/training/NYT_19980403_parsed.key";
var perceptron = new Perceptron(input, modelFile, tags);
perceptron.Train();
perceptron.ReMapFeatureToK();
perceptron.Dump();
perceptron.MapFeatures.Dump();
perceptron.ReadInputs();
var featureCache = new FeatureCache(perceptron.InputSentences, tags,
perceptron.MapFeatures.DictFeaturesToK);
featureCache.CreateCache();
var gradient = new ComputeGradient(perceptron.InputSentences, perceptron.TagsList,
tags, .1, featureCache);
gradient.RunIterations(perceptron.WeightVector, 10);
gradient.Dump(modelFile, perceptron.MapFeatures.DictKToFeatures);
}