private Dictionary <string, bool> Solve(List <Clause> clauses, SolverType type)
{
Dictionary <string, bool> solution;
SolverStatus status;
if (type == SolverType.SAT)
{
solution = SolveSAT(clauses, out status);
}
else if (type == SolverType.mhs)
{
solution = SolveMHS(clauses, out status);
}
else if (type == SolverType.MaxSAT)
{
solution = SolveMaxSAT(clauses, out status);
}
else
{
throw new RepairException("Invalid solver type!");
}
ClauseLogger.Log(clauses, type, solution);
if (status == SolverStatus.Unsatisfiable)
{
throw new RepairException("The program could not be repaired because of unsatisfiable clauses!");
}
return(solution);
}
public void testParseCommandLine()
{
Train train = new Train();
foreach (SolverType solver in SolverType.values())
{
train.parse_command_line(new [] { "-B", "5.3", "-s", "" + solver.getId(), "-p", "0.01", "model-filename" });
Parameter param = train.Parameter;
Assert.AreEqual(solver, param.solverType);
// check default eps
if (solver.getId() == 0 || solver.getId() == 2 || //
solver.getId() == 5 || solver.getId() == 6)
{
Assert.AreEqual(0.01, param.eps);
}
else if (solver.getId() == 7)
{
Assert.AreEqual(0.1, param.eps);
}
else if (solver.getId() == 11)
{
Assert.AreEqual(0.001, param.eps);
}
else
{
Assert.AreEqual(0.1, param.eps);
}
// check if bias is set
Assert.AreEqual(5.3, train.Bias);
Assert.AreEqual(0.01, param.p);
}
}
private Dictionary <string, bool> Solve(List <RepairableError> errors, SolverType type)
{
try
{
List <Clause> clauses = GenerateClauses(errors);
return(Solve(clauses, type));
}
// the repair exception could be because of not considering all the barriers in the loop
// refer CUDA/transitiveclosure as an example
catch (RepairException)
{
bool overapproximated = false;
foreach (Error error in errors)
{
RaceError race = error as RaceError;
if (race != null && race.OverapproximatedBarriers.Any())
{
overapproximated = true;
race.Overapproximated = true;
}
}
// if none of the clauses were overapproximated, return the original result
if (!overapproximated)
{
throw;
}
List <Clause> clauses = GenerateClauses(errors);
return(Solve(clauses, type));
}
}
//public class CplexStatusCallback : CpxIncumbentCallbackFunction, IStatusCallback
//{
// /// <summary>
// /// Creates a new callback.
// /// </summary>
// /// <param name="solver">The active solver.</param>
// public CplexStatusCallback(SolverWrapper solver) { _solver = solver; }
// /// <summary>
// /// The active solver.
// /// </summary>
// private SolverWrapper _solver;
// /// <summary>
// /// Logs messages of the solver.
// /// </summary>
// public Action<string> Logger { get; set; }
// /// <summary>
// /// Notifies about new incumbents.
// /// </summary>
// public Action NewIncumbent { get; set; }
// /// <summary>
// /// Logs new incumbent values.
// /// </summary>
// public Action<double> LogIncumbent { get; set; }
// public int CallIt(IntPtr xenv, IntPtr cbdata, int wherefrom, object cbhandle, double objval, double[] x, ref int isfeas_p, ref int useraction)
// {
// return 0;
// }
//}
#endregion
public static void Test(SolverType type)
{
Console.WriteLine("Is64BitProcess: " + Environment.Is64BitProcess);
LinearModel wrapper = new LinearModel(type, (string s) => { Console.Write(s); });
Console.WriteLine("Setting up model and optimizing it with " + wrapper.Type);
string x = "x"; string y = "y"; string z = "z";
List <string> variableNames = new List <string>()
{
x, y, z
};
VariableCollection <string> variables = new VariableCollection <string>(wrapper, VariableType.Binary, 0, 1, (string s) => { return(s); });
wrapper.SetObjective(0.5 * variables[x] + variables[y] + 4 * variables[z], OptimizationSense.Maximize);
wrapper.AddConstr(LinearExpression.Sum(variableNames.Select(v => 2 * variables[v])) <= 4, "Summation");
wrapper.AddConstr(variables[x] + variables[y] + variables[z] <= 2.0, "limit");
wrapper.AddConstr(variables[x] == variables[z], "equal");
wrapper.AddConstr(variables[x] * 2 == 2, "times");
wrapper.Update();
wrapper.Optimize();
Console.WriteLine("Solution:");
if (wrapper.HasSolution())
{
Console.WriteLine("Obj: " + wrapper.GetObjectiveValue());
foreach (var variableName in variableNames)
{
Console.WriteLine(variableName + ": " + variables[variableName].GetValue());
}
}
else
{
Console.WriteLine("No solution!");
}
}
public SolverInfo(SolverType slvr = SolverType.QP, Objective opt = Objective.MinimizeRisk,
bool trace = false)
{
// Initialize with default values if none are passed as parameters
Solver = slvr;
OptimizationObjective = opt;
Trace = trace;
}
public MaltLiblinearModel(int[] labels, int nr_class, int nr_feature, double[][] w, SolverType solverType)
{
this.labels = labels;
this.nr_class = nr_class;
this.nr_feature = nr_feature;
this.w = w;
this.solverType = solverType;
}
/// <summary>Constructor</summary>
/// <param name="solverType">solver type</param>
/// <param name="variableNumber">number of variables</param>
public OrdinaryDifferentialEquations(SolverType solverType, uint variableNumber)
{
dym = new double[variableNumber];
dyt = new double[variableNumber];
yt = new double[variableNumber];
dysav = new double[variableNumber];
ysav = new double[variableNumber];
ytemp = new double[variableNumber];
}
public WorldCinfo()
{
Gravity = 9.8f;
GravityDirection = -Vector3.UnitY;
WorldSize = 150;
CollisionTolerance = 0.1f;
HavokSimulationType = SimulationType.SIMULATION_TYPE_DISCRETE;
HavokSolverType = SolverType.SOLVER_TYPE_4ITERS_MEDIUM;
FireCollisionCallbacks = false;
EnableDeactivation = true;
}
public static string ToRString(this SolverType solvr)
{
switch (solvr)
{
case SolverType.QP:
return("solveRquadprog");
default:
return("solveRquadprog");
}
}
public static Solver GetSolver(SolverType type, WarehouseInstance instance)
{
return(type switch
{
SolverType.CBS => new CBS(instance),
SolverType.PrioritizedPlanner => new PrioritizedPlanner(instance),
SolverType.PrioritizedPlannerClassesL => new PrioritizedPlanner(instance, PrioritizedPlanner.Heuristic.ClassesLow),
SolverType.PrioritizedPlannerClassesH => new PrioritizedPlanner(instance, PrioritizedPlanner.Heuristic.ClassesHigh),
SolverType.Heuristic => null,
_ => throw new NotImplementedException("Solver not implemented."),
});
/// <summary>
/// Factory method for solver creation
/// </summary>
public static BaseSolver Create(SolverType type)
{
switch (type)
{
case SolverType.DummySolver:
return(new DummySolver());
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
}
public void IntiFromInfo(CubicTTTInitInfo info)
{
xSize = info.xSize;
ySize = info.ySize;
zSize = info.zSize;
winNumber = info.winNumber;
solverType = info.solverType;
p1 = info.p1;
p2 = info.p2;
p1Piece = info.p1Piece;
p2Piece = info.p2Piece;
}
public ImplicitEulerScheme(FdmLinearOpComposite map,
List <BoundaryCondition <FdmLinearOp> > bcSet,
double relTol = 1e-8,
SolverType solverType = SolverType.BiCGstab)
{
dt_ = null;
iterations_ = 0;
relTol_ = relTol;
map_ = map;
bcSet_ = new BoundaryConditionSchemeHelper(bcSet);
solverType_ = solverType;
}
private void cb_SolverType_SelectedIndexChanged(object sender, EventArgs e)
{
if (this.cb_SolverType.SelectedIndex == 0)
{
solverType = SolverType.PressureBased;
this.lb_help.Text = "帮助：\r\n" + HelpDescriptions.PressureBase;
}
else
{
solverType = SolverType.DensityBased;
this.lb_help.Text = "帮助：\r\n" + HelpDescriptions.DensityBase;
}
}
public static IGraphicalMazeSolver GetSolver(SolverType algorithmType)
{
switch (algorithmType)
{
case SolverType.ShortestPath:
return(ShortestPathGraphicalMazeSolver.Create());
case SolverType.WallFollower:
return(WallFollowerGraphicalMazeSolver.Create());
default:
throw new NotImplementedException();
}
}
private SolverType GetSolverType(Norm norm, Loss loss, bool dual, Multiclass multiclass)
{
if (multiclass == Multiclass.CrammerSinger)
{
return(SolverType.getById(SolverType.MCSVM_CS));
}
if (multiclass != Multiclass.Ovr)
{
throw new ArgumentException("Invalid multiclass value");
}
if (norm == Norm.L2 && loss == Loss.LogisticRegression && !dual)
{
return(SolverType.getById(SolverType.L2R_LR));
}
if (norm == Norm.L2 && loss == Loss.L2 && dual)
{
return(SolverType.getById(SolverType.L2R_L2LOSS_SVC_DUAL));
}
if (norm == Norm.L2 && loss == Loss.L2 && !dual)
{
return(SolverType.getById(SolverType.L2R_L2LOSS_SVC));
}
if (norm == Norm.L2 && loss == Loss.L1 && dual)
{
return(SolverType.getById(SolverType.L2R_L1LOSS_SVC_DUAL));
}
if (norm == Norm.L1 && loss == Loss.L2 && !dual)
{
return(SolverType.getById(SolverType.L1R_L2LOSS_SVC));
}
if (norm == Norm.L1 && loss == Loss.LogisticRegression && !dual)
{
return(SolverType.getById(SolverType.L1R_LR));
}
if (norm == Norm.L2 && loss == Loss.LogisticRegression && dual)
{
return(SolverType.getById(SolverType.L2R_LR_DUAL));
}
throw new ArgumentException("Given combination of penalty, loss, dual params is not supported");
}
public void testLoadSaveModel()
{
Model model = null;
foreach (SolverType solverType in SolverType.values())
{
model = createRandomModel();
model.solverType = solverType;
var tempFile = Path.GetTempFileName();
Linear.saveModel(new FileInfo(tempFile), model);
Model loadedModel = Linear.loadModel(new FileInfo(tempFile));
Assert.AreEqual(model, loadedModel);
}
}
//bla bla bla
public static ODESolver MakeSolver(SolverType solver)
{
switch (solver)
{
case SolverType.FE:
return(new SolverFE());
case SolverType.Heun:
return(new SolverHeun());
case SolverType.RK4:
return(new SolverRK4());
}
return(null);
}
public static void SetParams(out string modulesConfig, out string inputRawFile, out string classLabelsOutputFileName, out string featuresDictOutputFile, out string modelOutputFileName, out string libSvmOutputFileName, out SolverType liblinearSolver, out double liblinearC, out double liblinearEps, out int minFeaturesFrequency, out bool normalize, out ScaleRange scaleRange)
{
modulesConfig = ConfigurationManager.AppSettings["PIPELINE_MODULES_CONFIG"] ?? "annotate_words,plain_bow,nsuff_3,chngram_3,word2gram,doc_end";
inputRawFile = ConfigurationManager.AppSettings["TRAIN_RAW_FILE"] ?? "data\troll-comments.txt";
classLabelsOutputFileName = ConfigurationManager.AppSettings["MODEL_CLASSLABELS_FILE"] != null ? ConfigurationManager.AppSettings["MODEL_CLASSLABELS_FILE"] : inputRawFile + ".classlabels";
featuresDictOutputFile = ConfigurationManager.AppSettings["MODEL_FEATURES_FILE"] != null ? ConfigurationManager.AppSettings["MODEL_FEATURES_FILE"] : inputRawFile + ".features";
modelOutputFileName = ConfigurationManager.AppSettings["MODEL_MODEL_FILE"] != null ? ConfigurationManager.AppSettings["MODEL_MODEL_FILE"] : inputRawFile + ".model";
libSvmOutputFileName = inputRawFile + ".libsvm";
liblinearSolver = SolverType.L1R_LR;
liblinearC = ConfigurationManager.AppSettings["LIBLINEAR_C"] != null ? double.Parse(ConfigurationManager.AppSettings["LIBLINEAR_C"]) : 1.0;
liblinearEps = ConfigurationManager.AppSettings["LIBLINEAR_EPS"] != null ? double.Parse(ConfigurationManager.AppSettings["LIBLINEAR_EPS"]) : 0.01;
minFeaturesFrequency = 5;
normalize = false;
scaleRange = ScaleRange.ZeroToOne;
}
public static Solver FromType(SolverType solverType)
{
switch (solverType)
{
case SolverType.Line:
return(new LineSolver());
case SolverType.Plane:
return(new PlaneSolver());
case SolverType.Cylinder:
return(new CylinderSolver());
}
throw (new ApplicationException(String.Format("Unknown solver type: '{0}'.", solverType)));
}
/// <summary>
/// Determine the barrier assignments based on the error traces.
/// </summary>
/// <param name="errors">The errors.</param>
/// <param name="type">The solver type used.</param>
/// <returns>The barrier assignments.</returns>
public Dictionary <string, bool> Solve(List <RepairableError> errors, ref SolverType type)
{
try
{
return(Solve(errors, type));
}
catch (RepairException)
{
// fall back to MaxSAT if MHS fails
if (type != SolverType.mhs)
{
throw;
}
type = SolverType.MaxSAT;
return(Solve(errors, type));
}
}
public void testCrossValidation()
{
int numClasses = random.Next(10) + 1;
Problem prob = createRandomProblem(numClasses);
Parameter param = new Parameter(SolverType.getById(SolverType.L2R_LR), 10, 0.01);
int nr_fold = 10;
double[] target = new double[prob.l];
Linear.crossValidation(prob, param, nr_fold, target);
foreach (double clazz in target)
{
Assert.IsTrue(clazz >= 0);
Assert.IsTrue(clazz <= numClasses);
}
}
public void testSetSolverType()
{
foreach (SolverType type in SolverType.values())
{
_param.setSolverType(type);
Assert.AreEqual(type, _param.getSolverType());
}
try {
_param.setSolverType(null);
Assert.Fail("ArgumentException expected");
}
catch (ArgumentException e)
{
Assert.IsTrue(e.Message.Contains("must"));
Assert.IsTrue(e.Message.Contains("not"));
Assert.IsTrue(e.Message.Contains("null"));
}
}
public LinearModel(SolverType type, Action <string> logger, int threadCount = 0)
{
Type = type;
_logger = logger;
switch (type)
{
case SolverType.CPLEX:
{
CplexModel = new Cplex();
_cplexStatusCallback = new CplexStatusCallback(this)
{
};
CplexModel.SetOut(new CplexOutputHandler(logger));
CplexModel.Use(_cplexStatusCallback);
if (threadCount > 0)
{
CplexModel.SetParam(Cplex.IntParam.Threads, threadCount);
}
}
break;
case SolverType.Gurobi:
{
GRBEnv gurobiEnvironment = new GRBEnv();
GurobiModel = new GRBModel(gurobiEnvironment);
GurobiModel.GetEnv().Set(GRB.IntParam.UpdateMode, 1); // Enable immediate updates to better support the lazy initialization of the wrappers (at minor performance and memory costs)
GurobiModel.GetEnv().Set(GRB.IntParam.OutputFlag, 0);
if (threadCount > 0)
{
GurobiModel.GetEnv().Set(GRB.IntParam.Threads, threadCount);
}
_gurobiStatusCallback = new GurobiStatusCallback(this)
{
Logger = logger
};
GurobiModel.SetCallback(_gurobiStatusCallback);
}
break;
default: throw new ArgumentException("Unknown solver type: " + Type);
}
}
private string SolverTypeToStr(SolverType s)
{
string solverS = "";
switch (s)
{
case SolverType.CompleteSolver:
solverS = "Local Complete Solver";
break;
case SolverType.RandomSolver:
solverS = "Random Solver";
break;
case SolverType.RemoteSolver331:
solverS = "Remote Solver";
break;
}
return(solverS);
}
/// <summary>
/// Creates a new LP from a given file.
/// </summary>
/// <param name="file">The file to parse the model from.</param>
/// <param name="logger">The logger to use.</param>
/// <param name="solverType">The solver to use.</param>
/// <param name="solverArgs">The args to pass to the solver.</param>
/// <param name="roundToDecimals">The number of decimals to round all read data to (a negative value deactivates rounding).</param>
public MDPLP(string file, Action <string> logger, SolverType solverType, string[] solverArgs, int roundToDecimals = -1)
{
// Init
_solverType = solverType;
_solverArgs =
solverArgs == null ?
_solverArgs = new KeyValuePair <string, string> [0] :
solverArgs.Select(a => { string[] arg = a.Split('='); return(new KeyValuePair <string, string>(arg[0], arg[1])); }).ToArray();
Log = (string msg) => { logger?.Invoke(msg); _logBuffer.Append(msg); };
LogLine = (string msg) => { logger?.Invoke(msg + Environment.NewLine); _logBuffer.AppendLine(msg); };
// Read file
Filename = Path.GetFileName(file);
LogLine("Creating instance from file " + file + " ...");
Read(file, roundToDecimals);
// Output some model statistics
LogLine("Model statistics:");
LogLine("m: " + M);
LogLine("n: " + N);
LogLine("NNZs: " + NNZs);
}
public void SetSolverType(SolverType type)
{
switch (type)
{
case SolverType.NonLinearConjugateGradient:
Solver = new NonLinearConjugateGradient(SolverParameters);
solverType = SolverType.NonLinearConjugateGradient;
break;
case SolverType.RedBlackProjectedGaussSeidel:
Solver = new RedBlackProjectedGaussSeidel(SolverParameters);
solverType = SolverType.RedBlackProjectedGaussSeidel;
break;
case SolverType.FischerNewton:
Solver = new FischerNewton(SolverParameters);
solverType = SolverType.FischerNewton;
break;
case SolverType.Lemke:
Solver = new Lemke(SolverParameters);
solverType = SolverType.Lemke;
break;
case SolverType.ProjectedSymmetricGS:
Solver = new ProjectedSymmetricGS(SolverParameters, LinearSystemBuilder);
solverType = SolverType.ProjectedSymmetricGS;
break;
case SolverType.NonLinearGaussSeidel:
Solver = new NonLinearGaussSeidel(SolverParameters, LinearSystemBuilder, IntegrateVelocityEngine);
solverType = SolverType.NonLinearGaussSeidel;
break;
case SolverType.ProjectedGaussSeidel:
default:
Solver = new ProjectedGaussSeidel(SolverParameters);
solverType = SolverType.ProjectedGaussSeidel;
break;
}
}
private void MeshFlieForm_Load(object sender, EventArgs e)
{
cb_SolverType.Items.AddRange(new string[] { "基于压力法(压力基)", "基于密度法(密度基)" });
cb_VelocityFormulation.Items.AddRange(new string[] { "绝对速度", "相对速度" });
cb_TimeType.Items.AddRange(new string[] { "稳态", "瞬态" });
cb_GravityDirection.Items.AddRange(new string[] { "X", "Y", "Z" });
cb_SolverType.SelectedIndex = 0;
cb_VelocityFormulation.SelectedIndex = 0;
cb_TimeType.SelectedIndex = 0;
cb_GravityDirection.SelectedIndex = 2;
solverType = SolverType.PressureBased;
velocityFormulation = VelocityFormulation.Absolute;
timeType = TimeType.Steady;
isGravity = false;
gravityDirection = GravityDirection.Z;
this.lb_help.Text = "帮助：\r\n";
}
private void comboBox1_SelectedIndexChanged(object sender, EventArgs e)
{
switch (comboBox1.SelectedIndex)
{
case 0:
type = SolverType.Line;
break;
case 1:
type = SolverType.Plane;
break;
case 2:
type = SolverType.Cylinder;
break;
default:
break;
}
solver = Solver.FromType(type);
}
/// <summary>
/// Determine the optimized barrier assignments based on the error traces.
/// </summary>
/// <param name="errors">The errors.</param>
/// <param name="solution">The solution obtained so far.</param>
/// <param name="type">The solver type used.</param>
/// <returns>The barrier assignments.</returns>
public Dictionary <string, bool> Optimize(
List <RepairableError> errors, Dictionary <string, bool> solution, out SolverType type)
{
List <Clause> clauses = GenerateClauses(errors);
Dictionary <string, int> coeffs = new Dictionary <string, int>();
foreach (Barrier barrier in ProgramMetadata.Barriers.Values)
{
coeffs.Add(barrier.Name, barrier.Weight);
}
type = SolverType.Optimizer;
while (true)
{
int total_weight = 0;
foreach (string key in solution.Where(x => x.Value == true).Select(x => x.Key))
{
total_weight += coeffs[key];
}
using (Watch watch = new Watch(Measure.Optimization))
{
SolverStatus status;
Optimizer optimizer = new Optimizer(clauses, coeffs);
Dictionary <string, bool> assignments = optimizer.Solve(--total_weight, out status);
if (status == SolverStatus.Unsatisfiable)
{
ClauseLogger.Log(clauses, type, solution);
return(solution);
}
else
{
solution = assignments;
}
}
}
}
internal static void EinsteinCsp(
ISelectVariableHeuristic <EinsteinValue, House> selectVariable,
IOrderDomainHeuristic <EinsteinValue, House> orderDomain,
SolverType solverType, bool showResult = false)
{
var einstein = new EinsteinCsp();
var solver = solverType switch
{
SolverType.Ac3 => new ArcConsistencySolver <EinsteinValue, House>(selectVariable, orderDomain),
SolverType.Backtracking => new BacktrackingSolver <EinsteinValue, House>(selectVariable, orderDomain),
_ => (CspSolver <EinsteinValue, House>) new ForwardCheckingSolver <EinsteinValue, House>(selectVariable,
orderDomain)
};
var benchResult = BenchmarkCsp.BenchmarkFirstOnly(solver, einstein);
if (showResult)
{
foreach (var house in HouseExtension.Houses)
{
Console.WriteLine("_____________________");
Console.WriteLine(house);
Console.WriteLine("_____________________");
foreach (var einsteinValue in benchResult.FirstSolution
.Where(r => r.Value == house)
.Select(r => r.Key))
{
Console.WriteLine(einsteinValue);
}
Console.WriteLine("\n\n");
}
}
Console.WriteLine($"__________________________\n{solverType}");
benchResult.Report();
benchResult = BenchmarkCsp.BenchmarkAll(solver, einstein);
benchResult.Report();
}
}
public static Model createRandomModel()
{
Model model = new Model();
model.solverType = SolverType.getById(SolverType.L2R_LR);
model.bias = 2;
model.label = new int[] { 1, int.MaxValue, 2 };
model.w = new double[model.label.Length * 300];
for (int i = 0; i < model.w.Length; i++)
{
// precision should be at least 1e-4
model.w[i] = Math.Round(random.NextDouble() * 100000.0) / 10000.0;
}
// force at least one value to be zero
model.w[random.Next(model.w.Length)] = 0.0;
model.w[random.Next(model.w.Length)] = -0.0;
model.nr_feature = model.w.Length / model.label.Length - 1;
model.nr_class = model.label.Length;
return(model);
}
public WorldCinfo()
{
Gravity = 9.8f;
GravityDirection = -Vector3.UnitY;
WorldSize = 150;
CollisionTolerance = 0.1f;
HavokSimulationType = SimulationType.SIMULATION_TYPE_DISCRETE;
HavokSolverType = SolverType.SOLVER_TYPE_4ITERS_MEDIUM;
FireCollisionCallbacks = false;
EnableDeactivation = true;
}
public static void Train(string modulesConfig, string inputRawFile, SolverType liblinearSolver, double liblinearC, double liblinearEps,
string classLabelsOutputFileName, string featuresDictOutputFile, string modelOutputFileName, string libSvmOutputFileName,
int minFeaturesFrequency,bool normalize, ScaleRange scaleRange)
{
//Define feature extraction modules
#region 1 - Define all possible feature extraction modules
List<FeatureExtractionModule> modules = PipelineConfiguration.GetExtractionModules();
//Print possible module options
foreach (var module in modules)
{
Debug.Write(module.Name + ",");
}
#endregion
//configure which modules to use
#region 2 - Configure which module configurations
//string modulesConfig = "annotate_words,plain_bow,npref_2,npref_3,npref_4,nsuff_2,nsuff_3,nsuff_4,chngram_2,chngram_3,chngram_4,plain_word_stems,word2gram,word3gram, word4gram,count_punct,emoticons_dnevnikbg,doc_start,doc_end";
//string settingsConfig = "annotate_words,plain_word_stems, npref_4, nsuff_3";
FeatureExtractionPipeline pipeline = PipelineConfiguration.BuildPipeline(modulesConfig, modules);
#endregion
Console.WriteLine("Input file:{0}", inputRawFile);
//char fieldSeparator = '\t';
#region 3 - Build features dictionary - process documents and extract all possible features
//build features dictionary
var featureStatisticsDictBuilder = new FeatureStatisticsDictionaryBuilder();
Console.WriteLine("Building a features dictionary...");
var timeStart = DateTime.Now;
int itemsCnt = 0;
Dictionary<string, int> classLabels = new Dictionary<string, int>();
int maxClassLabelIndex = 0;
using (var filereader = new LabeledTextDocumentFileReader(inputRawFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
//build class labels dictionary
if (!classLabels.ContainsKey(classLabel))
{
classLabels[classLabel] = maxClassLabelIndex;
maxClassLabelIndex++;
}
Dictionary<string, double> docFeatures = new Dictionary<string, double>();
pipeline.ProcessDocument(docContent, docFeatures);
featureStatisticsDictBuilder.UpdateInfoStatisticsFromSingleDoc(docFeatures);
itemsCnt++;
if (itemsCnt % 500 == 0)
{
Console.WriteLine("{0} processed so far", itemsCnt);
}
}
}
//order classes by name - until now they are ordered by first occurence in dataset
var ordered = classLabels.OrderBy(cv => cv.Key);
var orderedClassLabels = new Dictionary<string, int>();
int classIndexCounter = 0;
foreach (var item in ordered)
{
orderedClassLabels.Add(item.Key, classIndexCounter);
classIndexCounter++;
}
classLabels = orderedClassLabels;
LexiconReaderHelper.SaveDictionaryToFile(classLabels, classLabelsOutputFileName);
Console.WriteLine("Class labels saved to file {0}", classLabelsOutputFileName);
Console.WriteLine("Extracted {0} features from {1} documents", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, itemsCnt);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
RecomputeFeatureIndexes(featureStatisticsDictBuilder, minFeaturesFrequency);
Console.WriteLine("Selected {0} features with min freq {1} ", featureStatisticsDictBuilder.FeatureInfoStatistics.Count, minFeaturesFrequency);
//save fetures for later use
if (System.IO.File.Exists(featuresDictOutputFile))
{
System.IO.File.Delete(featuresDictOutputFile);
}
featureStatisticsDictBuilder.SaveToFile(featuresDictOutputFile);
Console.WriteLine("Features saved to file {0}", featuresDictOutputFile);
#endregion
//4- Load features from file
featureStatisticsDictBuilder.LoadFromFile(featuresDictOutputFile);
classLabels = LexiconReaderHelper.LoadDictionaryFromFile(classLabelsOutputFileName);
#region 5 - Build items with features from text documents and features dictionary
//Build libsvm file from text insput file and features dictionary
var sparseItemsWithIndexFeatures = new List<SparseItemInt>();
timeStart = DateTime.Now;
Console.WriteLine("Exporting to libsvm file format...");
using (System.IO.TextWriter textWriter = new System.IO.StreamWriter(libSvmOutputFileName, false))
{
LibSvmFileBuilder libSvmFileBuilder = new LibSvmFileBuilder(textWriter);
using (var filereader = new LabeledTextDocumentFileReader(inputRawFile))
{
while (!filereader.EndOfSource())
{
var doc = filereader.ReadDocument();
//class label and doc contents
string classLabel = doc.ClassLabel;
string docContent = doc.DocContent;
int classLabelIndex = classLabels[classLabel];
SparseItemInt sparseItem = ProcessingHelpers.ProcessTextAndGetSparseItem(pipeline, featureStatisticsDictBuilder, minFeaturesFrequency, normalize, scaleRange, docContent, classLabelIndex);
libSvmFileBuilder.AppendItem(sparseItem.Label, sparseItem.Features);
sparseItemsWithIndexFeatures.Add(sparseItem);
//B - Or extract indexed features and append
//libSvmFileBuilder.PreprocessStringFeaturesAndAppendItem(classLabelIndex, docFeatures, featureStatisticsDictBuilder.FeatureInfoStatistics, minFeaturesFrequency);
}
}
}
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
Console.WriteLine("Libsvm file saved to {0}", libSvmOutputFileName);
Console.WriteLine();
#endregion
#region 6 - Train and test classifier
//LIBLINEAR - TRAIN AND EVAL CLASSIFIER
//Build problem X and Y
//Split data on train and test dataset
int trainRate = 4;
int testRate = 1;
int allItemsCnt = sparseItemsWithIndexFeatures.Count;
int trainCnt = (int)((double)allItemsCnt * trainRate / (double)(trainRate + testRate));
int testCnt = allItemsCnt - trainCnt;
var trainItems = sparseItemsWithIndexFeatures.Take(trainCnt).ToList();
var testItems = sparseItemsWithIndexFeatures.Skip(trainCnt).Take(testCnt).ToList();
string trainDataModelFileName = inputRawFile + ".train.model";
FeatureNode[][] problemXTrain = null;
double[] problemYTrain = null;
SetLibLinearProblemXandYFromSparseItems(trainItems, out problemXTrain, out problemYTrain);
TrainLibLinearProblemAndSveModel(liblinearSolver, liblinearC, liblinearEps, featureStatisticsDictBuilder, trainDataModelFileName, problemXTrain, problemYTrain);
var modelFileLoad = new java.io.File(trainDataModelFileName);
var modelLoaded = Model.load(modelFileLoad);
//evaluation
List<FeatureNode[]> problemXEvalList = new List<FeatureNode[]>();
List<double> problemYEvalList = new List<double>();
//SetLibLinearProblemXandYFromSparseItems(testItems, out problemXEval, out problemYEval);
//EVALUATE
List<double> predictedY = new List<double>();
foreach (var item in testItems)
{
var itemFeatureNodes = ProcessingHelpers.ConvertToSortedFeatureNodeArray(item.Features);
//fill eval list
problemXEvalList.Add(itemFeatureNodes);
problemYEvalList.Add((double)item.Label);
//predict
double prediction = Linear.predict(modelLoaded, itemFeatureNodes);
predictedY.Add(prediction);
}
int[][] matrix = ResultCalculationMetricsHelpers.BuildConfusionMatrix(problemYEvalList.ToArray(), predictedY, classLabels.Count);
Console.WriteLine("Class labels:");
foreach (var label in classLabels)
{
Console.WriteLine(string.Format("{1} - {0}", label.Key, label.Value));
}
Console.WriteLine();
ResultCalculationMetricsHelpers.PrintMatrix(matrix, true);
for (int i = 0; i < matrix.Length; i++)
{
int truePositivesCnt = matrix[i][i];
int falsePositievesCnt = matrix[i].Sum() - matrix[i][i];
int falseNegativesCnt = matrix.Select(m => m[i]).Sum() - matrix[i][i];
double precision;
double recall;
double fScore;
ResultCalculationMetricsHelpers.CalculatePRF(truePositivesCnt, falsePositievesCnt, falseNegativesCnt, out precision, out recall, out fScore);
Console.WriteLine(string.Format("[{0} - {4}] P={1:0.0000}, R={2:0.0000}, F={3:0.0000} ", i, precision, recall, fScore, orderedClassLabels.ToList().ToDictionary(kv => kv.Value, kv => kv.Key)[i]));
}
int truePositivesCntOverall = 0;
int testedCnt = 0;
for (int i = 0; i < matrix.Length; i++)
{
truePositivesCntOverall += matrix[i][i];
testedCnt += matrix[i].Sum();
}
double accuracyOverall = (double)truePositivesCntOverall / (double)testedCnt;
Console.WriteLine(string.Format("[{0}] Accuracy={1:0.0000}", "Overall", accuracyOverall));
//----TRAIN MODEL IWTH ALL DATA
Console.WriteLine("------------------------------------------------");
Console.WriteLine("Train model on all data");
FeatureNode[][] problemXAll = null;
double[] problemYAll = null;
var allItems = sparseItemsWithIndexFeatures;
SetLibLinearProblemXandYFromSparseItems(allItems, out problemXAll, out problemYAll);
TrainLibLinearProblemAndSveModel(liblinearSolver, liblinearC, liblinearEps, featureStatisticsDictBuilder, modelOutputFileName, problemXAll, problemYAll);
//CROSSVALIDATION
//int crossValidationFold = 5;
//Console.WriteLine("Training with {0} items and 5 fold crossvalidation...", sparseItemsWithIndexFeatures.Count);
//timeStart = DateTime.Now;
//double[] target = new double[problem.l];
//Linear.crossValidation(problem, parameter, crossValidationFold, target);
//Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
//WriteResult(target);
#endregion
//Console.ReadKey();
//var instancesToTest = new Feature[] { new FeatureNode(1, 0.0), new FeatureNode(2, 1.0) };
//var prediction = Linear.predict(model, instancesToTest);
}
private static void TrainLibLinearProblemAndSveModel(SolverType liblinearSolver, double liblinearC, double liblinearEps, FeatureStatisticsDictionaryBuilder featureStatisticsDictBuilder, string trainDataModelFileName, FeatureNode[][] problemXTrain, double[] problemYTrain)
{
//Create liblinear problem
var problem = new Problem();
problem.l = problemXTrain.Length;
problem.n = featureStatisticsDictBuilder.FeatureInfoStatistics.Count;
problem.x = problemXTrain;
problem.y = problemYTrain;
Console.WriteLine("Training a classifier with {0} items...", problemXTrain.Length);
DateTime timeStart = DateTime.Now;
var parameter = new Parameter(liblinearSolver, liblinearC, liblinearEps);
Model model = Linear.train(problem, parameter);
Console.WriteLine("Done - {0}", (DateTime.Now - timeStart));
var modelFile = new java.io.File(trainDataModelFileName);
model.save(modelFile);
Console.WriteLine("Train data model saved to {0}", trainDataModelFileName);
}
public void SetSolver(SolverType type)
{
switch (type)
{
case SolverType.ProjectedGaussSeidel:
solver = new ProjectedGaussSeidel(SolverParam);
break;
case SolverType.NonLinearConjugateGradient:
solver = new NonLinearConjugateGradient(SolverParam);
break;
case SolverType.ConjugateGradient:
solver = new ConjugateGradient(SolverParam);
break;
default:
solver = new ProjectedGaussSeidel(SolverParam);
break;
}
}
