public void TestFilterCSVHeaders()
{
GenerateTestFileHeadings(true);
var norm = new SegregateCSV();
norm.Targets.Add(new SegregateTargetPercent(OutputName1, 75));
norm.Targets.Add(new SegregateTargetPercent(OutputName2, 25));
norm.Analyze(InputName, true, CSVFormat.English);
norm.Process();
var tr = new StreamReader(OutputName1.ToString());
Assert.AreEqual("\"a\",\"b\"", tr.ReadLine());
Assert.AreEqual("one,1", tr.ReadLine());
Assert.AreEqual("two,2", tr.ReadLine());
Assert.AreEqual("three,3", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
tr = new StreamReader(OutputName2.ToString());
Assert.AreEqual("\"a\",\"b\"", tr.ReadLine());
Assert.AreEqual("four,4", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
InputName.Delete();
OutputName1.Delete();
OutputName1.Delete();
}
private static void Segregate()
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(trainingFile, 50));
seg.Targets.Add(new SegregateTargetPercent(crossValidationFile, 30));
seg.Targets.Add(new SegregateTargetPercent(evaluationFile, 20));
seg.ProduceOutputHeaders = true;
seg.Analyze(shuffledFile, true, CSVFormat.English);
seg.Process();
}
/// <inheritdoc/>
public override sealed bool ExecuteCommand(String args)
{
// get filenames
String sourceID = Prop.GetPropertyString(
ScriptProperties.SegregateConfigSourceFile);
FileInfo sourceFile = Script.ResolveFilename(sourceID);
EncogLogging.Log(EncogLogging.LevelDebug, "Beginning segregate");
EncogLogging.Log(EncogLogging.LevelDebug, "source file:" + sourceID);
// get formats
CSVFormat format = Script.DetermineFormat();
// prepare to segregate
bool headers = Script.ExpectInputHeaders(sourceID);
var seg = new SegregateCSV {Script = Script};
Analyst.CurrentQuantTask = seg;
foreach (AnalystSegregateTarget target in Script.Segregate.SegregateTargets)
{
FileInfo filename = Script.ResolveFilename(target.File);
seg.Targets.Add(new SegregateTargetPercent(filename, target.Percent));
// mark generated
Script.MarkGenerated(target.File);
EncogLogging.Log(
EncogLogging.LevelDebug,
"target file:" + target.File + ", Percent: "
+ Format.FormatPercent(target.Percent));
}
seg.Report = new AnalystReportBridge(Analyst);
seg.Analyze(sourceFile, headers, format);
seg.Process();
Analyst.CurrentQuantTask = null;
return seg.ShouldStop();
}
/// <summary>
/// Metodo responsavel por segregar as informacoes do dataset em infos para Treinamento do Modelo e para Teste do Modelo
/// </summary>
/// <param name="source">FileInfo com o caminho do Dataset misturado</param>
private static void Segregate(FileInfo source)
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(Config.TrainingClassificationFile, 75));
seg.Targets.Add(new SegregateTargetPercent(Config.EvaluateClassificationFile, 25));
seg.ProduceOutputHeaders = true;
seg.Analyze(source, true, CSVFormat.English);
seg.Process();
}
public static void Segregate(FileOps fileOps)
{
var segregator = new SegregateCSV{ProduceOutputHeaders = true};
segregator.Targets.Add(new SegregateTargetPercent(fileOps.TrainingFile,75));
segregator.Targets.Add(new SegregateTargetPercent(fileOps.EvaluationFile, 25));
segregator.Analyze(fileOps.ShuffledFile,true,CSVFormat.English);
segregator.Process();
}