public void TestFilterCSVNoHeaders()
{
GenerateTestFileHeadings(false);
var norm = new SegregateCSV();
norm.Targets.Add(new SegregateTargetPercent(OutputName1, 75));
norm.Targets.Add(new SegregateTargetPercent(OutputName2, 25));
norm.Analyze(InputName, false, CSVFormat.English);
norm.ProduceOutputHeaders = false;
norm.Process();
var tr = new StreamReader(OutputName1.ToString());
Assert.AreEqual("one,1", tr.ReadLine());
Assert.AreEqual("two,2", tr.ReadLine());
Assert.AreEqual("three,3", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
tr = new StreamReader(OutputName2.ToString());
Assert.AreEqual("four,4", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
InputName.Delete();
OutputName1.Delete();
OutputName2.Delete();
}
public void Segregate(FileInfo dataToSegregate, FileInfo trainingFile, FileInfo evaluateFile)
{
var segregator = new SegregateCSV();
segregator.Targets.Add(new SegregateTargetPercent(trainingFile, this.dataPercentForTraining));
segregator.Targets.Add(new SegregateTargetPercent(evaluateFile, this.dataPercentForEvaluate));
segregator.ProduceOutputHeaders = true;
segregator.Analyze(dataToSegregate, true, CSVFormat.English);
segregator.Process();
}
private static void Segregate(FileInfo source)
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(Config.TrainingFile, 75));
seg.Targets.Add(new SegregateTargetPercent(Config.EvaluateFile, 25));
seg.ProduceOutputHeaders = true;
seg.Analyze(source, true, CSVFormat.English);
seg.Process();
}
private void Segregate(FileInfo source, FileInfo TrainingFile, FileInfo EvaluateWasteFile)
{
//Segregate source file into training and evaluation file
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(TrainingFile, 100));
seg.Targets.Add(new SegregateTargetPercent(EvaluateWasteFile, 0));
seg.ProduceOutputHeaders = true;
seg.Analyze(source, true, CSVFormat.English);
seg.Process();
}
private void Segregate(FileInfo source)
{
//Segregate source file into training,validation and evaluation file
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(Config.TrainingFile, 60));
seg.Targets.Add(new SegregateTargetPercent(Config.CrossValidationFile, 20));
seg.Targets.Add(new SegregateTargetPercent(Config.EvaluateFile, 20));
seg.ProduceOutputHeaders = true;
seg.Analyze(source, true, CSVFormat.English);
seg.Process();
}
/// <see cref="INetwork.Segregate"/>
public INetwork Segregate()
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(TrainingFile, 95));
seg.Targets.Add(new SegregateTargetPercent(EvaluateFile, 5));
seg.ProduceOutputHeaders = true;
seg.Analyze(BaseFile, true, CSVFormat.English);
seg.Process();
return(this);
}
/// <summary>
/// Segregate source file into training and evaluation file
/// </summary>
/// <param name="shuffledSource"></param>
/// <param name="trainingFile"></param>
/// <param name="testingFile"></param>
/// <param name="trainingPercentage"></param>
/// <param name="testingPercentage"></param>
public void SegregateToTrainAndEvalSets(FileInfo shuffledSource, FileInfo trainingFile, FileInfo testingFile, int trainingPercentage, int testingPercentage)
{
try
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(trainingFile, trainingPercentage));
seg.Targets.Add(new SegregateTargetPercent(testingFile, testingPercentage));
seg.ProduceOutputHeaders = true;
seg.Analyze(shuffledSource, true, CSVFormat.English);
seg.Process();
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
/// <inheritdoc/>
public override sealed bool ExecuteCommand(String args)
{
// get filenames
String sourceID = Prop.GetPropertyString(
ScriptProperties.SegregateConfigSourceFile);
FileInfo sourceFile = Script.ResolveFilename(sourceID);
EncogLogging.Log(EncogLogging.LevelDebug, "Beginning segregate");
EncogLogging.Log(EncogLogging.LevelDebug, "source file:" + sourceID);
// get formats
CSVFormat inputFormat = Script
.DetermineInputFormat(sourceID);
CSVFormat outputFormat = Script.DetermineOutputFormat();
// prepare to segregate
bool headers = Script.ExpectInputHeaders(sourceID);
var seg = new SegregateCSV {
Script = Script
};
Analyst.CurrentQuantTask = seg;
foreach (AnalystSegregateTarget target in Script.Segregate.SegregateTargets)
{
FileInfo filename = Script.ResolveFilename(target.File);
seg.Targets.Add(new SegregateTargetPercent(filename, target.Percent));
// mark generated
Script.MarkGenerated(target.File);
EncogLogging.Log(
EncogLogging.LevelDebug,
"target file:" + target.File + ", Percent: "
+ Format.FormatPercent(target.Percent));
}
seg.Report = new AnalystReportBridge(Analyst);
seg.Analyze(sourceFile, headers, inputFormat);
seg.OutputFormat = outputFormat;
seg.Process();
Analyst.CurrentQuantTask = null;
return(seg.ShouldStop());
}