public void TestFilterCSVHeaders()
{
GenerateTestFileHeadings(true);
var norm = new SegregateCSV();
norm.Targets.Add(new SegregateTargetPercent(OutputName1, 75));
norm.Targets.Add(new SegregateTargetPercent(OutputName2, 25));
norm.Analyze(InputName, true, CSVFormat.English);
norm.Process();
var tr = new StreamReader(OutputName1.ToString());
Assert.AreEqual("\"a\",\"b\"", tr.ReadLine());
Assert.AreEqual("one,1", tr.ReadLine());
Assert.AreEqual("two,2", tr.ReadLine());
Assert.AreEqual("three,3", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
tr = new StreamReader(OutputName2.ToString());
Assert.AreEqual("\"a\",\"b\"", tr.ReadLine());
Assert.AreEqual("four,4", tr.ReadLine());
Assert.IsNull(tr.ReadLine());
tr.Close();
InputName.Delete();
OutputName1.Delete();
OutputName1.Delete();
}
private static void Segregate()
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(trainingFile, 50));
seg.Targets.Add(new SegregateTargetPercent(crossValidationFile, 30));
seg.Targets.Add(new SegregateTargetPercent(evaluationFile, 20));
seg.ProduceOutputHeaders = true;
seg.Analyze(shuffledFile, true, CSVFormat.English);
seg.Process();
}
/// <inheritdoc/>
public override sealed bool ExecuteCommand(String args)
{
// get filenames
String sourceID = Prop.GetPropertyString(
ScriptProperties.SegregateConfigSourceFile);
FileInfo sourceFile = Script.ResolveFilename(sourceID);
EncogLogging.Log(EncogLogging.LevelDebug, "Beginning segregate");
EncogLogging.Log(EncogLogging.LevelDebug, "source file:" + sourceID);
// get formats
CSVFormat inputFormat = Script
.DetermineInputFormat(sourceID);
CSVFormat outputFormat = Script.DetermineOutputFormat();
// prepare to segregate
bool headers = Script.ExpectInputHeaders(sourceID);
var seg = new SegregateCSV {Script = Script};
Analyst.CurrentQuantTask = seg;
foreach (AnalystSegregateTarget target in Script.Segregate.SegregateTargets)
{
FileInfo filename = Script.ResolveFilename(target.File);
seg.Targets.Add(new SegregateTargetPercent(filename, target.Percent));
// mark generated
Script.MarkGenerated(target.File);
EncogLogging.Log(
EncogLogging.LevelDebug,
"target file:" + target.File + ", Percent: "
+ Format.FormatPercent(target.Percent));
}
seg.Report = new AnalystReportBridge(Analyst);
seg.Analyze(sourceFile, headers, inputFormat);
seg.OutputFormat = outputFormat;
seg.Process();
Analyst.CurrentQuantTask = null;
return seg.ShouldStop();
}
/// <summary>
/// Metodo responsavel por segregar as informacoes do dataset em infos para Treinamento do Modelo e para Teste do Modelo
/// </summary>
/// <param name="source">FileInfo com o caminho do Dataset misturado</param>
private static void Segregate(FileInfo source)
{
var seg = new SegregateCSV();
seg.Targets.Add(new SegregateTargetPercent(Config.TrainingClassificationFile, 75));
seg.Targets.Add(new SegregateTargetPercent(Config.EvaluateClassificationFile, 25));
seg.ProduceOutputHeaders = true;
seg.Analyze(source, true, CSVFormat.English);
seg.Process();
}
public override sealed bool ExecuteCommand(string args)
{
FileInfo info;
CSVFormat format;
CSVFormat format2;
bool flag;
SegregateCSV ecsv;
AnalystSegregateTarget target;
FileInfo info2;
AnalystSegregateTarget[] segregateTargets;
int num;
string propertyString = base.Prop.GetPropertyString("SEGREGATE:CONFIG_sourceFile");
if (((uint) num) >= 0)
{
info = base.Script.ResolveFilename(propertyString);
goto Label_01C5;
}
Label_0026:
if (((uint) num) >= 0)
{
ecsv.Analyze(info, flag, format);
}
ecsv.OutputFormat = format2;
if (0 != 0)
{
goto Label_00ED;
}
ecsv.Process();
base.Analyst.CurrentQuantTask = null;
goto Label_021F;
Label_008C:
if (num < segregateTargets.Length)
{
goto Label_00E6;
}
if ((((uint) num) + ((uint) num)) <= uint.MaxValue)
{
goto Label_01AD;
}
if (15 != 0)
{
goto Label_00E1;
}
Label_00B6:
EncogLogging.Log(0, "target file:" + target.File + ", Percent: " + Format.FormatPercent((double) target.Percent));
num++;
goto Label_008C;
Label_00E1:
num = 0;
goto Label_008C;
Label_00E6:
target = segregateTargets[num];
Label_00ED:
info2 = base.Script.ResolveFilename(target.File);
ecsv.Targets.Add(new SegregateTargetPercent(info2, target.Percent));
base.Script.MarkGenerated(target.File);
goto Label_00B6;
Label_01AD:
if ((((uint) flag) - ((uint) num)) <= uint.MaxValue)
{
if (0 != 0)
{
goto Label_00E6;
}
ecsv.Report = new AnalystReportBridge(base.Analyst);
if (2 != 0)
{
goto Label_0026;
}
goto Label_021F;
}
Label_01C5:
EncogLogging.Log(0, "Beginning segregate");
EncogLogging.Log(0, "source file:" + propertyString);
format = base.Script.DetermineInputFormat(propertyString);
format2 = base.Script.DetermineOutputFormat();
flag = base.Script.ExpectInputHeaders(propertyString);
ecsv = new SegregateCSV {
Script = base.Script
};
if (((uint) flag) <= uint.MaxValue)
{
base.Analyst.CurrentQuantTask = ecsv;
if (((uint) flag) > uint.MaxValue)
{
goto Label_0026;
}
segregateTargets = base.Script.Segregate.SegregateTargets;
if (0 == 0)
{
goto Label_00E1;
}
}
goto Label_01AD;
Label_021F:
return ecsv.ShouldStop();
}
public static void Segregate(FileOps fileOps)
{
var segregator = new SegregateCSV{ProduceOutputHeaders = true};
segregator.Targets.Add(new SegregateTargetPercent(fileOps.TrainingFile,75));
segregator.Targets.Add(new SegregateTargetPercent(fileOps.EvaluationFile, 25));
segregator.Analyze(fileOps.ShuffledFile,true,CSVFormat.English);
segregator.Process();
}
