public static List<Sample> Resampling (Segment segment)
{
List<Sample> result = new List<Sample>();
Matrix[] Evaluation_Matrix = new Matrix[10];
Matrix[] Training_Segment_Matrix = new Matrix[10];
Matrix[] Training_Root_Matrix = new Matrix[10];
for (int i = 0; i < 10; i++)
{
Evaluation_Matrix[i] = new Matrix(segment.data.rows, segment.data.cols);
Training_Segment_Matrix[i] = segment.data.Duplicate();
Training_Root_Matrix[i] = segment.data.Duplicate();
}
int count_segment = 0;
int need = segment.data.CountCells() / 10;
Random rand = new Random();
for (int i = 0; i < segment.data.rows; i++)
{
for (int j = 0; j < segment.data.cols; j++)
{
if (segment.data[i, j] != 0)
{
bool isUsed = false;
while (!isUsed)
{
int k = rand.Next(0, 10);
if (Evaluation_Matrix[k].CountCells() < need + 1)
{
isUsed = true;
Evaluation_Matrix[k][i, j] = segment.data[i, j];
Training_Segment_Matrix[k][i, j] = 0;
Training_Root_Matrix[k][i, j] = 0;
}
}
}
}
}
for (int i = 0; i < 10; i++)
{
Segment Evaluation_Segment = new Segment(segment, Evaluation_Matrix[i]);
Segment Training_Segment = new Segment(segment, Training_Segment_Matrix[i]);
Sample sample = new Sample(Evaluation_Segment, Training_Segment);
sample.Training_root = new Segment(segment, Training_Root_Matrix[i]);
result.Add(sample);
}
return result;
}
private void btnRun_Click(object sender, EventArgs e)
{
if (txtInput.Text == "")
{
MessageBox.Show("Choose a file!");
return;
}
Stopwatch swglobal = new Stopwatch();
Stopwatch swtemp = new Stopwatch();
txtLogML.AppendText("Process starts: " + DateTime.Now);
swglobal.Start();
txtLogML.AppendText("\r\nReading Evaluation Matrix ... ");
swtemp.Start();
DataTable EvalTable = ExcelHelper.GetDataFromExcel(txtInput.Text, "Eval");
swtemp.Stop();
txtLogML.AppendText(swtemp.Elapsed.ToString());
txtLogML.AppendText("\r\nReading Training Matrix ... ");
swtemp.Start();
DataTable TrainingTable = ExcelHelper.GetDataFromExcel(txtInput.Text, "Training");
swtemp.Stop();
txtLogML.AppendText(swtemp.Elapsed.ToString());
txtLogML.AppendText("\r\nStarting evaluation");
Sample sample = new Sample(EvalTable, TrainingTable);
double performamce_segment = 0;
double correlation_avg_segment = 0;
txtLogML.AppendText("\r\nTraining data ... ");
swtemp.Start();
correlation_avg_segment = sample.Train();
swtemp.Stop();
txtLogML.AppendText(swtemp.Elapsed.ToString());
txtLogML.AppendText("\r\nTesting data (This may take a long time) ... ");
swtemp.Start();
DataTable result = new DataTable();
performamce_segment = sample.Test(correlation_avg_segment, ref result);
swtemp.Stop();
txtLogML.AppendText(swtemp.Elapsed.ToString());
txtLogML.AppendText("\r\nDone evaluate sample:");
txtLogML.AppendText("\r\nMAE: " + performamce_segment);
txtLogML.AppendText("\r\nCorreation average: " + correlation_avg_segment);
txtLogML.AppendText("\r\nProcess finishs: " + DateTime.Now);
txtLogML.AppendText("\r\nStarting export predict ... ");
swtemp.Start();
ExcelHelper.SetDataToExcel(txtInput.Text, "Prediction", result);
swtemp.Stop();
txtLogML.AppendText(swtemp.Elapsed.ToString());
swglobal.Stop();
txtLogML.AppendText("Process ended " + DateTime.Now + " - Total time: " + swglobal.Elapsed.ToString());
MessageBox.Show("Done!");
}
private void btnTemp_Click(object sender, EventArgs e)
{
Segment root = Segment.GetRoot();
Sample sample = new Sample(root, root);
sample.Export("D:\\haizz.xlsx");
}
public static bool GetStrongSegments()
{
//List<Segment> AllSegment = Segment.GetAllSegment();
Segment root = Segment.GetRoot();
double sum_Correlation_Avg_root = 0;
int count_Correlation_Avg_root = 0;
DbHelper.RunScripts("truncate table segments", "Data Warehouse");
//foreach (Segment segment in AllSegment)
//{
foreach(Time time in Time.GetAll())
foreach(Budget budget in Budget.GetAllData())
foreach(Companion companion in Companion.GetAllData())
foreach (Weather weather in Weather.GetAllData())
{
if (time.period_of_day == Time.Period_Of_Day.All && time.period_of_week == Time.Period_Of_Week.All && time.season == Time.Season.All && budget.id == 0 && companion.id == 0 && weather.id == 0)
continue;
else
{
Segment segment = new Segment();
segment.time = time;
segment.budget = budget;
segment.companion = companion;
//segment.familiarity = familiarity;
//segment.mood = mood;
//segment.temperature = temperature;
//segment.travelLength = travelLength;
segment.weather = weather;
segment.GetData();
if (segment.data.CountCells() < 10)
continue;
//result.Add(segment);
//Resample segment
double performamce_segment = 0;
double performance_root = 0;
double correlation_avg_segment = 0;
List<Sample> resamples = Sample.Resampling(segment);
foreach (Sample sample in resamples)
{
double temp_corr_avg_segment = sample.Train();
if (!Double.IsNaN(temp_corr_avg_segment))
{
correlation_avg_segment += temp_corr_avg_segment;
performamce_segment += sample.Test(temp_corr_avg_segment);
}
Sample sample_root = new Sample(sample.Evaluation, sample.Training_root);
double temp_corr_root = sample_root.Train();
if (!Double.IsNaN(temp_corr_root))
{
sum_Correlation_Avg_root += temp_corr_root;
count_Correlation_Avg_root += 1;
performance_root += sample_root.Test(sum_Correlation_Avg_root / count_Correlation_Avg_root);
}
}
//if (performamce_segment < performance_root)
//if (!Double.IsNaN(performamce_segment) && !Double.IsNaN(correlation_avg_segment))
if (Double.IsNaN(performamce_segment))
performamce_segment = -9999;
if (Double.IsNaN(correlation_avg_segment))
correlation_avg_segment = -9999;
if (Double.IsNaN(performance_root))
performance_root = -9999;
try
{
DbHelper.RunScripts(string.Format("pr_insertSegment "
+ "'" + segment.time.period_of_day.ToString() + "'"
+ ", '" + segment.time.period_of_week.ToString() + "'"
+ ", '" + segment.time.season.ToString() + "'"
+ ", " + segment.budget.id
+ ", " + segment.companion.id
+ ", " + segment.weather.id
+ ", " + performamce_segment / 10
+ ", " + correlation_avg_segment / 10
+ ", " + performance_root / 10
+ ", " + sum_Correlation_Avg_root / count_Correlation_Avg_root)
, "Data Warehouse");
}
catch (Exception ex)
{ }
}
//Remove segment "child" and have performance less than its parents
/*
Segment[] candidates = Segment.GetCandidates();
for (int i = 0; i < candidates.Length - 1; i++)
{
for (int j = i + 1; j < candidates.Length; j++)
{
if (candidates[j].IsChildOf(candidates[i]))
{
DbHelper.RunScripts(string.Format("delete from segments where id = " + candidates[j].id), "Data Warehouse");
}
}
}*/
}
DbHelper.RunScripts(string.Format("pr_insertSegment "
+ "'All', 'All', 'All'"
+ ", " + 0
+ ", " + 0 + ", " + 0
+ ", " + 9999
+ ", " + sum_Correlation_Avg_root / count_Correlation_Avg_root), "Data Warehouse");
return true;
}