public static void Evol3_BuildAprioriInstance()
{
Helpers.Utils.Debug("Evol3 Apriori..");
Stopwatch sw = new Stopwatch();
sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("UHS_EXTENDED_STORIES_2",
new string[]{
"MNTHS_TO_1",
"MNTHS_TO_2",
"AGE",
"STATUS",
"T_NONE",
"T_HORMONE",
"T_SURGERY",
"T_PRI_CHEMO",
"T_ADJ_CHEMO",
"T_ADJ_RT",
"T_OOPH",
"T_PLASTIC",
"T_HER"
}, -1, -1, false, false);
long em = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Gathered MySQL ArffInstance {0} ms.", em));
uhsAi.integrityCheck();
uhsAi.rebuildAttributeValueByRange(0, 120);
uhsAi.rebuildAttributeValueByRange(1, 120);
uhsAi.rebuildAttributeValueByRange(2, 16);
uhsAi.deletePatternMatchingDatasets(new List<string> { "[-19<->101]" });
uhsAi.deletePatternMatchingDatasets(new List<string> { "*", "[-67<->53]" });
uhsAi.removeUnusedAttributeValues();
uhsAi.Relation = "uhs_ext_story_apriori_evol3";
long em2 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Prepared apriori set in {0} ms.", (em2 - em)));
new ArffSaver(uhsAi).saveInstance("uhs_ext_story_apriori_evol3");
Helpers.Utils.Debug(string.Format("Evol done, tooks {0} ms.", sw.ElapsedMilliseconds));
}
/// <summary>
/// Additional query parameter parsing & running Ceka methods
/// </summary>
/// <returns></returns>
public override int Run()
{
int minRange = -1;
int maxRange = -1;
bool firstColNull = false;
bool secondColNull = true;
string table = "";
int c = 0;
try
{
foreach (KeyValuePair<string, string> param in options.ParsedParameters)
{
switch (param.Key)
{
case "min-range": minRange = int.Parse(param.Value); c++; break;
case "max-range": maxRange = int.Parse(param.Value); c++; break;
case "first-column-null": firstColNull = bool.Parse(param.Value); c++; break;
case "second-column-null": secondColNull = bool.Parse(param.Value); c++; break;
case "table": table = param.Value; c++; break;
default: if (options.Verbose) clout("Parameter " + param.Key + " is not supported for SQL building!"); break;
}
}
}
catch (Exception ex)
{
if (options.Verbose) clout(ex.Message);
}
if (c != 5 && options.Verbose)
clout("Parameter set was not full! Probably using default parameters for SQL building!");
CekaMySQL sql = new CekaMySQL(options.ConStr);
ArffInstance uhsAi = sql.tableToInstance(table, options.Columns.ToArray<string>(), minRange, maxRange, firstColNull, secondColNull);
uhsAi.integrityCheck();
new ArffSaver(uhsAi).saveInstance(options.OutFile);
return 0;
}
public static void TestAprioriOnAliveSetNoFilter()
{
Helpers.Utils.Debug("Testing apriori on alive dataset..");
Stopwatch sw = new Stopwatch(); sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("uhs_patient_story".ToUpper(),
new string[]{
"INITIAL_PRESENTATION",
"INT_1_PRES",
"INT_2_PRES",
"STATUS"
}, -1, -1, false, true);
long t = sw.ElapsedMilliseconds;
Helpers.Utils.Debug("Gathered MySQL ArffInstance in " + t + "ms.");
uhsAi.removeDatasetsPerAttributeValue("STATUS", "Y");
uhsAi.Relation = "uhs_arff_alive";
Helpers.Utils.Debug("Cleaned ArffInstance in " + (sw.ElapsedMilliseconds - t) + "ms.");
new Ceka.Algorithms.Associaters.Apriori(uhsAi, 0.2f, 0.1f, true, true, "apriori_result_alive", true);
Helpers.Utils.Debug("Apriori test done, took " + sw.ElapsedMilliseconds + "ms."); sw.Stop();
}
public static void RunAprioriOnDeepCleanedUhsAliveDataset(float support, float confidence)
{
Helpers.Utils.Debug("Running Apriori on deep cleaned UHS alive dataset from DB..");
Stopwatch sw = new Stopwatch(); sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("uhs_patient_story".ToUpper(),
new string[]{
"INITIAL_PRESENTATION",
"INT_1_PRES",
"INT_2_PRES",
"STATUS"
}, -1, -1, false, true);
long em = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Gathered MySQL ArffInstance in {0} ms.", em));
uhsAi.removeDatasetsPerAttributeValue("STATUS", "N");
List<string> pattern1 = new List<string>() { "Primary|breast|cancer|(and/or|DCIS)", "Primary|breast|cancer|(and/or|DCIS)", "*" };
List<string> pattern2 = new List<string>() { "Primary|breast|cancer|(and/or|DCIS)", "UNDEFINED", "*" };
List<string> pattern3 = new List<string>() { "Primary|breast|cancer|(and/or|DCIS)", "*", "UNDEFINED" };
List<string> pattern4 = new List<string>() { "*", "UNDEFINED", "UNDEFINED" };
uhsAi.deletePatternMatchingDatasets(pattern1);
uhsAi.deletePatternMatchingDatasets(pattern2);
uhsAi.deletePatternMatchingDatasets(pattern3);
uhsAi.deletePatternMatchingDatasets(pattern4);
uhsAi.Datasets.removeEmptyValueDatasets();
long t = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Cleansed dataset in {0} ms.", t));
new Ceka.Algorithms.Associaters.Apriori(uhsAi, support, confidence, true, true, AprioriSaveTypes.WEKA);
Helpers.Utils.Debug(string.Format("Finished, took {0} ms.", sw.ElapsedMilliseconds));
sw.Stop();
}
public static void GetDefaultMultithreadedAprioriFlexJsonResultOfAliveAndDead()
{
Helpers.Utils.Debug("Getting threaded Apriori Flex Results of Alive and Dead datasets..");
Stopwatch sw = new Stopwatch(); sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("uhs_patient_story".ToUpper(),
new string[]{
"INITIAL_PRESENTATION",
"INT_1_PRES",
"INT_2_PRES",
"STATUS"
}, -1, -1, false, true);
Helpers.Utils.Debug("Gathered MySQL ArffInstance in " + sw.ElapsedMilliseconds + "ms.");
uhsAi.Relation = "uhs_arff_total";
Thread t1 = new Thread(new ThreadStart(delegate()
{
ArffInstance uhsAi2 = uhsAi.toCopy();
uhsAi2.removeDatasetsPerAttributeValue("STATUS", "Y");
uhsAi2.Relation = "uhs_arff_alive";
new Ceka.Algorithms.Associaters.Apriori(uhsAi2, 0.2f, 0.1f, "apriori_result_alive");
}));
Thread t2 = new Thread(new ThreadStart(delegate()
{
ArffInstance uhsAi3 = uhsAi.toCopy();
uhsAi3.removeDatasetsPerAttributeValue("STATUS", "N");
uhsAi3.Relation = "uhs_arff_dead";
new Ceka.Algorithms.Associaters.Apriori(uhsAi3, 0.2f, 0.1f, "apriori_result_dead");
}));
t1.Start();
t2.Start();
new Ceka.Algorithms.Associaters.Apriori(uhsAi, 0.2f, 0.1f, "apriori_result_total");
t1.Join();
t2.Join();
Helpers.Utils.Debug("Json Apriori Flex Result Dump done, took " + sw.ElapsedMilliseconds + "ms."); sw.Stop();
}
public static void GetDefaultDumpOfComplexUHSSets()
{
Helpers.Utils.Debug("Dumping default (complex) Arff UHS files..");
Stopwatch sw = new Stopwatch();
sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("UHS_EXTENDED_STORIES_2",
new string[]{
"AGE",
"STATUS",
"T_NONE",
"T_HORMONE",
"T_SURGERY",
"T_PRI_CHEMO",
"T_ADJ_CHEMO",
"T_ADJ_RT",
"T_OOPH",
"T_PLASTIC",
"T_HER"
}, -1, -1, true, true);
long em = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Gathered MySQL ArffInstance and stored total file in {0} ms.", em));
new ArffSaver(uhsAi).saveInstance("uhs_ext_stories_evol1");
Helpers.Utils.Debug(string.Format("Dumping done, tooks {0} ms.", sw.ElapsedMilliseconds));
}
public static void GetDefaultDumpOfAliveAndDeadDeepClean()
{
Helpers.Utils.Debug("Dumping default alive and dead ARFF files..");
Stopwatch sw = new Stopwatch(); sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToInstance("uhs_patient_story".ToUpper(),
new string[]{
"INITIAL_PRESENTATION",
"INT_1_PRES",
"INT_2_PRES",
"STATUS"
}, -1, -1, false, true);
long em = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Gathered MySQL ArffInstance in {0} ms.", em));
List<string> pattern1 = new List<string>() { "Primary|breast|cancer|(and/or|DCIS)", "Primary|breast|cancer|(and/or|DCIS)", "UNDEFINED" };
List<string> pattern2 = new List<string>() { "Primary|breast|cancer|(and/or|DCIS)", "UNDEFINED", "UNDEFINED" };
uhsAi.deletePatternMatchingDatasets(pattern1);
uhsAi.deletePatternMatchingDatasets(pattern2);
long emx = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Pattern match deletion took {0} ms.", (emx - em)));
new ArffSaver(uhsAi).saveInstance("uhs_arff_total");
ArffInstance uhsAi2 = uhsAi.toCopy();
long em2 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Copied ArffInstance (+save of total set) in {0} ms.", (em2 - emx)));
uhsAi.removeDatasetsPerAttributeValue("STATUS", "Y");
uhsAi.Relation = "uhs_arff_alive";
long em3 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Separated Alive Set in {0} ms.", (em3 - em2)));
uhsAi2.removeDatasetsPerAttributeValue("STATUS", "N");
uhsAi2.Relation = "uhs_arff_dead";
long em4 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Separated Dead Set in {0} ms.", (em4 - em3)));
new ArffSaver(uhsAi).saveInstance("uhs_arff_alive");
new ArffSaver(uhsAi2).saveInstance("uhs_arff_dead");
long em5 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Stored both sets in files, took {0} ms.", (em5 - em4)));
sql.close();
Helpers.Utils.Debug(string.Format("Dumping done, took {0} ms.", sw.ElapsedMilliseconds));
}
public static void Evol4_BuildClassifierInstance()
{
Helpers.Utils.Debug("Evol4 Classifier..");
Stopwatch sw = new Stopwatch();
sw.Start();
string mysqlConStr = "SERVER=localhost;" +
"DATABASE=uhs;" +
"UID=root;" +
"PASSWORD=pascal;";
CekaMySQL sql = new CekaMySQL(mysqlConStr);
ArffInstance uhsAi = sql.tableToValidatedInstance("UHS_EXTENDED_STORIES_2",
new string[]{
"MNTHS_TO_1",
"MNTHS_TO_2",
"AGE",
"STATUS",
"T_NONE",
"T_HORMONE",
"T_SURGERY",
"T_PRI_CHEMO",
"T_ADJ_CHEMO",
"T_ADJ_RT",
"T_OOPH",
"T_PLASTIC",
"T_HER"
}, new string[]{
"MNTHS_TO_1",
"MNTHS_TO_2"
}, -1, -1, false, false);
long em = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Gathered MySQL ArffInstance + preparing.. in {0} ms.", em));
uhsAi.deletePatternMatchingDatasets(new List<string>(){
"*", "0"
});
int ageIndex = uhsAi.getIndexOfAttribute("AGE");
uhsAi.rebuildAttributeValueByRange(ageIndex, 29);
uhsAi.refineBackRangedAttribute(ageIndex, 3, 5);
uhsAi.Relation = "uhs_ext_story_classifier_evol4";
long em2 = sw.ElapsedMilliseconds;
Helpers.Utils.Debug(string.Format("Prepared classifier set in {0} ms.", (em2 - em)));
new ArffSaver(uhsAi).saveInstance("uhs_ext_story_classifier_evol4");
Helpers.Utils.Debug(string.Format("Evol done, tooks {0} ms.", sw.ElapsedMilliseconds));
}