public ExperimentResults<int> AreaPercentage_HC_RangeQuery(
string dataFile, int order, double pcMin, double pcMax,
double inc, int trials)
{
HCQueryEvaluator qe = new HCQueryEvaluator(dataFile, order, CurveType.UpLeft);
ExperimentResults<int> result = new ExperimentResults<int>(
"range_hc_ap_" + order + '_' + pcMin + '_' + pcMax);
//Percentage
for (double p = pcMin; p <= pcMax; p += inc)
{
//Calculate the range for this percenage.
double area = TotalArea * 0.01 * p;
double kmRange = Math.Sqrt(area / Math.PI);
double degRange = kmRange / kmPerDeg;
result.AddCategory(p + " percent");
//For each experiment, run trial number of times
for (int i = 0; i < trials; i++)
{
result.AddResult(qe.RangeQuery(
qp.GetRandomQueryPoint(),
(float)degRange));
}
}
return result;
}
public ExperimentResults<int> AreaPercentage_MBR_RangeQuery(
string dataFile, int fanout, double pcMin, double pcMax,
double inc, int trials)
{
MbrQueryEvaluator qe = new MbrQueryEvaluator(dataFile, fanout);
ExperimentResults<int> result = new ExperimentResults<int>(
"range_mbr_ap_" + fanout + '_' + pcMin + '_' + pcMax);
//Percentage
for (double p = pcMin; p <= pcMax; p += inc)
{
//Calculate the range for this percenage.
double area = TotalArea * 0.01 * p;
double kmRange = Math.Sqrt(area / Math.PI);
double degRange = kmRange / kmPerDeg;
result.AddCategory(p + " percent");
//For each experiment, run trial number of times
for (int i = 0; i < trials; i++)
{
result.AddResult(qe.RangeQuery(
qp.GetRandomQueryPoint(),
(float)degRange));
}
}
Console.WriteLine("Query Done.");
return result;
}
private async void RefreshResultsAsync()
{
var handle = uiService.StartIndicateLongOperation("Loading experiment results...");
try
{
experimentResults = null;
allResults = Results = null;
var res = await Task.Run(() => manager.GetResults(id));
experimentResults = res;
allResults = Results = res.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray();
RefreshJobStatus();
}
catch (Exception ex)
{
experimentResults = null;
allResults = Results = null;
uiService.ShowError(ex.Message, "Failed to load experiment results");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
public static async void SaveMatrix(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IUIService uiService)
{
if (filename == null)
{
throw new ArgumentNullException("filename");
}
if (experiments == null)
{
throw new ArgumentNullException("experiments");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (uiService == null)
{
throw new ArgumentNullException("uiService");
}
var handle = uiService.StartIndicateLongOperation("Save tex...");
try
{
using (StreamWriter f = new StreamWriter(filename, false))
{
f.WriteLine("% -*- mode: latex; TeX-master: \"main.tex\"; -*-");
f.WriteLine();
f.WriteLine(@"\documentclass{article}");
f.WriteLine(@"\usepackage{multirow}");
f.WriteLine(@"\usepackage{rotating}");
f.WriteLine(@"\begin{document}");
int count = experiments.Length;
ExperimentResults[] b = new ExperimentResults[count];
b = await DownloadResultsAsync(experiments, manager);
MakeMatrix(experiments, b, f, 0, "SAT+UNSAT");
MakeMatrix(experiments, b, f, 1, "SAT");
MakeMatrix(experiments, b, f, 2, "UNSAT");
f.WriteLine(@"\end{document}");
}
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to save matrix");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
private static async Task <ExperimentResults[]> DownloadResultsAsync(ExperimentStatusViewModel[] experiments, ExperimentManager manager)
{
var count = experiments.Length;
var t = new Task <ExperimentResults> [count];
for (int i = 0; i < count; i++)
{
int index = i;
t[index] = Task.Run(() => manager.GetResults(experiments[index].ID));
}
var b = new ExperimentResults[count];
for (int j = 0; j < count; j++)
{
int index = j;
b[index] = await t[index];
}
return(b);
}
void DoExperiments()
{
Parameters.Reset();
var basepath = GetExperimentBasePath();
var settingsPath = basepath + ".settings.txt";
var meta = $"Date: {DateTime.Now:yyyy-MM-dd H:mm:ss}\n{Parameters.FixedParameters()}";
File.WriteAllText(settingsPath, meta);
Results = new ExperimentResults();
var resultsPath = basepath + ".results.csv";
var header = $"{Parameters.VariableParametersCSVHeader()},{ExperimentResults.CSVHeader()}\n";
File.WriteAllText(resultsPath, header);
DoNextExperiment = LeaveResultOnScreen == false;
StartNextExperiment();
}
//-------------------------------------------------------------------------------------------------//
protected void btnDownloadResults_Click(object sender, EventArgs e)
{
// Set the content type of the file to be downloaded
Response.ContentType = Consts.StrRsp_ContentType_TextXml;
// Clear all response headers
Response.Clear();
// Add response header
Response.AddHeader(Consts.StrRsp_Disposition, Consts.StrRsp_Attachment_ExperimentResultsXml);
//
// Retrieve all experiment results, convert to XML and write out
//
ExperimentResults experimentResults = new ExperimentResults();
string xmlExperimentResults = experimentResults.RetrieveAllToXml();
Response.Write(xmlExperimentResults);
// End the http response
Response.End();
}
public ExperimentResults<int> RTreeKnnQuery(
string dataFile, int fanout, int min, int max,
int inc, int trials)
{
RTreeQueryEvaluator qe = new RTreeQueryEvaluator(dataFile, fanout);
ExperimentResults<int> result = new ExperimentResults<int>(
"knn_rtree_" + fanout + '_' + min + '_' + max);
for (int k = 1; k <= 10; k+=inc)
{
result.AddCategory(k + " NN");
//For each experiment, run trial number of times
for (int i = 0; i < trials; i++)
{
result.AddResult(qe.KnnQuery(
qp.GetRandomQueryPoint(),
k));
}
}
return result;
}
void StartNextExperiment()
{
/*
* Start Next if necessary
*/
if (_repeatsRemaining == 0)
{
Results = new ExperimentResults();
if (Parameters.MoveNext())
{
SetupFromParameters(Parameters);
StartSingleExperiment();
}
else
{
Debug.LogFormat("All experiments finished");
}
}
else
{
StartSingleExperiment();
}
}
public ExperimentResults<int> HCKnnQuery(
string dataFile, int order, int min, int max,
int inc, int trials)
{
HCQueryEvaluator qe = new HCQueryEvaluator(dataFile, order, CurveType.UpLeft);
ExperimentResults<int> result = new ExperimentResults<int>(
"knn_hc_" + order + '_' + min + '_' + max);
//Area
for (int k = 1; k <= 10; k+=inc)
{
result.AddCategory(k + " NN");
//For each experiment, run trial number of times
for (int i = 0; i < trials; i++)
{
result.AddResult(qe.KnnQuery(
qp.GetRandomQueryPoint(),
k));
}
}
return result;
}
/// <summary>
/// Assumes an Experiment is loaded, this initiates an Experiment run.
/// This can be very long-running.
/// </summary>
/// <param name="explanation"></param>
/// <returns></returns>
public bool RunModelExperiment(string resultFilepath, out string explanation)
{
explanation = "";
string marker = "Begin";
ExperimentResults experimentResults = new ExperimentResults();
if (CurrentProject == null)
{
explanation = $"Cannot run plan. No Project is currently loaded";
return(false);
}
if (CurrentModel == null)
{
explanation = $"Cannot run plan. No Model is currently loaded";
return(false);
}
// Check for Experiment
if (CurrentExperiment == null)
{
explanation = $"Model={CurrentModel.Name} has no Experiment selected.";
return(false);
}
try
{
CurrentExperiment.ReplicationEnded += (s, e) =>
{
int repNumber = e.ReplicationNumber;
double runTime = e.ActualRuntimeInSeconds;
};
CurrentExperiment.ScenarioEnded += (s, e) =>
{
LogIt($"ScenarioEnded: Results:");
foreach (var result in e.Results)
{
// Log the results to get a feel for what is being returned.
LogIt($"ScenarioEnded: ObjType='{result.ObjectType}', ObjName='{result.ObjectName}', DS='{result.DataSource}', " +
$"Cat='{result.StatisticCategory}', DataItem='{result.DataItem}', Stat='{result.StatisticCategory}', " +
$"Avg='{result.Average:0.00}', Min='{result.Minimum:f0.00}', Max='{result.Maximum:f0.00}");
experimentResults.AddScenarioResults(e.Scenario, e.Results);
}
};
CurrentExperiment.RunCompleted += (s, e) =>
{
LogIt($"RunCompleted:");
};
CurrentExperiment.RunProgressChanged += (s, e) =>
{
LogIt($"ProgressChanged:");
string ss = e.ToString();
};
// Run the experiment. Events will be thrown when replications start and end,
// so you would have to handle those elsewhere.
marker = "Starting Experiment (Experiment.Run)";
CurrentExperiment.Reset();
CurrentExperiment.Run();
if (string.IsNullOrEmpty(resultFilepath))
{
LogIt($"Info: No result Filepath specified. Results not written.");
}
else
{
string folder = Path.GetDirectoryName(resultFilepath);
if (Directory.Exists(folder))
{
StringBuilder sb = new StringBuilder();
// Build a tab-delimited file
if (!experimentResults.OutputCsv(resultFilepath, out explanation))
{
explanation = $"Cannot Write CSV results to={resultFilepath}";
return(false);
}
}
}
marker = "End";
return(true);
}
catch (Exception ex)
{
explanation = $"Project={CurrentProject.Name} Model={CurrentModel.Name} Marker={marker} Err={ex.Message} InnerEx={ex.InnerException}";
return(false);
}
}
public static async Task <bool> TryReclassifyResults(BenchmarkResultViewModel[] resultsToUpdate, ResultStatus newStatus, TimeSpan benchmarkTimeout, ExperimentResults results)
{
if (results == null)
{
throw new ArgumentNullException(nameof(results));
}
if (resultsToUpdate == null)
{
throw new ArgumentNullException(nameof(resultsToUpdate));
}
if (resultsToUpdate.Length == 0)
{
return(true);
}
var changes = await results.TryUpdateStatus(resultsToUpdate.Select(r => r.result), newStatus);
if (changes != null) // ok
{
foreach (var c in changes)
{
BenchmarkResultViewModel vm = resultsToUpdate.First(r => r.result == c.Key);
// replace and notify about new status
vm.UpdateResult(c.Value);
}
return(true);
}
return(false);
}
public ExperimentResults<int> AreaSqMile_RTree_RangeQuery(
string dataFile, int fanout, double min, double max,
double inc, int trials)
{
RTreeQueryEvaluator qe = new RTreeQueryEvaluator(dataFile, fanout);
ExperimentResults<int> result = new ExperimentResults<int>(
"range_rtree_sqmi_" + fanout + '_' + min + '_' + max);
//Percentage
for (double a = min; a <= max; a += inc)
{
//Calculate the range for this percenage.
double miRange = Math.Sqrt(a / Math.PI);
double degRange = miRange / 69;
result.AddCategory(a + " Sq Mi");
//For each experiment, run trial number of times
for (int i = 0; i < trials; i++)
{
result.AddResult(qe.RangeQuery(
qp.GetRandomQueryPoint(),
(float)degRange));
}
}
return result;
}
public static async void SaveMetaCSV(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IDomainResolver domainResolver, IUIService uiService)
{
if (filename == null)
{
throw new ArgumentNullException("filename");
}
if (experiments == null)
{
throw new ArgumentNullException("experiments");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (domainResolver == null)
{
throw new ArgumentNullException("domain");
}
if (uiService == null)
{
throw new ArgumentNullException("uiService");
}
var handle = uiService.StartIndicateLongOperation("Save meta csv...");
try
{
StreamWriter f = new StreamWriter(filename, false);
f.WriteLine("\"ID\",\"# Total\",\"# SAT\",\"# UNSAT\",\"# UNKNOWN\",\"# Timeout\",\"# Memout\",\"# Bug\",\"# Error\",\"# Unique\",\"Parameters\",\"Note\"");
var count = experiments.Length;
var b = new ExperimentResults[count];
b = await DownloadResultsAsync(experiments, manager);
var unique = computeUnique(experiments, b);
for (var i = 0; i < count; i++)
{
var domain = domainResolver.GetDomain(experiments[i].Definition.DomainName ?? "Z3");
var aggr = domain.Aggregate(b[i].Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds)));
var statistics = new ExperimentStatistics(aggr);
var def = experiments[i].Definition;
string ps = def.Parameters.Trim(' ');
string note = experiments[i].Note.Trim(' ');
int? sat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeySat], CultureInfo.InvariantCulture);
int? unsat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnsat], CultureInfo.InvariantCulture);
int? unknown = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnknown], CultureInfo.InvariantCulture);
int? bugs = statistics == null ? null : (int?)statistics.AggregatedResults.Bugs;
int? errors = statistics == null ? null : (int?)statistics.AggregatedResults.Errors;
int? timeouts = statistics == null ? null : (int?)statistics.AggregatedResults.Timeouts;
int? memouts = statistics == null ? null : (int?)statistics.AggregatedResults.MemoryOuts;
f.WriteLine(experiments[i].ID + "," +
experiments[i].BenchmarksTotal + "," +
sat + "," +
unsat + "," +
unknown + "," +
timeouts + "," +
memouts + "," +
bugs + "," +
errors + "," +
unique[i] + "," +
"\"" + ps + "\"," +
"\"" + note + "\"");
}
f.WriteLine();
f.Close();
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to save meta CSV");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
