public ExperimentResults<int> AreaPercentage_HC_RangeQuery( string dataFile, int order, double pcMin, double pcMax, double inc, int trials) { HCQueryEvaluator qe = new HCQueryEvaluator(dataFile, order, CurveType.UpLeft); ExperimentResults<int> result = new ExperimentResults<int>( "range_hc_ap_" + order + '_' + pcMin + '_' + pcMax); //Percentage for (double p = pcMin; p <= pcMax; p += inc) { //Calculate the range for this percenage. double area = TotalArea * 0.01 * p; double kmRange = Math.Sqrt(area / Math.PI); double degRange = kmRange / kmPerDeg; result.AddCategory(p + " percent"); //For each experiment, run trial number of times for (int i = 0; i < trials; i++) { result.AddResult(qe.RangeQuery( qp.GetRandomQueryPoint(), (float)degRange)); } } return result; }
public ExperimentResults<int> AreaPercentage_MBR_RangeQuery( string dataFile, int fanout, double pcMin, double pcMax, double inc, int trials) { MbrQueryEvaluator qe = new MbrQueryEvaluator(dataFile, fanout); ExperimentResults<int> result = new ExperimentResults<int>( "range_mbr_ap_" + fanout + '_' + pcMin + '_' + pcMax); //Percentage for (double p = pcMin; p <= pcMax; p += inc) { //Calculate the range for this percenage. double area = TotalArea * 0.01 * p; double kmRange = Math.Sqrt(area / Math.PI); double degRange = kmRange / kmPerDeg; result.AddCategory(p + " percent"); //For each experiment, run trial number of times for (int i = 0; i < trials; i++) { result.AddResult(qe.RangeQuery( qp.GetRandomQueryPoint(), (float)degRange)); } } Console.WriteLine("Query Done."); return result; }
private async void RefreshResultsAsync() { var handle = uiService.StartIndicateLongOperation("Loading experiment results..."); try { experimentResults = null; allResults = Results = null; var res = await Task.Run(() => manager.GetResults(id)); experimentResults = res; allResults = Results = res.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray(); RefreshJobStatus(); } catch (Exception ex) { experimentResults = null; allResults = Results = null; uiService.ShowError(ex.Message, "Failed to load experiment results"); } finally { uiService.StopIndicateLongOperation(handle); } }
public static async void SaveMatrix(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IUIService uiService) { if (filename == null) { throw new ArgumentNullException("filename"); } if (experiments == null) { throw new ArgumentNullException("experiments"); } if (manager == null) { throw new ArgumentNullException("manager"); } if (uiService == null) { throw new ArgumentNullException("uiService"); } var handle = uiService.StartIndicateLongOperation("Save tex..."); try { using (StreamWriter f = new StreamWriter(filename, false)) { f.WriteLine("% -*- mode: latex; TeX-master: \"main.tex\"; -*-"); f.WriteLine(); f.WriteLine(@"\documentclass{article}"); f.WriteLine(@"\usepackage{multirow}"); f.WriteLine(@"\usepackage{rotating}"); f.WriteLine(@"\begin{document}"); int count = experiments.Length; ExperimentResults[] b = new ExperimentResults[count]; b = await DownloadResultsAsync(experiments, manager); MakeMatrix(experiments, b, f, 0, "SAT+UNSAT"); MakeMatrix(experiments, b, f, 1, "SAT"); MakeMatrix(experiments, b, f, 2, "UNSAT"); f.WriteLine(@"\end{document}"); } } catch (Exception ex) { uiService.ShowError(ex, "Failed to save matrix"); } finally { uiService.StopIndicateLongOperation(handle); } }
private static async Task <ExperimentResults[]> DownloadResultsAsync(ExperimentStatusViewModel[] experiments, ExperimentManager manager) { var count = experiments.Length; var t = new Task <ExperimentResults> [count]; for (int i = 0; i < count; i++) { int index = i; t[index] = Task.Run(() => manager.GetResults(experiments[index].ID)); } var b = new ExperimentResults[count]; for (int j = 0; j < count; j++) { int index = j; b[index] = await t[index]; } return(b); }
void DoExperiments() { Parameters.Reset(); var basepath = GetExperimentBasePath(); var settingsPath = basepath + ".settings.txt"; var meta = $"Date: {DateTime.Now:yyyy-MM-dd H:mm:ss}\n{Parameters.FixedParameters()}"; File.WriteAllText(settingsPath, meta); Results = new ExperimentResults(); var resultsPath = basepath + ".results.csv"; var header = $"{Parameters.VariableParametersCSVHeader()},{ExperimentResults.CSVHeader()}\n"; File.WriteAllText(resultsPath, header); DoNextExperiment = LeaveResultOnScreen == false; StartNextExperiment(); }
//-------------------------------------------------------------------------------------------------// protected void btnDownloadResults_Click(object sender, EventArgs e) { // Set the content type of the file to be downloaded Response.ContentType = Consts.StrRsp_ContentType_TextXml; // Clear all response headers Response.Clear(); // Add response header Response.AddHeader(Consts.StrRsp_Disposition, Consts.StrRsp_Attachment_ExperimentResultsXml); // // Retrieve all experiment results, convert to XML and write out // ExperimentResults experimentResults = new ExperimentResults(); string xmlExperimentResults = experimentResults.RetrieveAllToXml(); Response.Write(xmlExperimentResults); // End the http response Response.End(); }
public ExperimentResults<int> RTreeKnnQuery( string dataFile, int fanout, int min, int max, int inc, int trials) { RTreeQueryEvaluator qe = new RTreeQueryEvaluator(dataFile, fanout); ExperimentResults<int> result = new ExperimentResults<int>( "knn_rtree_" + fanout + '_' + min + '_' + max); for (int k = 1; k <= 10; k+=inc) { result.AddCategory(k + " NN"); //For each experiment, run trial number of times for (int i = 0; i < trials; i++) { result.AddResult(qe.KnnQuery( qp.GetRandomQueryPoint(), k)); } } return result; }
void StartNextExperiment() { /* * Start Next if necessary */ if (_repeatsRemaining == 0) { Results = new ExperimentResults(); if (Parameters.MoveNext()) { SetupFromParameters(Parameters); StartSingleExperiment(); } else { Debug.LogFormat("All experiments finished"); } } else { StartSingleExperiment(); } }
public ExperimentResults<int> HCKnnQuery( string dataFile, int order, int min, int max, int inc, int trials) { HCQueryEvaluator qe = new HCQueryEvaluator(dataFile, order, CurveType.UpLeft); ExperimentResults<int> result = new ExperimentResults<int>( "knn_hc_" + order + '_' + min + '_' + max); //Area for (int k = 1; k <= 10; k+=inc) { result.AddCategory(k + " NN"); //For each experiment, run trial number of times for (int i = 0; i < trials; i++) { result.AddResult(qe.KnnQuery( qp.GetRandomQueryPoint(), k)); } } return result; }
/// <summary> /// Assumes an Experiment is loaded, this initiates an Experiment run. /// This can be very long-running. /// </summary> /// <param name="explanation"></param> /// <returns></returns> public bool RunModelExperiment(string resultFilepath, out string explanation) { explanation = ""; string marker = "Begin"; ExperimentResults experimentResults = new ExperimentResults(); if (CurrentProject == null) { explanation = $"Cannot run plan. No Project is currently loaded"; return(false); } if (CurrentModel == null) { explanation = $"Cannot run plan. No Model is currently loaded"; return(false); } // Check for Experiment if (CurrentExperiment == null) { explanation = $"Model={CurrentModel.Name} has no Experiment selected."; return(false); } try { CurrentExperiment.ReplicationEnded += (s, e) => { int repNumber = e.ReplicationNumber; double runTime = e.ActualRuntimeInSeconds; }; CurrentExperiment.ScenarioEnded += (s, e) => { LogIt($"ScenarioEnded: Results:"); foreach (var result in e.Results) { // Log the results to get a feel for what is being returned. LogIt($"ScenarioEnded: ObjType='{result.ObjectType}', ObjName='{result.ObjectName}', DS='{result.DataSource}', " + $"Cat='{result.StatisticCategory}', DataItem='{result.DataItem}', Stat='{result.StatisticCategory}', " + $"Avg='{result.Average:0.00}', Min='{result.Minimum:f0.00}', Max='{result.Maximum:f0.00}"); experimentResults.AddScenarioResults(e.Scenario, e.Results); } }; CurrentExperiment.RunCompleted += (s, e) => { LogIt($"RunCompleted:"); }; CurrentExperiment.RunProgressChanged += (s, e) => { LogIt($"ProgressChanged:"); string ss = e.ToString(); }; // Run the experiment. Events will be thrown when replications start and end, // so you would have to handle those elsewhere. marker = "Starting Experiment (Experiment.Run)"; CurrentExperiment.Reset(); CurrentExperiment.Run(); if (string.IsNullOrEmpty(resultFilepath)) { LogIt($"Info: No result Filepath specified. Results not written."); } else { string folder = Path.GetDirectoryName(resultFilepath); if (Directory.Exists(folder)) { StringBuilder sb = new StringBuilder(); // Build a tab-delimited file if (!experimentResults.OutputCsv(resultFilepath, out explanation)) { explanation = $"Cannot Write CSV results to={resultFilepath}"; return(false); } } } marker = "End"; return(true); } catch (Exception ex) { explanation = $"Project={CurrentProject.Name} Model={CurrentModel.Name} Marker={marker} Err={ex.Message} InnerEx={ex.InnerException}"; return(false); } }
public static async Task <bool> TryReclassifyResults(BenchmarkResultViewModel[] resultsToUpdate, ResultStatus newStatus, TimeSpan benchmarkTimeout, ExperimentResults results) { if (results == null) { throw new ArgumentNullException(nameof(results)); } if (resultsToUpdate == null) { throw new ArgumentNullException(nameof(resultsToUpdate)); } if (resultsToUpdate.Length == 0) { return(true); } var changes = await results.TryUpdateStatus(resultsToUpdate.Select(r => r.result), newStatus); if (changes != null) // ok { foreach (var c in changes) { BenchmarkResultViewModel vm = resultsToUpdate.First(r => r.result == c.Key); // replace and notify about new status vm.UpdateResult(c.Value); } return(true); } return(false); }
public ExperimentResults<int> AreaSqMile_RTree_RangeQuery( string dataFile, int fanout, double min, double max, double inc, int trials) { RTreeQueryEvaluator qe = new RTreeQueryEvaluator(dataFile, fanout); ExperimentResults<int> result = new ExperimentResults<int>( "range_rtree_sqmi_" + fanout + '_' + min + '_' + max); //Percentage for (double a = min; a <= max; a += inc) { //Calculate the range for this percenage. double miRange = Math.Sqrt(a / Math.PI); double degRange = miRange / 69; result.AddCategory(a + " Sq Mi"); //For each experiment, run trial number of times for (int i = 0; i < trials; i++) { result.AddResult(qe.RangeQuery( qp.GetRandomQueryPoint(), (float)degRange)); } } return result; }
public static async void SaveMetaCSV(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IDomainResolver domainResolver, IUIService uiService) { if (filename == null) { throw new ArgumentNullException("filename"); } if (experiments == null) { throw new ArgumentNullException("experiments"); } if (manager == null) { throw new ArgumentNullException("manager"); } if (domainResolver == null) { throw new ArgumentNullException("domain"); } if (uiService == null) { throw new ArgumentNullException("uiService"); } var handle = uiService.StartIndicateLongOperation("Save meta csv..."); try { StreamWriter f = new StreamWriter(filename, false); f.WriteLine("\"ID\",\"# Total\",\"# SAT\",\"# UNSAT\",\"# UNKNOWN\",\"# Timeout\",\"# Memout\",\"# Bug\",\"# Error\",\"# Unique\",\"Parameters\",\"Note\""); var count = experiments.Length; var b = new ExperimentResults[count]; b = await DownloadResultsAsync(experiments, manager); var unique = computeUnique(experiments, b); for (var i = 0; i < count; i++) { var domain = domainResolver.GetDomain(experiments[i].Definition.DomainName ?? "Z3"); var aggr = domain.Aggregate(b[i].Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds))); var statistics = new ExperimentStatistics(aggr); var def = experiments[i].Definition; string ps = def.Parameters.Trim(' '); string note = experiments[i].Note.Trim(' '); int? sat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeySat], CultureInfo.InvariantCulture); int? unsat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnsat], CultureInfo.InvariantCulture); int? unknown = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnknown], CultureInfo.InvariantCulture); int? bugs = statistics == null ? null : (int?)statistics.AggregatedResults.Bugs; int? errors = statistics == null ? null : (int?)statistics.AggregatedResults.Errors; int? timeouts = statistics == null ? null : (int?)statistics.AggregatedResults.Timeouts; int? memouts = statistics == null ? null : (int?)statistics.AggregatedResults.MemoryOuts; f.WriteLine(experiments[i].ID + "," + experiments[i].BenchmarksTotal + "," + sat + "," + unsat + "," + unknown + "," + timeouts + "," + memouts + "," + bugs + "," + errors + "," + unique[i] + "," + "\"" + ps + "\"," + "\"" + note + "\""); } f.WriteLine(); f.Close(); } catch (Exception ex) { uiService.ShowError(ex, "Failed to save meta CSV"); } finally { uiService.StopIndicateLongOperation(handle); } }