static async Task Run()
{
string connectionString = await GetConnectionString();
RepositoryContent binary = await GetRecentNightlyBuild();
if (binary == null)
{
Trace.WriteLine("Repository has no new build.");
return;
}
Trace.WriteLine("Last nightly build contains " + binary.Name);
AzureExperimentManager manager = AzureExperimentManager.Open(connectionString);
string lastNightlyExecutable = await GetLastNightlyExperiment(manager);
if (lastNightlyExecutable == binary.Name)
{
Trace.WriteLine("No changes found since last nightly experiment.");
return;
}
using (MemoryStream stream = new MemoryStream(binary.Size))
{
await Download(binary, stream);
stream.Position = 0;
Trace.WriteLine("Opening an experiment manager...");
await SubmitExperiment(manager, stream, binary.Name);
}
}
static async Task Run()
{
string connectionString = await GetConnectionString();
Tuple <ReleaseAsset, string> asset_hash = await GetNightlyBuild();
if (asset_hash == null || asset_hash.Item1 == null)
{
Trace.WriteLine("Repository has no new build.");
return;
}
ReleaseAsset asset = asset_hash.Item1;
string hash = asset_hash.Item2;
AzureExperimentManager manager = AzureExperimentManager.Open(connectionString);
DateTime last_submission_time = await GetLastNightlyExperimentSubmissionTime(manager);
if (last_submission_time >= asset.UpdatedAt)
{
Trace.WriteLine("No changes found since last nightly experiment.");
return;
}
using (MemoryStream stream = new MemoryStream(asset.Size))
{
await Download(asset, stream);
stream.Position = 0;
Trace.WriteLine("Opening an experiment manager...");
await SubmitExperiment(manager, stream, asset.Name, hash);
}
}
static async Task SubmitExperiment(AzureExperimentManager manager, Stream source, string fileName, string commit_hash)
{
Trace.WriteLine("Uploading new executable...");
string packageName = await manager.Storage.UploadNewExecutable(source, fileName, Settings.Creator);
Trace.WriteLine("Successfully uploaded as " + packageName);
ExperimentDefinition definition =
ExperimentDefinition.Create(
packageName,
ExperimentDefinition.DefaultContainerUri,
Settings.BenchmarkDirectory,
Settings.BenchmarkFileExtension,
Settings.Parameters,
TimeSpan.FromSeconds(Settings.BenchmarkTimeoutSeconds),
TimeSpan.FromSeconds(Settings.ExperimentTimeoutSeconds),
Settings.Domain,
Settings.BenchmarkCategory,
Settings.MemoryLimitMegabytes,
1, 0 /* Run each benchmark once (i.e. adaptive run is off) */);
Trace.WriteLine(string.Format("Starting nightly experiment in Batch pool \"{0}\"...", Settings.AzureBatchPoolId));
manager.BatchPoolID = Settings.AzureBatchPoolId;
string note = commit_hash != null?
string.Format("{0} for https://github.com/{1}/{2}/commit/{3}", Settings.ExperimentNote, Settings.GitHubOwner, Settings.GitHubZ3Repository, commit_hash) :
Settings.ExperimentNote;
var summaryName = Settings.SummaryName == "" ? null : Settings.SummaryName;
var experimentId = await manager.StartExperiment(definition, Settings.Creator, note, summaryName);
Trace.WriteLine(string.Format("Done, experiment id {0}.", experimentId));
}
protected async void Page_Load(object sender, EventArgs e)
{
if (!IsPostBack)
{
try
{
_defaultParams = new Dictionary <string, string>();
string summaryName = Request.Params.Get("summary");
summaryName = summaryName == null ? Properties.Settings.Default.SummaryName : summaryName;
_defaultParams.Add("summary", summaryName);
var connectionString = await SiteMaster.GetConnectionString();
var expManager = AzureExperimentManager.Open(connectionString);
var summaryManager = new AzureSummaryManager(connectionString, Helpers.GetDomainResolver());
vm = await Helpers.GetTimeline(summaryName, expManager, summaryManager, connectionString);
tags = await Helpers.GetTags(summaryName, summaryManager);
BuildEntries();
}
catch (Exception ex)
{
Label l = new Label();
l.Text = "Error loading dataset: " + ex.Message;
phMain.Controls.Add(l);
l = new Label();
l.Text = "Stacktrace: " + ex.StackTrace;
phMain.Controls.Add(l);
}
}
}
static async Task <string> GetLastNightlyExperiment(AzureExperimentManager manager)
{
Trace.WriteLine("Looking for most recent nightly experiment...");
// Returns a list ordered by submission time
var experiments = await manager.FindExperiments(new ExperimentManager.ExperimentFilter()
{
CreatorEquals = Settings.Creator
});
var mostRecent = experiments.FirstOrDefault();
if (mostRecent == null)
{
return(null);
}
var metadata = await manager.Storage.GetExecutableMetadata(mostRecent.Definition.Executable);
string fileName = null;
if (metadata.TryGetValue(AzureExperimentStorage.KeyFileName, out fileName))
{
Trace.WriteLine("Last nightly experiment was run for " + fileName);
}
return(fileName);
}
private Task <AzureExperimentManagerViewModel> ConnectAsync(string connectionString)
{
return(Task.Run(() => // run in thread pool
{
AzureExperimentManager azureManager = AzureExperimentManager.Open(connectionString);
return new AzureExperimentManagerViewModel(azureManager, uiService, domainResolver);
}));
}
private Timeline(AzureExperimentManager expManager, AzureSummaryManager summaryManager, string summaryName, ExperimentViewModel[] experiments, RecordsTable records)
{
this.expManager = expManager;
this.summaryManager = summaryManager;
this.summaryName = summaryName;
this.experiments = experiments.OrderBy(exp => exp.SubmissionTime).ToArray();
Records = records;
}
public async Task GetExperimentsFromCloud()
{
ValidatesConnectionString();
AzureExperimentStorage storage = new AzureExperimentStorage(storageConnectionString);
AzureExperimentManager manager = AzureExperimentManager.OpenWithoutStart(storage);
var experiments = (await manager.FindExperiments()).ToArray();
Assert.IsTrue(experiments.Length > 0);
}
public static void GeneratePlot(AzureExperimentManager aeMan, Tags tags, string prefix, ComparableExperiment refE, int refId, int id, string outputPath)
{
Console.WriteLine("Generating plot for #{0}...", id);
System.Diagnostics.Debug.Print("Starting task for job #{0}", id);
ComparableExperiment e = Helpers.GetComparableExperiment(id, aeMan).Result;
Comparison cmp = new Comparison(refE, e, prefix.Replace('|', '/'), tags);
Chart chart = Charts.BuildComparisonChart(prefix, cmp);
chart.SaveImage(Path.Combine(outputPath, String.Format("{0}-{1}.png", refId, id)), ChartImageFormat.Png);
e = null; cmp = null; chart = null;
System.GC.Collect();
System.Diagnostics.Debug.Print("Ending task for job #{0}", id);
}
public static async Task Run(string prefix, string outputPath, string repositoryPath)
{
Console.WriteLine("Connecting...");
var sStorage = new SecretStorage(Settings.Default.AADApplicationId, Settings.Default.AADApplicationCertThumbprint, Settings.Default.KeyVaultUrl);
string cString = await sStorage.GetSecret(Settings.Default.ConnectionStringSecretId);
AzureSummaryManager sMan = new AzureSummaryManager(cString, Helpers.GetDomainResolver());
AzureExperimentManager aeMan = AzureExperimentManager.Open(cString);
Tags tags = await Helpers.GetTags(Settings.Default.SummaryName, sMan);
Console.WriteLine("Loading timeline...");
Timeline timeline = await Helpers.GetTimeline(Settings.Default.SummaryName, aeMan, sMan, cString);
// Numbers: 4.5.0 = 8023; suspect 8308 -> 8312
Directory.CreateDirectory(outputPath);
int refId = 8023;
Console.WriteLine("Loading reference #{0}...", refId);
ComparableExperiment refE = await Helpers.GetComparableExperiment(refId, aeMan);
List <Task> tasks = new List <Task>();
for (int i = 0; i < timeline.Experiments.Length; i++)
{
ExperimentViewModel e = timeline.Experiments[i];
if (e.Id < refId)
{
continue;
}
//tasks.Add(GeneratePlot(aeMan, tags, prefix, refE, refId, e.Id, outputPath));
//GeneratePlot(aeMan, tags, prefix, refE, refId, e.Id, outputPath);
if (i > 0 && e.Id != refId)
{
GenerateLog(repositoryPath, outputPath, timeline.Experiments[i - 1].Id, e.Id, timeline.Experiments[i - 1].SubmissionTime, e.SubmissionTime);
}
//if (e.Id == 8099) break;
}
//ParallelOptions popts = new ParallelOptions();
//popts.MaxDegreeOfParallelism = 1;
//ParallelLoopResult r = Parallel.ForEach(tasks, popts, t => t.Wait());
//Task.WaitAll(tasks.ToArray());
//await Task.WhenAll(tasks);
}
public AzureExperimentManagerViewModel(AzureExperimentManager manager, IUIService uiService, IDomainResolver domainResolver)
{
if (manager == null)
{
throw new ArgumentNullException("manager");
}
this.manager = manager;
if (uiService == null)
{
throw new ArgumentNullException("uiService");
}
this.uiService = uiService;
if (domainResolver == null)
{
throw new ArgumentNullException("domainResolver");
}
this.domainResolver = domainResolver;
}
protected async void Page_Load(object sender, EventArgs e)
{
try
{
_defaultParams = new Dictionary <string, string>();
string summaryName = Request.Params.Get("summary");
summaryName = summaryName == null ? Properties.Settings.Default.SummaryName : summaryName;
_defaultParams.Add("summary", summaryName);
var connectionString = await SiteMaster.GetConnectionString();
expManager = AzureExperimentManager.Open(connectionString);
summaryManager = new AzureSummaryManager(connectionString, Helpers.GetDomainResolver());
timeline = await Helpers.GetTimeline(summaryName, expManager, summaryManager, connectionString);
string fn = Request.Params.Get("filename");
if (fn != null)
{
txtFilename.Text = fn;
}
string db = Request.Params.Get("daysback");
if (db != null)
{
txtDaysBack.Text = db;
}
if (txtFilename.Text != "")
{
BuildEntries();
}
}
catch (Exception ex)
{
Label l = new Label();
l.Text = "Error loading dataset: " + ex.Message;
phMain.Controls.Add(l);
l = new Label();
l.Text = "Stacktrace: " + ex.StackTrace;
phMain.Controls.Add(l);
}
}
static async Task <DateTime> GetLastNightlyExperimentSubmissionTime(AzureExperimentManager manager)
{
Trace.WriteLine("Looking for most recent nightly experiment...");
// Returns a list ordered by submission time
var experiments = await manager.FindExperiments(new ExperimentManager.ExperimentFilter()
{
CreatorEquals = Settings.Creator
});
var mostRecent = experiments.FirstOrDefault();
if (mostRecent == null)
{
return(DateTime.MinValue);
}
Trace.WriteLine("Last nightly experiment was submitted at " + mostRecent.Status.SubmissionTime);
return(mostRecent.Status.SubmissionTime);
}
public async Task GetResultsFromCloud()
{
ValidatesConnectionString();
AzureExperimentStorage storage = new AzureExperimentStorage(storageConnectionString);
AzureExperimentManager manager = AzureExperimentManager.OpenWithoutStart(storage);
Stopwatch sw1 = Stopwatch.StartNew();
var results = await manager.GetResults(TestExperimentId);
Assert.AreEqual(TestExperimentId, results.ExperimentId);
sw1.Stop();
Trace.WriteLine("1st time: " + sw1.ElapsedMilliseconds);
Assert.AreEqual(103814, results.Benchmarks.Length);
/// Again, should read from local disk:
Stopwatch sw2 = Stopwatch.StartNew();
var results2 = await manager.GetResults(TestExperimentId);
sw2.Stop();
Trace.WriteLine("2nd time: " + sw2.ElapsedMilliseconds);
Assert.AreEqual(103814, results2.Benchmarks.Length);
}
static void Main(string[] args)
{
Keys keys = JsonConvert.DeserializeObject <Keys>(File.ReadAllText("..\\..\\keys.json"));
var storage = new AzureExperimentStorage(keys.storageName, keys.storageKey);
var manager = AzureExperimentManager.Open(storage, keys.batchUri, keys.batchName, keys.batchKey);
//var refExp = new ReferenceExperiment(ExperimentDefinition.Create("referencez3.zip", ExperimentDefinition.DefaultContainerUri, "reference", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), "Z3", null, 2048), 20, 16.34375);
// storage.SaveReferenceExperiment(refExp).Wait();
var id = manager.StartExperiment(ExperimentDefinition.Create("z3.zip", ExperimentDefinition.DefaultContainerUri, "QF_BV", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), TimeSpan.FromSeconds(0), "Z3", "asp", 2048, 1, 0), "Dmitry K").Result;
Console.WriteLine("Experiment id:" + id);
//manager.RestartBenchmarks(159, new string[] {
// "15Puzzle/15-puzzle.init10.smt2",
// "15Puzzle/15-puzzle.init11.smt2",
// "15Puzzle/15-puzzle.init12.smt2",
// "15Puzzle/15-puzzle.init13.smt2",
// "15Puzzle/15-puzzle.init14.smt2",
// "15Puzzle/15-puzzle.init15.smt2",
// "15Puzzle/15-puzzle.init2.smt2",
// "15Puzzle/15-puzzle.init3.smt2",
// "15Puzzle/15-puzzle.init4.smt2",
// "15Puzzle/15-puzzle.init5.smt2",
// "15Puzzle/15-puzzle.init6.smt2",
// "15Puzzle/15-puzzle.init7.smt2",
// "15Puzzle/15-puzzle.init8.smt2",
// "15Puzzle/15-puzzle.init9.smt2",
// "15Puzzle/15puzzle_ins.lp.smt2"
//}).Wait();
Console.WriteLine("Done.");
Console.ReadLine();
}
public static async Task <Timeline> Initialize(string connectionString, string summaryName, AzureExperimentManager expManager, AzureSummaryManager summaryManager)
{
var summRec = await summaryManager.GetTimelineAndRecords(summaryName);
var summ = summRec.Item1;
var records = summRec.Item2;
var now = DateTime.Now;
var expTasks =
summ
.Select(async expSum =>
{
var exp = await expManager.TryFindExperiment(expSum.Id);
if (exp == null)
{
return(null);
}
bool isFinished;
if (exp.Status.SubmissionTime.Subtract(now).TotalDays >= 3)
{
isFinished = true;
}
else
{
try
{
var jobState = await expManager.GetExperimentJobState(new[] { exp.ID });
isFinished = jobState[0] != ExperimentExecutionState.Active;
}
catch
{
isFinished = true;
}
}
return(new ExperimentViewModel(expSum, isFinished, exp.Status.SubmissionTime, exp.Definition.BenchmarkTimeout));
});
var experiments = await Task.WhenAll(expTasks);
return(new Timeline(expManager, summaryManager, summaryName, experiments, records));
}
protected async void Page_Load(object sender, EventArgs e)
{
try
{
string summaryName = Request.Params.Get("summary");
summaryName = summaryName == null ? Settings.Default.SummaryName : summaryName;
string connectionString = await SiteMaster.GetConnectionString();
AzureSummaryManager summaryManager = new AzureSummaryManager(connectionString, Helpers.GetDomainResolver());
string px = null, py = null;
Tags tags = await Helpers.GetTags(summaryName, summaryManager);
if (!IsPostBack)
{
var last2 = await Helpers.GetTwoLastExperimentsId(summaryName, summaryManager);
string penultimate = last2.Item1.ToString();
string latest = last2.Item2.ToString();
lstTagX.Items.Add(new ListItem("Latest (" + latest + ")", latest));
lstTagX.Items.Add(new ListItem("Penultimate (" + penultimate + ")", penultimate));
//lstTagX.Items.Add(new ListItem("Records (best ever)", "RECORDS"));
lstTagY.Items.Add(new ListItem("Latest (" + latest + ")", latest));
lstTagY.Items.Add(new ListItem("Penultimate (" + penultimate + ")", penultimate));
foreach (KeyValuePair <string, int> kvp in tags)
{
lstTagX.Items.Add(new ListItem(kvp.Key + " (" + kvp.Value.ToString() + ")", kvp.Value.ToString()));
lstTagY.Items.Add(new ListItem(kvp.Key + " (" + kvp.Value.ToString() + ")", kvp.Value.ToString()));
}
px = Request.Params.Get("jobX");
py = Request.Params.Get("jobY");
Prefix = Request.Params.Get("prefix");
if (px == null)
{
rbnTagX.Checked = true;
}
else
{
txtIDX.Text = px; rbnIDX.Checked = true;
}
if (py == null)
{
rbnTagY.Checked = true;
}
else
{
txtIDY.Text = py; rbnIDY.Checked = true;
}
if (px != null)
{
if (tags.HasID(int.Parse(px, CultureInfo.InvariantCulture)))
{
rbnTagX.Checked = true;
lstTagX.SelectedValue = px;
}
else
{
lstTagX.SelectedValue = penultimate;
}
}
else
{
lstTagX.SelectedValue = penultimate;
}
if (py != null)
{
if (tags.HasID(int.Parse(py, CultureInfo.InvariantCulture)))
{
rbnTagY.Checked = true;
lstTagY.Items.FindByText(py.ToString());
lstTagY.SelectedValue = py.ToString();
}
else
{
lstTagY.SelectedValue = latest;
}
}
else
{
lstTagY.SelectedValue = latest;
}
}
JX = px != null ? px : rbnTagX.Checked ? lstTagX.SelectedValue : txtIDX.Text;
JY = py != null ? py : rbnTagY.Checked ? lstTagY.SelectedValue : txtIDY.Text;
ComparableExperiment jX = null, jY = null;
try
{
AzureExperimentManager expMan = AzureExperimentManager.Open(connectionString);
var t1 = Task.Run(() => Helpers.GetComparableExperiment(int.Parse(JX, CultureInfo.InvariantCulture), expMan));
var t2 = Task.Run(() => Helpers.GetComparableExperiment(int.Parse(JY, CultureInfo.InvariantCulture), expMan));
jX = await t1;
jY = await t2;
}
catch (Exception)
{
}
txtIDX.Text = JX;
txtIDY.Text = JY;
cmp = new Comparison(jX, jY, Prefix.Replace('|', '/'), tags);
phPre.Controls.Add(buildHeader("CHART_PRE", ""));
phChart.Controls.Add(Charts.BuildComparisonChart(Prefix, cmp));
phHisto.Controls.Add(Charts.BuildComparisonHistogramm(cmp));
phMain.Controls.Add(buildTabPanels());
//phMain.Controls.Add(buildFooter());
}
catch (Exception ex)
{
Label l = new Label();
l.Text = "Error loading dataset: " + ex.Message;
phMain.Controls.Add(l);
l = new Label();
l.Text = "Stacktrace: " + ex.StackTrace;
phMain.Controls.Add(l);
}
}
public static async Task <Timeline> Initialize(string connectionString, string summaryName, AzureExperimentManager expManager, AzureSummaryManager summaryManager, bool onlySummary = false)
{
var summRec = await summaryManager.GetTimelineAndRecords(summaryName, onlySummary);
var summ = summRec.Item1;
var records = summRec.Item2;
var now = DateTime.Now;
var expTasks =
summ
.Select(async expSum =>
{
//var exp = await expManager.TryFindExperiment(expSum.Id);
//if (exp == null) return null;
bool isFinished;
var date = expSum.Date;
if ((now - date).TotalDays >= 1)
{
isFinished = true;
}
else
{
try
{
var jobState = await expManager.GetExperimentJobState(new[] { expSum.Id });
isFinished = jobState[0] != ExperimentExecutionState.Active;
}
catch
{
isFinished = true;
}
}
return(new ExperimentViewModel(expSum, isFinished));
});
var experiments = await Task.WhenAll(expTasks);
return(new Timeline(expManager, summaryManager, summaryName, experiments, records));
}