/// <summary>
/// Download the results of a job.
/// </summary>
/// <param name="options">Download options.</param>
/// <param name="ct">Cancellation token.</param>
/// <param name="ShowProgress">Function which reports progress (in range [0, 1]) to the user.</param>
public async Task DownloadResultsAsync(DownloadOptions options, CancellationToken ct, Action<double> ShowProgress)
{
if (!Directory.Exists(options.Path))
Directory.CreateDirectory(options.Path);
List<CloudBlockBlob> outputs = await GetJobOutputs(options.JobID, ct);
// Build up a list of files to download.
List<CloudBlockBlob> toDownload = new List<CloudBlockBlob>();
CloudBlockBlob results = outputs.Find(b => string.Equals(b.Name, resultsFileName, StringComparison.InvariantCultureIgnoreCase));
if (results != null)
toDownload.Add(results);
else
// Always download debug files if no results archive can be found.
options.DownloadDebugFiles = true;
if (options.DownloadDebugFiles)
toDownload.AddRange(outputs.Where(blob => debugFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))));
// Now download the necessary files.
for (int i = 0; i < toDownload.Count; i++)
{
ShowProgress(i / toDownload.Count);
CloudBlockBlob blob = toDownload[i];
// todo: Download in parallel?
await blob.DownloadToFileAsync(Path.Combine(options.Path, blob.Name), FileMode.Create, ct);
if (ct.IsCancellationRequested)
return;
}
ShowProgress(100);
if (options.ExtractResults)
{
string archive = Path.Combine(options.Path, resultsFileName);
string resultsDir = Path.Combine(options.Path, "results");
if (File.Exists(archive))
{
// Extract the result files.
using (ZipArchive zip = ZipFile.Open(archive, ZipArchiveMode.Read, Encoding.UTF8))
zip.ExtractToDirectory(resultsDir);
try
{
// Merge results into a single .db file.
DBMerger.MergeFiles(Path.Combine(resultsDir, "*.db"), false, "combined.db");
}
catch (Exception err)
{
throw new Exception($"Results were successfully extracted to {resultsDir} but an error wasn encountered while attempting to merge the individual .db files", err);
}
// TBI: merge into csv file.
if (options.ExportToCsv)
throw new NotImplementedException();
}
}
}
/// <summary>
/// Download one or more files and process the .zip and .db files.
/// </summary>
/// <param name="outputs"></param>
/// <param name="ShowProgress"></param>
/// <param name="resultTempDirectory"></param>
/// <param name="ct"></param>
/// <returns></returns>
private static async Task DownloadAndProcessFiles(List <CloudBlockBlob> outputs, Action <double> ShowProgress, string resultTempDirectory, CancellationToken ct)
{
// Download all run outputs.
for (int i = 0; i < outputs.Count; i++)
{
ShowProgress(i / outputs.Count);
CloudBlockBlob blob = outputs[i];
// todo: Download in parallel?
var fullFileName = Path.Combine(resultTempDirectory, blob.Name);
await blob.DownloadToFileAsync(fullFileName, FileMode.Create, ct);
if (ct.IsCancellationRequested)
{
return;
}
}
ShowProgress(100);
// Extract any zip files.
foreach (var zipFileName in Directory.GetFiles(resultTempDirectory, "*.zip"))
{
if (Path.GetFileName(zipFileName) != "model.zip")
{
// Extract the result files.
using (ZipArchive zip = ZipFile.Open(zipFileName, ZipArchiveMode.Read, Encoding.UTF8))
zip.ExtractToDirectory(resultTempDirectory);
}
}
try
{
// Merge results into a single .db file.
var dbFiles = Directory.GetFiles(resultTempDirectory, "*.db");
var resultsDB = Path.Combine(resultTempDirectory, "Results.db");
DBMerger.MergeFiles(Path.Combine(resultTempDirectory, "*.db"), false, resultsDB);
if (File.Exists(resultsDB))
{
// Remove the individual .db files.
foreach (string dbFileName in dbFiles)
{
File.Delete(dbFileName);
}
// Delete the zip file containing the .db files.
foreach (var zipFileName in Directory.GetFiles(resultTempDirectory, "*.zip"))
{
if (Path.GetFileName(zipFileName) != "model.zip")
{
File.Delete(zipFileName);
}
}
}
}
catch (Exception err)
{
throw new Exception($"Results were successfully extracted to {resultTempDirectory} but an error wasn encountered while attempting to merge the individual .db files", err);
}
}
/// <summary>
/// Main program entry point.
/// </summary>
/// <param name="args"> Command line arguments</param>
/// <returns> Program exit code (0 for success)</returns>
public static int Main(string[] args)
{
if (args.Contains("/?") || args.Length < 1 || args.Length > 10)
{
WriteUsageMessage();
return(1);
}
arguments = args;
try
{
if (version)
{
WriteVersion();
}
else if (upgrade)
{
UpgradeFile(fileName, recurse);
}
else if (edit)
{
EditFile(fileName, recurse);
}
else if (mergeDBFiles)
{
DBMerger.MergeFiles(fileName, recurse, Path.Combine(Path.GetDirectoryName(fileName), "merged.db"));
}
else
{
// Run simulations
var runner = new Runner(fileName, ignorePaths, recurse, runTests, runType,
numberOfProcessors: numberOfProcessors,
simulationNamePatternMatch: simulationNameRegex);
runner.SimulationCompleted += OnJobCompleted;
runner.SimulationGroupCompleted += OnSimulationGroupCompleted;
runner.AllSimulationsCompleted += OnAllJobsCompleted;
runner.Run();
// If errors occurred, write them to the console.
if (exitCode != 0)
{
Console.WriteLine("ERRORS FOUND!!");
}
if (verbose)
{
Console.WriteLine("Elapsed time was " + runner.ElapsedTime.TotalSeconds.ToString("F1") + " seconds");
}
}
}
catch (Exception err)
{
Console.WriteLine(err.ToString());
exitCode = 1;
}
return(exitCode);
}
public void DBsThatHaveTheSameSimulationsMergeCorrectly()
{
var database1 = new SQLite();
database1.OpenDatabase(":memory:", readOnly: false);
CreateTable(database1, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
});
CreateTable(database1, "Report",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 10.0, "str1" },
new object[] { 2, 11.0, "str2" }
});
var database2 = new SQLite();
database2.OpenDatabase(":memory:", readOnly: false);
CreateTable(database2, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder3" },
new object[] { 2, "Sim1", "Folder4" }
});
CreateTable(database2, "Report",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 20.0, "str3" },
new object[] { 2, 21.0, "str4" }
});
DBMerger.Merge(database2, database1);
Assert.IsTrue(
Utilities.CreateTable(new string[] { "ID", "Name", "FolderName" },
new List <object[]> {
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
})
.IsSame(Utilities.GetTableFromDatabase(database1, "_Simulations")));
Assert.IsTrue(
Utilities.CreateTable(new string[] { "SimulationID", "A", "B" },
new List <object[]> {
new object[] { 1, 10, "str1" },
new object[] { 1, 20, "str3" },
new object[] { 2, 11, "str2" },
new object[] { 2, 21, "str4" }
})
.IsSame(Utilities.GetTableFromDatabase(database1, "Report")));
}
public void DBsThatHaveTablesWithNoSimulationIDMergeCorrectly()
{
var database1 = new SQLite();
database1.OpenDatabase(":memory:", readOnly: false);
CreateTable(database1, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
});
CreateTable(database1, "_Units",
columnNames: new List <string> {
"TableName", "ColumnHeading", "Units"
},
columnTypes: new List <string> {
"string", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { "Report", "A", "g/m2" }
});
var database2 = new SQLite();
database2.OpenDatabase(":memory:", readOnly: false);
CreateTable(database2, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder3" },
new object[] { 2, "Sim1", "Folder4" }
});
CreateTable(database2, "_Units",
columnNames: new List <string> {
"TableName", "ColumnHeading", "Units"
},
columnTypes: new List <string> {
"string", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { "Report", "B", "kg/ha" }
});
DBMerger.Merge(database2, database1);
Assert.IsTrue(
Utilities.CreateTable(new string[] { "ID", "Name", "FolderName" },
new List <object[]> {
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
})
.IsSame(Utilities.GetTableFromDatabase(database1, "_Simulations")));
Assert.IsTrue(
Utilities.CreateTable(new string[] { "TableName", "ColumnHeading", "Units" },
new List <object[]> {
new object[] { "Report", "A", "g/m2" },
new object[] { "Report", "B", "kg/ha" }
})
.IsSame(Utilities.GetTableFromDatabase(database1, "_Units")));
}
public void DBsThatHaveTheSameSimulationsMergeCorrectly()
{
var database1 = new SQLite();
database1.OpenDatabase(":memory:", readOnly: false);
CreateTable(database1, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
});
CreateTable(database1, "Report",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 10.0, "str1" },
new object[] { 2, 11.0, "str2" }
});
var database2 = new SQLite();
database2.OpenDatabase(":memory:", readOnly: false);
CreateTable(database2, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder3" },
new object[] { 2, "Sim1", "Folder4" }
});
CreateTable(database2, "Report",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 20.0, "str3" },
new object[] { 2, 21.0, "str4" }
});
DBMerger.Merge(database2, database1);
Assert.AreEqual(Utilities.TableToString(database1, "_Simulations"),
$"ID,Name,FolderName{Environment.NewLine}" +
$" 1,Sim1, Folder1{Environment.NewLine}" +
$" 2,Sim2, Folder2{Environment.NewLine}");
Assert.AreEqual(Utilities.TableToString(database1, "Report"),
$"SimulationID, A, B{Environment.NewLine}" +
$" 1,10.000,str1{Environment.NewLine}" +
$" 1,20.000,str3{Environment.NewLine}" +
$" 2,11.000,str2{Environment.NewLine}" +
$" 2,21.000,str4{Environment.NewLine}");
}
public void DBsThatHaveTablesWithNoSimulationIDMergeCorrectly()
{
var database1 = new SQLite();
database1.OpenDatabase(":memory:", readOnly: false);
CreateTable(database1, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
});
CreateTable(database1, "_Units",
columnNames: new List <string> {
"TableName", "ColumnHeading", "Units"
},
columnTypes: new List <string> {
"string", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { "Report", "A", "g/m2" }
});
var database2 = new SQLite();
database2.OpenDatabase(":memory:", readOnly: false);
CreateTable(database2, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder3" },
new object[] { 2, "Sim1", "Folder4" }
});
CreateTable(database2, "_Units",
columnNames: new List <string> {
"TableName", "ColumnHeading", "Units"
},
columnTypes: new List <string> {
"string", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { "Report", "B", "kg/ha" }
});
DBMerger.Merge(database2, database1);
Assert.AreEqual(Utilities.TableToString(database1, "_Simulations"),
$"ID,Name,FolderName{Environment.NewLine}" +
$" 1,Sim1, Folder1{Environment.NewLine}" +
$" 2,Sim2, Folder2{Environment.NewLine}");
Assert.AreEqual(Utilities.TableToString(database1, "_Units"),
$"TableName,ColumnHeading,Units{Environment.NewLine}" +
$" Report, A, g/m2{Environment.NewLine}" +
$" Report, B,kg/ha{Environment.NewLine}");
}
public void DBThatHaveTheDifferentTablesMergeCorrectly()
{
var database1 = new SQLite();
database1.OpenDatabase(":memory:", readOnly: false);
CreateTable(database1, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim1", "Folder1" },
new object[] { 2, "Sim2", "Folder2" }
});
CreateTable(database1, "Report1",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 10.0, "str1" },
new object[] { 2, 11.0, "str2" }
});
var database2 = new SQLite();
database2.OpenDatabase(":memory:", readOnly: false);
CreateTable(database2, "_Simulations",
columnNames: new List <string> {
"ID", "Name", "FolderName"
},
columnTypes: new List <string> {
"int", "string", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, "Sim3", "Folder3" },
new object[] { 2, "Sim4", "Folder4" }
});
CreateTable(database2, "Report2",
columnNames: new List <string> {
"SimulationID", "A", "B"
},
columnTypes: new List <string> {
"int", "float", "string"
},
rowValues: new List <object[]>
{
new object[] { 1, 20.0, "str3" },
new object[] { 2, 21.0, "str4" }
});
DBMerger.Merge(database2, database1);
Assert.AreEqual(Utilities.TableToString(database1, "_Simulations"),
"ID,Name,FolderName\r\n" +
" 1,Sim1, Folder1\r\n" +
" 2,Sim2, Folder2\r\n" +
" 3,Sim3, Folder3\r\n" +
" 4,Sim4, Folder4\r\n");
Assert.AreEqual(Utilities.TableToString(database1, "Report1"),
"SimulationID, A, B\r\n" +
" 1,10.000,str1\r\n" +
" 2,11.000,str2\r\n");
Assert.AreEqual(Utilities.TableToString(database1, "Report2"),
"SimulationID, A, B\r\n" +
" 3,20.000,str3\r\n" +
" 4,21.000,str4\r\n");
}
/// <summary>
/// Run Models with the given set of options.
/// </summary>
/// <param name="options"></param>
public static void Run(Options options)
{
try
{
string[] files = options.Files.SelectMany(f => DirectoryUtilities.FindFiles(f, options.Recursive)).ToArray();
if (files == null || files.Length < 1)
{
throw new ArgumentException($"No files were specified");
}
if (options.NumProcessors == 0)
{
throw new ArgumentException($"Number of processors cannot be 0");
}
if (options.Upgrade)
{
foreach (string file in files)
{
UpgradeFile(file);
if (options.Verbose)
{
Console.WriteLine("Successfully upgraded " + file);
}
}
}
else if (options.ListSimulationNames)
{
foreach (string file in files)
{
ListSimulationNames(file, options.SimulationNameRegex);
}
}
else if (options.MergeDBFiles)
{
string[] dbFiles = files.Select(f => Path.ChangeExtension(f, ".db")).ToArray();
string outFile = Path.Combine(Path.GetDirectoryName(dbFiles[0]), "merged.db");
DBMerger.MergeFiles(dbFiles, outFile);
}
else
{
Runner runner;
if (string.IsNullOrEmpty(options.EditFilePath))
{
// Run simulations
runner = new Runner(files,
options.RunTests,
options.RunType,
numberOfProcessors: options.NumProcessors,
simulationNamePatternMatch: options.SimulationNameRegex);
}
else
{
runner = new Runner(files.Select(f => EditFile.Do(f, options.EditFilePath)),
true,
true,
options.RunTests,
runType: options.RunType,
numberOfProcessors: options.NumProcessors,
simulationNamePatternMatch: options.SimulationNameRegex);
}
runner.SimulationCompleted += OnJobCompleted;
if (options.Verbose)
{
runner.SimulationCompleted += WriteCompleteMessage;
}
if (options.ExportToCsv)
{
runner.SimulationGroupCompleted += OnSimulationGroupCompleted;
}
runner.AllSimulationsCompleted += OnAllJobsCompleted;
runner.Run();
// If errors occurred, write them to the console.
if (exitCode != 0)
{
Console.WriteLine("ERRORS FOUND!!");
}
if (options.Verbose)
{
Console.WriteLine("Elapsed time was " + runner.ElapsedTime.TotalSeconds.ToString("F1") + " seconds");
}
}
}
catch (Exception err)
{
Console.WriteLine(err.ToString());
exitCode = 1;
}
}
