//public async Task<string> WatchRunStatusAsync(Action<string> progressAction = default, System.Threading.CancellationToken cancelToken = default)
//{
// var api = new RunsApi();
// var proj = this.Project;
// var simuId = this.RunID;
// var run = api.GetRun(proj.Owner.Name, proj.Name, simuId);
// var status = run.Status;
// var startTime = status.StartedAt;
// while (status.FinishedAt <= status.StartedAt)
// {
// var currentSeconds = Math.Round((DateTime.UtcNow - startTime).TotalSeconds);
// // wait 5 seconds before calling api to re-check the status
// var isCreatedOrScheduled = status.Status == RunStatusEnum.Created || status.Status == RunStatusEnum.Scheduled;
// var totalDelaySeconds = isCreatedOrScheduled ? 3 : 5;
// for (int i = 0; i < totalDelaySeconds; i++)
// {
// // suspended by user
// cancelToken.ThrowIfCancellationRequested();
// progressAction?.Invoke($"{status.Status}: [{GetUserFriendlyTimeCounter(TimeSpan.FromSeconds(currentSeconds))}]");
// await Task.Delay(1000);
// currentSeconds++;
// }
// // suspended by user
// cancelToken.ThrowIfCancellationRequested();
// // update status
// await Task.Delay(1000);
// run = api.GetRun(proj.Owner.Name, proj.Name, simuId);
// status = run.Status;
// //_simulation = new Simulation(proj, simuId);
// }
// this.Run = run;
// // suspended by user
// cancelToken.ThrowIfCancellationRequested();
// var totalTime = status.FinishedAt - startTime;
// var finishMessage = status.Status.ToString();
// //progressAction?.Invoke($"Task: {status.Status}");
// finishMessage = $"{finishMessage}: [{GetUserFriendlyTimeCounter(totalTime)}]";
// progressAction?.Invoke(finishMessage);
// return finishMessage;
// string GetUserFriendlyTimeCounter(TimeSpan timeDelta)
// {
// string format = @"hh\:mm\:ss";
// if (timeDelta.Days > 0)
// format = @"d\ hh\:mm\:ss";
// else if (timeDelta.Hours > 0)
// format = @"hh\:mm\:ss";
// else if (timeDelta.Minutes > 0)
// format = @"mm\:ss";
// else
// format = @"ss";
// return timeDelta.ToString(format);
// }
//}
///// <summary>
///// Download all log for a simulation and combine it into one text format
///// </summary>
///// <param name="progressAction"></param>
///// <param name="cancelToken"></param>
///// <returns></returns>
//public async Task<string> GetSimulationOutputLogAsync(Action<string> progressAction = default, System.Threading.CancellationToken cancelToken = default)
//{
// // get task log ids
// if (cancelToken.IsCancellationRequested) return string.Empty;
// progressAction?.Invoke($"Getting log IDs");
// var proj = this.Project;
// var simuId = this.RunID;
// var api = new RunsApi();
// var job = api.GetRun(proj.Owner.Name, proj.Name, simuId);
// var status = job.Status;
// if (status.Status == "Running") throw new ArgumentException("Simulation is still running, please wait until it's done!");
// var taskDic = status.Steps.OrderBy(_ => _.Value.StartedAt).ToDictionary(_ => _.Key, _ => $"[{_.Key}]\n{_.Value.StartedAt.ToLocalTime()} : {_.Value.Name}");
// var taskIDs = taskDic.Keys;
// //Download file
// if (cancelToken.IsCancellationRequested) return string.Empty;
// progressAction?.Invoke($"Downloading logs");
// //var url = api.GetSimulationLogs(proj.Owner.Name, proj.Name, simuId).ToString();
// var url = "";
// if (string.IsNullOrEmpty(url)) throw new ArgumentNullException("Failed to call GetSimulationLogs");
// var dir = Path.Combine(Helper.GenTempFolder(), simuId);
// var downloadfile = await Helper.DownloadFromUrlAsync(url, dir);
// //unzip file
// if (cancelToken.IsCancellationRequested) return string.Empty;
// progressAction?.Invoke($"Reading logs");
// Helper.Unzip(downloadfile, dir, true);
// //read logs
// if (cancelToken.IsCancellationRequested) return string.Empty;
// var taskFiles = Directory.GetFiles(dir, "*.log", SearchOption.AllDirectories);
// var totalCount = taskIDs.Count;
// var current = 0;
// foreach (var logFile in taskFiles)
// {
// if (cancelToken.IsCancellationRequested) break;
// var logID = new DirectoryInfo(Path.GetDirectoryName(logFile)).Name;
// if (!taskIDs.Contains(logID)) continue;
// var logHeader = taskDic[logID];
// var logContent = File.ReadAllText(logFile);
// logContent = string.IsNullOrWhiteSpace(logContent) ? "No log available for this task." : logContent;
// taskDic[logID] = $"{logHeader} \n{logContent}";
// current++;
// progressAction?.Invoke($"Reading logs [{current}/{totalCount}]");
// }
// var fullLog = string.Join("\n\n", taskDic.Values);
// return fullLog;
//}
//private static async Task<string> DownloadFile(string url, string dir)
//{
// var request = new RestRequest(Method.GET);
// var client = new RestClient(url.ToString());
// var response = await client.ExecuteAsync(request);
// if (response.StatusCode != HttpStatusCode.OK)
// throw new Exception($"Unable to download file");
// // prep file path
// var fileName = Path.GetFileName(url).Split(new[] { '?' })[0];
// var tempDir = string.IsNullOrEmpty(dir) ? Path.Combine(Path.GetTempPath(), "Pollination", Path.GetRandomFileName()) : dir;
// Directory.CreateDirectory(tempDir);
// var file = Path.Combine(tempDir, fileName);
// var b = response.RawBytes;
// File.WriteAllBytes(file, b);
// if (!File.Exists(file)) throw new ArgumentException($"Failed to download {fileName}");
// return file;
//}
//private static void CheckOutputLogs(RunsApi api, Project proj, string simuId)
//{
// //var api = new RunsApi();
// var steps = api.GetRunSteps(proj.Owner.Name, proj.Name, simuId.ToString());
// foreach (var item in steps.Resources)
// {
// var stepLog = api.GetRunStepLogs(proj.Owner.Name, proj.Name, simuId.ToString(), item.Id);
// Console.WriteLine(stepLog);
// }
//}
/// <summary>
/// Load a RunInfo from a local run's folder.
/// This folder must contains recipe.json for RecipeInterface, and input.json for input arguments
/// </summary>
/// <param name="folder"></param>
/// <returns></returns>
public static RunInfo LoadFromLocalFolder(string folder)
{
RunInfo runInfo = null;
if (Directory.Exists(folder))
{
var recipeFile = Path.Combine(folder, "recipe.json");
var recipeJson = File.ReadAllText(recipeFile);
var recipe = RecipeInterface.FromJson(recipeJson);
runInfo = new RunInfo(recipe, folder);
}
return(runInfo);
}
public RunInfo GetRunInfo()
{
if (this.RunSource.StartsWith("LOCAL:"))
{
var folder = this.RunSource.Substring(6);
return(RunInfo.LoadFromLocalFolder(folder));
}
else
{
var run = GetRun();
var proj = GetRoject();
var runInfo = new RunInfo(proj, run);
return(runInfo);
}
}
public RunInfo GetRunInfo(int runIndex)
{
if (_runInfoCache.ContainsKey(runIndex))
{
return(_runInfoCache[runIndex]);
}
var jobInfo = this;
// only get the first run asset for now
var job = jobInfo.CloudJob;
//check run index if valid
var page = runIndex + 1;
var totalRuns = job.Status.RunsCompleted + job.Status.RunsFailed + job.Status.RunsPending + job.Status.RunsRunning;
if (totalRuns == 0)
{
throw new ArgumentException($"[Error] Job status: [{job.Status.Status}]. There is no run available in this job");
}
if (page > totalRuns)
{
throw new ArgumentException($"[Error] This job has {totalRuns} runs in total, a valid run index could from 0 to { totalRuns - 1};");
}
var api = new PollinationSDK.Api.RunsApi();
var runs = api.ListRuns(jobInfo.Project.Owner.Name, jobInfo.Project.Name, jobId: new List <string>()
{
job.Id
}, page: page, perPage: 1).Resources;
var firstRun = runs.FirstOrDefault();
var isRunFinished = firstRun.Status.FinishedAt > firstRun.Status.StartedAt;
if (!isRunFinished)
{
throw new ArgumentException($"[Warning] Run status: {firstRun.Status.Status}. If this run [{firstRun.Id.Substring(0, 5)}] is scheduled but not finished, please check it again in a few seconds;");
}
var runInfo = new RunInfo(jobInfo.Project, firstRun);
_runInfoCache.Add(runIndex, runInfo);
return(runInfo);
}
public ScheduledJobInfo(RunInfo localRun)
{
this.LocalJob = localRun;
this.Recipe = localRun.Recipe;
}
/// <summary>
/// Download all assets (input/output) with one call
/// </summary>
/// <param name="runInfo"></param>
/// <param name="saveAsDir"></param>
/// <returns></returns>
private static List <Task <string> > DownloadAssets(RunInfo runInfo, IEnumerable <RunAssetBase> assets, string saveAsDir, Action <string> reportProgressAction, System.Threading.CancellationToken cancelToken = default)
{
var tasks = new List <Task <string> >();
if (assets == null || !assets.Any())
{
return(tasks);
}
var api = new PollinationSDK.Api.RunsApi();
var inputDir = Path.Combine(saveAsDir, "inputs");
var outputDir = Path.Combine(saveAsDir, "outputs");
var total = assets.Count();
var completed = 0;
foreach (var asset in assets)
{
try
{
if (asset.IsPathAsset() && !asset.IsSaved())
{
var assetName = asset.Name;
var isInputAsset = asset is RunInputAsset;
var dir = isInputAsset ? inputDir : outputDir;
dir = Path.Combine(dir, assetName);
Action <int> individualProgress = (percent) => {
reportProgressAction?.Invoke($"{assetName}: {percent}%");
};
Action overAllProgress = () => {
completed++;
reportProgressAction?.Invoke($"OVERALL: {completed}/{total}");
};
var task = Task.Run(async() => {
var url = string.Empty;
if (isInputAsset)
{
url = api.DownloadRunArtifact(runInfo.Project.Owner.Name, runInfo.Project.Name, runInfo.RunID, path: asset.RelativePath).ToString();
}
else
{
url = api.GetRunOutput(runInfo.Project.Owner.Name, runInfo.Project.Name, runInfo.RunID, assetName).ToString();
}
Helper.Logger.Information($"DownloadAssets: downloading {assetName} from \n -{url}\n");
var t = Helper.DownloadUrlAsync(url, dir, individualProgress, overAllProgress, cancelToken);
await t.ConfigureAwait(false);
var path = t.Result;
Helper.Logger.Information($"DownloadAssets: saved {assetName} to {path}");
return(path);
});
tasks.Add(task);
}
else
{
tasks.Add(Task.Run(() => asset.LocalPath));
completed++;
}
}
catch (Exception e)
{
//canceled by user
if (e is OperationCanceledException)
{
return(null);
}
throw new ArgumentException($"Failed to download asset {asset.Name}.\n -{e.Message}");
}
}
return(tasks);
}
