public RunInfo GetRunInfo(int runIndex)
{
if (_runInfoCache.ContainsKey(runIndex))
{
return(_runInfoCache[runIndex]);
}
var jobInfo = this;
// only get the first run asset for now
var job = jobInfo.CloudJob;
//check run index if valid
var page = runIndex + 1;
var totalRuns = job.Status.RunsCompleted + job.Status.RunsFailed + job.Status.RunsPending + job.Status.RunsRunning;
if (totalRuns == 0)
{
throw new ArgumentException($"[Error] Job status: [{job.Status.Status}]. There is no run available in this job");
}
if (page > totalRuns)
{
throw new ArgumentException($"[Error] This job has {totalRuns} runs in total, a valid run index could from 0 to { totalRuns - 1};");
}
var api = new PollinationSDK.Api.RunsApi();
var runs = api.ListRuns(jobInfo.Project.Owner.Name, jobInfo.Project.Name, jobId: new List <string>()
{
job.Id
}, page: page, perPage: 1).Resources;
var firstRun = runs.FirstOrDefault();
var isRunFinished = firstRun.Status.FinishedAt > firstRun.Status.StartedAt;
if (!isRunFinished)
{
throw new ArgumentException($"[Warning] Run status: {firstRun.Status.Status}. If this run [{firstRun.Id.Substring(0, 5)}] is scheduled but not finished, please check it again in a few seconds;");
}
var runInfo = new RunInfo(jobInfo.Project, firstRun);
_runInfoCache.Add(runIndex, runInfo);
return(runInfo);
}
private static async Task <List <string> > DownloadSimulationInputAssets(RunInfo runInfo, string saveAsDir = default, Action <int> reportProgressAction = default)
{
try
{
var simuApi = new PollinationSDK.Api.RunsApi();
//var url = simuApi.GetSimulationInputs(runInfo.Project.Owner.Name, runInfo.Project.Name, runInfo.RunID).ToString();
var url = "";
var dir = string.IsNullOrEmpty(saveAsDir) ? GenTempFolder() : saveAsDir;
var simuID = runInfo.RunID.Substring(0, 8);
dir = Path.Combine(dir, simuID, "inputs");
// downloaded folder
return(await Download(url, dir, reportProgressAction));
}
catch (Exception)
{
throw;
}
}
private static void DownloadAssets(Project proj)
{
var runApi = new PollinationSDK.Api.RunsApi();
var firstRun = runApi.ListRuns(proj.Owner.Name, proj.Name, page: 1, perPage: 1).Resources.First();
var runInfo = new RunInfo(proj, firstRun);
// get all output assets to download
var assets = runInfo.GetOutputAssets("grasshopper").OfType <RunAssetBase>().ToList();
var inputAssets = runInfo.GetInputAssets();
assets = assets.Concat(inputAssets).ToList();
var savedPath = System.IO.Path.GetTempPath();
var task = Task.Run(async() => await DownloadAsync(runInfo, assets, savedPath));
var filePaths = task.GetAwaiter().GetResult();
foreach (var item in filePaths)
{
Console.WriteLine($"Asset: {item.Name}\nSaved Path: {item.LocalPath}");
}
}
/// <summary>
/// Download an artifact(file/folder) items independently.
/// </summary>
/// <param name="simu"></param>
/// <param name="artifact"></param>
/// <param name="saveAsDir"></param>
/// <returns></returns>
private static List <Task <string> > DownloadArtifactWithItems(RunInfo simu, DAGPathOutput artifact, string saveAsDir, Action <int> reportProgressAction)
{
var file = string.Empty;
var outputDirOrFile = string.Empty;
try
{
var api = new PollinationSDK.Api.RunsApi();
var files = api.ListRunArtifacts(simu.Project.Owner.Name, simu.Project.Name, simu.RunID, page: 1, perPage: 100);
var found = files.FirstOrDefault(_ => _.FileName == artifact.Name);
if (found == null)
{
throw new ArgumentException($"{artifact.Name} doesn't exist in {simu.Project.Owner.Name}/{simu.Project.Name}/{simu.RunID}");
}
var dir = string.IsNullOrEmpty(saveAsDir) ? GenTempFolder() : saveAsDir;
var simuID = simu.RunID.Substring(0, 8);
var tasks = new List <Task <string> >();
dir = Path.Combine(dir, simuID);
ListFilesFromFolder(ref tasks, dir, simu.Project.Owner.Name, simu.Project.Name, simu.RunID, found, api);
return(tasks);
}
catch (Exception e)
{
throw new ArgumentException($"Failed to download artifact {artifact.Name}.\n -{e.Message}");
}
// loop through files in folder
void ListFilesFromFolder(ref List <Task <string> > ts, string saveDir, string owner, string projName, string simuID, FileMeta artfact, RunsApi RunsApi)
{
var dir = saveDir;
var api = RunsApi;
if (artfact.Type == "file")
{
var url = api.DownloadRunArtifact(owner, projName, simuID, artfact.Key).ToString();
var task = DownloadUrlAsync(url, dir, reportProgressAction);
ts.Add(task);
}
else if (artfact.Type == "folder")
{
dir = Path.Combine(dir, artfact.FileName);
var files = api.ListRunArtifacts(owner, projName, simuID, page: 1, perPage: 100, path: new[] { artifact.Name }.ToList());
foreach (var item in files)
{
// get all files in folder
if (item.Type == "folder")
{
ListFilesFromFolder(ref ts, dir, owner, projName, simuID, item, api);
continue;
}
// item is file
var url = api.DownloadRunArtifact(owner, projName, simuID, item.Key).ToString();
var task = DownloadUrlAsync(url, dir, reportProgressAction);
ts.Add(task);
}
}
}
}
/// <summary>
/// Download all assets (input/output) with one call
/// </summary>
/// <param name="runInfo"></param>
/// <param name="saveAsDir"></param>
/// <returns></returns>
private static List <Task <string> > DownloadAssets(RunInfo runInfo, IEnumerable <RunAssetBase> assets, string saveAsDir, Action <string> reportProgressAction, System.Threading.CancellationToken cancelToken = default)
{
var tasks = new List <Task <string> >();
if (assets == null || !assets.Any())
{
return(tasks);
}
var api = new PollinationSDK.Api.RunsApi();
var inputDir = Path.Combine(saveAsDir, "inputs");
var outputDir = Path.Combine(saveAsDir, "outputs");
var total = assets.Count();
var completed = 0;
foreach (var asset in assets)
{
try
{
if (asset.IsPathAsset() && !asset.IsSaved())
{
var assetName = asset.Name;
var isInputAsset = asset is RunInputAsset;
var dir = isInputAsset ? inputDir : outputDir;
dir = Path.Combine(dir, assetName);
Action <int> individualProgress = (percent) => {
reportProgressAction?.Invoke($"{assetName}: {percent}%");
};
Action overAllProgress = () => {
completed++;
reportProgressAction?.Invoke($"OVERALL: {completed}/{total}");
};
var task = Task.Run(async() => {
var url = string.Empty;
if (isInputAsset)
{
url = api.DownloadRunArtifact(runInfo.Project.Owner.Name, runInfo.Project.Name, runInfo.RunID, path: asset.RelativePath).ToString();
}
else
{
url = api.GetRunOutput(runInfo.Project.Owner.Name, runInfo.Project.Name, runInfo.RunID, assetName).ToString();
}
Helper.Logger.Information($"DownloadAssets: downloading {assetName} from \n -{url}\n");
var t = Helper.DownloadUrlAsync(url, dir, individualProgress, overAllProgress, cancelToken);
await t.ConfigureAwait(false);
var path = t.Result;
Helper.Logger.Information($"DownloadAssets: saved {assetName} to {path}");
return(path);
});
tasks.Add(task);
}
else
{
tasks.Add(Task.Run(() => asset.LocalPath));
completed++;
}
}
catch (Exception e)
{
//canceled by user
if (e is OperationCanceledException)
{
return(null);
}
throw new ArgumentException($"Failed to download asset {asset.Name}.\n -{e.Message}");
}
}
return(tasks);
}
