public void TestConnectionWrapperCheckValuesAre()
{
var wrapper = new ConnectionWrapper();
Mock <IConfiguration> mockAStorage = new Mock <IConfiguration>();
APIConnectionInformation info = new APIConnectionInformation();
info.Token = "Token";
info.ApiUri = "https://apiuri.com";
info.StorageUri = "https://storageuri.com";
info.AccountEmail = "*****@*****.**";
info.ForcePathStyle = true;
info.StorageUnsafeSsl = true;
mockAStorage.Setup(foo => foo.ApiConnection).Returns(info);
IConfiguration config = mockAStorage.Object;
QarnotSDK.Connection connection = wrapper.CreateConnection(config);
Assert.AreEqual("Token", connection.Token);
Assert.AreEqual(new Uri("https://apiuri.com/"), connection.Uri);
Assert.AreEqual(new Uri("https://storageuri.com/"), connection.StorageUri);
Assert.AreEqual(true, connection.ForceStoragePathStyle);
Assert.AreEqual("*****@*****.**", connection.StorageAccessKey);
Assert.IsInstanceOf <UnsafeS3HttpClientFactory>(connection.S3HttpClientFactory);
}
private QPool CreatePool(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
QarnotSDK.QPool pool = new QarnotSDK.QPool(connect, config.Name, config.Profile, config.InstanceCount, config.Shortname, config.TasksDefaultWaitForPoolResourcesSynchronization);
pool.SetTags(config.Tags.ToArray());
pool.Resources = this.Tools.CreateResources(config.Resources, connect, ct);
foreach (var item in this.Tools.CreateConstants(config.Constants))
{
pool.SetConstant(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateConstraints(config.Constraints))
{
pool.SetConstraint(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateLabels(config.Labels))
{
pool.SetLabel(item.Key, item.Value);
}
pool.IsElastic = config.IsElastic;
pool.ElasticMinimumTotalNodes = config.ElasticMinimumTotalNodes == default(int) ? pool.ElasticMinimumTotalNodes : config.ElasticMinimumTotalNodes;
pool.ElasticMaximumTotalNodes = config.ElasticMaximumTotalNodes == default(int) ? pool.ElasticMaximumTotalNodes : config.ElasticMaximumTotalNodes;
pool.ElasticMinimumIdlingNodes = config.ElasticMinimumIdlingNodes == default(int) ? pool.ElasticMinimumIdlingNodes : config.ElasticMinimumIdlingNodes;
pool.ElasticResizePeriod = config.ElasticResizePeriod == default(int) ? pool.ElasticResizePeriod : config.ElasticResizePeriod;
pool.ElasticResizeFactor = config.ElasticResizeFactor == default(int) ? pool.ElasticResizeFactor : config.ElasticResizeFactor;
pool.ElasticMinimumIdlingTime = config.ElasticMinimumIdlingTime == default(int) ? pool.ElasticMinimumIdlingTime : config.ElasticMinimumIdlingTime;
CLILogs.Info("create pool");
return(pool);
}
public QAbstractStorage CreateResult(string name, QarnotSDK.Connection connection, CancellationToken ct)
{
if (!string.IsNullOrEmpty(name))
{
return(connection.CreateBucketAsync(name, ct).Result as QAbstractStorage);
}
return(null);
}
private async Task <QBucket> CreateBucket(BucketConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
var bucket = await connect.CreateBucketAsync(config.Name, ct);
if (config.CacheTTL.HasValue)
{
bucket.WithCacheTTL(config.CacheTTL.Value);
}
return(bucket);
}
private async Task <QTask> CreateTask(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
QarnotSDK.QTask task = null;
if (config.InstanceCount > 0)
{
task = await this.CreateInstanceTaskAsync(config, connect);
}
else
{
task = await this.CreateRangeTaskAsync(config, connect);
}
task.SetTags(config.Tags.ToArray());
task.Resources = this.Tools.CreateResources(config.Resources, connect, ct);
task.Results = this.Tools.CreateResult(config.Result, connect, ct);
this.Tools.CreateConstants(config.Constants).ToList().ForEach(item => task.SetConstant(item.Key, item.Value));
foreach (var item in this.Tools.CreateConstraints(config.Constraints))
{
task.SetConstraint(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateLabels(config.Labels))
{
task.SetLabel(item.Key, item.Value);
}
task.SetTaskDependencies(config.Dependents.Select(id => new Guid(id)).ToArray());
task.ResultsWhitelist = config.Whitelist;
task.ResultsBlacklist = config.Blacklist;
if (config.MaxRetriesPerInstance.HasValue)
{
task.MaxRetriesPerInstance = config.MaxRetriesPerInstance.Value;
}
if (config.ExportApiAndStorageCredentialsInEnvironment.HasValue)
{
task.Privileges.ExportApiAndStorageCredentialsInEnvironment = config.ExportApiAndStorageCredentialsInEnvironment.Value;
}
if (config.DefaultResourcesCacheTTLSec.HasValue)
{
task.DefaultResourcesCacheTTLSec = config.DefaultResourcesCacheTTLSec.Value;
}
CLILogs.Info("create task");
return(task);
}
private QPool CreatePool(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
QarnotSDK.QPool pool = new QarnotSDK.QPool(connect, config.Name, config.Profile, config.InstanceCount, config.Shortname, config.TasksDefaultWaitForPoolResourcesSynchronization);
pool.SetTags(config.Tags.ToArray());
pool.Resources = this.Tools.CreateResources(config.Resources, connect, ct);
foreach (var item in this.Tools.CreateConstants(config.Constants))
{
pool.SetConstant(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateConstraints(config.Constraints))
{
pool.SetConstraint(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateLabels(config.Labels))
{
pool.SetLabel(item.Key, item.Value);
}
pool.IsElastic = config.IsElastic;
pool.ElasticMinimumTotalSlots = config.ElasticMinimumTotalSlots == default(int) ? pool.ElasticMinimumTotalSlots : config.ElasticMinimumTotalSlots;
pool.ElasticMaximumTotalSlots = config.ElasticMaximumTotalSlots == default(int) ? pool.ElasticMaximumTotalSlots : config.ElasticMaximumTotalSlots;
pool.ElasticMinimumIdlingSlots = config.ElasticMinimumIdlingSlots == default(int) ? pool.ElasticMinimumIdlingSlots : config.ElasticMinimumIdlingSlots;
pool.ElasticResizePeriod = config.ElasticResizePeriod == default(int) ? pool.ElasticResizePeriod : config.ElasticResizePeriod;
pool.ElasticResizeFactor = config.ElasticResizeFactor == default(int) ? pool.ElasticResizeFactor : config.ElasticResizeFactor;
pool.ElasticMinimumIdlingTime = config.ElasticMinimumIdlingTime == default(int) ? pool.ElasticMinimumIdlingTime : config.ElasticMinimumIdlingTime;
if (config.ExportApiAndStorageCredentialsInEnvironment.HasValue)
{
pool.Privileges.ExportApiAndStorageCredentialsInEnvironment = config.ExportApiAndStorageCredentialsInEnvironment.Value;
}
if (config.DefaultResourcesCacheTTLSec.HasValue)
{
pool.DefaultResourcesCacheTTLSec = config.DefaultResourcesCacheTTLSec.Value;
}
CLILogs.Info("create pool");
return(pool);
}
private async Task <QJob> CreateJobAsync(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
CLILogs.Info("create job");
QPool pool = null;
if (!string.IsNullOrEmpty(config.PoolUuidOrShortname))
{
pool = await connect.RetrievePoolByUuidAsync(config.PoolUuidOrShortname, ct);
}
QJob job = new QJob(connect, config.Name, pool, config.Shortname, config.IsDependents);
if (config.MaximumWallTime.HasValue && config.MaximumWallTime.Value != default(TimeSpan))
{
job.MaximumWallTime = config.MaximumWallTime.Value;
}
return(job);
}
private async Task <QTask> CreateTask(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
QarnotSDK.QTask task = null;
if (config.InstanceCount > 0)
{
task = await this.CreateInstanceTaskAsync(config, connect);
}
else
{
task = await this.CreateRangeTaskAsync(config, connect);
}
task.SetTags(config.Tags.ToArray());
task.Resources = this.Tools.CreateResources(config.Resources, connect, ct);
task.Results = this.Tools.CreateResult(config.Result, connect, ct);
this.Tools.CreateConstants(config.Constants).ToList().ForEach(item => task.SetConstant(item.Key, item.Value));
foreach (var item in this.Tools.CreateConstraints(config.Constraints))
{
task.SetConstraint(item.Key, item.Value);
}
foreach (var item in this.Tools.CreateLabels(config.Labels))
{
task.SetLabel(item.Key, item.Value);
}
task.SetTaskDependencies(config.Dependents.Select(id => new Guid(id)).ToArray());
task.ResultsWhitelist = config.Whitelist;
task.ResultsBlacklist = config.Blacklist;
CLILogs.Info("create task");
return(task);
}
private async Task <QTask> CreateRangeTaskAsync(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct = default(CancellationToken))
{
AdvancedRanges range = new AdvancedRanges(config.Range);
if (!string.IsNullOrEmpty(config.JobUuidOrShortname))
{
QarnotSDK.QJob job;
if (Guid.TryParse(config.JobUuidOrShortname, out var jobUuid))
{
job = await connect.RetrieveJobByUuidAsync(config.JobUuidOrShortname, ct);
}
else
{
job = await connect.RetrieveJobByShortnameAsync(config.JobUuidOrShortname, ct);
}
return(new QarnotSDK.QTask(connect, config.Name, job, range, config.Shortname));
}
else if (!string.IsNullOrEmpty(config.PoolUuidOrShortname))
{
QarnotSDK.QPool pool;
if (Guid.TryParse(config.PoolUuidOrShortname, out var poolUuid))
{
pool = new QarnotSDK.QPool(connect, poolUuid);
}
else
{
pool = await connect.RetrievePoolByShortnameAsync(config.PoolUuidOrShortname, ct);
}
return(new QarnotSDK.QTask(connect, config.Name, pool, range, config.Shortname, config.WaitForPoolResourcesSynchronization));
}
return(new QarnotSDK.QTask(connect, config.Name, config.Profile, range, config.Shortname));
}
public async Task <List <QarnotSDK.QPool> > RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
DefaultRunConfiguration config = configuration as DefaultRunConfiguration;
QarnotSDK.QPool pool = null;
List <QPool> listPool = null;
if (!string.IsNullOrEmpty(config.Name))
{
CLILogs.Debug("Retrieve QPools by name : " + config.Name);
pool = await connection.RetrievePoolByNameAsync(config.Name, cancellationToken : ct);
}
else if (!string.IsNullOrEmpty(config.Id))
{
CLILogs.Debug("Retrieve QPool by Uuid : " + config.Id);
pool = await connection.RetrievePoolByUuidAsync(config.Id, cancellationToken : ct);
}
else if (config.Tags != null && config.Tags.Count > 0)
{
if (config.TagsIntersect)
{
CLILogs.Debug("Retrieve QPools by Tags Intersect : " + config.Tags.ToString());
var poolTagFilter = new QDataDetail <QPool>();
var filterList = config.Tags.Select(tag => QFilter <QPool> .Contains(t => t.Tags, tag));
listPool = await connection.RetrievePoolsAsync(poolTagFilter, cancellationToken : ct);
}
else
{
CLILogs.Debug("Retrieve QPools by Tags : " + config.Tags.ToString());
listPool = await connection.RetrievePoolsByTagsAsync(config.Tags, cancellationToken : ct);
}
}
else
{
CLILogs.Debug("Retrieve all the QPools");
listPool = await connection.RetrievePoolsAsync(cancellationToken : ct);
}
return(listPool ?? new List <QPool>()
{
pool
});
}
public async Task <List <QarnotSDK.QBucket> > RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
BucketConfiguration config = configuration as BucketConfiguration;
if (config != null && !string.IsNullOrEmpty(config.Name))
{
CLILogs.Debug("Retrieve QBucket by Name : " + config.Name);
return(new List <QBucket>()
{
await connection.RetrieveBucketAsync(config.Name, cancellationToken : ct),
});
}
else
{
CLILogs.Debug("Retrieve all the QBuckets");
return(await connection.RetrieveBucketsAsync(cancellationToken : ct));
}
}
public async Task <List <QarnotSDK.QTask> > RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
DefaultRunConfiguration config = configuration as DefaultRunConfiguration;
QarnotSDK.QTask task = null;
List <QTask> listTask = null;
if (!string.IsNullOrEmpty(config.Name))
{
CLILogs.Debug("Retrieve QTasks by Name : " + config.Name);
task = await connection.RetrieveTaskByNameAsync(config.Name, cancellationToken : ct);
}
else if (!string.IsNullOrEmpty(config.Id))
{
CLILogs.Debug("Retrieve QTask by Uuid : " + config.Id);
task = await connection.RetrieveTaskByUuidAsync(config.Id, cancellationToken : ct);
}
else if (config.Tags != null && config.Tags.Count > 0)
{
if (config.TagsIntersect)
{
var taskTagFilter = new QDataDetail <QTask>();
var filterList = config.Tags.Select(tag => QFilter <QTask> .Contains(t => t.Tags, tag));
taskTagFilter.Filter = QFilter <QTask> .And(filterList.ToArray());
listTask = await connection.RetrieveTasksAsync(taskTagFilter, cancellationToken : ct);
}
else
{
CLILogs.Debug("Retrieve QTasks by Tags : " + config.Tags.ToString());
listTask = await connection.RetrieveTasksByTagsAsync(config.Tags, cancellationToken : ct);
}
}
else
{
CLILogs.Debug("Retrieve all the QTasks");
listTask = await connection.RetrieveTasksAsync(cancellationToken : ct);
}
return(listTask ?? new List <QTask>()
{
task
});
}
private async Task <QBucket> CreateBucket(BucketConfiguration config, QarnotSDK.Connection connect, CancellationToken ct)
{
return(await connect.CreateBucketAsync(config.Name, ct));
}
private async Task <QTask> CreateInstanceTaskAsync(CreateConfiguration config, QarnotSDK.Connection connect, CancellationToken ct = default(CancellationToken))
{
if (!string.IsNullOrEmpty(config.JobUuid))
{
QarnotSDK.QJob job = await connect.RetrieveJobByUuidAsync(config.JobUuid, ct);
return(new QarnotSDK.QTask(connect, config.Name, job, config.InstanceCount, config.Shortname));
}
else if (!string.IsNullOrEmpty(config.PoolUuid))
{
QarnotSDK.QPool pool = new QarnotSDK.QPool(connect, new Guid(config.PoolUuid));
return(new QarnotSDK.QTask(connect, config.Name, pool, config.InstanceCount, config.Shortname, config.WaitForPoolResourcesSynchronization));
}
return(new QarnotSDK.QTask(connect, config.Name, config.Profile, config.InstanceCount, config.Shortname));
}
public async override Task <CommandValues.GenericInfoCommandValue> Create(IConfiguration iconfig, QarnotSDK.Connection connect, CancellationToken ct)
{
CreateConfiguration config = iconfig as CreateConfiguration;
QTask task = await this.CreateTask(config, connect, ct);
await this.LaunchTaskAsync(task, ct);
await this.LaunchPostSubmitTaskMethodsAsync(task, config, ct);
return(this.PrintTaskInformation(task));
}
public async Task <QarnotSDK.QBucket> RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
BucketConfiguration config = configuration as BucketConfiguration;
CLILogs.Debug("Retrieve QBucket by Name : " + config.Name);
return(await connection.RetrieveBucketAsync(config.Name, ct));
}
public abstract Task <CommandValues.GenericInfoCommandValue> Create(IConfiguration iconfig, QarnotSDK.Connection connect, CancellationToken ct);
public async Task <QarnotSDK.UserInformation> RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
AccountConfiguration config = configuration as AccountConfiguration;
return(await connection.RetrieveUserInformationAsync(config.GetBucket, cancellationToken : ct));
}
public async override Task <CommandValues.GenericInfoCommandValue> Create(IConfiguration iconfig, QarnotSDK.Connection connect, CancellationToken ct)
{
CreateConfiguration config = iconfig as CreateConfiguration;
QJob job = await this.CreateJobAsync(config, connect, ct);
await this.LaunchJobAsync(job, ct);
return(this.PrintJobInformation(job));
}
public async Task <QarnotSDK.ApiSettings> RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
return(await connection.RetrieveApiSettingsAsync(cancellationToken : ct));
}
public async override Task <CommandValues.GenericInfoCommandValue> Create(IConfiguration iconfig, QarnotSDK.Connection connect, CancellationToken ct)
{
BucketConfiguration config = iconfig as BucketConfiguration;
if (!config.ApiConnection.DisableBucketPathsSanitization &&
(PathSanitization.IsThePathInvalid(config.RemoteRelativePath)))
{
return(new CommandValues.GenericInfoCommandValue()
{
Uuid = config.Name,
Message = "Creation failed. Invalid remote path",
});
}
QBucket bucket = await this.CreateBucket(config, connect, ct);
await this.LaunchBucket(bucket, ct);
List <Task> listOfTask = new List <Task>();
listOfTask.AddRange(config.LocalPathFiles.Select(localFile => bucket.UploadFileAsync(localFile, config.RemoteRelativePath, cancellationToken: ct)));
listOfTask.AddRange(config.LocalPathFolders.Select(localFolder => bucket.UploadFolderAsync(localFolder, config.RemoteRelativePath, cancellationToken: ct)));
await Task.WhenAll(listOfTask);
return(this.PrintBucketInformation(bucket));
}
public async Task <List <QarnotSDK.QJob> > RetrieveAsync(IConfiguration configuration, QarnotSDK.Connection connection, CancellationToken ct)
{
DefaultRunConfiguration config = configuration as DefaultRunConfiguration;
QarnotSDK.QJob job = null;
List <QJob> listJob = null;
if (!string.IsNullOrEmpty(config.Name))
{
CLILogs.Debug("Retrieve QJobs by name : " + config.Name);
QDataDetail <QJob> level = new QDataDetail <QJob>()
{
Filter = QFilter <QJob> .Eq(t => t.Name, config.Name),
};
listJob = await connection.RetrieveJobsAsync(level, cancellationToken : ct);
}
else if (!string.IsNullOrEmpty(config.Id))
{
CLILogs.Debug("Retrieve QJob by Uuid : " + config.Id);
job = await connection.RetrieveJobByUuidAsync(config.Id, cancellationToken : ct);
}
else
{
CLILogs.Debug("Retrieve all the QJobs");
listJob = await connection.RetrieveJobsAsync(cancellationToken : ct);
}
return(listJob ?? new List <QJob>()
{
job
});
}
public async override Task <CommandValues.GenericInfoCommandValue> Create(IConfiguration iconfig, QarnotSDK.Connection connect, CancellationToken ct)
{
CreateConfiguration config = iconfig as CreateConfiguration;
QarnotSDK.QPool pool = this.CreatePool(config, connect, ct);
await this.LaunchPoolAsync(pool, ct);
return(this.PrintPoolInformation(pool));
}
public List <QAbstractStorage> CreateResources(List <string> names, QarnotSDK.Connection connection, CancellationToken ct)
{
return(names.Select(name => connection.CreateBucketAsync(name, ct).Result as QAbstractStorage).ToList());
}