public static async Task <CloudStampyParameters> BuildChanges([QueueTrigger("build-jobs", Connection = "StampyStorageConnectionString")] CloudStampyParameters request)
{
if (string.IsNullOrWhiteSpace(request.JobId))
{
request.JobId = Guid.NewGuid().ToString();
}
var serviceCreationJob = request.Copy();
if ((request.JobType & StampyJobType.Build) == StampyJobType.Build)
{
var result = await ExecuteJob(request, StampyJobType.Build, TimeSpan.FromMinutes(120));
if (result.ResultDetails.TryGetValue("Build Share", out object val))
{
serviceCreationJob.BuildPath = (string)val;
}
else
{
eventsLogger.WriteError("Build Share is empty");
}
serviceCreationJob.FlowStatus = result.JobStatus == Status.Passed ? Status.InProgress : result.JobStatus;
}
serviceCreationJob.JobId = Guid.NewGuid().ToString();
return(serviceCreationJob);
}
public static IJob GetJob(ICloudStampyLogger logger, CloudStampyParameters args, StampyJobType requestedJobType)
{
IJob job = null;
switch (requestedJobType)
{
case StampyJobType.None:
break;
case StampyJobType.CreateService:
job = new ServiceCreationJob(logger, args);
break;
case StampyJobType.Build:
job = new BuildJob(logger, args);
break;
case StampyJobType.Deploy:
job = new DeploymentJob(logger, args);
break;
case StampyJobType.Test:
job = new TestJob(logger, args);
break;
case StampyJobType.RemoveResources:
job = new ScavengerJob(logger, args);
break;
default:
break;
}
return(job);
}
public AntaresDeploymentBaseJob(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs)
{
_logger = logger;
_result = new JobResult();
Parameters = cloudStampyArgs;
_statusMessageBuilder = new StringBuilder();
}
public OneBranchBuildClient(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs)
{
_logger = logger;
_cloudStampyArgs = cloudStampyArgs;
_buildLogsWritter = new AzureFileLogger(new LoggingConfiguration(), cloudStampyArgs, logger);
_buildLogsWriterUnfinishedJobs = new List <Task>();
_runningProcess = new List <ProcessAction>();
}
public AzureFileLogger(IConfiguration configuration, CloudStampyParameters p, ICloudStampyLogger logger)
{
_loggingTasks = new ConcurrentQueue <Task>();
_kustoLogger = logger;
_parameters = p;
_configuration = configuration;
_requests = new ConcurrentQueue <string>();
LogUrls = new Dictionary <string, string>();
}
public ServiceCreationJob(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs)
{
_logger = logger;
_result = new JobResult();
_parameters = cloudStampyArgs;
_statusMessageBuilder = new StringBuilder();
_azureFilesWriter = new AzureFileLogger(new LoggingConfiguration(), cloudStampyArgs, logger);
_azureLogsWriterUnfinishedJobs = new List <Task>();
}
public DeploymentJob(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs)
{
_logger = logger;
_result = new JobResult();
_parameters = cloudStampyArgs;
_statusMessageBuilder = new StringBuilder();
_deploymentContent = new ConcurrentQueue <string>();
_loggingTasks = new List <Task>();
}
public BuildJob(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs)
{
_logger = logger;
_args = cloudStampyArgs;
if (!string.IsNullOrWhiteSpace(_args.DpkPath))
{
_buildClient = new LabMachineBuildClient(_logger, _args);
}
else
{
_buildClient = new OneBranchBuildClient(_logger, _args);
}
}
public static async Task <CloudStampyParameters> DeployToCloudService([QueueTrigger("deployment-jobs", Connection = "StampyStorageConnectionString")] CloudStampyParameters myQueueItem)
{
var nextJob = myQueueItem.Copy();
if ((myQueueItem.JobType & StampyJobType.Deploy) == StampyJobType.Deploy && (myQueueItem.FlowStatus == Status.InProgress || myQueueItem.FlowStatus == default(Status)))
{
var jobResult = await ExecuteJob(myQueueItem, StampyJobType.Deploy, TimeSpan.FromMinutes(240));
nextJob.FlowStatus = jobResult.JobStatus == Status.Passed ? Status.InProgress : jobResult.JobStatus;
}
nextJob.JobId = Guid.NewGuid().ToString();
return(nextJob);
}
public static async Task <CloudStampyParameters> TestRunner([QueueTrigger("test-jobs", Connection = "StampyStorageConnectionString")] CloudStampyParameters request)
{
var nextJob = request.Copy();
nextJob.JobId = Guid.NewGuid().ToString();
if ((request.JobType & StampyJobType.Test) == StampyJobType.Test && (request.FlowStatus == Status.InProgress || request.FlowStatus == default(Status)))
{
if (request.TestCategories.Any() && !string.IsNullOrWhiteSpace(request.BuildPath) && !string.IsNullOrWhiteSpace(request.CloudName))
{
var resultTasks = new List <Task <JobResult> >();
var results = new List <JobResult>();
foreach (var testCategorySet in request.TestCategories)
{
foreach (var testCategory in testCategorySet)
{
var testJob = request.Copy();
testJob.JobId = Guid.NewGuid().ToString();
testJob.TestCategories = new List <List <string> > {
new List <string> {
testCategory
}
};
//test should timeout after 180 minutes.
//TODO double check if function runtime has default timeout
resultTasks.Add(ExecuteJob(testJob, StampyJobType.Test, TimeSpan.FromHours(5)));
}
var testCategoryResults = await Task.WhenAll(resultTasks);
resultTasks.Clear();
results.AddRange(testCategoryResults);
}
nextJob.FlowStatus = JobStatusHelper.DetermineOverallJobStatus(results);
if ((nextJob.JobType & StampyJobType.RemoveResources) != StampyJobType.RemoveResources)
{
nextJob.JobType = nextJob.JobType | StampyJobType.RemoveResources;
}
nextJob.ExpiryDate = DateTime.UtcNow.AddHours(1).ToString();
}
}
return(nextJob);
}
public ScavengerJob(ICloudStampyLogger logger, CloudStampyParameters cloudStampyArgs) : base(logger, cloudStampyArgs)
{
}
public static ITestClient GetTestClient(ICloudStampyLogger logger, CloudStampyParameters args)
{
return(new LabMachineTestClient(logger, args));
}
public static async Task CreateCloudService([QueueTrigger("service-creation-jobs", Connection = "StampyStorageConnectionString")] CloudStampyParameters request, [Queue("deployment-jobs")] ICollector <CloudStampyParameters> deploymentJobsQueue)
{
JobResult result;
if ((request.JobType & StampyJobType.CreateService) == StampyJobType.CreateService && (request.FlowStatus == Status.InProgress || request.FlowStatus == default(Status)))
{
if (string.IsNullOrWhiteSpace(request.CloudName))
{
const string alphanumericalchars = "abcdefghijklmnopqrstuvwxyz1234567890";
var rng = new RNGCryptoServiceProvider();
byte[] xx = new byte[16];
rng.GetBytes(xx);
char[] y = new char[8];
for (int i = 0; i < y.Length; i++)
{
y[i] = alphanumericalchars[(xx[i] % alphanumericalchars.Length)];
}
request.CloudName = $"stampy-{new string(y)}";
}
result = await ExecuteJob(request, StampyJobType.CreateService, TimeSpan.FromMinutes(10));
if (result.JobStatus == Status.Passed)
{
//TODO check the deployment template set in the parameters. Use that to determine what kind of service to create
//ServiceCreator creates the service using SetupPrivateStampWithGeo which essentially creates two cloud services. So enqueue a job to deploy to stamp and geomaster
var geomasterJob = new CloudStampyParameters
{
RequestId = request.RequestId,
JobId = Guid.NewGuid().ToString(),
BuildPath = request.BuildPath,
CloudName = $"{request.CloudName}geo",
DeploymentTemplate = "GeoMaster_StompDeploy.xml",
JobType = request.JobType,
TestCategories = request.TestCategories,
FlowStatus = result.JobStatus == Status.Passed ? Status.InProgress : result.JobStatus
};
var stampJob = new CloudStampyParameters
{
RequestId = request.RequestId,
JobId = Guid.NewGuid().ToString(),
BuildPath = request.BuildPath,
CloudName = $"{request.CloudName}",
DeploymentTemplate = "Antares_StompDeploy.xml",
JobType = request.JobType,
TestCategories = request.TestCategories,
FlowStatus = result.JobStatus == Status.Passed ? Status.InProgress : result.JobStatus
};
deploymentJobsQueue.Add(geomasterJob);
deploymentJobsQueue.Add(stampJob);
}
}
else
{
var p = request.Copy();
p.JobId = Guid.NewGuid().ToString();
deploymentJobsQueue.Add(p);
}
}
private static async Task <JobResult> ExecuteJob(CloudStampyParameters queueItem, StampyJobType requestedJobType, TimeSpan timeout)
{
JobResult jobResult = null;
Exception jobException = null;
var sw = new Stopwatch();
var job = JobFactory.GetJob(eventsLogger, queueItem, requestedJobType);
if (job != null)
{
try
{
sw.Start();
eventsLogger.WriteInfo(queueItem, "start job");
var jobResultTask = Task.Run(() => job.Execute());
var timeoutTask = Task.Run(async() => await Task.Delay(timeout));
Task finishedTask;
while (true)
{
var timerTask = Task.Run(async() => await Task.Delay(TimeSpan.FromMinutes(1)));
finishedTask = await Task.WhenAny(new Task[] { jobResultTask, timerTask, timeoutTask });
//log the progress of the job
resultsLogger.WriteJobProgress(queueItem.RequestId, queueItem.JobId, requestedJobType, job.JobStatus.ToString(), job.ReportUri);
if (finishedTask.Id == jobResultTask.Id)
{
//job is done
jobResult = await jobResultTask;
break;
}
else if (finishedTask.Id == timeoutTask.Id)
{
jobResult = new JobResult {
JobStatus = Status.Cancelled
};
eventsLogger.WriteError(queueItem, $"Cancel job. {job.GetType().ToString()} took longer than timeout of {timeout.TotalMinutes.ToString()}mins");
var isCancelled = await job.Cancel();
if (!isCancelled)
{
eventsLogger.WriteError(queueItem, "Error while cancelling job");
}
else
{
eventsLogger.WriteInfo(queueItem, "Cancelled job successfully");
}
break;
}
}
sw.Stop();
}
catch (Exception ex)
{
jobException = ex;
jobResult = new JobResult {
JobStatus = Status.Failed
};
eventsLogger.WriteError(queueItem, "Error while running job", ex);
throw new JobExecutionException(job.GetType().Name, "Error while running job", ex);
}
finally
{
resultsLogger.WriteResult(queueItem.RequestId, queueItem.JobId, requestedJobType, jobResult.JobStatus.ToString(), (int)sw.Elapsed.TotalMinutes, job.ReportUri, jobException);
}
}
else
{
eventsLogger.WriteError(queueItem, "Cannot run this job");
}
return(jobResult);
}
public TestJob(ICloudStampyLogger logger, CloudStampyParameters args)
{
_logger = logger;
_args = args;
}
public void WriteError(CloudStampyParameters stampyParameters, string source, string message, Exception ex = null)
{
Write(stampyParameters.RequestId, stampyParameters.JobId, source, message, ex);
}
public void WriteInfo(CloudStampyParameters stampyParameters, string source, string message)
{
Write(stampyParameters.RequestId, stampyParameters.JobId, source, message, null);
}
public LabMachineBuildClient(ICloudStampyLogger logger, CloudStampyParameters cloudStampArgs)
{
_logger = logger;
_args = cloudStampArgs;
}
public LabMachineTestClient(ICloudStampyLogger logger, CloudStampyParameters args)
{
_logger = logger;
_args = args;
}
