public void OneWithJobDescriptor()
{
CloudQueueMessage msg = queue.GetMessage();
Assert.That(msg, Is.Null);
JobDescriptor descriptor = factory.CreateDescriptor(new HelloWho()
{
Who = "ProducerTest-OneWithJobDescriptor"
});
producer.One(descriptor);
msg = queue.GetMessage();
Assert.That(msg, Is.Not.Null);
Assert.That(msg.AsString, Is.EqualTo("{\"Job\":\"HelloWho\",\"Properties\":{\"Who\":\"ProducerTest-OneWithJobDescriptor\"}}"));
descriptor = JsonConvert.DeserializeObject <JobDescriptor>(msg.AsString);
Assert.That(descriptor, Is.Not.Null.
And.Property("Job").EqualTo("HelloWho").
And.Property("Properties").Count.EqualTo(1));
Assert.That(descriptor.Properties["Who"].ToObject <string>(), Is.EqualTo("ProducerTest-OneWithJobDescriptor"));
queue.DeleteMessage(msg);
}
public void Roundtrip()
{
MockJob job = new MockJob()
{
Id = Guid.NewGuid(), Int32 = 987
};
JobDescriptor desc = factory.CreateDescriptor(job);
JToken token;
Assert.That(desc, Is.Not.Null);
Assert.That(desc.Job, Is.EqualTo("MockJob"));
Assert.That(desc.Properties, Is.Not.Null.And.Count.EqualTo(2));
Assert.That(desc.Properties.TryGetValue("Id", out token), Is.True);
Assert.That(token, Is.TypeOf <JValue>().And.Property("Type").EqualTo(JTokenType.Guid));
Assert.That(token.ToObject <Guid>(), Is.EqualTo(job.Id));
Assert.That(desc.Properties.TryGetValue("Int32", out token), Is.True);
Assert.That(token, Is.TypeOf <JValue>().And.Property("Type").EqualTo(JTokenType.Integer));
Assert.That(token.ToObject <Int32>(), Is.EqualTo(987));
Assert.That(desc.QueueMessageId, Is.Null);
IJob job2 = factory.CreateJob(desc);
Assert.That(job2, Is.Not.Null.And.TypeOf <MockJob>());
MockJob mockJob2 = (MockJob)job2;
Assert.That(mockJob2.Id, Is.EqualTo(job.Id));
}
private JobDescriptor Enqueue(JobCall job, JobStatus status, DateTime?scheduled, TimeZoneInfo?timeZone = null)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
if (status == JobStatus.Waiting && !scheduled.HasValue)
{
throw new ArgumentNullException(nameof(scheduled));
}
int jobId = ++_idSource;
var jobDescriptor = new JobDescriptor
{
Id = jobId.ToString(),
Status = status,
Scheduled = scheduled,
TimeZone = timeZone,
Call = job
};
lock (_syncLock)
{
_jobs.Add(jobDescriptor);
}
return(jobDescriptor);
}
public DetectAndHashJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
base(jobDescription, workerId, locator)
{
_codeLocationRepository = _serviceLocator.Locate<ICodeLocationRepository>();
_fileRepository = _serviceLocator.Locate<IFileRepository>();
_repositoryController = _serviceLocator.Locate<IRepositoryController>();
}
public DetectAndHashJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
base(jobDescription, workerId, locator)
{
_codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
_fileRepository = _serviceLocator.Locate <IFileRepository>();
_repositoryController = _serviceLocator.Locate <IRepositoryController>();
}
/// <summary>
/// Initializes a new instance of the <see cref="NullJob"/> class.
/// </summary>
/// <param name="jobDescriptor">The job descriptor.</param>
public NullJob(JobDescriptor jobDescriptor)
{
Url = jobDescriptor.Url;
Name = jobDescriptor.Name;
Status = jobDescriptor.Status;
Description = "Unable to load this job";
}
public void OneWithInitialVisibilityDelay()
{
TimeSpan visibilityDelay = TimeSpan.FromSeconds(1);
CloudQueueMessage msg = queue.GetMessage();
Assert.That(msg, Is.Null);
producer.One(new HelloWho()
{
Who = "ProducerTest-OneWithInitialVisibilityDelay"
}, visibilityDelay);
msg = queue.GetMessage();
Assert.That(msg, Is.Null);
Thread.Sleep(visibilityDelay);
msg = queue.GetMessage();
Assert.That(msg, Is.Not.Null);
Assert.That(msg.AsString, Is.EqualTo("{\"Job\":\"HelloWho\",\"Properties\":{\"Who\":\"ProducerTest-OneWithInitialVisibilityDelay\"}}"));
JobDescriptor descriptor = JsonConvert.DeserializeObject <JobDescriptor>(msg.AsString);
Assert.That(descriptor, Is.Not.Null.
And.Property("Job").EqualTo("HelloWho").
And.Property("Properties").Count.EqualTo(1));
Assert.That(descriptor.Properties["Who"].ToObject <string>(), Is.EqualTo("ProducerTest-OneWithInitialVisibilityDelay"));
queue.DeleteMessage(msg);
}
public Job(JobDescriptor jobDescription, Guid workerId, ILocator locator)
{
_serviceLocator = locator;
_jobRepository = locator.Locate<IJobRepository>();
_apiRepository = locator.Locate<IApiDataRepository>();
Id = jobDescription.Id;
WorkerId = workerId;
JobDescriptor = jobDescription;
}
public Job(JobDescriptor jobDescription, Guid workerId, ILocator locator)
{
_serviceLocator = locator;
_jobRepository = locator.Locate <IJobRepository>();
_apiRepository = locator.Locate <IApiDataRepository>();
Id = jobDescription.Id;
WorkerId = workerId;
JobDescriptor = jobDescription;
}
public void CanGetNextJobWhenEmpty()
{
// Arrange
var queue = serviceProviderMock.GetService <ITimeBasedQueue>();
// Act
JobDescriptor job = queue.Next();
// Assert
}
public static IJob CreateJob(JobDescriptor description, WorkerState worker, ILocator locator)
{
var jobType = Type.GetType(description.JobType);
if (jobType == null)
{
throw new TypeAccessException("Could not create a type from " + description.JobType);
}
var instance = Activator.CreateInstance(jobType, description, worker.Id, locator) as IJob;
return instance;
}
public static IJob CreateJob(JobDescriptor description, WorkerState worker, ILocator locator)
{
var jobType = Type.GetType(description.JobType);
if (jobType == null)
{
throw new TypeAccessException("Could not create a type from " + description.JobType);
}
var instance = Activator.CreateInstance(jobType, description, worker.Id, locator) as IJob;
return(instance);
}
public Task <string> EnqueueAsync(JobCall job)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
JobDescriptor jobDescriptor = Enqueue(job, JobStatus.Ready, null);
_queue.Enqueue(jobDescriptor);
_signal.Release();
return(Task.FromResult(jobDescriptor.Id));
}
public Task <string> ScheduleAsync(JobCall job, DateTime scheduled, TimeZoneInfo?timeZone = null)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
JobDescriptor jobDescriptor = Enqueue(job, JobStatus.Waiting, scheduled, timeZone);
// Refresh the queue observer.
_signal.Release();
return(Task.FromResult(jobDescriptor.Id));
}
public JsonResult List(string[] correlation)
{
var expected_correlation = GetExpectedCorrelation(correlation);
var model = new FunctionalTestRunnerModel(null);
var descriptor = new JobDescriptor
{
Name = string.Format("Functional Tests {0}", expected_correlation),
Runs = GetTests(model),
UserAgentNames = ie9Only,
};
return(Json(descriptor, JsonRequestBehavior.AllowGet));
}
#pragma warning disable IDE0051 // Remove unused private members
private static void SubmitNewJob()
{
#pragma warning restore IDE0051 // Remove unused private members
NewJobSubmitter newJobSubmitter = new NewJobSubmitter {
Command = "batch.py",
UserStandardOutputFileName = "stdout.txt",
TargetOperatingSystem = TargetOperatingSystem.ANY
};
JobDescriptor jobDescriptor = newJobSubmitter.SubmitNewJob();
Log("Job submitted");
jobDescriptor.JobStartedEvent.WaitOne();
Log("Job started");
jobDescriptor.JobCompletedEvent.WaitOne();
Log("Job completed");
}
public JsonResult List(string[] correlation)
{
var expected_correlation = GetExpectedCorrelation(correlation);
var model = new FunctionalTestRunnerModel(null);
var descriptor = new JobDescriptor
{
Name = string.Format("UI Framework Functional Tests (IE9) {0}", expected_correlation),
Runs = GetTests(model),
UserAgentNames = ie9Only,
};
return Json(descriptor, JsonRequestBehavior.AllowGet);
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <CloneJobArgs>();
var fullname = args.Url.Replace("https://github.com/", "")
.Replace("http://github.com/", "")
.Replace(".git", "");
var clonePathAbs = _codeLocationRepository.GetCodeLocationLocalPath(fullname);
return(await Task.Run(() =>
{
var options = new CloneOptions()
{
BranchName = args.Branch,
IsBare = args.Bare,
Checkout = true,
CredentialsProvider = (_url, _user, _cred) => new UsernamePasswordCredentials
{
Username = args.Username,
Password = _apiRepository.GetUserToken(args.Username)
}
};
try
{
var result = _repositoryController.Clone(args.Url, clonePathAbs, options);
// create an entry for the repository in our db, so we can link commits to it
var codelocation = _codeLocationRepository.CreateCodeLocation(fullname, args.RepoName, args.Username);
// let other workers know that this worker has this code location
_codeLocationRepository.RecordCodeLocationOnWorker(WorkerId, codelocation.Id);
// schedule an import
_jobRepository.ScheduleJob <ImportHistoryJob>(codelocation);
_jobRepository.CompleteJob(Id, WorkerId, new
{
localPath = result
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <UpdateUserCodeLocationsJobArgs>();
return(await Task.Run(() =>
{
try
{
var repos = _apiDataRepository.GetRepositories(args.UserName);
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
private static JobDescriptor GetNextMessage(DateTime stop, ITimeBasedQueue queue, out DateTime actualStop)
{
JobDescriptor result = null;
actualStop = DateTime.MaxValue;
int count = 0;
while (DateTime.Now < stop && result == null)
{
result = queue.Next();
actualStop = DateTime.Now;
count++;
Thread.Yield();
}
return(result);
}
public WhenCompletedSuccessfully()
{
var fixture = Fixtures.JobDescriptions.CloneJobs.Proggr_TestTesting();
_jobDescription = fixture.JobDescription;
// first thing the job will do is ask where the Code Location should be stored
_codeLocationRepository.Setup(m => m.GetCodeLocationLocalPath(fixture.RepositoryFullName))
.Returns(fixture.CloneDirectoryPath)
.Verifiable();
// then the job will attempt to clone the repository to the disk
_repositoryController.Setup(m => m.Clone(fixture.RepositoryUrl, fixture.CloneDirectoryPath, It.IsAny <CloneOptions>()))
.ReturnsAsync(fixture.CloneDirectoryPath)
.Verifiable();
// after this, it will create a db entry for the code location
_codeLocationRepository.Setup(m => m.CreateCodeLocation(fixture.RepositoryFullName, fixture.JobArgs.RepoName, fixture.JobArgs.Username, false))
.Returns(Fixtures.CodeLocations.CodeLocationA)
.Verifiable();
// next up, time to claim ownership of this repository
_codeLocationRepository.Setup(m => m.RecordCodeLocationOnWorker(TEST_WORKER_ID, Fixtures.CodeLocations.CodeLocationA.Id))
.Returns(true)
.Verifiable();
// next, the job should schedule an ImportHistoryJob
_jobRepository.Setup(m => m.ScheduleJob <ImportHistoryJob>(Fixtures.CodeLocations.CodeLocationA))
.Verifiable();
// then the final step is to mark the current job complete
_jobRepository.Setup(m => m.CompleteJob(It.IsAny <Guid>(), TEST_WORKER_ID, It.IsAny <object>()))
.Verifiable();
RegisterIoC();
}
public WhenCompletedSuccessfully()
{
var fixture = Fixtures.JobDescriptions.CloneJobs.Proggr_TestTesting();
_jobDescription = fixture.JobDescription;
// first thing the job will do is ask where the Code Location should be stored
_codeLocationRepository.Setup(m => m.GetCodeLocationLocalPath(fixture.RepositoryFullName))
.Returns(fixture.CloneDirectoryPath)
.Verifiable();
// then the job will attempt to clone the repository to the disk
_repositoryController.Setup(m => m.Clone(fixture.RepositoryUrl, fixture.CloneDirectoryPath, It.IsAny<CloneOptions>()))
.ReturnsAsync(fixture.CloneDirectoryPath)
.Verifiable();
// after this, it will create a db entry for the code location
_codeLocationRepository.Setup(m => m.CreateCodeLocation(fixture.RepositoryFullName, fixture.JobArgs.RepoName, fixture.JobArgs.Username, false))
.Returns(Fixtures.CodeLocations.CodeLocationA)
.Verifiable();
// next up, time to claim ownership of this repository
_codeLocationRepository.Setup(m => m.RecordCodeLocationOnWorker(TEST_WORKER_ID, Fixtures.CodeLocations.CodeLocationA.Id))
.Returns(true)
.Verifiable();
// next, the job should schedule an ImportHistoryJob
_jobRepository.Setup(m => m.ScheduleJob<ImportHistoryJob>(Fixtures.CodeLocations.CodeLocationA))
.Verifiable();
// then the final step is to mark the current job complete
_jobRepository.Setup(m => m.CompleteJob(It.IsAny<Guid>(), TEST_WORKER_ID, It.IsAny<object>()))
.Verifiable();
RegisterIoC();
}
private int DoTheParentJob()
{
var descriptors = new List <JobDescriptor>();
int i = 0;
for (; i < WORKERS_POOL_SIZE; ++i)
{
string[] modelFilesForTask = { "model.xml", "startpath" + i + ".xml" };
string[] arguments = { "--model", modelFilesForTask[0], "--startpath", modelFilesForTask[1] };
var descriptor = helper.SubmitNewCopyOfMyself(modelFilesForTask, arguments);
descriptors.Add(descriptor);
}
while (true)
{
JobDescriptor completedTaskDescriptor = Helper.WaitForAnyJobToEnd(descriptors);
completedTaskDescriptor.HardRemove();
// cleanup the active descriptors removing the completed one
descriptors.Remove(completedTaskDescriptor);
// gather results
var jobId = completedTaskDescriptor.JobId;
var exitCode = helper.GetExitCode(jobId);
if (exitCode == 0)
{
// everything is done, tearing down everything
descriptors.ForEach(descriptor => descriptor.HardRemove());
var counterExampleContent = helper.GetCounterExample(jobId);
// processing of the counterExample
return(0);
}
else
{
string[] modelFilesForTask = { "model.xml", "startpath" + ++i + ".xml" };
string[] arguments = { "--model", modelFilesForTask[0], "--startpath", modelFilesForTask[1] };
var descriptor = helper.SubmitNewCopyOfMyself(modelFilesForTask, arguments);
descriptors.Add(descriptor);
}
}
}
public UpdateUserCodeLocationsJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : base(jobDescription, workerId, locator)
{
_apiDataRepository = locator.Locate <IApiDataRepository>();
_codeLocationRepository = locator.Locate <ICodeLocationRepository>();
}
public void ScheduleJob(JobDescriptor jobDescriptor) => ScheduleJob(jobDescriptor as SimpleJobDescription);
public CloneJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
base(jobDescription, workerId, locator)
{
_codeLocationRepository = _serviceLocator.Locate<ICodeLocationRepository>();
_repositoryController = _serviceLocator.Locate<IRepositoryController>();
}
public ImportHistoryJob(JobDescriptor jobDescription, Guid workerId, ILocator locator)
: base(jobDescription, workerId, locator)
{
_codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
}
public JobDescriptor UpdateJobStatus(JobDescriptor job, string newStatus)
{
Database.Jobs.UpdateById(Id: job.Id, Status: newStatus);
job.Status = newStatus;
return(job);
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <CodeLocation>();
return(await Task.Run(() =>
{
try
{
if (!_codeLocationRepository.HasCodeLocationLocally(args.FullName))
{
// TODO: get the codelocation from the worker that cloned it? or run a quick clone job here...
}
var repo = new Repository(_codeLocationRepository.GetCodeLocationLocalPath(args.FullName), new RepositoryOptions()
{
});
repo.Fetch("origin");
int count = 0;
var start = DateTime.Now.Ticks;
Parallel.ForEach(repo.Commits, (commit) =>
{
count++;
var newCommit = new CommitData()
{
Sha = commit.Sha,
Message = commit.Message,
MessageShort = commit.MessageShort,
AuthorEmail = commit.Author.Email,
AuthorUserName = commit.Author.Name,
CommitterEmail = commit.Committer.Email,
CommitterUserName = commit.Committer.Name,
DateAuthored = commit.Author.When.DateTime.ToUniversalTime(),
DateCommitted = commit.Committer.When.DateTime.ToUniversalTime(),
RepositoryId = args.Id
};
_codeLocationRepository.AddCommit(newCommit, args);
});
var duration = new TimeSpan(DateTime.Now.Ticks - start).TotalMilliseconds;
_logger.Info($"{count} commits imported in {duration}ms");
_jobRepository.CompleteJob(Id, WorkerId, new { NumCommits = count, Duration = duration });
// break the list of commits into chunks, no bigger than 512 commits in length
// then create a DetectAndHashJob for each chunk.
var chunks = ToChunks(repo.Commits.Select(c => c.Sha).ToList(), DetectAndHashJob.MAX_NUMBER_OF_SHAS_TO_PROCESS);
Parallel.ForEach(chunks, (chunk) =>
{
// create a DetectAndHashJob for this chunk
_jobRepository.ScheduleJob <DetectAndHashJob>(new DetectAndHashJobArgs()
{
RepositoryId = args.Id,
Shas = chunk
});
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <DetectAndHashJobArgs>();
var codeLocation = _codeLocationRepository.GetCodeLocation(args.RepositoryId);
return(await Task.Run(() =>
{
try
{
var repoRoot = _codeLocationRepository.GetCodeLocationLocalPath(codeLocation.FullName);
var repo = new Repository(repoRoot, new RepositoryOptions()
{
});
Parallel.ForEach(args.Shas, async(sha) =>
{
var commit = await _repositoryController.GetCommit(repo, sha);
var patch = await CommitController.GetChangeSet(repo, commit);
Parallel.ForEach(patch, (change) =>
{
if (change.Mode == Mode.NonExecutableFile && change.Status != ChangeKind.Deleted &&
change.Status != ChangeKind.Ignored && change.Status != ChangeKind.Untracked)
{
var filePath = Path.Combine(repoRoot, change.Path);
string contents = null;
if (!change.IsBinaryComparison)
{
var treeEntry = commit.Tree[change.Path];
if (treeEntry.TargetType == TreeEntryTargetType.Blob)
{
// this is a file we can open and read
// get the file contents
var blob = (Blob)treeEntry.Target;
var contentStream = blob.GetContentStream();
using (var reader = new StreamReader(contentStream, Encoding.UTF8))
{
contents = reader.ReadToEnd();
}
}
}
var detection = _extDectector.Detect(filePath, contents);
// compute our hashes
byte[] raw, nowhitespace, nonewlines;
using (MD5 md5 = MD5.Create())
{
raw = md5.ComputeHash(Encoding.UTF8.GetBytes(contents));
nowhitespace = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace(" ", "")));
nonewlines = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace("\n", "")));
}
// save the results
_fileRepository.AddFile(new FileData
{
CommitId = sha,
Ext = detection.Extension,
FileName = filePath,
RelativePath = change.Path,
HashRaw = raw,
HashNoWhiteSpace = nowhitespace,
HashNoNewLines = nonewlines
});
}
});
// get the files that changed in this commit
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public UpdateUserCodeLocationsJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : base(jobDescription, workerId, locator)
{
_apiDataRepository = locator.Locate<IApiDataRepository>();
_codeLocationRepository = locator.Locate<ICodeLocationRepository>();
}
public CloneJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
base(jobDescription, workerId, locator)
{
_codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
_repositoryController = _serviceLocator.Locate <IRepositoryController>();
}
