Beispiel #1
0
        public void OneWithJobDescriptor()
        {
            CloudQueueMessage msg = queue.GetMessage();

            Assert.That(msg, Is.Null);

            JobDescriptor descriptor = factory.CreateDescriptor(new HelloWho()
            {
                Who = "ProducerTest-OneWithJobDescriptor"
            });

            producer.One(descriptor);

            msg = queue.GetMessage();
            Assert.That(msg, Is.Not.Null);
            Assert.That(msg.AsString, Is.EqualTo("{\"Job\":\"HelloWho\",\"Properties\":{\"Who\":\"ProducerTest-OneWithJobDescriptor\"}}"));

            descriptor = JsonConvert.DeserializeObject <JobDescriptor>(msg.AsString);
            Assert.That(descriptor, Is.Not.Null.
                        And.Property("Job").EqualTo("HelloWho").
                        And.Property("Properties").Count.EqualTo(1));
            Assert.That(descriptor.Properties["Who"].ToObject <string>(), Is.EqualTo("ProducerTest-OneWithJobDescriptor"));

            queue.DeleteMessage(msg);
        }
Beispiel #2
0
        public void Roundtrip()
        {
            MockJob job = new MockJob()
            {
                Id = Guid.NewGuid(), Int32 = 987
            };
            JobDescriptor desc = factory.CreateDescriptor(job);
            JToken        token;

            Assert.That(desc, Is.Not.Null);
            Assert.That(desc.Job, Is.EqualTo("MockJob"));
            Assert.That(desc.Properties, Is.Not.Null.And.Count.EqualTo(2));

            Assert.That(desc.Properties.TryGetValue("Id", out token), Is.True);
            Assert.That(token, Is.TypeOf <JValue>().And.Property("Type").EqualTo(JTokenType.Guid));
            Assert.That(token.ToObject <Guid>(), Is.EqualTo(job.Id));

            Assert.That(desc.Properties.TryGetValue("Int32", out token), Is.True);
            Assert.That(token, Is.TypeOf <JValue>().And.Property("Type").EqualTo(JTokenType.Integer));
            Assert.That(token.ToObject <Int32>(), Is.EqualTo(987));

            Assert.That(desc.QueueMessageId, Is.Null);

            IJob job2 = factory.CreateJob(desc);

            Assert.That(job2, Is.Not.Null.And.TypeOf <MockJob>());
            MockJob mockJob2 = (MockJob)job2;

            Assert.That(mockJob2.Id, Is.EqualTo(job.Id));
        }
Beispiel #3
0
    private JobDescriptor Enqueue(JobCall job, JobStatus status, DateTime?scheduled, TimeZoneInfo?timeZone = null)
    {
        if (job == null)
        {
            throw new ArgumentNullException(nameof(job));
        }

        if (status == JobStatus.Waiting && !scheduled.HasValue)
        {
            throw new ArgumentNullException(nameof(scheduled));
        }

        int jobId         = ++_idSource;
        var jobDescriptor = new JobDescriptor
        {
            Id        = jobId.ToString(),
            Status    = status,
            Scheduled = scheduled,
            TimeZone  = timeZone,
            Call      = job
        };

        lock (_syncLock)
        {
            _jobs.Add(jobDescriptor);
        }

        return(jobDescriptor);
    }
 public DetectAndHashJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : 
     base(jobDescription, workerId, locator)
 {
     _codeLocationRepository = _serviceLocator.Locate<ICodeLocationRepository>();
     _fileRepository = _serviceLocator.Locate<IFileRepository>();
     _repositoryController = _serviceLocator.Locate<IRepositoryController>();
 }
Beispiel #5
0
 public DetectAndHashJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
     base(jobDescription, workerId, locator)
 {
     _codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
     _fileRepository         = _serviceLocator.Locate <IFileRepository>();
     _repositoryController   = _serviceLocator.Locate <IRepositoryController>();
 }
Beispiel #6
0
 /// <summary>
 /// Initializes a new instance of the <see cref="NullJob"/> class.
 /// </summary>
 /// <param name="jobDescriptor">The job descriptor.</param>
 public NullJob(JobDescriptor jobDescriptor)
 {
     Url = jobDescriptor.Url;
     Name = jobDescriptor.Name;
     Status = jobDescriptor.Status;
     Description = "Unable to load this job";
 }
Beispiel #7
0
        public void OneWithInitialVisibilityDelay()
        {
            TimeSpan          visibilityDelay = TimeSpan.FromSeconds(1);
            CloudQueueMessage msg             = queue.GetMessage();

            Assert.That(msg, Is.Null);

            producer.One(new HelloWho()
            {
                Who = "ProducerTest-OneWithInitialVisibilityDelay"
            }, visibilityDelay);

            msg = queue.GetMessage();
            Assert.That(msg, Is.Null);

            Thread.Sleep(visibilityDelay);

            msg = queue.GetMessage();
            Assert.That(msg, Is.Not.Null);
            Assert.That(msg.AsString, Is.EqualTo("{\"Job\":\"HelloWho\",\"Properties\":{\"Who\":\"ProducerTest-OneWithInitialVisibilityDelay\"}}"));

            JobDescriptor descriptor = JsonConvert.DeserializeObject <JobDescriptor>(msg.AsString);

            Assert.That(descriptor, Is.Not.Null.
                        And.Property("Job").EqualTo("HelloWho").
                        And.Property("Properties").Count.EqualTo(1));
            Assert.That(descriptor.Properties["Who"].ToObject <string>(), Is.EqualTo("ProducerTest-OneWithInitialVisibilityDelay"));

            queue.DeleteMessage(msg);
        }
Beispiel #8
0
        public Job(JobDescriptor jobDescription, Guid workerId, ILocator locator)
        {
            _serviceLocator = locator;
            _jobRepository = locator.Locate<IJobRepository>();
            _apiRepository = locator.Locate<IApiDataRepository>();

            Id = jobDescription.Id;
            WorkerId = workerId;
            JobDescriptor = jobDescription;
        }
Beispiel #9
0
        public Job(JobDescriptor jobDescription, Guid workerId, ILocator locator)
        {
            _serviceLocator = locator;
            _jobRepository  = locator.Locate <IJobRepository>();
            _apiRepository  = locator.Locate <IApiDataRepository>();

            Id            = jobDescription.Id;
            WorkerId      = workerId;
            JobDescriptor = jobDescription;
        }
        public void CanGetNextJobWhenEmpty()
        {
            // Arrange
            var queue = serviceProviderMock.GetService <ITimeBasedQueue>();

            // Act
            JobDescriptor job = queue.Next();

            // Assert
        }
Beispiel #11
0
        public static IJob CreateJob(JobDescriptor description, WorkerState worker, ILocator locator)
        {
            var jobType = Type.GetType(description.JobType);
            if (jobType == null)
            {
                throw new TypeAccessException("Could not create a type from " + description.JobType);
            }
            var instance = Activator.CreateInstance(jobType, description, worker.Id, locator) as IJob;

            return instance;
        }
Beispiel #12
0
        public static IJob CreateJob(JobDescriptor description, WorkerState worker, ILocator locator)
        {
            var jobType = Type.GetType(description.JobType);

            if (jobType == null)
            {
                throw new TypeAccessException("Could not create a type from " + description.JobType);
            }
            var instance = Activator.CreateInstance(jobType, description, worker.Id, locator) as IJob;

            return(instance);
        }
Beispiel #13
0
    public Task <string> EnqueueAsync(JobCall job)
    {
        if (job == null)
        {
            throw new ArgumentNullException(nameof(job));
        }

        JobDescriptor jobDescriptor = Enqueue(job, JobStatus.Ready, null);

        _queue.Enqueue(jobDescriptor);
        _signal.Release();

        return(Task.FromResult(jobDescriptor.Id));
    }
Beispiel #14
0
    public Task <string> ScheduleAsync(JobCall job, DateTime scheduled, TimeZoneInfo?timeZone = null)
    {
        if (job == null)
        {
            throw new ArgumentNullException(nameof(job));
        }

        JobDescriptor jobDescriptor = Enqueue(job, JobStatus.Waiting, scheduled, timeZone);

        // Refresh the queue observer.
        _signal.Release();

        return(Task.FromResult(jobDescriptor.Id));
    }
Beispiel #15
0
        public JsonResult List(string[] correlation)
        {
            var expected_correlation = GetExpectedCorrelation(correlation);

            var model = new FunctionalTestRunnerModel(null);

            var descriptor = new JobDescriptor
            {
                Name           = string.Format("Functional Tests {0}", expected_correlation),
                Runs           = GetTests(model),
                UserAgentNames = ie9Only,
            };

            return(Json(descriptor, JsonRequestBehavior.AllowGet));
        }
Beispiel #16
0
#pragma warning disable IDE0051 // Remove unused private members
        private static void SubmitNewJob()
        {
#pragma warning restore IDE0051 // Remove unused private members
            NewJobSubmitter newJobSubmitter = new NewJobSubmitter {
                Command = "batch.py",
                UserStandardOutputFileName = "stdout.txt",
                TargetOperatingSystem      = TargetOperatingSystem.ANY
            };
            JobDescriptor jobDescriptor = newJobSubmitter.SubmitNewJob();
            Log("Job submitted");
            jobDescriptor.JobStartedEvent.WaitOne();
            Log("Job started");
            jobDescriptor.JobCompletedEvent.WaitOne();
            Log("Job completed");
        }
        public JsonResult List(string[] correlation)
        {
            var expected_correlation = GetExpectedCorrelation(correlation);

                var model = new FunctionalTestRunnerModel(null);

                var descriptor = new JobDescriptor
                {
                    Name = string.Format("UI Framework Functional Tests (IE9) {0}", expected_correlation),
                    Runs = GetTests(model),
                    UserAgentNames = ie9Only,
                };

                return Json(descriptor, JsonRequestBehavior.AllowGet);
        }
Beispiel #18
0
        public override async Task <JobResult> Run()
        {
            var args     = JobDescriptor.GetArgumentsJson <CloneJobArgs>();
            var fullname = args.Url.Replace("https://github.com/", "")
                           .Replace("http://github.com/", "")
                           .Replace(".git", "");
            var clonePathAbs = _codeLocationRepository.GetCodeLocationLocalPath(fullname);

            return(await Task.Run(() =>
            {
                var options = new CloneOptions()
                {
                    BranchName = args.Branch,
                    IsBare = args.Bare,
                    Checkout = true,
                    CredentialsProvider = (_url, _user, _cred) => new UsernamePasswordCredentials
                    {
                        Username = args.Username,
                        Password = _apiRepository.GetUserToken(args.Username)
                    }
                };
                try
                {
                    var result = _repositoryController.Clone(args.Url, clonePathAbs, options);

                    // create an entry for the repository in our db, so we can link commits to it
                    var codelocation = _codeLocationRepository.CreateCodeLocation(fullname, args.RepoName, args.Username);

                    // let other workers know that this worker has this code location
                    _codeLocationRepository.RecordCodeLocationOnWorker(WorkerId, codelocation.Id);

                    // schedule an import
                    _jobRepository.ScheduleJob <ImportHistoryJob>(codelocation);

                    _jobRepository.CompleteJob(Id, WorkerId, new
                    {
                        localPath = result
                    });

                    return Task.FromResult <JobResult>(new JobSuccessResult(this));
                }
                catch (Exception e)
                {
                    return Task.FromResult <JobResult>(new JobFailureResult(e, this));
                }
            }));
        }
        public override async Task <JobResult> Run()
        {
            var args = JobDescriptor.GetArgumentsJson <UpdateUserCodeLocationsJobArgs>();

            return(await Task.Run(() =>
            {
                try
                {
                    var repos = _apiDataRepository.GetRepositories(args.UserName);
                    return Task.FromResult <JobResult>(new JobSuccessResult(this));
                }
                catch (Exception e)
                {
                    return Task.FromResult <JobResult>(new JobFailureResult(e, this));
                }
            }));
        }
        private static JobDescriptor GetNextMessage(DateTime stop, ITimeBasedQueue queue, out DateTime actualStop)
        {
            JobDescriptor result = null;

            actualStop = DateTime.MaxValue;
            int count = 0;

            while (DateTime.Now < stop && result == null)
            {
                result     = queue.Next();
                actualStop = DateTime.Now;
                count++;
                Thread.Yield();
            }

            return(result);
        }
Beispiel #21
0
            public WhenCompletedSuccessfully()
            {
                var fixture = Fixtures.JobDescriptions.CloneJobs.Proggr_TestTesting();

                _jobDescription = fixture.JobDescription;

                // first thing the job will do is ask where the Code Location should be stored

                _codeLocationRepository.Setup(m => m.GetCodeLocationLocalPath(fixture.RepositoryFullName))
                .Returns(fixture.CloneDirectoryPath)
                .Verifiable();

                // then the job will attempt to clone the repository to the disk

                _repositoryController.Setup(m => m.Clone(fixture.RepositoryUrl, fixture.CloneDirectoryPath, It.IsAny <CloneOptions>()))
                .ReturnsAsync(fixture.CloneDirectoryPath)
                .Verifiable();

                // after this, it will create a db entry for the code location

                _codeLocationRepository.Setup(m => m.CreateCodeLocation(fixture.RepositoryFullName, fixture.JobArgs.RepoName, fixture.JobArgs.Username, false))
                .Returns(Fixtures.CodeLocations.CodeLocationA)
                .Verifiable();

                // next up, time to claim ownership of this repository

                _codeLocationRepository.Setup(m => m.RecordCodeLocationOnWorker(TEST_WORKER_ID, Fixtures.CodeLocations.CodeLocationA.Id))
                .Returns(true)
                .Verifiable();

                // next, the job should schedule an ImportHistoryJob

                _jobRepository.Setup(m => m.ScheduleJob <ImportHistoryJob>(Fixtures.CodeLocations.CodeLocationA))
                .Verifiable();

                // then the final step is to mark the current job complete

                _jobRepository.Setup(m => m.CompleteJob(It.IsAny <Guid>(), TEST_WORKER_ID, It.IsAny <object>()))
                .Verifiable();

                RegisterIoC();
            }
Beispiel #22
0
            public WhenCompletedSuccessfully()
            {
                var fixture = Fixtures.JobDescriptions.CloneJobs.Proggr_TestTesting();
                _jobDescription = fixture.JobDescription;

                // first thing the job will do is ask where the Code Location should be stored

                _codeLocationRepository.Setup(m => m.GetCodeLocationLocalPath(fixture.RepositoryFullName))
                    .Returns(fixture.CloneDirectoryPath)
                    .Verifiable();

                // then the job will attempt to clone the repository to the disk

                _repositoryController.Setup(m => m.Clone(fixture.RepositoryUrl, fixture.CloneDirectoryPath, It.IsAny<CloneOptions>()))
                    .ReturnsAsync(fixture.CloneDirectoryPath)
                    .Verifiable();

                // after this, it will create a db entry for the code location
                
                _codeLocationRepository.Setup(m => m.CreateCodeLocation(fixture.RepositoryFullName, fixture.JobArgs.RepoName, fixture.JobArgs.Username, false))
                    .Returns(Fixtures.CodeLocations.CodeLocationA)
                    .Verifiable();

                // next up, time to claim ownership of this repository

                _codeLocationRepository.Setup(m => m.RecordCodeLocationOnWorker(TEST_WORKER_ID, Fixtures.CodeLocations.CodeLocationA.Id))
                    .Returns(true)
                    .Verifiable();

                // next, the job should schedule an ImportHistoryJob

                _jobRepository.Setup(m => m.ScheduleJob<ImportHistoryJob>(Fixtures.CodeLocations.CodeLocationA))
                    .Verifiable();

                // then the final step is to mark the current job complete

                _jobRepository.Setup(m => m.CompleteJob(It.IsAny<Guid>(), TEST_WORKER_ID, It.IsAny<object>()))
                    .Verifiable();

                RegisterIoC();
            }
Beispiel #23
0
        private int DoTheParentJob()
        {
            var descriptors = new List <JobDescriptor>();
            int i           = 0;

            for (; i < WORKERS_POOL_SIZE; ++i)
            {
                string[] modelFilesForTask = { "model.xml", "startpath" + i + ".xml" };
                string[] arguments         = { "--model", modelFilesForTask[0], "--startpath", modelFilesForTask[1] };
                var      descriptor        = helper.SubmitNewCopyOfMyself(modelFilesForTask, arguments);
                descriptors.Add(descriptor);
            }

            while (true)
            {
                JobDescriptor completedTaskDescriptor = Helper.WaitForAnyJobToEnd(descriptors);
                completedTaskDescriptor.HardRemove();
                // cleanup the active descriptors removing the completed one
                descriptors.Remove(completedTaskDescriptor);
                // gather results
                var jobId    = completedTaskDescriptor.JobId;
                var exitCode = helper.GetExitCode(jobId);
                if (exitCode == 0)
                {
                    // everything is done, tearing down everything
                    descriptors.ForEach(descriptor => descriptor.HardRemove());
                    var counterExampleContent = helper.GetCounterExample(jobId);
                    // processing of the counterExample
                    return(0);
                }
                else
                {
                    string[] modelFilesForTask = { "model.xml", "startpath" + ++i + ".xml" };
                    string[] arguments         = { "--model", modelFilesForTask[0], "--startpath", modelFilesForTask[1] };
                    var      descriptor        = helper.SubmitNewCopyOfMyself(modelFilesForTask, arguments);
                    descriptors.Add(descriptor);
                }
            }
        }
 public UpdateUserCodeLocationsJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : base(jobDescription, workerId, locator)
 {
     _apiDataRepository      = locator.Locate <IApiDataRepository>();
     _codeLocationRepository = locator.Locate <ICodeLocationRepository>();
 }
 public void ScheduleJob(JobDescriptor jobDescriptor) => ScheduleJob(jobDescriptor as SimpleJobDescription);
Beispiel #26
0
 public CloneJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : 
     base(jobDescription, workerId, locator)
 {
     _codeLocationRepository = _serviceLocator.Locate<ICodeLocationRepository>();
     _repositoryController = _serviceLocator.Locate<IRepositoryController>();
 }
Beispiel #27
0
 public ImportHistoryJob(JobDescriptor jobDescription, Guid workerId, ILocator locator)
     : base(jobDescription, workerId, locator)
 {
     _codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
 }
Beispiel #28
0
 public JobDescriptor UpdateJobStatus(JobDescriptor job, string newStatus)
 {
     Database.Jobs.UpdateById(Id: job.Id, Status: newStatus);
     job.Status = newStatus;
     return(job);
 }
Beispiel #29
0
        public override async Task <JobResult> Run()
        {
            var args = JobDescriptor.GetArgumentsJson <CodeLocation>();

            return(await Task.Run(() =>
            {
                try
                {
                    if (!_codeLocationRepository.HasCodeLocationLocally(args.FullName))
                    {
                        // TODO: get the codelocation from the worker that cloned it? or run a quick clone job here...
                    }

                    var repo = new Repository(_codeLocationRepository.GetCodeLocationLocalPath(args.FullName), new RepositoryOptions()
                    {
                    });

                    repo.Fetch("origin");

                    int count = 0;
                    var start = DateTime.Now.Ticks;
                    Parallel.ForEach(repo.Commits, (commit) =>
                    {
                        count++;
                        var newCommit = new CommitData()
                        {
                            Sha = commit.Sha,
                            Message = commit.Message,
                            MessageShort = commit.MessageShort,
                            AuthorEmail = commit.Author.Email,
                            AuthorUserName = commit.Author.Name,
                            CommitterEmail = commit.Committer.Email,
                            CommitterUserName = commit.Committer.Name,
                            DateAuthored = commit.Author.When.DateTime.ToUniversalTime(),
                            DateCommitted = commit.Committer.When.DateTime.ToUniversalTime(),
                            RepositoryId = args.Id
                        };
                        _codeLocationRepository.AddCommit(newCommit, args);
                    });

                    var duration = new TimeSpan(DateTime.Now.Ticks - start).TotalMilliseconds;

                    _logger.Info($"{count} commits imported in {duration}ms");

                    _jobRepository.CompleteJob(Id, WorkerId, new { NumCommits = count, Duration = duration });

                    // break the list of commits into chunks, no bigger than 512 commits in length
                    // then create a DetectAndHashJob for each chunk.
                    var chunks = ToChunks(repo.Commits.Select(c => c.Sha).ToList(), DetectAndHashJob.MAX_NUMBER_OF_SHAS_TO_PROCESS);
                    Parallel.ForEach(chunks, (chunk) =>
                    {
                        // create a DetectAndHashJob for this chunk
                        _jobRepository.ScheduleJob <DetectAndHashJob>(new DetectAndHashJobArgs()
                        {
                            RepositoryId = args.Id,
                            Shas = chunk
                        });
                    });

                    return Task.FromResult <JobResult>(new JobSuccessResult(this));
                }
                catch (Exception e)
                {
                    return Task.FromResult <JobResult>(new JobFailureResult(e, this));
                }
            }));
        }
Beispiel #30
0
        public override async Task <JobResult> Run()
        {
            var args         = JobDescriptor.GetArgumentsJson <DetectAndHashJobArgs>();
            var codeLocation = _codeLocationRepository.GetCodeLocation(args.RepositoryId);

            return(await Task.Run(() =>
            {
                try
                {
                    var repoRoot = _codeLocationRepository.GetCodeLocationLocalPath(codeLocation.FullName);
                    var repo = new Repository(repoRoot, new RepositoryOptions()
                    {
                    });

                    Parallel.ForEach(args.Shas, async(sha) =>
                    {
                        var commit = await _repositoryController.GetCommit(repo, sha);
                        var patch = await CommitController.GetChangeSet(repo, commit);

                        Parallel.ForEach(patch, (change) =>
                        {
                            if (change.Mode == Mode.NonExecutableFile && change.Status != ChangeKind.Deleted &&
                                change.Status != ChangeKind.Ignored && change.Status != ChangeKind.Untracked)
                            {
                                var filePath = Path.Combine(repoRoot, change.Path);
                                string contents = null;
                                if (!change.IsBinaryComparison)
                                {
                                    var treeEntry = commit.Tree[change.Path];
                                    if (treeEntry.TargetType == TreeEntryTargetType.Blob)
                                    {
                                        // this is a file we can open and read
                                        // get the file contents
                                        var blob = (Blob)treeEntry.Target;
                                        var contentStream = blob.GetContentStream();
                                        using (var reader = new StreamReader(contentStream, Encoding.UTF8))
                                        {
                                            contents = reader.ReadToEnd();
                                        }
                                    }
                                }

                                var detection = _extDectector.Detect(filePath, contents);

                                // compute our hashes
                                byte[] raw, nowhitespace, nonewlines;
                                using (MD5 md5 = MD5.Create())
                                {
                                    raw = md5.ComputeHash(Encoding.UTF8.GetBytes(contents));
                                    nowhitespace = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace(" ", "")));
                                    nonewlines = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace("\n", "")));
                                }

                                // save the results
                                _fileRepository.AddFile(new FileData
                                {
                                    CommitId = sha,
                                    Ext = detection.Extension,
                                    FileName = filePath,
                                    RelativePath = change.Path,
                                    HashRaw = raw,
                                    HashNoWhiteSpace = nowhitespace,
                                    HashNoNewLines = nonewlines
                                });
                            }
                        });
                        // get the files that changed in this commit
                    });

                    return Task.FromResult <JobResult>(new JobSuccessResult(this));
                }
                catch (Exception e)
                {
                    return Task.FromResult <JobResult>(new JobFailureResult(e, this));
                }
            }));
        }
 public UpdateUserCodeLocationsJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) : base(jobDescription, workerId, locator)
 {
     _apiDataRepository = locator.Locate<IApiDataRepository>();
     _codeLocationRepository = locator.Locate<ICodeLocationRepository>();
 }
Beispiel #32
0
 public CloneJob(JobDescriptor jobDescription, Guid workerId, ILocator locator) :
     base(jobDescription, workerId, locator)
 {
     _codeLocationRepository = _serviceLocator.Locate <ICodeLocationRepository>();
     _repositoryController   = _serviceLocator.Locate <IRepositoryController>();
 }