public override async Task <JobResult> Run() { var args = JobDescriptor.GetArgumentsJson <CloneJobArgs>(); var fullname = args.Url.Replace("https://github.com/", "") .Replace("http://github.com/", "") .Replace(".git", ""); var clonePathAbs = _codeLocationRepository.GetCodeLocationLocalPath(fullname); return(await Task.Run(() => { var options = new CloneOptions() { BranchName = args.Branch, IsBare = args.Bare, Checkout = true, CredentialsProvider = (_url, _user, _cred) => new UsernamePasswordCredentials { Username = args.Username, Password = _apiRepository.GetUserToken(args.Username) } }; try { var result = _repositoryController.Clone(args.Url, clonePathAbs, options); // create an entry for the repository in our db, so we can link commits to it var codelocation = _codeLocationRepository.CreateCodeLocation(fullname, args.RepoName, args.Username); // let other workers know that this worker has this code location _codeLocationRepository.RecordCodeLocationOnWorker(WorkerId, codelocation.Id); // schedule an import _jobRepository.ScheduleJob <ImportHistoryJob>(codelocation); _jobRepository.CompleteJob(Id, WorkerId, new { localPath = result }); return Task.FromResult <JobResult>(new JobSuccessResult(this)); } catch (Exception e) { return Task.FromResult <JobResult>(new JobFailureResult(e, this)); } })); }
public override async Task <JobResult> Run() { var args = JobDescriptor.GetArgumentsJson <UpdateUserCodeLocationsJobArgs>(); return(await Task.Run(() => { try { var repos = _apiDataRepository.GetRepositories(args.UserName); return Task.FromResult <JobResult>(new JobSuccessResult(this)); } catch (Exception e) { return Task.FromResult <JobResult>(new JobFailureResult(e, this)); } })); }
public override async Task <JobResult> Run() { var args = JobDescriptor.GetArgumentsJson <DetectAndHashJobArgs>(); var codeLocation = _codeLocationRepository.GetCodeLocation(args.RepositoryId); return(await Task.Run(() => { try { var repoRoot = _codeLocationRepository.GetCodeLocationLocalPath(codeLocation.FullName); var repo = new Repository(repoRoot, new RepositoryOptions() { }); Parallel.ForEach(args.Shas, async(sha) => { var commit = await _repositoryController.GetCommit(repo, sha); var patch = await CommitController.GetChangeSet(repo, commit); Parallel.ForEach(patch, (change) => { if (change.Mode == Mode.NonExecutableFile && change.Status != ChangeKind.Deleted && change.Status != ChangeKind.Ignored && change.Status != ChangeKind.Untracked) { var filePath = Path.Combine(repoRoot, change.Path); string contents = null; if (!change.IsBinaryComparison) { var treeEntry = commit.Tree[change.Path]; if (treeEntry.TargetType == TreeEntryTargetType.Blob) { // this is a file we can open and read // get the file contents var blob = (Blob)treeEntry.Target; var contentStream = blob.GetContentStream(); using (var reader = new StreamReader(contentStream, Encoding.UTF8)) { contents = reader.ReadToEnd(); } } } var detection = _extDectector.Detect(filePath, contents); // compute our hashes byte[] raw, nowhitespace, nonewlines; using (MD5 md5 = MD5.Create()) { raw = md5.ComputeHash(Encoding.UTF8.GetBytes(contents)); nowhitespace = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace(" ", ""))); nonewlines = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace("\n", ""))); } // save the results _fileRepository.AddFile(new FileData { CommitId = sha, Ext = detection.Extension, FileName = filePath, RelativePath = change.Path, HashRaw = raw, HashNoWhiteSpace = nowhitespace, HashNoNewLines = nonewlines }); } }); // get the files that changed in this commit }); return Task.FromResult <JobResult>(new JobSuccessResult(this)); } catch (Exception e) { return Task.FromResult <JobResult>(new JobFailureResult(e, this)); } })); }
public override async Task <JobResult> Run() { var args = JobDescriptor.GetArgumentsJson <CodeLocation>(); return(await Task.Run(() => { try { if (!_codeLocationRepository.HasCodeLocationLocally(args.FullName)) { // TODO: get the codelocation from the worker that cloned it? or run a quick clone job here... } var repo = new Repository(_codeLocationRepository.GetCodeLocationLocalPath(args.FullName), new RepositoryOptions() { }); repo.Fetch("origin"); int count = 0; var start = DateTime.Now.Ticks; Parallel.ForEach(repo.Commits, (commit) => { count++; var newCommit = new CommitData() { Sha = commit.Sha, Message = commit.Message, MessageShort = commit.MessageShort, AuthorEmail = commit.Author.Email, AuthorUserName = commit.Author.Name, CommitterEmail = commit.Committer.Email, CommitterUserName = commit.Committer.Name, DateAuthored = commit.Author.When.DateTime.ToUniversalTime(), DateCommitted = commit.Committer.When.DateTime.ToUniversalTime(), RepositoryId = args.Id }; _codeLocationRepository.AddCommit(newCommit, args); }); var duration = new TimeSpan(DateTime.Now.Ticks - start).TotalMilliseconds; _logger.Info($"{count} commits imported in {duration}ms"); _jobRepository.CompleteJob(Id, WorkerId, new { NumCommits = count, Duration = duration }); // break the list of commits into chunks, no bigger than 512 commits in length // then create a DetectAndHashJob for each chunk. var chunks = ToChunks(repo.Commits.Select(c => c.Sha).ToList(), DetectAndHashJob.MAX_NUMBER_OF_SHAS_TO_PROCESS); Parallel.ForEach(chunks, (chunk) => { // create a DetectAndHashJob for this chunk _jobRepository.ScheduleJob <DetectAndHashJob>(new DetectAndHashJobArgs() { RepositoryId = args.Id, Shas = chunk }); }); return Task.FromResult <JobResult>(new JobSuccessResult(this)); } catch (Exception e) { return Task.FromResult <JobResult>(new JobFailureResult(e, this)); } })); }