public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <CloneJobArgs>();
var fullname = args.Url.Replace("https://github.com/", "")
.Replace("http://github.com/", "")
.Replace(".git", "");
var clonePathAbs = _codeLocationRepository.GetCodeLocationLocalPath(fullname);
return(await Task.Run(() =>
{
var options = new CloneOptions()
{
BranchName = args.Branch,
IsBare = args.Bare,
Checkout = true,
CredentialsProvider = (_url, _user, _cred) => new UsernamePasswordCredentials
{
Username = args.Username,
Password = _apiRepository.GetUserToken(args.Username)
}
};
try
{
var result = _repositoryController.Clone(args.Url, clonePathAbs, options);
// create an entry for the repository in our db, so we can link commits to it
var codelocation = _codeLocationRepository.CreateCodeLocation(fullname, args.RepoName, args.Username);
// let other workers know that this worker has this code location
_codeLocationRepository.RecordCodeLocationOnWorker(WorkerId, codelocation.Id);
// schedule an import
_jobRepository.ScheduleJob <ImportHistoryJob>(codelocation);
_jobRepository.CompleteJob(Id, WorkerId, new
{
localPath = result
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <UpdateUserCodeLocationsJobArgs>();
return(await Task.Run(() =>
{
try
{
var repos = _apiDataRepository.GetRepositories(args.UserName);
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <DetectAndHashJobArgs>();
var codeLocation = _codeLocationRepository.GetCodeLocation(args.RepositoryId);
return(await Task.Run(() =>
{
try
{
var repoRoot = _codeLocationRepository.GetCodeLocationLocalPath(codeLocation.FullName);
var repo = new Repository(repoRoot, new RepositoryOptions()
{
});
Parallel.ForEach(args.Shas, async(sha) =>
{
var commit = await _repositoryController.GetCommit(repo, sha);
var patch = await CommitController.GetChangeSet(repo, commit);
Parallel.ForEach(patch, (change) =>
{
if (change.Mode == Mode.NonExecutableFile && change.Status != ChangeKind.Deleted &&
change.Status != ChangeKind.Ignored && change.Status != ChangeKind.Untracked)
{
var filePath = Path.Combine(repoRoot, change.Path);
string contents = null;
if (!change.IsBinaryComparison)
{
var treeEntry = commit.Tree[change.Path];
if (treeEntry.TargetType == TreeEntryTargetType.Blob)
{
// this is a file we can open and read
// get the file contents
var blob = (Blob)treeEntry.Target;
var contentStream = blob.GetContentStream();
using (var reader = new StreamReader(contentStream, Encoding.UTF8))
{
contents = reader.ReadToEnd();
}
}
}
var detection = _extDectector.Detect(filePath, contents);
// compute our hashes
byte[] raw, nowhitespace, nonewlines;
using (MD5 md5 = MD5.Create())
{
raw = md5.ComputeHash(Encoding.UTF8.GetBytes(contents));
nowhitespace = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace(" ", "")));
nonewlines = md5.ComputeHash(Encoding.UTF8.GetBytes(contents.Replace("\n", "")));
}
// save the results
_fileRepository.AddFile(new FileData
{
CommitId = sha,
Ext = detection.Extension,
FileName = filePath,
RelativePath = change.Path,
HashRaw = raw,
HashNoWhiteSpace = nowhitespace,
HashNoNewLines = nonewlines
});
}
});
// get the files that changed in this commit
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
public override async Task <JobResult> Run()
{
var args = JobDescriptor.GetArgumentsJson <CodeLocation>();
return(await Task.Run(() =>
{
try
{
if (!_codeLocationRepository.HasCodeLocationLocally(args.FullName))
{
// TODO: get the codelocation from the worker that cloned it? or run a quick clone job here...
}
var repo = new Repository(_codeLocationRepository.GetCodeLocationLocalPath(args.FullName), new RepositoryOptions()
{
});
repo.Fetch("origin");
int count = 0;
var start = DateTime.Now.Ticks;
Parallel.ForEach(repo.Commits, (commit) =>
{
count++;
var newCommit = new CommitData()
{
Sha = commit.Sha,
Message = commit.Message,
MessageShort = commit.MessageShort,
AuthorEmail = commit.Author.Email,
AuthorUserName = commit.Author.Name,
CommitterEmail = commit.Committer.Email,
CommitterUserName = commit.Committer.Name,
DateAuthored = commit.Author.When.DateTime.ToUniversalTime(),
DateCommitted = commit.Committer.When.DateTime.ToUniversalTime(),
RepositoryId = args.Id
};
_codeLocationRepository.AddCommit(newCommit, args);
});
var duration = new TimeSpan(DateTime.Now.Ticks - start).TotalMilliseconds;
_logger.Info($"{count} commits imported in {duration}ms");
_jobRepository.CompleteJob(Id, WorkerId, new { NumCommits = count, Duration = duration });
// break the list of commits into chunks, no bigger than 512 commits in length
// then create a DetectAndHashJob for each chunk.
var chunks = ToChunks(repo.Commits.Select(c => c.Sha).ToList(), DetectAndHashJob.MAX_NUMBER_OF_SHAS_TO_PROCESS);
Parallel.ForEach(chunks, (chunk) =>
{
// create a DetectAndHashJob for this chunk
_jobRepository.ScheduleJob <DetectAndHashJob>(new DetectAndHashJobArgs()
{
RepositoryId = args.Id,
Shas = chunk
});
});
return Task.FromResult <JobResult>(new JobSuccessResult(this));
}
catch (Exception e)
{
return Task.FromResult <JobResult>(new JobFailureResult(e, this));
}
}));
}
