public static void DownloadProjects(
IRepositorySource source,
string outputDir,
IEnumerable <string> projects
)
{
if (!File.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
foreach (var project in projects)
{
string projectName =
project.Split('/').Reverse().Skip(1).First() + "_" +
project.Split('/').Last();
var output = Path.Combine(outputDir, projectName + ".zip");
if (source.DownloadZip(project, output))
{
Console.WriteLine("Downloaded {0}", project);
}
Thread.Sleep(10000);
}
}
public async Task CreateRepositoryAsync(IRepositorySource source, IRepository repository, CancellationToken cancellationToken)
{
await EnsureAccessToken(cancellationToken);
var json = JsonSerializer.Serialize(new Repository
{
Name = repository.Name,
});
var project = DevOpsProject;
if (project == null)
{
if (!(repository is AzureDevOpsRepository adoRepo))
{
throw new NotSupportedException($"Cannot create DevOps repository from {repository.GetType()}");
}
project = adoRepo.Project;
if (!(source is AzureDevOpsRepositoryTarget adoSource))
{
throw new NotSupportedException($"Can only mirror entire Azure DevOps organization to another Azure DevOps organization, but found source: {source.GetType()}");
}
// may be missing in target org
await CreateProjectAsync(adoSource, adoRepo, cancellationToken);
}
var response = await HttpClient.PostAsync($"https://dev.azure.com/{DevOpsOrganization}/{project}/_apis/git/repositories?api-version=5.1", new StringContent(json, Encoding.UTF8, "application/json"), cancellationToken);
response.EnsureSuccessStatusCode();
}
public static async Task <ICollection <ProjectFile> > GetProjectFiles(this IRepositorySource repository)
{
var files = new List <ProjectFile>();
foreach (var path in await repository.FindProjectFiles())
{
files.Add(await repository.GetProjectFile(path));
}
return(files);
}
public async Task ProcessAsync(
IRepositorySource source,
IMirrorService mirrorService,
IRepositoryTarget[] targets,
CancellationToken cancellationToken)
{
if (!targets.Any())
{
throw new NotSupportedException("At least one target is required!");
}
var repos = await source.GetRepositoriesAsync(cancellationToken);
_log.LogInformation($"Source {source.SourceId} has {repos.Length} repositories (to be mirrored by {mirrorService.MirrorId})");
var mirroredRepositories = (await mirrorService.GetExistingMirrorsAsync(cancellationToken))
.Select(x => x.Repository)
.ToArray();
var toMirror = repos.Where(r => !mirroredRepositories.Contains(r.Name)).ToArray();
if (toMirror.Length == 0)
{
_log.LogInformation("All repositories are already mirrored!");
return;
}
_log.LogInformation($"{repos.Length - toMirror.Length} repositories are already mirrored, setting up mirrors for the remaining {toMirror.Length} repositories..");
var cachedResponse = new Dictionary <IRepositoryTarget, IRepository[]>();
foreach (var target in targets)
{
cachedResponse[target] = await target.GetRepositoriesAsync(cancellationToken);
}
foreach (var repo in toMirror)
{
_log.LogInformation($"Creating mirror for repository {repo.Name}");
// create mirror last as it is the marker whether a repo mirror already exists
// creating repositories is also idempotent
foreach (var target in targets)
{
var existing = cachedResponse[target];
if (existing.Any(e => e.Name == repo.Name))
{
continue;
}
await target.CreateRepositoryAsync(source, repo, cancellationToken);
}
await mirrorService.SetupMirrorAsync(repo, cancellationToken);
_log.LogInformation($"Created mirror for repository {repo.Name}");
}
}
public async Task CreateRepositoryAsync(IRepositorySource source, IRepository repository, CancellationToken cancellationToken)
{
var json = JsonSerializer.Serialize(new
{
// must set path, if setting name then path will be lowerecased
// https://docs.gitlab.com/ee/api/projects.html#create-project
path = repository.Name,
visibility = "public",
description = repository.Description
});
var response = await _httpClient.PostAsync("projects", new StringContent(json, Encoding.UTF8, "application/json"), cancellationToken);
response.EnsureSuccessStatusCode();
}
private void WriteComponentLabels(IRepositorySource source)
{
string transformLabel = source.Transform != null ? source.Transform.Description : "none";
Console.ForegroundColor = ConsoleColor.DarkYellow;
_logger.WriteLine($"Source: {source.Description}");
_logger.WriteLine($"Destination: {source.Destination.Description}");
_logger.WriteLine($"Transform: {transformLabel}");
_logger.WriteLine($"Date: {DateTime.Now.ToString()}");
_logger.WriteLine($"Computer: {Environment.MachineName}");
_logger.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
}
public AzurePipelinesMirror(
MirrorViaConfig config,
IRepositorySource repositorySource,
ILogger log)
: base(config.Type, config.AccessToken)
{
_config = config;
_repositorySource = repositorySource;
_log = log;
if (DevOpsProject == null)
{
throw new NotSupportedException("Mirror must be set up in a specific project and does not support organization level!");
}
}
public static void Download()
{
// 1) Get Known list of Java projects
IRepositorySource source = ProjectSources.Github;
var projects = System.IO.File.ReadAllText(DataSources.JavaScriptProjectsListingsPath)
.Split('\n')
.Select(p => p.Replace("\"", "").Trim())
.ToList();
// 2) Throlled/Restartable Download.
var list = new List <string>();
for (int i = 0; i < 10000; i++)
{
list.Add(projects.GetRandomElement());
}
BuildRepository.DownloadProjects(source, "projects", list);
}
public async Task CreatePullRequestsAsync(
IRepositorySource ops,
string[] reviewers,
int?prLimit = null)
{
var changeSets = await GetChangeSets(ops);
_log.WriteLine("Opening PRs");
foreach (var pr in changeSets.Take(prLimit.GetValueOrDefault(int.MaxValue)))
{
try
{
await ops.SubmitPR(pr, reviewers);
}
catch
{
_log.WriteLine($"Unable to open PR for {pr.BranchName}");
}
}
}
public IMirrorService GetMirrorService(MirrorViaConfig mirrorConfig, IRepositorySource source)
=> new Uri(mirrorConfig.Type).Host.ToLowerInvariant() switch
{
"dev.azure.com" => new AzurePipelinesMirror(mirrorConfig, source, _log),
public IpcServer(IIpcEndpoint endpointProvider, IRepositorySource repositorySource)
{
EndpointProvider = endpointProvider ?? throw new ArgumentNullException(nameof(endpointProvider));
RepositorySource = repositorySource ?? throw new ArgumentNullException(nameof(repositorySource));
}
public DomainRepository(IRepositorySource <TDomain> repositorySource)
{
this.repositorySource = repositorySource;
}
public static void Run()
{
bool download = false;
bool analyze = false;
if (download)
{
// 1) Get Known list of C# projects
IRepositorySource source = ProjectSources.Github;
var projects = System.IO.File.ReadAllText(DataSources.CSharpProjectsListingsPath)
.Split('\n')
.Select(p => p.Replace("\"", ""));
// 2) Throlled/Restartable Download.
var list = new List <string>();
for (int i = 0; i < 10; i++)
{
list.Add(projects.GetRandomElement());
}
BuildRepository.DownloadProjects(source, "projects", list);
}
if (analyze)
{
// 3) Analyze project zips to count code grams
CodegramBuilder builder = new CodegramBuilder(".cs", ".xaml");
var codegrams = builder.Build("projects");
// 4) Save codegrams to DB.
CodegramsDB.Init(codegrams, "codegrams.db");
}
// 5) Test queries.
var reader = new CodegramsReader();
reader.Connect("codegrams.db");
Console.WriteLine(reader.SequenceIdentifierFrequency(new string[] { "using", "System" }));
Console.WriteLine(reader.SequenceWordFrequency(new string[] { "using", "System" }));
Console.WriteLine(reader.LineSalience(2, "using System"));
Console.WriteLine(reader.LineSalience(2, "Console.WriteLine"));
Console.WriteLine(reader.LineSalience(2, "CodegramsDB.Init"));
Console.WriteLine(reader.LineSalience(2, "using Codegrams.Persistance"));
var lines =
File.ReadAllText(@"C:\DEV\github\Codegrams\CodegramsClient\Program.cs")
.Split('\n')
.Select(line =>
new
{
Salience = reader.LineSalience(2, line),
Line = line
}
).OrderBy(salientLine => 1 - salientLine.Salience);
foreach (var line in lines)
{
Console.WriteLine(line.Salience + ":" + line.Line);
}
}
public async Task <IReadOnlyCollection <ChangeSet> > GetChangeSets(IRepositorySource ops)
{
_log.WriteLine("Fetching project files");
var projectFiles = await ops.GetProjectFiles();
var allPackages = projectFiles.SelectMany(x => x.ListPackages()).Select(x => x.Key).Distinct().ToArray();
_log.WriteLine("Fetching latest packages");
var nugetPackages = await Task.WhenAll(_nugetApis.Select(x => x.GetPackageVersions(allPackages, AllowBetaPackages)));
var latestPackageVersions =
nugetPackages
.SelectMany(x => x)
.GroupBy(x => x.Key)
.ToDictionary(x => x.Key, x => x.OrderByDescending(y => y.Value.Version).FirstOrDefault().Value);
HashSet <string> packagesToUpdate = new HashSet <string>();
var packages = projectFiles.SelectMany(x => x.ListPackages())
.GroupBy(x => x.Key)
.ToDictionary(x => x.Key, x => x.First().Value);
foreach (var item in latestPackageVersions)
{
var projectVersion = NuGetVersion.Parse(packages[item.Key]);
var lastVersion = item.Value;
if (lastVersion.Version > projectVersion)
{
// Major minor versions older than 21 days
var shouldUpdateMajorMinor =
(lastVersion.Version.Major > projectVersion.Major || lastVersion.Version.Minor > projectVersion.Minor) &&
lastVersion.Released <= DateTime.Now.AddDays(-PackagesMustBePublishedForThisManyDays);
var shouldUpdatePatch = lastVersion.Version.Major == projectVersion.Major &&
lastVersion.Version.Minor == projectVersion.Minor &&
lastVersion.Version > projectVersion;
if (shouldUpdateMajorMinor || shouldUpdatePatch)
{
packagesToUpdate.Add($"{item.Key}|{item.Value.Version}");
}
}
}
var packageGroupings =
packagesToUpdate
.Select(x => x.Split('|'))
.GroupBy(x => PackageGroupings.Select(y => y.GetGroupName(x[0], x[1])).First(y => y != null),
x => x[0]);
var changeSets = new List <ChangeSet>();
foreach (var group in packageGroupings)
{
var message =
new StringBuilder($"Auto-update for packages related to '{@group.Key}'");
message.AppendLine();
message.AppendLine();
var groupChangeSet = new ChangeSet
{
BranchName = $"refs/heads/auto-nuget-update/{@group.Key.ToLowerInvariant()}",
};
foreach (var package in @group)
{
var packageUpdateMessage =
$"Updates package '{package}' to version '{latestPackageVersions[package].Version}'";
if (@group.Count() == 1)
{
message = new StringBuilder(packageUpdateMessage);
}
else
{
message.AppendLine(packageUpdateMessage);
}
foreach (var projectFile in projectFiles)
{
projectFile.UpdatePackageReference(package,
latestPackageVersions[package].Version.ToString());
}
}
foreach (var projectFile in projectFiles)
{
var change = projectFile.ToChange();
if (change != null)
{
groupChangeSet.Changes.Add(change);
}
projectFile.Reset();
}
groupChangeSet.Message = message.ToString();
changeSets.Add(groupChangeSet);
}
return(changeSets);
}
public async Task CreatePullRequestsAsync(IRepositorySource ops, int?prLimit = null)
{
await CreatePullRequestsAsync(ops, new string[0], prLimit);
}
