/// <summary>
/// Mirrors the given repositories, queueing their plugins up for download. This will merge the repositories
/// together into a single filtered repository to save bandwidth.
/// </summary>
/// <param name="repositories">The repositories to mirror.</param>
/// <param name="ct">The cancellation token in use.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public async Task MirrorRepositoriesAsync(IReadOnlyCollection <IdeaPluginRepository> repositories, CancellationToken ct)
{
await Console.Out.WriteLineAsync("Merging requested versioned repositories (this might take a while)...");
var newCategories = new Dictionary <string, Dictionary <int, IdeaPlugin> >();
var totalRepositories = repositories.Count;
var mergedRepositories = 0;
foreach (var repository in repositories)
{
foreach (var category in repository.Categories)
{
if (!newCategories.TryGetValue(category.Name, out var newCategory))
{
newCategory = new Dictionary <int, IdeaPlugin>();
newCategories.Add(category.Name, newCategory);
}
foreach (var plugin in category.Plugins)
{
if (!newCategory.TryGetValue(plugin.GetIdentityHash(), out _))
{
newCategory.Add(plugin.GetIdentityHash(), plugin);
}
}
}
++mergedRepositories;
await Console.Out.WriteLineAsync($"Merged repository {mergedRepositories} out of {totalRepositories}.");
}
var mergedCategories = newCategories.Select
(
kvp =>
new IdeaPluginCategory
(
kvp.Key,
kvp.Value.Values.ToList()
)
);
var mergedRepository = new IdeaPluginRepository(mergedCategories.ToList());
await MirrorRepositoryAsync(mergedRepository, ct);
}
/// <summary>
/// Serves GET requests to this controller.
/// </summary>
/// <param name="build">The build to list plugins for.</param>
/// <returns>The plugins compatible with the given IDE version.</returns>
public ActionResult <IdeaPluginRepository> Get(string build)
{
if (!IDEVersion.TryParse(build, out var ideVersion))
{
return(BadRequest());
}
var categories = _database.Categories.ToList();
var ideaCategories = categories.Select(c => new IdeaPluginCategory(c.Name)).ToList();
foreach (var category in _database.Categories)
{
var ideaCategory = ideaCategories.First(c => c.Name == category.Name);
foreach (var plugin in category.Plugins)
{
var compatibleRelease = plugin.Releases
.OrderBy(r => r.UploadedAt)
.FirstOrDefault(r => r.CompatibleWith.IsInRange(ideVersion));
if (compatibleRelease is null)
{
continue;
}
var ideaPlugin = new IdeaPlugin
(
plugin.Name,
plugin.PluginID,
plugin.Description,
compatibleRelease.Version,
new IdeaVendor
{
Email = plugin.Vendor.Email,
Name = plugin.Vendor.Name,
URL = plugin.Vendor.URL
},
new IdeaVersion
{
UntilBuild = compatibleRelease.CompatibleWith.UntilBuild.IsValid ? compatibleRelease.CompatibleWith.UntilBuild.ToString() : null,
SinceBuild = compatibleRelease.CompatibleWith.SinceBuild.IsValid ? compatibleRelease.CompatibleWith.SinceBuild.ToString() : null,
},
compatibleRelease.ChangeNotes
)
{
ProjectURL = plugin.ProjectURL,
Tags = plugin.Tags.Aggregate((a, b) => a + ";" + b),
Depends = compatibleRelease.Dependencies,
Downloads = compatibleRelease.Downloads,
Size = compatibleRelease.Size,
Rating = plugin.Rating,
UploadDate = (compatibleRelease.UploadedAt.ToFileTimeUtc() / 10000).ToString(),
UpdateDate = (plugin.Releases.Select(r => r.UploadedAt).OrderBy(d => d).First().ToFileTimeUtc() / 10000).ToString()
};
ideaCategory.Plugins.Add(ideaPlugin);
}
}
var ideaRepository = new IdeaPluginRepository();
ideaRepository.Categories = ideaCategories;
return(new ActionResult <IdeaPluginRepository>(ideaRepository));
}
private static async Task ImportRepositoryAsync(IServiceProvider services, IdeaPluginRepository repository)
{
async Task ImportPluginReleaseScoped(IdeaPlugin pluginRelease)
{
using var scope = services.CreateScope();
await using var db = scope.ServiceProvider.GetRequiredService <PluginsDatabaseContext>();
if (await ImportPluginReleaseAsync(db, pluginRelease))
{
await db.SaveChangesAsync();
}
}
async Task ImportPluginScopedAsync(IdeaPlugin pluginDefinition, IdeaPluginCategory category)
{
using var scope = services.CreateScope();
await using var db = scope.ServiceProvider.GetRequiredService <PluginsDatabaseContext>();
if (await ImportPluginAsync(db, pluginDefinition, category))
{
await db.SaveChangesAsync();
}
}
async Task ImportCategoryScopedAsync(IdeaPluginCategory category)
{
using var scope = services.CreateScope();
await Console.Out.WriteLineAsync($"Importing category \"{category.Name}\"...");
await using var db = scope.ServiceProvider.GetRequiredService <PluginsDatabaseContext>();
if (await ImportCategoryAsync(db, category))
{
await db.SaveChangesAsync();
}
}
// Stage 1: Import categories
await Console.Out.WriteLineAsync("Importing categories and plugin definitions...");
await Task.WhenAll(repository.Categories.Select(ImportCategoryScopedAsync));
var importDefinitionTasks = new List <Task>();
// Stage 2: Import plugin definitions
foreach (var category in repository.Categories)
{
var pluginDefinitions = category.Plugins.GroupBy(p => p.ID).Select(g => g.First()).ToList();
await Console.Out.WriteLineAsync
(
$"Importing {pluginDefinitions.Count} plugins from \"{category.Name}\"..."
);
importDefinitionTasks.AddRange(pluginDefinitions.Select(p => ImportPluginScopedAsync(p, category)));
}
await Task.WhenAll(importDefinitionTasks);
await Console.Out.WriteLineAsync("Importing releases...");
var allReleases = repository.Categories.SelectMany(c => c.Plugins).ToList();
// Stage 3: Import plugin releases
var remaining = allReleases.Count;
foreach (var plugin in allReleases.Batch(16))
{
var enumeratedBatch = plugin.ToList();
var importReleaseTasks = enumeratedBatch.Select(ImportPluginReleaseScoped);
await Task.WhenAll(importReleaseTasks);
remaining -= enumeratedBatch.Count;
await Console.Out.WriteLineAsync
(
$"Imported batch ({enumeratedBatch.Count}) - {remaining} releases remaining."
);
}
}
/// <summary>
/// Mirrors the given repository, queueing its plugins up for download.
/// </summary>
/// <param name="repository">The repository to mirror.</param>
/// <param name="ct">The cancellation token in use.</param>
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
public async Task MirrorRepositoryAsync(IdeaPluginRepository repository, CancellationToken ct)
{
var totalSize = new ByteSize
(
repository.Categories.SelectMany(p => p.Plugins).Select(p => p.Size).Aggregate((a, b) => a + b)
);
await Console.Out.WriteLineAsync
(
$"Estimated total download size: " +
$"{totalSize.LargestWholeNumberValue:F1} {totalSize.LargestWholeNumberSymbol}\n"
);
var baseDirectory = Path.Combine(Program.Options.OutputFolder, "plugins");
if (Program.Options.VerboseOutput)
{
await Console.Out.WriteLineAsync("Creating directory tree...");
}
Directory.CreateDirectory(baseDirectory);
foreach (var category in repository.Categories)
{
Directory.CreateDirectory(Path.Combine(baseDirectory, category.Name.GenerateSlug()));
}
await Console.Out.WriteLineAsync("Done. Starting mirroring...");
var categoryDownloads = new List <(string CategoryName, Task <DownloadResult[]> Results)>();
foreach (var category in repository.Categories)
{
var finalizedDownloads = new List <Task <DownloadResult> >();
var targetDirectory = Path.Combine(baseDirectory, category.Name.GenerateSlug());
if (Program.Options.VerboseOutput)
{
await Console.Out.WriteLineAsync
(
$"Downloading {category.Plugins.Count} plugins from \"{category.Name}\"..."
);
}
foreach (var plugin in category.Plugins)
{
if (Program.Options.MirrorAllVersions)
{
try
{
var pluginVersions = await _api.ListVersionsAsync(plugin.ID, ct);
foreach (var versionedPlugin in pluginVersions)
{
var downloadTask = FinalizeDownload
(
DownloadPluginAsync(targetDirectory, versionedPlugin, ct)
);
finalizedDownloads.Add(downloadTask);
}
}
catch (TimeoutException)
{
await Console.Out.WriteLineAsync
(
$"[{nameof(RepositoryMirrorer)}]: Failed to fetch version information for " +
$"{plugin.Name}: The download timed out."
);
}
}
else
{
var downloadTask = FinalizeDownload(DownloadPluginAsync(targetDirectory, plugin, ct));
finalizedDownloads.Add(downloadTask);
}
}
categoryDownloads.Add((category.Name, Task.WhenAll(finalizedDownloads)));
}
// Finally, write the successfully mirrored repository data back out to disk
var categoryResults = await Task.WhenAll(categoryDownloads.Select(async pair =>
{
var(categoryName, task) = pair;
var results = await task;
return(categoryName, results);
}));
var mirroredRepository = new IdeaPluginRepository();
foreach (var(categoryName, downloadResults) in categoryResults)
{
var category = mirroredRepository.Categories.FirstOrDefault(c => c.Name == categoryName) ??
new IdeaPluginCategory(categoryName);
if (!mirroredRepository.Categories.Contains(category))
{
mirroredRepository.Categories.Add(category);
}
category.Plugins.AddRange(downloadResults.Where(r => r.IsSuccess).Select(r => r.Plugin !));
}
var repoPath = Path.Combine(baseDirectory, "repository.xml");
if (File.Exists(repoPath))
{
File.Delete(repoPath);
}
var serializer = new XmlSerializer(typeof(IdeaPluginRepository));
await using var output = File.OpenWrite(repoPath);
serializer.Serialize(output, mirroredRepository);
}