public async Task <PackagesLock> DownloadModuleLock(string outputPath, IEnumerable <string> loadOnStartup)
{
var modulesLock = await ModulesResource.FetchModules(PrismModule.ClientFramework, _restClient);
var requiredPackages = new Dictionary <PackageIdentity, IImmutableList <PackageIdentity> >();
void AddPackages(IEnumerable <PackageIdentity> identities)
{
foreach (var identity in identities)
{
if (!requiredPackages.ContainsKey(identity))
{
var dependencies = modulesLock[identity];
requiredPackages.Add(identity, dependencies);
AddPackages(dependencies);
}
}
}
AddPackages(loadOnStartup.Select(x => modulesLock.First(y => y.Key.Id.Equals(x, StringComparison.OrdinalIgnoreCase)).Key));
var requiredPackagesLock = new PackagesLock(requiredPackages);
_fileSystem.File.WriteAllText(_fileSystem.Path.Combine(outputPath, "modules.lock"), JsonConvert.SerializeObject(requiredPackagesLock));
return(requiredPackagesLock);
}
private PackageLockStatus CheckPackageLockStatus(PackagesLock currentLock, PackagesLock newLock)
{
foreach (var packageLockEntry in newLock)
{
if (currentLock.TryGetValue(packageLockEntry.Key, out var dependencies))
{
if (!packageLockEntry.Value.ScrambledEquals(dependencies))
{
return
(PackageLockStatus.Changed); //something seems fishy as the dependencies don't match even if the package versions are equal
}
}
else if (currentLock.Any(x => string.Equals(x.Key.Id, packageLockEntry.Key.Id, StringComparison.OrdinalIgnoreCase)))
{
//version changed
return(PackageLockStatus.Changed);
}
else
{
return(PackageLockStatus.PackagesAddedOrRemoved);
}
}
if (currentLock.Any(x => !newLock.Any(y => string.Equals(y.Key.Id, x.Key.Id, StringComparison.OrdinalIgnoreCase))))
{
return(PackageLockStatus.PackagesAddedOrRemoved);
}
//we don't really care if packages were removed
return(PackageLockStatus.Equal);
}
public bool Verify(PackagesLock packagesLock)
{
var packagesLockFile = _fileSystem.FileInfo.FromFileName(Environment.ExpandEnvironmentVariables(_options.ModulesLockPath));
if (!packagesLockFile.Exists)
{
_fileSystem.File.WriteAllText(packagesLockFile.FullName, JsonConvert.SerializeObject(packagesLock));
return(true);
}
var currentPackagesLock = JsonConvert.DeserializeObject <PackagesLock>(_fileSystem.File.ReadAllText(packagesLockFile.FullName));
var status = CheckPackageLockStatus(currentPackagesLock, packagesLock);
if (status == PackageLockStatus.Equal)
{
return(true);
}
_fileSystem.File.WriteAllText(packagesLockFile.FullName, JsonConvert.SerializeObject(packagesLock));
if (status == PackageLockStatus.PackagesAddedOrRemoved)
{
return(true); //packages can be added
}
//if status == PackageLockStatus.Changed
return(false);
}
public async Task Load(IEnumerable <PackageIdentity> primaryPackages, PackagesLock packagesLock)
{
var mapper = new ModuleMapper(_project.Framework, _project.ModulesDirectory, _project.Runtime, _project.Architecture);
var map = mapper.BuildMap(packagesLock);
var dependencyPaths = new Dictionary <AssemblyName, AssemblyInfo>(new AssemblyNameComparer());
_dependencyAssemblies = dependencyPaths;
while (map.TryPop(out var dependencyLayer))
{
foreach (var context in dependencyLayer.Where(x => !x.IsMazeModule))
{
foreach (var file in Directory.GetFiles(context.LibraryDirectory, "*.dll"))
{
dependencyPaths.Add(AssemblyName.GetAssemblyName(file), new AssemblyInfo(file));
}
}
await TaskCombinators.ThrottledAsync(dependencyLayer.Where(x => x.IsMazeModule), (context, token) => Task.Run(() =>
{
foreach (var file in Directory.GetFiles(context.LibraryDirectory, "*.dll"))
{
var assembly = _assemblyLoadContext.LoadFromAssemblyPath(file);
var typeDiscoveryService = new TypeDiscoveryService(assembly, context.Package);
typeDiscoveryService.DiscoverTypes(ModuleTypeMap);
}
}), CancellationToken.None);
}
}
public async Task <IReadOnlyList <PackageCarrier> > Load(PackagesLock packagesLock)
{
var mapper = new ModuleMapper(_framework, _modulesDirectory, Runtime.Windows, Environment.Is64BitProcess ? Architecture.x64 : Architecture.x86);
var packageStack = mapper.BuildMap(packagesLock);
var loadedPackages = new ConcurrentBag <PackageCarrier>();
while (packageStack.Any())
{
var dependencyLayer = packageStack.Pop();
await TaskCombinators.ThrottledAsync(dependencyLayer, (context, token) =>
{
return(Task.Run(() =>
{
foreach (var module in LoadModule(context))
{
loadedPackages.Add(module);
}
}));
}, CancellationToken.None);
}
Packages = Packages.AddRange(loadedPackages);
return(loadedPackages.ToList());
}
public void Synchronize(List <ClientConfigurationDataDto> configurations, PackagesLock packagesLock)
{
var configurationDirectory = _fileSystem.DirectoryInfo.FromDirectoryName(Environment.ExpandEnvironmentVariables(Program.ConfigDirectory));
var existingConfigurations = configurationDirectory.GetFiles("mazesettings*.json")
.Select(x => Regex.Match(x.Name, "mazesettings([0-9])\\.json")).Select(x => x.Success ? int.Parse(x.Groups[0].Value) : (int?)null);
foreach (var configurationId in existingConfigurations.Where(x => configurations.All(y => y.ClientGroupId != x)))
{
_fileSystem.File.Delete(Path.Combine(Program.ConfigDirectory, $"mazesettings{configurationId}.json"));
}
// var filesProvider = new MemoryFileProvider(configurations.ToDictionary(x => $"mazesettings{x.ClientGroupId}.json", x => x.Content));
// var configBuilder = new ConfigurationBuilder();
//#if DEBUG
// configBuilder.AddJsonFile("mazesettings.json");
//#endif
// foreach (var configuration in configurations)
// {
// configBuilder.AddJsonFile(filesProvider, $"mazesettings{configuration.ClientGroupId}.json", optional: false, reloadOnChange: false);
// }
//var configurationRoot = configBuilder.Build();
////var modules = configurationRoot.GetSection("LoadOnStartup").GetChildren();
////var packagesConfig = packagesLock; //TrimPackagesLock(modules.Select(x => x.Key), packagesLock);
//_fileSystem.File.WriteAllText(Environment.ExpandEnvironmentVariables(_options.ModulesLockPath),
// JsonConvert.SerializeObject(packagesConfig));
foreach (var configuration in configurations)
{
var filename = Path.Combine(Environment.ExpandEnvironmentVariables(Program.ConfigDirectory), $"mazesettings{configuration.ClientGroupId}.json");
_fileSystem.File.WriteAllText(filename, configuration.Content);
}
}
public async Task EnsurePackagesInstalled(PackagesLock packagesLock)
{
var resources = _project.PrimarySources.Concat(_project.DependencySources)
.ToImmutableDictionary(x => x.GetResource <FindPackageByIdResource>(), x => x);
var cache = GetDefaultResolutionContext().SourceCacheContext;
var actions = (await TaskCombinators.ThrottledAsync(packagesLock, async(package, token) =>
{
if (!_project.ModulesDirectory.ModuleExists(package.Key))
{
foreach (var(resource, repo) in resources)
{
var versions =
await resource.GetAllVersionsAsync(package.Key.Id, cache, _logger, CancellationToken.None);
if (versions.Any(x => x == package.Key.Version))
{
return(ResolvedAction.CreateInstall(package.Key, repo));
}
}
}
return(null);
}, CancellationToken.None)).Where(x => x != null).ToList();
if (actions.Any())
{
await ExecuteActionsAsync(actions, GetDefaultDownloadContext(), CancellationToken.None);
}
}
public async Task <PackageLockContext> Load(PackagesLock packagesLock)
{
if (packagesLock?.Any() != true)
{
return(new PackageLockContext(_catalog));
}
var packages = await _catalog.Load(packagesLock);
return(new PackageLockContext(packages, _packagesRegistrar, _catalog));
}
public async Task <Dictionary <PackageIdentity, PackagePreFetcherResult> > DownloadPackages(
PackagesLock packagesLock, PackageDownloadContext downloadContext, ILogger logger, CancellationToken token)
{
var packages = packagesLock.Keys.Concat(packagesLock.SelectMany(x => x.Value)).Distinct();
var toDownload = new Queue <PackageIdentity>();
var result = new Dictionary <PackageIdentity, PackagePreFetcherResult>();
foreach (var package in packages)
{
if (_modulesDirectory.ModuleExists(package))
{
continue;
}
toDownload.Enqueue(package);
}
if (toDownload.Any())
{
var maxParallelTasks = PackageManagementConstants.DefaultMaxDegreeOfParallelism;
var downloadResults = new List <PackagePreFetcherResult>(maxParallelTasks);
while (toDownload.Count > 0)
{
// Throttle tasks
if (downloadResults.Count == maxParallelTasks)
{
// Wait for a task to complete
// This will not throw, exceptions are stored in the result
await Task.WhenAny(downloadResults.Select(e => e.EnsureResultAsync()));
// Remove all completed tasks
downloadResults.RemoveAll(e => e.IsComplete);
}
var package = toDownload.Dequeue();
// Download the package if it does not exist in the packages folder already
// Start the download task
var task = Task.Run(
async() => await PackageDownloader.GetDownloadResourceResultAsync(_sourceRepository, package,
downloadContext, "NO_GLOBAL_CACHE_DIRECTORY", logger, token), token);
var downloadResult = new PackagePreFetcherResult(task, package, _sourceRepository.PackageSource);
downloadResults.Add(downloadResult);
result.Add(package, downloadResult);
}
}
return(result);
}
private static void SearchDependencies(PackageIdentity packageIdentity, PackagesLock packagesLock,
IDictionary <PackageIdentity, int> map, int level)
{
if (!map.TryGetValue(packageIdentity, out var currentLevel) || level > currentLevel)
{
map[packageIdentity] = level;
}
foreach (var dependency in packagesLock[packageIdentity])
{
SearchDependencies(dependency, packagesLock, map, level + 1);
}
}
public async Task Load(PackagesLock packagesLock, IServerInfo serverInfo, CancellationToken token)
{
var serverRepo = new ServerRepository(new Uri(serverInfo.ServerUri, "nuget"));
var context = new PackageDownloadContext(new SourceCacheContext {
DirectDownload = true, NoCache = true
},
Environment.ExpandEnvironmentVariables(_options.TempPath), true);
var packageDownloadManager = new PackageDownloadManager(_modulesDirectory, serverRepo);
var result =
await packageDownloadManager.DownloadPackages(packagesLock, context, NullLogger.Instance, token);
if (result.Any())
{
try
{
foreach (var preFetchResult in result.Values)
{
using (var downloadPackageResult = await preFetchResult.GetResultAsync())
{
// use the version exactly as specified in the nuspec file
var packageIdentity = await downloadPackageResult.PackageReader.GetIdentityAsync(token);
var packageExtractionContext = new PackageExtractionContext(
PackageSaveMode.Defaultv3,
PackageExtractionBehavior.XmlDocFileSaveMode,
ClientPolicyContext.GetClientPolicy(new NullSettings(), new NullLogger()),
new NullLogger());
downloadPackageResult.PackageStream.Position = 0;
await PackageExtractor.InstallFromSourceAsync(serverRepo.PackageSource.Source,
packageIdentity, stream => downloadPackageResult.PackageStream.CopyToAsync(stream),
_modulesDirectory.VersionFolderPathResolver, packageExtractionContext, token);
}
}
}
finally
{
foreach (var fetcherResult in result.Values)
{
await fetcherResult.EnsureResultAsync();
fetcherResult.Dispose();
}
}
}
}
/// <summary>
/// Build a module loader map that determines in which order which packages must be loaded
/// </summary>
/// <param name="packagesLock">The packages lock</param>
/// <returns>Return a stack of dependency lists that can be loaded in parallel</returns>
public Stack <List <PackageLoadingContext> > BuildMap(PackagesLock packagesLock)
{
var levelMap = new Dictionary <PackageIdentity, int>(PackageIdentity.Comparer);
foreach (var packageIdentity in packagesLock.Keys)
{
SearchDependencies(packageIdentity, packagesLock, levelMap, 0);
}
var result = new Stack <List <PackageLoadingContext> >();
//the first item of this foreach loop must be loaded last (because it has the lowest level)
foreach (var levelGroup in levelMap.GroupBy(x => x.Value).OrderBy(x => x.Key))
{
var levelList = new List <PackageLoadingContext>();
foreach (var packageIdentity in levelGroup.Select(x => x.Key))
{
if (!ModulesDirectory.ModuleExists(packageIdentity))
{
throw new FileNotFoundException($"The package {packageIdentity} was not found.");
}
var packageDirectory =
ModulesDirectory.VersionFolderPathResolver.GetInstallPath(packageIdentity.Id,
packageIdentity.Version);
var resolvedDirectory = LoadResolver.ResolveNuGetFolder(packageDirectory, MazeFramework, Runtime, Architecture);
if (resolvedDirectory == null) //maybe build only package?
{
continue;
}
levelList.Add(new PackageLoadingContext(packageIdentity, packageDirectory, resolvedDirectory.Directory.FullName,
resolvedDirectory.Framework, resolvedDirectory.Framework.Framework == MazeFramework.Framework));
}
result.Push(levelList);
}
return(result);
}
public async Task <PackagesLock> DownloadModuleLock(string outputPath, IEnumerable <string> loadOnStartup)
{
var modulesLock = await ModulesResource.FetchModules(PrismModule.ClientFramework, _restClient);
var requiredPackages = new Dictionary <PackageIdentity, IImmutableList <PackageIdentity> >();
void AddPackages(IEnumerable <PackageIdentity> identities)
{
foreach (var identity in identities)
{
if (!requiredPackages.ContainsKey(identity))
{
var dependencies = modulesLock[identity];
requiredPackages.Add(identity, dependencies);
AddPackages(dependencies);
}
}
}
var loadOnStartupPackages = new List <PackageIdentity>();
foreach (var module in loadOnStartup)
{
var lockedPackage = modulesLock.Where(y => y.Key.Id.Equals(module, StringComparison.OrdinalIgnoreCase));
if (!lockedPackage.Any())
{
_logger.LogWarning($"The module {module} is not installed on the server and cannot be loaded on startup. Please remove it from the config.");
continue;
}
loadOnStartupPackages.Add(lockedPackage.First().Key);
}
AddPackages(loadOnStartupPackages);
var requiredPackagesLock = new PackagesLock(requiredPackages);
_fileSystem.File.WriteAllText(_fileSystem.Path.Combine(outputPath, "modules.lock"), JsonConvert.SerializeObject(requiredPackagesLock));
return(requiredPackagesLock);
}
public ServerConnection(MazeRestClient restClient, PackagesLock packagesLock, MazeSocketOptions options)
{
_options = options;
RestClient = restClient;
PackagesLock = packagesLock;
}
public async Task <Dictionary <PackageIdentity, FileInfoBase> > LoadPackages(PackagesLock packagesLock, DirectoryInfoBase tempDirectory,
CancellationToken cancellationToken)
{
var result = new Dictionary <PackageIdentity, FileInfoBase>();
var packagesToDownload = new List <PackageIdentity>();
foreach (var packageId in packagesLock.Keys)
{
var filePath = _versionFolderPathResolver.GetPackageFilePath(packageId.Id, packageId.Version);
var file = _fileSystem.FileInfo.FromFileName(filePath);
if (file.Exists)
{
_logger.LogDebug("Package {packageId} could be resolved locally at {path}", packageId, file.FullName);
result.Add(packageId, file);
}
else
{
_logger.LogDebug("Package {packageId} could not be resolved locally", packageId);
packagesToDownload.Add(packageId);
}
cancellationToken.ThrowIfCancellationRequested();
}
if (packagesToDownload.Any())
{
_logger.LogInformation("{0} package(s) were not found locally and will be downloaded.", packagesToDownload.Count);
var nugetFolder = tempDirectory.CreateSubdirectory("nuget");
var tempFolderPathResolver = new VersionFolderPathResolverFlat(nugetFolder.FullName);
var serverRepo = new ServerRepository(new Uri(_restClient.Server.ServerUri, "nuget"));
var downloader = new PackageDownloadManager(new ModulesDirectory(tempFolderPathResolver), serverRepo);
var results = await downloader.DownloadPackages(
new PackagesLock(packagesToDownload.ToDictionary(x => x,
_ => (IImmutableList <PackageIdentity>)ImmutableList <PackageIdentity> .Empty)),
new PackageDownloadContext(new SourceCacheContext {
DirectDownload = true, NoCache = true
}, nugetFolder.FullName, true),
new NuGetLoggerWrapper(_logger), cancellationToken);
try
{
foreach (var preFetchResult in results.Values)
{
using (var downloadPackageResult = await preFetchResult.GetResultAsync())
{
var packageIdentity = await downloadPackageResult.PackageReader.GetIdentityAsync(cancellationToken);
_logger.LogInformation("Package {0} was successfully downloaded.", packageIdentity);
var fileStream = (FileStream)downloadPackageResult.PackageStream;
var file = _fileSystem.FileInfo.FromFileName(fileStream.Name);
if (!file.Exists)
{
_logger.LogError("Package {0} was not found at {1}. The download may have failed.", packageIdentity, file.FullName);
throw new FileNotFoundException($"The file {file.FullName} does not exist.");
}
var packagePath = _fileSystem.Path.Combine(_fileSystem.Path.GetDirectoryName(file.FullName), packageIdentity + ".nupkg");
using (var stream = _fileSystem.FileStream.Create(packagePath, FileMode.CreateNew))
await fileStream.CopyToAsync(stream);
result.Add(packageIdentity, _fileSystem.FileInfo.FromFileName(packagePath));
}
}
}
finally
{
foreach (var fetcherResult in results.Values)
{
await fetcherResult.EnsureResultAsync();
fetcherResult.Dispose();
}
}
}
return(result);
}
public async Task SetServerModulesLock(IReadOnlyList <PackageIdentity> primaryModules, PackagesLock serverLock)
{
await _modulesConfig.Replace(primaryModules);
await ModulesLock.Replace(new Dictionary <NuGetFramework, PackagesLock> {
{ Framework, serverLock }
});
}
public Task AddModulesLock(NuGetFramework framework, PackagesLock packagesLock)
{
return(ModulesLock.Add(framework, packagesLock));
}
