public async Task LoadProjectAsync(Path recipeFilePath) { var loadResult = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipeFilePath); if (loadResult.IsSuccess) { var packageDirectory = recipeFilePath.GetParent(); var targetPath = await GetTargetPathAsync(packageDirectory); var soupTargetDirectory = targetPath + new Path(".soup/"); var evaluateGraphFile = soupTargetDirectory + BuildConstants.GenerateEvaluateOperationGraphFileName; var fileSystemState = new FileSystemState(); if (!OperationGraphManager.TryLoadState(evaluateGraphFile, fileSystemState, out var evaluateGraph)) { NotifyError($"Failed to load Operation Graph: {evaluateGraphFile}"); return; } Graph = BuildGraph(fileSystemState, evaluateGraph); } else { NotifyError($"Failed to load Recipe file: {recipeFilePath}"); } }
public async Task LoadProjectAsync(Path recipeFilePath) { var loadResult = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipeFilePath); if (loadResult.IsSuccess) { var packageDirectory = recipeFilePath.GetParent(); var targetPath = await GetTargetPathAsync(packageDirectory); var soupTargetDirectory = targetPath + new Path(".soup/"); var generateInfoStateFile = soupTargetDirectory + BuildConstants.GenerateTaskInfoFileName; if (!ValueTableManager.TryLoadState(generateInfoStateFile, out var generateInfoTable)) { NotifyError($"Failed to load Value Table: {generateInfoStateFile}"); return; } Graph = BuildGraph(generateInfoTable); } else { NotifyError($"Failed to load Recipe file: {recipeFilePath}"); } }
/// <summary> /// Recursively restore all dependencies /// </summary> static async Task CheckRestoreRecursiveDependenciesAsync( Path recipeDirectory, Path packagesDirectory, Path stagingDirectory, IDictionary <string, IDictionary <string, PackageReference> > closure) { var recipePath = recipeDirectory + BuildConstants.RecipeFileName; var(isSuccess, recipe) = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipePath); if (!isSuccess) { throw new InvalidOperationException("Could not load the recipe file."); } if (closure.ContainsKey(recipe.Language) && closure[recipe.Language].ContainsKey(recipe.Name)) { Log.Diag("Recipe already processed."); } else { // Add the project to the closure if (!closure.ContainsKey(recipe.Language)) { closure.Add(recipe.Language, new Dictionary <string, PackageReference>()); } closure[recipe.Language].Add(recipe.Name, new PackageReference(recipeDirectory)); await RestoreRecursiveDependenciesAsync( recipeDirectory, recipe, packagesDirectory, stagingDirectory, closure); } }
/// <summary> /// Install a package /// </summary> public static async Task InstallPackageReferenceAsync(Path workingDirectory, string packageReference) { var recipePath = workingDirectory + BuildConstants.RecipeFileName; var(isSuccess, recipe) = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipePath); if (!isSuccess) { throw new InvalidOperationException("Could not load the recipe file."); } var packageStore = LifetimeManager.Get <IFileSystem>().GetUserProfileDirectory() + new Path(".soup/packages/"); Log.Info("Using Package Store: " + packageStore.ToString()); // Create the staging directory var stagingPath = EnsureStagingDirectoryExists(packageStore); try { // Parse the package reference to get the name var targetPackageReference = PackageReference.Parse(packageReference); string packageName = packageReference; if (!targetPackageReference.IsLocal) { packageName = targetPackageReference.GetName; } // Check if the package is already installed var packageNameNormalized = packageName.ToUpperInvariant(); if (recipe.HasRuntimeDependencies) { foreach (var dependency in recipe.RuntimeDependencies) { if (!dependency.IsLocal) { var dependencyNameNormalized = dependency.GetName.ToUpperInvariant(); if (dependencyNameNormalized == packageNameNormalized) { Log.Warning("Package already installed."); return; } } } } // Get the latest version if no version provided if (targetPackageReference.IsLocal) { var packageModel = await GetPackageModelAsync(recipe.Language, packageName); var latestVersion = new SemanticVersion(packageModel.Latest.Major, packageModel.Latest.Minor, packageModel.Latest.Patch); Log.HighPriority("Latest Version: " + latestVersion.ToString()); targetPackageReference = new PackageReference(packageModel.Name, latestVersion); } var closure = new Dictionary <string, IDictionary <string, PackageReference> >(); await CheckRecursiveEnsurePackageDownloadedAsync( recipe.Language, targetPackageReference.GetName, targetPackageReference.Version, packageStore, stagingPath, closure); // Cleanup the working directory Log.Info("Deleting staging directory"); LifetimeManager.Get <IFileSystem>().DeleteDirectory(stagingPath, true); // Register the package in the recipe Log.Info("Adding reference to recipe"); recipe.AddRuntimeDependency(targetPackageReference.ToString()); // Save the state of the recipe await RecipeExtensions.SaveToFileAsync(recipePath, recipe); } catch (Exception) { // Cleanup the staging directory and accept that we failed LifetimeManager.Get <IFileSystem>().DeleteDirectory(stagingPath, true); throw; } }
/// <summary> /// Ensure a package version is downloaded /// </summary> private static async Task CheckRecursiveEnsurePackageDownloadedAsync( string languageName, string packageName, SemanticVersion packageVersion, Path packagesDirectory, Path stagingDirectory, IDictionary <string, IDictionary <string, PackageReference> > closure) { if (closure.ContainsKey(languageName) && closure[languageName].ContainsKey(packageName)) { Log.Diag("Recipe already processed."); } else { // Add the new package to the closure if (!closure.ContainsKey(languageName)) { closure.Add(languageName, new Dictionary <string, PackageReference>()); } closure[languageName].Add(packageName, new PackageReference(packageName, packageVersion)); Log.HighPriority($"Install Package: {languageName} {packageName}@{packageVersion}"); var languageRootFolder = packagesDirectory + new Path(languageName); var packageRootFolder = languageRootFolder + new Path(packageName); var packageVersionFolder = packageRootFolder + new Path(packageVersion.ToString()) + new Path("/"); // Check if the package version already exists if (LifetimeManager.Get <IFileSystem>().Exists(packageVersionFolder)) { Log.HighPriority("Found local version"); } else { // Download the archive Log.HighPriority("Downloading package"); var archivePath = stagingDirectory + new Path(packageName + ".zip"); using (var httpClient = new HttpClient()) { var client = new Api.Client.PackageVersionClient(httpClient) { BaseUrl = SoupApiEndpoint, }; try { var result = await client.DownloadPackageVersionAsync(languageName, packageName, packageVersion.ToString()); // Write the contents to disk, scope cleanup using (var archiveWriteFile = LifetimeManager.Get <IFileSystem>().OpenWrite(archivePath, true)) { await result.Stream.CopyToAsync(archiveWriteFile.GetOutStream()); } } catch (Api.Client.ApiException ex) { if (ex.StatusCode == 404) { Log.HighPriority("Package Version Missing"); throw new HandledException(); } else { throw; } } } // Create the package folder to extract to var stagingVersionFolder = stagingDirectory + new Path($"{languageName}_{packageName}_{packageVersion}/"); LifetimeManager.Get <IFileSystem>().CreateDirectory2(stagingVersionFolder); // Unpack the contents of the archive ZipFile.ExtractToDirectory(archivePath.ToString(), stagingVersionFolder.ToString()); // Delete the archive file LifetimeManager.Get <IFileSystem>().DeleteFile(archivePath); // Ensure the package root folder exists if (!LifetimeManager.Get <IFileSystem>().Exists(packageRootFolder)) { // Create the folder LifetimeManager.Get <IFileSystem>().CreateDirectory2(packageRootFolder); } // Move the extracted contents into the version folder LifetimeManager.Get <IFileSystem>().Rename(stagingVersionFolder, packageVersionFolder); } var recipePath = packageVersionFolder + BuildConstants.RecipeFileName; var(isSuccess, recipe) = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipePath); if (!isSuccess) { throw new InvalidOperationException("Could not load the recipe file."); } // Install recursive dependencies await RestoreRecursiveDependenciesAsync( packageVersionFolder, recipe, packagesDirectory, stagingDirectory, closure); } }
/// <summary> /// Publish a package /// </summary> public static async Task PublishPackageAsync(Path workingDirectory) { Log.Info($"Publish Project: {workingDirectory}"); var recipePath = workingDirectory + BuildConstants.RecipeFileName; var(isSuccess, recipe) = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipePath); if (!isSuccess) { throw new InvalidOperationException("Could not load the recipe file."); } var packageStore = LifetimeManager.Get <IFileSystem>().GetUserProfileDirectory() + new Path(".soup/packages/"); Log.Info("Using Package Store: " + packageStore.ToString()); // Create the staging directory var stagingPath = EnsureStagingDirectoryExists(packageStore); try { var archivePath = stagingPath + new Path(recipe.Name + ".zip"); // Create the archive of the package using (var writeArchiveFile = LifetimeManager.Get <IFileSystem>().OpenWrite(archivePath, true)) using (var zipArchive = new ZipArchive(writeArchiveFile.GetOutStream(), ZipArchiveMode.Create, false)) { AddPackageFiles(workingDirectory, zipArchive); } // Authenticate the user Log.Info("Request Authentication Token"); var accessToken = await AuthenticationManager.EnsureSignInAsync(); // Publish the archive Log.Info("Publish package"); using (var httpClient = new HttpClient()) { var packageClient = new Api.Client.PackageClient(httpClient) { BaseUrl = SoupApiEndpoint, BearerToken = accessToken, }; // Check if the package exists bool packageExists = false; try { var package = await packageClient.GetPackageAsync(recipe.Language, recipe.Name); packageExists = true; } catch (Api.Client.ApiException ex) { if (ex.StatusCode == 404) { packageExists = false; } else { throw; } } // Create the package if it does not exist if (!packageExists) { var createPackageModel = new Api.Client.PackageCreateOrUpdateModel() { Description = string.Empty, }; await packageClient.CreateOrUpdatePackageAsync(recipe.Language, recipe.Name, createPackageModel); } var packageVersionClient = new Api.Client.PackageVersionClient(httpClient) { BaseUrl = SoupApiEndpoint, BearerToken = accessToken, }; using (var readArchiveFile = LifetimeManager.Get <IFileSystem>().OpenRead(archivePath)) { try { await packageVersionClient.PublishPackageVersionAsync( recipe.Language, recipe.Name, recipe.Version.ToString(), new Api.Client.FileParameter(readArchiveFile.GetInStream(), string.Empty, "application/zip")); Log.Info("Package published"); } catch (Api.Client.ApiException ex) { if (ex.StatusCode == 409) { Log.Info("Package version already exists"); } else { throw; } } } } // Cleanup the staging directory Log.Info("Cleanup staging directory"); LifetimeManager.Get <IFileSystem>().DeleteDirectory(stagingPath, true); } catch (Exception) { // Cleanup the staging directory and accept that we failed Log.Info("Publish Failed: Cleanup staging directory"); LifetimeManager.Get <IFileSystem>().DeleteDirectory(stagingPath, true); throw; } }
/// <summary> /// Execute the entire operation graph that is referenced by this build generate engine. /// </summary> public async Task GenerateAsync(Path soupTargetDirectory) { // Run all build operations in the correct order with incremental build checks Log.Diag("Build generate start"); // Load the parameters file var parametersFile = soupTargetDirectory + BuildConstants.GenerateParametersFileName; if (!ValueTableManager.TryLoadState(parametersFile, out var parametersState)) { Log.Error("Failed to load the parameter file: " + parametersFile.ToString()); throw new InvalidOperationException("Failed to load parameter file."); } // Load the read access file var readAccessFile = soupTargetDirectory + BuildConstants.GenerateReadAccessFileName; var readAccessList = new List <Path>(); if (!await PathListManager.TryLoadFileAsync(readAccessFile, readAccessList)) { Log.Error("Failed to load the read access file: " + readAccessFile.ToString()); throw new InvalidOperationException("Failed to load read access file."); } // Load the write access file var writeAccessFile = soupTargetDirectory + BuildConstants.GenerateWriteAccessFileName; var writeAccessList = new List <Path>(); if (!await PathListManager.TryLoadFileAsync(writeAccessFile, writeAccessList)) { Log.Error("Failed to load the write access file: " + writeAccessFile.ToString()); throw new InvalidOperationException("Failed to load write access file."); } // Get the required input state from the parameters var targetDirectory = new Path(parametersState["TargetDirectory"].AsString().ToString()); var packageDirectory = new Path(parametersState["PackageDirectory"].AsString().ToString()); // Load the recipe file var recipeFile = packageDirectory + BuildConstants.RecipeFileName; var(isSuccess, recipe) = await RecipeExtensions.TryLoadRecipeFromFileAsync(recipeFile); if (!isSuccess) { Log.Error("Failed to load the recipe: " + recipeFile.ToString()); throw new InvalidOperationException("Failed to load recipe."); } // Combine all the dependencies shared state var dependenciesSharedState = LoadDependenciesSharedState(parametersState); // Generate the set of build extension libraries var buildExtensionLibraries = GenerateBuildExtensionSet(recipe, dependenciesSharedState); // Start a new active state that is initialized to the recipe itself var activeState = new ValueTable(); // Initialize the Recipe Root Table var recipeState = recipe.Table; activeState.Add("Recipe", new Value(recipeState)); // Initialize the Parameters Root Table activeState.Add("Parameters", new Value(parametersState)); // Initialize the Dependencies Root Table activeState.Add("Dependencies", new Value(dependenciesSharedState)); // Keep the extension libraries open while running the build system // to ensure their memory is kept alive var evaluateGraph = new OperationGraph(); IValueTable sharedState = new ValueTable(); { // Create a new build system for the requested build var buildTaskManager = new BuildTaskManager(); // Run all build extension register callbacks foreach (var buildExtension in buildExtensionLibraries) { var library = LoadPlugin(buildExtension); FindAllCommands(library, buildTaskManager); } // Run the build var buildState = new BuildState( activeState, _fileSystemState, readAccessList, writeAccessList); buildTaskManager.Execute(buildState, soupTargetDirectory); // Grab the build results so the dependency libraries can be released asap evaluateGraph = buildState.BuildOperationGraph(); sharedState = buildState.SharedState; } // Save the operation graph so the evaluate phase can load it var evaluateGraphFile = soupTargetDirectory + BuildConstants.GenerateEvaluateOperationGraphFileName; OperationGraphManager.SaveState(evaluateGraphFile, evaluateGraph, _fileSystemState); // Save the shared state that is to be passed to the downstream builds var sharedStateFile = soupTargetDirectory + BuildConstants.GenerateSharedStateFileName; ValueTableManager.SaveState(sharedStateFile, sharedState); Log.Diag("Build generate end"); }