/// <summary> /// Decompresses a nupkg file to a temp directory, runs elfie to create an Idx file for the package, and stores the Idx file. /// </summary> async Task<Uri> StageAndIndexPackageAsync(Uri packageResourceUri, RegistrationIndexPackage package, CancellationToken cancellationToken) { Trace.TraceInformation("#StartActivity StageAndIndexPackageAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion); Uri idxResourceUri = null; // This is the temporary directory that we'll work in. string tempDirectory = Path.Combine(this._tempPath, Guid.NewGuid().ToString()); Trace.TraceInformation($"Temp directory: {tempDirectory}."); try { // Create the temp directory and expand the nupkg file Directory.CreateDirectory(tempDirectory); if (package.IsLocalPackage) { // This is a local package, so copy just the assemblies listed in the text file to the temp directory. this.StageLocalPackageContents(package.Id.LocalPath, tempDirectory); } else { // This is a NuGet package, so decompress the package. // This is the pointer to the package file in storage Trace.TraceInformation("Loading package from storage."); StorageContent packageStorage = this._storage.Load(packageResourceUri, new CancellationToken()).Result; // Expand the package contents to the temp directory. this.ExpandPackage(packageStorage, tempDirectory); } string idxFile = this.CreateIdxFile(package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion, tempDirectory); if (idxFile == null) { Trace.TraceInformation("The idx file was not created."); } else { Trace.TraceInformation($"Saving the idx file. {idxFile}"); idxResourceUri = this._storage.ComposeIdxResourceUrl(this._indexerVersion, package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion); this._storage.SaveFileContents(idxFile, idxResourceUri); } } catch (ZipException ze) { // The package couldn't be decompressed. SarifTraceListener.TraceWarning("NG007", $"Could not decompress the package. {packageResourceUri}", ze); idxResourceUri = null; } catch { // The idx creation failed, so delete any files which were saved to storage. if (idxResourceUri != null) { try { await this._storage.Delete(idxResourceUri, cancellationToken); idxResourceUri = null; } catch (Exception e) { // The resource couldn't be deleted. // Log the error and continue. The original exception will be rethrown. SarifTraceListener.TraceWarning("NG008", $"Could not delete the idx file from storage.", e); Trace.TraceWarning(e.ToString()); } } throw; } finally { Trace.TraceInformation("Deleting the temp directory."); try { Directory.Delete(tempDirectory, true); } catch (Exception e) { // If the temp directory couldn't be deleted just log the error and continue. SarifTraceListener.TraceWarning("NG009", $"Could not delete the temp directory {tempDirectory}.", e); } } Trace.TraceInformation("#StopActivity StageAndIndexPackageAsync"); return idxResourceUri; }
/// <summary> /// Downloads, decompresses and creates an idx file for a NuGet package. /// </summary> /// <param name="package">The package to process.</param> async Task ProcessPackageDetailsAsync(RegistrationIndexPackage package, CancellationToken cancellationToken) { Trace.TraceInformation("#StartActivity ProcessPackageDetailsAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion); Uri idxFile = null; Uri packageResourceUri = null; // If this is a NuGet package, download the package if (!package.IsLocalPackage) { packageResourceUri = await this.DownloadPackageAsync(package, cancellationToken); if (packageResourceUri == null) { SarifTraceListener.TraceWarning("NG005", $"Could not download package {package.CatalogEntry.PackageId}"); } } // If we successfully downloaded the package or the package is an local package, create the idx file for the package. if (packageResourceUri != null || package.IsLocalPackage) { idxFile = await this.StageAndIndexPackageAsync(packageResourceUri, package, cancellationToken); if (idxFile == null) { SarifTraceListener.TraceWarning("NG006", $"Could not create idx file for package {package.CatalogEntry.PackageId}"); } else { SarifTraceListener.TraceInformation($"Created Idx file for package {package.CatalogEntry.PackageId}."); } } Trace.TraceInformation("#StopActivity ProcessPackageDetailsAsync"); }
/// <summary> /// Downloads a package (nupkg) and saves the package to storage. /// </summary> /// <param name="package">The registration data for the package to download.</param> /// <param name="packageDownloadUrl">The download URL for the package to download.</param> /// <returns>The storage resource URL for the saved package.</returns> async Task<Uri> DownloadPackageAsync(RegistrationIndexPackage package, CancellationToken cancellationToken) { Trace.TraceInformation("#StartActivity DownloadPackageAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion); Uri packageResourceUri = null; int retryLimit = 3; for (int retry = 0; retry < retryLimit; retry++) { try { // Get the package file name from the download URL. string packageFileName = Path.GetFileName(package.PackageContent.LocalPath); // This is the storage path for the package. packageResourceUri = this._storage.ComposePackageResourceUrl(package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion, packageFileName); // Check if we already downloaded the package in a previous run. using (StorageContent packageStorageContent = await this._storage.Load(packageResourceUri, cancellationToken)) { if (packageStorageContent == null) { // The storage doesn't contain the package, so we have to download and save it. Trace.TraceInformation("Saving nupkg to " + packageResourceUri.AbsoluteUri); this._storage.SaveUrlContents(package.PackageContent, packageResourceUri); } } } catch (Exception e) { // If something went wrong, we should delete the package from storage so we don't have partially downloaded files. if (packageResourceUri != null) { try { await this._storage.Delete(packageResourceUri, cancellationToken); } catch { } } // If we received a 404 (file not found) when trying to download the file. // There's not much we can do here since the package download URL doesn't exist. // Return null, which indicates that the package doesn't exist, and continue. WebException webException = e as WebException; if (webException != null && webException.Response != null && ((HttpWebResponse)webException.Response).StatusCode == HttpStatusCode.NotFound) { SarifTraceListener.TraceWarning("NG011", $"The package download URL for the package {package.CatalogEntry.PackageId} could not be found (404). {package.PackageContent}", webException); packageResourceUri = null; break; } // For any other exception, retry the download. if (retry < retryLimit - 1) { SarifTraceListener.TraceWarning($"Exception downloading package for {package.CatalogEntry.PackageId} on attempt {retry} of {retryLimit - 1}. {e.Message}"); // Wait for a few seconds before retrying. int delay = Catalog2ElfieOptions.GetRetryDelay(retry); Thread.Sleep(delay * 1000); SarifTraceListener.TraceWarning($"Retrying package download for {package.CatalogEntry.PackageId}."); } else { // We retried a few times and failed. // Rethrow the exception so we track the failure. throw; } } } Trace.TraceInformation("#StopActivity DownloadPackageAsync"); return packageResourceUri; }
/// <summary> /// Gets the list of fake packages which represent assemblies on the local machine. /// </summary> /// <param name="assemblyPackageDirectory">The directory which contain the text files representing the local packages.</param> /// <returns>Returns faked package registration objects which contain the information for the local packages.</returns> IList<Tuple<RegistrationIndexPackage, long>> GetLocalAssemblyPackages(string assemblyPackageDirectory) { List<Tuple<RegistrationIndexPackage, long>> assemblyPackageFiles = new List<Tuple<RegistrationIndexPackage, long>>(); if (Directory.Exists(assemblyPackageDirectory)) { foreach (string file in Directory.GetFiles(assemblyPackageDirectory, "*.txt")) { // Create the fake registration data. RegistrationIndexPackage package = new RegistrationIndexPackage(); package.Id = new Uri(file); package.Type = "AssemblyPackage"; RegistrationIndexPackageDetails catalogEntry = new RegistrationIndexPackageDetails(); catalogEntry.Id = package.Id; catalogEntry.Type = package.Type; // The package id is the name of the text file. catalogEntry.PackageId = Path.GetFileNameWithoutExtension(file); // The package version is always 0.0.0.0. catalogEntry.PackageVersion = "0.0.0.0"; package.CatalogEntry = catalogEntry; // Give the assembly packages the largest download count so they are at the top of the ardb tree. long downloadCount = Int32.MaxValue; assemblyPackageFiles.Add(Tuple.Create((RegistrationIndexPackage)package, downloadCount)); } } return assemblyPackageFiles; }