/// <summary>
/// Decompresses a nupkg file to a temp directory, runs elfie to create an Idx file for the package, and stores the Idx file.
/// </summary>
async Task<Uri> StageAndIndexPackageAsync(Uri packageResourceUri, RegistrationIndexPackage package, CancellationToken cancellationToken)
{
Trace.TraceInformation("#StartActivity StageAndIndexPackageAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion);
Uri idxResourceUri = null;
// This is the temporary directory that we'll work in.
string tempDirectory = Path.Combine(this._tempPath, Guid.NewGuid().ToString());
Trace.TraceInformation($"Temp directory: {tempDirectory}.");
try
{
// Create the temp directory and expand the nupkg file
Directory.CreateDirectory(tempDirectory);
if (package.IsLocalPackage)
{
// This is a local package, so copy just the assemblies listed in the text file to the temp directory.
this.StageLocalPackageContents(package.Id.LocalPath, tempDirectory);
}
else
{
// This is a NuGet package, so decompress the package.
// This is the pointer to the package file in storage
Trace.TraceInformation("Loading package from storage.");
StorageContent packageStorage = this._storage.Load(packageResourceUri, new CancellationToken()).Result;
// Expand the package contents to the temp directory.
this.ExpandPackage(packageStorage, tempDirectory);
}
string idxFile = this.CreateIdxFile(package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion, tempDirectory);
if (idxFile == null)
{
Trace.TraceInformation("The idx file was not created.");
}
else
{
Trace.TraceInformation($"Saving the idx file. {idxFile}");
idxResourceUri = this._storage.ComposeIdxResourceUrl(this._indexerVersion, package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion);
this._storage.SaveFileContents(idxFile, idxResourceUri);
}
}
catch (ZipException ze)
{
// The package couldn't be decompressed.
SarifTraceListener.TraceWarning("NG007", $"Could not decompress the package. {packageResourceUri}", ze);
idxResourceUri = null;
}
catch
{
// The idx creation failed, so delete any files which were saved to storage.
if (idxResourceUri != null)
{
try
{
await this._storage.Delete(idxResourceUri, cancellationToken);
idxResourceUri = null;
}
catch (Exception e)
{
// The resource couldn't be deleted.
// Log the error and continue. The original exception will be rethrown.
SarifTraceListener.TraceWarning("NG008", $"Could not delete the idx file from storage.", e);
Trace.TraceWarning(e.ToString());
}
}
throw;
}
finally
{
Trace.TraceInformation("Deleting the temp directory.");
try
{
Directory.Delete(tempDirectory, true);
}
catch (Exception e)
{
// If the temp directory couldn't be deleted just log the error and continue.
SarifTraceListener.TraceWarning("NG009", $"Could not delete the temp directory {tempDirectory}.", e);
}
}
Trace.TraceInformation("#StopActivity StageAndIndexPackageAsync");
return idxResourceUri;
}
/// <summary>
/// Downloads, decompresses and creates an idx file for a NuGet package.
/// </summary>
/// <param name="package">The package to process.</param>
async Task ProcessPackageDetailsAsync(RegistrationIndexPackage package, CancellationToken cancellationToken)
{
Trace.TraceInformation("#StartActivity ProcessPackageDetailsAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion);
Uri idxFile = null;
Uri packageResourceUri = null;
// If this is a NuGet package, download the package
if (!package.IsLocalPackage)
{
packageResourceUri = await this.DownloadPackageAsync(package, cancellationToken);
if (packageResourceUri == null)
{
SarifTraceListener.TraceWarning("NG005", $"Could not download package {package.CatalogEntry.PackageId}");
}
}
// If we successfully downloaded the package or the package is an local package, create the idx file for the package.
if (packageResourceUri != null || package.IsLocalPackage)
{
idxFile = await this.StageAndIndexPackageAsync(packageResourceUri, package, cancellationToken);
if (idxFile == null)
{
SarifTraceListener.TraceWarning("NG006", $"Could not create idx file for package {package.CatalogEntry.PackageId}");
}
else
{
SarifTraceListener.TraceInformation($"Created Idx file for package {package.CatalogEntry.PackageId}.");
}
}
Trace.TraceInformation("#StopActivity ProcessPackageDetailsAsync");
}
/// <summary>
/// Downloads a package (nupkg) and saves the package to storage.
/// </summary>
/// <param name="package">The registration data for the package to download.</param>
/// <param name="packageDownloadUrl">The download URL for the package to download.</param>
/// <returns>The storage resource URL for the saved package.</returns>
async Task<Uri> DownloadPackageAsync(RegistrationIndexPackage package, CancellationToken cancellationToken)
{
Trace.TraceInformation("#StartActivity DownloadPackageAsync " + package.CatalogEntry.PackageId + " " + package.CatalogEntry.PackageVersion);
Uri packageResourceUri = null;
int retryLimit = 3;
for (int retry = 0; retry < retryLimit; retry++)
{
try
{
// Get the package file name from the download URL.
string packageFileName = Path.GetFileName(package.PackageContent.LocalPath);
// This is the storage path for the package.
packageResourceUri = this._storage.ComposePackageResourceUrl(package.CatalogEntry.PackageId, package.CatalogEntry.PackageVersion, packageFileName);
// Check if we already downloaded the package in a previous run.
using (StorageContent packageStorageContent = await this._storage.Load(packageResourceUri, cancellationToken))
{
if (packageStorageContent == null)
{
// The storage doesn't contain the package, so we have to download and save it.
Trace.TraceInformation("Saving nupkg to " + packageResourceUri.AbsoluteUri);
this._storage.SaveUrlContents(package.PackageContent, packageResourceUri);
}
}
}
catch (Exception e)
{
// If something went wrong, we should delete the package from storage so we don't have partially downloaded files.
if (packageResourceUri != null)
{
try
{
await this._storage.Delete(packageResourceUri, cancellationToken);
}
catch
{
}
}
// If we received a 404 (file not found) when trying to download the file.
// There's not much we can do here since the package download URL doesn't exist.
// Return null, which indicates that the package doesn't exist, and continue.
WebException webException = e as WebException;
if (webException != null && webException.Response != null && ((HttpWebResponse)webException.Response).StatusCode == HttpStatusCode.NotFound)
{
SarifTraceListener.TraceWarning("NG011", $"The package download URL for the package {package.CatalogEntry.PackageId} could not be found (404). {package.PackageContent}", webException);
packageResourceUri = null;
break;
}
// For any other exception, retry the download.
if (retry < retryLimit - 1)
{
SarifTraceListener.TraceWarning($"Exception downloading package for {package.CatalogEntry.PackageId} on attempt {retry} of {retryLimit - 1}. {e.Message}");
// Wait for a few seconds before retrying.
int delay = Catalog2ElfieOptions.GetRetryDelay(retry);
Thread.Sleep(delay * 1000);
SarifTraceListener.TraceWarning($"Retrying package download for {package.CatalogEntry.PackageId}.");
}
else
{
// We retried a few times and failed.
// Rethrow the exception so we track the failure.
throw;
}
}
}
Trace.TraceInformation("#StopActivity DownloadPackageAsync");
return packageResourceUri;
}
/// <summary>
/// Gets the list of fake packages which represent assemblies on the local machine.
/// </summary>
/// <param name="assemblyPackageDirectory">The directory which contain the text files representing the local packages.</param>
/// <returns>Returns faked package registration objects which contain the information for the local packages.</returns>
IList<Tuple<RegistrationIndexPackage, long>> GetLocalAssemblyPackages(string assemblyPackageDirectory)
{
List<Tuple<RegistrationIndexPackage, long>> assemblyPackageFiles = new List<Tuple<RegistrationIndexPackage, long>>();
if (Directory.Exists(assemblyPackageDirectory))
{
foreach (string file in Directory.GetFiles(assemblyPackageDirectory, "*.txt"))
{
// Create the fake registration data.
RegistrationIndexPackage package = new RegistrationIndexPackage();
package.Id = new Uri(file);
package.Type = "AssemblyPackage";
RegistrationIndexPackageDetails catalogEntry = new RegistrationIndexPackageDetails();
catalogEntry.Id = package.Id;
catalogEntry.Type = package.Type;
// The package id is the name of the text file.
catalogEntry.PackageId = Path.GetFileNameWithoutExtension(file);
// The package version is always 0.0.0.0.
catalogEntry.PackageVersion = "0.0.0.0";
package.CatalogEntry = catalogEntry;
// Give the assembly packages the largest download count so they are at the top of the ardb tree.
long downloadCount = Int32.MaxValue;
assemblyPackageFiles.Add(Tuple.Create((RegistrationIndexPackage)package, downloadCount));
}
}
return assemblyPackageFiles;
}