/// <summary>
/// Download one NPM package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> n/a </returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
// shouldn't happen here, but check
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Debug("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
try
{
var doc = await GetJsonCache($"{ENV_NPM_API_ENDPOINT}/{packageName}");
var tarball = doc.RootElement.GetProperty("versions").GetProperty(packageVersion).GetProperty("dist").GetProperty("tarball").GetString();
var result = await WebClient.GetAsync(tarball);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl?.ToString());
var targetName = $"npm-{packageName}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory ?? string.Empty, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
targetName += Path.GetExtension(tarball) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading NPM package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// try to resolve the source code for an npm package through different means
/// 1) Look at the metadata
/// 2) Try searching github
/// 3) Try calculating metrics for same name repos
/// </summary>
/// <param name="package_name"> </param>
/// <returns> </returns>
public async Task <Dictionary <PackageURL, double> > ResolvePackageLibraryAsync(PackageURL purl)
{
Logger.Trace("ResolvePackageLibraryAsync({0})", purl);
var repoMappings = new Dictionary <PackageURL, double>();
if (purl == null)
{
return(repoMappings);
}
var purlNoVersion = new PackageURL(purl.Type, purl.Namespace, purl.Name,
null, purl.Qualifiers, purl.Subpath);
Logger.Debug("Searching for source code for: {0}", purlNoVersion.ToString());
// Use reflection to find the correct downloader class
var projectManager = ProjectManagerFactory.CreateProjectManager(purl, null);
if (projectManager != null)
{
repoMappings = await projectManager.IdentifySourceRepository(purl);
if (repoMappings == null || !repoMappings.Any())
{
repoMappings = new Dictionary <PackageURL, double>();
Logger.Info("No repositories were found after searching metadata.");
}
}
else
{
throw new ArgumentException("Invalid Package URL type: {0}", purlNoVersion.Type);
}
return(repoMappings);
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_PYPI_ENDPOINT}/pypi/{packageName}/json");
var versionList = new List <string>();
if (doc.RootElement.TryGetProperty("releases", out JsonElement releases))
{
foreach (var versionObject in releases.EnumerateObject())
{
Logger.Debug("Identified {0} version {1}.", packageName, versionObject.Name);
versionList.Add(versionObject.Name);
}
}
// Add the current version (not included in releases)
if (doc.RootElement.TryGetProperty("info", out JsonElement info) &&
info.TryGetProperty("version", out JsonElement version))
{
Logger.Debug("Identified {0} version {1}.", packageName, version.GetString());
versionList.Add(version.GetString());
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Warn(ex, "Error enumerating PyPI packages: {0}", ex.Message);
return(Array.Empty <string>());
}
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_NUGET_ENDPOINT_API}/v3/registration3/{packageName}/index.json");
var versionList = new List <string>();
foreach (var catalogPage in doc.RootElement.GetProperty("items").EnumerateArray())
{
foreach (var item in catalogPage.GetProperty("items").EnumerateArray())
{
var catalogEntry = item.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, $"Error enumerating NuGet packages: {ex.Message}");
return(Array.Empty <string>());
}
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_NPM_ENDPOINT}/{packageName}");
var versionList = new List <string>();
foreach (var versionKey in doc.RootElement.GetProperty("versions").EnumerateObject())
{
Logger.Debug("Identified {0} version {1}.", packageName, versionKey.Name);
versionList.Add(versionKey.Name);
}
var latestVersion = doc.RootElement.GetProperty("dist-tags").GetProperty("latest").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, latestVersion);
versionList.Add(latestVersion);
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, $"Error enumerating NPM package: {ex.Message}");
return(Array.Empty <string>());
}
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
try
{
var packageName = $"{purl.Namespace}/{purl.Name}";
var doc = await GetJsonCache($"{ENV_COMPOSER_ENDPOINT}/p/{packageName}.json");
var versionList = new List <string>();
foreach (var topObject in doc.RootElement.GetProperty("packages").EnumerateObject())
{
foreach (var versionObject in topObject.Value.EnumerateObject())
{
Logger.Debug("Identified {0} version {1}.", packageName, versionObject.Name);
versionList.Add(versionObject.Name);
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, $"Error enumerating Composer package: {ex.Message}");
return(Array.Empty <string>());
}
}
/// <summary>
/// Download one Cocoapods package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
var prefix = GetCocoapodsPrefix(packageName);
var podspec = await GetJsonCache($"{ENV_COCOAPODS_SPECS_RAW_ENDPOINT}/Specs/{prefix}/{packageName}/{packageVersion}/{packageName}.podspec.json");
if (podspec.RootElement.TryGetProperty("source", out var source))
{
string url = null;
if (source.TryGetProperty("git", out var sourceGit) &&
source.TryGetProperty("tag", out var sourceTag))
{
var sourceGitString = sourceGit.GetString();
var sourceTagString = sourceTag.GetString();
if (sourceGitString.EndsWith(".git"))
{
sourceGitString = sourceGitString[0..^ 4];
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_RUBYGEMS_ENDPOINT_API}/api/v1/versions/{packageName}.json");
var versionList = new List <string>();
foreach (var gemObject in doc.RootElement.EnumerateArray())
{
if (gemObject.TryGetProperty("number", out JsonElement version))
{
var vString = version.ToString();
// RubyGems is mostly-semver-compliant
vString = Regex.Replace(vString, @"(\d)pre", @"$1-pre");
Logger.Debug("Identified {0} version {1}.", packageName, vString);
versionList.Add(vString);
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, "Error enumerating RubyGems package: {0}", ex.Message);
return(Array.Empty <string>());
}
}
/// <summary>
/// Identifies the available pools for a given Ubuntu project. For example, 'xenial'.
/// </summary>
/// <param name="purl"> Package URL to look up (only name is used). </param>
/// <returns> List of pool names </returns>
private async Task <IEnumerable <string> > GetPoolsForProject(PackageURL purl)
{
var pools = new HashSet <string>();
try
{
var searchResults = await GetHttpStringCache($"{ENV_UBUNTU_ENDPOINT}/search?keywords={purl.Name}&searchon=names&exact=1&suite=all§ion=all", neverThrow : true);
var document = await new HtmlParser().ParseDocumentAsync(searchResults);
foreach (var anchor in document.QuerySelectorAll("a.resultlink"))
{
var href = anchor.GetAttribute("href");
if (href != null)
{
var match = Regex.Match(href, "^/([^/]+)/.+");
if (match.Success)
{
var pool = match.Groups[1].Value.Trim();
Logger.Debug("Identified pool: {0}", pool);
pools.Add(pool);
}
}
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error fetching Ubuntu pools for {0}: {1}", purl.ToString(), ex.Message);
}
return(pools);
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
try
{
var packageName = purl.Name;
var versionList = new List <string>();
// Get the latest version
var html = await WebClient.GetAsync($"{ENV_CRAN_ENDPOINT}/web/packages/{packageName}/index.html");
html.EnsureSuccessStatusCode();
var parser = new HtmlParser();
var document = await parser.ParseDocumentAsync(await html.Content.ReadAsStringAsync());
var tds = document.QuerySelectorAll("td");
for (int i = 0; i < tds.Length; i++)
{
if (tds[i].TextContent == "Version:")
{
var value = tds[i + 1]?.TextContent?.Trim();
if (value != null)
{
versionList.Add(value);
}
break;
}
}
// Get the remaining versions
html = await WebClient.GetAsync($"{ENV_CRAN_ENDPOINT}/src/contrib/Archive/{packageName}/");
html.EnsureSuccessStatusCode();
document = await parser.ParseDocumentAsync(await html.Content.ReadAsStringAsync());
tds = document.QuerySelectorAll("a");
foreach (var td in tds)
{
var href = td.GetAttribute("href");
if (href.Contains(".tar.gz"))
{
var version = href.Replace(".tar.gz", "");
version = version.Replace(packageName + "_", "").Trim();
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Debug(ex, $"Error enumerating CRAN package: {ex.Message}");
throw;
}
}
// default = text
/// <summary>
/// Build a SARIF Result.Location object for the purl package
/// </summary>
/// <param name="purl">The <see cref="PackageURL"/> to build the location for.</param>
/// <returns>Location list with single location object</returns>
public static List <Location> BuildPurlLocation(PackageURL purl)
{
BaseProjectManager?projectManager = ProjectManagerFactory.ConstructPackageManager(purl, null);
if (projectManager == null)
{
Logger.Debug("Cannot determine the package type");
return(new List <Location>());
}
return(new List <Location>()
{
new Location()
{
PhysicalLocation = new PhysicalLocation()
{
Address = new Address()
{
FullyQualifiedName = projectManager.GetPackageAbsoluteUri(purl)?.AbsoluteUri,
AbsoluteAddress = PHYSICAL_ADDRESS_FLAG, // Sarif format needs non negative integer
Name = purl.ToString()
}
}
}
});
}
/// <summary>
/// Enumerates all possible versions of the package identified by purl.
/// </summary>
/// <param name="purl">Package URL specifying the package. Version is ignored.</param>
/// <returns>A list of package versions</returns>
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_CARGO_ENDPOINT}/api/v1/crates/{packageName}");
var versionList = new List <string>();
foreach (var versionObject in doc.RootElement.GetProperty("versions").EnumerateArray())
{
if (versionObject.TryGetProperty("num", out JsonElement version))
{
Logger.Debug("Identified {0} version {1}.", packageName, version.ToString());
versionList.Add(version.ToString());
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, "Error enumerating Cargo package versions: {0}", ex.Message);
return(Array.Empty <string>());
}
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
try
{
var packageNamespace = purl.Namespace.Replace('.', '/');
var packageName = purl.Name;
var content = await GetHttpStringCache($"{ENV_MAVEN_ENDPOINT}/{packageNamespace}/{packageName}/maven-metadata.xml");
var versionList = new List <string>();
var doc = new XmlDocument();
doc.LoadXml(content);
foreach (XmlNode versionObject in doc.GetElementsByTagName("version"))
{
Logger.Debug("Identified {0} version {1}.", packageName, versionObject.InnerText);
versionList.Add(versionObject.InnerText);
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, $"Error enumerating Maven packages: {ex.Message}");
return(Array.Empty <string>());
}
}
/// <summary>
/// Download one PyPI package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>the path or file written.</returns>
public override async Task <string> DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
string downloadedPath = null;
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(null);
}
try
{
var doc = await GetJsonCache($"{ENV_PYPI_ENDPOINT}/pypi/{packageName}/json");
if (!doc.RootElement.TryGetProperty("releases", out JsonElement releases))
{
return(null);
}
foreach (var versionObject in releases.EnumerateObject())
{
if (versionObject.Name != packageVersion || downloadedPath != null)
{
continue;
}
foreach (var release in versionObject.Value.EnumerateArray())
{
// For PyPI projects, we only download source distributions
if (release.GetProperty("packagetype").GetString() == "sdist")
{
var result = await WebClient.GetAsync(release.GetProperty("url").GetString());
result.EnsureSuccessStatusCode();
var targetName = $"pypi-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPath = await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync());
}
else
{
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPath = targetName;
}
break;
}
}
}
}
catch (Exception ex)
{
Logger.Warn(ex, "Error downloading PyPI package: {0}", ex.Message);
downloadedPath = null;
}
return(downloadedPath);
}
/// <summary>
/// Download one Hackage (Haskell) package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> n/a </returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Debug("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
try
{
var url = $"{ENV_HACKAGE_ENDPOINT}/package/{packageName}-{packageVersion}/{packageName}-{packageVersion}.tar.gz";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl?.ToString());
var targetName = $"hackage-{packageName}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading Hackage package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// Download one Composer (PHP) package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = $"{purl?.Namespace}/{purl?.Name}";
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(purl?.Namespace) || string.IsNullOrWhiteSpace(purl?.Name) ||
string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
try
{
var doc = await GetJsonCache($"{ENV_COMPOSER_ENDPOINT}/p/{packageName}.json");
foreach (var topObject in doc.RootElement.GetProperty("packages").EnumerateObject())
{
foreach (var versionObject in topObject.Value.EnumerateObject())
{
if (versionObject.Name != packageVersion)
{
continue;
}
var url = versionObject.Value.GetProperty("dist").GetProperty("url").GetString();
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
var targetName = $"composer-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync()));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
}
if (downloadedPaths.Count == 0)
{
Logger.Warn("Unable to find version {0} to download.", packageVersion);
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error downloading Composer package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// Download one Maven package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> n/a </returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageNamespace = purl?.Namespace?.Replace('.', '/');
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageNamespace) || string.IsNullOrWhiteSpace(packageName) ||
string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Debug("Unable to download [{0} {1} {2}]. Both must be defined.", packageNamespace, packageName, packageVersion);
return(downloadedPaths);
}
try
{
var suffixes = new string[] { "-javadoc", "-sources", "" };
foreach (var suffix in suffixes)
{
var url = $"{ENV_MAVEN_ENDPOINT}/{packageNamespace}/{packageName}/{packageVersion}/{packageName}-{packageVersion}{suffix}.jar";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug($"Downloading {purl}...");
var targetName = $"maven-{packageNamespace}/{packageName}{suffix}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading Maven package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// Download one GitHub package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
if (doExtract == false)
{
throw new NotImplementedException("GitHub does not support binary downloads yet.");
}
var packageNamespace = purl?.Namespace;
var packageName = purl?.Name;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageNamespace) || string.IsNullOrWhiteSpace(packageName))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageNamespace, packageName);
return(downloadedPaths);
}
try
{
var url = $"{ENV_GITHUB_ENDPOINT}/{purl.Namespace}/{purl.Name}";
var invalidChars = Path.GetInvalidFileNameChars();
// TODO: Externalize this normalization
var fsNamespace = new String(purl.Namespace.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var fsName = new String(purl.Name.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var fsVersion = new String(purl.Version.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var workingDirectory = string.IsNullOrWhiteSpace(purl.Version) ?
Path.Join(TopLevelExtractionDirectory, $"github-{fsNamespace}-{fsName}") :
Path.Join(TopLevelExtractionDirectory, $"github-{fsNamespace}-{fsName}-{fsVersion}");
Repository.Clone(url, workingDirectory);
var repo = new Repository(workingDirectory);
if (!string.IsNullOrWhiteSpace(purl.Version))
{
Commands.Checkout(repo, purl.Version);
downloadedPaths.Add(workingDirectory);
}
repo.Dispose();
}
catch (LibGit2Sharp.NotFoundException ex)
{
Logger.Warn(ex, "The version {0} is not a valid git reference: {1}", purl.Version, ex.Message);
}
catch (Exception ex)
{
Logger.Warn(ex, "Error downloading GitHub package: {0}", ex.Message);
}
return(downloadedPaths);
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
return(new List <string>()
{
"1.0"
});
}
/// <summary>
/// Download one GitHub package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <string> DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
if (doExtract == false)
{
throw new NotImplementedException("GitHub does not support binary downloads yet.");
}
var packageName = purl?.Name;
var packageVersion = purl?.Version;
string downloadedPath = null;
if (string.IsNullOrWhiteSpace(packageName))
{
Logger.Error("Unable to download [{0}]", packageName);
return(null);
}
try
{
var url = $"https://github.com/{purl.Namespace}/{purl.Name}";
var invalidChars = Path.GetInvalidFileNameChars();
var fsNamespace = new String(purl.Namespace.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var fsName = new String(purl.Name.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var fsVersion = new String(purl.Version.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
var workingDirectory = string.IsNullOrWhiteSpace(purl.Version) ?
Path.Join(TopLevelExtractionDirectory, $"github-{fsNamespace}-{fsName}") :
Path.Join(TopLevelExtractionDirectory, $"github-{fsNamespace}-{fsName}-{fsVersion}");
Repository.Clone(url, workingDirectory);
using var repo = new Repository(workingDirectory);
if (!string.IsNullOrWhiteSpace(purl.Version))
{
Commands.Checkout(repo, purl.Version);
downloadedPath = workingDirectory;
}
}
catch (LibGit2Sharp.NotFoundException ex)
{
Logger.Warn(ex, "The version {0} is not a valid git reference: {1}", purl.Version, ex.Message);
downloadedPath = null;
}
catch (Exception ex)
{
Logger.Error(ex, "Error downloading GitHub package: {0}", ex.Message);
downloadedPath = null;
}
return(downloadedPath);
}
/// <summary>
/// Download one Cargo package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> Path to the downloaded package </returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var fileName = purl?.ToStringFilename();
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion) || string.IsNullOrWhiteSpace(fileName))
{
Logger.Debug("Error with 'purl' argument. Unable to download [{0} {1}] @ {2}. Both must be defined.", packageName, packageVersion, fileName);
return(downloadedPaths);
}
var url = $"{ENV_CARGO_ENDPOINT}/api/v1/crates/{packageName}/{packageVersion}/download";
try
{
string targetName = $"cargo-{fileName}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
// if the cache is already present, no need to extract
if (doExtract && cached && Directory.Exists(extractionPath))
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
Logger.Debug("Downloading {0}", url);
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
extractionPath += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(extractionPath, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(extractionPath);
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading Cargo package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// Download one Maven package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <string> DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageNamespace = purl?.Namespace?.Replace('.', '/');
var packageName = purl?.Name;
var packageVersion = purl?.Version;
string downloadedPath = null;
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(null);
}
try
{
var suffixes = new string[] { "-javadoc", "-sources", "" };
foreach (var suffix in suffixes)
{
var url = $"{ENV_MAVEN_ENDPOINT}/{packageNamespace}/{packageName}/{packageVersion}/{packageName}-{packageVersion}{suffix}.jar";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug($"Downloading {purl}...");
var targetName = $"maven-{purl.Namespace}/{packageName}{suffix}@{packageVersion}";
if (doExtract)
{
downloadedPath = await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync());
}
else
{
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPath = targetName;
}
}
}
catch (Exception ex)
{
Logger.Error(ex, $"Error downloading Maven package: {ex.Message}");
downloadedPath = null;
}
return(downloadedPath);
}
/// <summary>
/// Download one NuGet package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
try
{
var doc = await GetJsonCache($"{ENV_NUGET_ENDPOINT_API}/v3/registration3/{packageName}/{packageVersion}.json");
var archive = doc.RootElement.GetProperty("packageContent").GetString();
var result = await WebClient.GetAsync(archive);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl.ToString());
var targetName = $"nuget-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync()));
}
else
{
targetName += Path.GetExtension(archive) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error downloading NuGet package: {0}", ex.Message);
}
return(downloadedPaths);
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
try
{
var packageName = purl.Name;
var versionList = new List <string>();
var html = await GetHttpStringCache($"{ENV_CPAN_ENDPOINT}/release/{packageName}");
var parser = new HtmlParser();
var document = await parser.ParseDocumentAsync(html);
foreach (var option in document.QuerySelectorAll("div.release select.extend option"))
{
if (!option.HasAttribute("value"))
{
continue;
}
var value = option.GetAttribute("value");
var match = Regex.Match(value, @".*-([^-]+)$");
if (match.Success)
{
Logger.Debug("Identified {0} version {1}.", packageName, match.Groups[1].Value);
versionList.Add(match.Groups[1].Value);
}
}
var result = SortVersions(versionList.Distinct());
return(result);
}
catch (Exception ex)
{
Logger.Debug(ex, "Error enumerating CPAN package: {0}", ex.Message);
throw;
}
}
/// <summary>
/// Download one NPM package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <string> DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
string downloadedPath;
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(null);
}
try
{
var doc = await GetJsonCache($"{ENV_NPM_ENDPOINT}/{packageName}");
var tarball = doc.RootElement.GetProperty("versions").GetProperty(packageVersion).GetProperty("dist").GetProperty("tarball").GetString();
var result = await WebClient.GetAsync(tarball);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl.ToString());
var targetName = $"npm-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPath = await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync());
}
else
{
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPath = targetName;
}
}
catch (Exception ex)
{
Logger.Error(ex, $"Error downloading NPM package: {ex.Message}");
downloadedPath = null;
}
return(downloadedPath);
}
public override async Task <IEnumerable <string> > EnumerateVersions(PackageURL purl)
{
Logger.Trace("EnumerateVersions {0}", purl?.ToString());
if (purl == null)
{
return(new List <string>());
}
try
{
var packageName = purl.Name;
var html = await WebClient.GetAsync($"{ENV_HACKAGE_ENDPOINT}/package/{packageName}");
html.EnsureSuccessStatusCode();
var parser = new HtmlParser();
var document = await parser.ParseDocumentAsync(await html.Content.ReadAsStringAsync());
var ths = document.QuerySelectorAll("th");
var versionList = new List <string>();
foreach (var th in ths)
{
if (th.TextContent.StartsWith("Versions"))
{
var td = th.NextElementSibling;
foreach (var version in td.QuerySelectorAll("a,strong"))
{
var versionString = version.TextContent.ToLower().Trim();
Logger.Debug("Identified {0} version {1}.", packageName, versionString);
versionList.Add(versionString);
}
break;
}
}
return(SortVersions(versionList.Distinct()));
}
catch (Exception ex)
{
Logger.Error(ex, $"Error enumerating Hackage package: {ex.Message}");
return(Array.Empty <string>());
}
}
/// <summary>
/// Download one RubyGems package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPaths);
}
try
{
var url = $"{ENV_RUBYGEMS_ENDPOINT}/downloads/{packageName}-{packageVersion}.gem";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
var targetName = $"rubygems-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync()));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error downloading RubyGems package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <summary>
/// Download one Cocoapods package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> n/a </returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var downloadedPaths = new List <string>();
string?url = null;
var foundValue = purl?.Qualifiers?.TryGetValue("url", out url) ?? false;
if (foundValue && url is not null)
{
Uri uri = new Uri(url);
Logger.Debug("Downloading {0} ({1})...", purl, uri);
var result = await WebClient.GetAsync(uri);
result.EnsureSuccessStatusCode();
var targetName = Path.GetFileName(uri.LocalPath);
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
return(downloadedPaths);
}
else
{
Logger.Debug("URL not found, {0}", purl);
return(downloadedPaths);
}
}
/// <summary>
/// Download one Cargo package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <string> DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageName = purl?.Name;
var packageVersion = purl?.Version;
string downloadedPath = null;
if (string.IsNullOrWhiteSpace(packageName) || string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1}]. Both must be defined.", packageName, packageVersion);
return(downloadedPath);
}
try
{
var url = $"{ENV_CARGO_ENDPOINT}/api/v1/crates/{packageName}/{packageVersion}/download";
Logger.Debug("Downloading {0}", url);
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
var targetName = $"cargo-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPath = await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync());
}
else
{
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPath = targetName;
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error downloading Cargo package: {0}", ex.Message);
downloadedPath = null;
}
return(downloadedPath);
}
/// <summary>
/// Download one CRAN package and extract it to the target directory.
/// </summary>
/// <param name="purl">Package URL of the package to download.</param>
/// <returns>n/a</returns>
public override async Task <IEnumerable <string> > DownloadVersion(PackageURL purl, bool doExtract = true)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
var packageNamespace = purl?.Namespace;
var packageName = purl?.Name;
var packageVersion = purl?.Version;
var downloadedPaths = new List <string>();
if (string.IsNullOrWhiteSpace(packageNamespace) || string.IsNullOrWhiteSpace(packageName) ||
string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Error("Unable to download [{0} {1} {2}]. All must be defined.", packageNamespace, packageName, packageVersion);
return(downloadedPaths);
}
// Current Version
try
{
var url = $"{ENV_CRAN_ENDPOINT}/src/contrib/{packageName}_{packageVersion}.tar.gz";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
var targetName = $"cran-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync()));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading CRAN package: {0}@{1}. Checking archives instead.", packageName, packageVersion);
}
if (downloadedPaths.Count > 0)
{
return(downloadedPaths);
}
// Archive Version - Only continue here if needed
try
{
var url = $"{ENV_CRAN_ENDPOINT}/src/contrib/{packageNamespace}/{packageName}/{packageName}_{packageVersion}.tar.gz";
var result = await WebClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
var targetName = $"cran-{packageName}@{packageVersion}";
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync()));
}
else
{
targetName += Path.GetExtension(url) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
}
catch (Exception ex)
{
Logger.Warn(ex, "Error downloading CRAN package: {0}", ex.Message);
}
return(downloadedPaths);
}
