private async Task <CSTRepository?> FetchGithubRepositoryMetadata(PackageURL purl)
{
try
{
GitHubClient github = new(new ProductHeaderValue("OSSGadget"));
GHRepository ghRepository = await github.Repository.Get(purl.Namespace, purl.Name);
if (ghRepository is null)
{
return(null);
}
Archived = ghRepository.Archived;
CreatedAt = ghRepository.CreatedAt;
UpdatedAt = ghRepository.UpdatedAt;
Description = OssUtilities.GetMaxClippedLength(ghRepository.Description);
IsFork = ghRepository.Fork;
Forks = ghRepository.ForksCount;
Homepage = OssUtilities.GetMaxClippedLength(ghRepository.Homepage);
Id = ghRepository.Id;
Language = OssUtilities.GetMaxClippedLength(ghRepository.Language);
Name = ghRepository.Name;
OpenIssuesCount = ghRepository.OpenIssuesCount;
Parent = ghRepository.Parent?.Url;
PushedAt = ghRepository.PushedAt;
Size = ghRepository.Size;
FollowersCount = ghRepository.StargazersCount;
Uri = OssUtilities.GetMaxClippedLength(ghRepository.Url);
StakeholdersCount = ghRepository.WatchersCount;
if (ghRepository.License is not null)
{
Licenses ??= new List <Model.License>();
Licenses.Add(new Model.License()
{
Name = ghRepository.License.Name,
Url = ghRepository.License.Url,
SPIX_ID = ghRepository.License.SpdxId
});
}
Owner ??= new Model.User()
{
Id = ghRepository.Owner.Id,
Name = ghRepository.Owner.Name,
Email = ghRepository.Owner.Email,
Url = ghRepository.Owner.Url,
Active = !ghRepository.Owner.Suspended
};
}
catch (Exception ex)
{
Logger.Debug($"Exception occurred while retrieving repository data: {ex}");
}
return(this);
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
protected async Task <Dictionary <PackageURL, double> > SearchRepoUrlsInPackageMetadata(PackageURL purl,
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
JsonDocument contentJSON)
{
Dictionary <PackageURL, double>?mapping = new();
if (purl.Name is string purlName && (purlName.StartsWith('_') || npm_internal_modules.Contains(purlName)))
{
// url = 'https://github.com/nodejs/node/tree/master/lib' + package.name,
mapping.Add(new PackageURL(purl.Type, purl.Namespace, purl.Name,
null, null, "node/tree/master/lib"), 1.0F);
return(mapping);
}
// if a version is provided, search that JSONElement, otherwise, just search the latest
// version, which is more likely best maintained
// TODO: If the latest version JSONElement doesnt have the repo infor, should we search all elements
// on that chance that one of them might have it?
JsonElement?versionJSON = string.IsNullOrEmpty(purl?.Version) ? GetLatestVersionElement(contentJSON) :
GetVersionElement(contentJSON, new Version(purl.Version));
if (versionJSON is JsonElement notNullVersionJSON)
{
try
{
if (!notNullVersionJSON.TryGetProperty("repository", out JsonElement repository))
{
return(mapping);
}
if (repository.ValueKind == JsonValueKind.Object)
{
string?repoType = OssUtilities.GetJSONPropertyStringIfExists(repository, "type")?.ToLower();
string?repoURL = OssUtilities.GetJSONPropertyStringIfExists(repository, "url");
// right now we deal with only github repos
if (repoType == "git" && repoURL is not null)
{
PackageURL gitPURL = GitHubProjectManager.ParseUri(new Uri(repoURL));
// we got a repository value the author specified in the metadata - so no
// further processing needed
mapping.Add(gitPURL, 1.0F);
return(mapping);
}
}
}
catch (KeyNotFoundException) { /* continue onwards */ }
catch (UriFormatException) { /* the uri specified in the metadata invalid */ }
}
return(mapping);
}
/// <inheritdoc />
public override async Task <PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool useCache = true)
{
PackageMetadata metadata = new();
string? content = await GetMetadataAsync(purl, useCache);
if (string.IsNullOrEmpty(content))
{
return(null);
}
JsonDocument contentJSON = JsonDocument.Parse(content);
JsonElement root = contentJSON.RootElement;
JsonElement infoElement = root.GetProperty("info");
metadata.Name = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "name");
metadata.Description = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "summary"); // Summary is the short description. Description is usually the readme.
metadata.LatestPackageVersion = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "version"); // Ran in the root, always points to latest version.
metadata.PackageManagerUri = ENV_PYPI_ENDPOINT;
metadata.PackageUri = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "package_url");
metadata.Keywords = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(infoElement, "keywords"));
// author
User author = new()
{
Name = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "author"),
Email = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "author_email"),
};
metadata.Authors ??= new List <Model.User>();
metadata.Authors.Add(author);
// maintainers
User maintainer = new()
{
Name = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "maintainer"),
Email = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "maintainer_email"),
};
metadata.Maintainers ??= new List <User>();
metadata.Maintainers.Add(maintainer);
// repository
Dictionary <PackageURL, double>?repoMappings = await SearchRepoUrlsInPackageMetadata(purl, content);
foreach (KeyValuePair <PackageURL, double> repoMapping in repoMappings)
{
Repository repository = new()
{
Rank = repoMapping.Value,
Type = repoMapping.Key.Type
};
await repository.ExtractRepositoryMetadata(repoMapping.Key);
metadata.Repository ??= new List <Repository>();
metadata.Repository.Add(repository);
}
// license
string?licenseType = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "license");
if (!string.IsNullOrWhiteSpace(licenseType))
{
metadata.Licenses ??= new List <License>();
metadata.Licenses.Add(new License()
{
Name = licenseType
});
}
// get the version, either use the provided one, or if null then use the LatestPackageVersion.
metadata.PackageVersion = purl.Version ?? metadata.LatestPackageVersion;
// if we found any version at all, get the information.
if (metadata.PackageVersion is not null)
{
Version versionToGet = new(metadata.PackageVersion);
JsonElement?versionElement = GetVersionElement(contentJSON, versionToGet);
if (versionElement is not null)
{
// fill the version specific entries
if (versionElement.Value.ValueKind == JsonValueKind.Array) // I think this should always be true.
{
foreach (JsonElement releaseFile in versionElement.Value.EnumerateArray())
{
// digests
if (OssUtilities.GetJSONPropertyIfExists(releaseFile, "digests")?.EnumerateObject()
is JsonElement.ObjectEnumerator digests)
{
metadata.Signature ??= new List <Digest>();
foreach (JsonProperty digest in digests)
{
metadata.Signature.Add(new Digest()
{
Algorithm = digest.Name,
Signature = digest.Value.ToString()
});
}
}
// TODO: Want to figure out how to store info for .whl files as well.
if (OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "packagetype") == "sdist")
{
// downloads
if (OssUtilities.GetJSONPropertyIfExists(releaseFile, "downloads")?.GetInt64() is long downloads &&
downloads != -1)
{
metadata.Downloads ??= new Downloads()
{
Overall = downloads
};
}
metadata.Size = OssUtilities.GetJSONPropertyIfExists(releaseFile, "size")?.GetInt64();
metadata.Active = !OssUtilities.GetJSONPropertyIfExists(releaseFile, "yanked")?.GetBoolean();
metadata.VersionUri = $"{ENV_PYPI_ENDPOINT}/project/{purl.Name}/{purl.Version}";
metadata.VersionDownloadUri = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "url");
string?uploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time");
if (uploadTime != null)
{
metadata.UploadTime = DateTime.Parse(uploadTime);
}
}
}
}
}
}
return(metadata);
}
/// <summary>
/// Download one Composer (PHP) package and extract it to the target directory.
/// </summary>
/// <param name="purl"> Package URL of the package to download. </param>
/// <returns> n/a </returns>
public override async Task <IEnumerable <string> > DownloadVersionAsync(PackageURL purl, bool doExtract, bool cached = false)
{
Logger.Trace("DownloadVersion {0}", purl?.ToString());
string? packageNamespace = purl?.Namespace;
string? packageName = purl?.Name;
string? packageVersion = purl?.Version;
List <string> downloadedPaths = new();
if (string.IsNullOrWhiteSpace(packageNamespace) || string.IsNullOrWhiteSpace(packageName) ||
string.IsNullOrWhiteSpace(packageVersion))
{
Logger.Debug("Unable to download [{0} {1} {2}]. All three must be defined.", packageNamespace, packageName, packageVersion);
return(downloadedPaths);
}
try
{
HttpClient httpClient = CreateHttpClient();
System.Text.Json.JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_COMPOSER_ENDPOINT}/p/{packageNamespace}/{packageName}.json");
foreach (System.Text.Json.JsonProperty topObject in doc.RootElement.GetProperty("packages").EnumerateObject())
{
foreach (System.Text.Json.JsonProperty versionObject in topObject.Value.EnumerateObject())
{
if (versionObject.Name != packageVersion)
{
continue;
}
string?url = versionObject.Value.GetProperty("dist").GetProperty("url").GetString();
System.Net.Http.HttpResponseMessage?result = await httpClient.GetAsync(url);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl);
string fsNamespace = OssUtilities.NormalizeStringForFileSystem(packageNamespace);
string fsName = OssUtilities.NormalizeStringForFileSystem(packageName);
string fsVersion = OssUtilities.NormalizeStringForFileSystem(packageVersion);
string targetName = $"composer-{fsNamespace}-{fsName}@{fsVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return(downloadedPaths);
}
if (doExtract)
{
downloadedPaths.Add(await ArchiveHelper.ExtractArchiveAsync(TopLevelExtractionDirectory, targetName, await result.Content.ReadAsStreamAsync(), cached));
}
else
{
extractionPath += ".zip";
await File.WriteAllBytesAsync(extractionPath, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(extractionPath);
}
}
}
if (downloadedPaths.Count == 0)
{
Logger.Debug("Unable to find version {0} to download.", packageVersion);
}
}
catch (Exception ex)
{
Logger.Debug(ex, "Error downloading Composer package: {0}", ex.Message);
}
return(downloadedPaths);
}
/// <inheritdoc />
public override async Task <PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool useCache = true)
{
PackageMetadata metadata = new();
string? content = await GetMetadataAsync(purl, useCache);
if (string.IsNullOrEmpty(content))
{
return(null);
}
// convert NPM package data to normalized form
JsonDocument contentJSON = JsonDocument.Parse(content);
JsonElement root = contentJSON.RootElement;
metadata.Name = root.GetProperty("name").GetString();
metadata.Description = OssUtilities.GetJSONPropertyStringIfExists(root, "description");
metadata.PackageManagerUri = ENV_NPM_ENDPOINT;
metadata.Platform = "NPM";
metadata.Language = "JavaScript";
metadata.PackageUri = $"{metadata.PackageManagerUri}/package/{metadata.Name}";
metadata.ApiPackageUri = $"{ENV_NPM_API_ENDPOINT}/{metadata.Name}";
List <Version> versions = GetVersions(contentJSON);
Version? latestVersion = GetLatestVersion(versions);
if (purl.Version != null)
{
// find the version object from the collection
metadata.PackageVersion = purl.Version;
}
else
{
metadata.PackageVersion = latestVersion is null ? purl.Version : latestVersion?.ToString();
}
// if we found any version at all, get the information
if (metadata.PackageVersion != null)
{
Version versionToGet = new(metadata.PackageVersion);
JsonElement?versionElement = GetVersionElement(contentJSON, versionToGet);
if (root.TryGetProperty("time", out JsonElement time))
{
string?uploadTime = OssUtilities.GetJSONPropertyStringIfExists(time, metadata.PackageVersion);
if (uploadTime != null)
{
metadata.UploadTime = DateTime.Parse(uploadTime);
}
}
if (versionElement != null)
{
// redo the generic values to version specific values
metadata.PackageUri = $"{ENV_NPM_ENDPOINT}/package/{metadata.Name}";
metadata.VersionUri = $"{ENV_NPM_ENDPOINT}/package/{metadata.Name}/v/{metadata.PackageVersion}";
metadata.ApiVersionUri = $"{ENV_NPM_API_ENDPOINT}/{metadata.Name}/{metadata.PackageVersion}";
// prioritize the version level description
if (OssUtilities.GetJSONPropertyStringIfExists(versionElement, "description") is string description)
{
metadata.Description = description;
}
JsonElement?distElement = OssUtilities.GetJSONPropertyIfExists(versionElement, "dist");
if (OssUtilities.GetJSONPropertyIfExists(distElement, "tarball") is JsonElement tarballElement)
{
metadata.VersionDownloadUri = tarballElement.ToString().IsBlank() ?
$"{ENV_NPM_API_ENDPOINT}/{metadata.Name}/-/{metadata.Name}-{metadata.PackageVersion}.tgz"
: tarballElement.ToString();
}
if (OssUtilities.GetJSONPropertyIfExists(distElement, "integrity") is JsonElement integrityElement &&
integrityElement.ToString() is string integrity &&
integrity.Split('-') is string[] pair &&
pair.Length == 2)
{
metadata.Signature ??= new List <Digest>();
metadata.Signature.Add(new Digest()
{
Algorithm = pair[0],
Signature = pair[1]
});
}
// size
if (OssUtilities.GetJSONPropertyIfExists(distElement, "unpackedSize") is JsonElement sizeElement &&
sizeElement.GetInt64() is long size)
{
metadata.Size = size;
}
// check for typescript
List <string>?devDependencies = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(versionElement, "devDependencies"));
if (devDependencies is not null && devDependencies.Count > 0 && devDependencies.Any(stringToCheck => stringToCheck.Contains("\"typescript\":")))
{
metadata.Language = "TypeScript";
}
// homepage
if (OssUtilities.GetJSONPropertyStringIfExists(versionElement, "homepage") is string homepage &&
!string.IsNullOrWhiteSpace(homepage))
{
metadata.Homepage = homepage;
}
// commit id
if (OssUtilities.GetJSONPropertyStringIfExists(versionElement, "gitHead") is string gitHead &&
!string.IsNullOrWhiteSpace(gitHead))
{
metadata.CommitId = gitHead;
}
// install scripts
List <string>?scripts = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(versionElement, "scripts"));
if (scripts is not null && scripts.Count > 0)
{
metadata.Scripts ??= new List <Command>();
scripts.ForEach((element) => metadata.Scripts.Add(new Command {
CommandLine = element
}));
}
// dependencies
List <string>?dependencies = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(versionElement, "dependencies"));
if (dependencies is not null && dependencies.Count > 0)
{
metadata.Dependencies ??= new List <Dependency>();
dependencies.ForEach((dependency) => metadata.Dependencies.Add(new Dependency()
{
Package = dependency
}));
}
// author(s)
JsonElement?authorElement = OssUtilities.GetJSONPropertyIfExists(versionElement, "_npmUser");
if (authorElement is not null)
{
User author = new()
{
Name = OssUtilities.GetJSONPropertyStringIfExists(authorElement, "name"),
Email = OssUtilities.GetJSONPropertyStringIfExists(authorElement, "email"),
Url = OssUtilities.GetJSONPropertyStringIfExists(authorElement, "url")
};
metadata.Authors ??= new List <User>();
metadata.Authors.Add(author);
}
// maintainers
JsonElement?maintainersElement = OssUtilities.GetJSONPropertyIfExists(versionElement, "maintainers");
if (maintainersElement?.EnumerateArray() is JsonElement.ArrayEnumerator maintainerEnumerator)
{
metadata.Maintainers ??= new List <User>();
maintainerEnumerator.ToList().ForEach((element) =>
{
metadata.Maintainers.Add(
new User
{
Name = OssUtilities.GetJSONPropertyStringIfExists(element, "name"),
Email = OssUtilities.GetJSONPropertyStringIfExists(element, "email"),
Url = OssUtilities.GetJSONPropertyStringIfExists(element, "url")
});
});
}
// repository
Dictionary <PackageURL, double> repoMappings = await SearchRepoUrlsInPackageMetadata(purl, content);
foreach (KeyValuePair <PackageURL, double> repoMapping in repoMappings)
{
Repository repository = new()
{
Rank = repoMapping.Value,
Type = repoMapping.Key.Type
};
await repository.ExtractRepositoryMetadata(repoMapping.Key);
metadata.Repository ??= new List <Repository>();
metadata.Repository.Add(repository);
}
// keywords
metadata.Keywords = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(versionElement, "keywords"));
// licenses
{
if (OssUtilities.GetJSONEnumerator(OssUtilities.GetJSONPropertyIfExists(versionElement, "licenses"))
is JsonElement.ArrayEnumerator enumeratorElement &&
enumeratorElement.ToList() is List <JsonElement> enumerator &&
enumerator.Any())
{
metadata.Licenses ??= new List <License>();
// TODO: Convert/append SPIX_ID values?
enumerator.ForEach((license) =>
{
metadata.Licenses.Add(new License()
{
Name = OssUtilities.GetJSONPropertyStringIfExists(license, "type"),
Url = OssUtilities.GetJSONPropertyStringIfExists(license, "url")
});
});
}
}
}
}
if (latestVersion is not null)
{
metadata.LatestPackageVersion = latestVersion.ToString();
}
return(metadata);
}