public override async ValueTask ScanAsync(ScanFileContext context)
{
var doc = await XmlUtilities.TryLoadDocumentWithoutClosingStream(context.Content, context.CancellationToken).ConfigureAwait(false);
if (doc == null || doc.Root == null)
{
return;
}
var ns = doc.Root.GetDefaultNamespace();
foreach (var package in doc.Descendants(ns + "PackageReference"))
{
var packageName = package.Attribute(s_includeName)?.Value;
if (string.IsNullOrEmpty(packageName))
{
continue;
}
var versionAttribute = package.Attribute(s_versionName)?.Value;
if (!string.IsNullOrEmpty(versionAttribute))
{
await context.ReportDependency(new Dependency(packageName, versionAttribute, DependencyType.NuGet, new XmlLocation(context.FullPath, package, "Version"))).ConfigureAwait(false);
}
else
{
var versionElement = package.Element(ns + "Version");
if (!string.IsNullOrEmpty(versionElement?.Value))
{
await context.ReportDependency(new Dependency(packageName, versionElement.Value, DependencyType.NuGet, new XmlLocation(context.FullPath, versionElement))).ConfigureAwait(false);
}
}
}
}
private static async Task ScanDirectoryParallelAsync <T>(string path, ScannerOptions options, DependencyFound onDependencyFound, CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
var filesToScanChannel = Channel.CreateBounded <FileToScan <T> >(new BoundedChannelOptions(10000)
{
AllowSynchronousContinuations = true,
SingleWriter = true,
SingleReader = false,
FullMode = BoundedChannelFullMode.Wait,
});
// Start enumerating
var enumeratorTask = Task.Run(async() =>
{
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
await filesToScanChannel.Writer.WriteAsync(enumerator.Current, cancellationToken).ConfigureAwait(false);
}
filesToScanChannel.Writer.Complete();
}, cancellationToken);
// Parse files
var tasks = new Task[options.DegreeOfParallelism + 1];
tasks[0] = enumeratorTask;
Array.Fill(tasks, Task.Run(async() =>
{
var reader = filesToScanChannel.Reader;
while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
while (reader.TryRead(out var entry))
{
var scanFileContext = new ScanFileContext(entry.FullPath, onDependencyFound, fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
}, cancellationToken), startIndex: 1, options.DegreeOfParallelism);
await Task.WhenAll(tasks).ConfigureAwait(false);
}
private static async Task ScanDirectoryAsync <T>(string path, ScannerOptions options, DependencyFound onDependencyFound, CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
var entry = enumerator.Current;
var scanFileContext = new ScanFileContext(entry.FullPath, onDependencyFound, fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
using var sr = await StreamUtilities.CreateReaderAsync(context.Content, context.CancellationToken).ConfigureAwait(false);
var lineNo = 0;
string?line;
while ((line = await sr.ReadLineAsync().ConfigureAwait(false)) != null)
{
lineNo++;
var match = s_pypiReferenceRegex.Match(line);
if (!match.Success)
{
continue;
}
// Name==1.2.2
var packageName = match.Groups["PACKAGENAME"].Value;
var versionGroup = match.Groups["VERSION"];
var version = versionGroup.Value;
var column = versionGroup.Index + 1;
await context.ReportDependency(new Dependency(packageName, version, DependencyType.PyPi, new TextLocation(context.FullPath, lineNo, column, versionGroup.Length))).ConfigureAwait(false);
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
using var repository = new LibGit2Sharp.Repository(Path.GetDirectoryName(context.FullPath));
foreach (var module in repository.Submodules)
{
await context.ReportDependency(new Dependency(module.Url, module.WorkDirCommitId.Sha, DependencyType.GitSubmodule, new NonUpdatableLocation(context.FullPath))).ConfigureAwait(false);
}
}
private static async ValueTask FindInImports(ScanFileContext context, Dependency dependency, string file, XDocument doc)
{
var imports = doc.Descendants().Where(element => element.Name.LocalName == "Import");
foreach (var import in imports)
{
await FindDependencyInAttributeValue(context, dependency, file, import.Attribute(s_projectName)).ConfigureAwait(false);
await FindDependencyInAttributeValue(context, dependency, file, import.Attribute(s_conditionName)).ConfigureAwait(false);
}
}
private static async ValueTask FindInErrors(ScanFileContext context, Dependency dependency, string file, XDocument doc)
{
var errors = doc.Descendants()
.Where(element => element.Name.LocalName == "Target")
.Elements()
.Where(element => element.Name.LocalName == "Error");
foreach (var error in errors)
{
await FindDependencyInAttributeValue(context, dependency, file, error.Attribute(s_textName)).ConfigureAwait(false);
await FindDependencyInAttributeValue(context, dependency, file, error.Attribute(s_conditionName)).ConfigureAwait(false);
}
}
private static async ValueTask FindInReferences(ScanFileContext context, Dependency dependency, string csprojPath, XDocument csproj)
{
var hints = csproj.Descendants()
.Where(element => element.Name.LocalName == "Reference")
.Elements()
.Where(element => element.Name.LocalName == "HintPath");
foreach (var hint in hints)
{
if (await FindDependencyInElementValue(context, dependency, csprojPath, hint).ConfigureAwait(false))
{
await FindDependencyInAssemblyName(context, dependency, csprojPath, hint.Parent?.Attribute(s_includeName)).ConfigureAwait(false);
}
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
try
{
using var sr = await StreamUtilities.CreateReaderAsync(context.Content, context.CancellationToken).ConfigureAwait(false);
using var jsonReader = new JsonTextReader(sr);
var doc = await JToken.ReadFromAsync(jsonReader, context.CancellationToken).ConfigureAwait(false);
foreach (var deps in doc.SelectTokens("$..dependencies").Concat(doc.SelectTokens("$.tools")).OfType <JObject>())
{
foreach (var dep in deps.Properties())
{
JToken valueElement = dep;
var packageName = dep.Name;
string?version;
if (dep.Value.Type == JTokenType.String)
{
version = dep.Value.Value <string>();
}
else if (dep.Value.Type == JTokenType.Object)
{
var token = dep.Value.SelectToken("$.version");
if (token == null)
{
continue;
}
version = token.Value <string>();
valueElement = token;
}
else
{
continue;
}
if (version != null)
{
await context.ReportDependency(new Dependency(packageName, version, DependencyType.NuGet, new JsonLocation(context.FullPath, LineInfo.FromJToken(dep), valueElement.Path))).ConfigureAwait(false);
}
}
}
}
catch (JsonException)
{
}
}
private static async ValueTask ScanDependenciesAsync(ScanFileContext context, JObject deps)
{
foreach (var dep in deps.Properties())
{
if (dep.Value is null)
{
continue;
}
var packageName = dep.Name;
string?version = null;
if (dep.Value.Type == JTokenType.String)
{
if (dep.Value != null)
{
version = dep.Value.Value <string>();
}
}
else if (dep.Value.Type == JTokenType.Object)
{
if (dep.Value != null)
{
var token = dep.Value.SelectToken("$.version");
if (token != null)
{
version = token.Value <string>();
}
}
}
else
{
continue;
}
if (version is null)
{
continue;
}
if (dep.Value != null)
{
var dependency = new Dependency(packageName, version, DependencyType.Npm, new JsonLocation(context.FullPath, LineInfo.FromJToken(dep), dep.Value.Path));
await context.ReportDependency(dependency).ConfigureAwait(false);
}
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
var doc = await XmlUtilities.TryLoadDocumentWithoutClosingStream(context.Content, context.CancellationToken).ConfigureAwait(false);
if (doc == null || doc.Root == null)
{
return;
}
foreach (var dependency in doc.Descendants(s_dependencyName))
{
var id = dependency.Attribute(s_idName)?.Value;
var version = dependency.Attribute(s_versionName)?.Value;
if (!string.IsNullOrEmpty(id) && !string.IsNullOrEmpty(version))
{
await context.ReportDependency(new Dependency(id, version, DependencyType.NuGet, new XmlLocation(context.FullPath, dependency, "version"))).ConfigureAwait(false);
}
}
}
private static async ValueTask FindDependencyInAssemblyName(ScanFileContext context, Dependency dependency, string file, XAttribute?attribute)
{
if (attribute == null)
{
return;
}
var value = attribute.Value;
var match = Regex.Match(value, "(?<=Version=)(?<Version>[0-9.]+)", RegexOptions.ExplicitCapture, TimeSpan.FromSeconds(1));
if (match.Success)
{
var version = match.Groups["Version"].Value;
if (Version.TryParse(version, out var v) && v != s_versionZero && v != s_versionOne)
{
Debug.Assert(attribute.Parent != null);
var location = new AssemblyVersionXmlLocation(file, attribute.Parent, attribute.Name.LocalName, column: match.Index, length: match.Value.Length);
await context.ReportDependency(new Dependency(dependency.Name, dependency.Version, dependency.Type, location)).ConfigureAwait(false);
}
}
}
private static async ValueTask <bool> FindDependencyInElementValue(ScanFileContext context, Dependency dependency, string file, XElement element)
{
if (element == null)
{
return(false);
}
var value = element.Value;
var indexOf = value.IndexOf(dependency.Name + '.' + dependency.Version, StringComparison.OrdinalIgnoreCase);
if (indexOf < 0)
{
return(false);
}
var versionStartColumn = indexOf + (dependency.Name + '.').Length;
var location = new XmlLocation(file, element, column: versionStartColumn, length: dependency.Version.Length);
await context.ReportDependency(new Dependency(dependency.Name, dependency.Version, dependency.Type, location)).ConfigureAwait(false);
return(true);
}
private static async ValueTask FindDependencyInAttributeValue(ScanFileContext context, Dependency dependency, string file, XAttribute?attribute)
{
if (attribute == null)
{
return;
}
var value = attribute.Value;
var indexOf = value.IndexOf(dependency.Name + '.' + dependency.Version, StringComparison.OrdinalIgnoreCase);
if (indexOf < 0)
{
return;
}
var versionStartColumn = indexOf + (dependency.Name + '.').Length;
Debug.Assert(attribute.Parent != null);
var location = new XmlLocation(file, attribute.Parent, attribute.Name.LocalName, column: versionStartColumn, length: dependency.Version.Length);
await context.ReportDependency(new Dependency(dependency.Name, dependency.Version, dependency.Type, location)).ConfigureAwait(false);
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
var doc = await XmlUtilities.LoadDocumentWithoutClosingStreamAsync(context.Content, context.CancellationToken).ConfigureAwait(false);
if (doc == null)
{
return;
}
IReadOnlyList <(string Path, XDocument Document)>?csprojs = null;
foreach (var package in doc.Descendants(s_packageName))
{
var packageName = package.Attribute(s_idName)?.Value;
var version = package.Attribute(s_versionName)?.Value;
if (string.IsNullOrEmpty(packageName) || string.IsNullOrEmpty(version))
{
continue;
}
var dependency = new Dependency(packageName, version, DependencyType.NuGet, new XmlLocation(context.FullPath, package, "version"));
await context.ReportDependency(dependency).ConfigureAwait(false);
if (SearchForReferencesInAssociatedCsprojFiles)
{
if (csprojs == null)
{
csprojs = await LoadAssociatedCsProjAsync(context).ConfigureAwait(false);
}
foreach (var(file, csproj) in csprojs)
{
await FindInReferences(context, dependency, file, csproj).ConfigureAwait(false);
await FindInImports(context, dependency, file, csproj).ConfigureAwait(false);
await FindInErrors(context, dependency, file, csproj).ConfigureAwait(false);
}
}
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
try
{
using var sr = await StreamUtilities.CreateReaderAsync(context.Content, context.CancellationToken).ConfigureAwait(false);
using var jsonReader = new JsonTextReader(sr);
var doc = await JToken.ReadFromAsync(jsonReader, context.CancellationToken).ConfigureAwait(false);
// https://docs.npmjs.com/files/package.json#dependencies
foreach (var deps in doc.SelectTokens("$.dependencies").OfType <JObject>())
{
await ScanDependenciesAsync(context, deps).ConfigureAwait(false);
}
// https://docs.npmjs.com/files/package.json#devdependencies
foreach (var deps in doc.SelectTokens("$.devDependencies").OfType <JObject>())
{
await ScanDependenciesAsync(context, deps).ConfigureAwait(false);
}
// https://docs.npmjs.com/files/package.json#peerdependencies
foreach (var deps in doc.SelectTokens("$.peerDependencies").OfType <JObject>())
{
await ScanDependenciesAsync(context, deps).ConfigureAwait(false);
}
// https://docs.npmjs.com/files/package.json#optionaldependencies
foreach (var deps in doc.SelectTokens("$.optionaldependencies").OfType <JObject>())
{
await ScanDependenciesAsync(context, deps).ConfigureAwait(false);
}
}
catch (JsonException)
{
}
}
public override async ValueTask ScanAsync(ScanFileContext context)
{
var doc = await XmlUtilities.TryLoadDocumentWithoutClosingStream(context.Content, context.CancellationToken).ConfigureAwait(false);
if (doc == null || doc.Root == null)
{
return;
}
var ns = doc.Root.GetDefaultNamespace();
foreach (var package in doc.Descendants(ns + "PackageReference"))
{
var packageName = package.Attribute(s_includeName)?.Value;
if (string.IsNullOrEmpty(packageName))
{
continue;
}
var versionAttribute = package.Attribute(s_versionName)?.Value;
if (!string.IsNullOrEmpty(versionAttribute))
{
await context.ReportDependency(new Dependency(packageName, versionAttribute, DependencyType.NuGet, new XmlLocation(context.FullPath, package, s_versionName.LocalName))).ConfigureAwait(false);
}
else
{
var versionElement = package.Element(ns + "Version");
if (!string.IsNullOrEmpty(versionElement?.Value))
{
await context.ReportDependency(new Dependency(packageName, versionElement.Value, DependencyType.NuGet, new XmlLocation(context.FullPath, versionElement))).ConfigureAwait(false);
}
}
}
foreach (var sdk in doc.Descendants(ns + "Sdk"))
{
var name = sdk.Attribute(s_nameName)?.Value;
if (string.IsNullOrEmpty(name))
{
continue;
}
var version = sdk.Attribute(s_versionName)?.Value;
if (string.IsNullOrEmpty(version))
{
continue;
}
await context.ReportDependency(new Dependency(name, version, DependencyType.NuGet, new XmlLocation(context.FullPath, sdk, s_versionName.LocalName))).ConfigureAwait(false);
}
foreach (var sdk in doc.Descendants().Where(element => element.Name == ns + "Import" || element.Name == ns + "Project"))
{
var value = sdk.Attribute(s_sdkName)?.Value;
if (string.IsNullOrEmpty(value))
{
continue;
}
var index = value.IndexOf('/', StringComparison.Ordinal);
if (index > 0)
{
var packageName = value.Substring(0, index);
var version = value[(index + 1)..];
private static async Task <IReadOnlyList <(string Path, XDocument Document)> > LoadAssociatedCsProjAsync(ScanFileContext context)
{
var directory = Path.GetDirectoryName(context.FullPath);
if (directory == null)
{
return(Array.Empty <(string, XDocument)>());
}
var files = context.FileSystem.GetFiles(directory, "*.csproj");
var result = new List <(string, XDocument)>();
foreach (var file in files)
{
using var stream = context.FileSystem.OpenRead(file);
var doc = await XmlUtilities.TryLoadDocumentWithoutClosingStream(stream, context.CancellationToken).ConfigureAwait(false);
if (doc == null)
{
continue;
}
result.Add((file, doc));
}
return(result);
}
public abstract ValueTask ScanAsync(ScanFileContext context);
private static async IAsyncEnumerable <Dependency> ScanDirectoryParallelAsync <T>(string path, ScannerOptions options, [EnumeratorCancellation] CancellationToken cancellationToken = default)
where T : struct, IEnabledScannersArray
{
var fileSystem = options.FileSystem;
var filesToScanChannel = Channel.CreateBounded <FileToScan <T> >(new BoundedChannelOptions(1000)
{
AllowSynchronousContinuations = true,
SingleWriter = true,
SingleReader = false,
FullMode = BoundedChannelFullMode.Wait,
});
var dependenciesChannel = Channel.CreateUnbounded <Dependency>(new UnboundedChannelOptions
{
AllowSynchronousContinuations = true,
SingleWriter = false,
SingleReader = true,
});
// Start enumerating
var enumeratorTask = Task.Run(async() =>
{
using var enumerator = new ScannerFileEnumerator <T>(path, options);
while (enumerator.MoveNext())
{
await filesToScanChannel.Writer.WriteAsync(enumerator.Current, cancellationToken).ConfigureAwait(false);
}
filesToScanChannel.Writer.Complete();
}, cancellationToken);
// Parse files
var exceptions = new ConcurrentBag <Exception>();
var tasks = new Task[options.DegreeOfParallelism];
Array.Fill(tasks, Task.Run(async() =>
{
var reader = filesToScanChannel.Reader;
while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
while (reader.TryRead(out var entry))
{
var scanFileContext = new ScanFileContext(entry.FullPath, d => dependenciesChannel.Writer.WriteAsync(d, cancellationToken), fileSystem, cancellationToken);
try
{
for (var i = 0; i < options.Scanners.Count; i++)
{
if (!entry.Scanners.Get(i))
{
continue;
}
var scanner = options.Scanners[i];
scanFileContext.ResetStream();
await scanner.ScanAsync(scanFileContext).ConfigureAwait(false);
}
}
finally
{
await scanFileContext.DisposeAsync().ConfigureAwait(false);
}
}
}
}, cancellationToken));
var whenAllTasks = Task.WhenAll(tasks);
var writerCompleteTask = whenAllTasks.ContinueWith(_ => dependenciesChannel.Writer.Complete(), cancellationToken);
await foreach (var value in dependenciesChannel.Reader.ReadAllAsync(cancellationToken).WithCancellation(cancellationToken).ConfigureAwait(false))
{
yield return(value);
}
await writerCompleteTask.ConfigureAwait(false);
await whenAllTasks.ConfigureAwait(false);
}
