private static void ApplyBaselineIgnorePatterns(UsageData actual, UsageData baseline)
{
Regex[] ignoreUsageRegexes = baseline.IgnorePatterns.NullAsEmpty()
.Select(p => p.CreateRegex())
.ToArray();
actual.IgnorePatterns = baseline.IgnorePatterns;
var ignoredUsages = actual.Usages
.Where(usage =>
{
string id = $"{usage.PackageIdentity.Id}/{usage.PackageIdentity.Version}";
return(ignoreUsageRegexes.Any(r => r.IsMatch(id)));
})
.ToArray();
actual.Usages = actual.Usages.Except(ignoredUsages).ToArray();
}
public override bool Execute()
{
var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile)));
string baselineText = string.IsNullOrEmpty(BaselineDataFile)
? "<UsageData />"
: File.ReadAllText(BaselineDataFile);
var baseline = UsageData.Parse(XElement.Parse(baselineText));
UsageValidationData data = GetUsageValidationData(baseline, used);
Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile));
File.WriteAllText(OutputBaselineFile, data.ActualUsageData.ToXml().ToString());
Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile));
File.WriteAllText(OutputReportFile, data.Report.ToString());
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
string baselineRelativeFileName = PrebuiltBaselineFile.Replace(RootDirectory, "");
string gitLogCommand = $"log --first-parent --pretty=format:%H,%f,%ci -- {PrebuiltBaselineFile}";
DateTime startTime = DateTime.Now;
Log.LogMessage(MessageImportance.High, "Generating summary usage burndown data...");
ParallelQuery <string> data = ExecuteGitCommand(RootDirectory, gitLogCommand).AsParallel().Select(commitLine =>
{
var splitLine = commitLine.Split(',');
var commit = new Commit()
{
Sha = splitLine[0],
Title = splitLine[1],
CommitDate = DateTime.Parse(splitLine[2])
};
string fileContents = GetFileContents(baselineRelativeFileName, commit.Sha);
Usage[] usages = UsageData.Parse(XElement.Parse(fileContents)).Usages.NullAsEmpty().ToArray();
commit.PackageVersionCount = usages.Count();
commit.PackageCount = usages.GroupBy(i => i.PackageIdentity.Id).Select(grp => grp.First()).Count();
return(commit);
})
.Select(c => c.ToString());
Directory.CreateDirectory(Path.GetDirectoryName(OutputFilePath));
File.WriteAllLines(OutputFilePath, data);
Log.LogMessage(
MessageImportance.High,
$"Generating summary usage burndown data at {OutputFilePath} done. Took {DateTime.Now - startTime}");
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile)));
var baseline = UsageData.Parse(XElement.Parse(File.ReadAllText(BaselineDataFile)));
Comparison <PackageIdentity> diff = Compare(
used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(),
baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct());
var report = new XElement("BaselineComparison");
bool tellUserToUpdateBaseline = false;
if (diff.Added.Any())
{
tellUserToUpdateBaseline = true;
Log.LogError(
$"{diff.Added.Length} new packages used not in baseline! See report " +
$"at {OutputReportFile} for more information. Package IDs are:\n" +
string.Join("\n", diff.Added.Select(u => u.ToString())));
// In the report, list full usage info, not only identity.
report.Add(
new XElement(
"New",
used.Usages
.Where(u => diff.Added.Contains(u.GetIdentityWithoutRid()))
.Select(u => u.ToXml())));
}
if (diff.Removed.Any())
{
tellUserToUpdateBaseline = true;
Log.LogMessage(
MessageImportance.High,
$"{diff.Removed.Length} packages in baseline weren't used!");
report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement())));
}
if (diff.Unchanged.Any())
{
Log.LogMessage(
MessageImportance.High,
$"{diff.Unchanged.Length} packages used as expected in the baseline.");
}
// Simplify the used data to what is necessary for a baseline, to reduce file size.
foreach (var usage in used.Usages)
{
usage.AssetsFile = null;
}
used.Usages = used.Usages.Distinct().ToArray();
Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile));
File.WriteAllText(OutputBaselineFile, used.ToXml().ToString());
Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile));
File.WriteAllText(OutputReportFile, report.ToString());
if (tellUserToUpdateBaseline)
{
Log.LogWarning(
"Prebuilt usages are different from the baseline. If detected changes are " +
"acceptable, update baseline with:\n" +
$"cp '{OutputBaselineFile}' '{BaselineDataFile}'");
}
return(!Log.HasLoggedErrors);
}
public UsageValidationData GetUsageValidationData(UsageData baseline, UsageData used)
{
// Remove prebuilts from the used data if the baseline says to ignore them. Do this
// first, so the new generated baseline doesn't list usages that are ignored by a
// pattern anyway.
ApplyBaselineIgnorePatterns(used, baseline);
// Find new, removed, and unchanged usage after filtering patterns.
Comparison <PackageIdentity> diff = Compare(
used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(),
baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct());
var report = new XElement("BaselineComparison");
bool tellUserToUpdateBaseline = false;
if (diff.Added.Any())
{
tellUserToUpdateBaseline = true;
Log.LogError(
$"{diff.Added.Length} new packages used not in baseline! See report " +
$"at {OutputReportFile} for more information. Package IDs are:\n" +
string.Join("\n", diff.Added.Select(u => u.ToString())));
// In the report, list full usage info, not only identity.
report.Add(
new XElement(
"New",
used.Usages
.Where(u => diff.Added.Contains(u.GetIdentityWithoutRid()))
.Select(u => u.ToXml())));
}
if (diff.Removed.Any())
{
tellUserToUpdateBaseline = true;
Log.LogMessage(
MessageImportance.High,
$"{diff.Removed.Length} packages in baseline weren't used!");
report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement())));
}
if (diff.Unchanged.Any())
{
Log.LogMessage(
MessageImportance.High,
$"{diff.Unchanged.Length} packages used as expected in the baseline.");
}
if (!AllowTestProjectUsage)
{
Usage[] testProjectUsages = used.Usages
.Where(WriteUsageReports.IsTestUsageByHeuristic)
.ToArray();
if (testProjectUsages.Any())
{
string[] projects = testProjectUsages
.Select(u => u.AssetsFile)
.Distinct()
.ToArray();
Log.LogError(
$"{testProjectUsages.Length} forbidden test usages found in " +
$"{projects.Length} projects:\n" +
string.Join("\n", projects));
}
}
// Simplify the used data to what is necessary for a baseline, to reduce file size.
foreach (var usage in used.Usages)
{
usage.AssetsFile = null;
}
used.ProjectDirectories = null;
used.Usages = used.Usages.Distinct().ToArray();
if (tellUserToUpdateBaseline)
{
string baselineNotFoundWarning = "";
if (string.IsNullOrEmpty(BaselineDataFile))
{
baselineNotFoundWarning =
$"not expected, because no baseline file exists at '{BaselineDataUpdateHintFile}'";
}
else
{
baselineNotFoundWarning =
$"different from the baseline found at '{BaselineDataFile}'";
}
Log.LogMessage(
MessageImportance.High,
$"Prebuilt usages are {baselineNotFoundWarning}. If it's acceptable to " +
"update the baseline, copy the contents of the automatically generated " +
$"baseline '{OutputBaselineFile}'.");
}
return(new UsageValidationData
{
Report = report,
ActualUsageData = used
});
}
public override bool Execute()
{
UsageData data = JObject.Parse(File.ReadAllText(DataFile)).ToObject <UsageData>();
IEnumerable <RepoOutput> sourceBuildRepoOutputs = GetSourceBuildRepoOutputs();
// Map package id to the build name that created them in a ProdCon build.
var prodConPackageOrigin = new Dictionary <string, string>(
StringComparer.OrdinalIgnoreCase);
foreach (ITaskItem item in ProdConPackageInfos ?? Enumerable.Empty <ITaskItem>())
{
AddProdConPackage(
prodConPackageOrigin,
item.GetMetadata("PackageId"),
item.GetMetadata("OriginBuildName"));
}
if (File.Exists(ProdConBuildManifestFile))
{
var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile));
foreach (var x in xml.Descendants("Package"))
{
AddProdConPackage(
prodConPackageOrigin,
x.Attribute("Id")?.Value,
x.Attribute("OriginBuildName")?.Value);
}
}
PreviousReleasePrebuilt[] previousReleasePrebuilts = PreviousReleasePrebuiltPackageInfos
?.Select(item => new PreviousReleasePrebuilt
{
Id = item.GetMetadata("PackageId"),
Version = item.GetMetadata("PackageVersion"),
Release = item.GetMetadata("Release")
})
.ToArray();
var poisonNupkgFilenames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (File.Exists(PoisonedReportFile))
{
foreach (string line in File.ReadAllLines(PoisonedReportFile))
{
string[] segments = line.Split('\\');
if (segments.Length > 2 &&
segments[0].Trim() == "intermediate" &&
segments[1].EndsWith(".nupkg"))
{
poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1]));
}
}
}
foreach (Usage usage in data.Usages)
{
string[] identity = usage.PackageIdentity.Split('/');
if (identity.Length != 2)
{
Log.LogWarning(
$"Data file contains invalid package identity '{usage.PackageIdentity}'. " +
"Expected 2 segments when split by '/'.");
continue;
}
string id = identity[0];
string version = identity[1];
string pvpIdent = WriteBuildOutputProps.GetPropertyName(id);
var sourceBuildCreator = new StringBuilder();
foreach (RepoOutput output in sourceBuildRepoOutputs)
{
foreach (PackageVersionPropsElement p in output.Built)
{
if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase))
{
if (sourceBuildCreator.Length != 0)
{
sourceBuildCreator.Append(" ");
}
sourceBuildCreator.Append(output.Repo);
sourceBuildCreator.Append(" ");
sourceBuildCreator.Append(p.Name);
sourceBuildCreator.Append("/");
sourceBuildCreator.Append(p.Version);
}
}
}
prodConPackageOrigin.TryGetValue(id, out string prodConCreator);
string previousReleasePrebuilt = previousReleasePrebuilts
?.FirstOrDefault(p => p.Matches(id, version))
?.Release;
usage.Annotation = new UsageAnnotation
{
SourceBuildPackageIdCreator = sourceBuildCreator.ToString(),
ProdConPackageIdCreator = prodConCreator,
InEarlierReleaseExactPrebuilt = previousReleasePrebuilt,
EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}")
};
}
Directory.CreateDirectory(OutputDirectory);
File.WriteAllText(
Path.Combine(OutputDirectory, "UsageSummary.xml"),
data.CreateSummaryReport().ToString());
File.WriteAllText(
Path.Combine(OutputDirectory, "UsageDetails.xml"),
data.CreateDetailedReport().ToString());
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
UsageData data = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile)));
IEnumerable <RepoOutput> sourceBuildRepoOutputs = GetSourceBuildRepoOutputs();
// Map package id to the build name that created them in a ProdCon build.
var prodConPackageOrigin = new Dictionary <string, string>(
StringComparer.OrdinalIgnoreCase);
foreach (ITaskItem item in ProdConPackageInfos.NullAsEmpty())
{
AddProdConPackage(
prodConPackageOrigin,
item.GetMetadata("PackageId"),
item.GetMetadata("OriginBuildName"));
}
if (File.Exists(ProdConBuildManifestFile))
{
var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile));
foreach (var x in xml.Descendants("Package"))
{
AddProdConPackage(
prodConPackageOrigin,
x.Attribute("Id")?.Value,
x.Attribute("OriginBuildName")?.Value);
}
}
var poisonNupkgFilenames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (File.Exists(PoisonedReportFile))
{
foreach (string line in File.ReadAllLines(PoisonedReportFile))
{
string[] segments = line.Split('\\');
if (segments.Length > 2 &&
segments[0].Trim() == "intermediate" &&
segments[1].EndsWith(".nupkg"))
{
poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1]));
}
}
}
var report = new XElement("AnnotatedUsages");
foreach (Usage usage in data.Usages.NullAsEmpty())
{
string id = usage.PackageIdentity.Id;
string version = usage.PackageIdentity.Version.OriginalVersion;
string pvpIdent = WriteBuildOutputProps.GetPropertyName(id);
var sourceBuildCreator = new StringBuilder();
foreach (RepoOutput output in sourceBuildRepoOutputs)
{
foreach (PackageVersionPropsElement p in output.Built)
{
if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase))
{
if (sourceBuildCreator.Length != 0)
{
sourceBuildCreator.Append(" ");
}
sourceBuildCreator.Append(output.Repo);
sourceBuildCreator.Append(" ");
sourceBuildCreator.Append(p.Name);
sourceBuildCreator.Append("/");
sourceBuildCreator.Append(p.Version);
}
}
}
prodConPackageOrigin.TryGetValue(id, out string prodConCreator);
var annotated = new AnnotatedUsage
{
Usage = usage,
Project = data.ProjectDirectories
?.FirstOrDefault(p => usage.AssetsFile?.StartsWith(p) ?? false),
SourceBuildPackageIdCreator = sourceBuildCreator.Length == 0
? null
: sourceBuildCreator.ToString(),
ProdConPackageIdCreator = prodConCreator,
EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}")
};
report.Add(annotated.ToXml());
}
Directory.CreateDirectory(OutputDirectory);
File.WriteAllText(
Path.Combine(OutputDirectory, "annotated-usage.xml"),
report.ToString());
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
DateTime startTime = DateTime.Now;
Log.LogMessage(MessageImportance.High, "Writing package usage data...");
string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty()
.Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal))
.ToArray();
if (projectDirectoriesOutsideRoot.Any())
{
throw new ArgumentException(
$"All ProjectDirectories must be in RootDir '{RootDir}', but found " +
string.Join(", ", projectDirectoriesOutsideRoot));
}
Log.LogMessage(MessageImportance.Low, "Finding set of RIDs...");
string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty()
.SelectMany(ReadRidsFromRuntimeJson)
.Distinct()
.ToArray();
Log.LogMessage(MessageImportance.Low, "Reading package identities...");
PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
PackageIdentity[] referencePackages = ReferencePackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
IEnumerable <PackageIdentity> prebuilt = restored.Except(sourceBuilt).Except(referencePackages);
PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty()
.Select(item => new PackageIdentity(
item.GetMetadata("PackageId"),
NuGetVersion.Parse(item.GetMetadata("PackageVersion"))))
.Concat(prebuilt)
.ToArray();
Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files...");
string[] assetFiles = Directory
.GetFiles(RootDir, "project.assets.json", SearchOption.AllDirectories)
.Select(GetPathRelativeToRoot)
.Except(IgnoredProjectAssetsJsonFiles.NullAsEmpty().Select(GetPathRelativeToRoot))
.ToArray();
if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile))
{
Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files...");
Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile));
using (var projectAssetArchive = new ZipArchive(
File.Open(
ProjectAssetsJsonArchiveFile,
FileMode.Create,
FileAccess.ReadWrite),
ZipArchiveMode.Create))
{
// Only one entry can be open at a time, so don't do this during the Parallel
// ForEach later.
foreach (var relativePath in assetFiles)
{
using (var stream = File.OpenRead(Path.Combine(RootDir, relativePath)))
using (Stream entryWriter = projectAssetArchive
.CreateEntry(relativePath, CompressionLevel.Optimal)
.Open())
{
stream.CopyTo(entryWriter);
}
}
}
}
Log.LogMessage(MessageImportance.Low, "Reading usage info...");
var usages = new ConcurrentBag <Usage>();
Parallel.ForEach(
assetFiles,
assetFile =>
{
JObject jObj;
using (var file = File.OpenRead(Path.Combine(RootDir, assetFile)))
using (var reader = new StreamReader(file))
using (var jsonReader = new JsonTextReader(reader))
{
jObj = (JObject)JToken.ReadFrom(jsonReader);
}
var properties = new HashSet <string>(
jObj.SelectTokens("$.targets.*").Children()
.Concat(jObj.SelectToken("$.libraries"))
.Select(t => ((JProperty)t).Name)
.Distinct(),
StringComparer.OrdinalIgnoreCase);
var directDependencies = jObj.SelectTokens("$.project.frameworks.*.dependencies").Children().Select(dep =>
new
{
name = ((JProperty)dep).Name,
target = dep.SelectToken("$..target")?.ToString(),
version = VersionRange.Parse(dep.SelectToken("$..version")?.ToString()),
autoReferenced = dep.SelectToken("$..autoReferenced")?.ToString() == "True",
})
.ToArray();
foreach (var identity in toCheck
.Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion)))
{
var directDependency =
directDependencies?.FirstOrDefault(
d => d.name == identity.Id &&
d.version.Satisfies(identity.Version));
usages.Add(Usage.Create(
assetFile,
identity,
directDependency != null,
directDependency?.autoReferenced == true,
possibleRids));
}
});
Log.LogMessage(MessageImportance.Low, "Searching for unused packages...");
foreach (PackageIdentity restoredWithoutUsagesFound in
toCheck.Except(usages.Select(u => u.PackageIdentity)))
{
usages.Add(Usage.Create(
null,
restoredWithoutUsagesFound,
false,
false,
possibleRids));
}
// Packages that were included in the tarball as prebuilts, but weren't even restored.
PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt
.Except(restored)
.ToArray();
Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'...");
var data = new UsageData
{
CreatedByRid = TargetRid,
Usages = usages.ToArray(),
NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts,
ProjectDirectories = ProjectDirectories
?.Select(GetPathRelativeToRoot)
.ToArray()
};
Directory.CreateDirectory(Path.GetDirectoryName(DataFile));
File.WriteAllText(DataFile, data.ToXml().ToString());
Log.LogMessage(
MessageImportance.High,
$"Writing package usage data... done. Took {DateTime.Now - startTime}");
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
string[] packageIdentities = NuGetPackageInfos
.Select(item => $"{item.GetMetadata("PackageId")}/{item.GetMetadata("PackageVersion")}")
.ToArray();
// Remove trailing slash from dir to avoid EnumerateFiles method adding slash twice,
// which would cause multiple unique paths per file and break the hash set.
ProjectDirectories = ProjectDirectories
.Where(Directory.Exists)
.Select(dir => dir.TrimEnd('/', '\\'))
.ToArray();
Log.LogMessage(MessageImportance.High, "Finding project.assets.json files...");
Dictionary <string, IEnumerable <string> > projectDirAssetFiles = ProjectDirectories
.ToDictionary(
dir => dir,
dir => Directory.EnumerateFiles(
dir,
"project.assets.json",
SearchOption.AllDirectories));
Log.LogMessage(MessageImportance.High, "Reading usage info...");
var assetFileUsages = new ConcurrentDictionary <string, Usage[]>();
Parallel.ForEach(
projectDirAssetFiles.Values.SelectMany(v => v).Distinct(),
assetsFile =>
{
string contents = File.ReadAllText(assetsFile);
assetFileUsages.TryAdd(
assetsFile,
packageIdentities
.Where(id => IsIdentityInText(id, contents))
.Select(id => new Usage
{
AssetsFile = assetsFile,
PackageIdentity = id
})
.ToArray());
});
Log.LogMessage(MessageImportance.High, "Associating usage info with projects...");
var usedAssetFiles = new HashSet <string>();
var usages = new List <Usage>();
foreach (string dir in ProjectDirectories)
{
foreach (string assetsFile in projectDirAssetFiles[dir])
{
if (usedAssetFiles.Add(assetsFile))
{
foreach (var usage in assetFileUsages[assetsFile])
{
usage.ProjectDirectory = dir;
usages.Add(usage);
}
}
}
}
Log.LogMessage(MessageImportance.High, $"Writing data to '{DataFile}'...");
Directory.CreateDirectory(Path.GetDirectoryName(DataFile));
string[] unused = packageIdentities
.Where(id => !usages.Any(u => IsIdentityEqual(id, u.PackageIdentity)))
.ToArray();
var data = new UsageData
{
UnusedPackages = unused,
Usages = usages.ToArray()
};
File.WriteAllText(DataFile, JsonConvert.SerializeObject(data, Formatting.Indented));
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
DateTime startTime = DateTime.Now;
Log.LogMessage(MessageImportance.High, "Writing package usage data...");
string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty()
.Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal))
.ToArray();
if (projectDirectoriesOutsideRoot.Any())
{
throw new ArgumentException(
$"All ProjectDirectories must be in RootDir '{RootDir}', but found " +
string.Join(", ", projectDirectoriesOutsideRoot));
}
Log.LogMessage(MessageImportance.Low, "Finding set of RIDs...");
string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty()
.SelectMany(ReadRidsFromRuntimeJson)
.Distinct()
.ToArray();
Log.LogMessage(MessageImportance.Low, "Reading package identities...");
PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty()
.Select(ReadNuGetPackageInfos.ReadIdentity)
.Distinct()
.ToArray();
IEnumerable <PackageIdentity> prebuilt = restored.Except(sourceBuilt);
PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty()
.Select(item => new PackageIdentity(
item.GetMetadata("PackageId"),
NuGetVersion.Parse(item.GetMetadata("PackageVersion"))))
.Concat(prebuilt)
.ToArray();
Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files...");
string[] assetFiles = Directory.GetFiles(
RootDir,
"project.assets.json",
SearchOption.AllDirectories);
if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile))
{
Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files...");
Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile));
using (var projectAssetArchive = new ZipArchive(
File.Open(
ProjectAssetsJsonArchiveFile,
FileMode.Create,
FileAccess.ReadWrite),
ZipArchiveMode.Create))
{
// Only one entry can be open at a time, so don't do this during the Parallel
// ForEach later.
foreach (var file in assetFiles)
{
string relativePath = file.Substring(RootDir.Length);
using (var stream = File.OpenRead(file))
using (Stream entryWriter = projectAssetArchive
.CreateEntry(relativePath, CompressionLevel.Optimal)
.Open())
{
stream.CopyTo(entryWriter);
}
}
}
}
Log.LogMessage(MessageImportance.Low, "Reading usage info...");
var usages = new ConcurrentBag <Usage>();
Parallel.ForEach(
assetFiles,
assetFile =>
{
var properties = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
using (var file = File.OpenRead(assetFile))
using (var reader = new StreamReader(file))
using (var jsonReader = new JsonTextReader(reader))
{
while (jsonReader.Read())
{
if (jsonReader.TokenType == JsonToken.PropertyName &&
jsonReader.Value is string value)
{
properties.Add(value);
}
}
}
foreach (var identity in toCheck
.Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion)))
{
usages.Add(Usage.Create(
// Store relative path for future report generation.
assetFile.Substring(RootDir.Length),
identity,
possibleRids));
}
});
Log.LogMessage(MessageImportance.Low, "Searching for unused packages...");
foreach (PackageIdentity restoredWithoutUsagesFound in
toCheck.Except(usages.Select(u => u.PackageIdentity)))
{
usages.Add(Usage.Create(
null,
restoredWithoutUsagesFound,
possibleRids));
}
// Packages that were included in the tarball as prebuilts, but weren't even restored.
PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt
.Except(restored)
.ToArray();
Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'...");
var data = new UsageData
{
CreatedByRid = TargetRid,
Usages = usages.ToArray(),
NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts,
ProjectDirectories = ProjectDirectories
?.Select(dir => dir.Substring(RootDir.Length))
.ToArray()
};
Directory.CreateDirectory(Path.GetDirectoryName(DataFile));
File.WriteAllText(DataFile, data.ToXml().ToString());
Log.LogMessage(
MessageImportance.High,
$"Writing package usage data... done. Took {DateTime.Now - startTime}");
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile)));
string baselineText = string.IsNullOrEmpty(BaselineDataFile)
? "<UsageData />"
: File.ReadAllText(BaselineDataFile);
var baseline = UsageData.Parse(XElement.Parse(baselineText));
Comparison <PackageIdentity> diff = Compare(
used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(),
baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct());
var report = new XElement("BaselineComparison");
bool tellUserToUpdateBaseline = false;
if (diff.Added.Any())
{
tellUserToUpdateBaseline = true;
Log.LogError(
$"{diff.Added.Length} new packages used not in baseline! See report " +
$"at {OutputReportFile} for more information. Package IDs are:\n" +
string.Join("\n", diff.Added.Select(u => u.ToString())));
// In the report, list full usage info, not only identity.
report.Add(
new XElement(
"New",
used.Usages
.Where(u => diff.Added.Contains(u.GetIdentityWithoutRid()))
.Select(u => u.ToXml())));
}
if (diff.Removed.Any())
{
tellUserToUpdateBaseline = true;
Log.LogMessage(
MessageImportance.High,
$"{diff.Removed.Length} packages in baseline weren't used!");
report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement())));
}
if (diff.Unchanged.Any())
{
Log.LogMessage(
MessageImportance.High,
$"{diff.Unchanged.Length} packages used as expected in the baseline.");
}
if (!AllowTestProjectUsage)
{
Usage[] testProjectUsages = used.Usages
.Where(WriteUsageReports.IsTestUsageByHeuristic)
.ToArray();
if (testProjectUsages.Any())
{
string[] projects = testProjectUsages
.Select(u => u.AssetsFile)
.Distinct()
.ToArray();
Log.LogError(
$"{testProjectUsages.Length} forbidden test usages found in " +
$"{projects.Length} projects:\n" +
string.Join("\n", projects));
}
}
// Simplify the used data to what is necessary for a baseline, to reduce file size.
foreach (var usage in used.Usages)
{
usage.AssetsFile = null;
}
used.Usages = used.Usages.Distinct().ToArray();
Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile));
File.WriteAllText(OutputBaselineFile, used.ToXml().ToString());
Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile));
File.WriteAllText(OutputReportFile, report.ToString());
if (tellUserToUpdateBaseline)
{
string baselineNotFoundWarning = "";
if (string.IsNullOrEmpty(BaselineDataFile))
{
baselineNotFoundWarning =
$"not expected, because no baseline file exists at '{BaselineDataUpdateHintFile}'";
}
else
{
baselineNotFoundWarning =
$"different from the baseline found at '{BaselineDataFile}'";
}
Log.LogWarning(
$"Prebuilt usages are {baselineNotFoundWarning}. If it's acceptable to " +
"update the baseline, copy the contents of the automatically generated " +
$"baseline '{OutputBaselineFile}'.");
}
return(!Log.HasLoggedErrors);
}
