private static void ApplyBaselineIgnorePatterns(UsageData actual, UsageData baseline) { Regex[] ignoreUsageRegexes = baseline.IgnorePatterns.NullAsEmpty() .Select(p => p.CreateRegex()) .ToArray(); actual.IgnorePatterns = baseline.IgnorePatterns; var ignoredUsages = actual.Usages .Where(usage => { string id = $"{usage.PackageIdentity.Id}/{usage.PackageIdentity.Version}"; return(ignoreUsageRegexes.Any(r => r.IsMatch(id))); }) .ToArray(); actual.Usages = actual.Usages.Except(ignoredUsages).ToArray(); }
public override bool Execute() { var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); string baselineText = string.IsNullOrEmpty(BaselineDataFile) ? "<UsageData />" : File.ReadAllText(BaselineDataFile); var baseline = UsageData.Parse(XElement.Parse(baselineText)); UsageValidationData data = GetUsageValidationData(baseline, used); Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile)); File.WriteAllText(OutputBaselineFile, data.ActualUsageData.ToXml().ToString()); Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile)); File.WriteAllText(OutputReportFile, data.Report.ToString()); return(!Log.HasLoggedErrors); }
public override bool Execute() { string baselineRelativeFileName = PrebuiltBaselineFile.Replace(RootDirectory, ""); string gitLogCommand = $"log --first-parent --pretty=format:%H,%f,%ci -- {PrebuiltBaselineFile}"; DateTime startTime = DateTime.Now; Log.LogMessage(MessageImportance.High, "Generating summary usage burndown data..."); ParallelQuery <string> data = ExecuteGitCommand(RootDirectory, gitLogCommand).AsParallel().Select(commitLine => { var splitLine = commitLine.Split(','); var commit = new Commit() { Sha = splitLine[0], Title = splitLine[1], CommitDate = DateTime.Parse(splitLine[2]) }; string fileContents = GetFileContents(baselineRelativeFileName, commit.Sha); Usage[] usages = UsageData.Parse(XElement.Parse(fileContents)).Usages.NullAsEmpty().ToArray(); commit.PackageVersionCount = usages.Count(); commit.PackageCount = usages.GroupBy(i => i.PackageIdentity.Id).Select(grp => grp.First()).Count(); return(commit); }) .Select(c => c.ToString()); Directory.CreateDirectory(Path.GetDirectoryName(OutputFilePath)); File.WriteAllLines(OutputFilePath, data); Log.LogMessage( MessageImportance.High, $"Generating summary usage burndown data at {OutputFilePath} done. Took {DateTime.Now - startTime}"); return(!Log.HasLoggedErrors); }
public override bool Execute() { var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); var baseline = UsageData.Parse(XElement.Parse(File.ReadAllText(BaselineDataFile))); Comparison <PackageIdentity> diff = Compare( used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(), baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct()); var report = new XElement("BaselineComparison"); bool tellUserToUpdateBaseline = false; if (diff.Added.Any()) { tellUserToUpdateBaseline = true; Log.LogError( $"{diff.Added.Length} new packages used not in baseline! See report " + $"at {OutputReportFile} for more information. Package IDs are:\n" + string.Join("\n", diff.Added.Select(u => u.ToString()))); // In the report, list full usage info, not only identity. report.Add( new XElement( "New", used.Usages .Where(u => diff.Added.Contains(u.GetIdentityWithoutRid())) .Select(u => u.ToXml()))); } if (diff.Removed.Any()) { tellUserToUpdateBaseline = true; Log.LogMessage( MessageImportance.High, $"{diff.Removed.Length} packages in baseline weren't used!"); report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement()))); } if (diff.Unchanged.Any()) { Log.LogMessage( MessageImportance.High, $"{diff.Unchanged.Length} packages used as expected in the baseline."); } // Simplify the used data to what is necessary for a baseline, to reduce file size. foreach (var usage in used.Usages) { usage.AssetsFile = null; } used.Usages = used.Usages.Distinct().ToArray(); Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile)); File.WriteAllText(OutputBaselineFile, used.ToXml().ToString()); Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile)); File.WriteAllText(OutputReportFile, report.ToString()); if (tellUserToUpdateBaseline) { Log.LogWarning( "Prebuilt usages are different from the baseline. If detected changes are " + "acceptable, update baseline with:\n" + $"cp '{OutputBaselineFile}' '{BaselineDataFile}'"); } return(!Log.HasLoggedErrors); }
public UsageValidationData GetUsageValidationData(UsageData baseline, UsageData used) { // Remove prebuilts from the used data if the baseline says to ignore them. Do this // first, so the new generated baseline doesn't list usages that are ignored by a // pattern anyway. ApplyBaselineIgnorePatterns(used, baseline); // Find new, removed, and unchanged usage after filtering patterns. Comparison <PackageIdentity> diff = Compare( used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(), baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct()); var report = new XElement("BaselineComparison"); bool tellUserToUpdateBaseline = false; if (diff.Added.Any()) { tellUserToUpdateBaseline = true; Log.LogError( $"{diff.Added.Length} new packages used not in baseline! See report " + $"at {OutputReportFile} for more information. Package IDs are:\n" + string.Join("\n", diff.Added.Select(u => u.ToString()))); // In the report, list full usage info, not only identity. report.Add( new XElement( "New", used.Usages .Where(u => diff.Added.Contains(u.GetIdentityWithoutRid())) .Select(u => u.ToXml()))); } if (diff.Removed.Any()) { tellUserToUpdateBaseline = true; Log.LogMessage( MessageImportance.High, $"{diff.Removed.Length} packages in baseline weren't used!"); report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement()))); } if (diff.Unchanged.Any()) { Log.LogMessage( MessageImportance.High, $"{diff.Unchanged.Length} packages used as expected in the baseline."); } if (!AllowTestProjectUsage) { Usage[] testProjectUsages = used.Usages .Where(WriteUsageReports.IsTestUsageByHeuristic) .ToArray(); if (testProjectUsages.Any()) { string[] projects = testProjectUsages .Select(u => u.AssetsFile) .Distinct() .ToArray(); Log.LogError( $"{testProjectUsages.Length} forbidden test usages found in " + $"{projects.Length} projects:\n" + string.Join("\n", projects)); } } // Simplify the used data to what is necessary for a baseline, to reduce file size. foreach (var usage in used.Usages) { usage.AssetsFile = null; } used.ProjectDirectories = null; used.Usages = used.Usages.Distinct().ToArray(); if (tellUserToUpdateBaseline) { string baselineNotFoundWarning = ""; if (string.IsNullOrEmpty(BaselineDataFile)) { baselineNotFoundWarning = $"not expected, because no baseline file exists at '{BaselineDataUpdateHintFile}'"; } else { baselineNotFoundWarning = $"different from the baseline found at '{BaselineDataFile}'"; } Log.LogMessage( MessageImportance.High, $"Prebuilt usages are {baselineNotFoundWarning}. If it's acceptable to " + "update the baseline, copy the contents of the automatically generated " + $"baseline '{OutputBaselineFile}'."); } return(new UsageValidationData { Report = report, ActualUsageData = used }); }
public override bool Execute() { UsageData data = JObject.Parse(File.ReadAllText(DataFile)).ToObject <UsageData>(); IEnumerable <RepoOutput> sourceBuildRepoOutputs = GetSourceBuildRepoOutputs(); // Map package id to the build name that created them in a ProdCon build. var prodConPackageOrigin = new Dictionary <string, string>( StringComparer.OrdinalIgnoreCase); foreach (ITaskItem item in ProdConPackageInfos ?? Enumerable.Empty <ITaskItem>()) { AddProdConPackage( prodConPackageOrigin, item.GetMetadata("PackageId"), item.GetMetadata("OriginBuildName")); } if (File.Exists(ProdConBuildManifestFile)) { var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile)); foreach (var x in xml.Descendants("Package")) { AddProdConPackage( prodConPackageOrigin, x.Attribute("Id")?.Value, x.Attribute("OriginBuildName")?.Value); } } PreviousReleasePrebuilt[] previousReleasePrebuilts = PreviousReleasePrebuiltPackageInfos ?.Select(item => new PreviousReleasePrebuilt { Id = item.GetMetadata("PackageId"), Version = item.GetMetadata("PackageVersion"), Release = item.GetMetadata("Release") }) .ToArray(); var poisonNupkgFilenames = new HashSet <string>(StringComparer.OrdinalIgnoreCase); if (File.Exists(PoisonedReportFile)) { foreach (string line in File.ReadAllLines(PoisonedReportFile)) { string[] segments = line.Split('\\'); if (segments.Length > 2 && segments[0].Trim() == "intermediate" && segments[1].EndsWith(".nupkg")) { poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1])); } } } foreach (Usage usage in data.Usages) { string[] identity = usage.PackageIdentity.Split('/'); if (identity.Length != 2) { Log.LogWarning( $"Data file contains invalid package identity '{usage.PackageIdentity}'. " + "Expected 2 segments when split by '/'."); continue; } string id = identity[0]; string version = identity[1]; string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); var sourceBuildCreator = new StringBuilder(); foreach (RepoOutput output in sourceBuildRepoOutputs) { foreach (PackageVersionPropsElement p in output.Built) { if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase)) { if (sourceBuildCreator.Length != 0) { sourceBuildCreator.Append(" "); } sourceBuildCreator.Append(output.Repo); sourceBuildCreator.Append(" "); sourceBuildCreator.Append(p.Name); sourceBuildCreator.Append("/"); sourceBuildCreator.Append(p.Version); } } } prodConPackageOrigin.TryGetValue(id, out string prodConCreator); string previousReleasePrebuilt = previousReleasePrebuilts ?.FirstOrDefault(p => p.Matches(id, version)) ?.Release; usage.Annotation = new UsageAnnotation { SourceBuildPackageIdCreator = sourceBuildCreator.ToString(), ProdConPackageIdCreator = prodConCreator, InEarlierReleaseExactPrebuilt = previousReleasePrebuilt, EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}") }; } Directory.CreateDirectory(OutputDirectory); File.WriteAllText( Path.Combine(OutputDirectory, "UsageSummary.xml"), data.CreateSummaryReport().ToString()); File.WriteAllText( Path.Combine(OutputDirectory, "UsageDetails.xml"), data.CreateDetailedReport().ToString()); return(!Log.HasLoggedErrors); }
public override bool Execute() { UsageData data = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); IEnumerable <RepoOutput> sourceBuildRepoOutputs = GetSourceBuildRepoOutputs(); // Map package id to the build name that created them in a ProdCon build. var prodConPackageOrigin = new Dictionary <string, string>( StringComparer.OrdinalIgnoreCase); foreach (ITaskItem item in ProdConPackageInfos.NullAsEmpty()) { AddProdConPackage( prodConPackageOrigin, item.GetMetadata("PackageId"), item.GetMetadata("OriginBuildName")); } if (File.Exists(ProdConBuildManifestFile)) { var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile)); foreach (var x in xml.Descendants("Package")) { AddProdConPackage( prodConPackageOrigin, x.Attribute("Id")?.Value, x.Attribute("OriginBuildName")?.Value); } } var poisonNupkgFilenames = new HashSet <string>(StringComparer.OrdinalIgnoreCase); if (File.Exists(PoisonedReportFile)) { foreach (string line in File.ReadAllLines(PoisonedReportFile)) { string[] segments = line.Split('\\'); if (segments.Length > 2 && segments[0].Trim() == "intermediate" && segments[1].EndsWith(".nupkg")) { poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1])); } } } var report = new XElement("AnnotatedUsages"); foreach (Usage usage in data.Usages.NullAsEmpty()) { string id = usage.PackageIdentity.Id; string version = usage.PackageIdentity.Version.OriginalVersion; string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); var sourceBuildCreator = new StringBuilder(); foreach (RepoOutput output in sourceBuildRepoOutputs) { foreach (PackageVersionPropsElement p in output.Built) { if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase)) { if (sourceBuildCreator.Length != 0) { sourceBuildCreator.Append(" "); } sourceBuildCreator.Append(output.Repo); sourceBuildCreator.Append(" "); sourceBuildCreator.Append(p.Name); sourceBuildCreator.Append("/"); sourceBuildCreator.Append(p.Version); } } } prodConPackageOrigin.TryGetValue(id, out string prodConCreator); var annotated = new AnnotatedUsage { Usage = usage, Project = data.ProjectDirectories ?.FirstOrDefault(p => usage.AssetsFile?.StartsWith(p) ?? false), SourceBuildPackageIdCreator = sourceBuildCreator.Length == 0 ? null : sourceBuildCreator.ToString(), ProdConPackageIdCreator = prodConCreator, EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}") }; report.Add(annotated.ToXml()); } Directory.CreateDirectory(OutputDirectory); File.WriteAllText( Path.Combine(OutputDirectory, "annotated-usage.xml"), report.ToString()); return(!Log.HasLoggedErrors); }
public override bool Execute() { DateTime startTime = DateTime.Now; Log.LogMessage(MessageImportance.High, "Writing package usage data..."); string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty() .Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal)) .ToArray(); if (projectDirectoriesOutsideRoot.Any()) { throw new ArgumentException( $"All ProjectDirectories must be in RootDir '{RootDir}', but found " + string.Join(", ", projectDirectoriesOutsideRoot)); } Log.LogMessage(MessageImportance.Low, "Finding set of RIDs..."); string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty() .SelectMany(ReadRidsFromRuntimeJson) .Distinct() .ToArray(); Log.LogMessage(MessageImportance.Low, "Reading package identities..."); PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); PackageIdentity[] referencePackages = ReferencePackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); IEnumerable <PackageIdentity> prebuilt = restored.Except(sourceBuilt).Except(referencePackages); PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty() .Select(item => new PackageIdentity( item.GetMetadata("PackageId"), NuGetVersion.Parse(item.GetMetadata("PackageVersion")))) .Concat(prebuilt) .ToArray(); Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files..."); string[] assetFiles = Directory .GetFiles(RootDir, "project.assets.json", SearchOption.AllDirectories) .Select(GetPathRelativeToRoot) .Except(IgnoredProjectAssetsJsonFiles.NullAsEmpty().Select(GetPathRelativeToRoot)) .ToArray(); if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile)) { Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files..."); Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile)); using (var projectAssetArchive = new ZipArchive( File.Open( ProjectAssetsJsonArchiveFile, FileMode.Create, FileAccess.ReadWrite), ZipArchiveMode.Create)) { // Only one entry can be open at a time, so don't do this during the Parallel // ForEach later. foreach (var relativePath in assetFiles) { using (var stream = File.OpenRead(Path.Combine(RootDir, relativePath))) using (Stream entryWriter = projectAssetArchive .CreateEntry(relativePath, CompressionLevel.Optimal) .Open()) { stream.CopyTo(entryWriter); } } } } Log.LogMessage(MessageImportance.Low, "Reading usage info..."); var usages = new ConcurrentBag <Usage>(); Parallel.ForEach( assetFiles, assetFile => { JObject jObj; using (var file = File.OpenRead(Path.Combine(RootDir, assetFile))) using (var reader = new StreamReader(file)) using (var jsonReader = new JsonTextReader(reader)) { jObj = (JObject)JToken.ReadFrom(jsonReader); } var properties = new HashSet <string>( jObj.SelectTokens("$.targets.*").Children() .Concat(jObj.SelectToken("$.libraries")) .Select(t => ((JProperty)t).Name) .Distinct(), StringComparer.OrdinalIgnoreCase); var directDependencies = jObj.SelectTokens("$.project.frameworks.*.dependencies").Children().Select(dep => new { name = ((JProperty)dep).Name, target = dep.SelectToken("$..target")?.ToString(), version = VersionRange.Parse(dep.SelectToken("$..version")?.ToString()), autoReferenced = dep.SelectToken("$..autoReferenced")?.ToString() == "True", }) .ToArray(); foreach (var identity in toCheck .Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion))) { var directDependency = directDependencies?.FirstOrDefault( d => d.name == identity.Id && d.version.Satisfies(identity.Version)); usages.Add(Usage.Create( assetFile, identity, directDependency != null, directDependency?.autoReferenced == true, possibleRids)); } }); Log.LogMessage(MessageImportance.Low, "Searching for unused packages..."); foreach (PackageIdentity restoredWithoutUsagesFound in toCheck.Except(usages.Select(u => u.PackageIdentity))) { usages.Add(Usage.Create( null, restoredWithoutUsagesFound, false, false, possibleRids)); } // Packages that were included in the tarball as prebuilts, but weren't even restored. PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt .Except(restored) .ToArray(); Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'..."); var data = new UsageData { CreatedByRid = TargetRid, Usages = usages.ToArray(), NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts, ProjectDirectories = ProjectDirectories ?.Select(GetPathRelativeToRoot) .ToArray() }; Directory.CreateDirectory(Path.GetDirectoryName(DataFile)); File.WriteAllText(DataFile, data.ToXml().ToString()); Log.LogMessage( MessageImportance.High, $"Writing package usage data... done. Took {DateTime.Now - startTime}"); return(!Log.HasLoggedErrors); }
public override bool Execute() { string[] packageIdentities = NuGetPackageInfos .Select(item => $"{item.GetMetadata("PackageId")}/{item.GetMetadata("PackageVersion")}") .ToArray(); // Remove trailing slash from dir to avoid EnumerateFiles method adding slash twice, // which would cause multiple unique paths per file and break the hash set. ProjectDirectories = ProjectDirectories .Where(Directory.Exists) .Select(dir => dir.TrimEnd('/', '\\')) .ToArray(); Log.LogMessage(MessageImportance.High, "Finding project.assets.json files..."); Dictionary <string, IEnumerable <string> > projectDirAssetFiles = ProjectDirectories .ToDictionary( dir => dir, dir => Directory.EnumerateFiles( dir, "project.assets.json", SearchOption.AllDirectories)); Log.LogMessage(MessageImportance.High, "Reading usage info..."); var assetFileUsages = new ConcurrentDictionary <string, Usage[]>(); Parallel.ForEach( projectDirAssetFiles.Values.SelectMany(v => v).Distinct(), assetsFile => { string contents = File.ReadAllText(assetsFile); assetFileUsages.TryAdd( assetsFile, packageIdentities .Where(id => IsIdentityInText(id, contents)) .Select(id => new Usage { AssetsFile = assetsFile, PackageIdentity = id }) .ToArray()); }); Log.LogMessage(MessageImportance.High, "Associating usage info with projects..."); var usedAssetFiles = new HashSet <string>(); var usages = new List <Usage>(); foreach (string dir in ProjectDirectories) { foreach (string assetsFile in projectDirAssetFiles[dir]) { if (usedAssetFiles.Add(assetsFile)) { foreach (var usage in assetFileUsages[assetsFile]) { usage.ProjectDirectory = dir; usages.Add(usage); } } } } Log.LogMessage(MessageImportance.High, $"Writing data to '{DataFile}'..."); Directory.CreateDirectory(Path.GetDirectoryName(DataFile)); string[] unused = packageIdentities .Where(id => !usages.Any(u => IsIdentityEqual(id, u.PackageIdentity))) .ToArray(); var data = new UsageData { UnusedPackages = unused, Usages = usages.ToArray() }; File.WriteAllText(DataFile, JsonConvert.SerializeObject(data, Formatting.Indented)); return(!Log.HasLoggedErrors); }
public override bool Execute() { DateTime startTime = DateTime.Now; Log.LogMessage(MessageImportance.High, "Writing package usage data..."); string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty() .Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal)) .ToArray(); if (projectDirectoriesOutsideRoot.Any()) { throw new ArgumentException( $"All ProjectDirectories must be in RootDir '{RootDir}', but found " + string.Join(", ", projectDirectoriesOutsideRoot)); } Log.LogMessage(MessageImportance.Low, "Finding set of RIDs..."); string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty() .SelectMany(ReadRidsFromRuntimeJson) .Distinct() .ToArray(); Log.LogMessage(MessageImportance.Low, "Reading package identities..."); PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty() .Select(ReadNuGetPackageInfos.ReadIdentity) .Distinct() .ToArray(); IEnumerable <PackageIdentity> prebuilt = restored.Except(sourceBuilt); PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty() .Select(item => new PackageIdentity( item.GetMetadata("PackageId"), NuGetVersion.Parse(item.GetMetadata("PackageVersion")))) .Concat(prebuilt) .ToArray(); Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files..."); string[] assetFiles = Directory.GetFiles( RootDir, "project.assets.json", SearchOption.AllDirectories); if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile)) { Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files..."); Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile)); using (var projectAssetArchive = new ZipArchive( File.Open( ProjectAssetsJsonArchiveFile, FileMode.Create, FileAccess.ReadWrite), ZipArchiveMode.Create)) { // Only one entry can be open at a time, so don't do this during the Parallel // ForEach later. foreach (var file in assetFiles) { string relativePath = file.Substring(RootDir.Length); using (var stream = File.OpenRead(file)) using (Stream entryWriter = projectAssetArchive .CreateEntry(relativePath, CompressionLevel.Optimal) .Open()) { stream.CopyTo(entryWriter); } } } } Log.LogMessage(MessageImportance.Low, "Reading usage info..."); var usages = new ConcurrentBag <Usage>(); Parallel.ForEach( assetFiles, assetFile => { var properties = new HashSet <string>(StringComparer.OrdinalIgnoreCase); using (var file = File.OpenRead(assetFile)) using (var reader = new StreamReader(file)) using (var jsonReader = new JsonTextReader(reader)) { while (jsonReader.Read()) { if (jsonReader.TokenType == JsonToken.PropertyName && jsonReader.Value is string value) { properties.Add(value); } } } foreach (var identity in toCheck .Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion))) { usages.Add(Usage.Create( // Store relative path for future report generation. assetFile.Substring(RootDir.Length), identity, possibleRids)); } }); Log.LogMessage(MessageImportance.Low, "Searching for unused packages..."); foreach (PackageIdentity restoredWithoutUsagesFound in toCheck.Except(usages.Select(u => u.PackageIdentity))) { usages.Add(Usage.Create( null, restoredWithoutUsagesFound, possibleRids)); } // Packages that were included in the tarball as prebuilts, but weren't even restored. PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt .Except(restored) .ToArray(); Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'..."); var data = new UsageData { CreatedByRid = TargetRid, Usages = usages.ToArray(), NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts, ProjectDirectories = ProjectDirectories ?.Select(dir => dir.Substring(RootDir.Length)) .ToArray() }; Directory.CreateDirectory(Path.GetDirectoryName(DataFile)); File.WriteAllText(DataFile, data.ToXml().ToString()); Log.LogMessage( MessageImportance.High, $"Writing package usage data... done. Took {DateTime.Now - startTime}"); return(!Log.HasLoggedErrors); }
public override bool Execute() { var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); string baselineText = string.IsNullOrEmpty(BaselineDataFile) ? "<UsageData />" : File.ReadAllText(BaselineDataFile); var baseline = UsageData.Parse(XElement.Parse(baselineText)); Comparison <PackageIdentity> diff = Compare( used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(), baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct()); var report = new XElement("BaselineComparison"); bool tellUserToUpdateBaseline = false; if (diff.Added.Any()) { tellUserToUpdateBaseline = true; Log.LogError( $"{diff.Added.Length} new packages used not in baseline! See report " + $"at {OutputReportFile} for more information. Package IDs are:\n" + string.Join("\n", diff.Added.Select(u => u.ToString()))); // In the report, list full usage info, not only identity. report.Add( new XElement( "New", used.Usages .Where(u => diff.Added.Contains(u.GetIdentityWithoutRid())) .Select(u => u.ToXml()))); } if (diff.Removed.Any()) { tellUserToUpdateBaseline = true; Log.LogMessage( MessageImportance.High, $"{diff.Removed.Length} packages in baseline weren't used!"); report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement()))); } if (diff.Unchanged.Any()) { Log.LogMessage( MessageImportance.High, $"{diff.Unchanged.Length} packages used as expected in the baseline."); } if (!AllowTestProjectUsage) { Usage[] testProjectUsages = used.Usages .Where(WriteUsageReports.IsTestUsageByHeuristic) .ToArray(); if (testProjectUsages.Any()) { string[] projects = testProjectUsages .Select(u => u.AssetsFile) .Distinct() .ToArray(); Log.LogError( $"{testProjectUsages.Length} forbidden test usages found in " + $"{projects.Length} projects:\n" + string.Join("\n", projects)); } } // Simplify the used data to what is necessary for a baseline, to reduce file size. foreach (var usage in used.Usages) { usage.AssetsFile = null; } used.Usages = used.Usages.Distinct().ToArray(); Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile)); File.WriteAllText(OutputBaselineFile, used.ToXml().ToString()); Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile)); File.WriteAllText(OutputReportFile, report.ToString()); if (tellUserToUpdateBaseline) { string baselineNotFoundWarning = ""; if (string.IsNullOrEmpty(BaselineDataFile)) { baselineNotFoundWarning = $"not expected, because no baseline file exists at '{BaselineDataUpdateHintFile}'"; } else { baselineNotFoundWarning = $"different from the baseline found at '{BaselineDataFile}'"; } Log.LogWarning( $"Prebuilt usages are {baselineNotFoundWarning}. If it's acceptable to " + "update the baseline, copy the contents of the automatically generated " + $"baseline '{OutputBaselineFile}'."); } return(!Log.HasLoggedErrors); }