private static TemplateDiscoveryMetadata CreateSearchMetadata(PackSourceCheckResult packSourceCheckResults)
{
List <ITemplateInfo> templateCache = packSourceCheckResults.PackCheckData.Where(r => r.AnyTemplates)
.SelectMany(r => r.FoundTemplates)
.Distinct(new TemplateIdentityEqualityComparer())
.ToList();
Dictionary <string, PackToTemplateEntry> packToTemplateMap = packSourceCheckResults.PackCheckData
.Where(r => r.AnyTemplates)
.ToDictionary(
r => r.PackInfo.Id,
r =>
{
PackToTemplateEntry packToTemplateEntry = new PackToTemplateEntry(
r.PackInfo.Version,
r.FoundTemplates.Select(t => new TemplateIdentificationEntry(t.Identity, t.GroupIdentity)).ToList());
if (r.PackInfo is NugetPackInfo npi)
{
packToTemplateEntry.TotalDownloads = npi.TotalDownloads;
}
return(packToTemplateEntry);
});
Dictionary <string, object> additionalData = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
foreach (IAdditionalDataProducer dataProducer in packSourceCheckResults.AdditionalDataProducers)
{
additionalData[dataProducer.DataUniqueName] = dataProducer.Data;
}
return(new TemplateDiscoveryMetadata(TemplateInfo.CurrentVersion, templateCache, packToTemplateMap, additionalData));
}
private static void WriteSearchMetadata(PackSourceCheckResult packSourceCheckResults, string outputFileName)
{
TemplateSearchCache searchMetadata = packSourceCheckResults.SearchCache;
File.WriteAllText(outputFileName, searchMetadata.ToJObject().ToString(Formatting.None));
Console.WriteLine($"Search cache file created: {outputFileName}");
}
private static TemplateDiscoveryMetadata CreateLegacySearchMetadata(PackSourceCheckResult packSourceCheckResults)
{
List <ITemplateInfo> templateCache = packSourceCheckResults.SearchCache.TemplatePackages.SelectMany(p => p.Templates)
.Distinct(new TemplateIdentityEqualityComparer())
.Select(t => (ITemplateInfo) new LegacyBlobTemplateInfo(t))
.ToList();
Dictionary <string, PackToTemplateEntry> packToTemplateMap = packSourceCheckResults.SearchCache.TemplatePackages
.ToDictionary(
r => r.Name,
r =>
{
PackToTemplateEntry packToTemplateEntry = new PackToTemplateEntry(
r.Version ?? "",
r.Templates.Select(t => new TemplateIdentificationEntry(t.Identity, t.GroupIdentity)).ToList());
packToTemplateEntry.TotalDownloads = r.TotalDownloads;
packToTemplateEntry.Owners = r.Owners;
packToTemplateEntry.Verified = r.Verified;
return(packToTemplateEntry);
});
Dictionary <string, object> additionalData = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
foreach (IAdditionalDataProducer dataProducer in packSourceCheckResults.AdditionalDataProducers)
{
if (dataProducer.Data != null)
{
additionalData[dataProducer.DataUniqueName] = dataProducer.Data;
}
}
return(new TemplateDiscoveryMetadata("1.0.0.3", templateCache, packToTemplateMap, additionalData));
}
internal static string WriteLegacySearchMetadata(PackSourceCheckResult packSourceCheckResults, string outputFileName)
{
var searchMetadata = CreateLegacySearchMetadata(packSourceCheckResults);
File.WriteAllText(outputFileName, searchMetadata.ToJObject().ToString(Newtonsoft.Json.Formatting.None));
Console.WriteLine($"Legacy search cache file created: {outputFileName}");
return(outputFileName);
}
private static bool TryWriteSearchMetadata(PackSourceCheckResult packSourceCheckResults, string reportPath)
{
try
{
TemplateDiscoveryMetadata searchMetadata = CreateSearchMetadata(packSourceCheckResults);
JObject toSerialize = JObject.FromObject(searchMetadata);
string outputFileName = Path.Combine(reportPath, SearchMetadataFilename);
File.WriteAllText(outputFileName, toSerialize.ToString());
return(true);
}
catch
{
return(false);
}
}
public static bool TryWriteResults(string outputBasePath, PackSourceCheckResult packSourceCheckResults)
{
try
{
string reportPath = Path.Combine(outputBasePath, CacheContentDirectory);
if (!Directory.Exists(reportPath))
{
Directory.CreateDirectory(reportPath);
}
return(TryWriteSearchMetadata(packSourceCheckResults, reportPath) &&
TryWriteNonTemplatePackList(reportPath, packSourceCheckResults.PackCheckData));
}
catch
{
return(false);
}
}
private static async Task Main(string[] args)
{
// setup the config with defaults
ScraperConfig config = new ScraperConfig()
{
PageSize = _defaultPageSize,
RunOnlyOnePage = _defaultRunOnlyOnePage,
SaveCandidatePacks = _defaultSaveCandidatePacks,
IncludePreviewPacks = _defaultIncludePreviewPacks
};
if (!TryParseArgs(args, config) || string.IsNullOrEmpty(config.BasePath))
{
// base path is the only required arg.
ShowUsageMessage();
return;
}
PackSourceChecker packSourceChecker;
// if or when we add other sources to scrape, input args can control which execute(s).
if (true)
{
if (!NugetPackScraper.TryCreateDefaultNugetPackScraper(config, out packSourceChecker))
{
Console.WriteLine("Unable to create the NugetPackScraper.");
return;
}
}
else
{
throw new NotImplementedException("no checker for the input options");
}
PackSourceCheckResult checkResults = await packSourceChecker.CheckPackagesAsync().ConfigureAwait(false);
PackCheckResultReportWriter.TryWriteResults(config.BasePath, checkResults);
}
private static async Task <int> ExecuteAsync(CommandArgs config)
{
Verbose.IsEnabled = config.Verbose;
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (s, e) =>
{
Console.WriteLine("Canceling...");
cts.Cancel();
e.Cancel = true;
};
try
{
IPackCheckerFactory factory = config.LocalPackagePath == null ? new NuGetPackSourceCheckerFactory() : new TestPackCheckerFactory();
PackSourceChecker packSourceChecker = await factory.CreatePackSourceCheckerAsync(config, cts.Token).ConfigureAwait(false);
PackSourceCheckResult checkResults = await packSourceChecker.CheckPackagesAsync(cts.Token).ConfigureAwait(false);
(string metadataPath, string legacyMetadataPath) = PackCheckResultReportWriter.WriteResults(config.OutputPath, checkResults);
if (config.TestEnabled)
{
CacheFileTests.RunTests(config, metadataPath, legacyMetadataPath);
}
return(0);
}
catch (TaskCanceledException)
{
Console.WriteLine("Operation was cancelled.");
return(2);
}
catch (Exception e)
{
Console.WriteLine($"Error occured: {e}");
return(1);
}
}
internal static (string MetadataPath, string LegacyMetadataPath) WriteResults(DirectoryInfo outputBasePath, PackSourceCheckResult packSourceCheckResults)
{
string reportPath = Path.Combine(outputBasePath.FullName, CacheContentDirectory);
if (!Directory.Exists(reportPath))
{
Directory.CreateDirectory(reportPath);
Console.WriteLine($"Created directory:{reportPath}");
}
string legacyMetadataFilePath = Path.GetFullPath(Path.Combine(reportPath, SearchMetadataFilename));
string metadataFilePath = Path.GetFullPath(Path.Combine(reportPath, SearchMetadataFilenameVer2));
WriteNonTemplatePackList(reportPath, packSourceCheckResults.FilteredPackages);
#pragma warning disable CS0612 // Type or member is obsolete
LegacyMetadataWriter.WriteLegacySearchMetadata(packSourceCheckResults, legacyMetadataFilePath);
#pragma warning restore CS0612 // Type or member is obsolete
WriteSearchMetadata(packSourceCheckResults, metadataFilePath);
return(metadataFilePath, legacyMetadataFilePath);
}