public async Task GenerateTemplateJsonReportAsync(string[] searchTerms, string jsonReportFilepath)
{
Debug.Assert(searchTerms != null && searchTerms.Length > 0);
Debug.Assert(!string.IsNullOrEmpty(jsonReportFilepath));
// 1: query nuget for search results
var foundPackages = await _nugetHelper.QueryNuGetAsync(_httpClient, searchTerms, GetPackagesToIgnore());
// 2: download nuget packages locally
var downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(foundPackages);
// 3: extract nuget package to local folder
var templatePackages = new List <NuGetPackage>();
var listPackagesWithNotemplates = new List <NuGetPackage>();
var pkgNamesWitoutPackages = new List <string>();
foreach (var pkg in downloadedPackages)
{
var extractPath = _remoteFile.ExtractZipLocally(pkg.LocalFilepath);
pkg.LocalExtractPath = extractPath;
// see if there is a .template
var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions {
RecurseSubdirectories = true
});
if (foundDirs.Count() > 0)
{
templatePackages.Add(pkg);
}
else
{
Console.WriteLine($"pkg has no templates: {pkg.Id}");
listPackagesWithNotemplates.Add(pkg);
pkgNamesWitoutPackages.Add(pkg.Id);
}
}
var reportsPath = Path.Combine(_remoteFile.CacheFolderpath, "reports", DateTime.Now.ToString("MM.dd.yy-H.m.s.ffff"));
if (!Directory.Exists(reportsPath))
{
Directory.CreateDirectory(reportsPath);
}
Path.Combine(reportsPath, "packages-without-templates.json");
Path.Combine(reportsPath, "package-names-without-templates.txt");
var templatePacks = new List <TemplatePack>();
foreach (var pkg in templatePackages)
{
// get nuspec file path
var nuspecFile = Directory.GetFiles(pkg.LocalExtractPath, $"{pkg.Id}.nuspec").FirstOrDefault();
if (nuspecFile == null)
{
Console.WriteLine($"warning: nuspec not found in folder {pkg.LocalFilepath}");
continue;
}
// get template folders
var contentDir = Path.Combine(pkg.LocalExtractPath, "content");
if (!Directory.Exists(contentDir))
{
continue;
}
var templateFolders = Directory.GetDirectories(contentDir, ".template.config", SearchOption.AllDirectories);
var templateFiles = new List <string>();
foreach (var folder in templateFolders)
{
var files = Directory.GetFiles(folder, "template.json", new EnumerationOptions {
RecurseSubdirectories = true
});
if (files != null && files.Length > 0)
{
templateFiles.AddRange(files);
}
}
try {
var tp = TemplatePack.CreateFromNuSpec(pkg, nuspecFile, templateFiles);
templatePacks.Add(tp);
}
catch (Exception ex) {
Console.WriteLine($"error creating template pack {nuspecFile} {ex.ToString()}");
}
}
templatePacks = templatePacks.OrderBy((tp) => - 1 * tp.DownloadCount).ToList();
// write to cache folder and then copy to dest
var cacheFile = Path.Combine(reportsPath, "template-report.json");
File.WriteAllText(cacheFile, JsonConvert.SerializeObject(templatePacks));
if (File.Exists(jsonReportFilepath))
{
File.Delete(jsonReportFilepath);
}
File.Move(cacheFile, jsonReportFilepath);
}
public async Task GenerateTemplateJsonReportAsync(string[] searchTerms, string jsonReportFilepath, List <string> specificPackagesToInclude, string previousReportPath)
{
Debug.Assert(searchTerms != null && searchTerms.Length > 0);
Debug.Assert(!string.IsNullOrEmpty(jsonReportFilepath));
Dictionary <string, TemplatePack> previousPacks = new Dictionary <string, TemplatePack>();
Console.WriteLine($"verbose: '{previousReportPath}', file exists '{File.Exists(previousReportPath)}'");
if (!string.IsNullOrEmpty(previousReportPath) && File.Exists(previousReportPath))
{
List <TemplatePack> previousReport = new List <TemplatePack>();
previousReport = JsonConvert.DeserializeObject <List <TemplatePack> >(File.ReadAllText(previousReportPath));
previousPacks = TemplatePack.ConvertToDictionary(previousReport);
}
Console.WriteLine($"num of previous template packs in previous report: '{previousPacks.Count}'");
// 1: query nuget for search results, we need to query all because we need to get the new download count
var packageIdsToIgnore = GetPackagesToIgnore();
var nugetSearchResultPackages = await _nugetHelper.QueryNuGetAsync(_httpClient, searchTerms, specificPackagesToInclude, packageIdsToIgnore);
// use below to write full list of search results to a file for debugging
// System.IO.File.WriteAllText(@"c:\temp\packages-found.json",JsonConvert.SerializeObject(nugetSearchResultPackages))
var pkgsToDownload = new List <NuGetPackage>();
// go through each found package, if pkg is in previous result with same version number, update download count and move on
// if not same version number, remove from dictionary and add to list to download
foreach (var pkg in nugetSearchResultPackages)
{
var id = TemplatePack.NormalisePkgId(pkg.Id);
if (previousPacks.ContainsKey(id))
{
var previousPackage = previousPacks[id];
// check version number to see if it is the same
if (string.Compare(pkg.Version, previousPackage.Version, StringComparison.OrdinalIgnoreCase) == 0)
{
// same version just update the download count
previousPackage.DownloadCount = pkg.TotalDownloads;
}
else
{
// removing from previousPacks becuase version mismatch, so latest version should be downloaded
previousPacks.Remove(id);
pkgsToDownload.Add(pkg);
}
}
else
{
pkgsToDownload.Add(pkg);
}
}
// 2: download nuget packages locally
var downloadedPackages = new List <NuGetPackage>();
// var pkgToRemoveAndExtractPathMap = new Dictionary<NuGetPackage, string>();
var pkgListSkippedExtractExists = new List <NuGetPackage>();
if (pkgsToDownload != null && pkgsToDownload.Count > 0)
{
// if the nuget pkg extract folder exists, don't download the package
var pkgsToRemoveFromDownloadList = new List <NuGetPackage>();
// TODO: the code that skips if the extract folder is there should be refactored
// probably into RemoteFile.cs somehow
var rf = (RemoteFile)_remoteFile;
foreach (var pkg in pkgsToDownload)
{
var key = TemplatePack.NormalisePkgId(pkg.Id);
if (packageIdsToIgnore.Contains(key))
{
pkgsToRemoveFromDownloadList.Add(pkg);
_reporter.WriteVerboseLine($"verbose: ignoring pkg id '{pkg.Id}' because it's on the ignore list");
continue;
}
var filepath = rf.GetLocalFilepathFor(pkg.GetPackageFilename());
var filename = new System.IO.FileInfo(filepath).Name;
var expectedExtractFolder = System.IO.Path.Combine(rf.CacheFolderpath, "extracted", filename);
if (Directory.Exists(expectedExtractFolder))
{
_reporter.WriteLine($"adding to exclude list because extract folder exists at '{expectedExtractFolder}'");
pkg.LocalExtractPath = expectedExtractFolder;
pkgsToRemoveFromDownloadList.Add(pkg);
pkgListSkippedExtractExists.Add(pkg);
// pkgToRemoveAndExtractPathMap.Add(pkg, expectedExtractFolder);
}
}
if (pkgsToRemoveFromDownloadList.Count > 0)
{
foreach (var pkg in pkgsToRemoveFromDownloadList)
{
pkgsToDownload.Remove(pkg);
}
}
if (pkgsToDownload.Count > 0)
{
downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(pkgsToDownload);
}
else
{
_reporter.WriteVerboseLine("no packages found to download");
}
}
// 3: extract nuget package to local folder
var templatePackages = new List <NuGetPackage>();
var listPackagesWithNotemplates = new List <NuGetPackage>();
var pkgNamesWitoutPackages = new List <string>();
foreach (var pkg in downloadedPackages)
{
var extractPath = string.Empty;
try {
extractPath = _remoteFile.ExtractZipLocally(pkg.LocalFilepath, false);
}
catch (Exception ex) {
listPackagesWithNotemplates.Add(pkg);
pkgNamesWitoutPackages.Add(pkg.Id);
_reporter.WriteLine($"ERROR: {ex.ToString()}");
continue;
}
if (string.IsNullOrEmpty(extractPath))
{
continue;
}
pkg.LocalExtractPath = extractPath;
// see if there is a .template
var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions {
RecurseSubdirectories = true
});
if (foundDirs.Count() > 0)
{
templatePackages.Add(pkg);
}
else
{
_reporter.WriteLine($"pkg has no templates: {pkg.Id}");
listPackagesWithNotemplates.Add(pkg);
pkgNamesWitoutPackages.Add(pkg.Id);
}
}
// look through the extract folders to see if other packages should be added to the exclude list
foreach (var pkg in pkgListSkippedExtractExists)
{
string extractPath = pkg.LocalExtractPath;
if (!string.IsNullOrEmpty(extractPath) && Directory.Exists(extractPath))
{
var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions {
RecurseSubdirectories = true
});
if (foundDirs.Count() > 0)
{
templatePackages.Add(pkg);
}
else
{
_reporter.WriteLine($"pkg has no templates (from extract path): {pkg.Id}");
listPackagesWithNotemplates.Add(pkg);
pkgNamesWitoutPackages.Add(pkg.Id);
}
}
}
var reportsPath = Path.Combine(_remoteFile.CacheFolderpath, "reports", DateTime.Now.ToString("MM.dd.yy-H.m.s.ffff"));
if (!Directory.Exists(reportsPath))
{
Directory.CreateDirectory(reportsPath);
}
var sb = new StringBuilder();
foreach (var pkg in packageIdsToIgnore)
{
sb.AppendLine(TemplatePack.NormalisePkgId(pkg));
}
foreach (var pkg in pkgNamesWitoutPackages)
{
sb.AppendLine(TemplatePack.NormalisePkgId(pkg));
}
File.WriteAllText(
Path.Combine(reportsPath, "newly-found-packages-without-templates.json"),
JsonConvert.SerializeObject(listPackagesWithNotemplates, Formatting.Indented));
File.WriteAllText(
Path.Combine(reportsPath, "newly-found-package-names-to-ignore.json"),
JsonConvert.SerializeObject(pkgNamesWitoutPackages, Formatting.Indented));
File.WriteAllText(
Path.Combine(reportsPath, "packages-to-ignore.txt"),
sb.ToString());
var ignoreFileOutputPath = Path.Combine(_remoteFile.CacheFolderpath, "packages-to-ignore.txt");
if (File.Exists(ignoreFileOutputPath))
{
File.Delete(ignoreFileOutputPath);
}
File.WriteAllText(
Path.Combine(_remoteFile.CacheFolderpath, "packages-to-ignore.txt"),
sb.ToString());
var templatePacks = new List <TemplatePack>();
foreach (var pkg in templatePackages)
{
// get nuspec file path
var nuspecFile = Directory.GetFiles(pkg.LocalExtractPath, $"{pkg.Id}.nuspec").FirstOrDefault();
if (nuspecFile == null)
{
_reporter.WriteLine($"warning: nuspec not found in folder {pkg.LocalFilepath}");
continue;
}
// get template folders
var contentDir = Path.Combine(pkg.LocalExtractPath);
if (!Directory.Exists(contentDir))
{
continue;
}
var templateFiles = TemplatePack.GetTemplateFilesUnder(contentDir);
try {
var tp = TemplatePack.CreateFromNuSpec(pkg, nuspecFile, templateFiles);
if (tp != null && tp.Templates != null && tp.Templates.Length > 0)
{
templatePacks.Add(tp);
}
else
{
_reporter.WriteLine($"Not adding package '{pkg.Id}', no templates found");
}
}
catch (Exception ex) {
_reporter.WriteLine($"error creating template pack {nuspecFile} {ex.ToString()}");
}
}
// add all the downloaded items to existing dictionary then get the full result
foreach (var pkg in templatePacks)
{
var id = TemplatePack.NormalisePkgId(pkg.Package);
if (previousPacks.ContainsKey(id))
{
// I believe it shouldn't get here, but just in case
previousPacks.Remove(id);
}
previousPacks.Add(id, pkg);
}
templatePacks = previousPacks.Values.ToList();
templatePacks = templatePacks.OrderBy((tp) => - 1 * tp.DownloadCount).ToList();
// write to cache folder and then copy to dest
var cacheFile = Path.Combine(reportsPath, "template-report.json");
File.WriteAllText(cacheFile, JsonConvert.SerializeObject(templatePacks, Formatting.Indented));
if (File.Exists(jsonReportFilepath))
{
File.Delete(jsonReportFilepath);
}
_reporter.WriteLine($"Writing report to '{jsonReportFilepath}'");
File.Copy(cacheFile, jsonReportFilepath);
}
public async Task GenerateTemplateJsonReportAsync(string[] searchTerms, string jsonReportFilepath, List <string> specificPackagesToInclude, string previousReportPath)
{
Debug.Assert(searchTerms != null && searchTerms.Length > 0);
Debug.Assert(!string.IsNullOrEmpty(jsonReportFilepath));
Dictionary <string, TemplatePack> previousPacks = new Dictionary <string, TemplatePack>();
if (!string.IsNullOrEmpty(previousReportPath) && File.Exists(previousReportPath))
{
List <TemplatePack> previousReport = new List <TemplatePack>();
previousReport = JsonConvert.DeserializeObject <List <TemplatePack> >(File.ReadAllText(previousReportPath));
previousPacks = TemplatePack.ConvertToDictionary(previousReport);
}
// 1: query nuget for search results, we need to query all because we need to get the new download count
var foundPackages = await _nugetHelper.QueryNuGetAsync(_httpClient, searchTerms, specificPackagesToInclude, GetPackagesToIgnore());
var pkgsToDownload = new List <NuGetPackage>();
// go through each found package, if pkg is in previous result with same version number, update download count and move on
// if not same version number, remove from dictionary and add to list to download
foreach (var pkg in foundPackages)
{
var id = TemplatePack.NormalisePkgId(pkg.Id);
if (previousPacks.ContainsKey(id))
{
var previousPackage = previousPacks[id];
// check version number to see if it is the same
if (string.Compare(pkg.Version, previousPackage.Version, StringComparison.OrdinalIgnoreCase) == 0)
{
// same version just update the download count
previousPackage.DownloadCount = pkg.TotalDownloads;
}
else
{
previousPacks.Remove(id);
pkgsToDownload.Add(pkg);
}
}
else
{
pkgsToDownload.Add(pkg);
}
}
// 2: download nuget packages locally
// var downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(foundPackages);
var downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(pkgsToDownload);
// 3: extract nuget package to local folder
var templatePackages = new List <NuGetPackage>();
var listPackagesWithNotemplates = new List <NuGetPackage>();
var pkgNamesWitoutPackages = new List <string>();
foreach (var pkg in downloadedPackages)
{
var extractPath = _remoteFile.ExtractZipLocally(pkg.LocalFilepath);
pkg.LocalExtractPath = extractPath;
// see if there is a .template
var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions {
RecurseSubdirectories = true
});
if (foundDirs.Count() > 0)
{
templatePackages.Add(pkg);
}
else
{
Console.WriteLine($"pkg has no templates: {pkg.Id}");
listPackagesWithNotemplates.Add(pkg);
pkgNamesWitoutPackages.Add(pkg.Id);
}
}
var reportsPath = Path.Combine(_remoteFile.CacheFolderpath, "reports", DateTime.Now.ToString("MM.dd.yy-H.m.s.ffff"));
if (!Directory.Exists(reportsPath))
{
Directory.CreateDirectory(reportsPath);
}
File.WriteAllText(
Path.Combine(reportsPath, "newly-found-packages-without-templates.json"),
JsonConvert.SerializeObject(listPackagesWithNotemplates, Formatting.Indented));
File.WriteAllText(
Path.Combine(reportsPath, "newly-found-package-names-to-ignore.json"),
JsonConvert.SerializeObject(pkgNamesWitoutPackages, Formatting.Indented));
var templatePacks = new List <TemplatePack>();
foreach (var pkg in templatePackages)
{
// get nuspec file path
var nuspecFile = Directory.GetFiles(pkg.LocalExtractPath, $"{pkg.Id}.nuspec").FirstOrDefault();
if (nuspecFile == null)
{
Console.WriteLine($"warning: nuspec not found in folder {pkg.LocalFilepath}");
continue;
}
// get template folders
var contentDir = Path.Combine(pkg.LocalExtractPath);
if (!Directory.Exists(contentDir))
{
continue;
}
var templateFiles = TemplatePack.GetTemplateFilesUnder(contentDir);
try {
var tp = TemplatePack.CreateFromNuSpec(pkg, nuspecFile, templateFiles);
templatePacks.Add(tp);
}
catch (Exception ex) {
Console.WriteLine($"error creating template pack {nuspecFile} {ex.ToString()}");
}
}
// add all the downloaded items to existing dictionary then get the full result
foreach (var pkg in templatePacks)
{
var id = TemplatePack.NormalisePkgId(pkg.Package);
if (previousPacks.ContainsKey(id))
{
// I believe it shouldn't get here, but just in case
previousPacks.Remove(id);
}
previousPacks.Add(id, pkg);
}
templatePacks = previousPacks.Values.ToList();
templatePacks = templatePacks.OrderBy((tp) => - 1 * tp.DownloadCount).ToList();
// write to cache folder and then copy to dest
var cacheFile = Path.Combine(reportsPath, "template-report.json");
File.WriteAllText(cacheFile, JsonConvert.SerializeObject(templatePacks, Formatting.Indented));
if (File.Exists(jsonReportFilepath))
{
File.Delete(jsonReportFilepath);
}
Console.WriteLine($"Writing report to '{jsonReportFilepath}'");
File.Copy(cacheFile, jsonReportFilepath);
}
