public async Task GenerateTemplateJsonReportAsync(string[] searchTerms, string jsonReportFilepath, List <string> specificPackagesToInclude, string previousReportPath) { Debug.Assert(searchTerms != null && searchTerms.Length > 0); Debug.Assert(!string.IsNullOrEmpty(jsonReportFilepath)); Dictionary <string, TemplatePack> previousPacks = new Dictionary <string, TemplatePack>(); Console.WriteLine($"verbose: '{previousReportPath}', file exists '{File.Exists(previousReportPath)}'"); if (!string.IsNullOrEmpty(previousReportPath) && File.Exists(previousReportPath)) { List <TemplatePack> previousReport = new List <TemplatePack>(); previousReport = JsonConvert.DeserializeObject <List <TemplatePack> >(File.ReadAllText(previousReportPath)); previousPacks = TemplatePack.ConvertToDictionary(previousReport); } Console.WriteLine($"num of previous template packs in previous report: '{previousPacks.Count}'"); // 1: query nuget for search results, we need to query all because we need to get the new download count var packageIdsToIgnore = GetPackagesToIgnore(); var nugetSearchResultPackages = await _nugetHelper.QueryNuGetAsync(_httpClient, searchTerms, specificPackagesToInclude, packageIdsToIgnore); // use below to write full list of search results to a file for debugging // System.IO.File.WriteAllText(@"c:\temp\packages-found.json",JsonConvert.SerializeObject(nugetSearchResultPackages)) var pkgsToDownload = new List <NuGetPackage>(); // go through each found package, if pkg is in previous result with same version number, update download count and move on // if not same version number, remove from dictionary and add to list to download foreach (var pkg in nugetSearchResultPackages) { var id = TemplatePack.NormalisePkgId(pkg.Id); if (previousPacks.ContainsKey(id)) { var previousPackage = previousPacks[id]; // check version number to see if it is the same if (string.Compare(pkg.Version, previousPackage.Version, StringComparison.OrdinalIgnoreCase) == 0) { // same version just update the download count previousPackage.DownloadCount = pkg.TotalDownloads; } else { // removing from previousPacks becuase version mismatch, so latest version should be downloaded previousPacks.Remove(id); pkgsToDownload.Add(pkg); } } else { pkgsToDownload.Add(pkg); } } // 2: download nuget packages locally var downloadedPackages = new List <NuGetPackage>(); // var pkgToRemoveAndExtractPathMap = new Dictionary<NuGetPackage, string>(); var pkgListSkippedExtractExists = new List <NuGetPackage>(); if (pkgsToDownload != null && pkgsToDownload.Count > 0) { // if the nuget pkg extract folder exists, don't download the package var pkgsToRemoveFromDownloadList = new List <NuGetPackage>(); // TODO: the code that skips if the extract folder is there should be refactored // probably into RemoteFile.cs somehow var rf = (RemoteFile)_remoteFile; foreach (var pkg in pkgsToDownload) { var key = TemplatePack.NormalisePkgId(pkg.Id); if (packageIdsToIgnore.Contains(key)) { pkgsToRemoveFromDownloadList.Add(pkg); _reporter.WriteVerboseLine($"verbose: ignoring pkg id '{pkg.Id}' because it's on the ignore list"); continue; } var filepath = rf.GetLocalFilepathFor(pkg.GetPackageFilename()); var filename = new System.IO.FileInfo(filepath).Name; var expectedExtractFolder = System.IO.Path.Combine(rf.CacheFolderpath, "extracted", filename); if (Directory.Exists(expectedExtractFolder)) { _reporter.WriteLine($"adding to exclude list because extract folder exists at '{expectedExtractFolder}'"); pkg.LocalExtractPath = expectedExtractFolder; pkgsToRemoveFromDownloadList.Add(pkg); pkgListSkippedExtractExists.Add(pkg); // pkgToRemoveAndExtractPathMap.Add(pkg, expectedExtractFolder); } } if (pkgsToRemoveFromDownloadList.Count > 0) { foreach (var pkg in pkgsToRemoveFromDownloadList) { pkgsToDownload.Remove(pkg); } } if (pkgsToDownload.Count > 0) { downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(pkgsToDownload); } else { _reporter.WriteVerboseLine("no packages found to download"); } } // 3: extract nuget package to local folder var templatePackages = new List <NuGetPackage>(); var listPackagesWithNotemplates = new List <NuGetPackage>(); var pkgNamesWitoutPackages = new List <string>(); foreach (var pkg in downloadedPackages) { var extractPath = string.Empty; try { extractPath = _remoteFile.ExtractZipLocally(pkg.LocalFilepath, false); } catch (Exception ex) { listPackagesWithNotemplates.Add(pkg); pkgNamesWitoutPackages.Add(pkg.Id); _reporter.WriteLine($"ERROR: {ex.ToString()}"); continue; } if (string.IsNullOrEmpty(extractPath)) { continue; } pkg.LocalExtractPath = extractPath; // see if there is a .template var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions { RecurseSubdirectories = true }); if (foundDirs.Count() > 0) { templatePackages.Add(pkg); } else { _reporter.WriteLine($"pkg has no templates: {pkg.Id}"); listPackagesWithNotemplates.Add(pkg); pkgNamesWitoutPackages.Add(pkg.Id); } } // look through the extract folders to see if other packages should be added to the exclude list foreach (var pkg in pkgListSkippedExtractExists) { string extractPath = pkg.LocalExtractPath; if (!string.IsNullOrEmpty(extractPath) && Directory.Exists(extractPath)) { var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions { RecurseSubdirectories = true }); if (foundDirs.Count() > 0) { templatePackages.Add(pkg); } else { _reporter.WriteLine($"pkg has no templates (from extract path): {pkg.Id}"); listPackagesWithNotemplates.Add(pkg); pkgNamesWitoutPackages.Add(pkg.Id); } } } var reportsPath = Path.Combine(_remoteFile.CacheFolderpath, "reports", DateTime.Now.ToString("MM.dd.yy-H.m.s.ffff")); if (!Directory.Exists(reportsPath)) { Directory.CreateDirectory(reportsPath); } var sb = new StringBuilder(); foreach (var pkg in packageIdsToIgnore) { sb.AppendLine(TemplatePack.NormalisePkgId(pkg)); } foreach (var pkg in pkgNamesWitoutPackages) { sb.AppendLine(TemplatePack.NormalisePkgId(pkg)); } File.WriteAllText( Path.Combine(reportsPath, "newly-found-packages-without-templates.json"), JsonConvert.SerializeObject(listPackagesWithNotemplates, Formatting.Indented)); File.WriteAllText( Path.Combine(reportsPath, "newly-found-package-names-to-ignore.json"), JsonConvert.SerializeObject(pkgNamesWitoutPackages, Formatting.Indented)); File.WriteAllText( Path.Combine(reportsPath, "packages-to-ignore.txt"), sb.ToString()); var ignoreFileOutputPath = Path.Combine(_remoteFile.CacheFolderpath, "packages-to-ignore.txt"); if (File.Exists(ignoreFileOutputPath)) { File.Delete(ignoreFileOutputPath); } File.WriteAllText( Path.Combine(_remoteFile.CacheFolderpath, "packages-to-ignore.txt"), sb.ToString()); var templatePacks = new List <TemplatePack>(); foreach (var pkg in templatePackages) { // get nuspec file path var nuspecFile = Directory.GetFiles(pkg.LocalExtractPath, $"{pkg.Id}.nuspec").FirstOrDefault(); if (nuspecFile == null) { _reporter.WriteLine($"warning: nuspec not found in folder {pkg.LocalFilepath}"); continue; } // get template folders var contentDir = Path.Combine(pkg.LocalExtractPath); if (!Directory.Exists(contentDir)) { continue; } var templateFiles = TemplatePack.GetTemplateFilesUnder(contentDir); try { var tp = TemplatePack.CreateFromNuSpec(pkg, nuspecFile, templateFiles); if (tp != null && tp.Templates != null && tp.Templates.Length > 0) { templatePacks.Add(tp); } else { _reporter.WriteLine($"Not adding package '{pkg.Id}', no templates found"); } } catch (Exception ex) { _reporter.WriteLine($"error creating template pack {nuspecFile} {ex.ToString()}"); } } // add all the downloaded items to existing dictionary then get the full result foreach (var pkg in templatePacks) { var id = TemplatePack.NormalisePkgId(pkg.Package); if (previousPacks.ContainsKey(id)) { // I believe it shouldn't get here, but just in case previousPacks.Remove(id); } previousPacks.Add(id, pkg); } templatePacks = previousPacks.Values.ToList(); templatePacks = templatePacks.OrderBy((tp) => - 1 * tp.DownloadCount).ToList(); // write to cache folder and then copy to dest var cacheFile = Path.Combine(reportsPath, "template-report.json"); File.WriteAllText(cacheFile, JsonConvert.SerializeObject(templatePacks, Formatting.Indented)); if (File.Exists(jsonReportFilepath)) { File.Delete(jsonReportFilepath); } _reporter.WriteLine($"Writing report to '{jsonReportFilepath}'"); File.Copy(cacheFile, jsonReportFilepath); }
public async Task GenerateTemplateJsonReportAsync(string[] searchTerms, string jsonReportFilepath, List <string> specificPackagesToInclude, string previousReportPath) { Debug.Assert(searchTerms != null && searchTerms.Length > 0); Debug.Assert(!string.IsNullOrEmpty(jsonReportFilepath)); Dictionary <string, TemplatePack> previousPacks = new Dictionary <string, TemplatePack>(); if (!string.IsNullOrEmpty(previousReportPath) && File.Exists(previousReportPath)) { List <TemplatePack> previousReport = new List <TemplatePack>(); previousReport = JsonConvert.DeserializeObject <List <TemplatePack> >(File.ReadAllText(previousReportPath)); previousPacks = TemplatePack.ConvertToDictionary(previousReport); } // 1: query nuget for search results, we need to query all because we need to get the new download count var foundPackages = await _nugetHelper.QueryNuGetAsync(_httpClient, searchTerms, specificPackagesToInclude, GetPackagesToIgnore()); var pkgsToDownload = new List <NuGetPackage>(); // go through each found package, if pkg is in previous result with same version number, update download count and move on // if not same version number, remove from dictionary and add to list to download foreach (var pkg in foundPackages) { var id = TemplatePack.NormalisePkgId(pkg.Id); if (previousPacks.ContainsKey(id)) { var previousPackage = previousPacks[id]; // check version number to see if it is the same if (string.Compare(pkg.Version, previousPackage.Version, StringComparison.OrdinalIgnoreCase) == 0) { // same version just update the download count previousPackage.DownloadCount = pkg.TotalDownloads; } else { previousPacks.Remove(id); pkgsToDownload.Add(pkg); } } else { pkgsToDownload.Add(pkg); } } // 2: download nuget packages locally // var downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(foundPackages); var downloadedPackages = await _nugetDownloader.DownloadAllPackagesAsync(pkgsToDownload); // 3: extract nuget package to local folder var templatePackages = new List <NuGetPackage>(); var listPackagesWithNotemplates = new List <NuGetPackage>(); var pkgNamesWitoutPackages = new List <string>(); foreach (var pkg in downloadedPackages) { var extractPath = _remoteFile.ExtractZipLocally(pkg.LocalFilepath); pkg.LocalExtractPath = extractPath; // see if there is a .template var foundDirs = Directory.EnumerateDirectories(extractPath, ".template.config", new EnumerationOptions { RecurseSubdirectories = true }); if (foundDirs.Count() > 0) { templatePackages.Add(pkg); } else { Console.WriteLine($"pkg has no templates: {pkg.Id}"); listPackagesWithNotemplates.Add(pkg); pkgNamesWitoutPackages.Add(pkg.Id); } } var reportsPath = Path.Combine(_remoteFile.CacheFolderpath, "reports", DateTime.Now.ToString("MM.dd.yy-H.m.s.ffff")); if (!Directory.Exists(reportsPath)) { Directory.CreateDirectory(reportsPath); } File.WriteAllText( Path.Combine(reportsPath, "newly-found-packages-without-templates.json"), JsonConvert.SerializeObject(listPackagesWithNotemplates, Formatting.Indented)); File.WriteAllText( Path.Combine(reportsPath, "newly-found-package-names-to-ignore.json"), JsonConvert.SerializeObject(pkgNamesWitoutPackages, Formatting.Indented)); var templatePacks = new List <TemplatePack>(); foreach (var pkg in templatePackages) { // get nuspec file path var nuspecFile = Directory.GetFiles(pkg.LocalExtractPath, $"{pkg.Id}.nuspec").FirstOrDefault(); if (nuspecFile == null) { Console.WriteLine($"warning: nuspec not found in folder {pkg.LocalFilepath}"); continue; } // get template folders var contentDir = Path.Combine(pkg.LocalExtractPath); if (!Directory.Exists(contentDir)) { continue; } var templateFiles = TemplatePack.GetTemplateFilesUnder(contentDir); try { var tp = TemplatePack.CreateFromNuSpec(pkg, nuspecFile, templateFiles); templatePacks.Add(tp); } catch (Exception ex) { Console.WriteLine($"error creating template pack {nuspecFile} {ex.ToString()}"); } } // add all the downloaded items to existing dictionary then get the full result foreach (var pkg in templatePacks) { var id = TemplatePack.NormalisePkgId(pkg.Package); if (previousPacks.ContainsKey(id)) { // I believe it shouldn't get here, but just in case previousPacks.Remove(id); } previousPacks.Add(id, pkg); } templatePacks = previousPacks.Values.ToList(); templatePacks = templatePacks.OrderBy((tp) => - 1 * tp.DownloadCount).ToList(); // write to cache folder and then copy to dest var cacheFile = Path.Combine(reportsPath, "template-report.json"); File.WriteAllText(cacheFile, JsonConvert.SerializeObject(templatePacks, Formatting.Indented)); if (File.Exists(jsonReportFilepath)) { File.Delete(jsonReportFilepath); } Console.WriteLine($"Writing report to '{jsonReportFilepath}'"); File.Copy(cacheFile, jsonReportFilepath); }