/// <summary>
/// Copies the idx files from storage to the local machine. This is to prep the local machine before running the ardb merger.
/// </summary>
/// <param name="packagesWithDownloadCounts">The list of packages to copy from storage.</param>
/// <param name="storage">The storage object that can access the idx files.</param>
/// <param name="indexerVersion">The indexer version used to create the idx files.</param>
/// <param name="outputDirectory">The directory to copy the idx files to.</param>
/// <returns>The local paths to the idx files. This will be used by the ardb merger to identify which packages to include.</returns>
IEnumerable<string> StageIdxFiles(IList<Tuple<RegistrationIndexPackage, long>> packagesWithDownloadCounts, IStorage storage, Version indexerVersion, string outputDirectory)
{
List<string> localIdxFileList = new List<string>();
Directory.CreateDirectory(outputDirectory);
// These are the list of packages that are required to be included in the ardb file.
Dictionary<string, bool> requiredPackagesState = new Dictionary<string, bool>();
IEnumerable<string> requiredPackages = Catalog2ElfieOptions.RequiredPackages;
if (requiredPackages == null || requiredPackages.Count() == 0)
{
Trace.TraceInformation("The configuration file does not define any required packages to verify.");
}
else
{
foreach (string part in requiredPackages)
{
requiredPackagesState[part.ToLowerInvariant()] = false;
}
}
foreach (Tuple<RegistrationIndexPackage, long> packageWithDownloadCount in packagesWithDownloadCounts)
{
// This is the URL of the idx file in storage. i.e. this is the source path
Uri idxResourceUri = storage.ComposeIdxResourceUrl(indexerVersion, packageWithDownloadCount.Item1.CatalogEntry.PackageId, packageWithDownloadCount.Item1.CatalogEntry.PackageVersion);
using (StorageContent idxContent = storage.Load(idxResourceUri, new CancellationToken()).Result)
{
if (idxContent != null)
{
// This is the path to the local file. i.e. this is the destination path
string localFilePath = Path.Combine(outputDirectory, Path.GetFileName(idxResourceUri.LocalPath));
localIdxFileList.Add(localFilePath);
// Copy the file from storage to the local path
using (Stream idxStream = idxContent.GetContentStream())
{
using (BinaryReader reader = new BinaryReader(idxStream))
{
// Restamp the idx file with the current download counts.
PackageDatabase idxDatabase = new PackageDatabase();
idxDatabase.ReadBinary(reader);
idxDatabase.Identity.DownloadCount = (int)packageWithDownloadCount.Item2;
// Save the file with the updated download counts.
using (FileStream fileStream = File.OpenWrite(localFilePath))
{
using (BinaryWriter writer = new BinaryWriter(fileStream))
{
idxDatabase.WriteBinary(writer);
requiredPackagesState[packageWithDownloadCount.Item1.CatalogEntry.PackageId.ToLowerInvariant()] = true;
}
}
}
}
}
}
}
Trace.TraceInformation($"Copied {localIdxFileList.Count.ToString("#,###")} idx files from storage to {outputDirectory}");
// Validate that the we have the idx files for the required packages.
if (requiredPackages != null && requiredPackages.Count() > 0)
{
Trace.TraceInformation($"Verify the {requiredPackages.Count()} required packages are in the ardb.");
}
IEnumerable<string> missingPackages = requiredPackagesState.Where(item => item.Value == false).Select(item => item.Key);
if (missingPackages.Count() > 0)
{
throw new InvalidOperationException($"The following required packages do not have idx files: {String.Join(";", missingPackages)}");
}
// Basic validation, just check that the package counts are about the right number.
int minimumPackageCount = Catalog2ElfieOptions.MinimumPackageCountInArdb;
SarifTraceListener.TraceInformation($"Verify the ardb package count {localIdxFileList.Count} > {minimumPackageCount}.");
SarifTraceListener.TraceInformation("NG913", localIdxFileList.Count.ToString());
if (localIdxFileList.Count < minimumPackageCount)
{
throw new InvalidOperationException($"The number of idx files to include in the ardb is less than the minimum set of packages. {localIdxFileList.Count} < {minimumPackageCount}");
}
return localIdxFileList;
}