public int Run(DumpOptions dumpOptions)
{
foreach (PackageDatabase db in BinarySerializableExtensions.LoadEach <PackageDatabase>(dumpOptions.IndexFilePath, true))
{
Console.Write(db.ToString());
}
return(0);
}
private static void CalculatePopularityAndSize(string databasePath)
{
using (new TraceWatch("Calculating statistics on '{0}'...", databasePath))
{
PopularityAndSizeReport report = new PopularityAndSizeReport();
// Load individual package databases
foreach (PackageDatabase db in BinarySerializableExtensions.LoadEach <PackageDatabase>(Directory.GetFiles(databasePath), true))
{
report.Add(db);
}
report.Write(databasePath + ".PopularityReport.csv");
}
}
private static void CountNonAsciiStrings(string databasePath)
{
int nonAsciiPackageDownloadCount = 0;
int packageCount = 0;
int stringCount = 0;
int packagesWithNonAscii = 0;
int stringsWithNonAscii = 0;
using (new TraceWatch("Counting Non-Ascii in '{0}'...", databasePath))
{
// Load individual package databases
foreach (PackageDatabase db in BinarySerializableExtensions.LoadEach <PackageDatabase>(Directory.GetFiles(databasePath), true))
{
bool isPackageAllAscii = true;
StringStore strings = ((IMemberDatabase)db).StringStore;
for (int i = 0; i < strings.Count; ++i)
{
if (strings[i].IsAscii() == false)
{
stringsWithNonAscii++;
isPackageAllAscii = false;
}
}
stringCount += strings.Count;
packageCount++;
if (!isPackageAllAscii)
{
packagesWithNonAscii++;
nonAsciiPackageDownloadCount += db.Identity.DownloadCount;
Trace.WriteLine(String.Format("Non-ASCII: {0} [{1} downloads]", db.Identity.PackageName, db.Identity.DownloadCount));
}
}
}
Console.WriteLine("{0:n0} / {1:n0} packages contain non-Ascii; {2:n0} / {3:n0} strings.", packagesWithNonAscii, packageCount, stringsWithNonAscii, stringCount);
Console.WriteLine("Total Non-Ascii Package Downloads: {0:n0}", nonAsciiPackageDownloadCount);
}
public static void Merge(MergerOptions options)
{
string individualLogs = options.PathToMerge + "Logs";
if (Directory.Exists(individualLogs))
{
Directory.Delete(individualLogs, true);
}
if (options.WithPackageOutcomes)
{
Directory.CreateDirectory(individualLogs);
}
using (new TraceWatch("Filtering '{0}' covering {1:p0} of downloads to Public Types Only...", options.PathToMerge, options.DownloadPercentage))
{
AddReferenceDatabase ardb = new AddReferenceDatabase(options.Version);
ardb.DatabaseVersion = options.DatabaseVersion;
// Determine indexes to include (up to popularity cutoff)
List <string> indexesToInclude = null;
if (Directory.Exists(options.PathToMerge))
{
indexesToInclude = IndexesByPopularityToCutoff(Directory.EnumerateFiles(options.PathToMerge, "*.idx", SearchOption.AllDirectories), options.DownloadPercentage);
if (indexesToInclude.Count == 0)
{
Trace.WriteLine("No indexes found to include. Stopping.");
return;
}
}
else if (File.Exists(options.PathToMerge))
{
// Text file listing IDS files passed
indexesToInclude = new List <string>(File.ReadAllLines(options.PathToMerge));
}
else
{
throw new ArgumentException(String.Format("Merge doesn't know how to crawl passed path, '{0}'.", options.PathToMerge));
}
HashSet <string> excludedPackageNames = ParsePackageNames(options.ExcludedPackageNames);
HashSet <string> filteringDisabledPackageNames = ParsePackageNames(options.DisableDuplicateFilteringPackageNames);
// Load individual package databases in approximate download count order (prefix of name is scale of download count)
ProgressWriter p = new ProgressWriter(indexesToInclude.Count);
foreach (PackageDatabase db in BinarySerializableExtensions.LoadEach <PackageDatabase>(indexesToInclude, true))
{
if (!String.IsNullOrEmpty(db.Identity.PackageName) && excludedPackageNames.Contains(db.Identity.PackageName))
{
Trace.WriteLine(String.Format("Excluded Package {0}", db.Identity.PackageName));
}
else
{
if (filteringDisabledPackageNames.Contains(db.Identity.PackageName))
{
ardb.AddReferenceAssemblyTypes(db);
}
else
{
DatabaseAddResult result = null;
ardb.AddUniqueMembers(db);
if (options.WithPackageOutcomes)
{
string log = Path.Combine(individualLogs, db.Identity.IndexFileName + ".log");
using (StreamWriter writer = new StreamWriter(log, false))
{
result.WriteMemberResults(writer);
}
}
}
}
p.IncrementProgress();
}
// Include the percentage included in the name
string outputFilePath = Path.Combine(options.OutputPath, Path.GetFileName(Path.GetFullPath(options.PathToMerge)) + "." + (options.DownloadPercentage * 100).ToString("g0"));
// Write the merged tree, if requested (debuggability)
if (options.WithMergedTreeLog)
{
string uniqueTreePath = options.PathToMerge + ".MergedTree.log";
using (new TraceWatch("Writing Unique Global Namespace tree to '{0}'...", uniqueTreePath))
{
using (StreamWriter writer = new StreamWriter(uniqueTreePath, false))
{
ardb.GetMergedMembers().WriteMergedTree(writer);
}
}
}
// Write the binary and text forms of the ARDB
using (new TraceWatch("Writing AddReferenceDatabase '{0}'...", outputFilePath))
{
ardb.FileWrite(outputFilePath + ".ardb");
CreateZip(outputFilePath + ".ardb");
Write.ToFile(ardb.WriteText, outputFilePath + ".ardb.txt");
CreateZip(outputFilePath + ".ardb.txt");
}
}
}
