private static void ComputeAndTestDeltas(string folderPath, string latestVersionPath)
{
using (new TraceWatch("Verifying latest file '{0}' loads...", latestVersionPath, folderPath))
{
AddReferenceDatabase latest = new AddReferenceDatabase(ArdbVersion.Current);
Read.FromFile(latest.ReadText, latestVersionPath);
}
byte[] latestBytes = File.ReadAllBytes(latestVersionPath);
using (new TraceWatch("Computing Deltas from '{0}' for all files in '{1}'...", latestVersionPath, folderPath))
{
foreach (string filePath in Directory.GetFiles(folderPath, "*.txt", SearchOption.AllDirectories))
{
if (String.Equals(filePath, latestVersionPath, StringComparison.OrdinalIgnoreCase))
{
continue;
}
Trace.WriteLine(filePath);
byte[] currentBytes = File.ReadAllBytes(filePath);
byte[] delta = null;
using (new TraceWatch("Computing ARDB text delta..."))
{
byte[] latestCopy = new byte[latestBytes.Length];
latestBytes.CopyTo(latestCopy, 0);
byte[] currentCopy = new byte[currentBytes.Length];
currentBytes.CopyTo(currentCopy, 0);
delta = Delta.CreatePatch(currentCopy, latestCopy);
Trace.WriteLine(String.Format("{0:n0}", ((long)delta.Length).SizeString()));
}
using (new TraceWatch("Validating Delta..."))
{
byte[] currentWithDelta = Delta.ApplyPatch(currentBytes, delta);
if (latestBytes.Length != currentWithDelta.Length)
{
throw new IOException(String.Format("Delta from '{0}' to '{1}' didn't reproduce latest. Wrong length. Aborting.", filePath, latestVersionPath));
}
for (int i = 0; i < latestBytes.Length; ++i)
{
if (latestBytes[i] != currentWithDelta[i])
{
throw new IOException(String.Format("Delta from '{0}' to '{1}' didn't reproduce latest; Aborting. Byte {2} wrong.\r\nExpect: {3}\r\nActual: {4}", filePath, latestVersionPath, i, latestBytes[i], currentWithDelta[i]));
}
}
}
using (new TraceWatch("Writing Delta..."))
{
File.WriteAllBytes(Path.ChangeExtension(filePath, ".Delta.bin"), delta);
}
}
}
}
public void AddReferenceDatabase_Prerelease()
{
// Verify unknown SymbolTypes are excluded ('!'),
// descendants of excluded elements are added under parent (Logger),
// and later siblings are correct (Extensions).
// We keep descendants rather than excluding them so that new types can be added higher in the tree (ex: Package Release, Assembly Hash Signature)
string sampleTree = @"Elfie V2
20160126
P SamplePackage
V 1.2.3-beta1
R 17
A Sample.dll
N Extensions
C IntExtensions
N Sample
C Logger
C TryLog
C Memory
C TraceWatch
".TrimStart().Replace(" ", "\t");
string expectedLoadResult = sampleTree;
AddReferenceDatabase db = new AddReferenceDatabase(ArdbVersion.Current);
db.ReadText(new StringReader(sampleTree));
string result = Write.ToString(db.WriteText);
Assert.AreEqual(expectedLoadResult, result);
}
private static void WriteArdbVariations(string folderPath)
{
foreach (string filePath in Directory.GetFiles(folderPath, "*.txt", SearchOption.AllDirectories))
{
AddReferenceDatabase ardb = new AddReferenceDatabase();
Read.FromFile(ardb.ReadText, filePath);
// Write only the Package Names [easy diff of which packages included]
using (StreamWriter writer = new StreamWriter(Path.ChangeExtension(filePath, "PackageOnly.txt"), false))
{
ardb.WriteText(writer, (s) => s.Type == SymbolType.Package);
}
// Write without extension methods [compare sizes]
using (StreamWriter writer = new StreamWriter(Path.ChangeExtension(filePath, "NoExtensionMethods.txt"), false))
{
ardb.WriteText(writer, (s) => s.Type != SymbolType.ExtensionMethod);
}
// Write without long names
using (StreamWriter writer = new StreamWriter(Path.ChangeExtension(filePath, "NoExtensionsOrLongNames.txt"), false))
{
ardb.WriteText(writer, (s) => s.Type != SymbolType.ExtensionMethod && (s.Type.IsAboveType() || s.Name.Length <= 40));
}
}
}
public void AddReferenceDatabase_BinaryWrongVersionLoading()
{
AddReferenceDatabase db = new AddReferenceDatabase(ArdbVersion.Current);
db.AddUniqueMembers(PackageDatabaseTests.BuildDefaultSample());
// Overwrite version with '9' and verify binary format won't load
Verify.RoundTrip(db, new AddReferenceDatabase(), (w) => w.Write(9));
}
public AddReferenceDatabase CreateDatabaseFromBytes(byte[] bytes)
{
using var memoryStream = new MemoryStream(bytes);
using var streamReader = new StreamReader(memoryStream);
var database = new AddReferenceDatabase(ArdbVersion.V1);
database.ReadText(streamReader);
return(database);
}
public AddReferenceDatabase CreateDatabaseFromBytes(byte[] bytes)
{
using (var memoryStream = new MemoryStream(bytes))
using (var streamReader = new StreamReader(memoryStream))
{
var database = new AddReferenceDatabase();
database.ReadText(streamReader);
return database;
}
}
public AddReferenceDatabase CreateDatabaseFromBytes(byte[] bytes)
{
using (var memoryStream = new MemoryStream(bytes))
using (var streamReader = new StreamReader(memoryStream))
{
var database = new AddReferenceDatabase();
database.ReadText(streamReader);
return(database);
}
}
private static void LoadSpeedTest(string sourcePath)
{
AddReferenceDatabase ardb = new AddReferenceDatabase();
int iterations = 100;
using (new TraceWatch("Loading '{0}' {1} times...", sourcePath, iterations))
{
for (int i = 0; i < iterations; ++i)
{
ardb.FileRead(sourcePath);
}
}
}
private static void ConvertAddReferenceDatabaseFromText(string textFilePath)
{
AddReferenceDatabase ardb = new AddReferenceDatabase();
using (new TraceWatch("Converting '{0}' from text to binary model...", textFilePath))
{
Read.FromFile(ardb.ReadText, textFilePath);
ardb.FileWrite(textFilePath.Substring(0, textFilePath.Length - 4));
}
int count = ardb.Count;
long bytes = ardb.Bytes;
Trace.WriteLine(String.Format("{0:n0} items in {1}, {2}/item.\r\n", count, bytes.SizeString(), (bytes / count).SizeString()));
}
private PackageWithTypeResult CreateResult(AddReferenceDatabase database, Symbol type)
{
var nameParts = ArrayBuilder<string>.GetInstance();
GetFullName(nameParts, type.FullName.Parent);
var packageName = type.PackageName.ToString();
var version = database.GetPackageVersion(type.Index).ToString();
return new PackageWithTypeResult(
packageName: packageName,
typeName: type.Name.ToString(),
version: version,
rank: GetRank(type),
containingNamespaceNames: nameParts.ToImmutableAndFree());
}
public void AddReferenceDatabase_PackagePrereleaseVersion()
{
AddReferenceDatabase ardb = new AddReferenceDatabase(ArdbVersion.Current);
// Build and add the sample PackageDatabase
PackageDatabase source = PackageDatabaseTests.BuildPreleaseSample();
ardb.AddUniqueMembers(source);
ardb.ConvertToImmutable();
MemberQuery query = new MemberQuery(PackageDatabaseTests.TYPE_LOGGER, false, false);
PartialArray <Symbol> results = new PartialArray <Symbol>(10);
query.TryFindMembers(ardb, ref results);
Assert.AreEqual(1, results.Count);
Assert.AreEqual(PackageDatabaseTests.PRERELEASE_VERSION, ardb.GetPackageVersion(results[0].Index).ToString());
}
private PackageWithTypeResult CreateResult(AddReferenceDatabase database, Symbol type)
{
var nameParts = new List <string>();
GetFullName(nameParts, type.FullName.Parent);
var packageName = type.PackageName.ToString();
var version = database.GetPackageVersion(type.Index).ToString();
return(new PackageWithTypeResult(
packageName: packageName,
typeName: type.Name.ToString(),
version: version,
containingNamespaceNames: nameParts));
}
public void AddReferenceDatabase_PackageReleaseVersion()
{
AddReferenceDatabase ardb = new AddReferenceDatabase(ArdbVersion.Current);
// Build and add the sample PackageDatabase
PackageDatabase source = PackageDatabaseTests.BuildDefaultSample();
ardb.AddUniqueMembers(source);
ardb.ConvertToImmutable();
MemberQuery query = new MemberQuery(PackageDatabaseTests.TYPE_LOGGER, false, false);
PartialArray <Symbol> results = new PartialArray <Symbol>(10);
query.TryFindMembers(ardb, ref results);
Assert.AreEqual(1, results.Count);
// We shouldn't persist non-prerelease version details in the ARDB
Assert.AreEqual(String.Empty, ardb.GetPackageVersion(results[0].Index).ToString());
}
private PackageWithTypeResult CreateResult(AddReferenceDatabase database, Symbol type)
{
var nameParts = new List <string>();
GetFullName(nameParts, type.FullName.Parent);
var packageName = type.PackageName.ToString();
var version = database.GetPackageVersion(type.Index).ToString();
return(new PackageWithTypeResult(
isDesktopFramework: packageName == MicrosoftAssemblyReferencesName,
packageName: packageName,
assemblyName: type.AssemblyNameWithoutExtension.ToString(),
typeName: type.Name.ToString(),
version: version,
containingNamespaceNames: nameParts));
}
public void AddReferenceDatabase_NewlineFlexibility()
{
PackageDatabase db = PackageDatabaseTests.BuildDefaultSample();
AddReferenceDatabase ardb = new AddReferenceDatabase(ArdbVersion.Current);
ardb.AddUniqueMembers(db);
string originalTextFormat = Write.ToString(ardb.WriteText);
// Ensure non-Windows newlines don't trip up ReadText
string modifiedNewlines = originalTextFormat.Replace("\r\n", "\n");
AddReferenceDatabase ardb2 = new AddReferenceDatabase(ArdbVersion.Current);
Read.FromString(ardb2.ReadText, modifiedNewlines);
// Verify everything in the ARDB round-tripped to validate read success
string roundTripped = Write.ToString(ardb2.WriteText);
Assert.AreEqual(originalTextFormat, roundTripped);
}
public void AddReferenceDatabase_DuplicateMerging()
{
int defaultArdbMemberCount, addedDefaultSampleDataMemberCount;
AddReferenceDatabase ardb = new AddReferenceDatabase(ArdbVersion.Current);
PackageDatabase db = PackageDatabaseTests.BuildDefaultSample();
defaultArdbMemberCount = ardb.DeclaredMembers.Count;
// Add the database once
ardb.AddUniqueMembers(db);
int memberCount = ardb.DeclaredMembers.Count;
addedDefaultSampleDataMemberCount = (ardb.DeclaredMembers.Count - defaultArdbMemberCount);
db = PackageDatabaseTests.BuildDefaultSample(Guid.NewGuid().ToString());
// Change only the package name. We should see no change
// in member count, since all package contents are duped.
memberCount = ardb.DeclaredMembers.Count;
ardb.AddUniqueMembers(db);
Assert.AreEqual(memberCount, ardb.DeclaredMembers.Count);
db = PackageDatabaseTests.BuildDefaultSample(Guid.NewGuid().ToString());
// Add allowing duplicates should add the same number of
// new members as fresh construction of the sample data
// + 2 (to account for a duplicated type and for
// an added assembly node).
memberCount = ardb.DeclaredMembers.Count + addedDefaultSampleDataMemberCount;
ardb.AddReferenceAssemblyTypes(db);
Assert.AreEqual(memberCount + 2, ardb.DeclaredMembers.Count);
// Write the ARDB tree [debuggability]
string result = Write.ToString(ardb.WriteText);
Trace.WriteLine(result);
}
private void VerifyQueryResults(AddReferenceDatabase ardb, ArdbVersion version)
{
// "Diagnostics."
MemberQuery query = new MemberQuery(PackageDatabaseTests.NS_DIAGNOSTICS + ".", false, false);
PartialArray <Symbol> results = new PartialArray <Symbol>(10);
query.TryFindMembers(ardb, ref results);
Assert.AreEqual(3, results.Count);
Assert.AreEqual("Logger, Memory, TraceWatch", PackageDatabaseTests.ResultNamesToString(results));
if (version == ArdbVersion.V1)
{
// V1 has no TFM data. This call also verifies that most current client
// can query older format without raising an exception.
Assert.AreEqual(String8.Empty, ardb.GetFrameworkTargets(results[0].Index));
return;
}
for (int i = 0; i < results.Count; i++)
{
Symbol symbol = results[i];
string fx = ardb.GetFrameworkTargets(symbol.Index).ToString();
if (symbol.Name.ToString() == "TraceWatch")
{
Assert.AreEqual(PackageDatabaseTests.NET20, fx);
}
else if (symbol.Name.ToString() == "Memory")
{
Assert.AreEqual(PackageDatabaseTests.NET35, fx);
}
else
{
Assert.AreEqual(@"<tfms><tfm>net20</tfm><tfm>net35</tfm></tfms>", fx);
}
}
}
private static void ReportEmptyPackages(AddReferenceDatabase ardb)
{
Console.WriteLine("Packages with few Types:");
Symbol root = ardb.QueryRoot;
for (Symbol package = root.FirstChild(); package.IsValid; package = package.NextSibling())
{
int typeCount = 0;
package.Walk((s) => { if (s.Type.IsType())
{
typeCount++;
}
});
if (typeCount < 5)
{
package.Name.WriteTo(Console.Out);
Console.WriteLine("\t" + typeCount.ToString());
}
}
Console.WriteLine("\r\nDone.");
}
private DatabaseAddResult CallAddUniqueMembers(AddReferenceDatabase ardb, PackageDatabase source)
{
return(ardb.AddUniqueMembers(source));
}
private PackageWithTypeResult CreateResult(AddReferenceDatabase database, Symbol type)
{
var nameParts = new List<string>();
GetFullName(nameParts, type.FullName.Parent);
var packageName = type.PackageName.ToString();
var version = database.GetPackageVersion(type.Index).ToString();
return new PackageWithTypeResult(
packageName: packageName,
typeName: type.Name.ToString(),
version: version,
rank: GetRank(type),
containingNamespaceNames: nameParts);
}
private void AddReferenceDatabaseBasicHelper(ArdbVersion version)
{
AddReferenceDatabase ardb = new AddReferenceDatabase(version);
DatabaseAddResult result;
// Build and add the sample PackageDatabase
PackageDatabase source = PackageDatabaseTests.BuildDefaultSample("V1");
result = CallAddUniqueMembers(ardb, source);
// Verify at least something was added
int ardbCountFirstAdd = ardb.Count;
Assert.IsTrue(result.WasMemberAdded[0].Value);
// Add the sample again; verify nothing was added
source = PackageDatabaseTests.BuildDefaultSample("V2");
result = CallAddUniqueMembers(ardb, source);
Assert.IsFalse(result.WasMemberAdded[0].Value);
Assert.AreEqual(ardbCountFirstAdd, ardb.Count);
// Add a namespace with a private class; verify nothing added
source = PackageDatabaseTests.BuildDefaultSample("V3");
MutableSymbol diagnostics = source.MutableRoot.FindByFullName(BuildDiagnosticsNamespaceFor(source.Identity.PackageName), PackageDatabaseTests.SEPARATOR_CHAR);
MutableSymbol internalNs = diagnostics.AddChild(new MutableSymbol("Internal", SymbolType.Namespace));
internalNs.AddChild(new MutableSymbol("Tracer", SymbolType.Class)
{
Modifiers = SymbolModifier.Internal
});
result = CallAddUniqueMembers(ardb, source);
Assert.IsFalse(result.WasMemberAdded[0].Value);
Assert.AreEqual(ardbCountFirstAdd, ardb.Count);
// Add a new public class (existing namespace); verify it is added
source = PackageDatabaseTests.BuildDefaultSample("V4");
diagnostics = source.MutableRoot.FindByFullName(BuildDiagnosticsNamespaceFor(source.Identity.PackageName), PackageDatabaseTests.SEPARATOR_CHAR);
diagnostics.AddChild(new MutableSymbol("TraceWatch", SymbolType.Class)
{
Modifiers = SymbolModifier.Public | SymbolModifier.Static
});
result = CallAddUniqueMembers(ardb, source);
Assert.IsTrue(result.WasMemberAdded[0].Value);
Assert.IsTrue(result.WasMemberAdded[result.WasMemberAdded.Length - 1].Value);
Assert.AreNotEqual(ardbCountFirstAdd, ardb.Count);
// Verify a query [expect Diagnostics. to match Logger, Memory, and TraceWatch
ardb.ConvertToImmutable();
VerifyQueryResults(ardb, version);
// Double-convert ARDB. Verify queries still work correctly.
ardb.ConvertToImmutable();
VerifyQueryResults(ardb, version);
// Round trip to string; verify query still right, count matches
string sampleArdbFilePath = "Sample.ardb.txt";
Write.ToFile(ardb.WriteText, sampleArdbFilePath);
AddReferenceDatabase reloaded = new AddReferenceDatabase(version);
Read.FromFile(reloaded.ReadText, sampleArdbFilePath);
VerifyQueryResults(reloaded, version);
Assert.AreEqual(ardb.Count, reloaded.Count);
string sampleRewriteArdbFilePath = "Sample.Rewrite.ardb.txt";
Write.ToFile(reloaded.WriteText, sampleRewriteArdbFilePath);
Assert.AreEqual(File.ReadAllText(sampleArdbFilePath), File.ReadAllText(sampleRewriteArdbFilePath));
}
public PopularityAndSizeReport()
{
_addReferenceDB = new AddReferenceDatabase(ArdbVersion.Current);
_results = new List <PopularityDetails>();
}
public AddReferenceDatabaseWrapper(AddReferenceDatabase database)
{
Database = database;
}
private PackageWithTypeResult CreateResult(AddReferenceDatabase database, Symbol type)
{
var nameParts = new List<string>();
GetFullName(nameParts, type.FullName.Parent);
var packageName = type.PackageName.ToString();
var version = database.GetPackageVersion(type.Index).ToString();
return new PackageWithTypeResult(
isDesktopFramework: packageName == MicrosoftAssemblyReferencesName,
packageName: packageName,
assemblyName: type.AssemblyNameWithoutExtension.ToString(),
typeName: type.Name.ToString(),
version: version,
containingNamespaceNames: nameParts);
}
public static void Merge(MergerOptions options)
{
string individualLogs = options.PathToMerge + "Logs";
if (Directory.Exists(individualLogs))
{
Directory.Delete(individualLogs, true);
}
if (options.WithPackageOutcomes)
{
Directory.CreateDirectory(individualLogs);
}
using (new TraceWatch("Filtering '{0}' covering {1:p0} of downloads to Public Types Only...", options.PathToMerge, options.DownloadPercentage))
{
AddReferenceDatabase ardb = new AddReferenceDatabase(options.Version);
ardb.DatabaseVersion = options.DatabaseVersion;
// Determine indexes to include (up to popularity cutoff)
List <string> indexesToInclude = null;
if (Directory.Exists(options.PathToMerge))
{
indexesToInclude = IndexesByPopularityToCutoff(Directory.EnumerateFiles(options.PathToMerge, "*.idx", SearchOption.AllDirectories), options.DownloadPercentage);
if (indexesToInclude.Count == 0)
{
Trace.WriteLine("No indexes found to include. Stopping.");
return;
}
}
else if (File.Exists(options.PathToMerge))
{
// Text file listing IDS files passed
indexesToInclude = new List <string>(File.ReadAllLines(options.PathToMerge));
}
else
{
throw new ArgumentException(String.Format("Merge doesn't know how to crawl passed path, '{0}'.", options.PathToMerge));
}
HashSet <string> excludedPackageNames = ParsePackageNames(options.ExcludedPackageNames);
HashSet <string> filteringDisabledPackageNames = ParsePackageNames(options.DisableDuplicateFilteringPackageNames);
// Load individual package databases in approximate download count order (prefix of name is scale of download count)
ProgressWriter p = new ProgressWriter(indexesToInclude.Count);
foreach (PackageDatabase db in BinarySerializableExtensions.LoadEach <PackageDatabase>(indexesToInclude, true))
{
if (!String.IsNullOrEmpty(db.Identity.PackageName) && excludedPackageNames.Contains(db.Identity.PackageName))
{
Trace.WriteLine(String.Format("Excluded Package {0}", db.Identity.PackageName));
}
else
{
if (filteringDisabledPackageNames.Contains(db.Identity.PackageName))
{
ardb.AddReferenceAssemblyTypes(db);
}
else
{
DatabaseAddResult result = null;
ardb.AddUniqueMembers(db);
if (options.WithPackageOutcomes)
{
string log = Path.Combine(individualLogs, db.Identity.IndexFileName + ".log");
using (StreamWriter writer = new StreamWriter(log, false))
{
result.WriteMemberResults(writer);
}
}
}
}
p.IncrementProgress();
}
// Include the percentage included in the name
string outputFilePath = Path.Combine(options.OutputPath, Path.GetFileName(Path.GetFullPath(options.PathToMerge)) + "." + (options.DownloadPercentage * 100).ToString("g0"));
// Write the merged tree, if requested (debuggability)
if (options.WithMergedTreeLog)
{
string uniqueTreePath = options.PathToMerge + ".MergedTree.log";
using (new TraceWatch("Writing Unique Global Namespace tree to '{0}'...", uniqueTreePath))
{
using (StreamWriter writer = new StreamWriter(uniqueTreePath, false))
{
ardb.GetMergedMembers().WriteMergedTree(writer);
}
}
}
// Write the binary and text forms of the ARDB
using (new TraceWatch("Writing AddReferenceDatabase '{0}'...", outputFilePath))
{
ardb.FileWrite(outputFilePath + ".ardb");
CreateZip(outputFilePath + ".ardb");
Write.ToFile(ardb.WriteText, outputFilePath + ".ardb.txt");
CreateZip(outputFilePath + ".ardb.txt");
}
}
}
public AddReferenceDatabaseWrapper(AddReferenceDatabase database) => Database = database;
