private async void compareVersions_Click(object sender, EventArgs e)
{
await lockWindowAndRunTask(async() =>
{
string newBranch = getBranch();
bool fetchPrevious = (newBranch == "roblox");
string newApiFilePath = await getApiDumpFilePath(newBranch);
string oldApiFilePath = await getApiDumpFilePath("roblox", fetchPrevious);
setStatus("Reading the " + (fetchPrevious ? "Previous" : "Production") + " API...");
var oldApi = new ReflectionDatabase(oldApiFilePath);
oldApi.Branch = fetchPrevious ? "roblox-prev" : "roblox";
setStatus("Reading the " + (fetchPrevious ? "Production" : "New") + " API...");
var newApi = new ReflectionDatabase(newApiFilePath);
newApi.Branch = newBranch;
setStatus("Comparing APIs...");
string format = getApiDumpFormat();
string result = await ReflectionDiffer.CompareDatabases(oldApi, newApi, format);
if (result.Length > 0)
{
FileInfo info = new FileInfo(newApiFilePath);
string dirName = info.DirectoryName;
string fileBase = Path.Combine(dirName, $"{newBranch}-diff.");
string filePath = fileBase + format.ToLower();
if (format == "PNG")
{
string htmlPath = $"{fileBase}.html";
writeFile(htmlPath, result);
setStatus("Rendering Image...");
Bitmap apiRender = await RenderApiDump(htmlPath);
apiRender.Save(filePath);
Process.Start(filePath);
}
else
{
writeAndViewFile(filePath, result);
}
}
else
{
MessageBox.Show("No differences were found!", "Well, this is awkward...", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
clearOldVersionFiles();
});
}
private async void viewApiDumpClassic_Click(object sender, EventArgs e)
{
await lockWindowAndRunTask(async() =>
{
string branch = getBranch();
string format = getApiDumpFormat();
string apiFilePath = await getApiDumpFilePath(branch);
if (format == "JSON")
{
Process.Start(apiFilePath);
return;
}
var api = new ReflectionDatabase(apiFilePath);
var dumper = new ReflectionDumper(api);
string result;
if (format == "HTML" || format == "PNG")
{
result = dumper.DumpApi(ReflectionDumper.DumpUsingHtml, PostProcessHtml);
}
else
{
result = dumper.DumpApi(ReflectionDumper.DumpUsingTxt);
}
FileInfo info = new FileInfo(apiFilePath);
string directory = info.DirectoryName;
string resultPath = Path.Combine(directory, branch + "-api-dump." + format.ToLower());
writeAndViewFile(resultPath, result);
});
}
public ReflectionDumper(ReflectionDatabase database = null)
{
Database = database;
}
private static async Task ProcessArgs(string[] args)
{
if (args.Length < 2)
{
return;
}
Dictionary <string, string> argMap = new Dictionary <string, string>();
string currentArg = "";
foreach (string arg in args)
{
if (arg.StartsWith("-"))
{
if (!string.IsNullOrEmpty(currentArg))
{
argMap.Add(currentArg, "");
}
currentArg = arg;
}
else if (!string.IsNullOrEmpty(currentArg))
{
argMap.Add(currentArg, arg);
currentArg = "";
}
}
if (!string.IsNullOrEmpty(currentArg))
{
argMap.Add(currentArg, "");
}
string format = "TXT";
if (argMap.ContainsKey("-format"))
{
format = argMap["-format"];
}
string bin = Directory.GetCurrentDirectory();
bool isDiffLog = argMap.ContainsKey("-difflog");
if (argMap.ContainsKey("-export"))
{
string branch = argMap["-export"];
string apiFilePath;
if (int.TryParse(branch, out int exportVersion))
{
apiFilePath = await ApiDumpTool.GetApiDumpFilePath("roblox", exportVersion).ConfigureAwait(false);
}
else if (branch == "roblox" || branch.StartsWith("sitetest", StringComparison.InvariantCulture) && branch.EndsWith(".robloxlabs", StringComparison.InvariantCulture))
{
apiFilePath = await ApiDumpTool.GetApiDumpFilePath(branch).ConfigureAwait(false);
}
else
{
apiFilePath = branch;
}
string exportBin = Path.Combine(bin, "ExportAPI");
if (argMap.ContainsKey("-outdir"))
{
exportBin = argMap["-outdir"];
}
else
{
Directory.CreateDirectory(exportBin);
}
if (format.ToUpperInvariant() == "JSON")
{
string jsonPath = Path.Combine(exportBin, branch + ".json");
File.Copy(apiFilePath, jsonPath);
Environment.Exit(0);
return;
}
var api = new ReflectionDatabase(apiFilePath);
var dumper = new ReflectionDumper(api);
string result = "";
bool isPng = false;
if (format.ToUpperInvariant() == "PNG")
{
isPng = true;
format = "html";
result = dumper.DumpApi(ReflectionDumper.DumpUsingHtml);
}
else if (format.ToUpperInvariant() != "HTML")
{
format = "txt";
result = dumper.DumpApi(ReflectionDumper.DumpUsingTxt);
}
else
{
format = "html";
result = dumper.DumpApi(ReflectionDumper.DumpUsingHtml);
}
string exportPath = Path.Combine(exportBin, branch + '.' + format);
if (format == "html" || format == "png")
{
FileInfo info = new FileInfo(exportPath);
string dir = info.DirectoryName;
result = ApiDumpTool.PostProcessHtml(result, dir);
}
File.WriteAllText(exportPath, result);
if (isPng)
{
using (var bitmap = await ApiDumpTool.RenderApiDump(exportPath))
{
exportPath = Path.Combine(exportBin, branch + ".png");
bitmap.Save(exportPath);
}
}
if (argMap.ContainsKey("-start"))
{
Process.Start(exportPath);
}
Environment.Exit(0);
}
else if (argMap.ContainsKey("-compare") || isDiffLog)
{
int version = -1;
if (isDiffLog)
{
string diffLog = argMap["-difflog"];
if (int.TryParse(diffLog, out version))
{
argMap["-new"] = version.ToString();
argMap["-old"] = (version - 1).ToString();
}
else
{
Environment.Exit(1);
}
}
else if (!argMap.ContainsKey("-old") || !argMap.ContainsKey("-new"))
{
Environment.Exit(1);
}
string oldFile = "";
string oldArg = argMap["-old"];
if (int.TryParse(oldArg, out int oldVersion))
{
oldFile = await ApiDumpTool.GetApiDumpFilePath("roblox", oldVersion);
}
else if (oldArg == "roblox" || oldArg.StartsWith("sitetest") && oldArg.EndsWith(".robloxlabs"))
{
oldFile = await ApiDumpTool.GetApiDumpFilePath(oldArg);
}
else
{
oldFile = oldArg;
}
string newFile = "";
string newArg = argMap["-new"];
if (int.TryParse(newArg, out int newVersion))
{
newFile = await ApiDumpTool.GetApiDumpFilePath("roblox", newVersion);
}
else if (newArg == "roblox" || newArg.StartsWith("sitetest") && newArg.EndsWith(".robloxlabs"))
{
newFile = await ApiDumpTool.GetApiDumpFilePath(newArg);
}
else
{
newFile = newArg;
}
var oldApi = new ReflectionDatabase(oldFile);
var newApi = new ReflectionDatabase(newFile);
var invFormat = format.ToUpperInvariant();
bool isPng = (invFormat == "PNG");
if (invFormat == "PNG" || (invFormat == "HTML" && !isDiffLog))
{
format = "HTML";
}
else
{
format = "TXT";
}
string result = await ReflectionDiffer.CompareDatabases(oldApi, newApi, format, false);
string exportPath = "";
if (isDiffLog)
{
exportPath = Path.Combine(bin, version + ".md");
}
else if (argMap.ContainsKey("-out"))
{
exportPath = argMap["-out"];
}
else
{
exportPath = Path.Combine(bin, "custom-comp." + format.ToLowerInvariant());
}
if (format == "HTML")
{
FileInfo info = new FileInfo(exportPath);
string dir = info.DirectoryName;
result = ApiDumpTool.PostProcessHtml(result, dir);
}
if (isDiffLog)
{
string commitUrl = "";
var userAgent = new WebHeaderCollection
{
{ "User-Agent", "Roblox API Dump Tool" }
};
using (WebClient http = new WebClient()
{
Headers = userAgent
})
{
string commitsUrl = $"https://api.github.com/repos/{ClientTracker}/commits?sha=roblox";
string commitsJson = await http.DownloadStringTaskAsync(commitsUrl);
using (StringReader reader = new StringReader(commitsJson))
using (JsonTextReader jsonReader = new JsonTextReader(reader))
{
JArray data = JArray.Load(jsonReader);
string prefix = "0." + version;
foreach (JObject info in data)
{
var commit = info.Value <JObject>("commit");
string message = commit.Value <string>("message");
if (message.StartsWith(prefix))
{
string sha = info.Value <string>("sha");
commitUrl = $"https://github.com/{ClientTracker}/commit/{sha}";
break;
}
}
}
}
result = "## Client Difference Log\n\n"
+ $"{commitUrl}\n\n"
+ "## API Changes\n\n"
+ "```plain\n"
+ $"{result}\n"
+ "```\n\n"
+ $"(Click [here]({apiHistoryUrl}#{version}) for a syntax highlighted version!)";
}
File.WriteAllText(exportPath, result);
if (isPng)
{
using (var bitmap = await ApiDumpTool.RenderApiDump(exportPath))
{
exportPath = exportPath.Replace(".html", ".png");
bitmap.Save(exportPath);
}
}
if (argMap.ContainsKey("-start") || isDiffLog)
{
Process.Start(exportPath);
}
Environment.Exit(0);
}
else if (argMap.ContainsKey("-updatePages"))
{
string dir = argMap["-updatePages"];
if (!Directory.Exists(dir))
{
Environment.Exit(1);
}
StudioDeployLogs logs = await StudioDeployLogs.Get("roblox");
DeployLog currentLog;
if (argMap.ContainsKey("-version"))
{
string versionStr = argMap["-version"];
int version = int.Parse(versionStr);
var logQuery = logs.CurrentLogs_x64
.Where(log => log.Version == version)
.OrderBy(log => log.Changelist);
currentLog = logQuery.Last();
}
else
{
var versionGuid = await ApiDumpTool.GetVersion("roblox");
var logQuery = logs.CurrentLogs_x64.Where(log => log.VersionGuid == versionGuid);
currentLog = logQuery.FirstOrDefault();
}
DeployLog prevLog = logs.CurrentLogs_x64
.Where(log => log.Version < currentLog.Version)
.OrderBy(log => log.Changelist)
.LastOrDefault();
string currentPath = await ApiDumpTool.GetApiDumpFilePath("roblox", currentLog.VersionGuid);
string prevPath = await ApiDumpTool.GetApiDumpFilePath("roblox", prevLog.VersionGuid);
var currentData = new ReflectionDatabase(currentPath, "roblox", currentLog.VersionId);
var prevData = new ReflectionDatabase(prevPath, "roblox", prevLog.VersionId);
var postProcess = new ReflectionDumper.DumpPostProcesser((dump, workDir) =>
{
var head = "<head>\n"
+ $"\t<link rel=\"stylesheet\" href=\"api-dump.css\">\n"
+ "</head>\n\n";
return(head + dump.Trim());
});
// Write Roblox-API-Dump.html
ReflectionDumper dumper = new ReflectionDumper(currentData);
string currentApi = dumper.DumpApi(ReflectionDumper.DumpUsingHtml, postProcess);
string dumpPath = Path.Combine(dir, "Roblox-API-Dump.html");
File.WriteAllText(dumpPath, currentApi);
// Append to Roblox-API-History.html
string comparison = await ReflectionDiffer.CompareDatabases(prevData, currentData, "HTML", false);
string historyPath = Path.Combine(dir, "Roblox-API-History.html");
if (!File.Exists(historyPath))
{
Environment.Exit(1);
}
string history = File.ReadAllText(historyPath);
string appendMarker = $"<hr id=\"{currentLog.Version}\"/>";
if (!history.Contains(appendMarker))
{
string prevMarker = $"<hr id=\"{prevLog.Version}\"/>";
int index = history.IndexOf(prevMarker);
string insert = $"{appendMarker}\n{comparison}";
history = history.Insert(index, insert);
File.WriteAllText(historyPath, history);
}
Directory.SetCurrentDirectory(dir);
var git = new Action <string>((input) =>
{
var gitExecute = Process.Start(new ProcessStartInfo
{
FileName = "git",
Arguments = input,
CreateNoWindow = true,
UseShellExecute = false
});
gitExecute.WaitForExit();
});
git("add .");
git($"commit -m \"{currentLog}\"");
git("push");
Environment.Exit(0);
}
}
public static async Task <string> CompareDatabases(ReflectionDatabase oldApi, ReflectionDatabase newApi, string format = "TXT", bool postProcess = true)
{
currentFormat = format.ToLower();
// For the purposes of the differ, treat png like html.
// Its assumed that the result will be processed afterwards.
if (currentFormat == "png")
{
currentFormat = "html";
}
// Clean up old results.
if (results.Count > 0)
{
results.Clear();
}
// Grab the class lists.
var oldClasses = oldApi.Classes;
var newClasses = newApi.Classes;
// Record classes that were added.
foreach (string className in newClasses.Keys)
{
if (!oldClasses.ContainsKey(className))
{
ClassDescriptor classDesc = newClasses[className];
flagEntireClass(classDesc, Added, true);
}
}
// Record classes that were removed.
foreach (string className in oldClasses.Keys)
{
if (!newClasses.ContainsKey(className))
{
ClassDescriptor classDesc = oldClasses[className];
flagEntireClass(classDesc, Removed, false);
}
}
// Run pre-member-diff modifier tasks.
foreach (IDiffModifier preModifier in preModifiers)
{
preModifier.RunModifier(ref results);
}
// Compare class changes.
foreach (string className in oldClasses.Keys)
{
ClassDescriptor oldClass = oldClasses[className];
if (newClasses.ContainsKey(className))
{
ClassDescriptor newClass = newClasses[className];
// Capture the members of these classes.
var oldMembers = createLookupTable(oldClass.Members);
var newMembers = createLookupTable(newClass.Members);
// Compare the classes directly.
var classTagDiffs = CompareTags(oldClass, oldClass.Tags, newClass.Tags);
Compare(oldClass, "superclass", oldClass.Superclass, newClass.Superclass, true);
Compare(oldClass, "memory category", oldClass.MemoryCategory, newClass.MemoryCategory, true);
// Record members that were added.
foreach (string memberName in newMembers.Keys)
{
if (!oldMembers.ContainsKey(memberName))
{
// Add New Member
MemberDescriptor newMember = newMembers[memberName];
Added(newMember);
}
}
// Record members that were changed or removed.
foreach (string memberName in oldMembers.Keys)
{
MemberDescriptor oldMember = oldMembers[memberName];
if (newMembers.ContainsKey(memberName))
{
MemberDescriptor newMember = newMembers[memberName];
if (oldMember.ThreadSafety.Type != ThreadSafetyType.Unknown)
{
Compare(newMember, "thread safety", oldMember.ThreadSafety, newMember.ThreadSafety);
}
// Check if any tags added to this member were also added to its parent class.
var memberTagDiffs = CompareTags(newMember, oldMember.Tags, newMember.Tags);
MergeTagDiffs(classTagDiffs, memberTagDiffs);
// Compare the fields specific to these member types
// TODO: I'd like to move these routines into their respective
// members, but I'm not sure how to do so in a clean manner.
if (newMember is PropertyDescriptor)
{
var oldProp = oldMember as PropertyDescriptor;
var newProp = newMember as PropertyDescriptor;
var oldMerged = oldProp.Security.Merged;
var newMerged = newProp.Security.Merged;
if (oldMerged && newMerged)
{
// Just compare them as a security change alone.
var oldSecurity = oldProp.Security.Value;
var newSecurity = newProp.Security.Value;
Compare(newMember, "security", oldSecurity, newSecurity);
}
else
{
// Compare the read/write permissions individually.
var oldSecurity = oldProp.Security;
var newSecurity = newProp.Security;
string oldRead = oldSecurity.Read.Value,
newRead = newSecurity.Read.Value;
string oldWrite = oldSecurity.Write.Value,
newWrite = newSecurity.Write.Value;
Compare(newMember, "read permissions", oldRead, newRead);
Compare(newMember, "write permissions", oldWrite, newWrite);
}
var oldSerial = oldProp.Serialization.Describe(true);
var newSerial = newProp.Serialization.Describe(true);
Compare(newMember, "serialization", oldSerial, newSerial);
Compare(newMember, "value-type", oldProp.ValueType, newProp.ValueType);
Compare(newMember, "category", oldProp.Category, newProp.Category, true);
}
else if (newMember is FunctionDescriptor)
{
var oldFunc = oldMember as FunctionDescriptor;
var newFunc = newMember as FunctionDescriptor;
Compare(newMember, "security", oldFunc.Security, newFunc.Security);
Compare(newMember, "parameters", oldFunc.Parameters, newFunc.Parameters);
Compare(newMember, "return-type", oldFunc.ReturnType, newFunc.ReturnType);
}
else if (newMember is CallbackDescriptor)
{
var oldCall = oldMember as CallbackDescriptor;
var newCall = newMember as CallbackDescriptor;
Compare(newMember, "security", oldCall.Security, newCall.Security);
Compare(newMember, "parameters", oldCall.Parameters, newCall.Parameters);
Compare(newMember, "expected return-type", oldCall.ReturnType, newCall.ReturnType);
}
else if (newMember is EventDescriptor)
{
var oldEvent = oldMember as EventDescriptor;
var newEvent = newMember as EventDescriptor;
Compare(newMember, "security", oldEvent.Security, newEvent.Security);
Compare(newMember, "parameters", oldEvent.Parameters, newEvent.Parameters);
}
}
else
{
// Remove old member.
Removed(oldMember, false);
}
}
}
}
// Grab the enum lists.
var oldEnums = oldApi.Enums;
var newEnums = newApi.Enums;
// Record enums that were added.
foreach (string enumName in newEnums.Keys)
{
if (!oldEnums.ContainsKey(enumName))
{
EnumDescriptor newEnum = newEnums[enumName];
flagEntireEnum(newEnum, Added, true);
}
}
// Record enums that were changed or removed.
foreach (string enumName in oldEnums.Keys)
{
EnumDescriptor oldEnum = oldEnums[enumName];
if (newEnums.ContainsKey(enumName))
{
EnumDescriptor newEnum = newEnums[enumName];
var enumTagDiffs = CompareTags(newEnum, oldEnum.Tags, newEnum.Tags);
// Grab the enum-item lists.
var oldItems = createLookupTable(oldEnum.Items);
var newItems = createLookupTable(newEnum.Items);
// Record enum-items that were added.
foreach (var itemName in newItems.Keys)
{
if (!oldItems.ContainsKey(itemName))
{
EnumItemDescriptor item = newItems[itemName];
Added(item);
}
}
foreach (var itemName in oldItems.Keys)
{
EnumItemDescriptor oldItem = oldItems[itemName];
if (newItems.ContainsKey(itemName))
{
EnumItemDescriptor newItem = newItems[itemName];
Compare(newItem, "value", oldItem.Value, newItem.Value);
// Check if any tags that were added to this item were also added to its parent enum.
var itemTagDiffs = CompareTags(newItem, oldItem.Tags, newItem.Tags);
MergeTagDiffs(enumTagDiffs, itemTagDiffs);
}
else
{
// Remove old enum-item.
Removed(oldItem, false);
}
}
}
else
{
// Remove old enum.
flagEntireEnum(oldEnum, Removed, false);
}
}
// Exit early if no diffs were recorded.
if (results.Count == 0)
{
return("");
}
// Select diffs that are not parented to other diffs.
List <Diff> diffs = results
.Where(diff => !diff.HasParent)
.ToList();
// Run post-member-diff modifier tasks.
foreach (IDiffModifier postModifier in postModifiers)
{
postModifier.RunModifier(ref diffs);
}
// Remove diffs that were disposed during the modifier tasks,
diffs = diffs
.Where(diff => !diff.Disposed)
.OrderBy(diff => diff)
.ToList();
// Setup actions for generating the final result, based on the requested format.
DiffResultLineAdder addLineToResults;
DiffResultFinalizer finalizeResults;
List <string> lines = diffs
.Select(diff => diff.ToString())
.ToList();
if (currentFormat == "html")
{
var htmlDumper = new ReflectionDumper();
var diffLookup = diffs.ToDictionary(diff => diff.ToString());
addLineToResults = new DiffResultLineAdder((line, addBreak) =>
{
if (addBreak)
{
htmlDumper.Write(HTML_BREAK);
htmlDumper.NextLine();
}
if (diffLookup.ContainsKey(line))
{
Diff diff = diffLookup[line];
diff.WriteDiffHtml(htmlDumper);
}
if (line.EndsWith(NL))
{
htmlDumper.Write(HTML_BREAK);
htmlDumper.NextLine();
}
});
finalizeResults = new DiffResultFinalizer(() =>
{
if (newApi.Branch == "roblox")
{
htmlDumper.NextLine();
}
string result = htmlDumper.ExportResults();
if (postProcess)
{
result = ApiDumpTool.PostProcessHtml(result);
}
return(result);
});
if (newApi.Branch == "roblox")
{
var deployLog = await ApiDumpTool.GetLastDeployLog("roblox");
htmlDumper.OpenHtmlTag("h2");
htmlDumper.Write("Version " + deployLog.VersionId);
htmlDumper.CloseHtmlTag("h2");
htmlDumper.NextLine(2);
}
}
else
{
var final = new List <string>();
addLineToResults = new DiffResultLineAdder((line, addBreak) =>
{
if (addBreak)
{
final.Add("");
}
final.Add(line);
});
finalizeResults = new DiffResultFinalizer(() =>
{
string[] finalLines = final.ToArray();
return(string.Join(NL, finalLines).Trim());
});
}
// Generate the final diff results
string prevLead = "";
string lastLine = NL;
foreach (string line in lines)
{
string[] words = line.Split(' ');
if (words.Length >= 2)
{
// Capture the first two words in this line.
string first = words[0];
string second = words[1];
if (second.ToLower() == "the" && words.Length > 2)
{
second = words[2];
}
string lead = (first + ' ' + second).Trim();
bool addBreak = false;
bool lastLineNoBreak = !lastLine.EndsWith(NL);
// If the first two words of this line aren't the same as the last...
if (lead != prevLead)
{
// Add a break if the last line doesn't have a break.
// (and if there actually were two previous words)
if (prevLead != "" && lastLineNoBreak)
{
addBreak = true;
}
prevLead = lead;
}
// If we didn't add a break, but this line has a break and the
// previous line doesn't, then we will add a break.
if (!addBreak && lastLineNoBreak && line.EndsWith(NL))
{
addBreak = true;
}
// Handle writing this line depending on the format we're using.
addLineToResults(line, addBreak);
lastLine = line;
}
}
return(finalizeResults());
}
