static PS3DialogueFragment SearchForBestPS3Dialogues(List <PS3DialogueFragment> ps3DialogueFragments, MangaGamerDialogue mgDialogueToSearch)
{
string mangaGamerChoiceForDiff = Differ.PrepareStringForDiff(mgDialogueToSearch.data);
float lowestDistance = float.MaxValue;
PS3DialogueFragment bestFrag = null;
foreach (var frag in ps3DialogueFragments)
{
string fragmentForDiff = Differ.PrepareStringForDiff(frag.data);
//if ps3 fragment is of length 0, skip it
if (fragmentForDiff.Length == 0)
{
continue;
}
float rawLevenshtein = (float)Fastenshtein.Levenshtein.Distance(mangaGamerChoiceForDiff, fragmentForDiff);
float scaledLevenshtein = rawLevenshtein / Math.Max(mangaGamerChoiceForDiff.Length, fragmentForDiff.Length);
if (scaledLevenshtein < lowestDistance)
{
lowestDistance = scaledLevenshtein;
bestFrag = frag;
}
}
return(bestFrag);
}
public PS3DialogueFragment(PS3DialogueInstruction parent, string dataFragment, int fragmentID, PS3DialogueFragment previousFragmentInSeries)
{
this.parent = parent;
this.data = dataFragment;
this.fragmentID = fragmentID;
this.previousFragmentInSeries = previousFragmentInSeries;
this.ID = parent.ID;
this.previousLinesOrInstructions = new List <string>();
//only associate the first fragment with previous instructions of parent
if (fragmentID == 0)
{
this.previousLinesOrInstructions = parent.previousLinesOrInstructions;
}
}
static void SaveMergedMGScript(List <AlignmentPoint> alignmentPoints, string outputPath, List <string> mgLeftovers)
{
using (StreamWriter swOut = FileUtils.CreateDirectoriesAndOpen(outputPath, FileMode.Create))
{
//iterate through each dialogue in PS3 list
List <string> currentPS3ToSave = new List <string>();
List <string> currentMangaGamerToSave = new List <string>();
foreach (AlignmentPoint alignmentPoint in alignmentPoints)
{
if (alignmentPoint.ps3DialogFragment != null)
{
PS3DialogueFragment ps3 = alignmentPoint.ps3DialogFragment;
currentPS3ToSave.AddRange(ps3.previousLinesOrInstructions);
if (ps3.fragmentID == 0)
{
currentPS3ToSave.Add(ps3.parent.translatedRawXML);
}
//add a comment detailing the fragment information. NOTE: double hypen (--) is not allowed in XML comments
currentPS3ToSave.Add($"<!-- [{ps3.ID}.{ps3.fragmentID} > {(ps3.otherDialogue == null ? "NULL" : ps3.otherDialogue.ID.ToString())}]: {ps3.data} -->");
}
if (alignmentPoint.mangaGamerDialogue != null)
{
MangaGamerDialogue mg = alignmentPoint.mangaGamerDialogue;
currentMangaGamerToSave.AddRange(mg.previousLinesOrInstructions);
currentMangaGamerToSave.Add(mg.data);
}
if (alignmentPoint.ps3DialogFragment != null && alignmentPoint.mangaGamerDialogue != null)
{
WriteAssociatedPS3StringChunksFormatted(swOut, currentMangaGamerToSave, currentPS3ToSave);
currentPS3ToSave.Clear();
currentMangaGamerToSave.Clear();
}
}
//write out any leftover ps3 lines
WriteAssociatedPS3StringChunksFormatted(swOut, currentMangaGamerToSave, currentPS3ToSave);
//write out any leftover manga gamer lines
StringUtils.WriteStringList(swOut, mgLeftovers);
}
}
public static void PrintSideBySideDiff(List <AlignmentPoint> alignmentPoints, out string mg_debug, out string ps3_debug)
{
StringWriter swMG = new StringWriter();
StringWriter swPS3 = new StringWriter();
//iterate through each dialogue in PS3 list
List <string> currentPS3ToSave = new List <string>();
List <string> currentMangaGamerToSave = new List <string>();
foreach (AlignmentPoint alignmentPoint in alignmentPoints)
{
//add the previous lines (instructions
if (alignmentPoint.ps3DialogFragment != null)
{
PS3DialogueFragment ps3 = alignmentPoint.ps3DialogFragment;
currentPS3ToSave.AddRange(ps3.previousLinesOrInstructions);
currentPS3ToSave.Add($">>>> [{ps3.ID}.{ps3.fragmentID} -> {(ps3.otherDialogue == null ? "NoMatch" : ps3.otherDialogue.ID.ToString())}]: {ps3.data}");
}
if (alignmentPoint.mangaGamerDialogue != null)
{
MangaGamerDialogue mg = alignmentPoint.mangaGamerDialogue;
currentMangaGamerToSave.AddRange(mg.previousLinesOrInstructions);
currentMangaGamerToSave.Add($">>>> [{mg.ID} -> {(mg.otherDialogue == null ? "NoMatch" : mg.otherDialogue.ID.ToString())}]: {mg.data}");
}
if (alignmentPoint.ps3DialogFragment != null && alignmentPoint.mangaGamerDialogue != null)
{
//Finally, top-pad the file with enough spaces so they line up (printing could be its own function)
WriteSideBySideTopPad(swMG, swPS3, currentMangaGamerToSave, currentPS3ToSave);
currentPS3ToSave.Clear();
currentMangaGamerToSave.Clear();
}
}
WriteSideBySideTopPad(swMG, swPS3, currentMangaGamerToSave, currentPS3ToSave);
currentPS3ToSave.Clear();
currentMangaGamerToSave.Clear();
mg_debug = swMG.ToString();
ps3_debug = swPS3.ToString();
}
/// <summary>
/// After basic matching, some alignment points will still remain unmatched. This function takes
/// a single chunk of unmatched alignment points (not the whole list), and attempts to re-match them.
/// </summary>
/// <param name="unmatchedSequence"></param>
/// <returns></returns>
public static List <AlignmentPoint> ReMatchUnmatchedDialogue(List <AlignmentPoint> unmatchedSequence)
{
//if empty list given, just return an empty list
if (unmatchedSequence.Count == 0)
{
return(new List <AlignmentPoint>());
}
List <MangaGamerDialogue> unmatchedMGs = new List <MangaGamerDialogue>();
List <PS3DialogueFragment> unmatchedPS3Fragments = new List <PS3DialogueFragment>();
List <PS3DialogueInstruction> unmatchedPS3s = new List <PS3DialogueInstruction>();
HashSet <int> alreadySeenPS3ParentIDs = new HashSet <int>();
Dictionary <int, PS3DialogueFragment> ps3DialogueIDToFirstFragmentMapping = new Dictionary <int, PS3DialogueFragment>();
DebugUtils.Print("------------------------------------");
foreach (AlignmentPoint ap in unmatchedSequence)
{
if (ap.mangaGamerDialogue != null)
{
DebugUtils.Print($"MG line: {ap.mangaGamerDialogue.data}");
unmatchedMGs.Add(ap.mangaGamerDialogue);
}
if (ap.ps3DialogFragment != null)
{
unmatchedPS3Fragments.Add(ap.ps3DialogFragment);
if (!alreadySeenPS3ParentIDs.Contains(ap.ps3DialogFragment.parent.ID))
{
ps3DialogueIDToFirstFragmentMapping.Add(ap.ps3DialogFragment.parent.ID, ap.ps3DialogFragment);
alreadySeenPS3ParentIDs.Add(ap.ps3DialogFragment.parent.ID);
DebugUtils.Print($"PS3 parent of below missing fragments [{ap.ps3DialogFragment.parent.ID}]: {ap.ps3DialogFragment.parent.data}");
unmatchedPS3s.Add(ap.ps3DialogFragment.parent);
}
DebugUtils.Print($"PS3 child [{ap.ps3DialogFragment.parent.ID}]: {ap.ps3DialogFragment.data}");
}
}
//Try and match the unmatched lines
List <InOrderLevenshteinMatcher.LevenshteinResult> greedyMatchResults = InOrderLevenshteinMatcher.DoMatching(unmatchedMGs, unmatchedPS3s);
//Use the match results to set associations
foreach (var result in greedyMatchResults)
{
MangaGamerDialogue mgToAssign = unmatchedMGs[result.mgIndex];
//want to get the first ps3 fragment associated with the Dialogue. Use hashmap we made earlier.
PS3DialogueFragment ps3FragmentToAssign = ps3DialogueIDToFirstFragmentMapping[unmatchedPS3s[result.ps3Index].ID];
mgToAssign.Associate(ps3FragmentToAssign);
}
//iterate through the list and add alignment points appropriately
List <AlignmentPoint> reAssociatedAlignmentPoints = GetAlignmentPointsFromMGPS3Array(unmatchedMGs, unmatchedPS3Fragments);
//Debug: Print out re-assigned alignment points for debugging
foreach (AlignmentPoint ap in reAssociatedAlignmentPoints)
{
DebugUtils.Print(ap.ToString());
}
return(reAssociatedAlignmentPoints);
}
//This function performs the diff given the two lists of dialogue.
//It then UPDATES the values in the mangaGamerDialogueList and the ps3DialogueList (the DialogueBase.other value is updated on each dialogue object!)
//If a dialogue cannot be associated, it is set to NULL.
public static List <AlignmentPoint> DoDiff(string tempFolderPath, List <MangaGamerDialogue> mangaGamerDialogueList, List <PS3DialogueFragment> ps3DialogueFragments, string debugFilenamePrefix = "")
{
//Convert PS3 Dialogue list into list of subsections before performing diff - this can be re-assembled later!
string mangaGamerDiffInputPath = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffInputA.txt");
string PS3DiffInputPath = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffInputB.txt");
string diffOutputPath = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffOutput.txt");
//write the diff-prepared manga gamer dialogue to a file
WriteListOfDialogueToFile(mangaGamerDialogueList, mangaGamerDiffInputPath);
//write the diff-prepared ps3 dialogue to a file
WriteListOfDialogueToFile(ps3DialogueFragments, PS3DiffInputPath);
//do the diff
string diffResult = RunDiffTool(mangaGamerDiffInputPath, PS3DiffInputPath);
//save the diff to file for debugging
using (StreamWriter sw = FileUtils.CreateDirectoriesAndOpen(diffOutputPath, FileMode.Create))
{
sw.Write(diffResult);
}
List <AlignmentPoint> alignmentPoints = new List <AlignmentPoint>();
using (StringReader reader = new StringReader(diffResult))
{
string line;
//skip the header information
while ((line = reader.ReadLine()) != null)
{
if (line[0] == '@')
{
break;
}
}
//TODO: need to think of the best way to categorize all mangagamer lines...
//a ' ' means lines are the same
//a '+' means line is present in PS3 ONLY (it was 'added' in the ps3 version)
//a '-' means line is present in mangagamer ONLY (it was 'removed' from the ps3 version)
List <AlignmentPoint> unmatchedSequence = new List <AlignmentPoint>();
int mgIndex = 0;
int ps3Index = 0;
while ((line = reader.ReadLine()) != null)
{
//classify the line type:
char lineType = line[0];
if (lineType == ' ') //lines match
{
PS3DialogueFragment dummyPS3Instruction = ps3DialogueFragments[ps3Index];
MangaGamerDialogue currentMangaGamer = mangaGamerDialogueList[mgIndex];
//associate the fragment with the mangagamer dialogue
currentMangaGamer.Associate(dummyPS3Instruction);
//re-match the unmatched sequence, then clear it for next iteration
alignmentPoints.AddRange(DialogueReMatcher.ReMatchUnmatchedDialogue(unmatchedSequence));
unmatchedSequence.Clear();
//add the just found instruction
alignmentPoints.Add(new AlignmentPoint(currentMangaGamer, dummyPS3Instruction));
mgIndex++;
ps3Index++;
}
else if (lineType == '+') //only exist in ps3
{
PS3DialogueFragment currentDialog = ps3DialogueFragments[ps3Index];
unmatchedSequence.Add(new AlignmentPoint(null, currentDialog));
ps3Index++;
}
else if (lineType == '-') //only exist in mangagamer
{
MangaGamerDialogue currentDialog = mangaGamerDialogueList[mgIndex];
unmatchedSequence.Add(new AlignmentPoint(currentDialog, null));
mgIndex++;
}
}
//Deal with any leftover unmatched sequences at the end of the file
alignmentPoints.AddRange(DialogueReMatcher.ReMatchUnmatchedDialogue(unmatchedSequence));
}
return(alignmentPoints);
}
public AlignmentPoint(MangaGamerDialogue mg, PS3DialogueFragment p)
{
mangaGamerDialogue = mg;
ps3DialogFragment = p;
}
/// <summary>
/// processes a single mangagamer script, attempting to merge the matching ps3 instructions
/// </summary>
/// <param name="pS3DialogueInstructionsPreFilter"></param>
/// <param name="config"></param>
/// <param name="mgInfo"></param>
/// <param name="guessedInputInfos"></param>
static List <PartialSubScriptToMerge> ProcessSingleFile(
List <PS3DialogueInstruction> pS3DialogueInstructionsPreFilter,
MergerConfiguration config,
InputInfo mgInfo,
List <InputInfo> guessedInputInfos,
Counter counter)
{
string fullPath = Path.Combine(config.input_folder, mgInfo.path);
string pathNoExt = Path.GetFileNameWithoutExtension(fullPath);
string debug_side_by_side_diff_path_MG = Path.Combine(config.output_folder, pathNoExt + "_side_by_side_debug.html");
string debug_alignment_statistics = Path.Combine(config.output_folder, pathNoExt + "_statistics_debug.txt");
List <PS3DialogueInstruction> pS3DialogueInstructions = GetFilteredPS3Instructions(pS3DialogueInstructionsPreFilter, mgInfo.ps3_regions);
//load all the mangagamer lines from the mangagamer file
List <MangaGamerDialogue> allMangaGamerDialogue = MangaGamerScriptReader.GetDialogueLinesFromMangaGamerScript(fullPath, out List <string> mg_leftovers);
//PS3 Dialogue fragments
List <PS3DialogueFragment> ps3DialogueFragments = new List <PS3DialogueFragment>();
int ps3DialogueIndex = 0;
foreach (PS3DialogueInstruction ps3Dialogue in pS3DialogueInstructions)
{
List <string> splitDialogueStrings = PS3DialogueTools.SplitPS3StringNoNames(ps3Dialogue.data);
PS3DialogueFragment previousPS3DialogueFragment = null;
for (int i = 0; i < splitDialogueStrings.Count; i++)
{
//dummy instructions index into the ps3DialogueList (for now...)
PS3DialogueFragment ps3DialogueFragment = new PS3DialogueFragment(ps3Dialogue, splitDialogueStrings[i], i, previousPS3DialogueFragment);
ps3DialogueFragments.Add(ps3DialogueFragment);
previousPS3DialogueFragment = ps3DialogueFragment;
}
ps3DialogueIndex++;
}
//If no ps3 regions specified, scan for regions, then print and let user fill in?
if (mgInfo.ps3_regions.Count == 0)
{
Console.WriteLine($"The file [{mgInfo.path}] does not have the PS3 region marked in the conf.toml file!");
Console.WriteLine($"Scanning for PS3 region...");
//print the first few and last mangagamer instructions
//skip if length too small?
Console.WriteLine("------- Finding first 5 entries -------");
int?startResult = AnalyseEntries(ps3DialogueFragments, allMangaGamerDialogue, amount: 10, isStart: true);
if (startResult.HasValue)
{
Console.WriteLine($"Best guess at start PS3 ID: {startResult.Value}");
}
else
{
Console.WriteLine("Not sure about start PS3 ID. Please inspect manually.");
}
Console.WriteLine("------- Finding last 5 entries -------");
int?endResult = AnalyseEntries(ps3DialogueFragments, allMangaGamerDialogue, amount: 10, isStart: false);
if (endResult.HasValue)
{
Console.WriteLine($"Best guess at last PS3 ID: {endResult.Value}");
}
else
{
Console.WriteLine("Not sure about last PS3 ID. Please inspect manually.");
}
string result_start_id = "<START_REGION>";
string result_end_id = "<END_REGION>";
if (startResult.HasValue && endResult.HasValue)
{
Console.WriteLine("AUTOREGION SUCCESS: You can copy this into the conf.toml file\n\n");
result_start_id = startResult.Value.ToString();
result_end_id = endResult.Value.ToString();
}
else
{
Console.WriteLine($"AUTOREGION FAIL: Region couldn't be determined confidently. Please use the above information and the ps3 script" +
$"to determine the PS3 region manually, then place the results in the conf.toml file as per below");
}
Console.WriteLine("[[input]]");
Console.WriteLine($@"path = ""{mgInfo.path}""");
Console.WriteLine($"ps3_regions = [[{result_start_id}, {result_end_id}]]");
Console.WriteLine("No output will be generated for this script until the program is run again.");
Console.WriteLine("Press ENTER to move to the next script...");
Console.ReadKey();
return(new List <PartialSubScriptToMerge>());
}
//Diff the dialogue
List <AlignmentPoint> allAlignmentPoints = Differ.DoDiff(config.temp_folder, allMangaGamerDialogue, ps3DialogueFragments, debugFilenamePrefix: pathNoExt);
//Sanity check the alignment points by making sure there aren't missing any values
SanityCheckAlignmentPoints(allAlignmentPoints, allMangaGamerDialogue, ps3DialogueFragments);
//trim alignment points to reduce output
List <AlignmentPoint> alignmentPoints = config.trim_after_diff ? TrimAlignmentPoints(allAlignmentPoints) : allAlignmentPoints;
//Write statistics on how well matched the alignment points are
WriteAlignmentStatistics(alignmentPoints, debug_alignment_statistics);
//DEBUG: generate the side-by-side diff
DifferDebugUtilities.PrintHTMLSideBySideDiff(alignmentPoints, debug_side_by_side_diff_path_MG);
//Insert PS3 instructions
string mergedOutputPath = Path.Combine(config.output_folder, pathNoExt + "_merged.xml.txt");
SaveMergedMGScript(alignmentPoints, mergedOutputPath, mg_leftovers);
// >>>> UnMerge ModCallScriptSection: Before using the results, we need to reverse the step we did earlier, by unmerging any merged files back into multiple files.
//Use the inserted instructions
string finalOutputWithMergedForkedScripts = Path.Combine(config.output_folder, pathNoExt + ".txt");
MergedScriptPostProcessing.PostProcessingMain.InsertMGLinesUsingPS3XML(mergedOutputPath, finalOutputWithMergedForkedScripts, config, counter);
List <PartialSubScriptToMerge> partialSubScriptsToMerge = ForkingScriptMerger.GetForkedScriptContentFromMergedScript(config.pre_input_folder, finalOutputWithMergedForkedScripts);
ForkingScriptMerger.RemoveForkedScriptContentFromMergedScript(finalOutputWithMergedForkedScripts);
return(partialSubScriptsToMerge);
}