Beispiel #1
0
        static PS3DialogueFragment SearchForBestPS3Dialogues(List <PS3DialogueFragment> ps3DialogueFragments, MangaGamerDialogue mgDialogueToSearch)
        {
            string mangaGamerChoiceForDiff = Differ.PrepareStringForDiff(mgDialogueToSearch.data);

            float lowestDistance         = float.MaxValue;
            PS3DialogueFragment bestFrag = null;

            foreach (var frag in ps3DialogueFragments)
            {
                string fragmentForDiff = Differ.PrepareStringForDiff(frag.data);

                //if ps3 fragment is of length 0, skip it
                if (fragmentForDiff.Length == 0)
                {
                    continue;
                }

                float rawLevenshtein    = (float)Fastenshtein.Levenshtein.Distance(mangaGamerChoiceForDiff, fragmentForDiff);
                float scaledLevenshtein = rawLevenshtein / Math.Max(mangaGamerChoiceForDiff.Length, fragmentForDiff.Length);

                if (scaledLevenshtein < lowestDistance)
                {
                    lowestDistance = scaledLevenshtein;
                    bestFrag       = frag;
                }
            }

            return(bestFrag);
        }
Beispiel #2
0
        public PS3DialogueFragment(PS3DialogueInstruction parent, string dataFragment, int fragmentID, PS3DialogueFragment previousFragmentInSeries)
        {
            this.parent     = parent;
            this.data       = dataFragment;
            this.fragmentID = fragmentID;
            this.previousFragmentInSeries = previousFragmentInSeries;
            this.ID = parent.ID;

            this.previousLinesOrInstructions = new List <string>();

            //only associate the first fragment with previous instructions of parent
            if (fragmentID == 0)
            {
                this.previousLinesOrInstructions = parent.previousLinesOrInstructions;
            }
        }
Beispiel #3
0
        static void SaveMergedMGScript(List <AlignmentPoint> alignmentPoints, string outputPath, List <string> mgLeftovers)
        {
            using (StreamWriter swOut = FileUtils.CreateDirectoriesAndOpen(outputPath, FileMode.Create))
            {
                //iterate through each dialogue in PS3 list
                List <string> currentPS3ToSave        = new List <string>();
                List <string> currentMangaGamerToSave = new List <string>();

                foreach (AlignmentPoint alignmentPoint in alignmentPoints)
                {
                    if (alignmentPoint.ps3DialogFragment != null)
                    {
                        PS3DialogueFragment ps3 = alignmentPoint.ps3DialogFragment;
                        currentPS3ToSave.AddRange(ps3.previousLinesOrInstructions);
                        if (ps3.fragmentID == 0)
                        {
                            currentPS3ToSave.Add(ps3.parent.translatedRawXML);
                        }
                        //add a comment detailing the fragment information. NOTE: double hypen (--) is not allowed in XML comments
                        currentPS3ToSave.Add($"<!-- [{ps3.ID}.{ps3.fragmentID} > {(ps3.otherDialogue == null ? "NULL" : ps3.otherDialogue.ID.ToString())}]: {ps3.data} -->");
                    }

                    if (alignmentPoint.mangaGamerDialogue != null)
                    {
                        MangaGamerDialogue mg = alignmentPoint.mangaGamerDialogue;
                        currentMangaGamerToSave.AddRange(mg.previousLinesOrInstructions);
                        currentMangaGamerToSave.Add(mg.data);
                    }

                    if (alignmentPoint.ps3DialogFragment != null && alignmentPoint.mangaGamerDialogue != null)
                    {
                        WriteAssociatedPS3StringChunksFormatted(swOut, currentMangaGamerToSave, currentPS3ToSave);

                        currentPS3ToSave.Clear();
                        currentMangaGamerToSave.Clear();
                    }
                }

                //write out any leftover ps3 lines
                WriteAssociatedPS3StringChunksFormatted(swOut, currentMangaGamerToSave, currentPS3ToSave);

                //write out any leftover manga gamer lines
                StringUtils.WriteStringList(swOut, mgLeftovers);
            }
        }
        public static void PrintSideBySideDiff(List <AlignmentPoint> alignmentPoints, out string mg_debug, out string ps3_debug)
        {
            StringWriter swMG  = new StringWriter();
            StringWriter swPS3 = new StringWriter();

            //iterate through each dialogue in PS3 list
            List <string> currentPS3ToSave        = new List <string>();
            List <string> currentMangaGamerToSave = new List <string>();

            foreach (AlignmentPoint alignmentPoint in alignmentPoints)
            {
                //add the previous lines (instructions

                if (alignmentPoint.ps3DialogFragment != null)
                {
                    PS3DialogueFragment ps3 = alignmentPoint.ps3DialogFragment;
                    currentPS3ToSave.AddRange(ps3.previousLinesOrInstructions);
                    currentPS3ToSave.Add($">>>> [{ps3.ID}.{ps3.fragmentID} -> {(ps3.otherDialogue == null ? "NoMatch" : ps3.otherDialogue.ID.ToString())}]: {ps3.data}");
                }

                if (alignmentPoint.mangaGamerDialogue != null)
                {
                    MangaGamerDialogue mg = alignmentPoint.mangaGamerDialogue;
                    currentMangaGamerToSave.AddRange(mg.previousLinesOrInstructions);
                    currentMangaGamerToSave.Add($">>>> [{mg.ID} -> {(mg.otherDialogue == null ? "NoMatch" : mg.otherDialogue.ID.ToString())}]: {mg.data}");
                }

                if (alignmentPoint.ps3DialogFragment != null && alignmentPoint.mangaGamerDialogue != null)
                {
                    //Finally, top-pad the file with enough spaces so they line up (printing could be its own function)
                    WriteSideBySideTopPad(swMG, swPS3, currentMangaGamerToSave, currentPS3ToSave);
                    currentPS3ToSave.Clear();
                    currentMangaGamerToSave.Clear();
                }
            }

            WriteSideBySideTopPad(swMG, swPS3, currentMangaGamerToSave, currentPS3ToSave);
            currentPS3ToSave.Clear();
            currentMangaGamerToSave.Clear();

            mg_debug  = swMG.ToString();
            ps3_debug = swPS3.ToString();
        }
Beispiel #5
0
        /// <summary>
        /// After basic matching, some alignment points will still remain unmatched. This function takes
        /// a single chunk of unmatched alignment points (not the whole list), and attempts to re-match them.
        /// </summary>
        /// <param name="unmatchedSequence"></param>
        /// <returns></returns>
        public static List <AlignmentPoint> ReMatchUnmatchedDialogue(List <AlignmentPoint> unmatchedSequence)
        {
            //if empty list given, just return an empty list
            if (unmatchedSequence.Count == 0)
            {
                return(new List <AlignmentPoint>());
            }

            List <MangaGamerDialogue>     unmatchedMGs          = new List <MangaGamerDialogue>();
            List <PS3DialogueFragment>    unmatchedPS3Fragments = new List <PS3DialogueFragment>();
            List <PS3DialogueInstruction> unmatchedPS3s         = new List <PS3DialogueInstruction>();

            HashSet <int> alreadySeenPS3ParentIDs = new HashSet <int>();
            Dictionary <int, PS3DialogueFragment> ps3DialogueIDToFirstFragmentMapping = new Dictionary <int, PS3DialogueFragment>();

            DebugUtils.Print("------------------------------------");
            foreach (AlignmentPoint ap in unmatchedSequence)
            {
                if (ap.mangaGamerDialogue != null)
                {
                    DebugUtils.Print($"MG line: {ap.mangaGamerDialogue.data}");
                    unmatchedMGs.Add(ap.mangaGamerDialogue);
                }

                if (ap.ps3DialogFragment != null)
                {
                    unmatchedPS3Fragments.Add(ap.ps3DialogFragment);

                    if (!alreadySeenPS3ParentIDs.Contains(ap.ps3DialogFragment.parent.ID))
                    {
                        ps3DialogueIDToFirstFragmentMapping.Add(ap.ps3DialogFragment.parent.ID, ap.ps3DialogFragment);
                        alreadySeenPS3ParentIDs.Add(ap.ps3DialogFragment.parent.ID);
                        DebugUtils.Print($"PS3 parent of below missing fragments [{ap.ps3DialogFragment.parent.ID}]: {ap.ps3DialogFragment.parent.data}");
                        unmatchedPS3s.Add(ap.ps3DialogFragment.parent);
                    }

                    DebugUtils.Print($"PS3 child [{ap.ps3DialogFragment.parent.ID}]: {ap.ps3DialogFragment.data}");
                }
            }

            //Try and match the unmatched lines
            List <InOrderLevenshteinMatcher.LevenshteinResult> greedyMatchResults = InOrderLevenshteinMatcher.DoMatching(unmatchedMGs, unmatchedPS3s);

            //Use the match results to set associations
            foreach (var result in greedyMatchResults)
            {
                MangaGamerDialogue mgToAssign = unmatchedMGs[result.mgIndex];
                //want to get the first ps3 fragment associated with the Dialogue. Use hashmap we made earlier.
                PS3DialogueFragment ps3FragmentToAssign = ps3DialogueIDToFirstFragmentMapping[unmatchedPS3s[result.ps3Index].ID];
                mgToAssign.Associate(ps3FragmentToAssign);
            }

            //iterate through the list and add alignment points appropriately
            List <AlignmentPoint> reAssociatedAlignmentPoints = GetAlignmentPointsFromMGPS3Array(unmatchedMGs, unmatchedPS3Fragments);

            //Debug: Print out re-assigned alignment points for debugging
            foreach (AlignmentPoint ap in reAssociatedAlignmentPoints)
            {
                DebugUtils.Print(ap.ToString());
            }

            return(reAssociatedAlignmentPoints);
        }
Beispiel #6
0
        //This function performs the diff given the two lists of dialogue.
        //It then UPDATES the values in the mangaGamerDialogueList and the ps3DialogueList (the DialogueBase.other value is updated on each dialogue object!)
        //If a dialogue cannot be associated, it is set to NULL.
        public static List <AlignmentPoint> DoDiff(string tempFolderPath, List <MangaGamerDialogue> mangaGamerDialogueList, List <PS3DialogueFragment> ps3DialogueFragments, string debugFilenamePrefix = "")
        {
            //Convert PS3 Dialogue list into list of subsections before performing diff - this can be re-assembled later!
            string mangaGamerDiffInputPath = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffInputA.txt");
            string PS3DiffInputPath        = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffInputB.txt");
            string diffOutputPath          = Path.Combine(tempFolderPath, debugFilenamePrefix + "_diffOutput.txt");

            //write the diff-prepared manga gamer dialogue to a file
            WriteListOfDialogueToFile(mangaGamerDialogueList, mangaGamerDiffInputPath);

            //write the diff-prepared ps3 dialogue to a file
            WriteListOfDialogueToFile(ps3DialogueFragments, PS3DiffInputPath);

            //do the diff
            string diffResult = RunDiffTool(mangaGamerDiffInputPath, PS3DiffInputPath);

            //save the diff to file for debugging
            using (StreamWriter sw = FileUtils.CreateDirectoriesAndOpen(diffOutputPath, FileMode.Create))
            {
                sw.Write(diffResult);
            }

            List <AlignmentPoint> alignmentPoints = new List <AlignmentPoint>();

            using (StringReader reader = new StringReader(diffResult))
            {
                string line;
                //skip the header information
                while ((line = reader.ReadLine()) != null)
                {
                    if (line[0] == '@')
                    {
                        break;
                    }
                }

                //TODO: need to think of the best way to categorize all mangagamer lines...
                //a ' ' means lines are the same
                //a '+' means line is present in PS3 ONLY           (it was 'added' in the ps3 version)
                //a '-' means line is present in mangagamer ONLY    (it was 'removed' from the ps3 version)
                List <AlignmentPoint> unmatchedSequence = new List <AlignmentPoint>();
                int mgIndex  = 0;
                int ps3Index = 0;
                while ((line = reader.ReadLine()) != null)
                {
                    //classify the line type:
                    char lineType = line[0];
                    if (lineType == ' ') //lines match
                    {
                        PS3DialogueFragment dummyPS3Instruction = ps3DialogueFragments[ps3Index];
                        MangaGamerDialogue  currentMangaGamer   = mangaGamerDialogueList[mgIndex];

                        //associate the fragment with the mangagamer dialogue
                        currentMangaGamer.Associate(dummyPS3Instruction);

                        //re-match the unmatched sequence, then clear it for next iteration
                        alignmentPoints.AddRange(DialogueReMatcher.ReMatchUnmatchedDialogue(unmatchedSequence));
                        unmatchedSequence.Clear();

                        //add the just found instruction
                        alignmentPoints.Add(new AlignmentPoint(currentMangaGamer, dummyPS3Instruction));

                        mgIndex++;
                        ps3Index++;
                    }
                    else if (lineType == '+') //only exist in ps3
                    {
                        PS3DialogueFragment currentDialog = ps3DialogueFragments[ps3Index];
                        unmatchedSequence.Add(new AlignmentPoint(null, currentDialog));
                        ps3Index++;
                    }
                    else if (lineType == '-') //only exist in mangagamer
                    {
                        MangaGamerDialogue currentDialog = mangaGamerDialogueList[mgIndex];
                        unmatchedSequence.Add(new AlignmentPoint(currentDialog, null));
                        mgIndex++;
                    }
                }

                //Deal with any leftover unmatched sequences at the end of the file
                alignmentPoints.AddRange(DialogueReMatcher.ReMatchUnmatchedDialogue(unmatchedSequence));
            }

            return(alignmentPoints);
        }
Beispiel #7
0
 public AlignmentPoint(MangaGamerDialogue mg, PS3DialogueFragment p)
 {
     mangaGamerDialogue = mg;
     ps3DialogFragment  = p;
 }
Beispiel #8
0
        /// <summary>
        /// processes a single mangagamer script, attempting to merge the matching ps3 instructions
        /// </summary>
        /// <param name="pS3DialogueInstructionsPreFilter"></param>
        /// <param name="config"></param>
        /// <param name="mgInfo"></param>
        /// <param name="guessedInputInfos"></param>
        static List <PartialSubScriptToMerge> ProcessSingleFile(
            List <PS3DialogueInstruction> pS3DialogueInstructionsPreFilter,
            MergerConfiguration config,
            InputInfo mgInfo,
            List <InputInfo> guessedInputInfos,
            Counter counter)
        {
            string fullPath  = Path.Combine(config.input_folder, mgInfo.path);
            string pathNoExt = Path.GetFileNameWithoutExtension(fullPath);

            string debug_side_by_side_diff_path_MG = Path.Combine(config.output_folder, pathNoExt + "_side_by_side_debug.html");
            string debug_alignment_statistics      = Path.Combine(config.output_folder, pathNoExt + "_statistics_debug.txt");

            List <PS3DialogueInstruction> pS3DialogueInstructions = GetFilteredPS3Instructions(pS3DialogueInstructionsPreFilter, mgInfo.ps3_regions);

            //load all the mangagamer lines from the mangagamer file
            List <MangaGamerDialogue> allMangaGamerDialogue = MangaGamerScriptReader.GetDialogueLinesFromMangaGamerScript(fullPath, out List <string> mg_leftovers);

            //PS3 Dialogue fragments
            List <PS3DialogueFragment> ps3DialogueFragments = new List <PS3DialogueFragment>();
            int ps3DialogueIndex = 0;

            foreach (PS3DialogueInstruction ps3Dialogue in pS3DialogueInstructions)
            {
                List <string>       splitDialogueStrings        = PS3DialogueTools.SplitPS3StringNoNames(ps3Dialogue.data);
                PS3DialogueFragment previousPS3DialogueFragment = null;
                for (int i = 0; i < splitDialogueStrings.Count; i++)
                {
                    //dummy instructions index into the ps3DialogueList (for now...)
                    PS3DialogueFragment ps3DialogueFragment = new PS3DialogueFragment(ps3Dialogue, splitDialogueStrings[i], i, previousPS3DialogueFragment);
                    ps3DialogueFragments.Add(ps3DialogueFragment);
                    previousPS3DialogueFragment = ps3DialogueFragment;
                }
                ps3DialogueIndex++;
            }

            //If no ps3 regions specified, scan for regions, then print and let user fill in?
            if (mgInfo.ps3_regions.Count == 0)
            {
                Console.WriteLine($"The file [{mgInfo.path}] does not have the PS3 region marked in the conf.toml file!");
                Console.WriteLine($"Scanning for PS3 region...");

                //print the first few and last mangagamer instructions
                //skip if length too small?
                Console.WriteLine("------- Finding first 5 entries -------");
                int?startResult = AnalyseEntries(ps3DialogueFragments, allMangaGamerDialogue, amount: 10, isStart: true);
                if (startResult.HasValue)
                {
                    Console.WriteLine($"Best guess at start PS3 ID: {startResult.Value}");
                }
                else
                {
                    Console.WriteLine("Not sure about start PS3 ID. Please inspect manually.");
                }

                Console.WriteLine("------- Finding last 5 entries -------");
                int?endResult = AnalyseEntries(ps3DialogueFragments, allMangaGamerDialogue, amount: 10, isStart: false);
                if (endResult.HasValue)
                {
                    Console.WriteLine($"Best guess at last PS3 ID: {endResult.Value}");
                }
                else
                {
                    Console.WriteLine("Not sure about last PS3 ID. Please inspect manually.");
                }

                string result_start_id = "<START_REGION>";
                string result_end_id   = "<END_REGION>";

                if (startResult.HasValue && endResult.HasValue)
                {
                    Console.WriteLine("AUTOREGION SUCCESS: You can copy this into the conf.toml file\n\n");
                    result_start_id = startResult.Value.ToString();
                    result_end_id   = endResult.Value.ToString();
                }
                else
                {
                    Console.WriteLine($"AUTOREGION FAIL: Region couldn't be determined confidently. Please use the above information and the ps3 script" +
                                      $"to determine the PS3 region manually, then place the results in the conf.toml file as per below");
                }

                Console.WriteLine("[[input]]");
                Console.WriteLine($@"path = ""{mgInfo.path}""");
                Console.WriteLine($"ps3_regions = [[{result_start_id}, {result_end_id}]]");
                Console.WriteLine("No output will be generated for this script until the program is run again.");
                Console.WriteLine("Press ENTER to move to the next script...");

                Console.ReadKey();
                return(new List <PartialSubScriptToMerge>());
            }

            //Diff the dialogue
            List <AlignmentPoint> allAlignmentPoints = Differ.DoDiff(config.temp_folder, allMangaGamerDialogue, ps3DialogueFragments, debugFilenamePrefix: pathNoExt);

            //Sanity check the alignment points by making sure there aren't missing any values
            SanityCheckAlignmentPoints(allAlignmentPoints, allMangaGamerDialogue, ps3DialogueFragments);

            //trim alignment points to reduce output
            List <AlignmentPoint> alignmentPoints = config.trim_after_diff ? TrimAlignmentPoints(allAlignmentPoints) : allAlignmentPoints;

            //Write statistics on how well matched the alignment points are
            WriteAlignmentStatistics(alignmentPoints, debug_alignment_statistics);

            //DEBUG: generate the side-by-side diff
            DifferDebugUtilities.PrintHTMLSideBySideDiff(alignmentPoints, debug_side_by_side_diff_path_MG);

            //Insert PS3 instructions
            string mergedOutputPath = Path.Combine(config.output_folder, pathNoExt + "_merged.xml.txt");

            SaveMergedMGScript(alignmentPoints, mergedOutputPath, mg_leftovers);

            // >>>> UnMerge ModCallScriptSection: Before using the results, we need to reverse the step we did earlier, by unmerging any merged files back into multiple files.

            //Use the inserted instructions
            string finalOutputWithMergedForkedScripts = Path.Combine(config.output_folder, pathNoExt + ".txt");

            MergedScriptPostProcessing.PostProcessingMain.InsertMGLinesUsingPS3XML(mergedOutputPath, finalOutputWithMergedForkedScripts, config, counter);

            List <PartialSubScriptToMerge> partialSubScriptsToMerge = ForkingScriptMerger.GetForkedScriptContentFromMergedScript(config.pre_input_folder, finalOutputWithMergedForkedScripts);

            ForkingScriptMerger.RemoveForkedScriptContentFromMergedScript(finalOutputWithMergedForkedScripts);

            return(partialSubScriptsToMerge);
        }