/// <summary> /// Reads comma-separated value ata from <paramref /// name="sourceText"/>, and produces a collection of <see /// cref="StringTableEntry"/> structs. /// </summary> /// <param name="sourceText">A string containing CSV-formatted /// data.</param> /// <returns>The parsed collection of <see /// cref="StringTableEntry"/> structs.</returns> /// <throws cref="CsvHelperException">Thrown when an error occurs /// when parsing the string.</throws> public static IEnumerable <StringTableEntry> ParseFromCSV(string sourceText) { using (var stringReader = new System.IO.StringReader(sourceText)) using (var csv = new CsvReader(stringReader, GetConfiguration())) { /* * Do the below instead of GetRecords<T> due to incompatibility with IL2CPP * See more: https://github.com/JoshClose/CsvHelper/issues/1337 */ var records = new List <StringTableEntry>(); csv.Read(); csv.ReadHeader(); while (csv.Read()) { var record = new StringTableEntry { Language = csv.GetField <string>("language"), ID = csv.GetField <string>("id"), Text = csv.GetField <string>("text"), File = csv.GetField <string>("file"), Node = csv.GetField <string>("node"), LineNumber = csv.GetField <string>("lineNumber"), Lock = csv.GetField <string>("lock"), Comment = csv.GetField <string>("comment"), }; records.Add(record); } return(records); } }
public StringTableEntry(StringTableEntry s) { ID = s.ID; Text = s.Text; File = s.File; Node = s.Node; LineNumber = s.LineNumber; Lock = s.Lock; Comment = s.Comment; Language = s.Language; }
/// <summary> /// Reads comma-separated value ata from <paramref /// name="sourceText"/>, and produces a collection of <see /// cref="StringTableEntry"/> structs. /// </summary> /// <param name="sourceText">A string containing CSV-formatted /// data.</param> /// <returns>The parsed collection of <see /// cref="StringTableEntry"/> structs.</returns> /// <exception cref="ArgumentException">Thrown when an error occurs /// when parsing the string.</exception> public static IEnumerable <StringTableEntry> ParseFromCSV(string sourceText) { try { using (var stringReader = new System.IO.StringReader(sourceText)) using (var csv = new CsvReader(stringReader, GetConfiguration())) { /* * Do the below instead of GetRecords<T> due to * incompatibility with IL2CPP See more: * https://github.com/YarnSpinnerTool/YarnSpinner-Unity/issues/36#issuecomment-691489913 */ var records = new List <StringTableEntry>(); csv.Read(); csv.ReadHeader(); while (csv.Read()) { // Fetch values; if they can't be found, they'll be // defaults. csv.TryGetField <string>("language", out var language); csv.TryGetField <string>("lock", out var lockString); csv.TryGetField <string>("comment", out var comment); csv.TryGetField <string>("id", out var id); csv.TryGetField <string>("text", out var text); csv.TryGetField <string>("file", out var file); csv.TryGetField <string>("node", out var node); csv.TryGetField <string>("lineNumber", out var lineNumber); var record = new StringTableEntry { Language = language ?? string.Empty, ID = id ?? string.Empty, Text = text ?? string.Empty, File = file ?? string.Empty, Node = node ?? string.Empty, LineNumber = lineNumber ?? string.Empty, Lock = lockString ?? string.Empty, Comment = comment ?? string.Empty, }; records.Add(record); } return(records); } } catch (CsvHelperException e) { throw new System.ArgumentException($"Error reading CSV file: {e}"); } }
/// <summary> /// Returns an <see cref="IEnumerable"/> containing the string /// table entries for the base language for the specified Yarn /// script. /// </summary> /// <param name="serializedObject">A serialized object that /// represents a <see cref="YarnScript"/>.</param> /// <returns>The string table entries.</returns> private static IEnumerable <StringTableEntry> GetBaseLanguageStringsForSelectedObject(SerializedObject serializedObject) { var baseLanguageProperty = serializedObject.FindProperty("baseLanguage"); // Get the TextAsset that contains the base string table CSV TextAsset textAsset = baseLanguageProperty.objectReferenceValue as TextAsset; if (textAsset == null) { throw new System.NullReferenceException($"The base language table asset for {serializedObject.targetObject.name} is either null or not a TextAsset. Did the script fail to compile?"); } var baseLanguageTableText = textAsset.text; // Parse this CSV into StringTableEntry structs return(StringTableEntry.ParseFromCSV(baseLanguageTableText) .OrderBy(entry => entry.File) .ThenBy(entry => int.Parse(entry.LineNumber))); }
private void UpdateTargetObjectMappingTable() { var canvasText = serializedObject.targetObject as YarnLinesAsCanvasText; if (!(_yarnProgramProperty.objectReferenceValue is YarnProgram yarnProgram)) { // No program means no strings available, so clear it and // bail out if (canvasText.stringsToViews.Count != 0) { // Modify the dictionary directly, to make Unity // realise that the property is dirty _stringsToViewsProperty.FindPropertyRelative("keys").ClearArray(); // And clear the in-memory representation as well canvasText.stringsToViews.Clear(); serializedObject.ApplyModifiedProperties(); } return; } var stringTableAsset = yarnProgram.baseLocalisationStringTable.text; var stringIDs = StringTableEntry.ParseFromCSV(stringTableAsset) .Select(e => e.ID); var extraneousIDs = canvasText.stringsToViews.Keys.Except(stringIDs).ToList(); var missingIDs = stringIDs.Except(canvasText.stringsToViews.Keys).ToList(); foreach (var id in extraneousIDs) { canvasText.stringsToViews.Remove(id); } foreach (var id in missingIDs) { canvasText.stringsToViews.Add(id, null); } }
/// <summary> /// Writes a .csv file to disk at the path indicated by <paramref /// name="destination"/>, containing all of the lines found in the /// scripts referred to by <paramref name="yarnProjectImporter"/>. /// </summary> /// <remarks> /// The file generated is in a format ready to be added to the <see /// cref="YarnProjectImporter.languagesToSourceAssets"/> list. /// </remarks> /// <param name="yarnProjectImporter">The YarnProjectImporter to /// extract strings from.</param> /// <param name="destination">The path to write the file /// to.</param> /// <returns><see langword="true"/> if the file was written /// successfully, <see langword="false"/> otherwise.</returns> /// <exception cref="CsvHelper.CsvHelperException">Thrown when an /// error is encountered when generating the CSV data.</exception> /// <exception cref="IOException">Thrown when an error is /// encountered when writing the data to disk.</exception> internal static bool WriteStringsFile(string destination, YarnProjectImporter yarnProjectImporter) { // Perform a strings-only compilation to get a full strings // table, and generate the CSV. var stringTable = yarnProjectImporter.GenerateStringsTable(); // If there was an error, bail out here if (stringTable == null) { return(false); } // Convert the string tables to CSV... var outputCSV = StringTableEntry.CreateCSV(stringTable); // ...and write it to disk. File.WriteAllText(destination, outputCSV); return(true); }
/// <summary> /// Verifies the TextAsset referred to by <paramref /// name="destinationLocalizationAsset"/>, and updates it if /// necessary. /// </summary> /// <param name="baseLocalizationStrings">A collection of <see /// cref="StringTableEntry"/></param> /// <param name="language">The language that <paramref /// name="destinationLocalizationAsset"/> provides strings /// for.false</param> /// <param name="destinationLocalizationAsset">A TextAsset /// containing localized strings in CSV format.</param> /// <returns>Whether <paramref /// name="destinationLocalizationAsset"/> was modified.</returns> private static bool UpdateLocalizationFile(IEnumerable <StringTableEntry> baseLocalizationStrings, string language, TextAsset destinationLocalizationAsset) { var translatedStrings = StringTableEntry.ParseFromCSV(destinationLocalizationAsset.text); // Convert both enumerables to dictionaries, for easier lookup var baseDictionary = baseLocalizationStrings.ToDictionary(entry => entry.ID); var translatedDictionary = translatedStrings.ToDictionary(entry => entry.ID); // The list of line IDs present in each localisation var baseIDs = baseLocalizationStrings.Select(entry => entry.ID); var translatedIDs = translatedStrings.Select(entry => entry.ID); // The list of line IDs that are ONLY present in each // localisation var onlyInBaseIDs = baseIDs.Except(translatedIDs); var onlyInTranslatedIDs = translatedIDs.Except(baseIDs); // Tracks if the translated localisation needed modifications // (either new lines added, old lines removed, or changed lines // flagged) var modificationsNeeded = false; // Remove every entry whose ID is only present in the // translated set. This entry has been removed from the base // localization. foreach (var id in onlyInTranslatedIDs.ToList()) { translatedDictionary.Remove(id); modificationsNeeded = true; } // Conversely, for every entry that is only present in the base // localisation, we need to create a new entry for it. foreach (var id in onlyInBaseIDs) { StringTableEntry baseEntry = baseDictionary[id]; var newEntry = new StringTableEntry(baseEntry) { // Empty this text, so that it's apparent that a // translated version needs to be provided. Text = string.Empty, Language = language, }; translatedDictionary.Add(id, newEntry); modificationsNeeded = true; } // Finally, we need to check for any entries in the translated // localisation that: // 1. have the same line ID as one in the base, but // 2. have a different Lock (the hash of the text), which // indicates that the base text has changed. // First, get the list of IDs that are in both base and // translated, and then filter this list to any where the lock // values differ var outOfDateLockIDs = baseDictionary.Keys .Intersect(translatedDictionary.Keys) .Where(id => baseDictionary[id].Lock != translatedDictionary[id].Lock); // Now loop over all of these, and update our translated // dictionary to include a note that it needs attention foreach (var id in outOfDateLockIDs) { // Get the translated entry as it currently exists var entry = translatedDictionary[id]; // Include a note that this entry is out of date entry.Text = $"(NEEDS UPDATE) {entry.Text}"; // Update the lock to match the new one entry.Lock = baseDictionary[id].Lock; // Put this modified entry back in the table translatedDictionary[id] = entry; modificationsNeeded = true; } // We're all done! if (modificationsNeeded == false) { // No changes needed to be done to the translated string // table entries. Stop here. return(false); } // We need to produce a replacement CSV file for the translated // entries. var outputStringEntries = translatedDictionary.Values .OrderBy(entry => entry.File) .ThenBy(entry => int.Parse(entry.LineNumber)); var outputCSV = StringTableEntry.CreateCSV(outputStringEntries); // Write out the replacement text to this existing file, // replacing its existing contents var outputFile = AssetDatabase.GetAssetPath(destinationLocalizationAsset); File.WriteAllText(outputFile, outputCSV, System.Text.Encoding.UTF8); // Tell the asset database that the file needs to be reimported AssetDatabase.ImportAsset(outputFile); // Signal that the file was changed return(true); }
private void ImportYarn(AssetImportContext ctx) { var sourceText = File.ReadAllText(ctx.assetPath); string fileName = System.IO.Path.GetFileNameWithoutExtension(ctx.assetPath); Yarn.Program compiledProgram = null; IDictionary <string, Yarn.Compiler.StringInfo> stringTable = null; compilationErrorMessage = null; try { // Compile the source code into a compiled Yarn program (or // generate a parse error) compilationStatus = Yarn.Compiler.Compiler.CompileString(sourceText, fileName, out compiledProgram, out stringTable); isSuccesfullyCompiled = true; compilationErrorMessage = string.Empty; } catch (Yarn.Compiler.ParseException e) { isSuccesfullyCompiled = false; compilationErrorMessage = e.Message; ctx.LogImportError(e.Message); } // Create a container for storing the bytes if (programContainer == null) { programContainer = ScriptableObject.CreateInstance <YarnProgram>(); } byte[] compiledBytes = null; if (compiledProgram != null) { using (var memoryStream = new MemoryStream()) using (var outputStream = new Google.Protobuf.CodedOutputStream(memoryStream)) { // Serialize the compiled program to memory compiledProgram.WriteTo(outputStream); outputStream.Flush(); compiledBytes = memoryStream.ToArray(); } } programContainer.compiledProgram = compiledBytes; // Add this container to the imported asset; it will be // what the user interacts with in Unity ctx.AddObjectToAsset("Program", programContainer, YarnEditorUtility.GetYarnDocumentIconTexture()); ctx.SetMainObject(programContainer); if (stringTable?.Count > 0) { var lines = stringTable.Select(x => new StringTableEntry { ID = x.Key, Language = baseLanguageID, Text = x.Value.text, File = x.Value.fileName, Node = x.Value.nodeName, LineNumber = x.Value.lineNumber.ToString(), Lock = GetHashString(x.Value.text, 8), }).OrderBy(entry => int.Parse(entry.LineNumber)); var stringTableCSV = StringTableEntry.CreateCSV(lines); var textAsset = new TextAsset(stringTableCSV); textAsset.name = $"{fileName} ({baseLanguageID})"; ctx.AddObjectToAsset("Strings", textAsset); programContainer.baseLocalizationId = baseLanguageID; baseLanguage = textAsset; programContainer.localizations = localizations.Append(new YarnProgram.YarnTranslation(baseLanguageID, textAsset)).ToArray(); programContainer.baseLocalizationId = baseLanguageID; stringIDs = lines.Select(l => l.ID).ToArray(); } }
public override void OnImportAsset(AssetImportContext ctx) { #if YARNSPINNER_DEBUG UnityEngine.Profiling.Profiler.enabled = true; #endif var project = ScriptableObject.CreateInstance <YarnProject>(); project.name = Path.GetFileNameWithoutExtension(ctx.assetPath); // Start by creating the asset - no matter what, we need to // produce an asset, even if it doesn't contain valid Yarn // bytecode, so that other assets don't lose their references. ctx.AddObjectToAsset("Project", project); ctx.SetMainObject(project); foreach (var script in sourceScripts) { string path = AssetDatabase.GetAssetPath(script); if (string.IsNullOrEmpty(path)) { // This is, for some reason, not a valid script we can // use. Don't add a dependency on it. continue; } ctx.DependsOnSourceAsset(path); } // Parse declarations var localDeclarationsCompileJob = CompilationJob.CreateFromFiles(ctx.assetPath); localDeclarationsCompileJob.CompilationType = CompilationJob.Type.DeclarationsOnly; IEnumerable <Declaration> localDeclarations; compileError = null; try { var result = Compiler.Compiler.Compile(localDeclarationsCompileJob); localDeclarations = result.Declarations; } catch (ParseException e) { ctx.LogImportError($"Error in Yarn Project: {e.Message}"); compileError = $"Error in Yarn Project {ctx.assetPath}: {e.Message}"; return; } // Store these so that we can continue displaying them after // this import step, in case there are compile errors later. // We'll replace this with a more complete list later if // compilation succeeds. serializedDeclarations = localDeclarations .Where(decl => decl.DeclarationType == Declaration.Type.Variable) .Select(decl => new SerializedDeclaration(decl)).ToList(); // We're done processing this file - we've parsed it, and // pulled any information out of it that we need to. Now to // compile the scripts associated with this project. var scriptImporters = sourceScripts.Where(s => s != null).Select(s => AssetImporter.GetAtPath(AssetDatabase.GetAssetPath(s)) as YarnImporter); // First step: check to see if there's any parse errors in the // files. var scriptsWithParseErrors = scriptImporters.Where(script => script.isSuccessfullyParsed == false); if (scriptsWithParseErrors.Count() != 0) { // Parse errors! We can't continue. string failingScriptNameList = string.Join("\n", scriptsWithParseErrors.Select(script => script.assetPath)); compileError = $"Parse errors exist in the following files:\n{failingScriptNameList}"; return; } // Get paths to the scripts we're importing, and also map them // to their corresponding importer var pathsToImporters = scriptImporters.ToDictionary(script => script.assetPath, script => script); if (pathsToImporters.Count == 0) { return; // nothing further to do here } // We now now compile! var job = CompilationJob.CreateFromFiles(pathsToImporters.Keys); job.VariableDeclarations = localDeclarations; CompilationResult compilationResult; try { compilationResult = Compiler.Compiler.Compile(job); } catch (TypeException e) { ctx.LogImportError($"Error compiling: {e.Message}"); compileError = e.Message; var importer = pathsToImporters[e.FileName]; importer.parseErrorMessage = e.Message; EditorUtility.SetDirty(importer); return; } catch (ParseException e) { ctx.LogImportError(e.Message); compileError = e.Message; var importer = pathsToImporters[e.FileName]; importer.parseErrorMessage = e.Message; EditorUtility.SetDirty(importer); return; } if (compilationResult.Program == null) { ctx.LogImportError("Internal error: Failed to compile: resulting program was null, but compiler did not throw a parse exception."); return; } // Store _all_ declarations - both the ones in this // .yarnproject file, and the ones inside the .yarn files serializedDeclarations = localDeclarations .Concat(compilationResult.Declarations) .Where(decl => decl.DeclarationType == Declaration.Type.Variable) .Select(decl => new SerializedDeclaration(decl)).ToList(); // Clear error messages from all scripts - they've all passed // compilation foreach (var importer in pathsToImporters.Values) { importer.parseErrorMessage = null; } // Will we need to create a default localization? This variable // will be set to false if any of the languages we've // configured in languagesToSourceAssets is the default // language. var shouldAddDefaultLocalization = true; foreach (var pair in languagesToSourceAssets) { // Don't create a localization if the language ID was not // provided if (string.IsNullOrEmpty(pair.languageID)) { Debug.LogWarning($"Not creating a localization for {project.name} because the language ID wasn't provided. Add the language ID to the localization in the Yarn Project's inspector."); continue; } IEnumerable <StringTableEntry> stringTable; // Where do we get our strings from? If it's the default // language, we'll pull it from the scripts. If it's from // any other source, we'll pull it from the CSVs. if (pair.languageID == defaultLanguage) { // We'll use the program-supplied string table. stringTable = GenerateStringsTable(); // We don't need to add a default localization. shouldAddDefaultLocalization = false; } else { try { if (pair.stringsFile == null) { // We can't create this localization because we // don't have any data for it. Debug.LogWarning($"Not creating a localization for {pair.languageID} in the Yarn Project {project.name} because a text asset containing the strings wasn't found. Add a .csv file containing the translated lines to the Yarn Project's inspector."); continue; } stringTable = StringTableEntry.ParseFromCSV(pair.stringsFile.text); } catch (System.ArgumentException e) { Debug.LogWarning($"Not creating a localization for {pair.languageID} in the Yarn Project {project.name} because an error was encountered during text parsing: {e}"); continue; } } var newLocalization = ScriptableObject.CreateInstance <Localization>(); newLocalization.LocaleCode = pair.languageID; newLocalization.AddLocalizedStrings(stringTable); project.localizations.Add(newLocalization); newLocalization.name = pair.languageID; if (pair.assetsFolder != null) { var assetsFolderPath = AssetDatabase.GetAssetPath(pair.assetsFolder); if (assetsFolderPath == null) { // This was somehow not a valid reference? Debug.LogWarning($"Can't find assets for localization {pair.languageID} in {project.name} because a path for the provided assets folder couldn't be found."); } else { var stringIDsToAssets = FindAssetsForLineIDs(stringTable.Select(s => s.ID), assetsFolderPath); #if YARNSPINNER_DEBUG var stopwatch = System.Diagnostics.Stopwatch.StartNew(); #endif newLocalization.AddLocalizedObjects(stringIDsToAssets.AsEnumerable()); #if YARNSPINNER_DEBUG stopwatch.Stop(); Debug.Log($"Imported {stringIDsToAssets.Count()} assets for {project.name} \"{pair.languageID}\" in {stopwatch.ElapsedMilliseconds}ms"); #endif } } ctx.AddObjectToAsset("localization-" + pair.languageID, newLocalization); if (pair.languageID == defaultLanguage) { // If this is our default language, set it as such project.baseLocalization = newLocalization; } else { // This localization depends upon a source asset. Make // this asset get re-imported if this source asset was // modified ctx.DependsOnSourceAsset(AssetDatabase.GetAssetPath(pair.stringsFile)); } } if (shouldAddDefaultLocalization) { // We didn't add a localization for the default language. // Create one for it now. var developmentLocalization = ScriptableObject.CreateInstance <Localization>(); developmentLocalization.LocaleCode = defaultLanguage; var stringTableEntries = compilationResult.StringTable.Select(x => new StringTableEntry { ID = x.Key, Language = defaultLanguage, Text = x.Value.text, File = x.Value.fileName, Node = x.Value.nodeName, LineNumber = x.Value.lineNumber.ToString(), Lock = YarnImporter.GetHashString(x.Value.text, 8), }); developmentLocalization.AddLocalizedStrings(stringTableEntries); project.baseLocalization = developmentLocalization; project.localizations.Add(project.baseLocalization); developmentLocalization.name = $"Default ({defaultLanguage})"; ctx.AddObjectToAsset("default-language", developmentLocalization); } // Store the compiled program byte[] compiledBytes = null; using (var memoryStream = new MemoryStream()) using (var outputStream = new Google.Protobuf.CodedOutputStream(memoryStream)) { // Serialize the compiled program to memory compilationResult.Program.WriteTo(outputStream); outputStream.Flush(); compiledBytes = memoryStream.ToArray(); } project.compiledYarnProgram = compiledBytes; #if YARNSPINNER_DEBUG UnityEngine.Profiling.Profiler.enabled = false; #endif }