/// <summary>
/// Reads comma-separated value ata from <paramref
/// name="sourceText"/>, and produces a collection of <see
/// cref="StringTableEntry"/> structs.
/// </summary>
/// <param name="sourceText">A string containing CSV-formatted
/// data.</param>
/// <returns>The parsed collection of <see
/// cref="StringTableEntry"/> structs.</returns>
/// <throws cref="CsvHelperException">Thrown when an error occurs
/// when parsing the string.</throws>
public static IEnumerable <StringTableEntry> ParseFromCSV(string sourceText)
{
using (var stringReader = new System.IO.StringReader(sourceText))
using (var csv = new CsvReader(stringReader, GetConfiguration()))
{
/*
* Do the below instead of GetRecords<T> due to incompatibility with IL2CPP
* See more: https://github.com/JoshClose/CsvHelper/issues/1337
*/
var records = new List <StringTableEntry>();
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
var record = new StringTableEntry
{
Language = csv.GetField <string>("language"),
ID = csv.GetField <string>("id"),
Text = csv.GetField <string>("text"),
File = csv.GetField <string>("file"),
Node = csv.GetField <string>("node"),
LineNumber = csv.GetField <string>("lineNumber"),
Lock = csv.GetField <string>("lock"),
Comment = csv.GetField <string>("comment"),
};
records.Add(record);
}
return(records);
}
}
public StringTableEntry(StringTableEntry s)
{
ID = s.ID;
Text = s.Text;
File = s.File;
Node = s.Node;
LineNumber = s.LineNumber;
Lock = s.Lock;
Comment = s.Comment;
Language = s.Language;
}
/// <summary>
/// Reads comma-separated value ata from <paramref
/// name="sourceText"/>, and produces a collection of <see
/// cref="StringTableEntry"/> structs.
/// </summary>
/// <param name="sourceText">A string containing CSV-formatted
/// data.</param>
/// <returns>The parsed collection of <see
/// cref="StringTableEntry"/> structs.</returns>
/// <exception cref="ArgumentException">Thrown when an error occurs
/// when parsing the string.</exception>
public static IEnumerable <StringTableEntry> ParseFromCSV(string sourceText)
{
try
{
using (var stringReader = new System.IO.StringReader(sourceText))
using (var csv = new CsvReader(stringReader, GetConfiguration()))
{
/*
* Do the below instead of GetRecords<T> due to
* incompatibility with IL2CPP See more:
* https://github.com/YarnSpinnerTool/YarnSpinner-Unity/issues/36#issuecomment-691489913
*/
var records = new List <StringTableEntry>();
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
// Fetch values; if they can't be found, they'll be
// defaults.
csv.TryGetField <string>("language", out var language);
csv.TryGetField <string>("lock", out var lockString);
csv.TryGetField <string>("comment", out var comment);
csv.TryGetField <string>("id", out var id);
csv.TryGetField <string>("text", out var text);
csv.TryGetField <string>("file", out var file);
csv.TryGetField <string>("node", out var node);
csv.TryGetField <string>("lineNumber", out var lineNumber);
var record = new StringTableEntry
{
Language = language ?? string.Empty,
ID = id ?? string.Empty,
Text = text ?? string.Empty,
File = file ?? string.Empty,
Node = node ?? string.Empty,
LineNumber = lineNumber ?? string.Empty,
Lock = lockString ?? string.Empty,
Comment = comment ?? string.Empty,
};
records.Add(record);
}
return(records);
}
}
catch (CsvHelperException e)
{
throw new System.ArgumentException($"Error reading CSV file: {e}");
}
}
/// <summary>
/// Returns an <see cref="IEnumerable"/> containing the string
/// table entries for the base language for the specified Yarn
/// script.
/// </summary>
/// <param name="serializedObject">A serialized object that
/// represents a <see cref="YarnScript"/>.</param>
/// <returns>The string table entries.</returns>
private static IEnumerable <StringTableEntry> GetBaseLanguageStringsForSelectedObject(SerializedObject serializedObject)
{
var baseLanguageProperty = serializedObject.FindProperty("baseLanguage");
// Get the TextAsset that contains the base string table CSV
TextAsset textAsset = baseLanguageProperty.objectReferenceValue as TextAsset;
if (textAsset == null)
{
throw new System.NullReferenceException($"The base language table asset for {serializedObject.targetObject.name} is either null or not a TextAsset. Did the script fail to compile?");
}
var baseLanguageTableText = textAsset.text;
// Parse this CSV into StringTableEntry structs
return(StringTableEntry.ParseFromCSV(baseLanguageTableText)
.OrderBy(entry => entry.File)
.ThenBy(entry => int.Parse(entry.LineNumber)));
}
private void UpdateTargetObjectMappingTable()
{
var canvasText = serializedObject.targetObject as YarnLinesAsCanvasText;
if (!(_yarnProgramProperty.objectReferenceValue is YarnProgram yarnProgram))
{
// No program means no strings available, so clear it and
// bail out
if (canvasText.stringsToViews.Count != 0)
{
// Modify the dictionary directly, to make Unity
// realise that the property is dirty
_stringsToViewsProperty.FindPropertyRelative("keys").ClearArray();
// And clear the in-memory representation as well
canvasText.stringsToViews.Clear();
serializedObject.ApplyModifiedProperties();
}
return;
}
var stringTableAsset = yarnProgram.baseLocalisationStringTable.text;
var stringIDs = StringTableEntry.ParseFromCSV(stringTableAsset)
.Select(e => e.ID);
var extraneousIDs = canvasText.stringsToViews.Keys.Except(stringIDs).ToList();
var missingIDs = stringIDs.Except(canvasText.stringsToViews.Keys).ToList();
foreach (var id in extraneousIDs)
{
canvasText.stringsToViews.Remove(id);
}
foreach (var id in missingIDs)
{
canvasText.stringsToViews.Add(id, null);
}
}
/// <summary>
/// Writes a .csv file to disk at the path indicated by <paramref
/// name="destination"/>, containing all of the lines found in the
/// scripts referred to by <paramref name="yarnProjectImporter"/>.
/// </summary>
/// <remarks>
/// The file generated is in a format ready to be added to the <see
/// cref="YarnProjectImporter.languagesToSourceAssets"/> list.
/// </remarks>
/// <param name="yarnProjectImporter">The YarnProjectImporter to
/// extract strings from.</param>
/// <param name="destination">The path to write the file
/// to.</param>
/// <returns><see langword="true"/> if the file was written
/// successfully, <see langword="false"/> otherwise.</returns>
/// <exception cref="CsvHelper.CsvHelperException">Thrown when an
/// error is encountered when generating the CSV data.</exception>
/// <exception cref="IOException">Thrown when an error is
/// encountered when writing the data to disk.</exception>
internal static bool WriteStringsFile(string destination, YarnProjectImporter yarnProjectImporter)
{
// Perform a strings-only compilation to get a full strings
// table, and generate the CSV.
var stringTable = yarnProjectImporter.GenerateStringsTable();
// If there was an error, bail out here
if (stringTable == null)
{
return(false);
}
// Convert the string tables to CSV...
var outputCSV = StringTableEntry.CreateCSV(stringTable);
// ...and write it to disk.
File.WriteAllText(destination, outputCSV);
return(true);
}
/// <summary>
/// Verifies the TextAsset referred to by <paramref
/// name="destinationLocalizationAsset"/>, and updates it if
/// necessary.
/// </summary>
/// <param name="baseLocalizationStrings">A collection of <see
/// cref="StringTableEntry"/></param>
/// <param name="language">The language that <paramref
/// name="destinationLocalizationAsset"/> provides strings
/// for.false</param>
/// <param name="destinationLocalizationAsset">A TextAsset
/// containing localized strings in CSV format.</param>
/// <returns>Whether <paramref
/// name="destinationLocalizationAsset"/> was modified.</returns>
private static bool UpdateLocalizationFile(IEnumerable <StringTableEntry> baseLocalizationStrings, string language, TextAsset destinationLocalizationAsset)
{
var translatedStrings = StringTableEntry.ParseFromCSV(destinationLocalizationAsset.text);
// Convert both enumerables to dictionaries, for easier lookup
var baseDictionary = baseLocalizationStrings.ToDictionary(entry => entry.ID);
var translatedDictionary = translatedStrings.ToDictionary(entry => entry.ID);
// The list of line IDs present in each localisation
var baseIDs = baseLocalizationStrings.Select(entry => entry.ID);
var translatedIDs = translatedStrings.Select(entry => entry.ID);
// The list of line IDs that are ONLY present in each
// localisation
var onlyInBaseIDs = baseIDs.Except(translatedIDs);
var onlyInTranslatedIDs = translatedIDs.Except(baseIDs);
// Tracks if the translated localisation needed modifications
// (either new lines added, old lines removed, or changed lines
// flagged)
var modificationsNeeded = false;
// Remove every entry whose ID is only present in the
// translated set. This entry has been removed from the base
// localization.
foreach (var id in onlyInTranslatedIDs.ToList())
{
translatedDictionary.Remove(id);
modificationsNeeded = true;
}
// Conversely, for every entry that is only present in the base
// localisation, we need to create a new entry for it.
foreach (var id in onlyInBaseIDs)
{
StringTableEntry baseEntry = baseDictionary[id];
var newEntry = new StringTableEntry(baseEntry)
{
// Empty this text, so that it's apparent that a
// translated version needs to be provided.
Text = string.Empty,
Language = language,
};
translatedDictionary.Add(id, newEntry);
modificationsNeeded = true;
}
// Finally, we need to check for any entries in the translated
// localisation that:
// 1. have the same line ID as one in the base, but
// 2. have a different Lock (the hash of the text), which
// indicates that the base text has changed.
// First, get the list of IDs that are in both base and
// translated, and then filter this list to any where the lock
// values differ
var outOfDateLockIDs = baseDictionary.Keys
.Intersect(translatedDictionary.Keys)
.Where(id => baseDictionary[id].Lock != translatedDictionary[id].Lock);
// Now loop over all of these, and update our translated
// dictionary to include a note that it needs attention
foreach (var id in outOfDateLockIDs)
{
// Get the translated entry as it currently exists
var entry = translatedDictionary[id];
// Include a note that this entry is out of date
entry.Text = $"(NEEDS UPDATE) {entry.Text}";
// Update the lock to match the new one
entry.Lock = baseDictionary[id].Lock;
// Put this modified entry back in the table
translatedDictionary[id] = entry;
modificationsNeeded = true;
}
// We're all done!
if (modificationsNeeded == false)
{
// No changes needed to be done to the translated string
// table entries. Stop here.
return(false);
}
// We need to produce a replacement CSV file for the translated
// entries.
var outputStringEntries = translatedDictionary.Values
.OrderBy(entry => entry.File)
.ThenBy(entry => int.Parse(entry.LineNumber));
var outputCSV = StringTableEntry.CreateCSV(outputStringEntries);
// Write out the replacement text to this existing file,
// replacing its existing contents
var outputFile = AssetDatabase.GetAssetPath(destinationLocalizationAsset);
File.WriteAllText(outputFile, outputCSV, System.Text.Encoding.UTF8);
// Tell the asset database that the file needs to be reimported
AssetDatabase.ImportAsset(outputFile);
// Signal that the file was changed
return(true);
}
private void ImportYarn(AssetImportContext ctx)
{
var sourceText = File.ReadAllText(ctx.assetPath);
string fileName = System.IO.Path.GetFileNameWithoutExtension(ctx.assetPath);
Yarn.Program compiledProgram = null;
IDictionary <string, Yarn.Compiler.StringInfo> stringTable = null;
compilationErrorMessage = null;
try
{
// Compile the source code into a compiled Yarn program (or
// generate a parse error)
compilationStatus = Yarn.Compiler.Compiler.CompileString(sourceText, fileName, out compiledProgram, out stringTable);
isSuccesfullyCompiled = true;
compilationErrorMessage = string.Empty;
}
catch (Yarn.Compiler.ParseException e)
{
isSuccesfullyCompiled = false;
compilationErrorMessage = e.Message;
ctx.LogImportError(e.Message);
}
// Create a container for storing the bytes
if (programContainer == null)
{
programContainer = ScriptableObject.CreateInstance <YarnProgram>();
}
byte[] compiledBytes = null;
if (compiledProgram != null)
{
using (var memoryStream = new MemoryStream())
using (var outputStream = new Google.Protobuf.CodedOutputStream(memoryStream))
{
// Serialize the compiled program to memory
compiledProgram.WriteTo(outputStream);
outputStream.Flush();
compiledBytes = memoryStream.ToArray();
}
}
programContainer.compiledProgram = compiledBytes;
// Add this container to the imported asset; it will be
// what the user interacts with in Unity
ctx.AddObjectToAsset("Program", programContainer, YarnEditorUtility.GetYarnDocumentIconTexture());
ctx.SetMainObject(programContainer);
if (stringTable?.Count > 0)
{
var lines = stringTable.Select(x => new StringTableEntry
{
ID = x.Key,
Language = baseLanguageID,
Text = x.Value.text,
File = x.Value.fileName,
Node = x.Value.nodeName,
LineNumber = x.Value.lineNumber.ToString(),
Lock = GetHashString(x.Value.text, 8),
}).OrderBy(entry => int.Parse(entry.LineNumber));
var stringTableCSV = StringTableEntry.CreateCSV(lines);
var textAsset = new TextAsset(stringTableCSV);
textAsset.name = $"{fileName} ({baseLanguageID})";
ctx.AddObjectToAsset("Strings", textAsset);
programContainer.baseLocalizationId = baseLanguageID;
baseLanguage = textAsset;
programContainer.localizations = localizations.Append(new YarnProgram.YarnTranslation(baseLanguageID, textAsset)).ToArray();
programContainer.baseLocalizationId = baseLanguageID;
stringIDs = lines.Select(l => l.ID).ToArray();
}
}
public override void OnImportAsset(AssetImportContext ctx)
{
#if YARNSPINNER_DEBUG
UnityEngine.Profiling.Profiler.enabled = true;
#endif
var project = ScriptableObject.CreateInstance <YarnProject>();
project.name = Path.GetFileNameWithoutExtension(ctx.assetPath);
// Start by creating the asset - no matter what, we need to
// produce an asset, even if it doesn't contain valid Yarn
// bytecode, so that other assets don't lose their references.
ctx.AddObjectToAsset("Project", project);
ctx.SetMainObject(project);
foreach (var script in sourceScripts)
{
string path = AssetDatabase.GetAssetPath(script);
if (string.IsNullOrEmpty(path))
{
// This is, for some reason, not a valid script we can
// use. Don't add a dependency on it.
continue;
}
ctx.DependsOnSourceAsset(path);
}
// Parse declarations
var localDeclarationsCompileJob = CompilationJob.CreateFromFiles(ctx.assetPath);
localDeclarationsCompileJob.CompilationType = CompilationJob.Type.DeclarationsOnly;
IEnumerable <Declaration> localDeclarations;
compileError = null;
try
{
var result = Compiler.Compiler.Compile(localDeclarationsCompileJob);
localDeclarations = result.Declarations;
}
catch (ParseException e)
{
ctx.LogImportError($"Error in Yarn Project: {e.Message}");
compileError = $"Error in Yarn Project {ctx.assetPath}: {e.Message}";
return;
}
// Store these so that we can continue displaying them after
// this import step, in case there are compile errors later.
// We'll replace this with a more complete list later if
// compilation succeeds.
serializedDeclarations = localDeclarations
.Where(decl => decl.DeclarationType == Declaration.Type.Variable)
.Select(decl => new SerializedDeclaration(decl)).ToList();
// We're done processing this file - we've parsed it, and
// pulled any information out of it that we need to. Now to
// compile the scripts associated with this project.
var scriptImporters = sourceScripts.Where(s => s != null).Select(s => AssetImporter.GetAtPath(AssetDatabase.GetAssetPath(s)) as YarnImporter);
// First step: check to see if there's any parse errors in the
// files.
var scriptsWithParseErrors = scriptImporters.Where(script => script.isSuccessfullyParsed == false);
if (scriptsWithParseErrors.Count() != 0)
{
// Parse errors! We can't continue.
string failingScriptNameList = string.Join("\n", scriptsWithParseErrors.Select(script => script.assetPath));
compileError = $"Parse errors exist in the following files:\n{failingScriptNameList}";
return;
}
// Get paths to the scripts we're importing, and also map them
// to their corresponding importer
var pathsToImporters = scriptImporters.ToDictionary(script => script.assetPath, script => script);
if (pathsToImporters.Count == 0)
{
return; // nothing further to do here
}
// We now now compile!
var job = CompilationJob.CreateFromFiles(pathsToImporters.Keys);
job.VariableDeclarations = localDeclarations;
CompilationResult compilationResult;
try
{
compilationResult = Compiler.Compiler.Compile(job);
}
catch (TypeException e)
{
ctx.LogImportError($"Error compiling: {e.Message}");
compileError = e.Message;
var importer = pathsToImporters[e.FileName];
importer.parseErrorMessage = e.Message;
EditorUtility.SetDirty(importer);
return;
}
catch (ParseException e)
{
ctx.LogImportError(e.Message);
compileError = e.Message;
var importer = pathsToImporters[e.FileName];
importer.parseErrorMessage = e.Message;
EditorUtility.SetDirty(importer);
return;
}
if (compilationResult.Program == null)
{
ctx.LogImportError("Internal error: Failed to compile: resulting program was null, but compiler did not throw a parse exception.");
return;
}
// Store _all_ declarations - both the ones in this
// .yarnproject file, and the ones inside the .yarn files
serializedDeclarations = localDeclarations
.Concat(compilationResult.Declarations)
.Where(decl => decl.DeclarationType == Declaration.Type.Variable)
.Select(decl => new SerializedDeclaration(decl)).ToList();
// Clear error messages from all scripts - they've all passed
// compilation
foreach (var importer in pathsToImporters.Values)
{
importer.parseErrorMessage = null;
}
// Will we need to create a default localization? This variable
// will be set to false if any of the languages we've
// configured in languagesToSourceAssets is the default
// language.
var shouldAddDefaultLocalization = true;
foreach (var pair in languagesToSourceAssets)
{
// Don't create a localization if the language ID was not
// provided
if (string.IsNullOrEmpty(pair.languageID))
{
Debug.LogWarning($"Not creating a localization for {project.name} because the language ID wasn't provided. Add the language ID to the localization in the Yarn Project's inspector.");
continue;
}
IEnumerable <StringTableEntry> stringTable;
// Where do we get our strings from? If it's the default
// language, we'll pull it from the scripts. If it's from
// any other source, we'll pull it from the CSVs.
if (pair.languageID == defaultLanguage)
{
// We'll use the program-supplied string table.
stringTable = GenerateStringsTable();
// We don't need to add a default localization.
shouldAddDefaultLocalization = false;
}
else
{
try
{
if (pair.stringsFile == null)
{
// We can't create this localization because we
// don't have any data for it.
Debug.LogWarning($"Not creating a localization for {pair.languageID} in the Yarn Project {project.name} because a text asset containing the strings wasn't found. Add a .csv file containing the translated lines to the Yarn Project's inspector.");
continue;
}
stringTable = StringTableEntry.ParseFromCSV(pair.stringsFile.text);
}
catch (System.ArgumentException e)
{
Debug.LogWarning($"Not creating a localization for {pair.languageID} in the Yarn Project {project.name} because an error was encountered during text parsing: {e}");
continue;
}
}
var newLocalization = ScriptableObject.CreateInstance <Localization>();
newLocalization.LocaleCode = pair.languageID;
newLocalization.AddLocalizedStrings(stringTable);
project.localizations.Add(newLocalization);
newLocalization.name = pair.languageID;
if (pair.assetsFolder != null)
{
var assetsFolderPath = AssetDatabase.GetAssetPath(pair.assetsFolder);
if (assetsFolderPath == null)
{
// This was somehow not a valid reference?
Debug.LogWarning($"Can't find assets for localization {pair.languageID} in {project.name} because a path for the provided assets folder couldn't be found.");
}
else
{
var stringIDsToAssets = FindAssetsForLineIDs(stringTable.Select(s => s.ID), assetsFolderPath);
#if YARNSPINNER_DEBUG
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
#endif
newLocalization.AddLocalizedObjects(stringIDsToAssets.AsEnumerable());
#if YARNSPINNER_DEBUG
stopwatch.Stop();
Debug.Log($"Imported {stringIDsToAssets.Count()} assets for {project.name} \"{pair.languageID}\" in {stopwatch.ElapsedMilliseconds}ms");
#endif
}
}
ctx.AddObjectToAsset("localization-" + pair.languageID, newLocalization);
if (pair.languageID == defaultLanguage)
{
// If this is our default language, set it as such
project.baseLocalization = newLocalization;
}
else
{
// This localization depends upon a source asset. Make
// this asset get re-imported if this source asset was
// modified
ctx.DependsOnSourceAsset(AssetDatabase.GetAssetPath(pair.stringsFile));
}
}
if (shouldAddDefaultLocalization)
{
// We didn't add a localization for the default language.
// Create one for it now.
var developmentLocalization = ScriptableObject.CreateInstance <Localization>();
developmentLocalization.LocaleCode = defaultLanguage;
var stringTableEntries = compilationResult.StringTable.Select(x => new StringTableEntry
{
ID = x.Key,
Language = defaultLanguage,
Text = x.Value.text,
File = x.Value.fileName,
Node = x.Value.nodeName,
LineNumber = x.Value.lineNumber.ToString(),
Lock = YarnImporter.GetHashString(x.Value.text, 8),
});
developmentLocalization.AddLocalizedStrings(stringTableEntries);
project.baseLocalization = developmentLocalization;
project.localizations.Add(project.baseLocalization);
developmentLocalization.name = $"Default ({defaultLanguage})";
ctx.AddObjectToAsset("default-language", developmentLocalization);
}
// Store the compiled program
byte[] compiledBytes = null;
using (var memoryStream = new MemoryStream())
using (var outputStream = new Google.Protobuf.CodedOutputStream(memoryStream))
{
// Serialize the compiled program to memory
compilationResult.Program.WriteTo(outputStream);
outputStream.Flush();
compiledBytes = memoryStream.ToArray();
}
project.compiledYarnProgram = compiledBytes;
#if YARNSPINNER_DEBUG
UnityEngine.Profiling.Profiler.enabled = false;
#endif
}
