public void WriteRecord(StringTableEntry record, IEnumerable <CsvColumnType> columnTypes, CsvWriter writer)
{
int writtenTextColumns = 0;
foreach (var type in columnTypes)
{
switch (type)
{
case CsvColumnType.Id:
writer.WriteField(record.Key);
break;
case CsvColumnType.Speaker:
writer.WriteField(record.Speaker);
break;
case CsvColumnType.Comment:
writer.WriteField(record.Comment);
break;
case CsvColumnType.Fallback:
writer.WriteField(record.Fallback);
break;
case CsvColumnType.Text when writtenTextColumns < record.Text.Length:
writer.WriteField(record.Text[writtenTextColumns++]);
break;
default:
writer.WriteField(null);
break;
}
}
writer.NextRecord();
}
public override void PullMetadata(StringTableEntry entry, Comment metadata, string cellValue, string cellNote)
{
if (string.IsNullOrEmpty(cellValue))
{
if (metadata != null)
{
entry.RemoveMetadata(metadata);
}
// If the entry is empty then just remove the whole thing
if (string.IsNullOrEmpty(entry.Value) && entry.MetadataEntries.Count == 0)
{
entry.RemoveFromTable();
}
}
else
{
if (metadata != null)
{
metadata.CommentText = cellValue;
}
else
{
entry.AddMetadata(new Comment {
CommentText = cellValue
});
}
}
}
public override void ReadRow(SharedTableData.SharedTableEntry keyEntry, CsvReader reader)
{
if (m_ImportTable == null)
{
return;
}
// Get the entry or create one
StringTableEntry entry = m_ImportTable.GetEntry(keyEntry.Id) ?? m_ImportTable.AddEntry(keyEntry.Id, string.Empty);
// Get the metadata or add one
var metadata = entry.GetMetadata <MyCustomDataMetadata>();
if (metadata == null)
{
metadata = new MyCustomDataMetadata();
entry.AddMetadata(metadata);
}
if (m_SomeValueIndex != -1)
{
metadata.someValue = reader.GetField(m_SomeValueIndex);
}
if (m_SomeOtherValueIndex != -1)
{
metadata.someOtherValue = reader.GetField <int>(m_SomeOtherValueIndex);
}
}
public void AddString(string offsetName, string str, uint offsetLength = 4)
{
if (string.IsNullOrEmpty(offsetName))
{
return;
}
//if (string.IsNullOrEmpty(str))
//{
// WriteNulls(offsetLength);
// return;
//}
var tableEntry = new StringTableEntry(str);
bool newEntry = true;
// Make sure there aren't any existing entries of this string
foreach (var strEntry in strings)
{
if (strEntry.Data == str)
{
tableEntry = strEntry;
newEntry = false;
break;
}
}
// Add an offset to the string we're going to write into the string table later
AddOffset(offsetName, offsetLength);
tableEntry.OffsetNames.Add(offsetName);
if (newEntry)
{
strings.Add(tableEntry);
}
}
public void TestDeepCopyBytes()
{
string filename = Path.Combine(Environment.SystemDirectory, "atl.dll");
Assert.IsTrue(File.Exists(filename));
VersionResource existingVersionResource = new VersionResource();
existingVersionResource.Language = ResourceUtil.USENGLISHLANGID;
Console.WriteLine("Loading {0}", filename);
existingVersionResource.LoadFrom(filename);
DumpResource.Dump(existingVersionResource);
VersionResource versionResource = new VersionResource();
versionResource.FileVersion = existingVersionResource.FileVersion;
versionResource.ProductVersion = existingVersionResource.ProductVersion;
StringFileInfo existingVersionResourceStringFileInfo = (StringFileInfo)existingVersionResource["StringFileInfo"];
VarFileInfo existingVersionResourceVarFileInfo = (VarFileInfo)existingVersionResource["VarFileInfo"];
// copy string resources, data only
StringFileInfo stringFileInfo = new StringFileInfo();
versionResource["StringFileInfo"] = stringFileInfo;
{
Dictionary <string, StringTable> .Enumerator enumerator = existingVersionResourceStringFileInfo.Strings.GetEnumerator();
while (enumerator.MoveNext())
{
StringTable stringTable = new StringTable(enumerator.Current.Key);
stringFileInfo.Strings.Add(enumerator.Current.Key, stringTable);
Dictionary <string, StringTableEntry> .Enumerator resourceEnumerator = enumerator.Current.Value.Strings.GetEnumerator();
while (resourceEnumerator.MoveNext())
{
StringTableEntry stringResource = new StringTableEntry(resourceEnumerator.Current.Key);
stringResource.Value = resourceEnumerator.Current.Value.Value;
stringTable.Strings.Add(resourceEnumerator.Current.Key, stringResource);
}
}
}
// copy var resources, data only
VarFileInfo varFileInfo = new VarFileInfo();
versionResource["VarFileInfo"] = varFileInfo;
{
Dictionary <string, VarTable> .Enumerator enumerator = existingVersionResourceVarFileInfo.Vars.GetEnumerator();
while (enumerator.MoveNext())
{
VarTable varTable = new VarTable(enumerator.Current.Key);
varFileInfo.Vars.Add(enumerator.Current.Key, varTable);
Dictionary <UInt16, UInt16> .Enumerator translationEnumerator = enumerator.Current.Value.Languages.GetEnumerator();
while (translationEnumerator.MoveNext())
{
varTable.Languages.Add(translationEnumerator.Current.Key, translationEnumerator.Current.Value);
}
}
}
ByteUtils.CompareBytes(existingVersionResource.WriteAndGetBytes(), versionResource.WriteAndGetBytes());
}
public void YarnImporter_OnValidYarnFile_GetExpectedStrings()
{
string fileName = Path.GetRandomFileName();
List <StringTableEntry> expectedStrings = GetExpectedStrings(fileName);
string path = Application.dataPath + "/" + fileName + ".yarn";
createdFilePaths.Add(path);
File.WriteAllText(path, TestYarnScriptSource);
AssetDatabase.Refresh();
var result = AssetImporter.GetAtPath("Assets/" + fileName + ".yarn") as YarnImporter;
// Importing this Yarn script will have produced a CSV
// TextAsset containing the string table extracted from this
// script. Parse that - we'll check that it contains what we
// expect.
var generatedStringsTable = StringTableEntry.ParseFromCSV(result.baseLanguage.text);
// Simplify the results so that we can compare these string
// table entries based only on specific fields
System.Func <StringTableEntry, (string id, string text)> simplifier = e => (id : e.ID, text : e.Text);
var simpleResult = expectedStrings.Select(simplifier);
var simpleExpected = generatedStringsTable.Select(simplifier);
Assert.AreEqual(simpleExpected, simpleResult);
}
public static void UpdateContents(LocalizationDatabase database)
{
var allTextAssets = database.TrackedPrograms
.SelectMany(p => p.localizations)
.Select(localization => new
{
localization.languageName,
localization.text
});
foreach (var localization in database.Localizations)
{
if (localization == null)
{
// Ignore any null entries
continue;
}
localization.Clear();
}
foreach (var localizedTextAsset in allTextAssets)
{
string languageName = localizedTextAsset.languageName;
Localization localization;
try
{
localization = database.GetLocalization(languageName);
}
catch (KeyNotFoundException)
{
Debug.LogWarning($"{localizedTextAsset.text.name} is marked for language {languageName}, but this {nameof(LocalizationDatabase)} isn't set up for that language. TODO: offer a quick way to create one here.");
continue;
}
TextAsset textAsset = localizedTextAsset.text;
if (textAsset == null)
{
// A null reference. Early out here.
continue;
}
var records = StringTableEntry.ParseFromCSV(textAsset.text);
foreach (var record in records)
{
AddLineEntryToLocalization(localization, record);
}
EditorUtility.SetDirty(localization);
}
AssetDatabase.SaveAssets();
}
public void YarnImporterUtility_CanUpdateLocalizedCSVs_WhenBaseScriptChanges()
{
// Arrange:
// Set up a project with a Yarn file filled with tagged lines.
var project = SetUpProject(TestYarnScriptSource);
var importer = AssetImporter.GetAtPath(AssetDatabase.GetAssetPath(project)) as YarnProjectImporter;
var scriptPath = AssetDatabase.GetAssetPath(importer.sourceScripts[0]);
var destinationStringsFilePath = "Assets/" + Path.GetRandomFileName() + ".csv";
// Act:
// Create a .CSV File, and add it to the Yarn project.
YarnProjectUtility.WriteStringsFile(destinationStringsFilePath, importer);
createdFilePaths.Add(destinationStringsFilePath);
AssetDatabase.Refresh();
var stringsAsset = AssetDatabase.LoadAssetAtPath <TextAsset>(destinationStringsFilePath);
importer.languagesToSourceAssets.Add(new YarnProjectImporter.LanguageToSourceAsset {
languageID = "test", stringsFile = stringsAsset
});
EditorUtility.SetDirty(importer);
importer.SaveAndReimport();
// Capture the strings tables. We'll use them later.
var unmodifiedBaseStringsTable = importer.GenerateStringsTable();
var unmodifiedLocalizedStringsTable = StringTableEntry.ParseFromCSV(File.ReadAllText(destinationStringsFilePath));
// Next, modify the original source script.
File.WriteAllText(scriptPath, TestYarnScriptSourceModified);
AssetDatabase.Refresh();
// Finally, update the CSV.
LogAssert.Expect(LogType.Log, $"Updated the following files: {destinationStringsFilePath}");
YarnProjectUtility.UpdateLocalizationCSVs(importer);
AssetDatabase.Refresh();
// Doing it again should result in a no-op.
LogAssert.Expect(LogType.Log, "No files needed updating.");
YarnProjectUtility.UpdateLocalizationCSVs(importer);
// Capture the updated strings tables, so we can compare them.
var modifiedBaseStringsTable = importer.GenerateStringsTable();
var modifiedLocalizedStringsTable = StringTableEntry.ParseFromCSV(File.ReadAllText(destinationStringsFilePath));
// Assert: verify the base language string table contains the
// string table entries we expect, verify the localized string
// table contains the string table entries we expect
System.Func <StringTableEntry, string> CompareIDs = t => t.ID;
System.Func <StringTableEntry, string> CompareLocks = t => t.Lock;
var tests = new[] {
public StringTableEntry GetStringTableEntry(ushort index)
{
var ste = new StringTableEntry();
/* String Table Header */
ste.UnknownIndex = (ushort)FSHelpers.Read16(_dataCopy, (int)_stringTableOffset + 0x4 + (index * 0x4));
ste.StringOffset = (ushort)FSHelpers.Read16(_dataCopy, (int)_stringTableOffset + 0x6 + (index * 0x4));
return(ste);
}
public void WriteStringTableEntry(StringTableEntry ste)
{
if (StringTable != null)
{
StringTable.WriteStringTableEntry(this, ste);
}
else
{
WriteString(ste.GetString());
}
}
/// <summary>
/// Creates a new localization CSV file for the specified object, for
/// the given language.
/// </summary>
/// <param name="serializedObject">A serialized object that represents
/// a <see cref="YarnImporter"/>.</param>
/// <param name="language">The language to generate a localization CSV
/// for.</param>
internal static void CreateLocalizationForLanguageInProgram(SerializedObject serializedObject, string language)
{
if (serializedObject.isEditingMultipleObjects)
{
Debug.LogError($"{nameof(CreateLocalizationForLanguageInProgram)} was called, but multiple objects were selected. Select a single object and try again.");
return;
}
IEnumerable <StringTableEntry> baseLanguageStrings = GetBaseLanguageStringsForSelectedObject(serializedObject);
// Produce a new version of these string entries, but with
// different languages
var translatedStringTable = baseLanguageStrings.Select(s =>
{
// Copy the entry, but mark the new language
return(new StringTableEntry(s)
{
Language = language
});
});
// Convert this new list to a CSV
string generatedCSV;
try
{
generatedCSV = StringTableEntry.CreateCSV(translatedStringTable);
}
catch (CsvHelper.CsvHelperException e)
{
Debug.LogError($"Error creating {language} CSV: {e}");
return;
}
// Write out this CSV to a file
var path = AssetDatabase.GetAssetPath(serializedObject.targetObject);
var directory = Path.GetDirectoryName(path);
var csvFileName = $"{Path.GetFileNameWithoutExtension(path)} ({language}).csv";
var destinationPath = Path.Combine(directory, csvFileName);
destinationPath = AssetDatabase.GenerateUniqueAssetPath(destinationPath);
File.WriteAllText(destinationPath, generatedCSV);
// Import this file as a TextAsset object
AssetDatabase.ImportAsset(destinationPath);
var newTextAsset = AssetDatabase.LoadAssetAtPath <TextAsset>(destinationPath);
// Store this TextAsset in the importer, at the end of the
// localizations array
ArrayUtility.Add(ref (serializedObject.targetObject as YarnImporter).localizations, new YarnProgram.YarnTranslation(language, newTextAsset));
// Mark that we just changed the target object
EditorUtility.SetDirty(serializedObject.targetObject);
}
public override void StringTableUpdated(string stringTableName, StringTableEntry stringTableEntry)
{
if(stringTableName == "userinfo")
{
if(stringTableEntry.Data != null)
{
PlayerInfo playerInfo = new PlayerInfo(stringTableEntry.Data);
m_Players.Add(stringTableEntry.Index, playerInfo);
}
}
}
public override PushFields PushFields => PushFields.ValueAndNote; // For our example we use both value and note.
public override void PullMetadata(StringTableEntry entry, MyCustomDataMetadata metadata, string cellValue, string cellNote)
{
// Metadata will be null if the entry does not already contain any.
if (metadata == null)
{
metadata = new MyCustomDataMetadata();
entry.AddMetadata(metadata);
}
metadata.someValue = cellValue;
metadata.someNoteValue = cellNote;
}
public void ReadStringTableEntry(out StringTableEntry ste)
{
if (StringTable != null)
{
ste = StringTable.ReadStringTableEntry(this);
}
else
{
ReadString(out string buf);
ste = new StringTableEntry();
ste.Set(buf.Contains("\0") ? buf.Substring(0, buf.IndexOf('\0')) : buf);
}
}
public void Add(string stringTableName, StringTableEntry stringTableEntry)
{
StringTable stringTable;
if (m_Values.TryGetValue(stringTableName, out stringTable))
{
stringTable.Entries[stringTableEntry.Name] = stringTableEntry;
m_Demo.GameObservers.StringTableUpdated(stringTableName, stringTableEntry);
}
else
{
Console.WriteLine("Unable to Find StringTable");
}
}
void GeneratePushRequests(int sheetId, StringTableCollection collection, IList <SheetColumn> columnMapping, List <Request> requestsToSend, ITaskReporter reporter)
{
// Prepare the column requests.
// We use a request per column as its possible that some columns in the sheet will be preserved and we don't want to write over them.
reporter?.ReportProgress("Generating column headers", 0);
var columnSheetRequests = new List <PushColumnSheetRequest>(columnMapping.Count);
foreach (var col in columnMapping)
{
var colRequest = new PushColumnSheetRequest(sheetId, col);
columnSheetRequests.Add(colRequest);
colRequest.Column.PushBegin(collection);
colRequest.Column.PushHeader(collection, out var header, out var note);
colRequest.AddHeader(header, note);
}
var stringTables = collection.StringTables;
var tableEntries = new StringTableEntry[stringTables.Count];
reporter?.ReportProgress("Generating push data", 0.1f);
foreach (var keyEntry in collection.SharedData.Entries)
{
// Collect the table entry data.
for (int i = 0; i < stringTables.Count; ++i)
{
tableEntries[i] = stringTables[i].GetEntry(keyEntry.Id);
}
// Now process each sheet column so they can update their requests.
foreach (var colReq in columnSheetRequests)
{
if (tableEntries[0].SharedEntry.Metadata.HasMetadata <ExcludeEntryFromExport>())
{
continue;
}
colReq.Column.PushCellData(keyEntry, tableEntries, out var value, out var note);
colReq.AddRow(value, note);
}
}
foreach (var col in columnSheetRequests)
{
col.Column.PushEnd();
requestsToSend.AddRange(col.Requests);
}
}
public int GetStringOffset(string str)
{
// HACK
if (str == null)
{
str = "";
}
StringTableEntry result;
if (!StringTable.TryGetValue(str, out result))
{
result = new StringTableEntry(this, str);
StringTable.Add(str, result);
}
return(result.Offset);
}
/// <summary>
/// Returns an <see cref="IEnumerable"/> containing the string table
/// entries for the base language for the specified Yarn script.
/// </summary>
/// <param name="serializedObject">A serialized object that represents
/// a <see cref="YarnProgram"/>.</param>
/// <returns>The string table entries.</returns>
private static IEnumerable <StringTableEntry> GetBaseLanguageStringsForSelectedObject(SerializedObject serializedObject)
{
var baseLanguageProperty = serializedObject.FindProperty("baseLanguage");
// Get the TextAsset that contains the base string table CSV
TextAsset textAsset = baseLanguageProperty.objectReferenceValue as TextAsset;
if (textAsset == null)
{
throw new System.NullReferenceException($"The base language table asset for {serializedObject.targetObject.name} is either null or not a TextAsset. Did the script fail to compile?");
}
var baseLanguageTableText = textAsset.text;
// Parse this CSV into StringTableEntry structs
return(StringTableEntry.ParseFromCSV(baseLanguageTableText)
.OrderBy(entry => entry.File)
.ThenBy(entry => int.Parse(entry.LineNumber)));
}
private StringTableEntry StringTableElement()
{
var ident = Expression(false);
if (Lexer.Peek() == Tokens.Comma)
{
PopExpected(Tokens.Comma);
}
var text = PopExpected(Tokens.String);
var ste = new StringTableEntry
{
Context = ident.Context,
Identifier = ident,
TextValue = text
};
return(ste);
}
public void YarnImporterUtility_CanUpdateLocalizedCSVs_WhenBaseScriptChanges()
{
// Arrange: Import a yarn script and create a localization
// database for it, create an alternate localization for it
string fileName = Path.GetRandomFileName();
string path = Path.Combine("Assets", fileName + ".yarn");
createdFilePaths.Add(path);
File.WriteAllText(path, TestYarnScriptSource);
AssetDatabase.Refresh();
var importer = AssetImporter.GetAtPath(path) as YarnImporter;
var importerSerializedObject = new SerializedObject(importer);
YarnImporterUtility.CreateNewLocalizationDatabase(importerSerializedObject);
var localizationDatabaseSerializedObject = new SerializedObject(importer.localizationDatabase);
LocalizationDatabaseUtility.CreateLocalizationWithLanguage(localizationDatabaseSerializedObject, AlternateLocaleCode);
YarnImporterUtility.CreateLocalizationForLanguageInProgram(importerSerializedObject, AlternateLocaleCode);
var unmodifiedBaseStringsTable = StringTableEntry.ParseFromCSV((importerSerializedObject.targetObject as YarnImporter).baseLanguage.text);
var unmodifiedLocalizedStringsTable = StringTableEntry.ParseFromCSV((importerSerializedObject.targetObject as YarnImporter).localizations.First(l => l.languageName == AlternateLocaleCode).text.text);
// Act: modify the imported script so that lines are added,
// changed and deleted, and then update the localized CSV
// programmatically
File.WriteAllText(path, TestYarnScriptSourceModified);
AssetDatabase.Refresh();
YarnImporterUtility.UpdateLocalizationCSVs(importerSerializedObject);
var modifiedBaseStringsTable = StringTableEntry.ParseFromCSV((importerSerializedObject.targetObject as YarnImporter).baseLanguage.text);
var modifiedLocalizedStringsTable = StringTableEntry.ParseFromCSV((importerSerializedObject.targetObject as YarnImporter).localizations.First(l => l.languageName == AlternateLocaleCode).text.text);
// Assert: verify the base language string table contains the
// string table entries we expect, verify the localized string
// table contains the string table entries we expect
System.Func <StringTableEntry, string> CompareIDs = t => t.ID;
System.Func <StringTableEntry, string> CompareLocks = t => t.Lock;
var tests = new[] {
/// <summary>
/// Writes a .csv file to disk at the path indicated by <paramref
/// name="destination"/>, containing all of the lines found in the
/// scripts referred to by <paramref name="yarnProjectImporter"/>.
/// </summary>
/// <remarks>
/// The file generated is in a format ready to be added to the <see
/// cref="YarnProjectImporter.languagesToSourceAssets"/> list.
/// </remarks>
/// <param name="yarnProjectImporter">The YarnProjectImporter to
/// extract strings from.</param>
/// <param name="destination">The path to write the file
/// to.</param>
/// <returns><see langword="true"/> if the file was written
/// successfully, <see langword="false"/> otherwise.</returns>
/// <exception cref="CsvHelper.CsvHelperException">Thrown when an
/// error is encountered when generating the CSV data.</exception>
/// <exception cref="IOException">Thrown when an error is
/// encountered when writing the data to disk.</exception>
internal static bool WriteStringsFile(string destination, YarnProjectImporter yarnProjectImporter)
{
// Perform a strings-only compilation to get a full strings
// table, and generate the CSV.
var stringTable = yarnProjectImporter.GenerateStringsTable();
// If there was an error, bail out here
if (stringTable == null)
{
return(false);
}
// Convert the string tables to CSV...
var outputCSV = StringTableEntry.CreateCSV(stringTable);
// ...and write it to disk.
File.WriteAllText(destination, outputCSV);
return(true);
}
public void TestDoubleNullTerminator()
{
StringTableEntry sr = new StringTableEntry("dummy");
string guid = Guid.NewGuid().ToString();
sr.Value = guid;
Assert.AreEqual(guid + '\0', sr.Value);
Assert.AreEqual(guid, sr.StringValue);
Assert.AreEqual(guid.Length + 1, sr.Header.wValueLength);
sr.Value = guid + '\0';
Assert.AreEqual(guid + '\0', sr.Value);
Assert.AreEqual(guid, sr.StringValue);
Assert.AreEqual(guid.Length + 1, sr.Header.wValueLength);
sr.Value = guid + "\0\0";
Assert.AreEqual(guid + "\0\0", sr.Value);
Assert.AreEqual(guid + '\0', sr.StringValue);
Assert.AreEqual(guid.Length + 2, sr.Header.wValueLength);
sr.Value = '\0' + guid;
Assert.AreEqual('\0' + guid + '\0', sr.Value);
Assert.AreEqual('\0' + guid, sr.StringValue);
Assert.AreEqual(guid.Length + 2, sr.Header.wValueLength);
}
public void WriteRecord(StringTableEntry record, CsvColumnType[] columnTypes, TextWriter writer)
{
int writtenTextColumns = 0;
bool firstColumn = true;
foreach (var type in columnTypes)
{
if (!firstColumn)
{
writer.Write(',');
}
switch (type)
{
case CsvColumnType.Id:
WriteField(record.Key, writer);
break;
case CsvColumnType.Speaker when record.Speaker != null:
WriteField(record.Speaker, writer);
break;
case CsvColumnType.Comment when record.Comment != null:
WriteField(record.Comment, writer);
break;
case CsvColumnType.Fallback when record.Fallback != null:
WriteField(record.Fallback, writer);
break;
case CsvColumnType.Text when writtenTextColumns < record.Text.Length:
WriteField(record.Text[writtenTextColumns++], writer);
break;
}
firstColumn = false;
}
writer.WriteLine();
}
private static void AddLineEntryToLocalization(Localization localization, StringTableEntry entry)
{
// Add the entry for this line's text
try
{
localization.AddLocalizedString(entry.ID, entry.Text);
}
catch (ArgumentException)
{
// An ArgumentException will be thrown by the internal
// dictionary of the Localization if there's a duplicate key
Debug.LogError($"Can't add line {entry.ID} (\"{entry.Text}\") because this localization already has an entry for this line. Are you trying to add two copies of the same file?");
return;
}
if (localization.AssetSourceFolder == null)
{
// No asset source folder specified, so don't go looking for
// assets to add
return;
}
// Remove "line:" from the ID before looking for assets
var id = entry.ID.Replace("line:", "");
// Look inside assetSourceFolder for any asset that includes this
// line ID
var assetGUIDs = AssetDatabase.FindAssets(id, new[] { AssetDatabase.GetAssetPath(localization.AssetSourceFolder) });
if (assetGUIDs.Length == 0)
{
// We didn't find any asset with this name, so early out here
return;
}
if (assetGUIDs.Length > 1)
{
string assetPaths = string.Join(", ", assetGUIDs.Select(g => AssetDatabase.GUIDToAssetPath(g)));
Debug.LogWarning($"Multiple assets found for line {id} in language {localization.LocaleCode}: {assetPaths}");
}
// Select the GUID for the first asset that we've found for this
// line
var assetGUID = assetGUIDs[0];
if (ProjectSettings.AddressableVoiceOverAudioClips)
{
#if ADDRESSABLES
// Add the asset reference
AddAddressableAssetReferenceToLocalization(localization, entry, assetGUID);
#else
// Something's gone wrong if we're in this situation
throw new System.InvalidOperationException($"Internal error: {nameof(ProjectSettings.AddressableVoiceOverAudioClips)} returned true, but ADDRESSABLES was not defined. Please file a bug.");
#endif
}
else
{
// Add the asset reference directly
AddAssetReferenceToLocalization(localization, entry, assetGUID);
}
}
public void TestDeleteDeepCopyAndSaveVersionResource(string binaryName)
{
Uri uri = new Uri(Assembly.GetExecutingAssembly().CodeBase);
string filename = Path.Combine(Path.GetDirectoryName(HttpUtility.UrlDecode(uri.AbsolutePath)), "Binaries\\" + binaryName);
Assert.IsTrue(File.Exists(filename));
string targetFilename = Path.Combine(Path.GetTempPath(), binaryName);
File.Copy(filename, targetFilename, true);
Console.WriteLine(targetFilename);
VersionResource existingVersionResource = new VersionResource();
existingVersionResource.Language = ResourceUtil.USENGLISHLANGID;
existingVersionResource.LoadFrom(targetFilename);
DumpResource.Dump(existingVersionResource);
existingVersionResource.DeleteFrom(targetFilename);
VersionResource versionResource = new VersionResource();
versionResource.FileVersion = existingVersionResource.FileVersion;
versionResource.ProductVersion = existingVersionResource.ProductVersion;
StringFileInfo existingVersionResourceStringFileInfo = (StringFileInfo)existingVersionResource["StringFileInfo"];
VarFileInfo existingVersionResourceVarFileInfo = (VarFileInfo)existingVersionResource["VarFileInfo"];
// copy string resources, data only
StringFileInfo stringFileInfo = new StringFileInfo();
versionResource["StringFileInfo"] = stringFileInfo;
{
Dictionary <string, StringTable> .Enumerator enumerator = existingVersionResourceStringFileInfo.Strings.GetEnumerator();
while (enumerator.MoveNext())
{
StringTable stringTable = new StringTable(enumerator.Current.Key);
stringFileInfo.Strings.Add(enumerator.Current.Key, stringTable);
Dictionary <string, StringTableEntry> .Enumerator resourceEnumerator = enumerator.Current.Value.Strings.GetEnumerator();
while (resourceEnumerator.MoveNext())
{
StringTableEntry stringResource = new StringTableEntry(resourceEnumerator.Current.Key);
stringResource.Value = resourceEnumerator.Current.Value.Value;
stringTable.Strings.Add(resourceEnumerator.Current.Key, stringResource);
}
}
}
// copy var resources, data only
VarFileInfo varFileInfo = new VarFileInfo();
versionResource["VarFileInfo"] = varFileInfo;
{
Dictionary <string, VarTable> .Enumerator enumerator = existingVersionResourceVarFileInfo.Vars.GetEnumerator();
while (enumerator.MoveNext())
{
VarTable varTable = new VarTable(enumerator.Current.Key);
varFileInfo.Vars.Add(enumerator.Current.Key, varTable);
Dictionary <UInt16, UInt16> .Enumerator translationEnumerator = enumerator.Current.Value.Languages.GetEnumerator();
while (translationEnumerator.MoveNext())
{
varTable.Languages.Add(translationEnumerator.Current.Key, translationEnumerator.Current.Value);
}
}
}
versionResource.SaveTo(targetFilename);
Console.WriteLine("Reloading {0}", targetFilename);
VersionResource newVersionResource = new VersionResource();
newVersionResource.LoadFrom(targetFilename);
DumpResource.Dump(newVersionResource);
AssertOneVersionResource(targetFilename);
Assert.AreEqual(newVersionResource.FileVersion, versionResource.FileVersion);
Assert.AreEqual(newVersionResource.ProductVersion, versionResource.ProductVersion);
StringFileInfo testedStringFileInfo = (StringFileInfo)newVersionResource["StringFileInfo"];
foreach (KeyValuePair <string, StringTableEntry> stringResource in testedStringFileInfo.Default.Strings)
{
Assert.AreEqual(stringResource.Value.Value, stringFileInfo[stringResource.Key]);
}
VarFileInfo newVarFileInfo = (VarFileInfo)newVersionResource["VarFileInfo"];
foreach (KeyValuePair <UInt16, UInt16> varResource in newVarFileInfo.Default.Languages)
{
Assert.AreEqual(varResource.Value, varFileInfo[varResource.Key]);
}
}
public void TestDeepCopyBytes(string binaryName)
{
Uri uri = new Uri(Assembly.GetExecutingAssembly().CodeBase);
string filename = Path.Combine(Path.GetDirectoryName(HttpUtility.UrlDecode(uri.AbsolutePath)), "Binaries\\" + binaryName);
Assert.IsTrue(File.Exists(filename));
VersionResource existingVersionResource = new VersionResource();
existingVersionResource.Language = ResourceUtil.USENGLISHLANGID;
Console.WriteLine("Loading {0}", filename);
existingVersionResource.LoadFrom(filename);
DumpResource.Dump(existingVersionResource);
VersionResource versionResource = new VersionResource();
versionResource.FileVersion = existingVersionResource.FileVersion;
versionResource.ProductVersion = existingVersionResource.ProductVersion;
StringFileInfo existingVersionResourceStringFileInfo = (StringFileInfo)existingVersionResource["StringFileInfo"];
VarFileInfo existingVersionResourceVarFileInfo = (VarFileInfo)existingVersionResource["VarFileInfo"];
// copy string resources, data only
StringFileInfo stringFileInfo = new StringFileInfo();
versionResource["StringFileInfo"] = stringFileInfo;
{
Dictionary <string, StringTable> .Enumerator enumerator = existingVersionResourceStringFileInfo.Strings.GetEnumerator();
while (enumerator.MoveNext())
{
StringTable stringTable = new StringTable(enumerator.Current.Key);
stringFileInfo.Strings.Add(enumerator.Current.Key, stringTable);
Dictionary <string, StringTableEntry> .Enumerator resourceEnumerator = enumerator.Current.Value.Strings.GetEnumerator();
while (resourceEnumerator.MoveNext())
{
StringTableEntry stringResource = new StringTableEntry(resourceEnumerator.Current.Key);
stringResource.Value = resourceEnumerator.Current.Value.Value;
stringTable.Strings.Add(resourceEnumerator.Current.Key, stringResource);
}
}
}
// copy var resources, data only
VarFileInfo varFileInfo = new VarFileInfo();
versionResource["VarFileInfo"] = varFileInfo;
{
Dictionary <string, VarTable> .Enumerator enumerator = existingVersionResourceVarFileInfo.Vars.GetEnumerator();
while (enumerator.MoveNext())
{
VarTable varTable = new VarTable(enumerator.Current.Key);
varFileInfo.Vars.Add(enumerator.Current.Key, varTable);
Dictionary <UInt16, UInt16> .Enumerator translationEnumerator = enumerator.Current.Value.Languages.GetEnumerator();
while (translationEnumerator.MoveNext())
{
varTable.Languages.Add(translationEnumerator.Current.Key, translationEnumerator.Current.Value);
}
}
}
ByteUtils.CompareBytes(existingVersionResource.WriteAndGetBytes(), versionResource.WriteAndGetBytes());
}
private void Update(BitStream bitStream, string tableName, int tableFlags, int numberOfEntries, int maxEntries, int userDataSize, int userDataSizeInBits, bool userDataIsFixedSize)
{
bool encodeUsingDictionaries = bitStream.ReadBit();
if (encodeUsingDictionaries)
{
Console.WriteLine("Cannot Decode StringTable Update");
return;
}
List<String> entryHistory = new List<String>();
int lastEntriesIndex = -1;
for (int i = 0; i < numberOfEntries; i++)
{
int entryIndex = lastEntriesIndex + 1;
int entryLength = (int)Math.Log(maxEntries, 2);
if (!bitStream.ReadBit())
{
entryIndex = (int)bitStream.ReadBits(entryLength);
}
lastEntriesIndex = entryIndex;
if (entryIndex < 0 || entryIndex >= maxEntries)
{
Console.WriteLine("Invalid StringTableEntry Index");
return;
}
string entryName = "";
if (bitStream.ReadBit())
{
bool substringCheck = bitStream.ReadBit();
if (substringCheck)
{
int index = (int)bitStream.ReadBits(5);
int bytesToCopy = (int)bitStream.ReadBits(5);
entryName = entryHistory[index].Substring(0, bytesToCopy);
entryName += bitStream.ReadString();
}
else
{
entryName = bitStream.ReadString();
}
}
if (entryHistory.Count > 31)
{
entryHistory.RemoveAt(0);
}
entryHistory.Add(entryName);
byte[] entryData = null;
if (bitStream.ReadBit())
{
if (userDataIsFixedSize)
{
entryData = bitStream.ReadBitsToArray(userDataSizeInBits);
}
else
{
int bytesToRead = (int)bitStream.ReadBits(14);
entryData = bitStream.ReadBytes(bytesToRead);
}
}
StringTableEntry stringTableEntry = new StringTableEntry(entryName, entryIndex, entryData);
Add(tableName, stringTableEntry);
}
}
public int GetStringOffset(string str)
{
// HACK
if (str == null)
str = "";
StringTableEntry result;
if (!StringTable.TryGetValue(str, out result)) {
result = new StringTableEntry(this, str);
StringTable.Add(str, result);
}
return result.Offset;
}
private bool ProcessTick()
{
DemoCommand demoCommand = (DemoCommand)m_BitStream.ReadByte();
int tickNumber = m_BitStream.ReadInt32();
int playerSlot = m_BitStream.ReadByte();
switch (demoCommand)
{
case DemoCommand.DataTables:
{
int length = m_BitStream.ReadInt32();
byte[] data = m_BitStream.ReadBytes(length);
BitStream bitStream = new BitStream(data);
while (true)
{
int dataTableType = bitStream.ReadVarint32();
int dataTablelength = bitStream.ReadVarint32();
byte[] dataTableData = bitStream.ReadBytes(dataTablelength);
DataTable dataTable = new DataTable(this.DataTables.Count, dataTableData);
if (dataTable.IsEnd)
{
break;
}
else
{
this.DataTables.Add(dataTable);
}
}
int count = bitStream.ReadShort();
for (int i = 0; i < count; i++)
{
int serverClassID = bitStream.ReadShort();
string serverClassName = bitStream.ReadString();
string dataTableName = bitStream.ReadString();
DataTable dataTable = this.DataTables.GetDataTable(dataTableName);
ServerClass serverClass = new ServerClass(serverClassID, serverClassName, dataTable.ID, dataTable.Name);
this.ServerClasses.Add(serverClass);
}
this.ServerClasses.ProcessAllServerClasses();
break;
}
case DemoCommand.Packet:
case DemoCommand.SignOn:
{
m_BitStream.SeekBytes(160, SeekOrigin.Current);
int length = m_BitStream.ReadInt32();
byte[] data = m_BitStream.ReadBytes(length);
BitStream packetReader = new BitStream(data);
while (packetReader.PositionInBytes < packetReader.LengthInBytes)
{
int packetID = packetReader.ReadVarint32();
int packetLength = packetReader.ReadVarint32();
byte[] packetData = packetReader.ReadBytes(packetLength);
PacketType packetType;
if (!Enum.TryParse<PacketType>(packetID.ToString(), out packetType))
{
Console.WriteLine("Unknown Packet Type: {0}", packetID);
break;
}
this.PacketHandlers.OnPacketReceived(packetType, packetData);
}
break;
}
case DemoCommand.Stop:
{
return true;
}
case DemoCommand.StringTables:
{
int length = m_BitStream.ReadInt32();
byte[] data = m_BitStream.ReadBytes(length);
BitStream bitStream = new BitStream(data);
int stringTableCount = bitStream.ReadByte();
for (int i = 0; i < stringTableCount; i++)
{
string stringTableName = bitStream.ReadString();
int entryCount = bitStream.ReadUShort();
for (int j = 0; j < entryCount; j++)
{
string entryName = bitStream.ReadString();
bool entryHasData = bitStream.ReadBit();
byte[] entryData = null;
if (entryHasData)
{
int entryDataLength = bitStream.ReadShort();
entryData = bitStream.ReadBytes(entryDataLength);
}
StringTableEntry stringTableEntry = new StringTableEntry(entryName, entryData);
this.StringTables.Add(stringTableName, stringTableEntry);
}
bool hasExtraTableData = bitStream.ReadBit();
if (hasExtraTableData)
{
int extraTableDataLength = bitStream.ReadShort();
byte[] extraTableData = bitStream.ReadBytes(extraTableDataLength);
}
}
break;
}
case DemoCommand.SyncTick:
{
break;
}
default:
{
Console.WriteLine("Unknown Demo Command: {0}", demoCommand);
break;
}
}
return false;
}
void GeneratePushRequests(int sheetId, StringTableCollection collection, IList <SheetColumn> columnMapping, List <Request> requestsToSend, ITaskReporter reporter)
{
// Prepare the tables - Sort the keys and table entries
reporter?.ReportProgress("Sorting entries", 0.2f);
var sortedKeyEntries = collection.SharedData.Entries.OrderBy(e => e.Id);
var sortedTableEntries = new List <IOrderedEnumerable <StringTableEntry> >();
foreach (var table in collection.StringTables)
{
if (table != null)
{
var s = table.Values.OrderBy(e => e.KeyId);
sortedTableEntries.Add(s);
}
}
// Extract all the data so we can generate a request to send
// We go through each Key, extract the table entries for that key if they exists and then pass this to each column.
var currentTableRowIterator = sortedTableEntries.Select(o =>
{
var itr = o.GetEnumerator();
itr.MoveNext();
return(itr);
}).ToArray();
// Prepare the column requests.
// We use a request per column as its possible that some columns in the sheet will be preserved and we don't want to write over them.
reporter?.ReportProgress("Generating column headers", 0.25f);
var columnSheetRequests = new List <PushColumnSheetRequest>(columnMapping.Count);
foreach (var col in columnMapping)
{
var colRequest = new PushColumnSheetRequest(sheetId, col);
columnSheetRequests.Add(colRequest);
colRequest.Column.PushBegin(collection);
colRequest.Column.PushHeader(collection, out var header, out var note);
colRequest.AddHeader(header, note);
}
// Populate the requests from the string tables
var currentTableRow = new StringTableEntry[sortedTableEntries.Count];
foreach (var keyRow in sortedKeyEntries)
{
// Extract the string table entries for this row
for (int i = 0; i < currentTableRow.Length; ++i)
{
var tableRowItr = currentTableRowIterator[i];
if (tableRowItr.Current?.KeyId == keyRow.Id)
{
currentTableRow[i] = tableRowItr.Current;
tableRowItr.MoveNext();
}
else
{
currentTableRow[i] = null;
}
}
// Now process each sheet column so they can update their requests.
foreach (var colReq in columnSheetRequests)
{
colReq.Column.PushCellData(keyRow, currentTableRow, out var value, out var note);
colReq.AddRow(value, note);
}
}
foreach (var col in columnSheetRequests)
{
col.Column.PushEnd();
requestsToSend.AddRange(col.Requests);
}
}
public static void UpdateContents(LineDatabase database)
{
// First, get all scripts whose importers are configured to use
// this database - we need to add them to our TrackedPrograms list
foreach (var updatedGUID in database.RecentlyUpdatedGUIDs)
{
var path = AssetDatabase.GUIDToAssetPath(updatedGUID);
if (string.IsNullOrEmpty(path))
{
// The corresponding asset can't be found! No-op.
continue;
}
var importer = AssetImporter.GetAtPath(path);
if (!(importer is YarnImporter yarnImporter))
{
Debug.LogWarning($"Yarn Spinner internal error: line database was told to load asset {path}, but this does not have a {nameof(YarnImporter)}. Ignoring.");
continue;
}
var textAsset = AssetDatabase.LoadAssetAtPath <TextAsset>(path);
if (textAsset == null)
{
Debug.LogWarning($"Yarn Spinner internal error: failed to get a {nameof(TextAsset)} at {path}. Ignoring.");
continue;
}
if (yarnImporter.lineDatabase == database)
{
// We need to add or update content based on this asset.
database.AddTrackedProject(textAsset);
}
else
{
// This asset used to refer to this database, but now no
// longer does. Remove the reference.
database.RemoveTrackedProject(textAsset);
}
}
database.RecentlyUpdatedGUIDs.Clear();
var allTrackedImporters = database.TrackedScripts
.Select(p => AssetDatabase.GetAssetPath(p))
.Select(path => AssetImporter.GetAtPath(path) as YarnImporter);
var allLocalizationAssets = allTrackedImporters
.Where(p => p != null)
.SelectMany(p => p.AllLocalizations)
.Select(localization => new
{
languageID = localization.languageName,
text = localization.text
});
// Erase the contents of all localizations, because we're about to
// replace them
foreach (var localization in database.Localizations)
{
if (localization == null)
{
// Ignore any null entries
continue;
}
localization.Clear();
}
foreach (var localizedTextAsset in allLocalizationAssets)
{
string languageName = localizedTextAsset.languageID;
Localization localization;
try
{
localization = database.GetLocalization(languageName);
}
catch (KeyNotFoundException)
{
Debug.LogWarning($"{localizedTextAsset.text.name} is marked for language {languageName}, but this {nameof(LineDatabase)} isn't set up for that language. TODO: offer a quick way to create one here.");
continue;
}
TextAsset textAsset = localizedTextAsset.text;
if (textAsset == null)
{
// A null reference. Early out here.
continue;
}
var records = StringTableEntry.ParseFromCSV(textAsset.text);
foreach (var record in records)
{
AddLineEntryToLocalization(localization, record);
}
EditorUtility.SetDirty(localization);
}
AssetDatabase.SaveAssets();
}
private static void AddAssetReferenceToLocalization(Localization localization, StringTableEntry entry, string assetGUID)
{
var asset = AssetDatabase.LoadAssetAtPath <UnityEngine.Object>(AssetDatabase.GUIDToAssetPath(assetGUID));
localization.AddLocalizedObject(entry.ID, asset);
}
/// <summary>
/// Verifies the TextAsset referred to by <paramref name="loc"/>, and
/// updates it if necessary.
/// </summary>
/// <param name="baseLocalizationStrings">A collection of <see
/// cref="StringTableEntry"/></param>
/// <param name="language">The language that <paramref name="loc"/>
/// provides strings for.false</param>
/// <param name="loc">A TextAsset containing localized strings in CSV
/// format.</param>
/// <returns>Whether <paramref name="loc"/> was modified.</returns>
private static bool UpdateLocalizationFile(IEnumerable <StringTableEntry> baseLocalizationStrings, string language, TextAsset loc)
{
var translatedStrings = StringTableEntry.ParseFromCSV(loc.text);
// Convert both enumerables to dictionaries, for easier lookup
var baseDictionary = baseLocalizationStrings.ToDictionary(entry => entry.ID);
var translatedDictionary = translatedStrings.ToDictionary(entry => entry.ID);
// The list of line IDs present in each localisation
var baseIDs = baseLocalizationStrings.Select(entry => entry.ID);
var translatedIDs = translatedStrings.Select(entry => entry.ID);
// The list of line IDs that are ONLY present in each localisation
var onlyInBaseIDs = baseIDs.Except(translatedIDs);
var onlyInTranslatedIDs = translatedIDs.Except(baseIDs);
// Tracks if the translated localisation needed modifications
// (either new lines added, old lines removed, or changed lines
// flagged)
var modificationsNeeded = false;
// Remove every entry whose ID is only present in the translated
// set. This entry has been removed from the base localization.
foreach (var id in onlyInTranslatedIDs.ToList())
{
translatedDictionary.Remove(id);
modificationsNeeded = true;
}
// Conversely, for every entry that is only present in the base
// localisation, we need to create a new entry for it.
foreach (var id in onlyInBaseIDs)
{
StringTableEntry baseEntry = baseDictionary[id];
var newEntry = new StringTableEntry(baseEntry)
{
// Empty this text, so that it's apparent that a translated
// version needs to be provided.
Text = string.Empty,
};
translatedDictionary.Add(id, newEntry);
modificationsNeeded = true;
}
// Finally, we need to check for any entries in the translated
// localisation that:
// 1. have the same line ID as one in the base, but
// 2. have a different Lock (the hash of the text), which indicates
// that the base text has changed.
// First, get the list of IDs that are in both base and translated,
// and then filter this list to any where the lock values differ
var outOfDateLockIDs = baseDictionary.Keys
.Intersect(translatedDictionary.Keys)
.Where(id => baseDictionary[id].Lock != translatedDictionary[id].Lock);
// Now loop over all of these, and update our translated dictionary
// to include a note that it needs attention
foreach (var id in outOfDateLockIDs)
{
// Get the translated entry as it currently exists
var entry = translatedDictionary[id];
// Include a note that this entry is out of date
entry.Text = $"(NEEDS UPDATE) {entry.Text}";
// Update the lock to match the new one
entry.Lock = baseDictionary[id].Lock;
// Put this modified entry back in the table
translatedDictionary[id] = entry;
modificationsNeeded = true;
}
// We're all done!
if (modificationsNeeded == false)
{
// No changes needed to be done to the translated string table
// entries. Stop here.
return(false);
}
// We need to produce a replacement CSV file for the translated
// entries.
var outputStringEntries = translatedDictionary.Values
.OrderBy(entry => entry.File)
.ThenBy(entry => int.Parse(entry.LineNumber));
var outputCSV = StringTableEntry.CreateCSV(outputStringEntries);
// Write out the replacement text to this existing file, replacing
// its existing contents
var outputFile = AssetDatabase.GetAssetPath(loc);
File.WriteAllText(outputFile, outputCSV);
// Tell the asset database that the file needs to be reimported
AssetDatabase.ImportAsset(outputFile);
// Signal that the file was changed
return(true);
}
public void StringTableUpdated(string stringTableName, StringTableEntry stringTableEntry)
{
for (int i = 0; i < this.m_Values.Count; i++)
{
GameObserver gameObserver = this.m_Values[i];
gameObserver.StringTableUpdated(stringTableName, stringTableEntry);
}
}
public string GetString(StringTableEntry entry)
{
return(FSHelpers.ReadString(_dataCopy, (int)_stringTableOffset + entry.StringOffset));
}
private static void AddAddressableAssetReferenceToLocalization(Localization localization, StringTableEntry entry, string assetGUID)
{
if (AddressableAssetSettingsDefaultObject.SettingsExists == false)
{
// Do nothing - the user hasn't set up their addressable assets
// settings object, so we have no place to record any new
// addresses.
return;
}
var settings = AddressableAssetSettingsDefaultObject.Settings;
// Get or create the asset reference for this asset.
var assetReference = settings.CreateAssetReference(assetGUID);
// Get the entry in the addressable assets settings - we want to
// update its address to be one that a LineProvider will ask for
var addressableAssetEntry = settings.FindAssetEntry(assetGUID);
addressableAssetEntry?.SetAddress(entry.ID + "-" + localization.LocaleCode);
localization.AddLocalizedObjectAddress(entry.ID, assetReference);
}
