public DatabaseMigration(IProgressReport progress)
{
this.progress = progress;
databases = new Dictionary <string, List<string>> ();
databasesDirectories = new List<string> ();
scoreNameToID = new ConcurrentDictionary <string, Guid> ();
penaltyNameToID = new ConcurrentDictionary<string, Guid> ();
teamNameToID = new ConcurrentDictionary<string, Guid> ();
dashboardNameToID = new ConcurrentDictionary<string, Guid> ();
}
public void Report(IProgressReport reportModel)
{
var r = (ScenarioFailedReport)reportModel;
_pepinoClient.Post(
string.Format("/scenarios/{0}/failed", r.ScenarioId),
new
{
r.Message,
r.Time
});
}
public void Report(IProgressReport reportModel)
{
IProgressReportingStrategy strategy =
_progressReportingStrategies.FirstOrDefault(x => x.ReportsFor(reportModel));
try
{
if (strategy != null) strategy.Report(reportModel);
}
catch (Exception ex)
{
_logger.LogException(ex);
}
}
public void Report(IProgressReport reportModel)
{
var r = (StepStoppedReport) reportModel;
_pepinoClient.Post(
string.Format("/scenarios/{0}/criteria/{1}/steps/{2}/stopped", r.ScenarioId, r.CriterionId, r.StepId),
new
{
r.Result.Passing,
r.Result.Duration,
r.Result.Time,
r.Result.ScreenShot,
r.Result.Message
});
}
public static void Fill(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
{
const int threshold = 0x400;
if (count > threshold)
{
input.FillLarge(value, offset, count, progress);
return;
}
Span <byte> buf = stackalloc byte[(int)count];
buf.Fill(value);
input.Write(offset, buf);
}
public static MapInfo[] ReadAll(IFileSystem fs, BdatCollection tables, IProgressReport progress = null)
{
progress?.LogMessage("Reading map info and gimmick sets");
Dictionary <string, MapInfo> maps = MapInfo.ReadAll(fs);
BdatTable <FLD_maplist> mapList = tables.FLD_maplist;
BdatTable <MNU_MapInfo> areaList = tables.MNU_MapInfo;
foreach (MapInfo mapInfo in maps.Values)
{
FLD_maplist map = mapList.FirstOrDefault(x => x.resource == mapInfo.Name && x._nameID != null);
if (map == null)
{
continue;
}
if (map._nameID != null)
{
mapInfo.DisplayName = map._nameID.name;
}
foreach (MapAreaInfo areaInfo in mapInfo.Areas)
{
MNU_MapInfo area = areaList.FirstOrDefault(x =>
x.level_name == areaInfo.Name || x.level_name2 == areaInfo.Name);
if (area == null)
{
progress?.LogMessage($"Found area {areaInfo.Name} that is not in the BDAT tables");
continue;
}
// Some areas use one of 2 maps depending on the game state.
// These 2 maps are always the same except one has a small addition or removal.
// We want the map with the most objects on it, so we use the second map for
// Gormott (ma05a), and the first map for everywhere else.
if (!string.IsNullOrWhiteSpace(area.level_name2) &&
area.level_name == areaInfo.Name &&
mapInfo.Name == "ma05a")
{
areaInfo.Priority = int.MaxValue;
}
else if (!string.IsNullOrWhiteSpace(area.level_name2) &&
area.level_name2 == areaInfo.Name &&
mapInfo.Name != "ma05a")
{
areaInfo.Priority = int.MaxValue;
}
else
{
areaInfo.Priority = area.level_priority;
}
if (area._disp_name?.name != null)
{
areaInfo.DisplayName = area._disp_name.name;
}
}
Dictionary <string, Lvb> gimmickSet = ReadGimmickSet(fs, tables, map.Id);
AssignGimmickAreas(gimmickSet, mapInfo);
}
return(maps.Values.ToArray());
}
/// <summary>
/// Loads keys from key files into an existing <see cref="KeySet"/>. Missing keys will be
/// derived from existing keys if possible. Any <see langword="null"/> file names will be skipped.
/// </summary>
/// <param name="keySet">The <see cref="KeySet"/> where the loaded keys will be placed.</param>
/// <param name="prodKeysFilename">The path of the file containing common prod keys. Can be <see langword="null"/>.</param>
/// <param name="devKeysFilename">The path of the file containing common dev keys. Can be <see langword="null"/>.</param>
/// <param name="titleKeysFilename">The path of the file containing title keys. Can be <see langword="null"/>.</param>
/// <param name="consoleKeysFilename">The path of the file containing device-unique keys. Can be <see langword="null"/>.</param>
/// <param name="logger">An optional logger that key-parsing errors will be written to.</param>
public static void ReadKeyFile(KeySet keySet, string prodKeysFilename = null, string devKeysFilename = null,
string titleKeysFilename = null, string consoleKeysFilename = null, IProgressReport logger = null)
{
KeySet.Mode originalMode = keySet.CurrentMode;
List <KeyInfo> keyInfos = DefaultKeySet.CreateKeyList();
if (prodKeysFilename != null)
{
keySet.SetMode(KeySet.Mode.Prod);
using var storage = new FileStream(prodKeysFilename, FileMode.Open, FileAccess.Read);
ReadMainKeys(keySet, storage, keyInfos, logger);
}
if (devKeysFilename != null)
{
keySet.SetMode(KeySet.Mode.Dev);
using var storage = new FileStream(devKeysFilename, FileMode.Open, FileAccess.Read);
ReadMainKeys(keySet, storage, keyInfos, logger);
}
keySet.SetMode(originalMode);
if (consoleKeysFilename != null)
{
using var storage = new FileStream(consoleKeysFilename, FileMode.Open, FileAccess.Read);
ReadMainKeys(keySet, storage, keyInfos, logger);
}
if (titleKeysFilename != null)
{
using var storage = new FileStream(titleKeysFilename, FileMode.Open, FileAccess.Read);
ReadTitleKeys(keySet, storage, logger);
}
keySet.DeriveKeys(logger);
}
static void InitAddins (IProgressReport progress)
{
Guid id = Guid.NewGuid ();
progress.Report (0.1f, "Initializing addins", id);
AddinsManager.Initialize (App.Current.PluginsConfigDir, App.Current.PluginsDir);
progress.Report (0.5f, "Addins parsed", id);
AddinsManager.LoadConfigModifierAddins ();
AddinsManager.LoadExportProjectAddins ();
AddinsManager.LoadMultimediaBackendsAddins (App.Current.MultimediaToolkit);
AddinsManager.LoadUIBackendsAddins (App.Current.GUIToolkit);
AddinsManager.LoadServicesAddins ();
progress.Report (1, "Addins initialized", id);
}
public void Save(String target, IProgressReport progress)
{
using (FileStream stream = new FileStream(target, FileMode.OpenOrCreate, FileAccess.Write))
{
using (BinaryWriter writer = new BinaryWriter(stream))
{
writer.Write(StringRes.USER_PROFILE_TAG);
writer.Write(StringRes.USER_PROFILE_VERSION);
using (XBinaryWriter xwriter = new XBinaryWriter(stream))
{
Save(xwriter, progress);
}
}
}
}
public override void Load(BinaryReader reader, IProgressReport progress)
{
int count = reader.ReadInt32();
ProgressReportHelper.Report(progress, ProgressType.Start, count);
for (int i = 0; i < count; ++i)
{
WordImpl word = (WordImpl)this.vocabulary.CreateWord();
word.PrepareLoad();
word.Load(reader);
this.Items.Add(word);
word.Parent = this.vocabulary;
ProgressReportHelper.Report(progress, ProgressType.Status, i);
}
this.Items.Sort();
ProgressReportHelper.Report(progress, ProgressType.End, 0);
}
public void Report(IProgressReport reportModel)
{
_reported = reportModel;
}
public static Keyset ReadKeyFile(string filename, string titleKeysFilename = null, string consoleKeysFilename = null, IProgressReport logger = null, bool dev = false)
{
var keyset = new Keyset();
keyset.KeysetForDev = dev;
ReadKeyFile(keyset, filename, titleKeysFilename, consoleKeysFilename, logger);
return(keyset);
}
public static void ReadKeyFile(Keyset keyset, string filename, string titleKeysFilename = null, string consoleKeysFilename = null, IProgressReport logger = null)
{
Dictionary <string, KeyValue> keyDictionary = CreateFullKeyDictionary();
if (filename != null)
{
ReadMainKeys(keyset, filename, keyDictionary, logger);
}
if (consoleKeysFilename != null)
{
ReadMainKeys(keyset, consoleKeysFilename, keyDictionary, logger);
}
if (titleKeysFilename != null)
{
ReadTitleKeys(keyset, titleKeysFilename, logger);
}
keyset.ExternalKeySet.TrimExcess();
keyset.DeriveKeys(logger);
}
private static void CopyBenchmark(IStorage src, IStorage dst, int iterations, string label, IProgressReport logger)
{
// Warmup
src.CopyTo(dst);
logger.SetTotal(iterations);
Stopwatch encryptWatch = Stopwatch.StartNew();
for (int i = 0; i < iterations; i++)
{
src.CopyTo(dst);
logger.ReportAdd(1);
}
encryptWatch.Stop();
logger.SetTotal(0);
src.GetSize(out long srcSize).ThrowIfFailure();
string rate = Util.GetBytesReadable((long)(srcSize * iterations / encryptWatch.Elapsed.TotalSeconds));
logger.LogMessage($"{label}{rate}/s");
}
// Benchmarks encrypting each block separately, initializing a new cipher object for each one
private static void CipherBenchmarkSeparate(ReadOnlySpan <byte> src, Span <byte> dst, CipherTaskSeparate function,
int iterations, string label, bool dotNetCrypto, IProgressReport logger)
{
Debug.Assert(src.Length == dst.Length);
var watch = new Stopwatch();
var runTimes = new double[iterations];
ReadOnlySpan <byte> key1 = stackalloc byte[0x10];
ReadOnlySpan <byte> key2 = stackalloc byte[0x10];
ReadOnlySpan <byte> iv = stackalloc byte[0x10];
logger.SetTotal(iterations);
const int blockSize = BlockSizeSeparate;
int blockCount = src.Length / blockSize;
for (int i = 0; i < iterations; i++)
{
watch.Restart();
for (int b = 0; b < blockCount; b++)
{
function(src.Slice(b * blockSize, blockSize), dst.Slice(b * blockSize, blockSize),
key1, key2, iv, dotNetCrypto);
}
watch.Stop();
logger.ReportAdd(1);
runTimes[i] = watch.Elapsed.TotalSeconds;
}
logger.SetTotal(0);
long srcSize = src.Length;
double fastestRun = runTimes.Min();
double averageRun = runTimes.Average();
double slowestRun = runTimes.Max();
string fastestRate = Util.GetBytesReadable((long)(srcSize / fastestRun));
string averageRate = Util.GetBytesReadable((long)(srcSize / averageRun));
string slowestRate = Util.GetBytesReadable((long)(srcSize / slowestRun));
logger.LogMessage($"{label}{averageRate}/s, fastest run: {fastestRate}/s, slowest run: {slowestRate}/s");
}
public void Report(IProgressReport reportModel)
{
var r = (StepStartedReport) reportModel;
_pepinoClient.Post(string.Format("/scenarios/{0}/criteria/{1}/steps/{2}/started", r.ScenarioId, r.CriterionId, r.StepId));
}
public ProgressReportAccessLog(IProgressReport logger)
{
Logger = logger;
}
public static void CopyToStream(this IStorage input, Stream output, long length, IProgressReport progress = null, int bufferSize = 0x8000)
{
long remaining = length;
long inOffset = 0;
using var buffer = new RentedArray <byte>(bufferSize);
int rentedBufferSize = buffer.Array.Length;
progress?.SetTotal(length);
while (remaining > 0)
{
int toWrite = (int)Math.Min(rentedBufferSize, remaining);
input.Read(inOffset, buffer.Array.AsSpan(0, toWrite));
output.Write(buffer.Array, 0, toWrite);
remaining -= toWrite;
inOffset += toWrite;
progress?.ReportAdd(toWrite);
}
}
private static void ReadMainKeys(Keyset keyset, string filename, Dictionary <string, KeyValue> keyDict, IProgressReport logger = null)
{
if (filename == null)
{
return;
}
using (var reader = new StreamReader(new FileStream(filename, FileMode.Open, FileAccess.Read)))
{
string line;
while ((line = reader.ReadLine()) != null)
{
string[] a = line.Split(',', '=');
if (a.Length != 2)
{
continue;
}
string key = a[0].Trim();
string valueStr = a[1].Trim();
if (!keyDict.TryGetValue(key, out KeyValue kv))
{
logger?.LogMessage($"Failed to match key {key}");
continue;
}
byte[] value = valueStr.ToBytes();
if (value.Length != kv.Size)
{
logger?.LogMessage($"Key {key} had incorrect size {value.Length}. (Expected {kv.Size})");
continue;
}
byte[] dest = kv.GetKey(keyset);
Array.Copy(value, dest, value.Length);
}
}
}
public void ExportWav(string baseName, IProgressReport progress = null)
{
int bytesWrote = 0;
if (MultiChannel)
{
/*using (Stream file = File.Create(baseName + ".wav"))
* {
* ExportWav(AudioStreams[0], file);
* }*/
if (progress != null)
{
// Calculate how many bytes are going to be written
progress = new SubProgressReport(progress, (((MultiChannelAudio)AudioStreams[0]).Channels.Sum(audio => audio.GetSize())));
}
for (int i = 0; i < ((MultiChannelAudio)AudioStreams[0]).Channels.Count; ++i)
{
using (Stream file = File.Create(baseName + "." + AudioIds[i] + ".wav"))
{
if (progress != null)
{
progress.SetMessage("Decoding " + AudioIds[i]);
}
int audioSize = ((MultiChannelAudio)AudioStreams[0]).Channels[i].GetSize();
try
{
ExportWav(((MultiChannelAudio)AudioStreams[0]).Channels[i], file, progress == null ? null : new SubProgressReport(progress, bytesWrote, audioSize));
}
catch (Exception e)
{
file.Close();
// Delete uncompleted file
File.Delete(baseName + "." + AudioIds[i] + ".wav");
throw e;
}
bytesWrote += audioSize;
}
}
}
else
{
for (int i = 0; i < AudioStreams.Count; ++i)
{
if (progress != null)
{
// Calculate how many bytes are going to be written
progress = new SubProgressReport(progress, AudioStreams.Sum(audio => audio.GetSize()));
}
using (Stream file = File.Create(baseName + "." + AudioIds[i] + ".wav"))
{
if (progress != null)
{
progress.SetMessage("Decoding " + AudioIds[i]);
}
int audioSize = AudioStreams[i].GetSize();
try
{
ExportWav(AudioStreams[i], file, progress == null ? null : new SubProgressReport(progress, bytesWrote, audioSize));
}
catch (OperationCanceledException e)
{
file.Close();
// Delete uncompleted file
File.Delete(baseName + "." + AudioIds[i] + ".wav");
throw e;
}
bytesWrote += audioSize;
}
}
}
}
public static Keyset ReadKeyFile(string filename, string titleKeysFilename = null, string consoleKeysFilename = null, IProgressReport logger = null)
{
var keyset = new Keyset();
if (filename != null)
{
ReadMainKeys(keyset, filename, AllKeyDict, logger);
}
if (consoleKeysFilename != null)
{
ReadMainKeys(keyset, consoleKeysFilename, AllKeyDict, logger);
}
if (titleKeysFilename != null)
{
ReadTitleKeys(keyset, titleKeysFilename, logger);
}
keyset.DeriveKeys(logger);
return(keyset);
}
public static Result CopyDirectory(this FileSystemClient fs, string sourcePath, string destPath,
CreateFileOptions options = CreateFileOptions.None, IProgressReport logger = null)
{
Result rc = fs.OpenDirectory(out DirectoryHandle sourceHandle, sourcePath.ToU8Span(), OpenDirectoryMode.All);
if (rc.IsFailure())
{
return(rc);
}
try
{
foreach (DirectoryEntryEx entry in fs.EnumerateEntries(sourcePath, "*", SearchOptions.Default))
{
string subSrcPath = PathTools.Normalize(PathTools.Combine(sourcePath, entry.Name));
string subDstPath = PathTools.Normalize(PathTools.Combine(destPath, entry.Name));
if (entry.Type == DirectoryEntryType.Directory)
{
fs.EnsureDirectoryExists(subDstPath);
rc = fs.CopyDirectory(subSrcPath, subDstPath, options, logger);
if (rc.IsFailure())
{
return(rc);
}
}
if (entry.Type == DirectoryEntryType.File)
{
logger?.LogMessage(subSrcPath);
fs.CreateOrOverwriteFile(subDstPath, entry.Size, options);
rc = fs.CopyFile(subSrcPath, subDstPath, logger);
if (rc.IsFailure())
{
return(rc);
}
}
}
}
finally
{
if (sourceHandle.IsValid)
{
fs.CloseDirectory(sourceHandle);
}
}
return(Result.Success);
}
public void Load(String source, IProgressReport progress)
{
Debug.Assert(File.Exists(source));
using (FileStream stream = File.OpenRead(source))
{
using (BinaryReader reader = new BinaryReader(stream))
{
String tag = reader.ReadString();
if (tag != StringRes.USER_PROFILE_TAG)
{
throw new FileFormatException(source, StringRes.NOT_USER_STATISTICS_FILE);
}
String version = reader.ReadString();
if (version != StringRes.USER_PROFILE_VERSION)
{
throw new FileFormatException(source, String.Format(StringRes.VERSION_NOT_SUPPORTED, version));
}
using (XBinaryReader xreader = new XBinaryReader(stream))
{
Load(xreader, progress);
}
}
}
}
public static Result CopyFile(this FileSystemClient fs, string sourcePath, string destPath, IProgressReport logger = null)
{
Result rc = fs.OpenFile(out FileHandle sourceHandle, sourcePath.ToU8Span(), OpenMode.Read);
if (rc.IsFailure())
{
return(rc);
}
try
{
rc = fs.OpenFile(out FileHandle destHandle, destPath.ToU8Span(),
OpenMode.Write | OpenMode.AllowAppend);
if (rc.IsFailure())
{
return(rc);
}
try
{
const int maxBufferSize = 0x10000;
rc = fs.GetFileSize(out long fileSize, sourceHandle);
if (rc.IsFailure())
{
return(rc);
}
int bufferSize = (int)Math.Min(maxBufferSize, fileSize);
logger?.SetTotal(fileSize);
byte[] buffer = ArrayPool <byte> .Shared.Rent(bufferSize);
try
{
for (long offset = 0; offset < fileSize; offset += bufferSize)
{
int toRead = (int)Math.Min(fileSize - offset, bufferSize);
Span <byte> buf = buffer.AsSpan(0, toRead);
rc = fs.ReadFile(out long _, sourceHandle, offset, buf);
if (rc.IsFailure())
{
return(rc);
}
rc = fs.WriteFile(destHandle, offset, buf, WriteOption.None);
if (rc.IsFailure())
{
return(rc);
}
logger?.ReportAdd(toRead);
}
}
finally
{
ArrayPool <byte> .Shared.Return(buffer);
logger?.SetTotal(0);
}
rc = fs.FlushFile(destHandle);
if (rc.IsFailure())
{
return(rc);
}
}
finally
{
if (destHandle.IsValid)
{
fs.CloseFile(destHandle);
}
}
}
finally
{
if (sourceHandle.IsValid)
{
fs.CloseFile(sourceHandle);
}
}
return(Result.Success);
}
public virtual void Load(BinaryReader reader, IProgressReport progress)
{
Load(reader);
}
public static void GenerateSite(IFileSystem fs, string outDir, IProgressReport progress)
{
Directory.CreateDirectory(outDir);
GenerateBdatHtml(fs, outDir, progress);
}
public bool ReportsFor(IProgressReport reportModel)
{
return reportModel is ScenarioFailedReport;
}
public static void Fill(this IStorage input, byte value, IProgressReport progress = null)
{
input.Fill(value, 0, input.GetSize(), progress);
}
public static void PrintSeparateTables(BdatStringCollection bdats, string htmlDir, IProgressReport progress = null)
{
progress?.LogMessage("Writing BDAT tables as HTML");
progress?.SetTotal(bdats.Tables.Count);
string bdatHtmlDir = Path.Combine(htmlDir, "bdat");
Directory.CreateDirectory(bdatHtmlDir);
if (bdats.Game == Game.XB2)
{
PrintIndex(bdats, htmlDir);
}
PrintBdatIndex(bdats, bdatHtmlDir);
foreach (string tableName in bdats.Tables.Keys)
{
string outDir = bdatHtmlDir;
string tableFilename = bdats[tableName].Filename;
var indexPath = tableFilename == null ? "index.html" : "../index.html";
var sb = new Indenter(2);
sb.AppendLine("<!DOCTYPE html>");
sb.AppendLineAndIncrease("<html>");
sb.AppendLineAndIncrease("<head>");
sb.AppendLine("<meta charset=\"utf-8\" />");
sb.AppendLine($"<title>{tableName}</title>");
sb.AppendLineAndIncrease("<style>");
sb.AppendLine(CssSticky);
sb.AppendLine(CssSortable);
sb.DecreaseAndAppendLine("</style>");
sb.DecreaseAndAppendLine("</head>");
sb.AppendLineAndIncrease("<body>");
sb.AppendLine($"<a href=\"{indexPath}\">Return to BDAT index</a><br/>");
sb.AppendLine("<input type=\"button\" value=\"Open all references\" onclick=\"openAll(true)\" />");
sb.AppendLine("<input type=\"button\" value=\"Close all references\" onclick=\"openAll(false)\" />");
PrintTable(bdats[tableName], sb);
sb.AppendLineAndIncrease("<script>");
sb.AppendLine(JsOpenAll);
sb.AppendLine(JsSortable);
sb.AppendLine(JsAnchor);
sb.DecreaseAndAppendLine("</script>");
sb.DecreaseAndAppendLine("</body>");
sb.DecreaseAndAppendLine("</html>");
if (tableFilename != null)
{
outDir = Path.Combine(outDir, tableFilename);
}
string filename = Path.Combine(outDir, tableName + ".html");
Directory.CreateDirectory(outDir);
File.WriteAllText(filename, sb.ToString());
progress?.ReportAdd(1);
}
}
private static void FillLarge(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
{
const int bufferSize = 0x4000;
long remaining = count;
if (remaining < 0)
{
throw new ArgumentException("Storage must have an explicit length");
}
progress?.SetTotal(remaining);
long pos = offset;
byte[] buffer = ArrayPool <byte> .Shared.Rent(bufferSize);
try
{
buffer.AsSpan(0, (int)Math.Min(remaining, bufferSize)).Fill(value);
while (remaining > 0)
{
int toFill = (int)Math.Min(bufferSize, remaining);
Span <byte> buf = buffer.AsSpan(0, toFill);
input.Write(buf, pos);
remaining -= toFill;
pos += toFill;
progress?.ReportAdd(toFill);
}
}
finally
{
ArrayPool <byte> .Shared.Return(buffer);
}
progress?.SetTotal(0);
}
public static void Fill(this IStorage input, byte value, IProgressReport progress = null)
{
input.GetSize(out long inputSize).ThrowIfFailure();
input.Fill(value, 0, inputSize, progress);
}
public static void CopyToStream(this IStorage input, Stream output, long length, IProgressReport progress = null)
{
const int bufferSize = 0x8000;
long remaining = length;
long inOffset = 0;
var buffer = new byte[bufferSize];
progress?.SetTotal(length);
while (remaining > 0)
{
int toWrite = (int)Math.Min(buffer.Length, remaining);
input.Read(buffer.AsSpan(0, toWrite), inOffset);
output.Write(buffer, 0, toWrite);
remaining -= toWrite;
inOffset += toWrite;
progress?.ReportAdd(toWrite);
}
}
private static void FillLarge(this IStorage input, byte value, long offset, long count, IProgressReport progress = null)
{
const int bufferSize = 0x4000;
long remaining = count;
if (remaining < 0)
{
throw new ArgumentException("Storage must have an explicit length");
}
progress?.SetTotal(remaining);
long pos = offset;
using var buffer = new RentedArray <byte>(bufferSize);
int rentedBufferSize = buffer.Array.Length;
buffer.Array.AsSpan(0, (int)Math.Min(remaining, rentedBufferSize)).Fill(value);
while (remaining > 0)
{
int toFill = (int)Math.Min(rentedBufferSize, remaining);
Span <byte> buf = buffer.Array.AsSpan(0, toFill);
input.Write(pos, buf);
remaining -= toFill;
pos += toFill;
progress?.ReportAdd(toFill);
}
progress?.SetTotal(0);
}
public SubProgressReport(IProgressReport report, long units = 100)
: this(report, 0, report.GetFullValue(), units)
{
}
public bool ReportsFor(IProgressReport reportModel)
{
return reportModel is StepStoppedReport;
}
public override void Load(BinaryReader reader, IProgressReport progress)
{
this.name = reader.ReadString();
this.description = reader.ReadString();
this.targetLangauge.Load(reader);
this.nativeLanguage.Load(reader);
this.wordTypes.Load(reader);
this.categories.Load(reader);
this.words.Load(reader, progress);
}
public bool ReportsFor(IProgressReport reportModel)
{
return _report.GetType() == reportModel.GetType();
}
public override void Save(BinaryWriter writer, IProgressReport progress)
{
writer.Write(this.name);
writer.Write(this.description);
this.targetLangauge.Save(writer);
this.nativeLanguage.Save(writer);
this.wordTypes.Save(writer);
this.categories.Save(writer);
this.words.Save(writer, progress);
}
public static void PrintAllTables(BdatStringCollection bdats, string schemaName, IProgressReport progress = null)
{
string dbName;
string dbUsername;
string dbPassword;
Console.Write("Enter Database Name: ");
dbName = Console.ReadLine();
Console.Write("Enter User Name: ");
dbUsername = Console.ReadLine();
Console.Write("Enter User Password: "******"Host=localhost;Username={dbUsername};Password={dbPassword};Database={dbName};";
using (NpgsqlConnection conn = new NpgsqlConnection(connString))
{
try
{
conn.Open();
}
catch (PostgresException exception)
{
if (exception.SqlState == "28P01")
{
Console.WriteLine($"Password authentication for user {dbUsername} failed.");
}
if (exception.SqlState == "3D000")
{
Console.WriteLine($"Database {dbName} does not exist.");
}
Environment.Exit(1);
}
using (NpgsqlCommand cmd = new NpgsqlCommand())
{
cmd.Connection = conn;
cmd.CommandText = $"CREATE SCHEMA {schemaName};";
try
{
cmd.ExecuteNonQuery();
}
catch (PostgresException exception)
{
if (exception.SqlState == "42P06")
{
Console.WriteLine($"Schema name {schemaName} is already in use. Delete the schema and retry or provide a different schema name.");
Environment.Exit(1);
}
}
}
progress?.LogMessage("Writing BDAT tables to postgresql database");
progress?.SetTotal(bdats.Tables.Count);
foreach (string tableName in bdats.Tables.Keys)
{
BdatStringTable table = bdats[tableName];
string createQuery = CreateTableQuery(schemaName, table);
using (var cmd = new NpgsqlCommand())
{
cmd.Connection = conn;
cmd.CommandText = createQuery;
cmd.ExecuteNonQuery();
}
PrintTable(table, conn, schemaName);
progress?.ReportAdd(1);
}
}
}
private static void ExportWilayTextures(byte[] file, string name, string outDir, IProgressReport progress = null)
{
var wilay = new WilayRead(file);
Directory.CreateDirectory(outDir);
for (int i = 0; i < wilay.Textures.Length; i++)
{
byte[] png = wilay.Textures[i].ToPng();
if (png == null)
{
progress?.LogMessage($"{wilay.Textures[i].Format} decoding not implemented. Converting to DDS.");
byte[] dds = Dds.CreateDds(wilay.Textures[i]);
File.WriteAllBytes(Path.Combine(outDir, name + "_" + i + ".dds"), dds);
continue;
}
File.WriteAllBytes(Path.Combine(outDir, name + "_" + i + ".png"), png);
}
}
public void Load(String source, IProgressReport progress)
{
Debug.Assert(File.Exists(source));
using (FileStream stream = File.OpenRead(source))
{
using (BinaryReader reader = new BinaryReader(stream))
{
String tag = reader.ReadString();
if (tag != StringRes.VOCABULARY_TAG)
{
throw new FileFormatException(source, StringRes.NOT_VOCABULARY_FILE);
}
String version = reader.ReadString();
if (version != StringRes.VOCABULARY_VERSION)
{
throw new FileFormatException(source, String.Format(StringRes.VERSION_NOT_SUPPORTED, version));
}
using (XBinaryReader xreader = new XBinaryReader(stream))
{
PrepareLoad();
Load(xreader, progress);
FinalizeLoad();
}
}
}
Modify(false);
}
public static void Extract(this IFileSystem source, string destinationPath, IProgressReport logger = null)
{
var destFs = new LocalFileSystem(destinationPath);
source.CopyFileSystem(destFs, logger);
}
public void Save(String target, IProgressReport progress)
{
using (FileStream stream = new FileStream(target, FileMode.OpenOrCreate, FileAccess.Write))
{
using (BinaryWriter writer = new BinaryWriter(stream))
{
writer.Write(StringRes.VOCABULARY_TAG);
writer.Write(StringRes.VOCABULARY_VERSION);
using (XBinaryWriter xwriter = new XBinaryWriter(stream))
{
PrepareSave();
Save(xwriter, progress);
FinalizeSave();
}
}
}
Modify(false);
}
public static Result CopyDirectoryWithProgress(FileSystemClient fs, U8Span sourcePath, U8Span destPath,
CreateFileOptions options = CreateFileOptions.None, IProgressReport logger = null)
{
try
{
logger?.SetTotal(GetTotalSize(fs, sourcePath));
return(CopyDirectoryWithProgressInternal(fs, sourcePath, destPath, options, logger));
}
finally
{
logger?.SetTotal(0);
}
}
public override void Save(BinaryWriter writer, IProgressReport progress)
{
ProgressReportHelper.Report(progress, ProgressType.Start, this.Items.Count);
int i = 0;
writer.Write(this.Items.Count);
foreach (WordImpl word in this.Items)
{
word.Save(writer);
ProgressReportHelper.Report(progress, ProgressType.Status, ++i);
}
ProgressReportHelper.Report(progress, ProgressType.End, 0);
}
private static Result CopyDirectoryWithProgressInternal(FileSystemClient fs, U8Span sourcePath, U8Span destPath,
CreateFileOptions options, IProgressReport logger)
{
string sourcePathStr = sourcePath.ToString();
string destPathStr = destPath.ToString();
Result rc = fs.OpenDirectory(out DirectoryHandle sourceHandle, sourcePath, OpenDirectoryMode.All);
if (rc.IsFailure())
{
return(rc);
}
using (sourceHandle)
{
foreach (DirectoryEntryEx entry in fs.EnumerateEntries(sourcePathStr, "*", SearchOptions.Default))
{
string subSrcPath = PathTools.Normalize(PathTools.Combine(sourcePathStr, entry.Name));
string subDstPath = PathTools.Normalize(PathTools.Combine(destPathStr, entry.Name));
if (entry.Type == DirectoryEntryType.Directory)
{
fs.EnsureDirectoryExists(subDstPath);
rc = CopyDirectoryWithProgressInternal(fs, subSrcPath.ToU8Span(), subDstPath.ToU8Span(), options, logger);
if (rc.IsFailure())
{
return(rc);
}
}
if (entry.Type == DirectoryEntryType.File)
{
logger?.LogMessage(subSrcPath);
rc = fs.CreateOrOverwriteFile(subDstPath, entry.Size, options);
if (rc.IsFailure())
{
return(rc);
}
rc = CopyFileWithProgress(fs, subSrcPath.ToU8Span(), subDstPath.ToU8Span(), logger);
if (rc.IsFailure())
{
return(rc);
}
}
}
}
return(Result.Success);
}
public void Load(BinaryReader reader, IProgressReport progress)
{
this.name = reader.ReadString();
this.created = new DateTime(reader.ReadInt64());
this.saved = new DateTime(reader.ReadInt64());
this.defaultVocabulary = reader.ReadString();
this.sleepInterval = reader.ReadInt32();
this.beep = reader.ReadBoolean();
int count = reader.ReadInt32();
ProgressReportHelper.Report(progress, ProgressType.Start, count);
for (int i = 0; i < count; ++i)
{
StatisticsImpl statistics = new StatisticsImpl(this, Guid.Empty);
statistics.Load(reader);
this.statistics.Add(statistics);
ProgressReportHelper.Report(progress, ProgressType.Status, i);
}
this.statistics.Sort();
ProgressReportHelper.Report(progress, ProgressType.End, 0);
}
public static Result CopyFileWithProgress(FileSystemClient fs, U8Span sourcePath, U8Span destPath, IProgressReport logger = null)
{
Result rc = fs.OpenFile(out FileHandle sourceHandle, sourcePath, OpenMode.Read);
if (rc.IsFailure())
{
return(rc);
}
using (sourceHandle)
{
rc = fs.OpenFile(out FileHandle destHandle, destPath, OpenMode.Write | OpenMode.AllowAppend);
if (rc.IsFailure())
{
return(rc);
}
using (destHandle)
{
const int maxBufferSize = 1024 * 1024;
rc = fs.GetFileSize(out long fileSize, sourceHandle);
if (rc.IsFailure())
{
return(rc);
}
int bufferSize = (int)Math.Min(maxBufferSize, fileSize);
byte[] buffer = ArrayPool <byte> .Shared.Rent(bufferSize);
try
{
for (long offset = 0; offset < fileSize; offset += bufferSize)
{
int toRead = (int)Math.Min(fileSize - offset, bufferSize);
Span <byte> buf = buffer.AsSpan(0, toRead);
rc = fs.ReadFile(out long _, sourceHandle, offset, buf);
if (rc.IsFailure())
{
return(rc);
}
rc = fs.WriteFile(destHandle, offset, buf);
if (rc.IsFailure())
{
return(rc);
}
logger?.ReportAdd(toRead);
}
}
finally
{
ArrayPool <byte> .Shared.Return(buffer);
}
rc = fs.FlushFile(destHandle);
if (rc.IsFailure())
{
return(rc);
}
}
}
return(Result.Success);
}
public void Save(BinaryWriter writer, IProgressReport progress)
{
this.saved = DateTime.Now;
writer.Write(this.name);
writer.Write(this.created.Ticks);
writer.Write(this.saved.Ticks);
writer.Write(this.defaultVocabulary);
writer.Write(this.sleepInterval);
writer.Write(this.beep);
ProgressReportHelper.Report(progress, ProgressType.Start, this.statistics.Count);
int i = 0;
writer.Write(this.statistics.Count);
foreach (StatisticsImpl statistics in this.statistics)
{
statistics.Save(writer);
ProgressReportHelper.Report(progress, ProgressType.Status, i++);
}
ProgressReportHelper.Report(progress, ProgressType.Start, 0);
}
/// <summary>
/// Loads non-title keys from a <see cref="TextReader"/> into an existing <see cref="KeySet"/>.
/// Missing keys will not be derived.
/// </summary>
/// <param name="keySet">The <see cref="KeySet"/> where the loaded keys will be placed.</param>
/// <param name="reader">A <see cref="Stream"/> containing the keys to load.</param>
/// <param name="keyList">A list of all the keys that will be loaded into the key set.
/// <see cref="DefaultKeySet.CreateKeyList"/> will create a list containing all loadable keys.</param>
/// <param name="logger">An optional logger that key-parsing errors will be written to.</param>
public static void ReadMainKeys(KeySet keySet, Stream reader, List <KeyInfo> keyList,
IProgressReport logger = null)
{
if (reader == null)
{
return;
}
using var streamReader = new StreamReader(reader);
Span <char> buffer = stackalloc char[ReadBufferSize];
var ctx = new KvPairReaderContext(streamReader, buffer);
while (true)
{
ReaderStatus status = GetKeyValuePair(ref ctx);
if (status == ReaderStatus.Error)
{
logger?.LogMessage($"Invalid line in key data: \"{ctx.CurrentKey.ToString()}\"");
}
else if (status == ReaderStatus.ReadKey)
{
if (!TryGetKeyInfo(out SpecificKeyInfo info, keyList, ctx.CurrentKey))
{
logger?.LogMessage($"Failed to match key {ctx.CurrentKey.ToString()}");
continue;
}
Span <byte> key;
// Get the dev key in the key set if needed.
if (info.IsDev && keySet.CurrentMode == KeySet.Mode.Prod)
{
keySet.SetMode(KeySet.Mode.Dev);
key = info.Key.Getter(keySet, info.Index);
keySet.SetMode(KeySet.Mode.Prod);
}
else
{
key = info.Key.Getter(keySet, info.Index);
}
if (ctx.CurrentValue.Length != key.Length * 2)
{
logger?.LogMessage($"Key {ctx.CurrentKey.ToString()} has incorrect size {ctx.CurrentValue.Length}. Must be {key.Length * 2} hex digits.");
continue;
}
if (!StringUtils.TryFromHexString(ctx.CurrentValue, key))
{
key.Clear();
logger?.LogMessage($"Key {ctx.CurrentKey.ToString()} has an invalid value. Must be {key.Length * 2} hex digits.");
}
}
else if (status == ReaderStatus.Finished)
{
break;
}
}
}
public virtual void Save(BinaryWriter writer, IProgressReport progress)
{
Save(writer);
}
/// <summary>
/// Loads title keys from a <see cref="TextReader"/> into an existing <see cref="KeySet"/>.
/// </summary>
/// <param name="keySet">The <see cref="KeySet"/> where the loaded keys will be placed.</param>
/// <param name="reader">A <see cref="Stream"/> containing the keys to load.</param>
/// <param name="logger">An optional logger that key-parsing errors will be written to.</param>
public static void ReadTitleKeys(KeySet keySet, Stream reader, IProgressReport logger = null)
{
if (reader == null)
{
return;
}
using var streamReader = new StreamReader(reader);
Span <char> buffer = stackalloc char[ReadBufferSize];
var ctx = new KvPairReaderContext(streamReader, buffer);
// Estimate the number of keys by assuming each line is about 69 bytes.
// Subtract 2 from that so we estimate slightly high.
keySet.ExternalKeySet.EnsureCapacity((int)reader.Length / 67);
while (true)
{
ReaderStatus status = GetKeyValuePair(ref ctx);
if (status == ReaderStatus.Error)
{
logger?.LogMessage($"Invalid line in key data: \"{ctx.CurrentKey.ToString()}\"");
Debugger.Break();
}
else if (status == ReaderStatus.ReadKey)
{
if (ctx.CurrentKey.Length != TitleKeySize * 2)
{
logger?.LogMessage($"Rights ID {ctx.CurrentKey.ToString()} has incorrect size {ctx.CurrentKey.Length}. (Expected {TitleKeySize * 2})");
continue;
}
if (ctx.CurrentValue.Length != TitleKeySize * 2)
{
logger?.LogMessage($"Title key {ctx.CurrentValue.ToString()} has incorrect size {ctx.CurrentValue.Length}. (Expected {TitleKeySize * 2})");
continue;
}
var rightsId = new RightsId();
var titleKey = new AccessKey();
if (!StringUtils.TryFromHexString(ctx.CurrentKey, SpanHelpers.AsByteSpan(ref rightsId)))
{
logger?.LogMessage($"Invalid rights ID \"{ctx.CurrentKey.ToString()}\" in title key file");
continue;
}
if (!StringUtils.TryFromHexString(ctx.CurrentValue, SpanHelpers.AsByteSpan(ref titleKey)))
{
logger?.LogMessage($"Invalid title key \"{ctx.CurrentValue.ToString()}\" in title key file");
continue;
}
keySet.ExternalKeySet.Add(rightsId, titleKey).ThrowIfFailure();
}
else if (status == ReaderStatus.Finished)
{
break;
}
}
}
static void InitGStreamer (IProgressReport progress)
{
Guid id = Guid.NewGuid ();
progress.Report (0.1f, "Initializing GStreamer", id);
GStreamer.Init ();
progress.Report (1f, "GStreamer initialized", id);
}
public static void Report(IProgressReport progress, ProgressType type, int count)
{
if (progress != null)
{
progress.Report(type, count);
}
}
