protected override void LoadChanges(IList <Subtitle> subtitles)
{
if (HasChanges)
{
var subs = subtitles.Select(subtitle => subtitle as UnderRailSubtitle).ToList();
using (var fs = new FileStream(ChangesFile, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs, System.Text.Encoding.Unicode))
{
var version = input.ReadInt32();
if (version != ChangesFileVersion)
{
//File.Delete(ChangesFile);
return;
}
var subtitleCount = input.ReadInt32();
for (var i = 0; i < subtitleCount; i++)
{
var id = input.ReadString();
var text = input.ReadString();
var subtitle = subs.FirstOrDefault(x => x.Id == id);
if (subtitle != null)
{
subtitle.PropertyChanged -= SubtitlePropertyChanged;
subtitle.Translation = text;
subtitle.Loaded = subtitle.Translation;
subtitle.PropertyChanged += SubtitlePropertyChanged;
}
}
}
}
}
private static (IDataSourceVersion Version, string JsonKey, bool IsArray) ReadHeader(Stream stream)
{
IDataSourceVersion version;
string jsonKey;
bool isArray;
using (var reader = new ExtendedBinaryReader(stream, Encoding.UTF8, true))
{
string identifier = reader.ReadString();
if (identifier != SaCommon.NgaIdentifier)
{
throw new InvalidDataException($"Expected the NGA identifier ({SaCommon.NgaIdentifier}), but found another value: ({identifier})");
}
version = DataSourceVersion.Read(reader);
jsonKey = reader.ReadString();
isArray = reader.ReadBoolean();
ushort schemaVersion = reader.ReadUInt16();
if (schemaVersion != SaCommon.SchemaVersion)
{
throw new UserErrorException($"Expected the schema version {SaCommon.SchemaVersion}, but found another value: ({schemaVersion}) for {jsonKey}");
}
uint guard = reader.ReadUInt32();
if (guard != SaCommon.GuardInt)
{
throw new InvalidDataException($"Expected a guard integer ({SaCommon.GuardInt}), but found another value: ({guard})");
}
}
return(version, jsonKey, isArray);
}
private void ReadHeader()
{
var header = _reader.ReadString();
if (header != CustomIntervalCommon.DataHeader)
{
throw new GeneralException("Unrecognized header in custom interval database");
}
var schema = _reader.ReadUInt16();
if (schema != CustomIntervalCommon.SchemaVersion)
{
throw new GeneralException(
$"Custom interval database schema mismatch. Expected {CustomIntervalCommon.SchemaVersion}, observed {schema}");
}
_creationTime = _reader.ReadInt64();
_referenceName = _reader.ReadString();
_intervalType = _reader.ReadString();
DataVersion = new DataSourceVersion(_reader);
CheckGuard();
}
// for the SA reader
public SaIndex(ExtendedBinaryReader reader)
{
var header = reader.ReadString();
var version = reader.ReadUInt16();
reader.ReadInt64(); // skip timestamp
RefSeqName = reader.ReadString();
SupplementaryAnnotationCommon.CheckGuard(reader);
if (header != SupplementaryAnnotationCommon.IndexHeader || version != SupplementaryAnnotationCommon.IndexVersion)
{
throw new UserErrorException($"The header check failed for the supplementary annotation index file ({reader.BaseStream}): ID: exp: {SupplementaryAnnotationCommon.IndexHeader} obs: {header}, version: exp: {SupplementaryAnnotationCommon.IndexVersion} obs: {version}");
}
var count = reader.ReadInt32();
_saIndexNodes = new SaIndexNode[count];
for (var i = 0; i < count; i++)
{
_saIndexNodes[i] = new SaIndexNode(reader);
}
SupplementaryAnnotationCommon.CheckGuard(reader);
}
private void ReadHeaders()
{
if (reader.PeekChar() == -1)
{
// Empty dataset
FieldNames = new string[0];
RawTypes = new ClickHouseType[0];
return;
}
var count = reader.Read7BitEncodedInt();
FieldNames = new string[count];
RawTypes = new ClickHouseType[count];
CurrentRow = new object[count];
for (var i = 0; i < count; i++)
{
FieldNames[i] = reader.ReadString();
}
for (var i = 0; i < count; i++)
{
var chType = reader.ReadString();
RawTypes[i] = TypeConverter.ParseClickHouseType(chType);
}
}
// Methods
public override void Load(Stream fileStream)
{
// Header
var reader = new ExtendedBinaryReader(fileStream, Encoding.ASCII, false);
string sig = reader.ReadSignature();
if (sig != Signature)
{
throw new InvalidSignatureException(Signature, sig);
}
// Textures
uint totalFrameCount = reader.ReadUInt32();
byte textureCount = reader.ReadByte();
for (int i = 0; i < textureCount; ++i)
{
Textures.Add(new Texture()
{
FilePath = reader.ReadString(),
Unknown1 = reader.ReadByte()
});
}
// Animations
ushort animCount = reader.ReadUInt16();
for (int i2 = 0; i2 < animCount; ++i2)
{
var anim = new Animation()
{
Name = reader.ReadString()
};
ushort frameCount = reader.ReadUInt16();
anim.Unknown1 = reader.ReadInt16();
anim.Unknown2 = reader.ReadInt16();
// Animation Frames
for (int i3 = 0; i3 < frameCount; ++i3)
{
anim.Frames.Add(new Frame()
{
TextureIndex = reader.ReadByte(),
Unknown1 = reader.ReadInt16(),
Unknown2 = reader.ReadInt16(),
X = reader.ReadUInt16(),
Y = reader.ReadUInt16(),
Width = reader.ReadUInt16(),
Height = reader.ReadUInt16(),
OriginX = reader.ReadInt16(),
OriginY = reader.ReadInt16()
});
}
Animations.Add(anim);
}
}
public override void Rebuild(string outputFolder)
{
var outputPath = System.IO.Path.Combine(outputFolder, RelativePath);
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(outputPath));
var subtitles = GetSubtitles();
using (var fsInput = new FileStream(Path, FileMode.Open))
using (var input = new ExtendedBinaryReader(fsInput, FileEncoding, Endianness.LittleEndian))
using (var fsOutput = new FileStream(outputPath, FileMode.Create))
using (var output = new ExtendedBinaryWriter(fsOutput, FileEncoding, Endianness.LittleEndian))
{
var outputOffset = 0;
while (input.Position < input.Length)
{
var inputNextOffset = input.ReadInt16();
var sectionName = input.ReadString(Encoding.UTF8);
using (var tempMemoryStream = new MemoryStream())
using (var temp = new ExtendedBinaryWriter(tempMemoryStream, FileEncoding, Endianness.LittleEndian))
{
temp.WriteString(sectionName);
var aux = input.PeekInt16();
while (aux != -1)
{
var index = input.ReadByte();
temp.Write(index);
var size = input.ReadByte();
var offset = (int)input.Position;
WriteSubtitle(temp, subtitles, offset);
input.ReadString();
aux = input.PeekInt16();
}
input.Skip(2);
temp.Write((short)-1);
var bytes = tempMemoryStream.ToArray();
output.Write((short)(outputOffset + 2 + bytes.Length));
output.Write(bytes);
outputOffset = outputOffset + 2 + bytes.Length;
}
}
}
}
public static ProcessInvocationStartEvent ReadProcessInvocationStartEvent(ExtendedBinaryReader reader)
{
return(new ProcessInvocationStartEvent
{
InvocationUID = reader.Read7BitEncodedInt(),
InstanceUID = reader.Read7BitEncodedInt(),
InvocationCounter = reader.Read7BitEncodedInt(),
Type = reader.ReadString(),
Kind = reader.ReadNullableString(),
Name = reader.ReadString(),
Topic = reader.ReadNullableString(),
CallerInvocationUID = reader.ReadNullableInt32()
});
}
protected virtual void LoadChanges(IList <Subtitle> subtitles)
{
if (HasChanges)
{
using (var fs = new FileStream(ChangesFile, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs, Encoding.Unicode))
{
var version = input.ReadInt32();
if (version != ChangesFileVersion)
{
//File.Delete(ChangesFile);
return;
}
var subtitleCount = input.ReadInt32();
for (var i = 0; i < subtitleCount; i++)
{
var offset = input.ReadInt64();
var text = input.ReadString();
var subtitle = subtitles.FirstOrDefault(x => x.Offset == offset);
if (subtitle != null)
{
subtitle.PropertyChanged -= SubtitlePropertyChanged;
subtitle.Translation = text;
subtitle.Loaded = subtitle.Translation;
subtitle.PropertyChanged += SubtitlePropertyChanged;
}
}
}
}
}
public static NgaReader Read(Stream stream)
{
(IDataSourceVersion version, string jsonKey, bool isArray) = ReadHeader(stream);
Dictionary <string, List <string> > geneSymbolToJsonStrings;
using (var blockStream = new BlockStream(new Zstandard(), stream, CompressionMode.Decompress))
using (var reader = new ExtendedBinaryReader(blockStream))
{
int geneCount = reader.ReadOptInt32();
geneSymbolToJsonStrings = new Dictionary <string, List <string> >(geneCount);
for (var i = 0; i < geneCount; i++)
{
string geneSymbol = reader.ReadAsciiString();
int numEntries = reader.ReadOptInt32();
var entries = new List <string>(numEntries);
for (var j = 0; j < numEntries; j++)
{
entries.Add(reader.ReadString());
}
geneSymbolToJsonStrings[geneSymbol] = entries;
}
}
return(new NgaReader(version, jsonKey, isArray, geneSymbolToJsonStrings));
}
public LogEvent ReadLogEvent(ExtendedBinaryReader reader)
{
var evt = new LogEvent
{
TransactionId = GetTextById(reader.Read7BitEncodedInt()),
Text = reader.ReadString(),
Severity = (LogSeverity)reader.ReadByte(),
ProcessInvocationUID = reader.ReadNullableInt32()
};
var argCount = reader.Read7BitEncodedInt();
if (argCount > 0)
{
evt.Arguments = new KeyValuePair <string, object> [argCount];
for (var i = 0; i < argCount; i++)
{
var key = GetTextById(reader.Read7BitEncodedInt());
var value = reader.ReadObject();
evt.Arguments[i] = new KeyValuePair <string, object>(key, value);
}
}
return(evt);
}
private void ReadHeader()
{
var header = _reader.ReadString();
if (header != SaDataBaseCommon.DataHeader)
{
throw new FormatException("Unrecognized header in this database");
}
// ReSharper disable UnusedVariable
var dataVersion = _reader.ReadUInt16();
var schema = _reader.ReadUInt16();
if (schema != SaDataBaseCommon.SchemaVersion)
{
throw new UserErrorException($"Gene database schema mismatch. Expected {SaDataBaseCommon.SchemaVersion}, observed {schema}");
}
var genomeAssembly = (GenomeAssembly)_reader.ReadByte();
var creationTime = _reader.ReadInt64();
// ReSharper restore UnusedVariable
var dataSourseVersionsCount = _reader.ReadOptInt32();
for (var i = 0; i < dataSourseVersionsCount; i++)
{
DataSourceVersions.Add(DataSourceVersion.Read(_reader));
}
CheckGuard();
}
protected override IList <Subtitle> GetSubtitles()
{
var result = new List <Subtitle>();
using (var fs = new FileStream(Path, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs, FileEncoding))
{
while (input.Position < input.Length)
{
var nextOffset = input.ReadInt16();
var sectionName = input.ReadString(Encoding.UTF8);
var aux = input.PeekInt16();
while (aux != -1)
{
input.Skip(2);
var subtitle = ReadSubtitle(input);
subtitle.PropertyChanged += SubtitlePropertyChanged;
result.Add(subtitle);
aux = input.PeekInt16();
}
input.Skip(2);
}
}
LoadChanges(result);
return(result);
}
private static IList <FileInfo> ReadFileInfo(ExtendedBinaryReader log)
{
var result = new List <FileInfo>();
while (log.Position < log.Length)
{
var fileName = log.ReadString();
var originalCompressedSize = log.ReadInt32();
var maxSize = log.ReadInt32();
var unknown = log.ReadInt32();
var timestamp = log.ReadInt32();
var originalOffset = log.ReadInt32();
var compressed = log.ReadInt32() == 1;
var fi = new FileInfo
{
FileName = fileName,
CompressedSize = originalCompressedSize,
MaxSize = maxSize,
Unknown = unknown,
TimeStamp = timestamp,
OriginalOffset = originalOffset,
IsCompressed = compressed
};
result.Add(fi);
}
return(result);
}
private static IDictionary <int, SizeF[]> ReadBoxSizes(string path)
{
var result = new Dictionary <int, SizeF[]>();
var file = System.IO.Path.Combine(path, "resources_00001.-13");
using (var fs = new FileStream(file, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs))
{
input.BaseStream.Seek(0x150, SeekOrigin.Begin);
var heightCount = input.ReadInt32();
for (var i = 0; i < heightCount; i++)
{
var nameLength = input.ReadInt32();
var name = input.ReadString();
var msgEnum = input.ReadInt32();
var langCount = input.ReadInt32();
var sizes = new SizeF[langCount];
for (var j = 0; j < langCount; j++)
{
var sz = new SizeF(input.ReadSingle(), input.ReadSingle());
sizes[j] = sz;
}
result.Add(msgEnum, sizes);
}
}
return(result);
}
protected override IList <Subtitle> GetSubtitles()
{
var result = new List <Subtitle>();
var encryptedData = System.IO.File.ReadAllBytes(Path);
var data = EncryptionManager.DecryptData(encryptedData);
using (var ms = new MemoryStream(data))
using (var input = new ExtendedBinaryReader(ms, FileEncoding))
{
while (input.Position + 2 <= input.Length)
{
var offset = input.Position;
var id = input.ReadUInt16();
var text = input.ReadString();
text = text.Replace(",", "<Break>");
text = text.ToHalfWidthChars();
var subtitle = new Subtitle {
Offset = offset, Text = text, Translation = text, Loaded = text
};
subtitle.PropertyChanged += SubtitlePropertyChanged;
result.Add(subtitle);
}
}
LoadChanges(result);
return(result);
}
/// <summary>
/// checks if the header is good
/// </summary>
private static SupplementaryAnnotationHeader GetHeader(ExtendedBinaryReader reader, out long intervalsPosition, string saPath = null)
{
// check the header and data version
var header = System.Text.Encoding.ASCII.GetString(reader.ReadBytes(SupplementaryAnnotationCommon.DataHeader.Length));
var dataVersion = reader.ReadUInt16();
var schemaVersion = reader.ReadUInt16();
var genomeAssembly = (GenomeAssembly)reader.ReadByte();
if (header != SupplementaryAnnotationCommon.DataHeader || schemaVersion != SupplementaryAnnotationCommon.SchemaVersion)
{
throw new UserErrorException($"The header check failed for the supplementary annotation file ({saPath ?? "(resource)"}): ID: exp: {SupplementaryAnnotationCommon.DataHeader} obs: {header}, schema version: exp:{SupplementaryAnnotationCommon.SchemaVersion} obs: {schemaVersion}");
}
var creationTimeTicks = reader.ReadInt64();
var referenceSequenceName = reader.ReadString();
// skip over the offsets since they're not currently used
reader.ReadInt64(); // _dataSourceVersionsOffset
reader.ReadInt64(); // _dataOffset
intervalsPosition = reader.ReadInt64();
reader.ReadInt64(); // _eofOffset
// load the data source versions
var numDataSourceVersions = reader.ReadOptInt32();
var dataSourceVersions = new List <DataSourceVersion>();
for (var i = 0; i < numDataSourceVersions; i++)
{
dataSourceVersions.Add(DataSourceVersion.Read(reader));
}
return(new SupplementaryAnnotationHeader(referenceSequenceName, creationTimeTicks, dataVersion,
dataSourceVersions, genomeAssembly));
}
protected override void LoadChanges(IList <Subtitle> subtitles)
{
if (!HasChanges)
{
return;
}
List <SubtitleWithId> subs = subtitles.Select(subtitle => subtitle as SubtitleWithId).ToList();
var dictionary = new Dictionary <string, SubtitleWithId>(subs.Count);
foreach (SubtitleWithId subtitle in subs)
{
if (!dictionary.ContainsKey(subtitle.Id))
{
dictionary.Add(subtitle.Id, subtitle);
}
}
using (var fs = new FileStream(ChangesFile, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs, System.Text.Encoding.Unicode))
{
int version = input.ReadInt32();
if (version != ChangesFileVersion)
{
//File.Delete(ChangesFile);
return;
}
int subtitleCount = input.ReadInt32();
for (int i = 0; i < subtitleCount; i++)
{
string id = input.ReadString();
string text = input.ReadString();
if (dictionary.TryGetValue(id, out SubtitleWithId subtitle))
{
subtitle.PropertyChanged -= SubtitlePropertyChanged;
subtitle.Translation = text;
subtitle.Loaded = subtitle.Translation;
subtitle.PropertyChanged += SubtitlePropertyChanged;
}
}
}
}
public sealed override void Read(ExtendedBinaryReader reader)
{
if (Reference != null)
{
throw new InvalidOperationException("Already initialized");
}
Reference = reader.ReadString();
}
private void ReadHeaders()
{
var count = reader.Read7BitEncodedInt();
FieldNames = new string[count];
RawTypes = new ClickHouseType[count];
CurrentRow = new object[count];
for (var i = 0; i < count; i++)
{
FieldNames[i] = reader.ReadString();
}
for (var i = 0; i < count; i++)
{
var chType = reader.ReadString();
RawTypes[i] = TypeConverter.ParseClickHouseType(chType);
}
}
public override void Load(Stream fileStream)
{
var reader = new ExtendedBinaryReader(fileStream, true);
string sig = reader.ReadSignature();
if (sig != "SCSE")
{
throw new InvalidSignatureException("SCSE", sig);
}
int entryCount = reader.ReadInt32();
for (int i = 0; i < entryCount; ++i)
{
var entry = new Entry();
entry.OrcFileName = reader.ReadString();
entry.FriendlyName = reader.ReadString();
Entries.Add(entry);
}
}
private void CheckHeaderVersion()
{
string headerTag = _reader.ReadString();
int headerVersion = _reader.ReadInt32();
if (headerTag != ReferenceSequenceCommon.HeaderTag || headerVersion != ReferenceSequenceCommon.HeaderVersion)
{
throw new InvalidFileFormatException($"The header identifiers do not match the expected values: Obs: {headerTag} {headerVersion} vs Exp: {ReferenceSequenceCommon.HeaderTag} {ReferenceSequenceCommon.HeaderVersion}");
}
}
private IEnumerable <(string refAllele, string altAllele, string jsonString)> ExtractAnnotations(byte[] data)
{
using (var reader = new ExtendedBinaryReader(new MemoryStream(data)))
{
if (IsPositional)
{
var positionalAnno = reader.ReadString();
return(new List <(string, string, string)> {
(null, null, positionalAnno)
});
public void Read(ExtendedBinaryReader reader)
{
int count = reader.Read7BitEncodedInt();
for (int i = 0; i < count; i++)
{
string assemblyQualifiedName = reader.ReadString();
var type = Type.GetType(assemblyQualifiedName);
RegisterDefaultType(type);
}
}
public SuppInterval(ExtendedBinaryReader reader)
{
string ensemblName = reader.ReadAsciiString();
string ucscName = reader.ReadAsciiString();
ushort chromIndex = reader.ReadOptUInt16();
Chromosome = new Chromosome(ucscName, ensemblName, chromIndex);
Start = reader.ReadOptInt32();
End = reader.ReadOptInt32();
_jsonString = reader.ReadString();
}
private TableData GetData()
{
var result = new TableData();
using (var fs = new FileStream(Path, FileMode.Open))
using (var input = new ExtendedBinaryReader(fs, FileEncoding, Endianness.BigEndian))
{
var signature = input.ReadInt32();
if (signature != 0x20070319)
{
return(result);
}
var numColumns = input.ReadInt32();
var numRows = input.ReadInt32();
result.NumRows = numRows;
input.Skip(4);
for (var i = 0; i < numColumns; i++)
{
input.Seek(16 + i * 64, SeekOrigin.Begin);
var columnName = input.ReadString(Encoding.UTF8);
input.Seek(16 + i * 64 + 48, SeekOrigin.Begin);
var type = input.ReadInt32();
var dataCount = input.ReadInt32();
var sectionSize = input.ReadInt32();
input.Skip(4);
var column = ColumnFactory.GetColumn(type, numRows);
column.Name = columnName;
column.DataCount = dataCount;
column.Size = sectionSize;
column.PropertyChanged += SubtitlePropertyChanged;
result.Columns.Add(column);
}
for (var i = 0; i < numColumns; i++)
{
var column = result.Columns[i];
column.ReadData(input);
}
}
LoadChanges(result);
return(result);
}
public static SuppInterval Read(ExtendedBinaryReader reader)
{
string ensemblName = reader.ReadAsciiString();
string ucscName = reader.ReadAsciiString();
ushort chromIndex = reader.ReadOptUInt16();
var chromosome = new Chromosome(ucscName, ensemblName, null, null, 1, chromIndex);
var start = reader.ReadOptInt32();
var end = reader.ReadOptInt32();
var jsonString = reader.ReadString();
return(new SuppInterval(chromosome, start, end, jsonString));
}
public override void ReadData(ExtendedBinaryReader input)
{
var startPos = input.Position;
for (var i = 0; i < DataCount; i++)
{
var str = input.ReadString();
_data.Add(str);
_translatedData.Add(str);
_loadedData.Add(str);
}
_remainder = input.ReadBytes((int)(startPos + Size - input.Position));
}
public static ISaDataSource Read(ExtendedBinaryReader reader)
{
var keyName = reader.ReadString();
var vcfkeyName = reader.ReadString();
var altAllele = reader.ReadString();
var flags = reader.ReadByte();
var matchByAllele = (flags & 1) != 0;
var isArray = (flags & 2) != 0;
var vcfString = reader.ReadString();
var numJsonStrings = reader.ReadOptInt32();
string[] jsonStrings = null;
if (numJsonStrings > 0)
{
jsonStrings = new string[numJsonStrings];
for (int i = 0; i < numJsonStrings; i++)
{
jsonStrings[i] = reader.ReadString();
}
}
return(new SaDataSource(keyName, vcfkeyName, altAllele, matchByAllele, isArray, vcfString, jsonStrings));
}
private void LoadHeader()
{
var identifier = _reader.ReadString();
if (identifier != PhylopCommon.Header)
{
throw new InvalidDataException("Unrecognized file header: " + identifier);
}
var schemaVersion = _reader.ReadInt16();
if (schemaVersion != PhylopCommon.SchemaVersion)
{
throw new InvalidDataException("Expected phylop schema version:" + PhylopCommon.SchemaVersion + " observed schema version: " + schemaVersion);
}
var dataVersion = _reader.ReadInt16();
if (dataVersion != PhylopCommon.DataVersion)
{
Console.WriteLine("WARNING: Expected phylop data version:" + PhylopCommon.DataVersion + " observed data version: " + dataVersion);
}
_genomeAssembly = (GenomeAssembly)_reader.ReadByte();
_version = DataSourceVersion.Read(_reader);
// skip the reference name
_reader.ReadString();
_intervalListPosition = _reader.ReadInt64();
CheckGuard();
LoadChromosomeIntervals();
IsInitialized = true;
}