public override void Rebuild(string outputFolder)
{
var outputPath = System.IO.Path.Combine(outputFolder, RelativePath);
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(outputPath));
var subtitles = GetSubtitles();
var subs = subtitles.Select(subtitle => subtitle as SubtitleWithId).ToList();
var dictionary = new Dictionary <string, SubtitleWithId>(subs.Count);
foreach (SubtitleWithId subtitle in subs)
{
dictionary.Add(subtitle.Id, subtitle);
}
using (var fsInput = new FileStream(Path, FileMode.Open))
using (var input = new ExtendedBinaryReader(fsInput, FileEncoding))
using (var fsOutput = new FileStream(outputPath, FileMode.Create))
using (var output = new ExtendedBinaryWriter(fsOutput, FileEncoding))
{
var count = input.ReadInt32();
output.Write(count);
for (int i = 0; i < count; i++)
{
// UILabel
output.Write(input.ReadBytes(0x04));
output.Write(input.ReadBytes(0x08));
var strJap = input.ReadStringSerialized(0x04);
var strEng = input.ReadStringSerialized(0x04);
output.WriteStringSerialized(strJap, 0x04);
output.WriteStringSerialized(dictionary.TryGetValue((i + 1).ToString(), out SubtitleWithId subtitle) ? subtitle.Translation : strEng, 0x04);
}
}
}
public override void SaveChanges()
{
using (var fs = new FileStream(ChangesFile, FileMode.Create))
using (var output = new ExtendedBinaryWriter(fs, Encoding.Unicode))
{
output.Write(ChangesFileVersion);
foreach (var column in _data.Columns)
{
column.SaveChanges(output);
}
}
NeedSaving = false;
OnFileChanged();
}
public void Write(Dictionary <string, List <ISuppGeneItem> > geneToEntries)
{
_writer.WriteOpt(geneToEntries.Count);
foreach ((string geneSymbol, List <ISuppGeneItem> entries) in geneToEntries)
{
_writer.WriteOptAscii(geneSymbol);
_writer.WriteOpt(entries.Count);
foreach (ISuppGeneItem geneItem in entries)
{
_writer.Write(geneItem.GetJsonString());
}
}
}
public (int firstPosition, int lastPosition, int numBytes) Write(ExtendedBinaryWriter writer)
{
var compressedLength = _compressionAlgorithm.Compress(_uncompressedBlock, BlockOffset,
_compressedBlock, _compressedBlock.Length);
writer.WriteOpt(compressedLength);
writer.WriteOpt(_firstPosition);
//writer.WriteOpt(_lastPosition);
writer.WriteOpt(_count);
writer.Write(_compressedBlock, 0, compressedLength);
_writer.BaseStream.Position = 0;
return(_firstPosition, _lastPosition, compressedLength);
}
private long WriteString(ExtendedBinaryWriter output, IList <Subtitle> subtitles, IDictionary <long, long> used, long inputOffset, long outputOffset)
{
var result = outputOffset;
if (inputOffset == 0)
{
output.Write(0);
}
else
{
if (!used.ContainsKey(inputOffset))
{
used.Add(inputOffset, outputOffset);
var sub = subtitles.First(x => x.Offset == inputOffset);
result = WriteSubtitle(output, sub, outputOffset, true);
}
else
{
output.Write((int)used[inputOffset]);
}
}
return(result);
}
/// <summary>
/// writes out each transcript in the cache
/// </summary>
private void WriteTranscripts(DataStructures.GlobalCache cache)
{
if (cache.Transcripts == null)
{
_writer.WriteOpt(0);
}
else
{
// create index dictionaries for each data type
var geneIndices = CreateIndex(cache.Genes);
var intronIndices = CreateIndex(cache.Introns);
var microRnaIndices = CreateIndex(cache.MicroRnas);
var peptideIndices = CreateIndex(cache.PeptideSeqs);
// write the transcripts
_writer.WriteOpt(cache.Transcripts.Length);
foreach (var transcript in cache.Transcripts)
{
transcript.Write(_writer, geneIndices, intronIndices, microRnaIndices, peptideIndices);
}
}
_writer.Write(CacheConstants.GuardInt);
}
private long WriteItem(ExtendedBinaryWriter output, IList <Subtitle> subtitles, TownElement item, long outputOffset)
{
var returnPos = output.Position + 2;
var subtitle = subtitles.First(x => x.Offset == item.Title.Offset);
WriteSubtitle(output, subtitle, outputOffset, false);
output.Write(item.Unknown);
var currentOffset = output.Position;
output.Seek(returnPos, SeekOrigin.Begin);
return(currentOffset);
}
public override void FinishWrite(ExtendedBinaryWriter writer)
{
// BINA Header
writer.WriteSignature(Signature);
writer.WriteSignature(Version.ToString());
writer.Write((IsBigEndian) ? BigEndianFlag : LittleEndianFlag);
writer.Write(FileSize);
// TODO: Write Nodes Properly
writer.Write((ushort)1); // NodeCount
writer.Write((ushort)0); // Possibly IsFooterMagicPresent?
// DATA Header
writer.WriteSignature(DataSignature);
writer.Write(DataLength);
writer.Write(StringTableOffset);
writer.Write(StringTableLength);
writer.Write(FinalTableLength);
writer.Write((ushort)(writer.Offset - (writer.BaseStream.Position + 4)));
}
public void Write(ExtendedBinaryWriter writer, string offsetSuffix = "")
{
writer.AddOffset($"texNameOffset{offsetSuffix}");
writer.Write(TexFlags);
writer.AddOffset($"texTypeOffset{offsetSuffix}");
// Texture Name
writer.FillInOffset($"texNameOffset{offsetSuffix}", false, false);
writer.WriteNullTerminatedString(TextureName);
// Texture Type
writer.FillInOffset($"texTypeOffset{offsetSuffix}", false, false);
writer.WriteNullTerminatedString(Type);
writer.FixPadding(4);
}
public void Write(IEnumerable <PhylopItem> items)
{
var benchmark = new Benchmark();
int lastPosition = 0;
foreach (PhylopItem item in items)
{
if (item.Chromosome.Index != _chromIndex)
{
//flush out old chrom
if (_chromIndex != ushort.MaxValue)
{
WriteCompressed(lastPosition);
Console.WriteLine($"Chromosome {_chromName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
benchmark.Reset();
}
_chromIndex = item.Chromosome.Index;
_chromName = item.Chromosome.EnsemblName;
}
if (!_scoreMap.TryGetValue(item.Score, out byte _))
{
_scoreMap.Add(item.Score, _nextScoreCode++);
if (_nextScoreCode == byte.MaxValue)
{
throw new ArgumentOutOfRangeException($"No of distinct scores exceeded expected value of {_nextScoreCode}!!");
}
}
_memStream.Position = item.Position - 1;
_memWriter.Write(_scoreMap[item.Score]);
lastPosition = item.Position;
}
//closing the last chromosome
WriteCompressed(lastPosition);
Console.WriteLine($"Chromosome {_chromName} completed in {Benchmark.ToHumanReadable(benchmark.GetElapsedTime())}");
benchmark.Reset();
Console.WriteLine($"\nNumber of distinct scores oberved:{_scoreMap.Count}");
_index.Write(_scoreMap);
}
/// <summary>
/// Writes all of the nodes contained in this list to the underlying stream, prefixed
/// by the number of elements encoded using LEB128 encoding.
/// </summary>
/// <param name="writer">Writer used for writing to the underlying stream.</param>
public override void WriteValue(ExtendedBinaryWriter writer)
{
writer.Write7BitEncodedInt(_list.Count);
foreach (var node in _list)
{
if (node == null)
{
writer.Write((byte)BsfType.Null);
}
else
{
node.WriteType(writer);
node.WriteValue(writer);
}
}
}
public void Write(IEnumerable <ISuppIntervalItem> siItems)
{
var sortedItems = siItems.OrderBy(x => x.Chromosome.Index).ThenBy(x => x.Start).ThenBy(x => x.End).ToList();
Console.WriteLine($"Writing {sortedItems.Count} intervals to database...");
_writer.WriteOpt(sortedItems.Count);
foreach (ISuppIntervalItem item in sortedItems)
{
_writer.WriteOptAscii(item.Chromosome.EnsemblName);
_writer.WriteOptAscii(item.Chromosome.UcscName);
_writer.WriteOpt(item.Chromosome.Index);
_writer.WriteOpt(item.Start);
_writer.WriteOpt(item.End);
_writer.Write(item.GetJsonString());
}
}
public override void Save(ExtendedBinaryWriter writer)
{
ManualCount = 0;
var strings = new List <string>();
writer.WriteSignature("STSC");
writer.AddOffset("EntryPosition");
writer.Write(0x07); // Version
writer.WriteSignature(ScriptName);
writer.WriteNulls((uint)(0x20 - ScriptName.Length)); // Pad Script Name
writer.Write(0x000607E3);
writer.Write((short)0x06);
writer.Write((short)0x0A);
writer.Write((short)0x06);
writer.Write((short)0x17);
writer.Write(ScriptID);
writer.FillInOffset("EntryPosition");
foreach (var instruction in Instructions)
{
writer.Write((byte)STSCInstructions.DALRRInstructions.FindIndex(t => t?.Name == instruction.Name));
instruction.Write(writer, ref ManualCount, strings);
}
// Write String Table
for (int i = 0; i < strings.Count; ++i)
{
writer.FillInOffset($"Strings_{i}");
writer.WriteNullTerminatedString(strings[i]);
}
writer.FixPadding(0x04);
WriteStage1(writer);
WriteStage2(writer);
WriteStage3(writer);
WriteStage4(writer);
WriteStage5(writer);
WriteStage6(writer);
WriteStage7(writer);
WriteStage8(writer);
WriteStage9(writer);
WriteStage10(writer);
WriteStage11(writer);
writer.FixPadding(0x10);
}
public void Write(ExtendedBinaryWriter writer)
{
writer.WriteOptArray(_alleleOrigins?.Distinct().ToArray(), writer.WriteOptAscii);
writer.WriteOptAscii(SaAltAllele);
writer.WriteOptAscii(ReferenceAllele);
writer.WriteOptAscii(ID);
writer.Write((byte)ReviewStatus);
writer.WriteOptAscii(IsAlleleSpecific);
writer.WriteOptArray(_medgenIds?.Distinct().ToArray(), writer.WriteOptAscii);
writer.WriteOptArray(_omimIds?.Distinct().ToArray(), writer.WriteOptAscii);
writer.WriteOptArray(_orphanetIds?.Distinct().ToArray(), writer.WriteOptAscii);
writer.WriteOptArray(SupplementaryAnnotationUtilities.ConvertMixedFormatStrings(_phenotypes)?.Distinct().ToArray(), writer.WriteOptUtf8);
writer.WriteOptAscii(Significance);
writer.WriteOpt(LastUpdatedDate);
writer.WriteOptArray(_pubmedIds.ToArray(), writer.WriteOpt);
}
private void WriteHeader()
{
_writer.Write(CustomIntervalCommon.DataHeader);
_writer.Write(CustomIntervalCommon.SchemaVersion);
_writer.Write(DateTime.UtcNow.Ticks);
_writer.Write(_referenceName);
_writer.Write(_intervalType);
_version.Write(_writer);
// marking end of header with guard int
_writer.Write(CustomIntervalCommon.GuardInt);
}
public void Add(byte[] data, int length, int position)
{
if (!HasSpace(length))
{
return;
}
if (_writer.BaseStream.Position == 0)
{
_firstPosition = position;
_lastPosition = position;
}
_writer.WriteOpt(length);
_writer.WriteOpt(position - _lastPosition);
_writer.Write(data, 0, length);
_lastPosition = position;
_count++;
}
private void RebuildFontTable(string outputFile)
{
if (!System.IO.File.Exists(outputFile))
{
System.IO.File.Copy(Path, outputFile);
}
var data = GetFontTable();
using (var fs = new FileStream(outputFile, FileMode.Open))
using (var output = new ExtendedBinaryWriter(fs))
{
output.Seek(FontTableOffset, SeekOrigin.Begin);
for (var i = 0; i < 128; i++)
{
output.Write(data[i].Width);
}
}
}
private long WriteItem(ExtendedBinaryWriter output, IList <Subtitle> subtitles, ItemElement item, long outputOffset)
{
output.Write((ushort)outputOffset);
var returnPos = output.Position;
output.Seek(outputOffset, SeekOrigin.Begin);
var subtitle = subtitles.First(x => x.Offset == item.Title.Offset);
var currentOffset = outputOffset + 4;
currentOffset = WriteSubtitle(output, subtitle, currentOffset, true);
subtitle = subtitles.First(x => x.Offset == item.Description.Offset);
currentOffset = WriteSubtitle(output, subtitle, currentOffset, true);
output.Seek(returnPos, SeekOrigin.Begin);
return(currentOffset);
}
/// <summary>
/// Characters
/// </summary>
public void WriteStage5(ExtendedBinaryWriter writer)
{
writer.FillInOffset("Manual_Ptr_4l", true, false);
int i = 0;
foreach (var entry in _characters)
{
writer.AddOffset($"Name{i}");
writer.Write(entry.ID);
++i;
}
writer.FillInOffset("Manual_Ptr_4h", true, false);
i = 0;
foreach (var entry in _characters)
{
writer.FillInOffset($"Name{i}");
writer.WriteNullTerminatedString(entry.FriendlyName);
++i;
}
writer.FixPadding(0x4);
}
protected override long WriteSubtitle(ExtendedBinaryWriter output, IList <Subtitle> subtitles, long inputOffset, long outputOffset)
{
var sub = subtitles.First(x => x.Offset == inputOffset);
output.Write((ushort)outputOffset);
var returnPos = output.Position;
var evaluator = new MatchEvaluator(SubtitleMatchEvaluator);
var newSub = Regex.Replace(sub.Translation, @"<Color: (?<colorValue>[0-9A-Fa-f]{2})>", evaluator);
newSub = Regex.Replace(newSub, @"<Item: (?<itemValue>[0-9A-Fa-f]{4})>", evaluator);
output.Seek(outputOffset, SeekOrigin.Begin);
output.WriteString(newSub);
var result = output.Position;
output.Seek(returnPos, SeekOrigin.Begin);
return(result);
}
public override void Rebuild(string outputFolder)
{
var outputPath = System.IO.Path.Combine(outputFolder, RelativePath);
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(outputPath));
var subtitles = GetSubtitles();
var encryptedInputData = System.IO.File.ReadAllBytes(Path);
var inputData = EncryptionManager.DecryptData(encryptedInputData);
byte[] outputData;
using (var msInput = new MemoryStream(inputData))
using (var input = new ExtendedBinaryReader(msInput, FileEncoding))
using (var msOutput = new MemoryStream())
using (var output = new ExtendedBinaryWriter(msOutput, FileEncoding))
{
while (input.Position + 2 <= input.Length)
{
var offset = input.Position;
var id = input.ReadUInt16();
var inputText = input.ReadString();
var subtitle = subtitles.First(x => x.Offset == offset);
var outputText = subtitle.Translation.Replace("<Break>", "\u0001");
outputText = outputText.ToFullWidthChars();
outputText = outputText.Replace("\u0001", ",");
output.Write(id);
output.WriteString(outputText);
}
outputData = msOutput.ToArray();
}
var encryptedOutputData = EncryptionManager.EncryptData(outputData);
System.IO.File.WriteAllBytes(outputPath, encryptedOutputData);
}
public void BufferedBinaryReader_EndToEnd_DoNotLeaveOpen()
{
const int expectedResult = 5;
int observedResult;
byte[] data;
using (var ms = new MemoryStream())
using (var writer = new ExtendedBinaryWriter(ms))
{
writer.Write(expectedResult);
data = ms.ToArray();
}
using (var ms = new MemoryStream(data))
using (var reader = new ExtendedBinaryReader(ms))
{
observedResult = reader.ReadInt32();
}
Assert.Equal(expectedResult, observedResult);
}
protected override long WriteSubtitle(ExtendedBinaryWriter output, Subtitle subtitle, long offset, bool returnToPos)
{
var result = offset;
var pos = output.Position;
output.Write((ushort)offset);
output.Seek(offset, SeekOrigin.Begin);
if (!string.IsNullOrEmpty(subtitle.Text))
{
output.WriteString(subtitle.Translation);
result = output.Position;
}
if (returnToPos)
{
output.Seek(pos + 2, SeekOrigin.Begin);
}
return(result);
}
private void WriteHeader(ISupplementaryAnnotationHeader header)
{
_writer.Write(SaDataBaseCommon.DataHeader);
_writer.Write(SaDataBaseCommon.DataVersion);
_writer.Write(SaDataBaseCommon.SchemaVersion);
_writer.Write((byte)header.GenomeAssembly);
_writer.Write(DateTime.Now.Ticks);
var dataSourceVersions = header.DataSourceVersions.ToList();
_writer.WriteOpt(dataSourceVersions.Count);
foreach (var version in dataSourceVersions)
{
version.Write(_writer);
}
_writer.Write(SaDataBaseCommon.GuardInt);
}
public int Save(Stream fileStream, uint?sizeLimit, int startIndex = 0)
{
// Header
var files = GetFiles(false);
var writer = new ExtendedBinaryWriter(fileStream, Encoding.ASCII, false);
writer.Write(Sig1);
writer.Write(Sig2);
writer.Write(Sig3);
writer.Write(Padding);
// The absolute smallest an archive with a header and one
// valid file entry can be is 37 bytes.
if (sizeLimit.HasValue && sizeLimit < 37)
{
throw new ArgumentOutOfRangeException("sizeLimit", sizeLimit,
"The sizeLimit argument must at least be greater than 36.");
}
// Data
for (int i = startIndex; i < files.Count; ++i)
{
var file = files[i];
writer.Offset = (uint)fileStream.Position;
if (sizeLimit.HasValue && i > startIndex && fileStream.Position +
MinFileEntrySize + file.Data.Length > sizeLimit)
{
return(i);
}
writer.AddOffset("dataEndOffset");
writer.Write((uint)file.Data.LongLength);
writer.AddOffset("dataStartOffset");
writer.WriteNulls(8); // TODO: Figure out what unknown1 and unknown2 are.
writer.WriteNullTerminatedString(file.Name);
writer.FixPadding(Padding);
writer.FillInOffset("dataStartOffset", false);
writer.Write(file.Data);
writer.FillInOffset("dataEndOffset", false);
}
return(-1);
}
internal ReferenceBuffer GetReferenceBuffer(ushort refIndex)
{
int bufferSize;
byte[] buffer;
using (var ms = new MemoryStream())
{
using (var writer = new ExtendedBinaryWriter(ms, Encoding.UTF8, true))
{
writer.Write(ReferenceSequenceCommon.ReferenceStartTag);
WriteMetadata(writer);
WriteBuffer(writer);
WriteMaskedEntries(writer);
WriteCytogeneticBands(writer);
}
bufferSize = (int)ms.Position;
buffer = ms.ToArray();
}
return(new ReferenceBuffer(refIndex, buffer, bufferSize));
}
public void Reset()
{
const string s = "The quick brown fox jumped over the lazy dog.";
string expectedValue = s.Substring(4);
string observedValue;
long observedBufferPosition;
byte observedByte;
using (var memoryStream = new MemoryStream())
{
using (var writer = new ExtendedBinaryWriter(memoryStream, Encoding.UTF8, true))
{
writer.Write(Encoding.ASCII.GetBytes(s));
}
memoryStream.Position = 0;
using (var reader = new BufferedBinaryReader(memoryStream))
{
reader.ReadInt32();
observedBufferPosition = reader.BufferPosition;
reader.BufferPosition = 2;
observedByte = reader.ReadByte();
memoryStream.Position = 4;
reader.Reset();
var bytes = reader.ReadBytes(s.Length - 4);
observedValue = Encoding.ASCII.GetString(bytes);
}
}
Assert.Equal(expectedValue, observedValue);
Assert.Equal((byte)'e', observedByte);
Assert.Equal(4, observedBufferPosition);
}
public override void Rebuild(string outputFolder)
{
var outputPath = System.IO.Path.Combine(outputFolder, RelativePath);
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(outputPath));
System.IO.File.Copy(Path, outputPath, true);
var subtitles = GetSubtitles();
using (var fs = new FileStream(outputPath, FileMode.Open))
using (var output = new ExtendedBinaryWriter(fs, FileEncoding, Endianness.BigEndian))
{
foreach (var subtitle in subtitles)
{
output.Seek(subtitle.Offset, SeekOrigin.Begin);
var sub = subtitle as Subtitle;
var zeros = new byte[sub.MaxLength];
output.Write(zeros);
WriteSubtitle(output, subtitle, (int)subtitle.Offset, false);
}
}
}
private long WriteItem(ExtendedBinaryWriter output, IList <Subtitle> subtitles, ShopElement item, long outputOffset)
{
output.Write((ushort)outputOffset);
var returnPos = output.Position;
output.Seek(outputOffset, SeekOrigin.Begin);
output.Write(item.Id);
output.Write(item.Unknown1);
output.Write((ushort)0x0000);
var subtitle = subtitles.First(x => x.Offset == item.Title.Offset);
var currentOffset = outputOffset + 2 + 0x0C + 4;
currentOffset = WriteSubtitle(output, subtitle, currentOffset, true);
output.Seek(outputOffset + 2 + 0x0C, SeekOrigin.Begin);
output.Write((ushort)currentOffset);
output.Seek(currentOffset, SeekOrigin.Begin);
output.Write(item.Unknown2);
currentOffset = output.Position;
output.Seek(returnPos, SeekOrigin.Begin);
return(currentOffset);
}
public override void Write(ExtendedBinaryWriter writer, object value)
{
var enumIndex = value is string enumStr ? (sbyte)Lookup(enumStr) : Convert.ToSByte(value);
writer.Write(enumIndex);
}
