static void Read(IO.EndianReader s, bool decrypt, IO.IEndianStreamable obj,
long size = 0, ulong userKey = 0, Action <IO.EndianReader> readLeftovers = null)
{
if (!decrypt)
{
obj.Read(s);
}
else
{
using (var ms = new System.IO.MemoryStream())
using (var sout = new IO.EndianWriter(ms, Shell.EndianFormat.Big))
using (var decrypted = new IO.EndianReader(ms, Shell.EndianFormat.Big))
{
long position = s.BaseStream.Position;
var tea = new Security.Cryptography.PhxTEA(s, sout);
tea.InitializeKey(Security.Cryptography.PhxTEA.kKeyGameFile, userKey);
tea.Decrypt(size);
decrypted.Seek(0);
obj.Read(decrypted);
if (readLeftovers != null)
{
readLeftovers(decrypted);
}
}
}
}
static void Write(IO.EndianWriter s, bool encrypt, IO.IEndianStreamable obj,
long size = 0, ulong userKey = 0, Action <IO.EndianWriter> writeLeftovers = null)
{
if (!encrypt)
{
obj.Write(s);
if (writeLeftovers != null)
{
writeLeftovers(s);
}
}
else
{
using (var ms = new System.IO.MemoryStream())
using (var sin = new IO.EndianWriter(ms, Shell.EndianFormat.Big))
using (var encrypted = new IO.EndianReader(ms, Shell.EndianFormat.Big))
{
obj.Write(sin);
if (writeLeftovers != null)
{
writeLeftovers(sin);
}
encrypted.Seek(0);
var tea = new Security.Cryptography.PhxTEA(encrypted, s);
tea.InitializeKey(Security.Cryptography.PhxTEA.kKeyGameFile, userKey);
tea.Encrypt(size);
}
}
}
/// <summary>
/// Validates the cache's header for proper signatures
/// </summary>
/// <param name="s">cache stream</param>
/// <param name="header_sizes">size of a single header</param>
/// <remarks>Assumes the header is at the start of the stream</remarks>
/// <returns>The header size entry that matches this cache or -1 if no matches</returns>
internal static int ValidateHeader(IO.EndianReader s, params int[] header_sizes)
{
bool invalid = true;
int result = -1;
foreach (int header_size in header_sizes)
{
s.Seek(0);
if (s.ReadTagUInt() == MiscGroups.head.ID && s.Length >= header_size) // it HAS to be more than the size of a header...
{
s.Seek(header_size - sizeof(uint));
if (s.ReadTagUInt() == MiscGroups.foot.ID)
{
invalid = false;
result = header_size;
}
}
if (!invalid)
{
break;
}
}
if (invalid)
{
throw new InvalidCacheFileException(s.FileName);
}
return(result);
}
/// <summary>
/// Closes the map resources
/// </summary>
/// <remarks>Eg, tag manager, string id manager, IO streams, etc</remarks>
public virtual void Close()
{
if (!CacheId.IsNull)
{
CloseTagIndexManager();
refManager = null;
StringIdManagerDispose();
if (!IsSharedReference && InputStream != null)
{
InputStream.Close();
}
InputStream = null;
if (!IsSharedReference && OutputStream != null)
{
OutputStream.Close();
}
OutputStream = null;
CacheId = DatumIndex.Null;
}
}
public void Enum_BinaryStreamerUpCastTest()
{
using (var ms = new System.IO.MemoryStream())
using (var br = new IO.EndianReader(ms))
using (var bw = new IO.EndianWriter(ms))
{
const System.TypeCode kExpectedValue = System.TypeCode.String;
var value = kExpectedValue;
TypeCodeStreamer64.Write(bw, value);
ms.Position = 0;
TypeCodeStreamer64.Read(br, out value);
Assert.IsTrue(value == kExpectedValue);
//////////////////////////////////////////////////////////////////////////
// Test the instance interface
var streamer_instance = TypeCodeStreamer64.Instance;
ms.Position = 0;
streamer_instance.Write(bw, value);
ms.Position = 0;
streamer_instance.Read(br, out value);
Assert.IsTrue(value == kExpectedValue);
}
}
protected void ReadGroupTags(Managers.BlamDefinition gd, IO.EndianReader s)
{
uint gt = s.ReadUInt32();
if (gt != uint.MaxValue)
{
GroupTag1 = gd.TagGroupFind(TagInterface.TagGroup.FromUInt(gt));
}
else
{
GroupTag1 = TagInterface.TagGroup.Null;
}
gt = s.ReadUInt32();
if (gt != uint.MaxValue)
{
GroupTag2 = gd.TagGroupFind(TagInterface.TagGroup.FromUInt(gt));
}
else
{
GroupTag2 = TagInterface.TagGroup.Null;
}
gt = s.ReadUInt32();
if (gt != uint.MaxValue)
{
GroupTag3 = gd.TagGroupFind(TagInterface.TagGroup.FromUInt(gt));
}
else
{
GroupTag3 = TagInterface.TagGroup.Null;
}
}
public void Enum_BinaryStreamerUsingUnderlyingTypeTest()
{
using (var ms = new System.IO.MemoryStream())
using (var br = new IO.EndianReader(ms))
using (var bw = new IO.EndianWriter(ms))
{
const UInt32Enum kExpectedValue = UInt32Enum.DeadBeef;
var value = kExpectedValue;
UInt32EnumStreamer.Write(bw, value);
ms.Position = 0;
UInt32EnumStreamer.Read(br, out value);
Assert.IsTrue(value == kExpectedValue);
//////////////////////////////////////////////////////////////////////////
// Test the instance interface
var streamer_instance = UInt32EnumStreamer.Instance;
ms.Position = 0;
streamer_instance.Write(bw, value);
ms.Position = 0;
streamer_instance.Read(br, out value);
Assert.IsTrue(value == kExpectedValue);
}
}
void ReadBitStream(IO.EndianReader s, byte[] hashBuffer)
{
int max_bit_stream_size = GetBitStreamSize();
bool is_probably_from_mcc = hashBuffer.EqualsZero();
byte[] bs_bytes;
using (var hasher = Program.GetGen3RuntimeDataHasher())
{
int bs_length = ReadBitStreamSize(s, hasher, max_bit_stream_size, is_probably_from_mcc);
bs_bytes = new byte[IntegerMath.Align(IntegerMath.kInt32AlignmentBit, bs_length)];
s.Read(bs_bytes, bs_length);
hasher.TransformFinalBlock(bs_bytes, 0, bs_length);
InvalidData = hasher.Hash.EqualsArray(hashBuffer) == false;
}
if (RequireValidHashes && InvalidData)
{
Data = null;
}
else
{
using (var ms = new System.IO.MemoryStream(bs_bytes))
using (var bs = new IO.BitStream(ms, System.IO.FileAccess.Read, streamName: "GameVariant"))
{
bs.StreamMode = System.IO.FileAccess.Read;
Data.Serialize(bs);
}
}
}
/// <summary></summary>
/// <param name="offsets">Buffer containing the offsets of the string values in <paramref name="buffer"/></param>
/// <param name="buffer">Buffer containing the string values</param>
/// <param name="is_packed">If true, reads the strings as null terminated strings, else as 128 character strings</param>
/// <param name="count">Number of dynamic strings for <paramref name="set_index"/></param>
/// <remarks>
/// Doesn't use <paramref name="offsets"/> for reading the strings, assumes the strings are in sequential order in <paramref name="buffer"/>.
///
/// Ignores <see cref="IsReadOnly"/>; will always add new IDs.
/// </remarks>
public void FromDebugStream(IO.EndianReader offsets, IO.EndianReader buffer,
bool is_packed, int count)
{
if (count <= 0)
{
throw new ArgumentOutOfRangeException("count", count, "Must be greater than zero");
}
// Assume the strings are in sequential order in [buffer]
offsets.Seek(sizeof(uint) * count, System.IO.SeekOrigin.Current);
int set_index = InitialId.Set, initial_index = InitialId.Index;
InitializeSet(count);
for (int x = 0; x < count; x++)
{
var str = is_packed ? buffer.ReadCString() : buffer.ReadAsciiString(128);
var sid = mOwner.Definition.Description.Generate(initial_index + x, str.Length, set_index);
m_set.Set.Add(sid, str);
int hc = str.GetHashCode();
StringId first_sid;
if (m_set.SetLookup.TryGetValue(hc, out first_sid))
{
Debug.Warn.If(false, "Hash collision! '{0}' ({1}) collides with '{2}' ({3})",
m_set.Set[first_sid], first_sid.ToString(),
str, sid.ToString());
}
else
{
m_set.SetLookup.Add(str.GetHashCode(), sid);
}
}
}
public void Read(IO.EndianReader s)
{
s.Owner = this;
Flags = s.Read(FileFlagsStreamer.Instance);
Version = s.ReadUInt16();
if (Version != kVersion)
{
throw new IO.VersionMismatchException(s.BaseStream,
kVersion, Version);
}
Read(s, EnumFlags.Test(Flags, FileFlags.EncryptHeader), Header, MediaHeader.kSizeOf);
GenerateHash();
if (EnumFlags.Test(Flags, FileFlags.CompressContent))
{
using (var cs = new CompressedStream(true))
{
Read(s, EnumFlags.Test(Flags, FileFlags.EncryptContent), cs,
userKey: Header.DataCryptKey, readLeftovers: ReadLeftovers);
cs.Decompress();
Content = cs.UncompressedData;
}
}
else
{
Content = s.ReadBytes((int)(s.BaseStream.Length - s.BaseStream.Position));
}
}
public static void XmbToXml(IO.EndianStream xmbStream, System.IO.Stream outputStream, Shell.ProcessorSize vaSize)
{
byte[] xmbBytes;
using (var xmb = new ECF.EcfFileXmb())
{
xmb.Serialize(xmbStream);
xmbBytes = xmb.FileData;
}
var context = new Xmb.XmbFileContext()
{
PointerSize = vaSize,
};
using (var ms = new System.IO.MemoryStream(xmbBytes, false))
using (var s = new IO.EndianReader(ms, xmbStream.ByteOrder))
{
s.UserData = context;
using (var xmbf = new Phoenix.Xmb.XmbFile())
{
xmbf.Read(s);
xmbf.ToXml(outputStream);
}
}
}
/// <summary>
/// Stream the field from a buffer
/// </summary>
/// <param name="input"></param>
public override void Read(IO.EndianReader input)
{
#if false//DEBUG
if (Value < 5)
{
input.Seek(Value, System.IO.SeekOrigin.Current);
}
else
{
byte[] data = input.ReadBytes(Value);
for (int x = 0, y = 0, z = 0; x < data.Length; x++)
{
if (data[x] != 0)
{
if (y == 0 || x == 0)
{
y = x;
}
z++;
}
else if (x != 0 && data[x - 1] != 0)
{
trace.WriteLine("Found data at offset 0x{0:X} covering 0x{1:X} bytes, in a pad field with a count of {1}", y, z, Value);
}
}
data = null;
}
#else
input.Seek(Value, System.IO.SeekOrigin.Current);
#endif
}
/// <summary>
/// Stream the field to a tag stream
/// </summary>
/// <param name="ts"></param>
/// <exception cref="Exceptions.InvalidTagStruct"></exception>
public override void Read(IO.ITagStream ts)
{
IO.EndianReader s = ts.GetInputStream();
base.relativeOffset = s.PositionUnsigned; // use relative offset since the 'fieldset header' is really...well the header
NeedsUpgrading = false;
try
{
if (DefinitionState.FieldSetRequiresVersioningHeader(ts))
{
IFieldSetVersioning fs = kState.FieldSetReadVersion(ts, FieldType.Struct, base.relativeOffset, 1);
if (NeedsUpgrading = fs.NeedsUpgrading)
{
int index, size_of;
fs.GetUpgradeParameters(out index, out size_of);
// It is ASSUMED that the group tag won't ever be needed for version construction
if (!fs.UseImplicitUpgrading)
{
Value = (T)kState.NewInstance(this, index, size_of);
}
else
{
Value.VersionImplicitUpgradeBegin(size_of, ts);
}
}
}
}
catch (Debug.ExceptionLog del) { throw del; }
catch (Exception ex)
{
throw new Exceptions.InvalidTagStruct(ex,
base.relativeOffset, this.owner, ts);
}
}
private void TransformXmbToXml(byte[] eraFileEntryBuffer, string fullPath, Shell.EndianFormat byteOrder, Shell.ProcessorSize vaSize)
{
byte[] xmb_buffer;
using (var xmb = new ECF.EcfFileXmb())
using (var ms = new System.IO.MemoryStream(eraFileEntryBuffer))
using (var es = new IO.EndianStream(ms, byteOrder, permissions: System.IO.FileAccess.Read))
{
es.StreamMode = System.IO.FileAccess.Read;
xmb.Serialize(es);
xmb_buffer = xmb.FileData;
}
string xmb_path = fullPath;
ResourceUtils.RemoveXmbExtension(ref xmb_path);
var context = new Xmb.XmbFileContext()
{
PointerSize = vaSize,
};
using (var ms = new System.IO.MemoryStream(xmb_buffer, false))
using (var s = new IO.EndianReader(ms, byteOrder))
{
s.UserData = context;
using (var xmbf = new Phoenix.Xmb.XmbFile())
{
xmbf.Read(s);
xmbf.ToXml(xmb_path);
}
}
}
private bool DecompressUIFileToDisk(byte[] eraFileEntryBuffer, string fullPath)
{
bool success = false;
using (var ms = new System.IO.MemoryStream(eraFileEntryBuffer, false))
using (var s = new IO.EndianReader(ms, Shell.EndianFormat.Little))
{
uint buffer_signature;
if (ResourceUtils.IsScaleformBuffer(s, out buffer_signature))
{
int decompressed_size = s.ReadInt32();
int compressed_size = (int)(ms.Length - ms.Position);
byte[] decompressed_data = ResourceUtils.DecompressScaleform(eraFileEntryBuffer, decompressed_size);
using (var fs = System.IO.File.Create(fullPath + ".bin"))
{
fs.Write(decompressed_data, 0, decompressed_data.Length);
}
success = true;
}
}
return(success);
}
/// <summary>Read a multi-byte CString from an endian stream</summary>
/// <param name="s">Endian stream to read from</param>
/// <param name="ms">Stream to write the character's bytes to</param>
void ReadCStringMultiByte(IO.EndianReader s, System.IO.MemoryStream ms)
{
byte[] characters;
if (!mStorage.IsFixedLength)
{
characters = new byte[mNullCharacterSize];
while (!ReadStringMultiByteIsNull(s.ByteOrder, s.Read(characters), 0))
{
ms.Write(characters, 0, characters.Length);
}
}
else
{
characters = s.ReadBytes(mFixedLengthByteLength);
int x;
for (x = 0; x < characters.Length - mNullCharacterSize; x += mNullCharacterSize)
{
if (ReadStringMultiByteIsNull(s.ByteOrder, characters, x))
{
break;
}
}
ms.Write(characters, 0, x);
}
}
/// <summary>
/// Stream the field header data from a buffer
/// </summary>
/// <param name="s"></param>
public override void ReadHeader(IO.EndianReader s)
{
headerOffset = s.PositionUnsigned; // nifty for debugging
var owner = Program.GetTagIndex(OwnerId);
Handle.Read(s, owner.StringIds.Definition.Description);
}
/// <summary>
/// Stream the field from a buffer
/// </summary>
/// <param name="input"></param>
public override void Read(IO.EndianReader input)
{
if (StringType == StringType.Normal)
{
Value = input.ReadTagString();
}
else if (StringType == StringType.Unicode)
{
Value = input.ReadUnicodeString(Length);
}
else if (StringType == StringType.Ascii)
{
Value = input.ReadAsciiString(Length);
}
else if (StringType == StringType.Halo1Profile)
{
Value = input.ReadUnicodeString(12);
}
else if (StringType == StringType.Halo2Profile)
{
Value = input.ReadUnicodeString(16);
}
else if (StringType == StringType.CString)
{
Value = input.ReadCString();
}
}
/// <summary>
/// Stream the field header data from a tag stream
/// </summary>
/// <param name="ts"></param>
/// <exception cref="Exceptions.InvalidStringId"></exception>
public override void ReadHeader(IO.ITagStream ts)
{
IO.EndianReader s = ts.GetInputStream();
headerOffset = s.PositionUnsigned; // nifty for debugging
OwnerId = ts.OwnerId;
string value = null;
try
{
if (fieldType == FieldType.OldStringId && ts.Flags.Test(IO.ITagStreamFlags.Halo2OldFormat_StringId))
{
value = s.ReadAsciiString(28); // max of 28 characters in the string id in old builds
}
var owner = Program.GetTagIndex(OwnerId);
Handle.Read(s, owner.StringIds.Definition.Description);
if (value != null /*&& Handle != Blam.StringID.Null*/)
{
owner.StringIds.TryAndGetStringId(value, out Handle);
Handle = new Blam.StringId(Handle.Description, Handle.Index, Handle.Length,
-1); // HACK used to tell Read that we already read the string data (as this is an old halo 2 tag)
}
}
catch (Exception ex)
{
throw new Exceptions.InvalidStringId(ex,
base.headerOffset, uint.MaxValue,
ts, Handle.Length, value);
}
}
void InitializeBspTags(IO.EndianReader s, Blam.CacheFile cache)
{
// Seek to the scenario's scenario_structure_bsps_block tag_block field
s.Seek(items[0].Offset + 1444, System.IO.SeekOrigin.Begin);
bspTags = new Item[s.ReadInt32()];
uint sbsp_offset = s.ReadPointer();
// Seek to the scenario_structure_bsps_block definitions
s.Seek(sbsp_offset, System.IO.SeekOrigin.Begin);
DatumIndex di = new DatumIndex();
CacheItemBase item = null;
// Process each definition's runtime data
for (int x = 0; x < bspTags.Length; x++)
{
s.Seek(28, System.IO.SeekOrigin.Current);
di.Read(s);
item = items[di.Index];
bspTags[x] = item as CacheItemBase;
// Seek back to the beginning of the definition so the following stream code works
s.Seek(-Halo1.Tags.scenario_structure_bsps_block.kSizeOf, System.IO.SeekOrigin.Current);
// We're actually selectively reading scenario_structure_bsps_block fields here
// The offset actually points to the bsp header, and the bsp comes after that header
item.Offset = s.ReadInt32() + Halo1.Tags.scenario_structure_bsps_header.kSizeOf;
item.Size = s.ReadInt32();
cache.BspAddressMasks.Add(s.ReadUInt32() - (uint)item.Offset); // won't count the header
item.BspIndex = bspCount++;
// Seek to the end of this definition, and thus, the start of the next definition
s.Seek(20, System.IO.SeekOrigin.Current);
}
}
void DebugVertices(StreamWriter s, Render.VertexBufferInterface.VertexBuffersGen3 gr, bool denormalize,
Render.VertexBufferInterface.VertexBuffersGen3.Definition def,
s_tag_d3d_vertex_buffer vb)
{
var stream_r = new Render.VertexBufferInterface.StreamReader(def);
using (var er = new IO.EndianReader(vb.VertexBuffer.Value, IO.EndianState.Big, this))
{
for (int x = 0; x < vb.VertexCount.Value; x++) // foreach vertex...
{
s.WriteLine("\tVertex\t{0}", x.ToString("X8"));
{
stream_r.Read(er);
if (denormalize)
{
foreach (string str in stream_r.GetDenormalizedStrings())
{
s.WriteLine("\t\t{0}", str);
}
}
else
{
foreach (string str in stream_r.GetNormalizedStrings())
{
s.WriteLine("\t\t{0}", str);
}
}
}
s.WriteLine();
}
s.WriteLine(); s.WriteLine();
}
}
/// <summary>
/// Stream the field from a buffer
/// </summary>
/// <param name="input"></param>
public override void Read(IO.EndianReader input)
{
Internal = input.ReadBytes(Value);
#if DEBUG
if (TracingEnabled && Value > 5)
{
byte[] data = input.ReadBytes(Value);
for (int x = 0, y = 0, z = 0; x < data.Length; x++)
{
if (data[x] != 0)
{
if (y == 0 || x == 0)
{
y = x;
}
z++;
}
else if (x != 0 && data[x - 1] != 0)
{
trace.WriteLine("Found data at offset 0x{0:X} covering 0x{1:X} bytes, in a pad field with a count of {1}", y, z, Value);
}
}
data = null;
}
#endif
}
/// <summary>Read a string from an endian stream using <see cref="Storage"/>'s specifications</summary>
/// <param name="s">Endian stream to read from</param>
/// <param name="length">Optional length specification</param>
/// <returns></returns>
internal string ReadString(IO.EndianReader s, int length)
{
Contract.Requires(s != null);
if (length < 0) // Not <= because FixedLength might just be zero itself, resulting in a redundant expression
{
length = mStorage.FixedLength;
}
byte[] bytes = null;
int actual_count = 0;
switch (mStorage.Type)
{
// Type streamers should set actual_count to -1 if we're to assume all the bytes are characters.
// Otherwise, set actual_count to a byte count for padded string cases (where we don't want to
// include null characters in the result string)
case StringStorageType.CString: bytes = ReadStrCString(s, length, out actual_count); break;
case StringStorageType.Pascal: bytes = ReadStrPascal(s, out actual_count); break;
case StringStorageType.CharArray: bytes = ReadStrCharArray(s, length, out actual_count); break;
default: throw new Debug.UnreachableException();
}
return(new string(actual_count != -1
? mBaseEncoding.GetChars(bytes, 0, actual_count) // for padded string cases
: mBaseEncoding.GetChars(bytes))); // for complete string cases
}
/// <summary>
/// Stream the field from a tag stream
/// </summary>
/// <param name="ts"></param>
/// <exception cref="Exceptions.InvalidStringId"></exception>
public override void Read(IO.ITagStream ts)
{
if (Handle.Set == -1) // HACK used to tell Read that we already read the string data (as this is an old halo 2 tag)
{
Handle = new Blam.StringId(Handle.Description, Handle.Index, Handle.Length, 0);
return;
}
if (!ts.Flags.Test(IO.ITagStreamFlags.DontStreamStringData)) // stream the string id value if we can
{
IO.EndianReader s = ts.GetInputStream();
relativeOffset = s.PositionUnsigned;
string value = null;
try
{
value = new string(s.ReadChars(Handle.Length));
if (Handle != Blam.StringId.Null)
{
Program.GetTagIndex(OwnerId).StringIds.TryAndGetStringId(value, out Handle);
}
}
catch (Exception ex)
{
throw new Exceptions.InvalidStringId(ex,
base.headerOffset, base.relativeOffset, ts,
Handle.Length, value);
}
}
}
protected CacheFileBase(string map_name)
{
InputStream = new IO.EndianReader(map_name, IO.EndianState.Little, this);
if (!CacheIsReadonly(map_name))
{
OutputStream = new IO.EndianWriter(map_name, IO.EndianState.Little, this);
}
}
/// <summary>
/// Process the reference data from a stream
/// </summary>
/// <param name="s">Stream</param>
/// <exception cref="Exceptions.InvalidTagBlock"></exception>
public override void ReadHeader(IO.EndianReader s)
{
headerOffset = s.PositionUnsigned; // nifty for debugging
Resize(s.ReadInt32()); // element count
s.ReadInt32(); // elements
s.ReadInt32(); // definition
}
/// <summary>
/// Opens the file for reading
/// </summary>
/// <remarks>Closes existing file streams</remarks>
public void OpenForRead()
{
Close();
InputStream = new BlamLib.IO.EndianReader(path, endianState, this);
SetupBaseAddress();
}
void ReadFloats(IO.EndianReader s)
{
Read(s, out PlatoonRadius);
Read(s, out ProjectionTime);
Read(s, out OverrideGroundIKRange);
Read(s, out OverrideGroundIKTiltFactor);
Read(s, out GameSpeed);
}
/// <summary>
/// Call this at the end of the header's Read method to update data that uses some kind of base address
/// </summary>
/// <param name="s"></param>
protected void ReadPostprocessForBaseAddresses(IO.EndianReader s)
{
uint base_address = GetFirstNonEmptyPartition().BaseAddress - (uint)cacheInterop[CacheSectionType.Tag].CacheOffset;
(s.Owner as Blam.CacheFile).AddressMask = base_address;
this.offsetToIndex = (int)(tagIndexAddress - base_address);
CalculatePartitionOffsets();
}
public void Read(IO.EndianReader s)
{
mWidthType = (StringStorageWidthType)s.ReadByte();
mType = (StringStorageType)s.ReadByte();
mByteOrder = (Shell.EndianFormat)s.ReadByte();
s.Seek(sizeof(byte));
mFixedLength = s.ReadInt16();
s.Seek(sizeof(ushort));
}
