public async Task <TcpMessage> Receive(CancellationToken token = default(CancellationToken))
{
var packetLengthBytes = new byte[4];
if (await _stream.ReadAsync(packetLengthBytes, 0, 4, token).ConfigureAwait(false) != 4)
{
throw new InvalidOperationException("Couldn't read the packet length");
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
var seqBytes = new byte[4];
if (await _stream.ReadAsync(seqBytes, 0, 4, token).ConfigureAwait(false) != 4)
{
throw new InvalidOperationException("Couldn't read the sequence");
}
int seq = BitConverter.ToInt32(seqBytes, 0);
int readBytes = 0;
var body = new byte[packetLength - 12];
int neededToRead = packetLength - 12;
do
{
var bodyByte = new byte[packetLength - 12];
var availableBytes = await _stream.ReadAsync(bodyByte, 0, neededToRead, token).ConfigureAwait(false);
neededToRead -= availableBytes;
Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
readBytes += availableBytes;
}while (readBytes != packetLength - 12);
var crcBytes = new byte[4];
if (await _stream.ReadAsync(crcBytes, 0, 4, token).ConfigureAwait(false) != 4)
{
throw new InvalidOperationException("Couldn't read the crc");
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + body.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(body, 0, rv, packetLengthBytes.Length + seqBytes.Length, body.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, body));
}
private bool NameCRCEquals(Int32 NameCRC)
{
var mstr = new MemoryStream(_FileNameBytes, false);
var crc = new Ionic.Crc.CRC32();
return(NameCRC == crc.GetCrc32(mstr));
}
public void Reset(Stream stream)
{
TraceOutput(TraceBits.Session, "-------------------------------------------------------");
TraceOutput(TraceBits.Session, "Reset {0:X8} firstDone({1})", this.GetHashCode(), _firstWriteDone);
if (!_firstWriteDone)
{
return;
}
// reset all status
_toWrite.Clear();
_toFill.Clear();
foreach (var workitem in _pool)
{
_toFill.Enqueue(workitem.index);
workitem.ordinal = -1;
}
_firstWriteDone = false;
_totalBytesProcessed = 0L;
_runningCrc = new Ionic.Crc.CRC32();
_isClosed = false;
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
_outStream = stream;
}
private static Int32 GetCrc(string fname)
{
using (var fs1 = File.OpenRead(fname))
{
var checker = new Ionic.Crc.CRC32(true);
return(checker.GetCrc32(fs1));
}
}
public async Task <TcpMessage> Receieve()
{
// packet length
var packetLengthBytes = new byte[4];
if (!await ReadBuffer(stream, packetLengthBytes))
{
return(null);
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
// seq
var seqBytes = new byte[4];
if (!await ReadBuffer(stream, seqBytes))
{
return(null);
}
int seq = BitConverter.ToInt32(seqBytes, 0);
// body
var bodyBytes = new byte[packetLength - 12];
if (!await ReadBuffer(stream, bodyBytes))
{
return(null);
}
// crc
var crcBytes = new byte[4];
if (!await ReadBuffer(stream, crcBytes))
{
return(null);
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + bodyBytes.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(bodyBytes, 0, rv, packetLengthBytes.Length + seqBytes.Length, bodyBytes.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, bodyBytes));
}
private static bool IsValidChecksum(byte[] block, int checksum)
{
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(block, 0, block.Length);
var validChecksum = crc32.Crc32Result;
return(checksum == validChecksum);
}
public TcpMessage Receieve()
{
lock (_locker)
{
var stream = _tcpClient.GetStream();
var packetLengthBytes = new byte[4];
if (stream.Read(packetLengthBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the packet length");
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
var seqBytes = new byte[4];
if (stream.Read(seqBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the sequence");
}
int seq = BitConverter.ToInt32(seqBytes, 0);
int readBytes = 0;
var body = new byte[packetLength - 12];
int neededToRead = packetLength - 12;
do
{
var bodyByte = new byte[packetLength - 12];
var availableBytes = stream.Read(bodyByte, 0, neededToRead);
neededToRead -= availableBytes;
Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
readBytes += availableBytes;
}while (readBytes != packetLength - 12);
var crcBytes = new byte[4];
if (stream.Read(crcBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the crc");
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + body.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(body, 0, rv, packetLengthBytes.Length + seqBytes.Length, body.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, body));
}
}
public static uint CalculateCRCForChunk(string chunkType, byte[] chunkData)
{
byte[] chunkTypeBytes = Encoding.UTF8.GetBytes(chunkType);
Ionic.Crc.CRC32 crc32calculator = new Ionic.Crc.CRC32();
crc32calculator.SlurpBlock(chunkTypeBytes, 0, chunkTypeBytes.Length);
crc32calculator.SlurpBlock(chunkData, 0, chunkData.Length);
return((uint)crc32calculator.Crc32Result);
}
private void _DeflateOne(Object wi)
{
// compress one buffer
WorkItem workitem = (WorkItem)wi;
try
{
int myItem = workitem.index;
Ionic.Crc.CRC32 crc = new Ionic.Crc.CRC32();
// calc CRC on the buffer
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
// deflate it
DeflateOneSegment(workitem);
// update status
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress wi({0}) ord({1}) len({2})",
workitem.index,
workitem.ordinal,
workitem.compressedBytesAvailable
);
lock (_latestLock)
{
if (workitem.ordinal > _latestCompressed)
{
_latestCompressed = workitem.ordinal;
}
}
lock (_toWrite)
{
_toWrite.Enqueue(workitem.index);
}
_newlyCompressedBlob.Set();
}
catch (System.Exception exc1)
{
lock (_eLock)
{
// expose the exception to the main thread
if (_pendingException != null)
{
_pendingException = exc1;
}
}
}
}
public async Task<TcpMessage> Receieve()
{
var stream = _tcpClient.GetStream();
var packetLengthBytes = new byte[4];
if (await stream.ReadAsync(packetLengthBytes, 0, 4) != 4)
throw new InvalidOperationException("Couldn't read the packet length");
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
var seqBytes = new byte[4];
if (await stream.ReadAsync(seqBytes, 0, 4) != 4)
throw new InvalidOperationException("Couldn't read the sequence");
int seq = BitConverter.ToInt32(seqBytes, 0);
int readBytes = 0;
var body = new byte[packetLength - 12];
int neededToRead = packetLength - 12;
do
{
var bodyByte = new byte[packetLength - 12];
var availableBytes = await stream.ReadAsync(bodyByte, 0, neededToRead);
neededToRead -= availableBytes;
Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
readBytes += availableBytes;
}
while (readBytes != packetLength - 12);
var crcBytes = new byte[4];
if (await stream.ReadAsync(crcBytes, 0, 4) != 4)
throw new InvalidOperationException("Couldn't read the crc");
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + body.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(body, 0, rv, packetLengthBytes.Length + seqBytes.Length, body.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return new TcpMessage(seq, body);
}
public ZlibBaseStream(System.IO.Stream stream,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
this._flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
this._stream = stream;
this._leaveOpen = leaveOpen;
this._flavor = flavor;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
this.crc = new Ionic.Crc.CRC32();
}
}
public ZlibBaseStream(System.IO.Stream stream,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
this._flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
this._stream = stream;
this._leaveOpen = leaveOpen;
this._flavor = flavor;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
this.crc = new Ionic.Crc.CRC32();
}
}
/// <summary>
/// Get the CRC of a file stream and return as a string
/// </summary>
/// <param name="stream">FileStream</param>
/// <param name="hashOption">Default = SHA1, also MD5 and CRC32</param>
/// <param name="headerlength">Number of bytes to skip prior to computing hash. Should try both platform header length and 0 when comparing to known checksum to account for Rom files with dumped intros. Default = 0.</param>
/// <returns>Returns hash as hex string</returns>
public static string GetHash(Stream stream, HashOption hashOption = HashOption.SHA1, int headerlength = 0)
{
string hash = "";
int streamLength = (int)stream.Length;
if (streamLength < headerlength)
{
return("");
}
// Read header and discard
stream.Seek(headerlength, SeekOrigin.Begin);
// Read everything after the header into a byte array
byte[] buffer = new byte[streamLength - headerlength];
stream.Read(buffer, 0, (streamLength - headerlength));
switch (hashOption)
{
case HashOption.CRC32:
var crc32 = new Ionic.Crc.CRC32();
foreach (byte b in buffer)
{
crc32.UpdateCRC(b);
}
hash = crc32.Crc32Result.ToString("x8").ToUpper();
break;
case HashOption.MD5:
var md5 = MD5.Create();
byte[] hashBytes = md5.ComputeHash(buffer);
hash = BitConverter.ToString(hashBytes).Replace("-", String.Empty).ToUpper();
break;
case HashOption.SHA1:
SHA1Managed managedSHA1 = new SHA1Managed();
byte[] shaBuffer = managedSHA1.ComputeHash(buffer);
foreach (byte b in shaBuffer)
{
hash += b.ToString("x2").ToUpper();
}
break;
}
return(hash);
}
public ZlibBaseStream(System.IO.Stream stream,
CompressionMode compressionMode,
CompressionLevel level,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
_flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
_stream = stream;
_leaveOpen = leaveOpen;
_compressionMode = compressionMode;
_flavor = flavor;
_level = level;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
crc = new Ionic.Crc.CRC32();
}
}
private void _InitializePoolOfWorkItems()
{
_toWrite = new Queue <int>();
_toFill = new Queue <int>();
_pool = new System.Collections.Generic.List <WorkItem>();
int nTasks = BufferPairsPerCore * Environment.ProcessorCount;
nTasks = Math.Min(nTasks, _maxBufferPairs);
for (int i = 0; i < nTasks; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy, i));
_toFill.Enqueue(i);
}
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new Ionic.Crc.CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
public ZlibBaseStream(System.IO.Stream stream,
CompressionMode compressionMode,
CompressionLevel level,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
this._flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
this._stream = stream;
this._leaveOpen = leaveOpen;
this._compressionMode = compressionMode;
this._flavor = flavor;
this._level = level;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
#if !NETFX_CORE
this.crc = new Ionic.Crc.CRC32();
#else
throw new NotSupportedException("ZlibStreamFlavor.GZIP is unsupported");
#endif
}
}
/// <summary>
/// Resets the stream for use with another stream.
/// </summary>
/// <remarks>
/// Because the ParallelDeflateOutputStream is expensive to create, it
/// has been designed so that it can be recycled and re-used. You have
/// to call Close() on the stream first, then you can call Reset() on
/// it, to use it again on another stream.
/// </remarks>
///
/// <param name="stream">
/// The new output stream for this era.
/// </param>
///
/// <example>
/// <code>
/// ParallelDeflateOutputStream deflater = null;
/// foreach (var inputFile in listOfFiles)
/// {
/// string outputFile = inputFile + ".compressed";
/// using (System.IO.Stream input = System.IO.File.OpenRead(inputFile))
/// {
/// using (var outStream = System.IO.File.Create(outputFile))
/// {
/// if (deflater == null)
/// deflater = new ParallelDeflateOutputStream(outStream,
/// CompressionLevel.Best,
/// CompressionStrategy.Default,
/// true);
/// deflater.Reset(outStream);
///
/// while ((n= input.Read(buffer, 0, buffer.Length)) != 0)
/// {
/// deflater.Write(buffer, 0, n);
/// }
/// }
/// }
/// }
/// </code>
/// </example>
public void Reset(Stream stream)
{
TraceOutput(TraceBits.Session, "-------------------------------------------------------");
TraceOutput(TraceBits.Session, "Reset {0:X8} firstDone({1})", this.GetHashCode(), _firstWriteDone);
if (!_firstWriteDone) return;
// reset all status
_toWrite.Clear();
_toFill.Clear();
foreach (var workitem in _pool)
{
_toFill.Enqueue(workitem.index);
workitem.ordinal = -1;
}
_firstWriteDone = false;
_totalBytesProcessed = 0L;
_runningCrc = new Ionic.Crc.CRC32();
_isClosed= false;
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
_outStream = stream;
}
private void _InitializePoolOfWorkItems()
{
_toWrite = new Queue<int>();
_toFill = new Queue<int>();
_pool = new System.Collections.Generic.List<WorkItem>();
int nTasks = BufferPairsPerCore * Environment.ProcessorCount;
nTasks = Math.Min(nTasks, _maxBufferPairs);
for(int i=0; i < nTasks; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy, i));
_toFill.Enqueue(i);
}
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new Ionic.Crc.CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
private void _DeflateOne(Object wi)
{
// compress one buffer
WorkItem workitem = (WorkItem) wi;
try
{
int myItem = workitem.index;
Ionic.Crc.CRC32 crc = new Ionic.Crc.CRC32();
// calc CRC on the buffer
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
// deflate it
DeflateOneSegment(workitem);
// update status
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress wi({0}) ord({1}) len({2})",
workitem.index,
workitem.ordinal,
workitem.compressedBytesAvailable
);
lock(_latestLock)
{
if (workitem.ordinal > _latestCompressed)
_latestCompressed = workitem.ordinal;
}
lock (_toWrite)
{
_toWrite.Enqueue(workitem.index);
}
_newlyCompressedBlob.Set();
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
}
private Int32 FigureCrc32()
{
if (_crcCalculated == false)
{
Stream input = null;
// get the original stream:
if (this._Source == ZipEntrySource.WriteDelegate)
{
var output = new Ionic.Crc.CrcCalculatorStream(Stream.Null);
// allow the application to write the data
this._WriteDelegate(this.FileName, output);
_Crc32 = output.Crc;
}
else if (this._Source == ZipEntrySource.ZipFile)
{
// nothing to do - the CRC is already set
}
else
{
if (this._Source == ZipEntrySource.Stream)
{
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.JitStream)
{
// allow the application to open the stream
if (this._sourceStream == null)
_sourceStream = this._OpenDelegate(this.FileName);
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.ZipOutputStream)
{
}
else
{
//input = File.OpenRead(LocalFileName);
input = File.Open(LocalFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
var crc32 = new Ionic.Crc.CRC32();
_Crc32 = crc32.GetCrc32(input);
if (_sourceStream == null)
{
#if NETCF
input.Close();
#else
input.Dispose();
#endif
}
}
_crcCalculated = true;
}
return _Crc32;
}
public async Task <TcpMessage> Receieve()
{
logger.Trace($"Wait for answer {_tcpClient.Available} ...");
var stream = _tcpClient.GetStream();
var packetLengthBytes = new byte[4];
if (await stream.ReadAsync(packetLengthBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the packet length");
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
logger.Debug("[IN] Packet length: {0}", packetLength);
var seqBytes = new byte[4];
if (await stream.ReadAsync(seqBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the sequence");
}
int seq = BitConverter.ToInt32(seqBytes, 0);
logger.Debug("[IN] Sequence: {0}", seq);
int readBytes = 0;
var body = new byte[packetLength - 12];
int neededToRead = packetLength - 12;
do
{
var bodyByte = new byte[packetLength - 12];
var availableBytes = await stream.ReadAsync(bodyByte, 0, neededToRead);
neededToRead -= availableBytes;
Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
readBytes += availableBytes;
}while (readBytes != packetLength - 12);
var crcBytes = new byte[4];
if (await stream.ReadAsync(crcBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the crc");
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + body.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(body, 0, rv, packetLengthBytes.Length + seqBytes.Length, body.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, body));
}
public async Task <TcpMessage> Receieve(int timeoutms)
{
logger.Trace($"Wait for event {_tcpClient.Available} ...");
var stream = _tcpClient.GetStream();
var packetLengthBytes = new byte[4];
var token = tokenSource.Token;
stream.ReadTimeout = timeoutms;
int bytes = 0;
try
{
bytes = stream.Read(packetLengthBytes, 0, 4);
} catch (System.IO.IOException io)
{
var socketError = io.InnerException as SocketException;
if (socketError != null && socketError.SocketErrorCode == SocketError.TimedOut)
{
throw new OperationCanceledException();
}
throw io;
}
if (bytes != 4)
{
throw new InvalidOperationException("Couldn't read the packet length");
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
logger.Debug("[IN]* Packet length: {0}", packetLength);
var seqBytes = new byte[4];
if (await stream.ReadAsync(seqBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the sequence");
}
int seq = BitConverter.ToInt32(seqBytes, 0);
logger.Debug("[IN]* sequence: {0}", seq);
int readBytes = 0;
var body = new byte[packetLength - 12];
int neededToRead = packetLength - 12;
do
{
var bodyByte = new byte[packetLength - 12];
var availableBytes = await stream.ReadAsync(bodyByte, 0, neededToRead);
neededToRead -= availableBytes;
Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
readBytes += availableBytes;
}while (readBytes != packetLength - 12);
var crcBytes = new byte[4];
if (await stream.ReadAsync(crcBytes, 0, 4) != 4)
{
throw new InvalidOperationException("Couldn't read the crc");
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + body.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(body, 0, rv, packetLengthBytes.Length + seqBytes.Length, body.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, body));
}
private static Int32 GetCrc(string fname)
{
using (var fs1 = File.OpenRead(fname))
{
var checker = new Ionic.Crc.CRC32(true);
return checker.GetCrc32(fs1);
}
}
public static PackedFileHeader FromReader(BinaryReader reader)
{
if (new string(reader.ReadChars(HeaderString.Length)) != HeaderString)
{
throw new InvalidDataException("Invalid file header.");
}
using var crcStream = new MemoryStream();
using var crcWriter = new BinaryWriter(crcStream);
var header = new PackedFileHeader();
header._dataOffset = reader.ReadUInt32();
header._compressedSize = reader.ReadUInt32();
header._headerVersion = reader.ReadUInt32();
header._decompressedSize = reader.ReadUInt32();
header._dataBlocksCount = reader.ReadUInt32();
crcWriter.WriteString(HeaderString);
crcWriter.Write(header._dataOffset);
crcWriter.Write(header._compressedSize);
crcWriter.Write(header._headerVersion);
crcWriter.Write(header._decompressedSize);
crcWriter.Write(header._dataBlocksCount);
if (header._headerVersion == 0x00)
{
if (header._dataOffset != 0x40)
{
throw new InvalidDataException("Invalid header size.");
}
if (reader.ReadUInt16() != 0)
{
throw new InvalidDataException();
}
header._gameVersion = reader.ReadUInt16();
crcWriter.Write((ushort)0);
crcWriter.Write((ushort)header._gameVersion);
}
else if (header._headerVersion == 0x01)
{
if (header._dataOffset != 0x44)
{
throw new InvalidDataException("Invalid header size.");
}
header._gameIdentifier = reader.ReadUInt32();
header._gameVersion = reader.ReadUInt32();
crcWriter.Write(header._gameIdentifier);
crcWriter.Write(header._gameVersion);
}
else
{
throw new NotSupportedException("Only header version 1 is supported.");
}
header._build = reader.ReadUInt16();
header._flags = reader.ReadUInt16();
header._gameLength = reader.ReadUInt32();
header._checksum = reader.ReadUInt32();
crcWriter.Write(header._build);
crcWriter.Write(header._flags);
crcWriter.Write(header._gameLength);
crcWriter.Write(0U);
crcStream.Position = 0;
var crc = new Ionic.Crc.CRC32().GetCrc32(crcStream);
if (crc != header._checksum)
{
throw new InvalidDataException("CRC failed");
}
return(header);
}
public async Task <TcpMessage> Receieve()
{
//var stream = _tcpClient.GetStream();
//var packetLengthBytes = new byte[4];
//if (await stream.ReadAsync(packetLengthBytes, 0, 4) != 4)
// throw new InvalidOperationException("Couldn't read the packet length");
//int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
//var seqBytes = new byte[4];
//if (await stream.ReadAsync(seqBytes, 0, 4) != 4)
// throw new InvalidOperationException("Couldn't read the sequence");
//int seq = BitConverter.ToInt32(seqBytes, 0);
//int readBytes = 0;
//var body = new byte[packetLength - 12];
//int neededToRead = packetLength - 12;
//do
//{
// var bodyByte = new byte[packetLength - 12];
// var availableBytes = await stream.ReadAsync(bodyByte, 0, neededToRead);
// neededToRead -= availableBytes;
// Buffer.BlockCopy(bodyByte, 0, body, readBytes, availableBytes);
// readBytes += availableBytes;
//}
//while (readBytes != packetLength - 12);
//var crcBytes = new byte[4];
//if (await stream.ReadAsync(crcBytes, 0, 4) != 4)
// throw new InvalidOperationException("Couldn't read the crc");
//int checksum = BitConverter.ToInt32(crcBytes, 0);
// packet length
var packetLengthBytes = new byte[4];
if (!await ReadBuffer(_stream, packetLengthBytes))
{
return(null);
}
int packetLength = BitConverter.ToInt32(packetLengthBytes, 0);
// seq
var seqBytes = new byte[4];
if (!await ReadBuffer(_stream, seqBytes))
{
return(null);
}
int seq = BitConverter.ToInt32(seqBytes, 0);
// body
var bodyBytes = new byte[packetLength - 12];
if (!await ReadBuffer(_stream, bodyBytes))
{
return(null);
}
// crc
var crcBytes = new byte[4];
if (!await ReadBuffer(_stream, crcBytes))
{
return(null);
}
int checksum = BitConverter.ToInt32(crcBytes, 0);
byte[] rv = new byte[packetLengthBytes.Length + seqBytes.Length + bodyBytes.Length];
Buffer.BlockCopy(packetLengthBytes, 0, rv, 0, packetLengthBytes.Length);
Buffer.BlockCopy(seqBytes, 0, rv, packetLengthBytes.Length, seqBytes.Length);
Buffer.BlockCopy(bodyBytes, 0, rv, packetLengthBytes.Length + seqBytes.Length, bodyBytes.Length);
var crc32 = new Ionic.Crc.CRC32();
crc32.SlurpBlock(rv, 0, rv.Length);
var validChecksum = crc32.Crc32Result;
if (checksum != validChecksum)
{
throw new InvalidOperationException("invalid checksum! skip");
}
return(new TcpMessage(seq, bodyBytes));
}
static void Main(string[] args)
{
//args = new string[4];
//args[0] = "Mob-WorldData.wad";
//args[1] = "-x";
//args[2] = "*";
//args[3] = "mobout";
System.Diagnostics.Stopwatch MainTimer = new System.Diagnostics.Stopwatch();
MainTimer.Start();
string wad = ""; //wad filename
string mode = ""; //Operating mode
string arg1 = ""; //Argument 1 for mode
string arg2 = ""; //Argument 2 for mode
bool HadArgs = false; //Whether the user supplied arguments, or just a filename
//Try grabbing wad name and mode
try
{
wad = args[0];
}
catch //If the wad/mode couldn't be grabbed
{
Console.WriteLine("Please specify wad!"); //Print error message
PrintHelp(); //Print usage info
}
try
{
mode = args[1];
HadArgs = true;
}
catch //If the wad/mode couldn't be grabbed
{
Console.WriteLine("No mode selected. Defaulting to extract all."); //Print error message
mode = "-x";
//PrintHelp(); //Print usage info
}
//If the user specified a mode, try grabbing arguments for specified mode
if (HadArgs)
{
try
{
if (mode == "-x") //If extract mode, grab two arguments
{
arg1 = args[2];
arg2 = args[3];
}
else if (mode == "-r" || mode == "-c" || mode == "-d" || mode == "-a") //Remove/Create/Diff/Add mode, one arg
{
arg1 = args[2];
}
else if (mode != "-i" && mode != "-w2z") //If the mode is not -i (takes no arguments), then we don't know what mode they specified
{
Console.WriteLine("Invalid mode!"); //Print error
PrintHelp(); //Print usage info
}
}
catch //If the arguments were missing, or something else went wrong
{
Console.WriteLine("Invalid arguments for specified mode!"); //Print an error
PrintHelp(); //Print usage info
}
}
else
{
if (!System.IO.File.Exists(wad))
{
Console.WriteLine("Wad file not found!");
PrintHelp();
}
arg1 = "*"; //Set extraction file-selection to all
System.Windows.Forms.FolderBrowserDialog fd = new System.Windows.Forms.FolderBrowserDialog();
fd.Description = "Choose where to save the extracted files";
MainTimer.Stop();
if (fd.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
MainTimer.Start();
arg2 = fd.SelectedPath;
ExclusionDir = arg2;
}
else
{
Quit();
}
}
FileList[] entries = new FileList[0]; //Pre-init the 'entries' array
if (mode == "-a") //If using file-add mode
{
try
{
arg2 = args[3];
}
catch
{
Console.WriteLine("No output wad specified. Overwriting original");
arg2 = wad;
}
string[] InFiles = arg1.Split(':');
for (int i = 0; i < InFiles.Length; i++)
{
if (!File.Exists(InFiles[i]))
{
if (!Directory.Exists(InFiles[i]))
{
Console.WriteLine("{0} could not be found!\nCancelling operation...", InFiles[i]);
Quit();
}
else
{
Console.WriteLine("{0} is a directory! Processing sub-entries...", InFiles[i]);
}
}
else
{
Console.WriteLine("{0} added", InFiles[i]);
}
}
Quit();
}
if (mode == "-c") //Create (wad) mode
{
//Make sure the input directory exists
if (!Directory.Exists(arg1))
{
Console.WriteLine("Input directory not found!");
PrintHelp();
}
string[] InFiles = Directory.GetFiles(arg1, "*.*", SearchOption.AllDirectories); //Grab a list of all files within the directory
entries = new FileList[InFiles.Count()]; //Make a new FileList for storing these files in the wad
Console.WriteLine("Filecount: {0}", InFiles.Count()); //Debug
//For each file
Parallel.For(0, InFiles.Count(), i =>
{
entries[i].Filename = InFiles[i].Substring(arg1.Length, InFiles[i].Length - arg1.Length); //Remove directory info that shouldn't be included in the wad (path up to the wad contents)
if (entries[i].Filename.IndexOf("\\") == 0) //If the entry starts with a \
{
entries[i].Filename = entries[i].Filename.Substring(1, entries[i].Filename.Length - 1); //Get rid of the slash
}
entries[i].Filename = entries[i].Filename.Replace('\\', '/');
//MemoryStream ms = new MemoryStream(File.ReadAllBytes(InFiles[i])); //Read
entries[i].Data = File.ReadAllBytes(InFiles[i]); //Read the file into memory
entries[i].Size = (uint)entries[i].Data.Length;
//Console.WriteLine("{0}, with a size of {1}, was read to memory", entries[i].Filename, entries[i].Size);
});
Console.WriteLine("Files read to memory");
List <byte> WadHeader = new List <byte>();
List <byte> WadBody = new List <byte>();
var crcparam = new CrcSharp.CrcParameters(32, 0x04c11db7, 0, 0, true, true);
var crc = new CrcSharp.Crc(crcparam);
for (int i = 0; i < entries.Length; i++)
{
entries[i].Offset = (uint)WadBody.Count(); //Save the offset for this entry
MemoryStream Compressed = new MemoryStream();
Ionic.Zlib.ZlibStream zls = new Ionic.Zlib.ZlibStream(Compressed, Ionic.Zlib.CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, false);
zls.Write(entries[i].Data, 0, entries[i].Data.Length);
zls.Close();
byte[] compdata = Compressed.ToArray();
WadBody.AddRange(compdata); //Add the compressed data to the body of the wad
entries[i].CRC = (uint)crc.CalculateAsNumeric(entries[i].Data); //Add the CRC of the compressed data (idk why they do this)
entries[i].CompressedSize = (uint)compdata.Length; //Save the size of the compressed data
//Console.WriteLine("CRC: {0} : {1}", entries[i].CRC.ToString("X8"), entries[i].Filename); //Debug
}
Console.WriteLine("Wad body added");
//Add wad header info
WadHeader.AddRange(new byte[] { 0x4B, 0x49, 0x57, 0x41, 0x44, 0x02, 0x00, 0x00, 0x00 }); //Add magic and wad version (2)
WadHeader.AddRange(BitConverter.GetBytes(entries.Length)); //Add the file count
WadHeader.Add(0x01); //Add wad2 byte (idk what it is, but it's required)
int[] offoff = new int[entries.Length]; //Keeps track of the offset for the 'Offset' field in the header (we can't know the offset until the header is finished, so we add it later)
//Add each file to the wad header
for (int i = 0; i < entries.Length; i++)
{
offoff[i] = WadHeader.Count(); //Save current offset, so that we can update the 'offset' field when the header is finished
WadHeader.AddRange(new byte[] { 0x00, 0x00, 0x00, 0x00 }); //Add filler offset
WadHeader.AddRange(BitConverter.GetBytes(entries[i].Size)); //Add uncompresses file size
WadHeader.AddRange(BitConverter.GetBytes(entries[i].CompressedSize)); //Add compressed file size
WadHeader.Add(0x01); //Set 'IsCompressed' to 1 (we compress all files, so this is always true)
WadHeader.AddRange(BitConverter.GetBytes(entries[i].CRC)); //Add the compressed-data crc (idk why they have this, because the extracted file has a crc anyway)
WadHeader.AddRange(BitConverter.GetBytes(entries[i].Filename.Length + 1)); //Add filename length
WadHeader.AddRange(ASCIIEncoding.ASCII.GetBytes(entries[i].Filename)); //Add the filename
WadHeader.Add(0x00); //Add padding
}
Console.WriteLine("Wad Header added");
//Fix the offsets
for (int i = 0; i < entries.Length; i++)
{
byte[] offsetbytes = BitConverter.GetBytes(WadHeader.Count() + entries[i].Offset);
WadHeader[offoff[i]] = offsetbytes[0];
WadHeader[offoff[i] + 1] = offsetbytes[1];
WadHeader[offoff[i] + 2] = offsetbytes[2];
WadHeader[offoff[i] + 3] = offsetbytes[3];
}
Console.WriteLine("Offsets fixed");
byte[] Output = WadHeader.Concat(WadBody).ToArray();
File.WriteAllBytes(wad, Output); //Save
Console.WriteLine("Wad Saved");
Quit();
}
if (mode != "-c") //If the tool is not in create mode, check if the wad exists
{
if (!System.IO.File.Exists(wad))
{
Console.WriteLine("Wad file not found!");
PrintHelp();
}
else //If the file exists
{
Console.WriteLine("Reading wad to memory...");
entries = ReadWad(wad); //Read the wad into the 'entries' array
}
}
if (mode == "-d" && !File.Exists(arg1)) //If using diff mode, and the second file doesn't exist
{
Console.WriteLine("Second wad does not exist!");
PrintHelp();
}
if (mode == "-w2z") //If using Wad2Zip mode
{
//Stopwatch for diagnostics
System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
Console.WriteLine("Calculating CRC's...");
//For each file in the wad; extract it in-memory, calculate the CRC of the extracted data, and update the entry's CRC field (because this is done in-memory, it should be less than a couple of seconds)
Parallel.For(0, entries.Length, i =>
{
byte[] filemem = new byte[0];
if (entries[i].IsCompressed) //If the file is marked as compressed
{
filemem = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data);
Array.Copy(entries[i].Data, 2, entries[i].Data, 0, entries[i].Data.Length - 6);
}
else //If the file isn't compressed
{
filemem = entries[i].Data;
}
Ionic.Crc.CRC32 crc = new Ionic.Crc.CRC32();
entries[i].CRC = (uint)crc.GetCrc32(new MemoryStream(filemem)); //Replace the entries' crc with the CRC of the compressed data (KI are shit and use their own incompatible polynomials for their checksum, so we need to recalculate it with a standard polynomial)
});
stopwatch.Stop();
//Stopwatch for timing the zip-process
System.Diagnostics.Stopwatch ziptimer = new System.Diagnostics.Stopwatch();
ziptimer.Start();
byte[] outputfile = Zipper(entries); //Add all file entries to a zip in-memory, and return the zip
ziptimer.Stop();
File.WriteAllBytes(wad + ".zip", outputfile); //Save the created zip to disk (input filename with .zip appended)
MainTimer.Stop();
Console.WriteLine("Updated CRC's in {0} Seconds", stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Zipped in {0} Seconds", ziptimer.Elapsed.TotalSeconds);
Console.WriteLine("Total program runtime: {0} Seconds", MainTimer.Elapsed.TotalSeconds);
Quit(); //Exit
}
if (mode == "-i" || mode == "-x" || mode == "-d")
{
if (mode == "-i") //If using info mode
{
StringBuilder sb = new StringBuilder(); //Make a new stringbuilder (stringbuilder is much faster than just appending strings normally)
for (int i = 0; i < entries.Length; i++) //For each file entry
{
sb.AppendLine(entries[i].Offset.ToString("X") + ":" + entries[i].CompressedSize + ":" + entries[i].Size + ":" + entries[i].Filename); //Add a newline to the output string, with the offset and filename (offset:filename)
}
Console.WriteLine(sb); //Print the output string
MainTimer.Stop();
Console.WriteLine("{0} files found in {1} Seconds", entries.Length, MainTimer.Elapsed.TotalSeconds);
Quit(); //Quit
}
else if (mode == "-d") //If using diff mode
{
Console.WriteLine("Diff-checking {0} and {1}", wad, arg1);
bool extract = false;
try //Try grabbing the output-folder argument
{
arg2 = args[3]; //Grab fourth argument (starting from 0)
try
{
arg2 = ResolveDir(arg2);
extract = true; //Enable file-extraction
}
catch //If something went wrong whlie creating the directory
{
Console.WriteLine("Error creating directory: {0}\nMaybe you're trying to write to a folder you don't have permission to write in?", arg2); //Let the user know something went wrong
Quit(); //Exit
}
}
catch //If there isn't an output-folder argument
{
Console.WriteLine("No output folder specified. No extraction will be performed."); //Print a message stating operating mode
}
Console.WriteLine("Checking differences...");
FileList[] entries2 = ReadWad(arg1);
bool[] ExtractIt = new bool[entries2.Length]; //Create a bool array, which keeps track of which files to extract from the second wad
StringBuilder MissingIn1 = new StringBuilder();
StringBuilder MissingIn2 = new StringBuilder();
StringBuilder DiffIn2 = new StringBuilder();
object threadsync = new object(); //Threadsync is used to prevent threads from overwriting eachothers data
//Check what files are missing
Parallel.For(0, entries.Length, i => //For each file entry in the first wad
{
bool Exists = false; //Mark the file as not existing in both wads (default, until proven otherwise)
for (int j = 0; j < entries2.Length; j++) //For each file in second wad
{
if (entries2[j].Filename == entries[i].Filename) //If the currently-processes filename in wad 1 is also the same filename in wad2
{
Exists = true; //Mark the file as existing in both
if (entries2[j].CRC != entries[i].CRC) //If the files have a different CRC
{
lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses
{
DiffIn2.AppendLine(entries[i].Filename); //Add the filename to the 'DiffIn2' sb, so we can print the results later
}
ExtractIt[j] = true; //Mark the file in wad2 for extraction, because it's different
}
break; //Stop searching for this file, because we've already checked it
}
}
if (!Exists) //If the file wasn't marked as existing
{
lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses
{
MissingIn2.AppendLine(entries[i].Filename); //Add the file to the 'MissingIn2' sb for printing later
}
}
});
//Check what files exist in both wads (and whether any files were added)
Parallel.For(0, entries2.Length, i =>
{
bool Exists = false;
for (int j = 0; j < entries.Length; j++)
{
if (entries2[i].Filename == entries[j].Filename)
{
Exists = true; //Mark the file as existing in both
break; //Stop searching for this file
}
}
if (!Exists) //If the file wasn't marked as existing
{
lock (threadsync) //Use a lock to prevent SB from getting corrupt by multiple-accesses
{
MissingIn1.AppendLine(entries[i].Filename); //Add the file to the 'MissingIn1' sb for printing later
}
ExtractIt[i] = true; //Mark the file in wad2 for extraction, because it doesn't exist in wad1
}
});
Console.WriteLine("Diff-checking complete!");
if (extract) //If the user specified an output directory (extract diff files)
{
Console.WriteLine("Extracting new/different files..."); //Inform the user that the files will now be extracted
Console.WriteLine("Pre-creating directories...");
for (int i = 0; i < entries2.Length; i++) //For each file in the second wad
{
if (ExtractIt[i]) //If the file is marked for extraction, check it for any subdirectories, and create them if necessary
{
PreCreate(entries2[i], arg2); //Call 'PreCreate' for that file, and any required subdirs for that file will be created
}
}
Console.WriteLine("Directories created!\nExtracting...");
//For each file in the second wad, check if it's marked for extraction; and if so, extract it.
Parallel.For(0, entries2.Length, i =>
{
if (ExtractIt[i]) //If the file was marked for extraction (new/different file)
{
if (entries2[i].IsCompressed) //If the file is marked as compressed
{
entries2[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries2[i].Data); //Decompress the file
}
using (FileStream output = new FileStream(arg2 + "\\" + entries2[i].Filename, FileMode.Create)) //Create the file that is being extracted (replaces old files if they exist)
{
output.Write(entries2[i].Data, 0, entries2[i].Data.Length); //Write the file from memory to disk
}
}
});
}
if (MissingIn1.Length > 0 || MissingIn2.Length > 0 || DiffIn2.Length > 0) //If there were any file differences
{
Console.WriteLine("Differences found!");
if (entries.Length != entries2.Length)
{
Console.WriteLine("------------------------------File count is different!------------------------------\nOld:{0}\tNew:{1}", entries.Length, entries2.Length);
}
if (MissingIn1.Length > 0)
{
Console.WriteLine("------------------------------Files missing in first wad------------------------------\n{0}", MissingIn1);
}
if (MissingIn2.Length > 0)
{
Console.WriteLine("------------------------------Files missing in second wad------------------------------\n{0}", MissingIn2);
}
if (DiffIn2.Length > 0)
{
Console.WriteLine("------------------------------Files changed------------------------------\n{0}", DiffIn2);
}
}
else
{
Console.WriteLine("No differences found!");
}
MainTimer.Stop();
Console.WriteLine("Total program runtime: {0} Seconds", MainTimer.Elapsed.TotalSeconds);
Quit();
}
else if (mode == "-x") //If using extract mode
{
arg2 = ResolveDir(arg2);
if (arg1 != "*") //If the user specified a file to extract (not all files)
{
for (int i = 0; i < entries.Length; i++) //For each file in the filelist
{
if (string.Equals(entries[i].Filename, arg1, StringComparison.OrdinalIgnoreCase) || string.Equals(entries[i].Filename.Replace('/', '\\'), arg1, StringComparison.OrdinalIgnoreCase)) //If the file entry matches the user-specified file (ignoring case and slash-direction)
{
Console.WriteLine("File found!");
PreCreate(entries[i], arg2); //Check if file is located in a subdirectory. If so, create the appropriate directory structure
if (entries[i].IsCompressed) //If the file is marked as compressed
{
entries[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data); //Decompress the data
}
using (FileStream output = new FileStream(arg2 + "\\" + entries[i].Filename, FileMode.Create)) //Create the file that is being extracted (replaces old files if they exist)
{
output.Write(entries[i].Data, 0, entries[i].Data.Length); //Write the file from memory to disk
Console.WriteLine("File extracted to: {0}", arg2 + "\\" + entries[i].Filename.Replace('/', '\\'));
}
Quit();
}
//If the filename doesn't match, read the next entry
}
//If all files have been scanned, and the user-specified file wasn't found; let the user know, and give them some advice
Console.WriteLine("'{0}' was not found in the specified wad!", arg1);
Console.WriteLine("Make sure you include the file's parent directories");
Console.WriteLine("eg: capabilities\\cpu.xml");
Quit();
}
else //If the file is '*": Don't 'really' need to say else here, because it would have exited previously anyway, but it just makes things more readable
{
//Create directories in advance, using a single thread (prevents race crash when parallel threads attempt to create the same directory at the same time)
Console.WriteLine("Pre-creating directories...");
for (int i = 0; i < entries.Length; i++)
{
PreCreate(entries[i], arg2); //Pre-create any subdirectories listed in the filename (arg2 is the base directory for extraction)
}
Console.WriteLine("Directories created!\nExtracting...");
System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
Parallel.For(0, entries.Length, i =>
{
if (entries[i].IsCompressed) //If the file is marked as compressed
{
entries[i].Data = Ionic.Zlib.ZlibStream.UncompressBuffer(entries[i].Data);
}
});
stopwatch.Stop();
Console.WriteLine("Extraction complete!\nWriting to disk... (this may take some time)");
System.Diagnostics.Stopwatch writetimer = new System.Diagnostics.Stopwatch();
writetimer.Start();
Parallel.For(0, entries.Length, i =>
{
File.WriteAllBytes(arg2 + "\\" + entries[i].Filename, entries[i].Data);
});
writetimer.Stop();
MainTimer.Stop();
Console.WriteLine("Extracted {0} files in {1} Seconds", entries.Length, stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Wrote files in {0} Ms", writetimer.ElapsedMilliseconds);
Console.WriteLine("Total program runtime: {0} seconds", MainTimer.Elapsed.TotalSeconds);
Quit(); //Exit
}
}
}
}
public void Zlib_ParallelDeflateStream()
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream Start", sw.Elapsed);
int sz = 256*1024 + this.rnd.Next(120000);
string FileToCompress = System.IO.Path.Combine(TopLevelDir, String.Format("Zlib_ParallelDeflateStream.{0}.txt", sz));
CreateAndFillFileText( FileToCompress, sz);
TestContext.WriteLine("{0}: Created file: {1}", sw.Elapsed, FileToCompress );
byte[] original = File.ReadAllBytes(FileToCompress);
int crc1 = DoCrc(FileToCompress);
TestContext.WriteLine("{0}: Original CRC: {1:X8}", sw.Elapsed, crc1 );
byte[] working = new byte[WORKING_BUFFER_SIZE];
int n = -1;
long originalLength;
MemoryStream ms1 = new MemoryStream();
{
using (FileStream fs1 = File.OpenRead(FileToCompress))
{
originalLength = fs1.Length;
using (var compressor = new Ionic.Zlib.ParallelDeflateOutputStream(ms1, true))
{
while ((n = fs1.Read(working, 0, working.Length)) != 0)
{
compressor.Write(working, 0, n);
}
}
}
ms1.Seek(0, SeekOrigin.Begin);
}
TestContext.WriteLine("{0}: Compressed {1} bytes into {2} bytes", sw.Elapsed,
originalLength, ms1.Length);
var crc = new Ionic.Crc.CRC32();
int crc2= 0;
byte[] decompressedBytes= null;
using (MemoryStream ms2 = new MemoryStream())
{
using (var decompressor = new DeflateStream(ms1, CompressionMode.Decompress, false))
{
while ((n = decompressor.Read(working, 0, working.Length)) != 0)
{
ms2.Write(working, 0, n);
}
}
TestContext.WriteLine("{0}: Decompressed", sw.Elapsed);
TestContext.WriteLine("{0}: Decompressed length: {1}", sw.Elapsed, ms2.Length);
ms2.Seek(0, SeekOrigin.Begin);
crc2 = crc.GetCrc32(ms2);
decompressedBytes = ms2.ToArray();
TestContext.WriteLine("{0}: Decompressed CRC: {1:X8}", sw.Elapsed, crc2 );
}
TestContext.WriteLine("{0}: Checking...", sw.Elapsed );
bool check = true;
if (originalLength != decompressedBytes.Length)
{
TestContext.WriteLine("Different lengths.");
check = false;
}
else
{
for (int i = 0; i < decompressedBytes.Length; i++)
{
if (original[i] != decompressedBytes[i])
{
TestContext.WriteLine("byte {0} differs", i);
check = false;
break;
}
}
}
Assert.IsTrue(check,"Data check failed");
TestContext.WriteLine("{0}: Done...", sw.Elapsed );
}
/// <summary>
/// Initializes a new instance of the <see cref="MpqArchive"/> class.
/// </summary>
/// <param name="sourceStream">The <see cref="Stream"/> containing pre-archive data. Can be <see langword="null"/>.</param>
/// <param name="inputFiles">The <see cref="MpqFile"/>s that should be added to the archive.</param>
/// <param name="createOptions"></param>
/// <param name="leaveOpen">If <see langword="false"/>, the given <paramref name="sourceStream"/> will be disposed when the <see cref="MpqArchive"/> is disposed.</param>
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="mpqFiles"/> collection is <see langword="null"/>.</exception>
public MpqArchive(Stream?sourceStream, IEnumerable <MpqFile> inputFiles, MpqArchiveCreateOptions createOptions, bool leaveOpen = false)
{
if (inputFiles is null)
{
throw new ArgumentNullException(nameof(inputFiles));
}
if (createOptions is null)
{
throw new ArgumentNullException(nameof(createOptions));
}
_isStreamOwner = !leaveOpen;
_baseStream = AlignStream(sourceStream);
_headerOffset = _baseStream.Position;
_blockSize = BlockSizeModifier << createOptions.BlockSize;
_archiveFollowsHeader = createOptions.WriteArchiveFirst;
var signatureName = Signature.FileName.GetStringHash();
var listFileName = ListFile.FileName.GetStringHash();
var attributesName = Attributes.FileName.GetStringHash();
var signatureCreateMode = createOptions.SignatureCreateMode.GetValueOrDefault(MpqFileCreateMode.Prune);
var listFileCreateMode = createOptions.ListFileCreateMode.GetValueOrDefault(MpqFileCreateMode.Overwrite);
var attributesCreateMode = createOptions.AttributesCreateMode.GetValueOrDefault(MpqFileCreateMode.Overwrite);
var haveSignature = false;
var haveListFile = false;
var haveAttributes = false;
var mpqFiles = new HashSet <MpqFile>(MpqFileComparer.Default);
foreach (var mpqFile in inputFiles)
{
if (mpqFile is MpqOrphanedFile)
{
continue;
}
if (mpqFile.Name == signatureName)
{
if (signatureCreateMode.HasFlag(MpqFileCreateMode.RemoveFlag))
{
continue;
}
else
{
haveSignature = true;
}
}
if (mpqFile.Name == listFileName)
{
if (listFileCreateMode.HasFlag(MpqFileCreateMode.RemoveFlag))
{
continue;
}
else
{
haveListFile = true;
}
}
if (mpqFile.Name == attributesName)
{
if (attributesCreateMode.HasFlag(MpqFileCreateMode.RemoveFlag))
{
continue;
}
else
{
haveAttributes = true;
}
}
if (!mpqFiles.Add(mpqFile))
{
// todo: logging?
}
}
var fileCount = (uint)mpqFiles.Count;
var wantGenerateSignature = !haveSignature && signatureCreateMode.HasFlag(MpqFileCreateMode.AddFlag);
var signature = wantGenerateSignature ? new Signature() : null;
if (wantGenerateSignature)
{
fileCount++;
}
var wantGenerateListFile = !haveListFile && listFileCreateMode.HasFlag(MpqFileCreateMode.AddFlag);
var listFile = wantGenerateListFile ? new ListFile() : null;
if (wantGenerateListFile)
{
fileCount++;
}
var wantGenerateAttributes = !haveAttributes && attributesCreateMode.HasFlag(MpqFileCreateMode.AddFlag);
var attributes = wantGenerateAttributes ? new Attributes(createOptions) : null;
if (wantGenerateAttributes)
{
fileCount++;
}
_hashTable = new HashTable(Math.Max(createOptions.HashTableSize ?? fileCount * 8, fileCount));
_blockTable = new BlockTable();
using (var writer = new BinaryWriter(_baseStream, new UTF8Encoding(false, true), true))
{
// Skip the MPQ header, since its contents will be calculated afterwards.
writer.Seek((int)MpqHeader.Size, SeekOrigin.Current);
// Write Archive
var fileIndex = 0U;
var fileOffset = _archiveFollowsHeader ? MpqHeader.Size : throw new NotImplementedException();
// var gaps = new List<(long Start, long Length)>();
var endOfStream = _baseStream.Position;
void InsertMpqFile(MpqFile mpqFile, bool updateEndOfStream, bool allowMultiple = true)
{
if (listFile is not null && mpqFile is MpqKnownFile knownFile)
{
listFile.FileNames.Add(knownFile.FileName);
}
mpqFile.AddToArchive(this, fileIndex, out var mpqEntry, out var mpqHash);
var hashTableEntries = _hashTable.Add(mpqHash, mpqFile.HashIndex, mpqFile.HashCollisions);
if (!allowMultiple && hashTableEntries > 1)
{
throw new Exception();
}
var crc32 = 0;
if (attributes is not null && attributes.Flags.HasFlag(AttributesFlags.Crc32) && allowMultiple)
{
mpqFile.MpqStream.Position = 0;
crc32 = new Ionic.Crc.CRC32().GetCrc32(mpqFile.MpqStream);
}
for (var i = 0; i < hashTableEntries; i++)
{
_blockTable.Add(mpqEntry);
if (attributes is not null)
{
if (attributes.Flags.HasFlag(AttributesFlags.Crc32))
{
attributes.Crc32s.Add(crc32);
}
if (attributes.Flags.HasFlag(AttributesFlags.DateTime))
{
attributes.DateTimes.Add(DateTime.Now);
}
if (attributes.Flags.HasFlag(AttributesFlags.Unk0x04))
{
attributes.Unk0x04s.Add(new byte[16]);
}
}
}
mpqFile.Dispose();
fileIndex += hashTableEntries;
if (updateEndOfStream)
{
endOfStream = _baseStream.Position;
}
}
// Find files that cannot be decrypted, and need to have a specific position in the archive, because that position is used to calculate the encryption seed.
var mpqFixedPositionFiles = mpqFiles.Where(mpqFile => mpqFile.IsFilePositionFixed).OrderBy(mpqFile => mpqFile.MpqStream.FilePosition).ToArray();
if (mpqFixedPositionFiles.Length > 0)
{
if (mpqFixedPositionFiles.First() !.MpqStream.FilePosition < 0)
{
throw new NotSupportedException($"Cannot place files in front of the header.");
}
foreach (var mpqFixedPositionFile in mpqFixedPositionFiles)
{
var position = mpqFixedPositionFile.MpqStream.FilePosition;
if (position < endOfStream)
{
throw new ArgumentException($"Fixed position files overlap with each other and/or the header. Archive cannot be created.", nameof(inputFiles));
}
if (position > endOfStream)
{
var gapSize = position - endOfStream;
// gaps.Add((endOfStream, gapSize));
writer.Seek((int)gapSize, SeekOrigin.Current);
}
InsertMpqFile(mpqFixedPositionFile, true);
}
}
foreach (var mpqFile in mpqFiles.Where(mpqFile => !mpqFile.IsFilePositionFixed))
{
// TODO: insert files into the gaps
// need to know compressed size of file first, and if file is also encrypted with blockoffsetadjustedkey, encryption needs to happen after gap selection
// therefore, can't use current AddToArchive method, which does both compression and encryption at same time
// var availableGaps = gaps.Where(gap => gap.Length >= )
var selectedPosition = endOfStream;
var selectedGap = false;
_baseStream.Position = selectedPosition;
InsertMpqFile(mpqFile, !selectedGap);
}
var signaturePosition = endOfStream + 8;
if (signature is not null)
{
_baseStream.Position = endOfStream;
using var signatureStream = new MemoryStream();
using var signatureWriter = new BinaryWriter(signatureStream);
signatureWriter.Write(signature);
signatureWriter.Flush();
using var signatureMpqFile = MpqFile.New(signatureStream, Signature.FileName);
signatureMpqFile.TargetFlags = MpqFileFlags.Exists;
InsertMpqFile(signatureMpqFile, true);
}
if (listFile is not null)
{
_baseStream.Position = endOfStream;
using var listFileStream = new MemoryStream();
using var listFileWriter = new StreamWriter(listFileStream);
listFileWriter.WriteListFile(listFile);
listFileWriter.Flush();
using var listFileMpqFile = MpqFile.New(listFileStream, ListFile.FileName);
listFileMpqFile.TargetFlags = MpqFileFlags.Exists | MpqFileFlags.CompressedMulti | MpqFileFlags.Encrypted | MpqFileFlags.BlockOffsetAdjustedKey;
InsertMpqFile(listFileMpqFile, true);
}
if (attributes is not null)
{
_baseStream.Position = endOfStream;
if (attributes.Flags.HasFlag(AttributesFlags.Crc32))
{
attributes.Crc32s.Add(0);
}
if (attributes.Flags.HasFlag(AttributesFlags.DateTime))
{
attributes.DateTimes.Add(DateTime.Now);
}
if (attributes.Flags.HasFlag(AttributesFlags.Unk0x04))
{
attributes.Unk0x04s.Add(new byte[16]);
}
using var attributesStream = new MemoryStream();
using var attributesWriter = new BinaryWriter(attributesStream);
attributesWriter.Write(attributes);
attributesWriter.Flush();
using var attributesMpqFile = MpqFile.New(attributesStream, Attributes.FileName);
attributesMpqFile.TargetFlags = MpqFileFlags.Exists | MpqFileFlags.CompressedMulti | MpqFileFlags.Encrypted | MpqFileFlags.BlockOffsetAdjustedKey;
InsertMpqFile(attributesMpqFile, true, false);
}
_baseStream.Position = endOfStream;
_hashTable.WriteTo(writer);
_blockTable.WriteTo(writer);
/*if (!_archiveFollowsHeader)
* {
* foreach (var mpqFile in mpqFiles)
* {
* mpqFile.WriteTo(writer, true);
* }
* }*/
writer.Seek((int)_headerOffset, SeekOrigin.Begin);
_mpqHeader = new MpqHeader((uint)_headerOffset, (uint)(endOfStream - fileOffset), _hashTable.Size, _blockTable.Size, createOptions.BlockSize, _archiveFollowsHeader);
_mpqHeader.WriteTo(writer);
if (wantGenerateSignature)
{
var archiveBytes = new byte[_mpqHeader.ArchiveSize];
_baseStream.Position = _headerOffset;
_baseStream.Read(archiveBytes);
using var rsa = RSA.Create();
rsa.ImportFromPem(createOptions.SignaturePrivateKey);
var signatureBytes = rsa.SignData(archiveBytes, HashAlgorithmName.MD5, RSASignaturePadding.Pkcs1);
_baseStream.Position = signaturePosition;
_baseStream.Write(signatureBytes.Reverse().ToArray());
}
}
}
public static void SetModded()
{
try {
if (global::Rust.Global.SteamServer == null)
return;
using (TimeWarning.New("UpdateServerInformation", 0.1f)) {
var steamServer = global::Rust.Global.SteamServer;
System.Reflection.Assembly assembly = typeof(ServerMgr).Assembly;
byte[] byteArray = System.IO.File.ReadAllBytes(assembly.Location);
Ionic.Crc.CRC32 cRC = new Ionic.Crc.CRC32();
cRC.SlurpBlock(byteArray, 0, byteArray.Length);
string _AssemblyHash = cRC.Crc32Result.ToString("x");
steamServer.ServerName = ConVar.Server.hostname;
steamServer.MaxPlayers = ConVar.Server.maxplayers;
steamServer.Passworded = false;
steamServer.MapName = UnityEngine.SceneManagement.SceneManager.GetActiveScene().name;
string gameTags = string.Format("mp{0},cp{1},qp{5},v{2}{3},h{4}", new object[] {
ConVar.Server.maxplayers,
BasePlayer.activePlayerList.Count,
global::Rust.Protocol.network,
ConVar.Server.pve ? ",pve" : string.Empty,
pluton.enabled ? ",modded,pluton" : string.Empty,
_AssemblyHash,
ServerMgr.Instance.connectionQueue.Queued
});
steamServer.GameTags = gameTags;
string[] array = ConVar.Server.description.SplitToChunks(100).ToArray();
for (int i = 0; i < 16; i++) {
if (i < array.Length) {
steamServer.SetKey(string.Format("description_{0:00}", i), array[i]);
} else {
steamServer.SetKey(string.Format("description_{0:00}", i), string.Empty);
}
}
steamServer.SetKey("hash", _AssemblyHash);
steamServer.SetKey("world.seed", global::World.Seed.ToString());
steamServer.SetKey("world.size", global::World.Size.ToString());
steamServer.SetKey("pve", ConVar.Server.pve.ToString());
steamServer.SetKey("headerimage", ConVar.Server.headerimage);
steamServer.SetKey("url", ConVar.Server.url);
steamServer.SetKey("uptime", ((int)Time.realtimeSinceStartup).ToString());
steamServer.SetKey("mem_ws", Performance.report.usedMemoryWorkingSetMB.ToString());
steamServer.SetKey("mem_pv", Performance.report.usedMemoryPrivateMB.ToString());
steamServer.SetKey("gc_mb", Performance.report.memoryAllocations.ToString());
steamServer.SetKey("gc_cl", Performance.report.memoryCollections.ToString());
steamServer.SetKey("fps", Performance.report.frameRate.ToString());
steamServer.SetKey("fps_avg", Performance.report.frameRateAverage.ToString("0.00"));
steamServer.SetKey("ent_cnt", BaseNetworkable.serverEntities.Count.ToString());
steamServer.SetKey("build", BuildInformation.VersionStampDays.ToString());
}
} catch (Exception ex) {
Logger.LogError("[Hooks] Error while setting the server modded.");
Logger.LogException(ex);
}
}
Int32 FigureCrc32()
{
if (_crcCalculated == false)
{
Stream input = null;
// get the original stream:
if (this._Source == ZipEntrySource.WriteDelegate)
{
var output = new Ionic.Crc.CrcCalculatorStream(Stream.Null);
// allow the application to write the data
this._WriteDelegate(this.FileName, output);
_Crc32 = output.Crc;
}
else if (this._Source == ZipEntrySource.ZipFile)
{
// nothing to do - the CRC is already set
}
else
{
if (this._Source == ZipEntrySource.Stream)
{
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.JitStream)
{
// allow the application to open the stream
if (this._sourceStream == null)
_sourceStream = this._OpenDelegate(this.FileName);
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.ZipOutputStream)
{
}
else
{
}
var crc32 = new Ionic.Crc.CRC32();
_Crc32 = crc32.GetCrc32(input);
}
_crcCalculated = true;
}
return _Crc32;
}
