protected override void OnReceive(StreamBlock block)
{
// agregarmos al buffer.
STrace.Debug(GetType().FullName, deviceId, String.Format("FileServer: bloque de datos={0}", block.TotalBytes));
Array.Copy(block.Data, 0, active_buffer, buffer_position, block.TotalBytes);
buffer_position += block.TotalBytes;
if (!headers_readed)
{
if (buffer_position >= 71)
{
// leer la cabecera... podemos
// Short IdDispositivo
// Byte Destino (0x00 = File, 0x01 = Queue)
// Integer Tamaño del Archivo (setea $FileSize)
// Bytes[64] Nombre de Archivo o Base64MessageQueue segun corresponda (Rellenado con 0)
// Bytes[$FileSize] Datos del archivo.
var pos = 0;
deviceId = UrbetrackCodec.DecodeShort(active_buffer, ref pos);
op = UrbetrackCodec.DecodeByte(active_buffer, ref pos);
chunk_size = UrbetrackCodec.DecodeInteger(active_buffer, ref pos);
label = UrbetrackCodec.DecodeBytesAsString(active_buffer, ref pos, 64);
headers_readed = true;
strm = new MemoryStream();
STrace.Debug(GetType().FullName, deviceId, String.Format("Iniciando Recepcion: queue={0} size={1}", label, chunk_size));
}
else
{
return; // aun no leimos suficiente.
}
}
while (buffer_position > payload_start)
{
STrace.Debug(GetType().FullName, deviceId, String.Format("Recibiendo bloque: size={0}", buffer_position - payload_start));
var data_block = new byte[buffer_position - payload_start];
Array.Copy(active_buffer, payload_start, data_block, 0, buffer_position - payload_start);
strm.Write(data_block, 0, buffer_position - payload_start);
buffer_position = payload_start = 0;
if (strm.Length != chunk_size)
{
continue;
}
bool local_result;
try
{
local_result = MessageReceived(deviceId, op, label, strm, true);
}
catch (Exception e)
{
STrace.Exception(GetType().FullName, e);
local_result = false;
}
Send(Encoding.ASCII.GetBytes(local_result ? "A" : "N"), 1, deviceId, "MemoryStream");
STrace.Debug(GetType().FullName, deviceId, String.Format("Respondo {0} y cierro.", local_result ? "A" : "N"));
Disconnect();
}
}
public override void OnReceive(StreamBlock block)
{
// agregarmos al buffer.
TRACE("bloque de datos={0}", block.TotalBytes);
Array.Copy(block.Data, 0, active_buffer, buffer_position, block.TotalBytes);
buffer_position += block.TotalBytes;
var ack_buffer = ActiveBufferPop(1);
if (ack_buffer == null)
{
return;
}
Parent.LastResult = ack_buffer[0] == 'A';
TRACE("SYNCRO.SET");
Parent.syncro.Set();
}
private static void LoadResourceInfoFromAssetFile(AssetBundleEx ab, AssetFileEx af, string classname)
{
List <ObjectData> odlist = af.extractlist[classname];
Dictionary <string, ResourceInfoEx> resourcetableoftype = resourcetable[classname];
foreach (ObjectData od in odlist)
{
ResourceInfoEx currentinfo = new ResourceInfoEx();
currentinfo.bundlename = ab.name;
currentinfo.assetfilename = af.sourceFile;
currentinfo.objectname = od.objectname;
currentinfo.classname = od.classname;
currentinfo.typename = od.typename;
currentinfo.propertyfield = od.propertyfield;
currentinfo.path = od.path;
object data = od.data;
if (data is StreamBlock)
{
StreamBlock sdi = (StreamBlock)od.data;
if (sdi != null)
{
currentinfo.offset = sdi.offset;
currentinfo.length = sdi.length;
}
}
if (currentinfo.objectname == null || currentinfo.objectname == "")
{
continue;
}
//if (currentinfo.path == null || currentinfo.path == "")
//{
// continue;
//}
if (resourcetableoftype.ContainsKey(currentinfo.objectname))
{
ResourceInfoEx otherinfo = resourcetableoftype[od.objectname];
//Logger.Log("current=" + currentinfo.objectname + "<" + currentinfo.typename + ">,other=" + otherinfo.objectname + "<" + otherinfo.typename + ">");
}
else
{
resourcetableoftype.Add(currentinfo.objectname, currentinfo);
}
}
}
public void CouldCreateUninitializedBlock()
{
var rm = BufferPool.Retain(1024, true);
Assert.Throws <InvalidOperationException>(() =>
{
var _ = new StreamBlock(rm, default);
});
var sb = new StreamBlock(new DirectBuffer(rm));
Assert.IsTrue(sb.IsValid);
Assert.IsFalse(sb.IsRented);
Assert.IsFalse(sb.IsInitialized);
rm.Dispose();
}
public override void OnReceive(StreamBlock block)
{
TRACE("bloque de datos={0}", block.TotalBytes);
Array.Copy(block.Data, 0, active_buffer, buffer_position, block.TotalBytes);
buffer_position += block.TotalBytes;
var hdrs = ActiveBufferPop(27);
var fields = Encoding.ASCII.GetString(hdrs).Split("/".ToCharArray());
if (fields.GetLength(0) != 4)
{
Send("TENGO UN PATO!");
Disconnect();
return;
}
IMEI = fields[2];
var d = Devices.i().Find(IMEI);
/*if (d == null)
* {
* T.TRACE("FOTASERVER: DEVICE w/IMEI={0} no encontrado");
* send("YOU ARE BANNED HERE!");
* close();
* return;
* }
*
* T.TRACE("FOTASERVER: DEVICE[{0}] conectado", d.LogId);
*/
if (fields[3] != "REQ")
{
return;
}
//T.TRACE("FOTASERVER: DEVICE[{0}] solicita firmware", d.LogId);
// var file = "rnr.bin";
var file = "rnr.bin";
//var path = @"C:\Documents and Settings\evecchio\Escritorio\rnros\trunk\torino\obj\";
var path = @"C:\Documents and Settings\evecchio\Escritorio\rnros\trunk\torino\obj";
Send("L4PNP");
SendFile(d, "TEST", file, path);
//close();
}
public unsafe void CountAndChecksum()
{
var path = TestUtils.GetPath();
var rounds = 10;
var count = 100_000;
var bufferSize = count * 16 + StreamBlock.DataOffset;
var pool = new BlockMemoryPool(path, 64, LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync, (Wpid)123,
maxBufferLength: BitUtil.FindNextPositivePowerOfTwo(bufferSize));
// TODO step on what happens when buffer is too large
var buffer = pool.RentMemory(bufferSize);
var rm = buffer.RetainBlockMemory(false);
var rm2 = rm.Clone();
var seed = 2u;
StreamBlock.TryInitialize(new DirectBuffer(rm2), (StreamLogId)1, default, 16, 1, seed, default);
public override void OnReceive(StreamBlock block)
{
if (State == States.FAILURE)
{
Disconnect();
return;
}
// agregarmos al buffer.
TRACE("bloque de datos={0}", block.TotalBytes);
Array.Copy(block.Data, 0, active_buffer, buffer_position, block.TotalBytes);
buffer_position += block.TotalBytes;
var continue_required = true;
while (continue_required && buffer_position > 0)
{
switch (State)
{
case States.WAITFOR_VERSION:
continue_required = OnReceiveWaitForVersion();
break;
case States.WAITFOR_CHUNK:
continue_required = OnReceiveWaitForChunk();
break;
case States.READING_CHUNK:
continue_required = OnReceiveReadingChunk();
break;
default:
Disconnect();
continue_required = false;
break;
}
}
}
/// <summary>
/// Helper for checking if we are at the end of the stream block
/// </summary>
/// <param name="reader"></param>
/// <returns></returns>
public static bool EndOfBlock(ExtendedBinaryReader reader, StreamBlock streamBlock)
{
return(reader.BaseStream.Position >= streamBlock.Length);
}
public void ReadFromFile(BinaryReader reader)
{
if (reader.ReadInt32() != 1299346515)
{
return;
}
if (reader.ReadInt32() != 0x6)
{
return;
}
int fileSize = reader.ReadInt32();
int unk0 = reader.ReadInt32();
int numGroups = reader.ReadInt32();
int groupOffset = reader.ReadInt32();
int numHeaders = reader.ReadInt32();
int headerOffset = reader.ReadInt32();
int numLines = reader.ReadInt32();
int lineOffset = reader.ReadInt32();
int numLoaders = reader.ReadInt32();
int loadersOffset = reader.ReadInt32();
int numBlocks = reader.ReadInt32();
int blockOffset = reader.ReadInt32();
int numHashes = reader.ReadInt32();
int hashOffset = reader.ReadInt32();
int poolSize = reader.ReadInt32();
int poolOffset = reader.ReadInt32();
Debug.Assert(reader.BaseStream.Position == groupOffset, "Failed to reach the starting offset for group declaration!");
groups = new StreamGroup[numGroups];
for (int i = 0; i < numGroups; i++)
{
StreamGroup map = new StreamGroup();
map.nameIDX = reader.ReadInt32();
map.Name = ReadFromBuffer((long)((ulong)map.nameIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Type = (GroupTypes)reader.ReadInt32();
map.Unk01 = reader.ReadInt32();
map.startOffset = reader.ReadInt32();
map.endOffset = reader.ReadInt32();
map.unk5 = reader.ReadInt32();
groups[i] = map;
}
Debug.Assert(reader.BaseStream.Position == headerOffset, "Did not reach the header starting offset");
groupHeaders = new string[numHeaders];
ulong[] ulongHeaders = new ulong[numHeaders];
for (int i = 0; i < numHeaders; i++)
{
ulongHeaders[i] = reader.ReadUInt64();
groupHeaders[i] = ReadFromBuffer((long)(ulongHeaders[i] + (ulong)poolOffset), reader.BaseStream.Position, reader);
}
Debug.Assert(reader.BaseStream.Position == lineOffset, "Did not reach the line data starting offset!");
lines = new StreamLine[numLines];
for (int i = 0; i < numLines; i++)
{
StreamLine map = new StreamLine();
map.nameIDX = reader.ReadInt32();
map.lineID = reader.ReadInt32();
map.groupID = reader.ReadInt32();
map.LoadType = reader.ReadInt32();
map.flagIDX = reader.ReadInt32();
map.Unk5 = reader.ReadInt32();
map.Unk10 = reader.ReadUInt64();
map.Unk11 = reader.ReadUInt64();
map.Unk12 = reader.ReadInt32();
map.Unk13 = reader.ReadInt32();
map.Unk14 = reader.ReadInt32();
map.Unk15 = reader.ReadInt32();
map.Group = groupHeaders[map.groupID];
map.Name = ReadFromBuffer((long)((ulong)map.nameIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Flags = ReadBufferSpecial((long)((ulong)map.flagIDX + (ulong)poolOffset), reader.BaseStream.Position, reader).TrimEnd('\0').Replace('\0', '|');
lines[i] = map;
}
Debug.Assert(reader.BaseStream.Position == loadersOffset, "Did not reach the loader data starting offset!");
loaders = new StreamLoader[numLoaders];
for (int i = 0; i < numLoaders; i++)
{
StreamLoader map = new StreamLoader();
map.start = reader.ReadInt32();
map.end = reader.ReadInt32();
map.Type = (GroupTypes)reader.ReadInt32();
map.LoaderSubID = reader.ReadInt32();
map.LoaderID = reader.ReadInt32();
map.LoadType = reader.ReadInt32();
map.pathIDX = reader.ReadInt32();
map.entityIDX = reader.ReadInt32();
map.Path = ReadFromBuffer((long)((ulong)map.pathIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Entity = ReadBufferSpecial((long)((ulong)map.entityIDX + (ulong)poolOffset), reader.BaseStream.Position, reader).TrimEnd('\0').Replace('\0', '|');
loaders[i] = map;
}
Debug.Assert(reader.BaseStream.Position == blockOffset, "Did not reach the block declaration starting offset!");
blocks = new StreamBlock[numBlocks];
for (int i = 0; i < numBlocks; i++)
{
StreamBlock map = new StreamBlock();
map.startOffset = reader.ReadInt32();
map.endOffset = reader.ReadInt32();
map.Hashes = new ulong[map.endOffset - map.startOffset];
blocks[i] = map;
}
Debug.Assert(reader.BaseStream.Position == hashOffset, "Did not reach the block hashes starting offset!");
hashes = new ulong[numHashes];
for (int i = 0; i < numHashes; i++)
{
hashes[i] = reader.ReadUInt64();
}
for (int i = 0; i < numBlocks; i++)
{
var block = blocks[i];
Array.Copy(hashes, block.startOffset, block.Hashes, 0, block.Hashes.Length);
}
Debug.Assert(reader.BaseStream.Position == poolOffset, "Did not reach the buffer pool starting offset!");
reader.BaseStream.Seek(poolSize, SeekOrigin.Current);
Debug.Assert(reader.BaseStream.Position == reader.BaseStream.Length, "Did not reach the end of the file!");
}
public void CouldAddNextChunkAfterCompleted()
{
Settings.DoAdditionalCorrectnessChecks = false;
var path = TestUtils.GetPath();
var processConfig = new ProcessConfig(path);
StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync;
StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; // LMDBEnvironmentFlags.MapAsync | LMDBEnvironmentFlags.NoMetaSync | LMDBEnvironmentFlags.WriteMap;
var slm = new StreamLogManager(processConfig, "CouldAddNextChunk", null, 512, true, true);
var bufferPool = slm.BufferPool;
var blockIndex = slm.BlockIndex;
var streamId = (StreamLogId)42;
short valueSize = 8;
var state = slm.StateStorage.GetState(streamId);
state.CheckInit(streamId, 8, SerializationFormat.Binary);
var sl = new StreamLog(slm, state, textId: "test_stream");
var count = 100;
SharedMemory currentBuffer = null;
using (Benchmark.Run("AddNext (KOPS)", count * 1000))
{
var version = 1UL;
var block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version, default);
Assert.AreEqual(2, block.SharedMemory.ReferenceCount);
// var firstBufferRef = block.SharedMemory.BufferRef;
for (int i = 0; i < count; i++)
{
version = (ulong)i + 1;
if (!block.Claim(version, 8).IsValid)
{
Assert.Fail("!slc.Claim(version, 8).IsValid");
}
block.Commit();
block.Complete();
var bufferRef = block.SharedMemory.BufferRef;
block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
var bufferRef1 = block.SharedMemory.BufferRef;
if (bufferRef1 == bufferRef)
{
Assert.Fail("bufferRef1 == bufferRef");
}
// As if someone has created the next buffer before the second call, forget the first call
var prev = block;
// block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
prev.DisposeFree();
var bufferRef2 = block.SharedMemory.BufferRef;
if (block.SharedMemory.ReferenceCount != 2)
{
Assert.Fail($"block.SharedMemory.RefCount {block.SharedMemory.ReferenceCount} != 1");
}
if (bufferRef1 != bufferRef2)
{
Assert.Fail($"bufferRef1 {bufferRef1} != bufferRef2 {bufferRef2}");
}
var nextVersion = block.NextVersion;
block.DisposeFree();
// block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, nextVersion);
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, nextVersion, default);
}
// SharedMemory.Free(buffer);
block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, version + 1);
// SharedMemory.Free(buffer);
block.DisposeFree();
}
bufferPool.PrintBuffers();
bufferPool.PrintBuffersAfterPoolDispose = true;
Benchmark.Dump();
Console.WriteLine("Finished");
sl.Dispose();
slm.Dispose();
}
public void BlockForLoopCopy() => AlignedBlockSrc = AlignedBlockDest;
public unsafe void StreamBlockHeaderLayout()
{
var ty = typeof(StreamBlock.StreamBlockHeader);
var rm = BufferPool.Retain(4096, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1L, 1, 1);
var sb = new StreamBlock(rm, (StreamLogId)1L);
Assert.AreEqual(0, StreamBlock.StreamBlockHeader.InitTimestampOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.InitTimestampOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.InitTimestamp)));
Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).InitTimestamp.GetType());
Assert.AreEqual(64, StreamBlock.StreamBlockHeader.VersionAndFlagsOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.VersionAndFlagsOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.VersionAndFlags)));
Assert.AreEqual(typeof(VersionAndFlags), default(StreamBlock.StreamBlockHeader).VersionAndFlags.GetType());
Assert.AreEqual(65, StreamBlock.StreamBlockHeader.AdditionalFlagsOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.AdditionalFlagsOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.AdditionalFlags)));
Assert.AreEqual(typeof(byte), default(StreamBlock.StreamBlockHeader).AdditionalFlags.GetType());
Assert.AreEqual(66, StreamBlock.StreamBlockHeader.ItemFixedSizeOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemFixedSizeOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemFixedSize)));
Assert.AreEqual(typeof(short), default(StreamBlock.StreamBlockHeader).ItemFixedSize.GetType());
Assert.AreEqual(68, StreamBlock.StreamBlockHeader.PayloadLengthOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.PayloadLengthOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PayloadLength)));
Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).PayloadLength.GetType());
Assert.AreEqual(72, StreamBlock.StreamBlockHeader.StreamLogIdOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.StreamLogIdOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.StreamLogId)));
Assert.AreEqual(typeof(StreamLogId), default(StreamBlock.StreamBlockHeader).StreamLogId.GetType());
Assert.AreEqual(typeof(StreamLogId), sb.StreamLogId.GetType());
Assert.AreEqual(typeof(long), sb.StreamLogIdLong.GetType());
Assert.AreEqual(80, StreamBlock.StreamBlockHeader.FirstVersionOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.FirstVersionOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.FirstVersion)));
Assert.AreEqual(typeof(ulong), default(StreamBlock.StreamBlockHeader).FirstVersion.GetType());
Assert.AreEqual(typeof(ulong), sb.FirstVersion.GetType());
Assert.AreEqual(1, sb.FirstVersion);
Assert.AreEqual(88, StreamBlock.StreamBlockHeader.CountOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.CountOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Count)));
Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).Count.GetType());
Assert.AreEqual(typeof(int), sb.CountVolatile.GetType());
Assert.AreEqual(92, StreamBlock.StreamBlockHeader.ChecksumOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ChecksumOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Checksum)));
Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).Checksum.GetType());
Assert.AreEqual(typeof(uint), sb.Checksum.GetType());
Assert.AreEqual(96, StreamBlock.StreamBlockHeader.WriteEndTimestampOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.WriteEndTimestampOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.WriteEnd)));
Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).WriteEnd.GetType());
Assert.AreEqual(typeof(Timestamp), sb.WriteEnd.GetType());
Assert.AreEqual(104, StreamBlock.StreamBlockHeader.HashOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.HashOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Hash)));
Assert.AreEqual(typeof(Hash16), default(StreamBlock.StreamBlockHeader).Hash.GetType());
Assert.AreEqual(typeof(Hash16), sb.Hash.GetType());
Assert.AreEqual(120, StreamBlock.StreamBlockHeader.PkContextIdOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.PkContextIdOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PkContextId)));
Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).PkContextId.GetType());
Assert.AreEqual(typeof(uint), sb.PkContextId.GetType());
Assert.AreEqual(124, StreamBlock.StreamBlockHeader.ItemDthOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemDthOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemDth)));
Assert.AreEqual(typeof(DataTypeHeader), default(StreamBlock.StreamBlockHeader).ItemDth.GetType());
Assert.AreEqual(typeof(DataTypeHeader), sb.ItemDth.GetType());
rm.Dispose();
}
public override void OnReceive(StreamBlock block)
{
// agregarmos al buffer.
TRACE("datos en buffer={0} recibidos={1}", buffer_position, block.TotalBytes);
Array.Copy(block.Data, 0, active_buffer, buffer_position, block.TotalBytes);
buffer_position += block.TotalBytes;
if (!headers_readed)
{
if (buffer_position >= 71)
{
// leer la cabecera... podemos
// Short DeviceId
// Byte destination (0x00 = File, 0x01 = Queue)
// Integer Tamaño del Archivo (setea $FileSize)
// Bytes[64] Name de Archivo o Base64MessageQueue segun corresponda (Rellenado con 0)
// Bytes[$FileSize] Datos del archivo.
var pos = 0;
idDispositivo = Decoder.DecodeShort(active_buffer, ref pos);
op = Decoder.DecodeByte(active_buffer, ref pos);
chunk_size = Decoder.DecodeInteger(active_buffer, ref pos);
bfilename = Decoder.DecodeBytes(active_buffer, ref pos, 64);
int zpos;
for (zpos = 0; zpos < 64; ++zpos)
{
if (bfilename[zpos] != 0)
{
continue;
}
}
filename = Encoding.ASCII.GetString(bfilename, 0, zpos);
headers_readed = true;
switch (op)
{
case 0x01:
strm = new MemoryStream();
break;
default:
AbortsReceivingMessages++;
throw new Exception("UIQS: Tipo de operacion desconocida.");
}
TRACE("Recibiedo {0}{1} size={2} dev={3}", (op == 0x01 ? "Mensaje para cola=" : "Archivo nombre="), filename, chunk_size, idDispositivo);
}
else
{
return; // aun no leimos suficiente.
}
}
while (buffer_position > payload_start)
{
T.TRACE(String.Format("UIQS: Recibiedo bloque: size={0}", buffer_position - payload_start));
payload_size += buffer_position - payload_start;
var data_block = new byte[buffer_position - payload_start];
Array.Copy(active_buffer, payload_start, data_block, 0, buffer_position - payload_start);
strm.Write(data_block, 0, buffer_position - payload_start);
buffer_position = payload_start = 0;
if (strm.Length != chunk_size)
{
continue;
}
bool local_result;
try
{
local_result = complete_receive();
}
catch (Exception e)
{
T.EXCEPTION(e, "RECEIVE DATA");
local_result = false;
}
Send(Encoding.ASCII.GetBytes(local_result ? "A" : "N"), 1);
TRACE("Respondo {0} y cierro.", local_result ? "A" : "N");
Disconnect();
}
}
public void ReadFromFile(BinaryReader reader)
{
if (reader.ReadInt32() != 1299346515)
{
return;
}
if (reader.ReadInt32() != 0x6)
{
return;
}
int fileSize = reader.ReadInt32();
int unk0 = reader.ReadInt32();
int numGroups = reader.ReadInt32();
int groupOffset = reader.ReadInt32();
int numHeaders = reader.ReadInt32();
int headerOffset = reader.ReadInt32();
int numLines = reader.ReadInt32();
int lineOffset = reader.ReadInt32();
int numLoaders = reader.ReadInt32();
int loadersOffset = reader.ReadInt32();
int numBlocks = reader.ReadInt32();
int blockOffset = reader.ReadInt32();
int numHashes = reader.ReadInt32();
int hashOffset = reader.ReadInt32();
int poolSize = reader.ReadInt32();
int poolOffset = reader.ReadInt32();
if (reader.BaseStream.Position != groupOffset)
{
throw new FormatException();
}
groups = new StreamGroup[numGroups];
for (int i = 0; i < numGroups; i++)
{
StreamGroup map = new StreamGroup();
map.nameIDX = reader.ReadInt32();
map.Name = ReadFromBuffer((long)((ulong)map.nameIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Type = reader.ReadInt32();
map.Unk01 = reader.ReadInt32();
map.startOffset = reader.ReadInt32();
map.endOffset = reader.ReadInt32();
map.unk5 = reader.ReadInt32();
groups[i] = map;
}
if (reader.BaseStream.Position != headerOffset)
{
throw new FormatException();
}
groupHeaders = new string[numHeaders];
ulong[] ulongHeaders = new ulong[numHeaders];
for (int i = 0; i < numHeaders; i++)
{
ulongHeaders[i] = reader.ReadUInt64();
groupHeaders[i] = ReadFromBuffer((long)(ulongHeaders[i] + (ulong)poolOffset), reader.BaseStream.Position, reader);
}
if (reader.BaseStream.Position != lineOffset)
{
throw new FormatException();
}
lines = new StreamLine[numLines];
for (int i = 0; i < numLines; i++)
{
StreamLine map = new StreamLine();
map.nameIDX = reader.ReadInt32();
map.lineID = reader.ReadInt32();
map.groupID = reader.ReadInt32();
map.LoadType = reader.ReadInt32();
map.flagIDX = reader.ReadInt32();
map.Unk5 = reader.ReadInt32();
map.Unk10 = reader.ReadUInt64();
map.Unk11 = reader.ReadUInt64();
map.Unk12 = reader.ReadInt32();
map.Unk13 = reader.ReadInt32();
map.Unk14 = reader.ReadInt32();
map.Unk15 = reader.ReadInt32();
map.Group = groupHeaders[map.groupID];
map.Name = ReadFromBuffer((long)((ulong)map.nameIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Flags = ReadBufferSpecial((long)((ulong)map.flagIDX + (ulong)poolOffset), reader.BaseStream.Position, reader).TrimEnd('\0').Replace('\0', '|');
lines[i] = map;
}
if (reader.BaseStream.Position != loadersOffset)
{
throw new FormatException();
}
loaders = new StreamLoader[numLoaders];
for (int i = 0; i < numLoaders; i++)
{
StreamLoader map = new StreamLoader();
map.start = reader.ReadInt32();
map.end = reader.ReadInt32();
map.type = reader.ReadInt32();
map.loaderSubID = reader.ReadInt32();
map.loaderID = reader.ReadInt32();
map.LoadType = reader.ReadInt32();
map.pathIDX = reader.ReadInt32();
map.entityIDX = reader.ReadInt32();
map.Path = ReadFromBuffer((long)((ulong)map.pathIDX + (ulong)poolOffset), reader.BaseStream.Position, reader);
map.Entity = ReadBufferSpecial((long)((ulong)map.entityIDX + (ulong)poolOffset), reader.BaseStream.Position, reader).TrimEnd('\0').Replace('\0', '|');
loaders[i] = map;
}
if (reader.BaseStream.Position != blockOffset)
{
throw new FormatException();
}
blocks = new StreamBlock[numBlocks];
for (int i = 0; i < numBlocks; i++)
{
StreamBlock map = new StreamBlock();
map.startOffset = reader.ReadInt32();
map.endOffset = reader.ReadInt32();
blocks[i] = map;
}
if (reader.BaseStream.Position != hashOffset)
{
throw new FormatException();
}
hashes = new ulong[numHashes];
for (int i = 0; i < numHashes; i++)
{
hashes[i] = reader.ReadUInt64();
}
if (reader.BaseStream.Position != poolOffset)
{
throw new FormatException();
}
reader.BaseStream.Seek(poolSize, SeekOrigin.Current);
if (reader.BaseStream.Position != reader.BaseStream.Length)
{
throw new FormatException("Borked this up");
}
}
public void CouldWriteManyFinalizedChunksToTable()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
Settings.DoDetectBufferLeaks = true;
#pragma warning restore 618
ProcessConfig.InitDefault();
var path = TestUtils.GetPath();
ulong count = 100_000;
var rng = new Random(42);
var values = new SmallDecimal[count + 1];
for (ulong i = 1; i <= count; i++)
{
values[(int)i] = new SmallDecimal(10 + Math.Round(rng.NextDouble() * 2, 4), 4);
}
var storage = new SQLiteStorage($@"Filename={Path.Combine(path, "blockstorage.db")}");
var totalPayload = 0;
var totalCapacity = 0L;
//Settings.ZstdCompressionLevel = 1;
//Settings.LZ4CompressionLevel = 1;
//Settings.ZlibCompressionLevel = 5;
using (Benchmark.Run("Chunks", (int)(count)))
{
var chunkSize = 4032;
var rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, 1);
var block = new StreamBlock(rm, (StreamLogId)1, 8, 1);
var chunkCount = 1;
for (ulong i = 1; i <= count; i++)
{
var claim = block.Claim(i, 8);
if (!claim.IsValid)
{
Assert.IsTrue(block.IsCompleted);
var(inserted, rowid) = storage.InsertBlock(block);
//var couldRead = file.TryReadChunk(chunk.FirstVersion, out var readChunk);
//Assert.IsTrue(couldRead);
//Assert.AreEqual(chunk.WriteEnd, readChunk.WriteEnd);
//var fv = readChunk.FirstVersion;
//for (int j = 0; j < readChunk.Count; j++)
//{
// var ii = fv + (ulong)j;
// var readValue = readChunk[j].Read<SmallDecimal>(0);
// if (readValue != values[ii])
// {
// Assert.Fail();
// }
//}
//readChunk.Dispose();
block.DisposeFree();
chunkCount++;
if (chunkCount % 100 == 0)
{
storage.Checkpoint(true);
}
//if (chunkCount % 50 == 0)
//{
// file.Complete();
// totalPayload += file.PayloadSize;
// totalCapacity += file.FileCapacity;
// fileCount++;
// file.Dispose();
// filePath = Path.Combine(path, $"{fileCount}.slc");
// file = new StreamLogChunkFile(filePath, true);
//}
rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, i);
block = new StreamBlock(rm, (StreamLogId)1, 8, i);
claim = block.Claim(i, 8);
}
claim.Write(0, values[i]);
block.Commit();
}
var(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
var blockR = storage.TryGetStreamBlock(block.StreamLogIdLong, block.FirstVersion);
var lastVersion = blockR.CurrentVersion;
Assert.AreEqual(block.CurrentVersion, lastVersion);
blockR.DisposeFree();
block.Complete();
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
Assert.AreEqual(block.CurrentVersion, lastVersion);
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsFalse(inserted1);
//var couldRead1 = file.TryReadChunk(chunk.FirstVersion, out var readChunk1);
//Assert.IsTrue(couldRead1);
//Assert.AreEqual(chunk.WriteStart, readChunk1.WriteStart, "WriteStart");
//Assert.AreEqual(chunk.WriteEnd, readChunk1.WriteEnd, "WriteEnd");
//readChunk1.Dispose();
block.DisposeFree();
//totalPayload += file.PayloadSize;
//totalCapacity += file.FileCapacity;
Console.WriteLine("Chunk count: " + chunkCount);
//Console.WriteLine("Payload: " + totalPayload);
//Console.WriteLine("File size: " + totalCapacity);
Console.WriteLine("Useful size: " + count * 8);
//Console.WriteLine("Effective Compression: " + (totalPayload / (1.0 * count * 8)));
storage.Dispose();
}
}
public StreamBlockWithTimestamp(StreamBlock block)
{
Block = block;
}
public void Read(Stream stream)
{
using (var br = new BinaryReader(stream, Encoding.UTF8, true))
{
if (!br.CheckHeaderString("MAP6", reversed: true))
{
throw new Exception("Invalid FourCC header!");
}
var mapFileNameWithoutExtension = Path.GetFileNameWithoutExtension("global.map");
m_nTotalNumBlocks = br.ReadInt32();
var streamBlockArrayIdx = 0;
var itrCount = 0u;
do
{
m_nPalettes = br.ReadInt32();
m_pPalettes[streamBlockArrayIdx] = new StreamBlock[m_nPalettes];
if (m_nPalettes == 0)
{
++streamBlockArrayIdx;
continue;
}
var i = 0;
uint unkFlagStuff = ((itrCount + 1) & 7) << 10;
do
{
var key = br.ReadInt32();
var strLen = br.ReadUInt16();
var tempName = br.ReadStringWithMaxLength(strLen);
var nameWithItr = string.Format("{0}{1}", tempName, itrCount);
var block = m_pPalettes[streamBlockArrayIdx][i] = new StreamBlock();
block.m_nId = key;
// Skipping two blocks on unused data
br.BaseStream.Position += 12 * 2;
block.SetFileNameFromFormat("{0}\\{1}", mapFileNameWithoutExtension, nameWithItr);
// SKipping two unused shorts
br.BaseStream.Position += 2 * 2;
block.Flags = (unkFlagStuff | block.Flags & 0xFFFFE3FF) & 0xFFFFFF3F;
block.FieldA4 = 0.0f;
block.FieldA8 = 0.0f;
block.FieldAC = 0.0f;
block.FieldCC = -10000.0f;
block.FieldD0 = -10000.0f;
block.FieldD4 = -10000.0f;
block.FieldDC = 10000.0f;
block.FieldE0 = 10000.0f;
block.FieldE4 = 10000.0f;
block.FieldC0 = BitConverter.ToSingle(BitConverter.GetBytes(0xFFFFFFFF), 0);
block.ReadTextureInfo(br);
block.ReadTOCs(br);
block.Flags = (block.Flags & 0xFFFFFFF8) | 0x100;
}while (++i < m_nPalettes);
++streamBlockArrayIdx;
++itrCount;
}while (itrCount < 2);
LoadBlocksFromMapFile(br, m_nTotalNumBlocks, mapFileNameWithoutExtension, false);
}
}
protected override void OnReceive(StreamBlock block)
{
if (block.TotalBytes < 5)
{
STrace.Debug(GetType().FullName, String.Format("chunk invalido={0}", block.Data));
return;
}
Array.Copy(block.Data, 0, _activeBuffer, _pendingCount, block.TotalBytes);
_pendingCount += block.TotalBytes;
var start = 0;
while (_pendingCount > 0)
{
int cant;
bool ignorenoise;
var completed = _codec.IsPacketCompleted(_activeBuffer, start, _pendingCount, out cant, out ignorenoise);
if (ignorenoise)
{
if (cant == _pendingCount)
{
_pendingCount = 0;
return;
}
start += cant;
_pendingCount -= cant;
completed = _codec.IsPacketCompleted(_activeBuffer, start, _pendingCount, out cant, out ignorenoise);
//Debug.Assert(ignorenoise == false);
}
if (!completed)
{
if ((start != 0) && (_pendingCount != 0))
{
var tba = new byte[_pendingCount];
Array.Copy(_activeBuffer, start, tba, 0, _pendingCount);
Array.Copy(tba, 0, _activeBuffer, 0, _pendingCount);
}
return;
}
var activeChunk = new Chunk(_activeBuffer, start, cant, SourceEndpoint, _deviceId);
start += cant;
_pendingCount -= cant;
if (_iLink == null)
{
_iLink = _parent.DataLinkLayer.OnLinkTranslation(_parent, SourceEndpoint, activeChunk);
if (_iLink != null)
{
_iLink.UserState = this;
_deviceId = _iLink.Device.GetDeviceId();
activeChunk.DeviceId = _deviceId;
SendTo(activeChunk);
}
}
if (_parent.DataLinkLayer.OnFrameReceived(_iLink, activeChunk, _parent.Parser))
{
SendTo(activeChunk);
}
OsSocket.Disconnect(false);
}
}
