public SpatialBlock(string fileName)
{
FileName = fileName;
var exists = File.Exists(fileName);
Directory.CreateDirectory(Path.GetDirectoryName(fileName));
stream = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
stream = new BufferedStream(stream, 65536);
writer = new BinaryWriter(stream, Encoding.UTF8, true);
reader = new BinaryReader(stream, Encoding.UTF8, true);
nodeIdWriter = new DeltaWriter(writer);
nodeLatWriter = new DeltaWriter(writer);
nodeLonWriter = new DeltaWriter(writer);
wayIdWriter = new DeltaWriter(writer);
wayNodeIdWriter = new DeltaWriter(writer);
wayLatWriter = new DeltaWriter(writer);
wayLonWriter = new DeltaWriter(writer);
wayIdReader = new DeltaReader(reader);
wayNodeIdReader = new DeltaReader(reader);
wayLatReader = new DeltaReader(reader);
wayLonReader = new DeltaReader(reader);
if (exists)
{
ReadLastNodeData();
}
}
static void TestDelta <TInputStream, TOutputStream>(NetworkSchema schema, List <object> values, List <object> baselineValues) where TInputStream : NetworkCompression.IInputStream, new()
where TOutputStream : NetworkCompression.IOutputStream, new()
{
var inputBuffer = new byte[1024 * 64];
var baselineBuffer = new byte[1024 * 64];
var deltaBuffer = new byte[1024 * 64];
var outputBuffer = new byte[1024 * 64];
NetworkTestUtils.WriteValues(values, inputBuffer, schema);
if (baselineValues != null)
{
NetworkTestUtils.WriteValues(baselineValues, baselineBuffer, schema);
}
else
{
baselineBuffer = new byte[1024 * 64];
}
var outputStream = new TOutputStream();
outputStream.Initialize(NetworkCompressionModel.DefaultModel, deltaBuffer, 0, null);
uint hash = 0;
DeltaWriter.Write(ref outputStream, schema, inputBuffer, baselineBuffer, zeroFieldsChanged, 0, ref hash);
outputStream.Flush();
var inputStream = new TInputStream();
inputStream.Initialize(NetworkCompressionModel.DefaultModel, deltaBuffer, 0);
hash = 0;
DeltaReader.Read(ref inputStream, schema, outputBuffer, baselineBuffer, zeroFieldsChanged, 0, ref hash);
NetworkTestUtils.ReadAndAssertValues(values, outputBuffer, schema);
}
public void Test()
{
var stream = new MemoryStream();
var breader = new BinaryReader(stream, Encoding.UTF8, true);
var bwriter = new BinaryWriter(stream, Encoding.UTF8, true);
var writer = new DeltaWriter(bwriter);
var reader = new DeltaReader(breader);
writer.WriteZigZag(10);
writer.WriteZigZag(15);
writer.WriteZigZag(7);
writer.WriteZigZag(12);
writer.WriteZigZag(1);
bwriter.Flush();
stream.Position = 0;
reader.ReadZigZag().Should().Be(10);
reader.ReadZigZag().Should().Be(15);
reader.ReadZigZag().Should().Be(7);
reader.ReadZigZag().Should().Be(12);
reader.ReadZigZag().Should().Be(1);
}
public RelationsFile(string fileName)
{
stream = File.Open(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
stream = new BufferedStream(stream, 65536);
reader = new BinaryReader(stream, Encoding.UTF8, true);
writer = new BinaryWriter(stream, Encoding.UTF8, true);
idWriter = new DeltaWriter(writer);
memberIdWriter = new DeltaWriter(writer);
memberMidLatWriter = new DeltaWriter(writer);
memberMidLonWriter = new DeltaWriter(writer);
}
public byte[] EncodeSnapshot(Snapshot snapshot, int lastAckedCommandTick, int clientAckedTick)
{
snapshots.Add(snapshot.Tick, snapshot);
if (snapshot.Tick % 20 == 0)
{
snapshots.RemoveLowerThan(clientAckedTick);
}
Snapshot _snapshot;
if (GameConfig.deltaCompression && clientAckedTick != -1)
{
_snapshot = DeltaWriter.WriteDeltaSnapshot(snapshot, snapshots[clientAckedTick]);
}
else
{
_snapshot = snapshot;
}
INetBuffer writer;
if (GameConfig.bitPacking)
{
writer = bitBuffer;
}
else
{
writer = byteBuffer;
}
writer.CompressionSchema = compressionSchema;
writer.Reset();
writer.Put((byte)MessageTypes.Snapshot);
writer.Put(EncodeCompressionSchema());
writer.Put(lastAckedCommandTick);
int baselineTickMask = 0;
if (clientAckedTick != -1)
{
baselineTickMask = snapshot.Tick - clientAckedTick;
}
writer.Put((byte)baselineTickMask);
_snapshot.Serialize(writer);
return(writer.GetBytes());
}
public WaysDataFile(string fileName)
{
var exists = File.Exists(fileName);
var indexFileName = fileName + ".idx";
if (!exists)
{
waysStream = File.Open(fileName, FileMode.Create);
infoStream = File.Open(indexFileName, FileMode.Create);
}
else
{
waysStream = new FileStream(fileName, FileMode.Open, FileAccess.ReadWrite,
FileShare.None, 4096, FileOptions.RandomAccess);
//
// infoStream = new FileStream(indexFileName, FileMode.Open, FileAccess.ReadWrite,
// FileShare.None, 65536, FileOptions.SequentialScan);
infoFile = MemoryMappedFile.CreateFromFile(indexFileName, FileMode.Open, "waysIndex");
infoStream = infoFile.CreateViewStream();
}
waysWriter = new BinaryWriter(waysStream, Encoding.UTF8, true);
idWriter = new DeltaWriter(waysWriter);
nodeIdWriter = new DeltaWriter(waysWriter);
latWriter = new DeltaWriter(waysWriter);
lonWriter = new DeltaWriter(waysWriter);
infoWriter = new BinaryWriter(infoStream, Encoding.UTF8, true);
infoReader = new BinaryReader(infoStream, Encoding.UTF8, true);
if (exists)
{
var info = new FileInfo(indexFileName);
infoLength = info.Length;
ReadOffsets();
}
}
void WriteCommands <TOutputStream>(ClientPackageInfo packageInfo, ref TOutputStream output) where TOutputStream : NetworkCompression.IOutputStream
{
AddMessage(NetworkMessage.Commands);
counters.commandsOut++;
var includeSchema = commandSequenceAck == 0;
output.WriteRawBits(includeSchema ? 1U : 0, 1);
if (includeSchema)
{
NetworkSchema.WriteSchema(commandSchema, ref output);
}
var sequence = commandSequence;
output.WriteRawBits(Sequence.ToUInt16(commandSequence), 16);
packageInfo.commandSequence = commandSequence;
packageInfo.commandTime = commandsOut[commandSequence].time;
CommandInfo previous = defaultCommandInfo;
CommandInfo command;
while (commandsOut.TryGetValue(sequence, out command))
{
// 1 bit to tell there is a command
output.WriteRawBits(1, 1);
output.WritePackedIntDelta(command.time, previous.time, NetworkConfig.commandTimeContext);
uint hash = 0;
DeltaWriter.Write(ref output, commandSchema, command.data, previous.data, zeroFieldsChanged, 0, ref hash);
previous = command;
--sequence;
}
output.WriteRawBits(0, 1);
}
unsafe void WriteSnapshot(ref RawOutputStream output)
{
Profiler.BeginSample("NetworkServer.WriteSnapshot()");
AddMessageContentFlag(NetworkMessage.Snapshot);
bool enableNetworkPrediction = network_prediction.IntValue != 0;
//bool enableHashing = debug_hashing.IntValue != 0;
// Check if the baseline from the client is too old. We keep N number of snapshots on the server
// so if the client baseline is older than that we cannot generate the snapshot. Furthermore, we require
// the client to keep the last N updates for any entity, so even though the client might have much older
// baselines for some entities we cannot guarantee it.
// TODO : Can we make this simpler?
var haveBaseline = maxSnapshotAck != 0;
if (_server.m_ServerSequence - maxSnapshotAck >= NetworkConfig.snapshotDeltaCacheSize - 2) // -2 because we want 3 baselines!
{
if (serverDebug.IntValue > 0)
{
GameDebug.Log("ServerSequence ahead of latest ack'ed snapshot by more than cache size. " + (haveBaseline ? "nobaseline" : "baseline"));
}
haveBaseline = false;
}
var baseline = haveBaseline ? maxSnapshotAck : 0;
int snapshot0Baseline = baseline;
int snapshot1Baseline = baseline;
int snapshot2Baseline = baseline;
int snapshot0BaselineClient = snapshotPackageBaseline;
int snapshot1BaselineClient = snapshotPackageBaseline;
int snapshot2BaselineClient = snapshotPackageBaseline;
if (enableNetworkPrediction && haveBaseline)
{
var end = snapshotPackageBaseline - NetworkConfig.clientAckCacheSize;
end = end < 0 ? 0 : end;
var a = snapshotPackageBaseline - 1;
while (a > end)
{
if (snapshotAcks[a % NetworkConfig.clientAckCacheSize])
{
var base1 = snapshotSeqs[a % NetworkConfig.clientAckCacheSize];
if (_server.m_ServerSequence - base1 < NetworkConfig.snapshotDeltaCacheSize - 2)
{
snapshot1Baseline = base1;
snapshot1BaselineClient = a;
snapshot2Baseline = snapshotSeqs[a % NetworkConfig.clientAckCacheSize];
snapshot2BaselineClient = a;
}
break;
}
a--;
}
a--;
while (a > end)
{
if (snapshotAcks[a % NetworkConfig.clientAckCacheSize])
{
var base2 = snapshotSeqs[a % NetworkConfig.clientAckCacheSize];
if (_server.m_ServerSequence - base2 < NetworkConfig.snapshotDeltaCacheSize - 2)
{
snapshot2Baseline = base2;
snapshot2BaselineClient = a;
}
break;
}
a--;
}
}
output.WriteRawBits(haveBaseline ? 1u : 0, 1);
output.WritePackedIntDelta(snapshot0BaselineClient, outSequence - 1, NetworkConfig.baseSequenceContext);
output.WriteRawBits(enableNetworkPrediction ? 1u : 0u, 1);
//output.WriteRawBits(enableHashing ? 1u : 0u, 1);
if (enableNetworkPrediction)
{
output.WritePackedIntDelta(haveBaseline ? snapshot1BaselineClient : 0, snapshot0BaselineClient - 1, NetworkConfig.baseSequence1Context);
output.WritePackedIntDelta(haveBaseline ? snapshot2BaselineClient : 0, snapshot1BaselineClient - 1, NetworkConfig.baseSequence2Context);
}
// NETTODO: For us serverTime == tick but network layer only cares about a growing int
output.WritePackedIntDelta(_server.serverTime, haveBaseline ? maxSnapshotTime : 0, NetworkConfig.serverTimeContext);
// NETTODO: a more generic way to send stats
var temp = _server.m_ServerSimTime * 10;
output.WriteRawBits((byte)temp, 8);
_server.m_TempTypeList.Clear();
_server.m_TempSpawnList.Clear();
_server.m_TempDespawnList.Clear();
_server.m_TempUpdateList.Clear();
_server.m_PredictionIndex = 0;
for (int id = 0, c = _server.m_Entities.Count; id < c; id++)
{
var entity = _server.m_Entities[id];
// Skip freed
if (entity.spawnSequence == 0)
{
continue;
}
bool spawnedSinceBaseline = (entity.spawnSequence > baseline);
bool despawned = (entity.despawnSequence > 0);
// Note to future self: This is a bit tricky... We consider lifetimes of entities
// re the baseline (last ack'ed, so in the past) and the snapshot we are building (now)
// There are 6 cases (S == spawn, D = despawn):
//
// --------------------------------- time ----------------------------------->
//
// BASELINE SNAPSHOT
// | |
// v v
// 1. S-------D IGNORE
// 2. S------------------D SEND DESPAWN
// 3. S-------------------------------------D SEND UPDATE
// 4. S-----D IGNORE
// 5. S-----------------D SEND SPAWN + UPDATE
// 6. S----------D INVALID (FUTURE)
//
if (despawned && entity.despawnSequence <= baseline)
{
continue; // case 1: ignore
}
if (despawned && !spawnedSinceBaseline)
{
_server.m_TempDespawnList.Add(id); // case 2: despawn
continue;
}
if (spawnedSinceBaseline && despawned)
{
continue; // case 4: ignore
}
if (spawnedSinceBaseline)
{
_server.m_TempSpawnList.Add(id); // case 5: send spawn + update
}
// case 5. and 3. fall through to here and gets updated
// Send data from latest tick
var tickToSend = _server.m_ServerSequence;
// If despawned, however, we have stopped generating updates so pick latest valid
if (despawned)
{
tickToSend = Mathf.Max(entity.updateSequence, entity.despawnSequence - 1);
}
{
var entityType = _server.m_EntityTypes[entity.typeId];
var snapshot = entity.snapshots[tickToSend];
// NOTE : As long as the server haven't gotten the spawn acked, it will keep sending
// delta relative to 0 as we cannot know if we have a valid baseline on the client or not
uint num_baselines = 1; // if there is no normal baseline, we use schema baseline so there is always one
uint *baseline0 = entityType.baseline;
int time0 = maxSnapshotTime;
if (haveBaseline && entity.spawnSequence <= maxSnapshotAck)
{
baseline0 = entity.snapshots[snapshot0Baseline].start;
}
if (enableNetworkPrediction)
{
uint *baseline1 = entityType.baseline;
uint *baseline2 = entityType.baseline;
int time1 = maxSnapshotTime;
int time2 = maxSnapshotTime;
if (haveBaseline && entity.spawnSequence <= maxSnapshotAck)
{
GameDebug.Assert(_server.m_Snapshots[snapshot0Baseline % _server.m_Snapshots.Length].serverTime == maxSnapshotTime, "serverTime == maxSnapshotTime");
GameDebug.Assert(entity.snapshots.Exists(snapshot0Baseline), "Exists(snapshot0Baseline)");
// Newly spawned entities might not have earlier baselines initially
if (snapshot1Baseline != snapshot0Baseline && entity.snapshots.Exists(snapshot1Baseline))
{
num_baselines = 2;
baseline1 = entity.snapshots[snapshot1Baseline].start;
time1 = _server.m_Snapshots[snapshot1Baseline % _server.m_Snapshots.Length].serverTime;
if (snapshot2Baseline != snapshot1Baseline && entity.snapshots.Exists(snapshot2Baseline))
{
num_baselines = 3;
baseline2 = entity.snapshots[snapshot2Baseline].start;
//time2 = entity.snapshots[snapshot2Baseline].serverTime;
time2 = _server.m_Snapshots[snapshot2Baseline % _server.m_Snapshots.Length].serverTime;
}
}
}
entity.prediction = _server.m_Prediction + _server.m_PredictionIndex;
NetworkPrediction.PredictSnapshot(entity.prediction, entity.fieldsChangedPrediction, entityType.schema, num_baselines, (uint)time0, baseline0, (uint)time1, baseline1, (uint)time2, baseline2, (uint)_server.serverTime, entity.GetFieldMask(ConnectionId));
_server.m_PredictionIndex += entityType.schema.GetByteSize() / 4;
//_server.statsProcessedOutgoing += entityType.schema.GetByteSize();
if (UnsafeUtility.MemCmp(entity.prediction, snapshot.start, entityType.schema.GetByteSize()) != 0)
{
_server.m_TempUpdateList.Add(id);
}
if (serverDebug.IntValue > 2)
{
GameDebug.Log((haveBaseline ? "Upd [BL]" : "Upd [ ]") +
"num_baselines: " + num_baselines + " serverSequence: " + tickToSend + " " +
snapshot0Baseline + "(" + snapshot0BaselineClient + "," + time0 + ") - " +
snapshot1Baseline + "(" + snapshot1BaselineClient + "," + time1 + ") - " +
snapshot2Baseline + "(" + snapshot2BaselineClient + "," + time2 + "). Sche: " +
_server.m_TempTypeList.Count + " Spwns: " + _server.m_TempSpawnList.Count + " Desp: " + _server.m_TempDespawnList.Count + " Upd: " + _server.m_TempUpdateList.Count);
}
}
else
{
var prediction = baseline0;
var fcp = entity.fieldsChangedPrediction;
for (int i = 0, l = fcp.Length; i < l; ++i)
{
fcp[i] = 0;
}
if (UnsafeUtility.MemCmp(prediction, snapshot.start, entityType.schema.GetByteSize()) != 0)
{
_server.m_TempUpdateList.Add(id);
}
if (serverDebug.IntValue > 2)
{
GameDebug.Log((haveBaseline ? "Upd [BL]" : "Upd [ ]") + snapshot0Baseline + "(" + snapshot0BaselineClient + "," + time0 + "). Sche: " + _server.m_TempTypeList.Count + " Spwns: " + _server.m_TempSpawnList.Count + " Desp: " + _server.m_TempDespawnList.Count + " Upd: " + _server.m_TempUpdateList.Count);
}
}
}
}
if (serverDebug.IntValue > 1 && (_server.m_TempSpawnList.Count > 0 || _server.m_TempDespawnList.Count > 0))
{
GameDebug.Log(ConnectionId + ": spwns: " + string.Join(",", _server.m_TempSpawnList) + " despwans: " + string.Join(",", _server.m_TempDespawnList));
}
foreach (var pair in _server.m_EntityTypes)
{
if (pair.Value.createdSequence > maxSnapshotAck)
{
_server.m_TempTypeList.Add(pair.Value);
}
}
output.WritePackedUInt((uint)_server.m_TempTypeList.Count, NetworkConfig.schemaCountContext);
foreach (var typeInfo in _server.m_TempTypeList)
{
output.WritePackedUInt(typeInfo.typeId, NetworkConfig.schemaTypeIdContext);
NetworkSchema.WriteSchema(typeInfo.schema, ref output);
GameDebug.Assert(typeInfo.baseline != null);
NetworkSchema.CopyFieldsFromBuffer(typeInfo.schema, typeInfo.baseline, ref output);
}
int previousId = 1;
output.WritePackedUInt((uint)_server.m_TempSpawnList.Count, NetworkConfig.spawnCountContext);
foreach (var id in _server.m_TempSpawnList)
{
output.WritePackedIntDelta(id, previousId, NetworkConfig.idContext);
previousId = id;
var entity = _server.m_Entities[id];
output.WritePackedUInt((uint)entity.typeId, NetworkConfig.spawnTypeIdContext);
output.WriteRawBits(entity.GetFieldMask(ConnectionId), 8);
}
output.WritePackedUInt((uint)_server.m_TempDespawnList.Count, NetworkConfig.despawnCountContext);
foreach (var id in _server.m_TempDespawnList)
{
output.WritePackedIntDelta(id, previousId, NetworkConfig.idContext);
previousId = id;
}
int numUpdates = _server.m_TempUpdateList.Count;
output.WritePackedUInt((uint)numUpdates, NetworkConfig.updateCountContext);
foreach (var id in _server.m_TempUpdateList)
{
var entity = _server.m_Entities[id];
var entityType = _server.m_EntityTypes[entity.typeId];
uint *prediction = null;
if (enableNetworkPrediction)
{
prediction = entity.prediction;
}
else
{
prediction = entityType.baseline;
if (haveBaseline && entity.spawnSequence <= maxSnapshotAck)
{
prediction = entity.snapshots[snapshot0Baseline].start;
}
}
output.WritePackedIntDelta(id, previousId, NetworkConfig.idContext);
previousId = id;
// TODO It is a mess that we have to repeat the logic about tickToSend from above here
int tickToSend = _server.m_ServerSequence;
if (entity.despawnSequence > 0)
{
tickToSend = Mathf.Max(entity.despawnSequence - 1, entity.updateSequence);
}
GameDebug.Assert(_server.m_ServerSequence - tickToSend < NetworkConfig.snapshotDeltaCacheSize);
if (!entity.snapshots.Exists(tickToSend))
{
GameDebug.Log("maxSnapAck: " + maxSnapshotAck);
GameDebug.Log("lastWritten: " + snapshotServerLastWritten);
GameDebug.Log("spawn: " + entity.spawnSequence);
GameDebug.Log("despawn: " + entity.despawnSequence);
GameDebug.Log("update: " + entity.updateSequence);
GameDebug.Log("tick: " + _server.m_ServerSequence);
GameDebug.Log("id: " + id);
GameDebug.Log("snapshots: " + entity.snapshots.ToString());
GameDebug.Log("WOULD HAVE crashed looking for " + tickToSend + " changing to " + (entity.despawnSequence - 1));
tickToSend = entity.despawnSequence - 1;
GameDebug.Assert(false, "Unable to find " + tickToSend + " in snapshots. Would update have worked?");
}
var snapshotInfo = entity.snapshots[tickToSend];
// NOTE : As long as the server haven't gotten the spawn acked, it will keep sending
// delta relative to 0 as we cannot know if we have a valid baseline on the client or not
uint entity_hash = 0;
DeltaWriter.Write(ref output, entityType.schema, snapshotInfo.start, prediction, entity.fieldsChangedPrediction, entity.GetFieldMask(ConnectionId), ref entity_hash);
}
if (!haveBaseline && serverDebug.IntValue > 0)
{
Debug.Log("Sending no-baseline snapshot. C: " + ConnectionId + " Seq: " + outSequence + " Max: " + maxSnapshotAck + " Total entities sent: " + _server.m_TempUpdateList.Count + " Type breakdown:");
//foreach (var c in _server.m_EntityTypes) {
// Debug.Log(c.Value.name + " " + c.Key + " #" + (c.Value.stats_count) + " " + (c.Value.stats_bits / 8) + " bytes");
//}
}
snapshotSeqs[outSequence % NetworkConfig.clientAckCacheSize] = _server.m_ServerSequence;
snapshotServerLastWritten = _server.m_ServerSequence;
Profiler.EndSample();
}
