private void __SetFormulaIgnoreNull(SidSrc sidsrc, CellValueLiteral formula)
{
if (this._formulaRecords == null)
{
this._formulaRecords = new WriteCache <SidSrc>();
}
if (formula.HasValue)
{
this._formulaRecords.Add(sidsrc, formula.Value);
}
}
public static void TestWriteCache_WriteNode_DoesntThrow()
{
var wc = new WriteCache(
WRITE_CACHE_FILE_INFO,
WRITE_CACHE_MAX_RECORD_COUNT,
null,
null
);
Assert.DoesNotThrow(() => wc.WriteNode(new StratusAsset {
Id = Guid.NewGuid(),
}));
}
public void TestWriteCache()
{
WriteCache wc = new WriteCache(true);
Assert.AreEqual("~:foo", wc.CacheWrite("~:foo", false));
Assert.AreEqual("^" + (char)WriteCache.BaseCharIdx, wc.CacheWrite("~:foo", false));
Assert.AreEqual("~$bar", wc.CacheWrite("~$bar", false));
Assert.AreEqual("^" + (char)(WriteCache.BaseCharIdx + 1), wc.CacheWrite("~$bar", false));
Assert.AreEqual("~#baz", wc.CacheWrite("~#baz", false));
Assert.AreEqual("^" + (char)(WriteCache.BaseCharIdx + 2), wc.CacheWrite("~#baz", false));
Assert.AreEqual("foobar", wc.CacheWrite("foobar", false));
Assert.AreEqual("foobar", wc.CacheWrite("foobar", false));
Assert.AreEqual("foobar", wc.CacheWrite("foobar", true));
Assert.AreEqual("^" + (char)(WriteCache.BaseCharIdx + 3), wc.CacheWrite("foobar", true));
Assert.AreEqual("abc", wc.CacheWrite("abc", false));
Assert.AreEqual("abc", wc.CacheWrite("abc", false));
Assert.AreEqual("abc", wc.CacheWrite("abc", true));
Assert.AreEqual("abc", wc.CacheWrite("abc", true));
}
/// <summary>
/// Caches the read.
/// </summary>
/// <param name="s">The s.</param>
/// <param name="asDictionaryKey">if set to <c>true</c> [as dictionary key].</param>
/// <param name="p">The p.</param>
/// <returns></returns>
public object CacheRead(string s, bool asDictionaryKey, AbstractParser p)
{
if (s.Length != 0)
{
if (CacheCode(s))
{
return(cache[CodeToIndex(s)]);
}
else if (WriteCache.IsCacheable(s, asDictionaryKey))
{
if (index == WriteCache.MaxCacheEntries)
{
Init();
}
return(cache[index++] = (p != null ? p.ParseString(s) : s));
}
}
return(p != null?p.ParseString(s) : s);
}
public static void TestWriteCache_WriteNode_TwiceDoesntReturnSameNode()
{
var wc = new WriteCache(
WRITE_CACHE_FILE_INFO,
WRITE_CACHE_MAX_RECORD_COUNT,
null,
null
);
var id = Guid.NewGuid();
var node1 = wc.WriteNode(new StratusAsset {
Id = id,
});
var node2 = wc.WriteNode(new StratusAsset {
Id = id,
});
Assert.AreNotSame(node1, node2);
}
public void TestSimple()
{
ISource <byte[], byte[]> src = new MemoryDictionarySource();
WriteCache <byte[]> writeCache = new WriteCache <byte[]>(src, WriteCache <byte[]> .CacheType.SIMPLE);
for (int i = 0; i < 10_000; ++i)
{
writeCache.Put(IntToKey(i), IntToValue(i));
}
// Everything is cached
Assert.Equal(ToHexString(IntToValue(0)), ToHexString(writeCache.GetCached(IntToKey(0)).Value()));
Assert.Equal(ToHexString(IntToValue(9_999)), ToHexString(writeCache.GetCached(IntToKey(9_999)).Value()));
// Everything is flushed
writeCache.Flush();
Assert.Null(writeCache.GetCached(IntToKey(0)));
Assert.Null(writeCache.GetCached(IntToKey(9_999)));
Assert.Equal(ToHexString(IntToValue(9_999)), ToHexString(writeCache.Get(IntToKey(9_999))));
Assert.Equal(ToHexString(IntToValue(0)), ToHexString(writeCache.Get(IntToKey(0))));
// Get not caches, only write cache
Assert.Null(writeCache.GetCached(IntToKey(0)));
// Deleting key that is currently in cache
writeCache.Put(IntToKey(0), IntToValue(12345));
Assert.Equal(ToHexString(IntToValue(12345)), ToHexString(writeCache.GetCached(IntToKey(0)).Value()));
writeCache.Delete(IntToKey(0));
Assert.True(null == writeCache.GetCached(IntToKey(0)) || null == writeCache.GetCached(IntToKey(0)).Value());
Assert.Equal(ToHexString(IntToValue(0)), ToHexString(src.Get(IntToKey(0))));
writeCache.Flush();
Assert.Null(src.Get(IntToKey(0)));
// Deleting key that is not currently in cache
Assert.True(null == writeCache.GetCached(IntToKey(1)) || null == writeCache.GetCached(IntToKey(1)).Value());
Assert.Equal(ToHexString(IntToValue(1)), ToHexString(src.Get(IntToKey(1))));
writeCache.Delete(IntToKey(1));
Assert.True(null == writeCache.GetCached(IntToKey(1)) || null == writeCache.GetCached(IntToKey(1)).Value());
Assert.Equal(ToHexString(IntToValue(1)), ToHexString(src.Get(IntToKey(1))));
writeCache.Flush();
Assert.Null(src.Get(IntToKey(1)));
}
public void Write(Stream stream)
{
var collisionList = new List <Collision>();
var collisionMeshList = new List <RawCollisionMesh>();
var collisionMeshGroupList = new List <RawCollisionMeshGroup>();
foreach (var node in CollisionMeshGroupList)
{
var start = node.Meshes.Count > 0 ? collisionMeshList.Count : 0;
collisionMeshGroupList.Add(new RawCollisionMeshGroup
{
Child1 = node.Child1,
Child2 = node.Child2,
Child3 = node.Child3,
Child4 = node.Child4,
Child5 = node.Child5,
Child6 = node.Child6,
Child7 = node.Child7,
Child8 = node.Child8,
BoundingBox = node.BoundingBox,
CollisionMeshStart = (ushort)start,
CollisionMeshEnd = (ushort)(start + node.Meshes.Count)
});
foreach (var mesh in node.Meshes)
{
collisionMeshList.Add(new RawCollisionMesh
{
BoundingBox = mesh.BoundingBox,
CollisionStart = (ushort)collisionList.Count,
CollisionEnd = (ushort)(collisionList.Count + mesh.Collisions.Count),
v10 = mesh.v10,
v12 = mesh.v12
});
collisionList.AddRange(mesh.Collisions);
}
}
var bbCache = new WriteCache <BoundingBoxInt16, string>(x => x.ToString());
bbCache.AddConstant(BoundingBoxInt16.Invalid, -1);
foreach (var item in collisionList)
{
bbCache.Add(item.BoundingBox);
}
var surfaceCache = new WriteCache <SurfaceFlags, int>(x => x.Flags);
foreach (var item in collisionList)
{
surfaceCache.Add(item.SurfaceFlags);
}
var planeCache = new WriteCache <Plane, string>(x => x.ToString());
foreach (var item in collisionList)
{
planeCache.Add(item.Plane);
}
var entries = new List <Entry>(8);
AddEntry(entries, HeaderSize, 1);
AddEntry(entries, CollisionMeshGroupList.Count * Col1Size, 0x10);
AddEntry(entries, collisionMeshList.Count * Col2Size, 0x10);
AddEntry(entries, collisionList.Count * Col3Size, 4);
AddEntry(entries, VertexList.Count * Col4Size, 0x10);
AddEntry(entries, planeCache.Count * Col5Size, 0x10);
AddEntry(entries, bbCache.Count * BoundingBoxSize, 0x10);
AddEntry(entries, surfaceCache.Count * Col7Size, 4);
stream.Position = 0;
BinaryMapping.WriteObject(stream, new Header
{
MagicCode = MagicCode,
Version = 1,
Unknown08 = Unknown08,
Unknown0c = Unknown0c,
Entries = entries.ToArray()
});
WriteCoctEntry(stream, collisionMeshGroupList);
WriteCoctEntry(stream, collisionMeshList);
WriteCoctEntry(stream, collisionList
.Select(x => new RawCollision
{
v00 = x.v00,
Vertex1 = x.Vertex1,
Vertex2 = x.Vertex2,
Vertex3 = x.Vertex3,
Vertex4 = x.Vertex4,
PlaneIndex = planeCache[x.Plane],
BoundingBoxIndex = bbCache[x.BoundingBox],
SurfaceFlagsIndex = surfaceCache[x.SurfaceFlags],
}));
stream.AlignPosition(0x10);
WriteValueEntry(stream, VertexList, WriteVector4);
WriteValueEntry(stream, planeCache, WritePlane);
WriteValueEntry(stream, bbCache, WriteBoundingBoxInt16);
WriteCoctEntry(stream, surfaceCache);
}
public DataService(
ILoggerFactory loggerFactory,
IShardRepository shardRepository,
IDataRouter dataRouter,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient,
IOptions <ClusterOptions> clusterOptions,
IOperationCacheRepository transactionCacheRepository,
IOptions <NodeOptions> nodeOptions)
{
_nodeStateService = nodeStateService;
_nodeOptions = nodeOptions.Value;
_clusterOptions = clusterOptions.Value;
_stateMachine = stateMachine;
_writeCache = new WriteCache(transactionCacheRepository, loggerFactory.CreateLogger <WriteCache>(), _nodeOptions.PersistWriteQueue);
_logger = loggerFactory.CreateLogger <DataService <State> >();
_shardRepository = shardRepository;
_clusterClient = clusterClient;
Reader = new Reader <State>(
loggerFactory.CreateLogger <Reader <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);;
Allocator = new Allocator <State>(
loggerFactory.CreateLogger <Allocator <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);
Writer = new Writer <State>(loggerFactory.CreateLogger <Writer <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient
);
Syncer = new Syncer <State>(shardRepository, loggerFactory.CreateLogger <Syncer <State> >(), stateMachine, clusterClient, nodeStateService, Writer);
_writeTask = new Task(async() =>
{
//Before you write you should first dequeue all transactions
if (_writeCache.TransitQueue.Count() > 0)
{
_logger.LogInformation("Found transactions in transit, attempting to reapply them...");
foreach (var operationKV in _writeCache.TransitQueue.ToDictionary(entry => entry.Key, entry => entry.Value))
{
var operation = operationKV.Value;
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to apply operation " + operation.Id + " with exception " + e.Message + Environment.NewLine + e.StackTrace);
try
{
await _writeCache.CompleteOperation(operation.Id);
}
catch (Exception completionError)
{
_logger.LogError("Error removing operation from transit queue with error " + completionError.Message + Environment.NewLine
+ completionError.StackTrace + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
}
}
}
while (true)
{
try
{
var operation = await _writeCache.DequeueOperation();
if (operation != null)
{
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to write operation with exception " + e.Message + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
await _writeCache.CompleteOperation(operation.Id);
}
else
{
//Release write thread for a small amount of time
await Task.Delay(100);
}
}
catch (Exception e)
{
_logger.LogError("Encountered critical write error " + e.Message + Environment.NewLine + e.StackTrace);
}
}
});
_writeTask.Start();
TaskUtility.RestartTask(ref _indexCreationTask, async() => await CreateIndexLoop());
_allocationTask = new Task(async() => await AllocateShards());
_allocationTask.Start();
_replicaValidationChecksTask = Task.Run(async() => await CheckAllReplicas());
_objectLockWatcher = new Task(async() => await CheckLocks());
_objectLockWatcher.Start();
if (_nodeOptions.EnablePerformanceLogging)
{
var performancePrinting = new Task(() =>
{
while (true)
{
Console.Clear();
Console.WriteLine("Performance Report...");
foreach (var value in totals)
{
Console.WriteLine(value.Key + ":" + (value.Value / totalRequests));
}
Console.WriteLine("Queue:" + _writeCache.OperationsInQueue);
Task.Delay(1000);
}
});
performancePrinting.Start();
}
}
public void Write(Stream stream)
{
var collisionList = new List <Collision>();
var collisionMeshList = new List <RawCollisionMesh>();
var collisionMeshGroupList = new List <RawNode>();
foreach (var node in Nodes)
{
var start = node.Meshes.Count > 0 ? collisionMeshList.Count : 0;
collisionMeshGroupList.Add(new RawNode
{
Child1 = node.Child1,
Child2 = node.Child2,
Child3 = node.Child3,
Child4 = node.Child4,
Child5 = node.Child5,
Child6 = node.Child6,
Child7 = node.Child7,
Child8 = node.Child8,
BoundingBox = node.BoundingBox,
CollisionMeshStart = (ushort)start,
CollisionMeshEnd = (ushort)(start + node.Meshes.Count)
});
foreach (var mesh in node.Meshes)
{
collisionMeshList.Add(new RawCollisionMesh
{
BoundingBox = mesh.BoundingBox,
CollisionStart = (ushort)collisionList.Count,
CollisionEnd = (ushort)(collisionList.Count + mesh.Collisions.Count),
Visibility = mesh.Visibility,
Group = mesh.Group
});
collisionList.AddRange(mesh.Collisions);
}
}
var bbCache = new WriteCache <BoundingBoxInt16, string>(x => x.ToString());
bbCache.AddConstant(BoundingBoxInt16.Invalid, -1);
foreach (var item in collisionList)
{
bbCache.Add(item.BoundingBox);
}
var surfaceCache = new WriteCache <Attributes, int>(x => x.Flags);
foreach (var item in collisionList)
{
surfaceCache.Add(item.Attributes);
}
var planeCache = new WriteCache <Plane, string>(x => x.ToString());
foreach (var item in collisionList)
{
planeCache.Add(item.Plane);
}
var entries = new List <Entry>(8);
AddEntry(entries, HeaderSize, 1);
AddEntry(entries, Nodes.Count * NodeSize, 0x10);
AddEntry(entries, collisionMeshList.Count * MeshSize, 0x10);
AddEntry(entries, collisionList.Count * PolygonSize, 4);
AddEntry(entries, VertexList.Count * Col4Size, 0x10);
AddEntry(entries, planeCache.Count * Col5Size, 0x10);
AddEntry(entries, bbCache.Count * BoundingBoxSize, 0x10);
AddEntry(entries, surfaceCache.Count * AttributesSize, 4);
stream.Position = 0;
BinaryMapping.WriteObject(stream, new Header
{
MagicCode = MagicCode,
Version = 1,
Depth = Depth,
Type = Type,
Entries = entries.ToArray()
});
WriteCoctEntry(stream, collisionMeshGroupList);
WriteCoctEntry(stream, collisionMeshList);
WriteCoctEntry(stream, collisionList
.Select(x => new RawCollision
{
Ground = x.Ground,
FloorLevel = x.FloorLevel,
Vertex1 = x.Vertex1,
Vertex2 = x.Vertex2,
Vertex3 = x.Vertex3,
Vertex4 = x.Vertex4,
PlaneIndex = planeCache[x.Plane],
BoundingBoxIndex = bbCache[x.BoundingBox],
AttributeIndex = surfaceCache[x.Attributes],
}));
stream.AlignPosition(0x10);
WriteValueEntry(stream, VertexList, WriteVector4);
WriteValueEntry(stream, planeCache, WritePlane);
WriteValueEntry(stream, bbCache, WriteBoundingBoxInt16);
WriteCoctEntry(stream, surfaceCache);
}
