public byte[] Sign(byte[] data, RSACryptoServiceProvider pk)
{
var signature = pk.SignData(data, 0, data.Length, CryptoConfig.MapNameToOID("SHA1"));
return(BufferUtils.Concat(data, signature));
}
private static void OnEntitySyncQueue(object state)
{
if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0)
{
try
{
var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>());
var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>());
if (temp.Count == 0)
{
return;
}
using (var client = RedisConnectionPool.GetClient())
{
var pipeline = client.CreatePipeline();
try
{
bool hasPost = false;
foreach (var key in temp)
{
var keyValues = key.Split('_', '|');
if (keyValues.Length != 3)
{
continue;
}
AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity;
int id = keyValues[1].ToInt();
string keyCode = keyValues[2];
string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode);
string hashId = GetRedisSyncQueueKey(id);
byte[] idBytes = BufferUtils.GetBytes(id);
var keyBytes = RedisConnectionPool.ToByteKey(redisKey);
bool isDelete;
byte[] entityBytes;
if (entity != null)
{
isDelete = entity.IsDelete;
entityBytes = _serializer.Serialize(entity);
}
else if (tempRemove.Contains(key))
{
entityBytes = new byte[0];
isDelete = true;
}
else
{
TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key);
continue;
}
byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0);
byte[] values =
BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length),
idBytes, stateBytes, entityBytes);
pipeline.QueueCommand(c => ((RedisClient)c).HSet(hashId, keyBytes, values));
hasPost = true;
}
if (hasPost)
{
pipeline.Flush();
}
}
finally
{
pipeline.Dispose();
}
}
}
catch (Exception ex)
{
TraceLog.WriteError("EntitySync queue {0}", ex);
}
finally
{
Interlocked.Exchange(ref _entityQueueRunning, 0);
}
}
}
/// <summary>
/// Constructs HeightMapBuilder and takes a height map in the form of a float array as input
/// </summary>
/// <param name="shader">The compute shader that has all the layer kernels</param>
/// <param name="heightMap">The height map to use. Height map length must have a whole square root result</param>
public HeightMapBuilder(ComputeShader shader, float[] heightMap) : this(shader, BufferUtils.CreateHeightBuffer(heightMap))
{
}
public void BufferUtilsShouldCreateANewBufferIfPassedInBufferIsNull()
{
char[] buffer = null;
BufferUtils.InitializeBufferIfRequired(buffer).Should()
.NotBeNull("If null buffer is passed then a new buffer should be created");
}
public void BufferUtilsShouldNotCreateANewBufferIfPassedInBuferIsNotNull()
{
char[] buffer = new char[10];
buffer = BufferUtils.InitializeBufferIfRequired(buffer);
buffer.ShouldBeEquivalentTo(buffer);
}
/// <summary>
/// Returns the string value with special characters escaped.
/// </summary>
/// <param name="writer">The text writer to write the output to.</param>
/// <param name="inputString">Input string value.</param>
/// <param name="buffer">Char buffer to use for streaming data</param>
internal static void WriteEscapedJsonString(TextWriter writer, string inputString, ref char[] buffer)
{
Debug.Assert(writer != null, "writer != null");
Debug.Assert(inputString != null, "The string value must not be null.");
writer.Write(JsonConstants.QuoteCharacter);
int firstIndex;
if (!JsonValueUtils.CheckIfStringHasSpecialChars(inputString, out firstIndex))
{
writer.Write(inputString);
}
else
{
int inputStringLength = inputString.Length;
Debug.Assert(firstIndex < inputStringLength, "First index of the special character should be within the string");
buffer = BufferUtils.InitializeBufferIfRequired(buffer);
int bufferLength = buffer.Length;
int bufferIndex = 0;
int currentIndex = 0;
// Let's copy and flush strings up to the first index of the special char
while (currentIndex < firstIndex)
{
int subStrLength = firstIndex - currentIndex;
Debug.Assert(subStrLength > 0, "SubStrLength should be greater than 0 always");
// If the first index of the special character is larger than the buffer length,
// flush everything to the buffer first and reset the buffer to the next chunk.
// Otherwise copy to the buffer and go on from there.
if (subStrLength >= bufferLength)
{
inputString.CopyTo(currentIndex, buffer, 0, bufferLength);
writer.Write(buffer, 0, bufferLength);
currentIndex += bufferLength;
}
else
{
inputString.CopyTo(currentIndex, buffer, 0, subStrLength);
bufferIndex = subStrLength;
currentIndex += subStrLength;
}
}
for (; currentIndex < inputStringLength; currentIndex++)
{
char c = inputString[currentIndex];
string escapedString = JsonValueUtils.SpecialCharToEscapedStringMap[c];
// Append the unhandled characters (that do not require special treament)
// to the buffer.
if (escapedString == null)
{
buffer[bufferIndex] = c;
bufferIndex++;
}
else
{
// Okay, an unhandled character was deteced.
// First lets check if we can fit it in the existing buffer, if not,
// flush the current buffer and reset. Add the escaped string to the buffer
// and continue.
int escapedStringLength = escapedString.Length;
Debug.Assert(escapedStringLength <= bufferLength, "Buffer should be larger than the escaped string");
if ((bufferIndex + escapedStringLength) > bufferLength)
{
writer.Write(buffer, 0, bufferIndex);
bufferIndex = 0;
}
escapedString.CopyTo(0, buffer, bufferIndex, escapedStringLength);
bufferIndex += escapedStringLength;
}
if (bufferIndex >= bufferLength)
{
Debug.Assert(bufferIndex == bufferLength,
"We should never encounter a situation where the buffer index is greater than the buffer length");
writer.Write(buffer, 0, bufferIndex);
bufferIndex = 0;
}
}
if (bufferIndex > 0)
{
writer.Write(buffer, 0, bufferIndex);
}
}
writer.Write(JsonConstants.QuoteCharacter);
}
/// <summary>
/// Attempts to parse a SIP message from a single buffer that can only contain a single message.
/// </summary>
/// <param name="buffer">The buffer that will be parsed for a SIP message.</param>
/// <param name="sipBodyEncoding">SIP payload encoding</param>
/// <param name="localSIPEndPoint">The end point the message was received on.</param>
/// <param name="remoteSIPEndPoint">The end point the message was received from.</param>
/// <param name="sipEncoding">SIP protocol encoding, according to RFC should be UTF-8 </param>
/// <returns>If successful a SIP message or null if not.</returns>
public static SIPMessageBuffer ParseSIPMessage(
byte[] buffer,
Encoding sipEncoding,
Encoding sipBodyEncoding,
SIPEndPoint localSIPEndPoint,
SIPEndPoint remoteSIPEndPoint)
{
if (buffer == null || buffer.Length < m_minFirstLineLength)
{
// Ignore.
return(null);
}
else if (buffer.Length > SIPConstants.SIP_MAXIMUM_RECEIVE_LENGTH)
{
throw new ApplicationException("SIP message received that exceeded the maximum allowed message length, ignoring.");
}
else if (!BufferUtils.HasString(buffer, 0, buffer.Length, SIP_MESSAGE_IDENTIFIER, m_CRLF))
{
// Message does not contain "SIP" anywhere on the first line, ignore.
return(null);
}
else
{
var sipMessage = new SIPMessageBuffer(sipEncoding, sipBodyEncoding);
sipMessage.RawBuffer = buffer;
sipMessage.LocalSIPEndPoint = localSIPEndPoint;
sipMessage.RemoteSIPEndPoint = remoteSIPEndPoint;
// For connection oriented transports the same connection should be used for responses and subsequent requests.
if (sipMessage.LocalSIPEndPoint != null && remoteSIPEndPoint.ConnectionID != null)
{
sipMessage.LocalSIPEndPoint.ConnectionID = remoteSIPEndPoint.ConnectionID;
}
string message = sipEncoding.GetString(buffer);
int endFistLinePosn = message.IndexOf(m_CRLF);
if (endFistLinePosn != -1)
{
sipMessage.FirstLine = message.Substring(0, endFistLinePosn);
if (sipMessage.FirstLine.Substring(0, 3) == SIP_RESPONSE_PREFIX)
{
sipMessage.SIPMessageType = SIPMessageTypesEnum.Response;
}
else
{
sipMessage.SIPMessageType = SIPMessageTypesEnum.Request;
}
int endHeaderPosn = message.IndexOf(m_CRLF + m_CRLF);
if (endHeaderPosn == -1)
{
// Assume flakey implementation if message does not contain the required CRLFCRLF sequence and treat the message as having no body.
string headerString = message.Substring(endFistLinePosn + 2, message.Length - endFistLinePosn - 2);
sipMessage.SIPHeaders = SIPHeader.SplitHeaders(headerString);
}
else
{
string headerString = message.Substring(endFistLinePosn + 2, endHeaderPosn - endFistLinePosn - 2);
sipMessage.SIPHeaders = SIPHeader.SplitHeaders(headerString);
if (message.Length > endHeaderPosn + 4)
{
sipMessage.Body = new byte[buffer.Length - (endHeaderPosn + 4)];
Buffer.BlockCopy(buffer, endHeaderPosn + 4, sipMessage.Body, 0, buffer.Length - (endHeaderPosn + 4));
}
}
return(sipMessage);
}
else
{
logger.LogWarning("Error ParseSIPMessage, there were no end of line characters in the string being parsed.");
return(null);
}
}
}
bool TryDecodeNextMessage(out OpCode opCode, out byte[] messagePayload)
{
while (true)
{
switch (decodeState)
{
case DecodeState.DecodeHeader:
if (FrameDecoder.TryDecodeHeader(readBuffer, ref dataStart, readBufferOffset, out isFin, out frameOpCode, out isMasked, out framePayloadLength, mask))
{
decodeState = DecodeState.DecodePayload;
switch (frameOpCode)
{
case OpCode.Continuation:
if (framePayloadLength > 0)
{
BufferUtils.Expand(ref messageBuffer, framePayloadLength); //TODO: limit, sanity guard
}
break;
case OpCode.Text:
case OpCode.Binary:
messageOpCode = frameOpCode;
messageBuffer = new byte[framePayloadLength];
messageBufferOffset = 0;
break;
case OpCode.Close:
case OpCode.Ping:
case OpCode.Pong:
if (!isFin)
{
throw new WebSocketException("Control frame cannot be fragmented");
}
break;
}
continue;
}
break;
case DecodeState.DecodePayload:
var decodeStart = dataStart;
if (FrameDecoder.TryDecodePayload(readBuffer, ref dataStart, readBufferOffset, framePayloadLength, isMasked, mask))
{
decodeState = DecodeState.DecodeHeader;
switch (frameOpCode)
{
case OpCode.Continuation:
case OpCode.Text:
case OpCode.Binary:
Array.Copy(readBuffer, decodeStart, messageBuffer, messageBufferOffset, framePayloadLength);
CompactReadBuffer(ref dataStart);
if (isFin)
{
opCode = messageOpCode;
messagePayload = messageBuffer;
messageBuffer = null;
messageBufferOffset = 0;
return(true);
}
else
{
messageBufferOffset += framePayloadLength;
continue;
}
case OpCode.Close:
case OpCode.Ping:
case OpCode.Pong:
opCode = frameOpCode;
messagePayload = new byte[framePayloadLength];
Array.Copy(readBuffer, decodeStart, messagePayload, 0, framePayloadLength);
CompactReadBuffer(ref dataStart);
return(true); // must be Fin
}
}
break;
}
opCode = OpCode.Continuation;
messagePayload = null;
return(false);
}
}
public uint ReadUInt32(Endian endian = Endian.Big)
{
Read(internalBuffer, 0, 4);
return(BufferUtils.ToUInt32(internalBuffer, 0, endian));
}
public void BufferUtilsShouldCreateANewBufferIfPassedInBufferIsNull()
{
char[] buffer = null;
Assert.NotNull(BufferUtils.InitializeBufferIfRequired(buffer));
}
public void BufferUtilsShouldCreateNonZeroLengthBuffer()
{
char[] buffer = null;
buffer = BufferUtils.InitializeBufferIfRequired(buffer);
Assert.True(buffer.Length > 0);
}
public void ReadAndExecuteCommands(long maxTimeMillis, long networkMaxDelayTime)
{
this.infoEnabled = this.log.IsInfoEnabled;
long now = CurrentTimeMillis();
long timeToStop = (maxTimeMillis <= 0L) ? 0x7fffffffffffffffL : (now + maxTimeMillis);
if (this.ExecuteDelayedCommands(now, timeToStop) && (this.IsConnected && this.socket.CanRead))
{
if (this.socket.AvailableLength == 0)
{
this.Disconnect(ProblemStatus.ClosedByServer);
}
else
{
int num3;
try
{
int availableLength = this.socket.AvailableLength;
this.readBuffer = BufferUtils.GetBufferWithValidSize(this.readBuffer, availableLength);
num3 = this.socket.Read(this.readBuffer, 0, availableLength);
if (this.infoEnabled)
{
this.log.InfoFormat("Received {0} byte(s).", num3);
}
}
catch (IOException exception)
{
this.OnSocketProblem(ProblemStatus.ReceiveError, exception);
return;
}
if (this.infoEnabled)
{
this.log.Info("Processing new commands");
}
try
{
CommandPacket packet;
this.protocolAdapter.AddChunk(this.readBuffer, num3);
while (this.TryDecode(out packet))
{
CommandPacket packet2;
bool flag = this.ExecuteCommands(packet, timeToStop, out packet2);
this.protocolAdapter.FinalizeDecodedCommandPacket(packet);
if (!flag)
{
if (this.delayedCommands == null)
{
this.delayedCommands = packet2;
this.delayUntilTime = now + networkMaxDelayTime;
continue;
}
this.delayedCommands.Append(packet2);
this.protocolAdapter.FinalizeDecodedCommandPacket(packet2);
}
}
}
catch (Exception exception2)
{
this.OnSocketProblem(ProblemStatus.DecodePacketError, exception2);
}
}
}
}
public void WriteUInt16(ushort value, Endian endian = Endian.Big)
{
BufferUtils.FromUInt16(value, internalBuffer, 0, endian);
Stream.Write(internalBuffer, 0, 2);
}
public void WriteUInt32(uint value, Endian endian = Endian.Big)
{
BufferUtils.FromUInt32(value, internalBuffer, 0, endian);
Stream.Write(internalBuffer, 0, 4);
}
public void Send(byte cmd, byte[] data, Protocal type = Protocal.Tcp)
{
Send(BufferUtils.AddFirst(cmd, data), type);
}
public ushort ReadUInt16(Endian endian = Endian.Big)
{
Read(internalBuffer, 0, 2);
return(BufferUtils.ToUInt16(internalBuffer, 0, endian));
}
private byte[] AddLength(byte[] data)
{
byte[] lengthBuff = BitConverter.GetBytes((UInt16)(data.Length + 2));
return(BufferUtils.Add(lengthBuff, data));
}
public object Save()
{
return(BufferUtils.GetUIntBitValue(ACC, Location, Latitude, Longitude, Operate, Encryption, Keep6, Keep7, Keep8, Keep9,
OilRoad, ElectricityRoad, DoorLock, Keep13, Keep14, Keep15, Keep16, Keep17, Keep18, Keep19, Keep20, Keep21, Keep22,
Keep23, Keep24, Keep25, Keep26, Keep27, Keep28, Keep29, Keep30, Keep31));
}
private static void OnEntitySyncQueue(object state)
{
if (Interlocked.CompareExchange(ref _entityQueueRunning, 1, 0) == 0)
{
try
{
var tempRemove = Interlocked.Exchange(ref _entityRemoteSet, new HashSet <string>());
var temp = Interlocked.Exchange(ref _entitySet, new HashSet <string>());
if (temp.Count == 0 || _queueWatchTimers == null)
{
return;
}
TraceLog.WriteWarn("OnEntitySyncQueue execute count:{0}, success:{1}/total {2}, fail:{3} start...", temp.Count, ExecuteSuccessCount, SendWaitCount, ExecuteFailCount);
RedisConnectionPool.Process(client =>
{
while (temp.Count > 0)
{
var dumpSet = temp.Take(ExecuteCount).ToArray();
var pipeline = client.CreatePipeline();
try
{
bool hasPost = false;
foreach (var key in dumpSet)
{
var keyValues = key.Split('_', '|');
if (keyValues.Length != 3)
{
TraceLog.WriteWarn("OnEntitySyncQueue:{0}", key);
ExecuteFailCount++;
continue;
}
AbstractEntity entity = CacheFactory.GetPersonalEntity(key) as AbstractEntity;
int id = keyValues[1].ToInt();
string keyCode = keyValues[2];
string redisKey = string.Format("{0}_{1}", keyValues[0], keyCode);
string hashId = GetRedisSyncQueueKey(id);
byte[] idBytes = BufferUtils.GetBytes(id);
var keyBytes = RedisConnectionPool.ToByteKey(redisKey);
bool isDelete;
byte[] entityBytes;
if (entity != null)
{
isDelete = entity.IsDelete;
entityBytes = _serializer.Serialize(entity);
//modify resean: set unchange status.
entity.Reset();
}
else if (tempRemove.Contains(key))
{
entityBytes = new byte[0];
isDelete = true;
}
else
{
ExecuteFailCount++;
TraceLog.WriteError("EntitySync queue key {0} faild object is null.", key);
continue;
}
byte[] stateBytes = BufferUtils.GetBytes(isDelete ? 1 : 0);
byte[] values =
BufferUtils.MergeBytes(BufferUtils.GetBytes(idBytes.Length + stateBytes.Length),
idBytes, stateBytes, entityBytes);
pipeline.QueueCommand(c =>
{
((RedisClient)c).HSet(hashId, keyBytes, values);
});
hasPost = true;
ExecuteSuccessCount++;
}
if (hasPost)
{
pipeline.Flush();
}
}
finally
{
pipeline.Dispose();
}
try
{
foreach (var key in dumpSet)
{
temp.Remove(key);
}
}
catch (Exception)
{
}
}
});
}
catch (Exception ex)
{
TraceLog.WriteError("EntitySync queue {0}", ex);
}
finally
{
Interlocked.Exchange(ref _entityQueueRunning, 0);
}
}
}
/// <summary>
/// Instantiate new GpuTextureBuilder with given resolution. Creates a new RenderTexture
/// </summary>
/// <param name="shader">Compute shader to use. Pass this along to chained GpuTextureBuilders</param>
/// <param name="resolution">Resolution of the new RenderTexture</param>
public TextureBuilder(ComputeShader shader, int resolution) : this(shader, BufferUtils.CreateRenderTexture(resolution))
{
}
private void WriteEscapedString(string value)
{
EnsureWriteBuffer();
var bufferPool = _arrayPool;
if (value != null)
{
int lastWritePosition = 0;
for (int i = 0; i < value.Length; i++)
{
var c = value[i];
if (c < _charEscapeFlags.Length && !_charEscapeFlags[c])
{
continue;
}
string escapedValue = null;
if (c < _charEscapeFlags.Length)
{
escapedValue = $"{Grammar.ReleaseCharacter.Value}{c}";
}
else
{
escapedValue = null;
}
if (escapedValue == null)
{
continue;
}
if (i > lastWritePosition)
{
int length = i - lastWritePosition;
int start = 0;
if (_writeBuffer == null || _writeBuffer.Length < length)
{
char[] newBuffer = BufferUtils.RentBuffer(bufferPool, length);
BufferUtils.ReturnBuffer(bufferPool, _writeBuffer);
_writeBuffer = newBuffer;
}
value.CopyTo(lastWritePosition, _writeBuffer, start, length - start);
// write unchanged chars before writing escaped text
_writer.Write(_writeBuffer, start, length - start);
}
lastWritePosition = i + 1;
_writer.Write(escapedValue);
}
if (lastWritePosition == 0)
{
// no escaped text, write entire string
_writer.Write(value);
}
else
{
int length = value.Length - lastWritePosition;
if (_writeBuffer == null || _writeBuffer.Length < length)
{
_writeBuffer = BufferUtils.EnsureBufferSize(bufferPool, length, _writeBuffer);
}
value.CopyTo(lastWritePosition, _writeBuffer, 0, length);
// write remaining text
_writer.Write(_writeBuffer, 0, length);
}
}
else
{
WriteNull();
}
}
private byte[] InsertCommand(byte command, byte[] data)
{
return(BufferUtils.AddFirst(command, data));
}
public void BufferUtilsShouldCreateNonZeroLengthBuffer()
{
char[] buffer = null;
buffer = BufferUtils.InitializeBufferIfRequired(buffer);
buffer.Length.Should().BeGreaterThan(0, "Created Buffer should be greater than zero");
}
/// <summary>
/// Constructs HeightMapBuilder and creates a new height map buffer based on the given resolution
/// </summary>
/// <param name="shader">The compute shader that has all the layer kernels</param>
/// <param name="resolution">Resolution for the new height map</param>
public HeightMapBuilder(ComputeShader shader, int resolution) : this(shader, BufferUtils.CreateHeightBuffer(resolution))
{
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function,
"post",
Route = null)] HttpRequestMessage req,
[EventHub(
"", // Not used if the name is in the connection string
Connection = Config.EventHubConnectionStringConfigField)] ICollector <EventData> messages,
ILogger log,
ExecutionContext context)
{
using (var payload = BufferUtils.GetStream())
{
try
{
string clientId = null;
string batchId = context.InvocationId.ToString("N");
string ingestTimestamp = DateTime.UtcNow.ToString("O");
// Extract content from body
string content = await GetContentAsStringAsync(req.Content);
using (StreamWriter writer = new StreamWriter(payload, new UTF8Encoding(false, true), 4 * 1024, true))
{
// Create a JSONL formatted payload (one json structure per line)
foreach (var se in ParseContent(content))
{
clientId = clientId ?? se["client_id"].Value <string>();
// Stamp event with ingestion metadata - Useful for debugging issues later
se.Add("ingest_ts", JToken.FromObject(ingestTimestamp));
se.Add("ingest_activity_id", JToken.FromObject(batchId));
se.Add("ingest_version", INGEST_VERSION);
// Check to make sure we can generate a unique id for this event
// Cosmos DB will reject creation of a new document with an existing id
Debug.Assert(se["client_id"] != null && se["client_ts"] != null && se["seq"] != null);
// The client can also generate a unique id for it, if it wishes
se["id"] = se["id"] ?? ComputeHash(se["client_id"], se["client_ts"], se["seq"]);
// Any newlines inside of values are automatically escaped.
string result = se.ToString(Formatting.None);
writer.WriteLine(result);
log.LogTrace(result);
}
}
// Choosing optimal because the expected data size isn't huge anyway
var compressedPayload = BufferUtils.GzipCompressToArray(payload, CompressionLevel.Optimal);
if (compressedPayload.Length > MAX_EH_PAYLOAD_SIZE)
{
return(req.CreateErrorResponse(HttpStatusCode.RequestEntityTooLarge,
"The content after compression was larger than the max allowed Event Hub payload size."));
}
// Send payload on to event hub if not disabled
if (Config.EnableEventHub)
{
EventData batch = new EventData(compressedPayload);
batch.SetGzipEncoding();
batch.SetVersion(PAYLOAD_VERSION);
messages.Add(batch);
}
}
catch (Exception ex)
{
log.LogError(ex, "Ingestion failed to send telemetry to event hub.\n" + ex.StackTrace);
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, "Ingestion failed to send telemetry to event hub."));
}
return(req.CreateResponse(HttpStatusCode.Accepted));
}
}
public void TestEqualsNull()
{
Assert.IsTrue(BufferUtils.Equals(null, null));
Assert.IsFalse(BufferUtils.Equals(null, new byte[0]));
Assert.IsFalse(BufferUtils.Equals(new byte[0], null));
}
