/// <inheritdoc />
public override bool Fallback(char charUnknown, int index)
{
if (Buffer.Any())
{
throw new ArgumentException("This method is called again before the GetNextChar method has read all the replacement string characters.");
}
return(Fallback((Int32)charUnknown));
}
public virtual async Task <string> WebSocketSendAsync(string message)
{
try
{
await _ioSemaphoreSlim.WaitAsync();
if (string.IsNullOrEmpty(message))
{
return(null);
}
byte[] requestBytes = Encoding.UTF8.GetBytes(message);
if (!IsWebSocketConnected())
{
await ClientWebSocket.ConnectAsync(Authentication.WebSocketUri, CancellationToken.None);
}
ArraySegment <byte> subscribeRequest = new ArraySegment <byte>(requestBytes);
await ClientWebSocket.SendAsync(subscribeRequest, WebSocketMessageType.Text, true,
CancellationToken.None);
if (!IsWebSocketConnected())
{
return(null);
}
ArraySegment <byte> receiveBuffer = new ArraySegment <byte>(new byte[512 * 512 * 5]);
WebSocketReceiveResult webSocketReceiveResult = await ClientWebSocket.ReceiveAsync(
receiveBuffer,
CancellationToken.None);
if (webSocketReceiveResult.MessageType == WebSocketMessageType.Close)
{
ClientWebSocket.Abort();
return(null);
}
if (webSocketReceiveResult.Count == 0 || !receiveBuffer.Any() || receiveBuffer.Array == null)
{
return(null);
}
return(Encoding.UTF8.GetString(receiveBuffer.Array, 0, webSocketReceiveResult.Count));
}
catch (Exception ex)
{
ProcessLogBroadcast?.Invoke(MessageType.Error,
$"Method: WebSocketSendAsync\r\nException Stack Trace: {ex.StackTrace}");
await ClientWebSocket.ConnectAsync(Authentication.WebSocketUri, CancellationToken.None);
}
finally
{
_ioSemaphoreSlim.Release();
}
return(null);
}
static bool IsChoku(ArraySegment <char> s)
{
if (!s.Any())
{
return(true);
}
else if (s.Count >= 2 && s.Array[s.Count - 2] == 'c' && s.Array[s.Count - 1] == 'h')
{
return(IsChoku(new ArraySegment <char>(s.Array, 0, s.Count - 2)));
}
else if (s.Array[s.Count - 1] == 'o' || s.Array[s.Count - 1] == 'k' || s.Array[s.Count - 1] == 'u')
{
return(IsChoku(new ArraySegment <char>(s.Array, 0, s.Count - 1)));
}
else
{
return(false);
}
}
void GetNodes(JsonElement current, ArraySegment <string> path, List <int> nodes)
{
if (!path.Any())
{
nodes.Add(current.GetArrayLength());
return;
}
var split = path.First();
if (split[0] == '[')
{
var splitValue = split.Substring(1, split.Length - 2);
if (splitValue == "*")
{
foreach (var next in current.EnumerateArray())
{
GetNodes(next, path.Slice(1), nodes);
}
}
else
{
var splitIndex = int.Parse(splitValue);
var next = current.EnumerateArray().Skip(splitIndex).First();
GetNodes(next, path.Slice(1), nodes);
}
}
else
{
foreach (var kv in current.EnumerateObject())
{
if (kv.Name == split)
{
var next = current.GetProperty(split);
GetNodes(next, path.Slice(1), nodes);
return;
}
}
// no prop
nodes.Add(-1);
}
}
IEnumerable<Message> BatchAndBuildMessages(ArraySegment<Metric> metrics)
{
if (!metrics.Any())
{
return Enumerable.Empty<Message>();
}
byte[] data = MetricsSerializer.MetricsToBytes(metrics).ToArray();
if (data.Length > MaxMessageSize)
{
var part1 = this.BatchAndBuildMessages(metrics.Slice(0, metrics.Count / 2));
var part2 = this.BatchAndBuildMessages(metrics.Slice(metrics.Count / 2));
return part1.Concat(part2);
}
else
{
return new Message[] { this.BuildMessage(data) };
}
}
public virtual async Task <string> WebSocketReceiveAsync()
{
try
{
await _ioSemaphoreSlim.WaitAsync();
if (ClientWebSocket == null || !IsWebSocketConnected())
{
return(null);
}
ArraySegment <byte> receiveBuffer = new ArraySegment <byte>(new byte[512 * 512 * 5]);
WebSocketReceiveResult webSocketReceiveResult = await ClientWebSocket.ReceiveAsync(
receiveBuffer,
CancellationToken.None);
if (webSocketReceiveResult.MessageType == WebSocketMessageType.Close)
{
ClientWebSocket.Abort();
ClientWebSocket.Dispose();
return(null);
}
if (webSocketReceiveResult.Count == 0 || !receiveBuffer.Any() || receiveBuffer.Array == null)
{
return(null);
}
return(Encoding.UTF8.GetString(receiveBuffer.Array, 0, webSocketReceiveResult.Count));
}
catch (Exception ex)
{
ProcessLogBroadcast?.Invoke(MessageType.Error,
$"Method: WebSocketReceiveAsync\r\nException Stack Trace: {ex.StackTrace}");
}
finally
{
_ioSemaphoreSlim.Release();
}
return(null);
}
public virtual async Task <string> WebSocketReceiveAsync()
{
try
{
await _ioSemaphoreSlim.WaitAsync();
if (ClientWebSocket == null)
{
return(null);
}
if (!IsWebSocketConnected())
{
return(null);
}
ArraySegment <byte> receiveBuffer = new ArraySegment <byte>(new byte[512 * 512 * 5]);
WebSocketReceiveResult webSocketReceiveResult = await ClientWebSocket.ReceiveAsync(
receiveBuffer,
CancellationToken.None);
if (webSocketReceiveResult.MessageType == WebSocketMessageType.Close)
{
ClientWebSocket.Abort();
ClientWebSocket.Dispose();
return(null);
}
if (webSocketReceiveResult.Count == 0 || !receiveBuffer.Any() || receiveBuffer.Array == null)
{
return(null);
}
return(Encoding.UTF8.GetString(receiveBuffer.Array, 0, webSocketReceiveResult.Count));
}
finally
{
_ioSemaphoreSlim.Release();
}
}
/// <summary>
/// This is the main entry point for your service replica.
/// This method executes when this replica of your service becomes primary and has write status.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
// load the connection string from event hub
var ehConfiguration = this.Context.CodePackageActivationContext
.GetConfigurationPackageObject("Config")
.Settings
.Sections["EventHubConfiguration"];
var eventHubConnectionString = ehConfiguration.Get("ConnectionString");
var eventHubConsumerGroup = ehConfiguration.Get("ConsumerGroup");
var eventHubMaxBatchSize = ehConfiguration.TryGetAsInt("MaxBatchSize", 100);
// we'll be storing our offset here, in case our service is killed
IReliableDictionary <string, string> offsetDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, string> >(OffsetDictionaryName);
IReliableDictionary <string, long> epochDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >(EpochDictionaryName);
// Each partition of this service corresponds to a partition in EventHub.
// Partitions are automatically assigned based on the partitioning scheme
// which is defined in the Orchestrator service, where this service is supposed
// to be created.
Int64RangePartitionInformation partitionInfo = (Int64RangePartitionInformation)this.Partition.PartitionInfo;
long servicePartitionKey = partitionInfo.LowKey;
// start connecting to the event hub
var client = EventHubClient.CreateFromConnectionString(eventHubConnectionString);
// verify the partition exists
var eventHubInfo = await client.GetRuntimeInformationAsync();
if (eventHubInfo.PartitionCount < servicePartitionKey || servicePartitionKey < 0)
{
ServiceEventSource.Current.ServiceMessage(this.Context,
"The instance is running on partition {0} but there are {1} partitions on the EventHub.",
servicePartitionKey, eventHubInfo.PartitionCount);
throw new ArgumentOutOfRangeException($"Partition {servicePartitionKey} does not exist on event hub with {eventHubInfo.PartitionCount} partitions.");
}
if (string.IsNullOrEmpty(eventHubConsumerGroup))
{
// TODO: replace with a better way
eventHubConsumerGroup = "$Default";
}
ServiceEventSource.Current.ServiceMessage(this.Context,
"Starting reading from Event Hub {0}, running on partition {1}",
client.EventHubName,
servicePartitionKey);
Tuple <EventPosition, long> offsetAndEpoch = await GetOffsetAndEpochAsync(offsetDictionary, epochDictionary);
PartitionReceiver partitionReceiever = client.CreateEpochReceiver(
eventHubConsumerGroup,
eventHubInfo.PartitionIds[servicePartitionKey],
offsetAndEpoch.Item1,
offsetAndEpoch.Item2);
// TODO: make this configurable
var waitTime = TimeSpan.FromSeconds(5);
long?count = 0;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <EventData> eventData = await partitionReceiever.ReceiveAsync(eventHubMaxBatchSize, waitTime);
if ((count = eventData?.Count() ?? 0) == 0)
{
continue;
}
ServiceEventSource.Current.ServiceMessage(this.Context,
"Data read from event hub partition {2}. I've read {0} messages, last offset was {1}. Total count: {3}.",
eventData.Count(),
eventData.Last().SystemProperties.Offset,
servicePartitionKey,
count
);
//we're enumerating through anyway so may as well ToArray it here.
var messages = eventData.Select(x => new MessageWrapper(x, _router)).ToArray();
var groups = messages.GroupBy(x => x.Id);
var keys = groups
.Select(x => new
{
// we're creating this object because we need both, the id and address
Id = x.Key,
Task = orchestratorClient.OrchestrateWorker(new WorkerDescription()
{
Identifier = x.Key
})
})
.ToArray();
// the operation to call the workers is async, and a bunch of them can run in parallel, so let's wait
// until they all finish to get the keys out
Task.WaitAll(keys.Select(x => x.Task).ToArray(), cancellationToken);
var workers = new Dictionary <string, IProcessor>();
// now we need to actually create the service proxies
foreach (var key in keys)
{
var address = key.Task.Result;
var processor = _proxyFactory.CreateServiceProxy <IProcessor>(new Uri(address), new ServicePartitionKey(1));
// we need to attach to the ID, not the address!
workers.Add(key.Id, processor);
}
var page = 0;
var pageSize = 100;
while (true)
{
ArraySegment <MessageWrapper> buffer = new ArraySegment <MessageWrapper>(messages, page * pageSize, pageSize);
if (!buffer.Any())
{
break;
}
var lastMessage = buffer.Last();
foreach (var bufferedGroup in buffer.GroupBy(x => x.Id))
{
var worker = workers[bufferedGroup.Key];
if (!await worker.Process(bufferedGroup.Select(x => x.DataPoint).ToArray()))
{
// what do we do?
ServiceEventSource.Current.ServiceMessage(this.Context,
"Processing failed for message page {0}. Throwing exception.", page);
throw new InvalidOperationException("Processing failed. False returned from worker process.");
}
}
page++;
await Checkpoint(lastMessage, offsetDictionary);
}
}
}
private async void Listen(WebClient client)
{
WebSocket webSocket = client.Socket;
try
{
byte[] byteBuffer = new byte[BufferSize];
ArraySegment <byte> receiveBuffer = new ArraySegment <byte>(byteBuffer);
if (webSocket.State == WebSocketState.Open)
{
WebSocketReceiveResult receiveResult;
try
{
//CancellationTokenSource ct = new CancellationTokenSource(10000);
//CancellationToken token = ct.Token;
receiveResult = await webSocket.ReceiveAsync(receiveBuffer, CancellationToken.None);
}
catch (Exception)
{
this.Listen(client);
return;
}
if (!receiveBuffer.Any())
{
this.Listen(client);
}
client.AppendBytes(receiveBuffer.Array.Where(b => !b.Equals((byte)0)).ToArray());
if (receiveResult.EndOfMessage == false)
{
this.Listen(client);
return;
}
InboundMessage message = new InboundMessage(client.CurrentBytesSentWithoutNewLine.ToArray());
client.ClearBytes();
Console.WriteLine("WEB Text: " + message.StringMessage);
if (message.StringMessage == "/quit") // Client wants to exit gracefully
{
client.Send(new OutboundMessage("BYE"));
CloseClientConnection(client);
return;
}
this.ClientService.ProcessMessage(client, message);
this.Listen(client);
}
}
catch (Exception e)
{
Console.WriteLine("Exception: {0}", e);
this.CloseClientConnection(client);
if (webSocket != null)
{
webSocket.Dispose();
}
}
}
