public override async Task ListActions(IAsyncStreamWriter <FlightActionType> responseStream, ServerCallContext context)
{
await responseStream.WriteAsync(new FlightActionType("get", "get a flight"));
await responseStream.WriteAsync(new FlightActionType("put", "add a flight"));
await responseStream.WriteAsync(new FlightActionType("delete", "delete a flight"));
await responseStream.WriteAsync(new FlightActionType("test", "test action"));
}
public override async Task Reverse(IAsyncStreamReader <IMessage <Box <string> > > requests, IAsyncStreamWriter <IMessage <Box <string> > > responses, ServerCallContext context)
{
while (await requests.MoveNext())
{
var msg = requests.Current;
string value = msg.Payload.Deserialize().value;
await responses.WriteAsync(Message.From(Box.Create(value.ToUpperInvariant())));
}
// Requests and responses do not have to correspond 1:1. Both of the streams can be handled independently.
await responses.WriteAsync(Message.From(Box.Create("BONUS!")));
}
public override async Task DoPut(
FlightServerRecordBatchStreamReader requestStream,
IAsyncStreamWriter <FlightPutResult> responseStream,
ServerCallContext context
)
{
var newTable = new List <RecordBatch> {
};
long numRows = 0;
await foreach (var batch in requestStream.ReadAllAsync(context.CancellationToken))
{
newTable.Add(batch);
numRows += batch.Length;
}
var descriptor = await requestStream.FlightDescriptor;
var ticket = DescriptorAsTicket(descriptor);
var schema = await requestStream.Schema;
_flightData.Flights.Add(ticket, new FlightInfo(
schema,
descriptor,
new List <FlightEndpoint> {
GetEndpoint(ticket, $"http://{context.Host}")
},
numRows,
-1 // Unknown
));
_flightData.Tables.Add(ticket, newTable);
await responseStream.WriteAsync(new FlightPutResult("Table saved."));
}
public override async Task ListActions(
IAsyncStreamWriter <FlightActionType> responseStream,
ServerCallContext context
)
{
await responseStream.WriteAsync(new FlightActionType("clear", "Clear the flights from the server"));
}
/// <summary>
/// IServerStreamWriter Methods.
/// </summary>
public Task WriteAsync(TResponse message)
{
var bytes = MessagePackSerializer.Serialize(message, context.SerializerOptions);
logger.WriteToStream(context, bytes, typeof(TResponse));
return(innerWriter.WriteAsync(bytes));
}
/// <summary>
/// IServerStreamWriter Methods.
/// </summary>
public Task WriteAsync(TResponse message)
{
var bytes = LZ4MessagePackSerializer.Serialize(message, context.FormatterResolver);
logger.WriteToStream(context, bytes, typeof(TResponse));
return(innerWriter.WriteAsync(bytes));
}
/// <summary>
/// Writes the specified objects.
/// </summary>
/// <typeparam name="T">The type of the object.</typeparam>
/// <param name="streamWriter"></param>
/// <param name="objects">The objects.</param>
/// <returns></returns>
public static async Task WritesAsync <T>(this IAsyncStreamWriter <T> streamWriter, IEnumerable <Task <T> > objects)
{
foreach (Task <T> request in objects)
{
await streamWriter.WriteAsync(await request.ConfigureAwait(false)).ConfigureAwait(false);
}
}
private async Task SendLoginResponses(string targetLogin,
IAsyncStreamWriter <ServerResponse> responseStream,
ServerResponse loginResponse)
{
if (loginResponse.Type == ResponseType.Init || loginResponse.Type == ResponseType.LoginExists)
{
await responseStream.WriteAsync(loginResponse);
if (loginResponse.Type == ResponseType.Init)
{
Responses.TryAdd(targetLogin, new ConcurrentQueue <ServerResponse>());
}
}
else if (loginResponse.Type == ResponseType.PlayerJoin)
{
await responseStream.WriteAsync(loginResponse);
}
}
public override async Task ListFlights(FlightCriteria request, IAsyncStreamWriter <FlightInfo> responseStream, ServerCallContext context)
{
var flightInfos = _flightStore.Flights.Select(x => x.Value.GetFlightInfo()).ToList();
foreach (var flightInfo in flightInfos)
{
await responseStream.WriteAsync(flightInfo);
}
}
private async Task SendSchema()
{
_currentFlightData = new Protocol.FlightData();
if (_flightDescriptor != null)
{
_currentFlightData.FlightDescriptor = _flightDescriptor.ToProtocol();
}
var offset = SerializeSchema(Schema);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
await WriteMessageAsync(MessageHeader.Schema, offset, 0, cancellationTokenSource.Token).ConfigureAwait(false);
await _clientStreamWriter.WriteAsync(_currentFlightData).ConfigureAwait(false);
HasWrittenSchema = true;
}
public override async Task Invert(IAsyncStreamReader <IMessage <Box <string> > > requests, IAsyncStreamWriter <IMessage <Box <string> > > responses, ServerCallContext context)
{
while (await requests.MoveNext())
{
var requestStr = requests.Current.Payload.Deserialize().value;
var resultStr = new string(requestStr.Reverse().ToArray());
await responses.WriteAsync(Message.From(Box.Create(resultStr)));
}
}
public override async Task Invert(IAsyncStreamReader <IMessage <Box <int> > > requests, IAsyncStreamWriter <IMessage <Box <int> > > responses, ServerCallContext context)
{
while (await requests.MoveNext())
{
var requestInt = requests.Current.Payload.Deserialize().value;
var resultInt = ~requestInt;
await responses.WriteAsync(Message.From(Box.Create(resultInt)));
}
}
public override async Task ListFlights(
FlightCriteria request,
IAsyncStreamWriter <FlightInfo> responseStream,
ServerCallContext context
)
{
foreach (var flight in _flightData.Flights.Values)
{
await responseStream.WriteAsync(flight);
}
}
public override async Task Countdown(IMessage <Box <uint> > request, IAsyncStreamWriter <IMessage <Box <uint> > > responses, ServerCallContext context)
{
uint num = request.Payload.Deserialize().value;
while (num > 0)
{
await responses.WriteAsync(Message.From(Box.Create(num)));
--num;
}
}
private async void DequeuePacketsToBeSent()
{
while (true)
{
while (_packetsToBeSent.Count > 0)
{
// Record.Log($"Send");
await _outStream.WriteAsync(_packetsToBeSent.Dequeue());
}
await Task.Delay(100);
}
}
public override async Task ListFlights(FlightCriteria request, IAsyncStreamWriter <FlightInfo> responseStream, ServerCallContext context)
{
var tables = _koraliumTransportService.GetTables();
var httpContext = context.GetHttpContext();
foreach (var table in tables)
{
var selectAllSql = table.SelectAllColumnsStatement();
await responseStream.WriteAsync(GetFlightInfo(selectAllSql, context));
}
}
public override async Task DoAction(FlightAction request, IAsyncStreamWriter <FlightResult> responseStream, ServerCallContext context)
{
switch (request.Type)
{
case "test":
await responseStream.WriteAsync(new FlightResult("test data"));
break;
default:
throw new NotImplementedException();
}
}
static async Task CommandLoopAsync(IAsyncStreamWriter <ActionMessage> stream, CancellationToken token)
{
Console.WriteLine("Use +XXXX to add a symbol, -XXXX to remove it.");
await Task.Yield();
while (!token.IsCancellationRequested)
{
Console.Write("> ");
var command = Console.ReadLine();
if (string.IsNullOrWhiteSpace(command))
{
break;
}
ActionMessage message;
switch (command[0])
{
case '+':
message = new ActionMessage {
Add = new AddSymbolRequest {
Symbol = command.Substring(1)
}
};
break;
case '-':
message = new ActionMessage {
Remove = new RemoveSymbolRequest {
Symbol = command.Substring(1)
}
};
break;
default:
Console.WriteLine($"Unknown command: {command}");
continue;
}
try
{
await stream.WriteAsync(message);
}
catch (RpcException ex)
{
Console.WriteLine($"Error sending command: {ex.Message}");
}
catch (OperationCanceledException)
{
// Ignored
}
}
}
async Task SendJoin(IAsyncStreamWriter <TransportMessage> requestStream)
{
var message = new TransportMessage
{
MessageId = NewId.NextGuid().ToString(),
Join = new Join {
Node = new Node().Initialize(_hostNode)
}
};
await requestStream.WriteAsync(message).ConfigureAwait(false);
_node.LogSent(message);
}
private static async Task <IResult> RewriteStream <T>(IAsyncStreamReader <T> source, IAsyncStreamWriter <T> destination) where T : class
{
try
{
while (await source.MoveNext())
{
await destination.WriteAsync(source.Current);
}
}
catch (Exception e)
{
return(Result.Failure(e.Message));
}
return(Result.Success());
}
public override async Task DoPut(FlightServerRecordBatchStreamReader requestStream, IAsyncStreamWriter <FlightPutResult> responseStream, ServerCallContext context)
{
var flightDescriptor = await requestStream.FlightDescriptor;
if (!_flightStore.Flights.TryGetValue(flightDescriptor, out var flightHolder))
{
flightHolder = new FlightHolder(flightDescriptor, await requestStream.Schema, $"http://{context.Host}");
_flightStore.Flights.Add(flightDescriptor, flightHolder);
}
while (await requestStream.MoveNext())
{
flightHolder.AddBatch(new RecordBatchWithMetadata(requestStream.Current, requestStream.ApplicationMetadata.FirstOrDefault()));
await responseStream.WriteAsync(FlightPutResult.Empty);
}
}
private async Task Send(Response response, string typeUrl)
{
try
{
var nonce = Interlocked.Increment(ref _streamNonce);
var resources = response.Resources.Select(Any.Pack);
var discoveryResponse = new DiscoveryResponse
{
VersionInfo = response.Version,
TypeUrl = typeUrl,
Nonce = nonce.ToString()
};
discoveryResponse.Resources.Add(resources);
Logger.LogDebug("[{0}] response {1} with nonce {2} version {3}", _streamId, typeUrl, nonce, response.Version);
_callbacks.ForEach(cb => cb.OnStreamResponse(_streamId, response.Request, discoveryResponse));
// Store the latest response *before* we send the response. This ensures that by the time the request
// is processed the map is guaranteed to be updated. Doing it afterwards leads to a race conditions
// which may see the incoming request arrive before the map is updated, failing the nonce check erroneously.
this.SetLatestResponse(typeUrl, discoveryResponse);
using (await _lock.LockAsync())
{
if (!_isClosing)
{
await _responseStream.WriteAsync(discoveryResponse);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public async Task RunJob(BuildTaskBase?buildTaskBase)
{
await SetupWorkspace();
logger.LogTrace("Copied workspace");
Directory.CreateDirectory(ArtifactDir);
var psi = buildTaskBase switch {
BuildTask buildTask => CreateBuildProcess(buildDir, buildTask),
ContainerBuildTask containerBuildTask => CreateContainerBuildProcess(buildDir, containerBuildTask),
_ => throw new Exception("Invalid build task")
};
cancellationToken.ThrowIfCancellationRequested();
var process = Process.Start(psi);
logger.LogTrace("Running external process");
var exitTask = process.WaitForExitAsync();
var writerLock = new AsyncLock();
async Task PipeData(Stream stream)
{
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
{
cancellationToken.ThrowIfCancellationRequested();
using (await writerLock.LockAsync()) {
cancellationToken.ThrowIfCancellationRequested();
await updateStream.WriteAsync(new BuildStatusUpdate {
BuildOutput = ByteString.CopyFrom(buffer.AsSpan(0, bytesRead)),
});
}
cancellationToken.ThrowIfCancellationRequested();
}
stream.Close();
}
var outputTask = Task.Run(() => PipeData(process.StandardOutput.BaseStream), cancellationToken);
var errorTask = Task.Run(() => PipeData(process.StandardError.BaseStream), cancellationToken);
await Task.WhenAll(outputTask, errorTask, exitTask);
var artifactDir = ArtifactDir;
var artifacts = Directory.EnumerateFiles(artifactDir, "*.json", SearchOption.AllDirectories)
.Select(file => new ArtifactInfo {
Name = file.Substring(artifactDir.Length)
.TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
});
var jobResult = new JobResult {
ExitCode = process.ExitCode,
};
jobResult.Artifacts.AddRange(artifacts);
await updateStream.WriteAsync(new BuildStatusUpdate {
JobFinished = jobResult,
});
logger.LogTrace("Finished job");
}
public Task WriteAsync(T message)
{
logger.WriteToStream(context);
return(writer.WriteAsync(message));
}
/// <summary>
/// IServerStreamWriter Methods.
/// </summary>
public Task WriteAsync(TResponse message)
{
var bytes = responseMarshaller.Serializer(message);
return(innerWriter.WriteAsync(bytes));
}
public Task WriteAsync(TIn message)
{
return(_inputStream.WriteAsync(_convertFunction(message)));
}
/// <inheritdoc cref="IServerChatClient.SendMessageAsync"/> public Task SendMessageAsync(ServerMessage message) => _responseStream.WriteAsync(message);
public async Task HandleStream(
IAsyncStreamReader <DiscoveryRequest> requestStream,
IAsyncStreamWriter <DiscoveryResponse> responseStream,
ServerCallContext context,
string defaultTypeUrl)
{
var streamId = Guid.NewGuid();
_logger.InfoF($"New Stream started {streamId}");
var clusters = new WatchAndNounce();
var endpoints = new WatchAndNounce();
var listeners = new WatchAndNounce();
var routes = new WatchAndNounce();
var watches = new Dictionary <string, WatchAndNounce>()
{
{ TypeStrings.ClusterType, clusters },
{ TypeStrings.EndpointType, endpoints },
{ TypeStrings.ListenerType, listeners },
{ TypeStrings.RouteType, routes },
};
try
{
var streamNonce = 0;
var requestTask = requestStream.MoveNext(_cancellationToken);
while (!_cancellationToken.IsCancellationRequested &&
!context.CancellationToken.IsCancellationRequested)
{
var resolved = await Task.WhenAny(
requestTask,
clusters.Watch.Response,
endpoints.Watch.Response,
listeners.Watch.Response,
routes.Watch.Response);
switch (resolved)
{
case Task <bool> r when ReferenceEquals(r, requestTask):
// New request from Envoy
if (!r.Result)
{
// MoveNext failed. The stream was finalized by Envoy or is broken
throw new OperationCanceledException();
}
var request = requestStream.Current;
if (request.Node?.Id == null)
{
_logger.Warning("Missing Node ID in envoy request. Check configuration");
throw new InvalidOperationException("Missing Node ID in envoy request. Check configuration");
}
if (defaultTypeUrl == TypeStrings.Any && string.IsNullOrEmpty(request.TypeUrl))
{
_logger.Warning("type URL is required for ADS");
throw new InvalidOperationException("type URL is required for ADS");
}
if (string.IsNullOrEmpty(request.TypeUrl))
{
request.TypeUrl = defaultTypeUrl;
}
_logger.DebugF($"<- New request on stream {streamId}, typeUrl {request.TypeUrl}, version {request.VersionInfo}, nonce: {request.ResponseNonce}");
var requestWatch = watches[request.TypeUrl];
if (requestWatch.Nonce == null || requestWatch.Nonce == request.ResponseNonce)
{
// if the nonce is not correct ignore the request.
requestWatch.Watch.Cancel();
requestWatch.Watch = _cache.CreateWatch(request);
if (_nodes.Add(request.Node.Id))
{
// send to this newly connected node envoy the current snapshot
_cache.SetSnapshot(request.Node.Id, _snapshot);
}
}
else
{
_logger.DebugF($"<- Ignoring request on stream {streamId}, expectedNonce: {requestWatch.Nonce}, recievedNonce: {request.ResponseNonce}");
}
requestTask = requestStream.MoveNext(_cancellationToken);
break;
case Task <DiscoveryResponse> responseTask:
// Watch was resolved. Send the update.
var response = responseTask.Result;
var responseWatch = watches[response.TypeUrl];
response.Nonce = (++streamNonce).ToString();
responseWatch.Nonce = streamNonce.ToString();
// the value is consumed we do not want to get it again from this watch.
responseWatch.Watch.Cancel();
responseWatch.Watch = Watch.Empty;
_logger.DebugF($"-> New response on stream {streamId}, typeUrl {response.TypeUrl}, version {response.VersionInfo}, nonce: {response.Nonce}");
await responseStream.WriteAsync(response);
break;
}
}
}
finally
{
// cleanup the cache before exit
clusters.Watch.Cancel();
endpoints.Watch.Cancel();
listeners.Watch.Cancel();
routes.Watch.Cancel();
_logger.InfoF($"Stream finalized {streamId}");
}
}