public override async Task StreamingCall(IAsyncStreamReader<SimpleRequest> requestStream, IServerStreamWriter<SimpleResponse> responseStream, ServerCallContext context)
{
await requestStream.ForEachAsync(async request =>
{
var response = new SimpleResponse { Payload = CreateZerosPayload(request.ResponseSize) };
await responseStream.WriteAsync(response);
});
}
public async Task DivMany(ServerCallContext context, IAsyncStreamReader<DivArgs> requestStream, IServerStreamWriter<DivReply> responseStream)
{
await requestStream.ForEach(async divArgs =>
{
await responseStream.WriteAsync(DivInternal(divArgs));
});
}
public Task ListenForNewPeople(ListenForNewPeopleRequest request, IServerStreamWriter<Person> responseStream, ServerCallContext context)
{
LogRpc(context);
_personRepository.PersonCreated += async (sender, arg) => await responseStream.WriteAsync(arg.Person);
_manualResetEvent.Wait();
return Task.FromResult(0);
}
public async Task OperationStream(IAsyncStreamReader<ServiceRequest> requestStream, IServerStreamWriter<ServiceResponse> responseStream, ServerCallContext context)
{
while (await requestStream.MoveNext(CancellationToken.None))
{
await responseStream.WriteAsync(new ServiceResponse { Id = requestStream.Current.Id });
}
}
public async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
foreach (var responseParam in request.ResponseParameters)
{
var response = new StreamingOutputCallResponse { Payload = CreateZerosPayload(responseParam.Size) };
await responseStream.WriteAsync(response);
}
}
/// <summary>
/// Gets all features contained within the given bounding rectangle.
/// </summary>
public override async Task ListFeatures(Rectangle request, IServerStreamWriter<Feature> responseStream, ServerCallContext context)
{
var responses = features.FindAll( (feature) => feature.Exists() && request.Contains(feature.Location) );
foreach (var response in responses)
{
await responseStream.WriteAsync(response);
}
}
public override async Task ServerReflectionInfo(IAsyncStreamReader<ServerReflectionRequest> requestStream, IServerStreamWriter<ServerReflectionResponse> responseStream, ServerCallContext context)
{
while (await requestStream.MoveNext())
{
var response = ProcessRequest(requestStream.Current);
await responseStream.WriteAsync(response);
}
}
public async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
foreach (var responseParam in request.ResponseParametersList)
{
var response = StreamingOutputCallResponse.CreateBuilder()
.SetPayload(CreateZerosPayload(responseParam.Size)).Build();
await responseStream.WriteAsync(response);
}
}
public async Task FullDuplexCall(IAsyncStreamReader<StreamingOutputCallRequest> requestStream, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
await requestStream.ForEachAsync(async request =>
{
foreach (var responseParam in request.ResponseParameters)
{
var response = new StreamingOutputCallResponse { Payload = CreateZerosPayload(responseParam.Size) };
await responseStream.WriteAsync(response);
}
});
}
public override async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
await EnsureEchoMetadataAsync(context);
EnsureEchoStatus(request.ResponseStatus, context);
foreach (var responseParam in request.ResponseParameters)
{
var response = new StreamingOutputCallResponse { Payload = CreateZerosPayload(responseParam.Size) };
await responseStream.WriteAsync(response);
}
}
public async Task FullDuplexCall(ServerCallContext context, IAsyncStreamReader<StreamingOutputCallRequest> requestStream, IServerStreamWriter<StreamingOutputCallResponse> responseStream)
{
await requestStream.ForEach(async request =>
{
foreach (var responseParam in request.ResponseParametersList)
{
var response = StreamingOutputCallResponse.CreateBuilder()
.SetPayload(CreateZerosPayload(responseParam.Size)).Build();
await responseStream.WriteAsync(response);
}
});
}
public override async Task Fib(FibArgs request, IServerStreamWriter<Num> responseStream, ServerCallContext context)
{
var limit = request.Limit > 0 ? request.Limit : long.MaxValue;
var fibEnumerator = FibInternal(limit).GetEnumerator();
// Keep streaming the sequence until the call is cancelled.
// Use CancellationToken from ServerCallContext to detect the cancellation.
while (!context.CancellationToken.IsCancellationRequested && fibEnumerator.MoveNext())
{
await responseStream.WriteAsync(fibEnumerator.Current);
await Task.Delay(100);
}
}
public async Task Fib(ServerCallContext context, FibArgs request, IServerStreamWriter<Num> responseStream)
{
if (request.Limit <= 0)
{
// TODO(jtattermusch): support cancellation
throw new NotImplementedException("Not implemented yet");
}
if (request.Limit > 0)
{
foreach (var num in FibInternal(request.Limit))
{
await responseStream.WriteAsync(num);
}
}
}
public async Task Chat(IAsyncStreamReader<ChatMessage> requestStream, IServerStreamWriter<ChatMessage> responseStream, ServerCallContext context)
{
Log(context);
Program.Log("Server starting to chat");
while (await requestStream.MoveNext())
{
ChatMessage clientChatMessage = requestStream.Current;
Program.Log($"Client says {clientChatMessage}");
ChatMessage serverChatMessage = Utility.GetRandomChatMessage(0);
await responseStream.WriteAsync(serverChatMessage);
}
// Returning from the method will automatically complete the response async enumerator on the client.
}
public async Task ListenChat(ChatMessageRequest request, IServerStreamWriter<ChatMessage> responseStream, ServerCallContext context)
{
Log(context);
using (IAsyncEnumerator<ChatMessage> enumerator = _chatMessageRepository.ListenAccountChatAsync(request.AccountId).GetEnumerator())
{
// Custom reponse header
await context.WriteResponseHeadersAsync(new Metadata { new Metadata.Entry("Some-response-header-key", "Some-response-header-value") });
// Async enumerator
while (await enumerator.MoveNext())
{
ChatMessage chatMessage = enumerator.Current;
await responseStream.WriteAsync(chatMessage);
}
// Custom response trailer
context.ResponseTrailers.Add(new Metadata.Entry("Some-response-tailer-key", "Some-response-trailer-value"));
}
}
public async Task Fib(FibArgs request, IServerStreamWriter<Num> responseStream, ServerCallContext context)
{
if (request.Limit <= 0)
{
// keep streaming the sequence until cancelled.
IEnumerator<Num> fibEnumerator = FibInternal(long.MaxValue).GetEnumerator();
while (!context.CancellationToken.IsCancellationRequested && fibEnumerator.MoveNext())
{
await responseStream.WriteAsync(fibEnumerator.Current);
await Task.Delay(100);
}
}
if (request.Limit > 0)
{
foreach (var num in FibInternal(request.Limit))
{
await responseStream.WriteAsync(num);
}
}
}
public override async Task RunClient(IAsyncStreamReader<ClientArgs> requestStream, IServerStreamWriter<ClientStatus> responseStream, ServerCallContext context)
{
GrpcPreconditions.CheckState(await requestStream.MoveNext());
var clientConfig = requestStream.Current.Setup;
var runner = ClientRunners.CreateStarted(clientConfig);
await responseStream.WriteAsync(new ClientStatus
{
Stats = runner.GetStats(false)
});
while (await requestStream.MoveNext())
{
var reset = requestStream.Current.Mark.Reset;
await responseStream.WriteAsync(new ClientStatus
{
Stats = runner.GetStats(reset)
});
}
await runner.StopAsync();
}
public override async Task RunServer(IAsyncStreamReader<ServerArgs> requestStream, IServerStreamWriter<ServerStatus> responseStream, ServerCallContext context)
{
GrpcPreconditions.CheckState(await requestStream.MoveNext());
var serverConfig = requestStream.Current.Setup;
var runner = ServerRunners.CreateStarted(serverConfig);
await responseStream.WriteAsync(new ServerStatus
{
Stats = runner.GetStats(false),
Port = runner.BoundPort,
Cores = Environment.ProcessorCount,
});
while (await requestStream.MoveNext())
{
var reset = requestStream.Current.Mark.Reset;
await responseStream.WriteAsync(new ServerStatus
{
Stats = runner.GetStats(reset)
});
}
await runner.StopAsync();
}
/// <inheritdoc/>
public override async Task DuplexStreamingServerHandler <TRequest, TResponse>(IAsyncStreamReader <TRequest> requestStream, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, DuplexStreamingServerMethod <TRequest, TResponse> continuation)
{
try
{
await base.DuplexStreamingServerHandler(requestStream, responseStream, context, continuation).ConfigureAwait(false);
}
catch (Exception ex) when(OnException(ex, out var status))
{
throw new RpcException(status, ex.Message);
}
}
/// <summary>
/// Fib generates numbers in the Fibonacci sequence. If FibArgs.limit > 0, Fib
/// generates up to limit numbers; otherwise it continues until the call is
/// canceled. Unlike Fib above, Fib has no final FibReply.
/// </summary>
public virtual global::System.Threading.Tasks.Task Fib(global::Math.FibArgs request, IServerStreamWriter <global::Math.Num> responseStream, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, ""));
}
/// <summary>
/// Receives a stream of message/location pairs, and responds with a stream of all previous
/// messages at each of those locations.
/// </summary>
public override async Task RouteChat(IAsyncStreamReader<RouteNote> requestStream, IServerStreamWriter<RouteNote> responseStream, ServerCallContext context)
{
while (await requestStream.MoveNext())
{
var note = requestStream.Current;
List<RouteNote> prevNotes = AddNoteForLocation(note.Location, note);
foreach (var prevNote in prevNotes)
{
await responseStream.WriteAsync(prevNote);
}
}
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="conn"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecordAsync(SqlDatabaseConnection conn, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = Constants.SchemaName;
var safeTargetTableName = config.GetGoldenTableName();
var targetTable =
Replication.Replication.GetGoldenReplicationTable(schema, safeSchemaName, safeTargetTableName);
// get record
var recordData = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = null;
// write data
if (recordData.Count == 2)
{
// delete record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await Replication.Replication.DeleteRecordAsync(conn, targetTable, record.RecordId);
}
else
{
// add in all default values
foreach (var property in schema.Properties)
{
if (!recordData.ContainsKey(property.Id) && !string.IsNullOrWhiteSpace(property.PublisherMetaJson))
{
Logger.Debug("adding default value");
var columnConfig = JsonConvert.DeserializeObject <WriteColumn>(property.PublisherMetaJson);
recordData[property.Id] = columnConfig.DefaultValue;
}
}
Logger.Debug(JsonConvert.SerializeObject(recordData, Formatting.Indented));
// update record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await Replication.Replication.UpsertRecordAsync(conn, targetTable, recordData);
}
// set triggers for async file write
LastWriteTime = DateTime.Now;
PendingWrites = true;
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
if (e.Message.Contains("library routine called out of sequence"))
{
throw;
}
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
/// <summary>
/// Adds and removes records to replication db
/// Adds and updates available shapes
/// </summary>
/// <param name="connFactory"></param>
/// <param name="schema"></param>
/// <param name="record"></param>
/// <param name="config"></param>
/// <param name="responseStream"></param>
/// <returns>Error message string</returns>
public static async Task <string> WriteRecord(IConnectionFactory connFactory, Schema schema, Record record,
ConfigureReplicationFormData config, IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
try
{
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await ReplicationSemaphoreSlim.WaitAsync();
// setup
var safeSchemaName = config.SchemaName;
var safeGoldenTableName = config.GoldenTableName;
var safeVersionTableName = config.VersionTableName;
var goldenTable = GetGoldenReplicationTable(schema, safeSchemaName, safeGoldenTableName);
var versionTable = GetVersionReplicationTable(schema, safeSchemaName, safeVersionTableName);
// transform data
var recordVersionIds = record.Versions.Select(v => v.RecordId).ToList();
var recordData = GetNamedRecordData(schema, record.DataJson);
recordData[Constants.ReplicationRecordId] = record.RecordId;
recordData[Constants.ReplicationVersionIds] = recordVersionIds;
// get previous golden record
List <string> previousRecordVersionIds;
if (await RecordExistsAsync(connFactory, goldenTable, record.RecordId))
{
var recordMap = await GetRecordAsync(connFactory, goldenTable, record.RecordId);
if (recordMap.ContainsKey(Constants.ReplicationVersionIds))
{
previousRecordVersionIds =
JsonConvert.DeserializeObject <List <string> >(recordMap[Constants.ReplicationVersionIds].ToString());
}
else
{
previousRecordVersionIds = recordVersionIds;
}
}
else
{
previousRecordVersionIds = recordVersionIds;
}
// write data
// check if 2 since we always add 2 things to the dictionary
if (recordData.Count == 2)
{
// delete everything for this record
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - DELETE");
await DeleteRecordAsync(connFactory, goldenTable, record.RecordId);
foreach (var versionId in previousRecordVersionIds)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
}
else
{
// update record and remove/add versions
Logger.Debug($"shapeId: {safeSchemaName} | recordId: {record.RecordId} - UPSERT");
await UpsertRecordAsync(connFactory, goldenTable, recordData);
// delete missing versions
var missingVersions = previousRecordVersionIds.Except(recordVersionIds);
foreach (var versionId in missingVersions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {versionId} - DELETE");
await DeleteRecordAsync(connFactory, versionTable, versionId);
}
// upsert other versions
foreach (var version in record.Versions)
{
Logger.Debug(
$"shapeId: {safeSchemaName} | recordId: {record.RecordId} | versionId: {version.RecordId} - UPSERT");
var versionData = GetNamedRecordData(schema, version.DataJson);
versionData[Constants.ReplicationVersionRecordId] = version.RecordId;
versionData[Constants.ReplicationRecordId] = record.RecordId;
await UpsertRecordAsync(connFactory, versionTable, versionData);
}
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
Logger.Error(e, $"Error replicating records {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
ReplicationSemaphoreSlim.Release();
}
}
public override async Task RequestIndexingFromSideChain(RequestCrossChainBlockData request,
IServerStreamWriter <ResponseSideChainBlockData> responseStream, ServerCallContext context)
{
Logger.LogTrace("Side Chain Server received IndexedInfo message.");
await WriteResponseStream(request, responseStream, false);
}
public override async Task DuplexStreamingServerHandler <TRequest, TResponse>(IAsyncStreamReader <TRequest> requestStream, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, DuplexStreamingServerMethod <TRequest, TResponse> continuation)
{
await Monitor <TRequest, TResponse>(requestStream, context, continuation, responseStream);
}
public override async Task ServerStreamingServerHandler <TRequest, TResponse>(TRequest request, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, ServerStreamingServerMethod <TRequest, TResponse> continuation)
{
await Monitor <TRequest, TResponse>(request, context, continuation, responseStream);
}
public override async Task FullDuplexCall(IAsyncStreamReader <StreamingOutputCallRequest> requestStream, IServerStreamWriter <StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
await EnsureEchoMetadataAsync(context);
await requestStream.ForEachAsync(async request =>
{
EnsureEchoStatus(request.ResponseStatus, context);
foreach (var responseParam in request.ResponseParameters)
{
var response = new StreamingOutputCallResponse {
Payload = CreateZerosPayload(responseParam.Size)
};
await responseStream.WriteAsync(response);
}
});
}
public override async Task StreamingOutputCall(StreamingOutputCallRequest request, IServerStreamWriter <StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
await EnsureEchoMetadataAsync(context, request.ResponseParameters.Any(rp => rp.Compressed?.Value ?? false));
EnsureEchoStatus(request.ResponseStatus, context);
foreach (var responseParam in request.ResponseParameters)
{
responseStream.WriteOptions = !(responseParam.Compressed?.Value ?? false)
? new WriteOptions(WriteFlags.NoCompress)
: null;
var response = new StreamingOutputCallResponse {
Payload = CreateZerosPayload(responseParam.Size)
};
await responseStream.WriteAsync(response);
}
}
public override Task HalfDuplexCall(IAsyncStreamReader <StreamingOutputCallRequest> requestStream, IServerStreamWriter <StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
throw new NotImplementedException();
}
public override async Task handle(IAsyncStreamReader <CrdtStreamIn> requestStream, IServerStreamWriter <CrdtStreamOut> responseStream, ServerCallContext context)
{
if (!await requestStream.MoveNext())
{
return;
}
switch (requestStream.Current.MessageCase)
{
case CrdtStreamIn.MessageOneofCase.Command:
case CrdtStreamIn.MessageOneofCase.Changed:
case CrdtStreamIn.MessageOneofCase.Deleted:
case CrdtStreamIn.MessageOneofCase.State:
case CrdtStreamIn.MessageOneofCase.StreamCancelled:
case CrdtStreamIn.MessageOneofCase.None:
throw new InvalidOperationException(
$"First message on entity stream is expected to be 'Init'. Received: [{requestStream.Current.MessageCase}]"
);
case CrdtStreamIn.MessageOneofCase.Init:
var init = requestStream.Current.Init;
if (!CrdtStatefulServices.TryGetValue(init.ServiceName, out var crdtService)
)
{
throw new InvalidOperationException($"Failed to locate service with name {init.ServiceName}");
}
try
{
var streamContext = new MessageStreamingContext <CrdtStreamIn, CrdtStreamOut>(requestStream, responseStream, context);
await RunEntity(init, crdtService, streamContext);
}
catch (Exception ex)
{
Logger.LogError(ex, "Stream processing failed for entity.");
// TODO: translate to response error
throw;
}
break;
default:
throw new InvalidOperationException(
$"First message on entity stream is expected to be 'Init'. Received unknown case."
);
}
}
public override async Task StreamingBothWays(IAsyncStreamReader <SimpleRequest> requestStream, IServerStreamWriter <SimpleResponse> responseStream, ServerCallContext context)
{
var response = new SimpleResponse
{
Payload = new Payload {
Body = ByteString.CopyFrom(new byte[100])
}
};
var clientComplete = false;
var readTask = Task.Run(async() =>
{
await foreach (var message in requestStream.ReadAllAsync())
{
// Nom nom nom
}
clientComplete = true;
});
// Write outgoing messages until client is complete
while (!clientComplete)
{
await responseStream.WriteAsync(response);
}
await readTask;
}
public override async Task StreamingCall(IAsyncStreamReader <SimpleRequest> requestStream, IServerStreamWriter <SimpleResponse> responseStream, ServerCallContext context)
{
await foreach (var item in requestStream.ReadAllAsync())
{
await responseStream.WriteAsync(CreateResponse(item));
}
}
/// <summary>
/// BiDirectional streaming
/// </summary>
/// <param name="requestStream"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task SaveAll(IAsyncStreamReader <EmployeeRequest> requestStream, IServerStreamWriter <EmployeeResponse> responseStream, ServerCallContext context)
{
while (await requestStream.MoveNext())
{
var employee = requestStream.Current.Employee;
lock (this)
{
Employees.employees.Add(employee);
}
await responseStream.WriteAsync(new EmployeeResponse
{
Employee = employee
});
}
}
public BatchAppendWorker(IPublisher publisher, IAuthorizationProvider authorizationProvider,
IAsyncStreamReader <BatchAppendReq> requestStream, IServerStreamWriter <BatchAppendResp> responseStream,
ClaimsPrincipal user, int maxAppendSize, TimeSpan writeTimeout, bool requiresLeader)
{
_publisher = publisher;
_authorizationProvider = authorizationProvider;
_requestStream = requestStream;
_responseStream = responseStream;
_user = user;
_maxAppendSize = maxAppendSize;
_writeTimeout = writeTimeout;
_requiresLeader = requiresLeader;
_channel = Channel.CreateUnbounded <BatchAppendResp>(new() {
AllowSynchronousContinuations = false,
SingleReader = false,
SingleWriter = false
});
}
/// <summary>
/// DivMany accepts an arbitrary number of division args from the client stream
/// and sends back the results in the reply stream. The stream continues until
/// the client closes its end; the server does the same after sending all the
/// replies. The stream ends immediately if either end aborts.
/// </summary>
public virtual global::System.Threading.Tasks.Task DivMany(IAsyncStreamReader <global::Math.DivArgs> requestStream, IServerStreamWriter <global::Math.DivReply> responseStream, ServerCallContext context)
{
throw new RpcException(new Status(StatusCode.Unimplemented, ""));
}
public override async Task GetAllGauges(EmptyMessage request, IServerStreamWriter<GaugeResponse> responseStream, ServerCallContext context)
{
long qps = GetQpsAndReset();
var response = new GaugeResponse
{
Name = GaugeName,
LongValue = qps
};
await responseStream.WriteAsync(response);
}
/// <summary>
/// Publishes a stream of data for a given schema
/// </summary>
/// <param name="request"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task ReadStream(ReadRequest request, IServerStreamWriter <Record> responseStream,
ServerCallContext context)
{
var schema = request.Schema;
var limit = request.Limit;
var limitFlag = request.Limit > 0;
Logger.SetLogPrefix(request.JobId);
Logger.Info($"Publishing records for schema: {schema.Name}");
try
{
var recordsCount = 0;
// check if query is empty
if (string.IsNullOrWhiteSpace(schema.Query))
{
Logger.Info("Query not defined.");
return;
}
// build read record couchDB query
ReadQuery readQuery = new ReadQuery
{
fields = new List <string>(),
selector = new JObject()
};
// set limit to couchDB query if limitFlag is on
if (limitFlag)
{
readQuery.limit = limit;
}
foreach (Property property in schema.Properties)
{
readQuery.fields.Add(property.Name);
}
var couchdbReadRecordQuery = JsonConvert.SerializeObject(readQuery);
// read record from couchDB
Logger.Info($"Reading records from {_server.Settings.DatabaseName} database");
var readRecordUri = $"{_server.Settings.DatabaseName}/_find";
var response = await _client.PostAsync(readRecordUri,
new StringContent(couchdbReadRecordQuery, Encoding.UTF8, "application/json"));
response.EnsureSuccessStatusCode();
var documents = JObject.Parse(await response.Content.ReadAsStringAsync())["docs"];
// build record map
if (documents.ToList().Count > 0)
{
foreach (JObject document in documents)
{
// build record map
var recordMap = new Dictionary <string, object>();
foreach (JProperty property in document.Properties())
{
recordMap[property.Name] = property.Value;
}
//create record
var record = new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(recordMap)
};
//publish record
await responseStream.WriteAsync(record);
recordsCount++;
}
Logger.Info($"Published {recordsCount} records");
}
else
{
Logger.Info("No record read from database.");
}
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
}
}
public override Task DuplexStreamingServerHandler <TRequest, TResponse>(IAsyncStreamReader <TRequest> requestStream, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, DuplexStreamingServerMethod <TRequest, TResponse> continuation)
{
return(continuation(requestStream, responseStream, context));
}
public TracingServerStreamWriter(IServerStreamWriter <T> writer, ServerCallContext context, Action <T, ServerCallContext> onWrite)
{
_writer = writer;
_context = context;
_onWrite = onWrite;
}
public override async Task DuplexStreamingServerHandler <TRequest, TResponse>(IAsyncStreamReader <TRequest> requestStream, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, DuplexStreamingServerMethod <TRequest, TResponse> continuation)
{
(string, string)tuple = InterceptCallContext(context);
var watch = Stopwatch.StartNew();
await continuation(requestStream, responseStream, context);
Logger.Log.GrpcTrace(m_loggingContext, string.Format(RespondedLogFormat, tuple.Item1, tuple.Item2, watch.ElapsedMilliseconds));
}
/// <summary>
/// Publishes a stream of data for a given schema
/// </summary>
/// <param name="request"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task ReadStream(ReadRequest request, IServerStreamWriter <Record> responseStream,
ServerCallContext context)
{
var schema = request.Schema;
var limit = request.Limit;
var limitFlag = request.Limit != 0;
var jobId = request.JobId;
var recordsCount = 0;
Logger.SetLogPrefix(request.JobId);
Logger.Info($"Publishing records for schema: {schema.Name}");
try
{
var conn = Utility.GetSqlConnection(jobId);
var filesByRootPath = _server.Settings.GetAllFilesByRootPath();
if (string.IsNullOrWhiteSpace(schema.Query))
{
// schema is not query based so we can stream each file as it is loaded
var rootPaths = _server.Settings.GetRootPathsFromQuery(Utility.GetDefaultQuery(schema));
var rootPath = rootPaths.First();
var files = filesByRootPath[rootPath.RootPathName()];
var schemaName = Constants.SchemaName;
var tableName = string.IsNullOrWhiteSpace(rootPath.Name)
? new DirectoryInfo(rootPath.RootPath).Name
: rootPath.Name;
if (files.Count > 0)
{
// load file and then stream file one by one
foreach (var file in files)
{
Utility.LoadDirectoryFilesIntoDb(Utility.GetImportExportFactory(rootPath.Mode), conn,
rootPath,
tableName, schemaName, new List <string> {
file
}, true);
var records = Read.ReadRecords(context, schema, jobId);
foreach (var record in records)
{
// stop publishing if the limit flag is enabled and the limit has been reached or the server is disconnected
if ((limitFlag && recordsCount == limit) || !_server.Connected)
{
break;
}
// publish record
await responseStream.WriteAsync(record);
recordsCount++;
}
}
}
else
{
Utility.DeleteDirectoryFilesFromDb(conn, tableName, schemaName, Utility.GetImportExportFactory(rootPath.Mode), rootPath, files);
}
}
else
{
// schema is query based
var rootPaths = _server.Settings.GetRootPathsFromQuery(schema.Query);
Logger.Info(
$"Query root paths {JsonConvert.SerializeObject(rootPaths.Select(r => r.RootPathName()).ToList(), Formatting.Indented)}");
Logger.Info($"Begin loading all files.");
// schema is query based so everything in query needs to be loaded first
var hasRecords = true;
foreach (var rootPath in rootPaths)
{
var files = filesByRootPath[rootPath.RootPathName()];
var schemaName = Constants.SchemaName;
var tableName = string.IsNullOrWhiteSpace(rootPath.Name)
? new DirectoryInfo(rootPath.RootPath).Name
: rootPath.Name;
if (files.Count > 0)
{
Utility.LoadDirectoryFilesIntoDb(Utility.GetImportExportFactory(rootPath.Mode), conn,
rootPath,
tableName, schemaName, files, true);
}
else
{
hasRecords = false;
Utility.DeleteDirectoryFilesFromDb(conn, tableName, schemaName, Utility.GetImportExportFactory(rootPath.Mode), rootPath, files);
}
}
Logger.Info("Completed loading all files.");
if (hasRecords)
{
var records = Read.ReadRecords(context, schema, jobId);
foreach (var record in records)
{
// stop publishing if the limit flag is enabled and the limit has been reached or the server is disconnected
if ((limitFlag && recordsCount == limit) || !_server.Connected)
{
Logger.Info($"Stopping read - limit reached: {limitFlag && recordsCount == limit}, server disconnected: {!_server.Connected}");
break;
}
// publish record
await responseStream.WriteAsync(record);
recordsCount++;
}
}
}
Logger.Info($"Published {recordsCount} records");
foreach (var rootPath in _server.Settings.RootPaths)
{
var files = filesByRootPath[rootPath.RootPathName()];
switch (rootPath.CleanupAction)
{
case Constants.CleanupActionDelete:
foreach (var file in files)
{
Logger.Info($"Deleting file {file}");
Utility.DeleteFileAtPath(file, rootPath, _server.Settings, true);
}
break;
case Constants.CleanupActionArchive:
foreach (var file in files)
{
Logger.Info($"Archiving file {file} to {rootPath.ArchivePath}");
Utility.ArchiveFileAtPath(file, rootPath, _server.Settings);
}
break;
}
}
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
}
}
async Task <byte[]> DuplexStreamingServerMethod <TRequest, TResponse>(IAsyncStreamReader <byte[]> requestStream, IServerStreamWriter <byte[]> responseStream, ServerCallContext context)
{
using var serviceLocatorScope = serviceProvider.CreateScope();
var isErrorOrInterrupted = false;
var serviceContext = new ServiceContext(ServiceType, MethodInfo, AttributeLookup, this.MethodType, context, serializerOptions, logger, this, serviceLocatorScope.ServiceProvider)
{
RequestStream = requestStream,
ResponseStream = responseStream
};
try
{
logger.BeginInvokeMethod(serviceContext, emptyBytes, typeof(Nil));
using (requestStream as IDisposable)
{
if (enableCurrentContext)
{
ServiceContext.currentServiceContext.Value = serviceContext;
}
await this.methodBody(serviceContext).ConfigureAwait(false);
return(emptyBytes);
}
}
catch (ReturnStatusException ex)
{
isErrorOrInterrupted = true;
context.Status = ex.ToStatus();
return(emptyBytes);
}
catch (Exception ex)
{
isErrorOrInterrupted = true;
if (isReturnExceptionStackTraceInErrorDetail)
{
context.Status = new Status(StatusCode.Unknown, ex.ToString());
logger.Error(ex, context);
return(emptyBytes);
}
else
{
throw;
}
}
finally
{
logger.EndInvokeMethod(serviceContext, emptyBytes, typeof(Nil), (DateTime.UtcNow - serviceContext.Timestamp).TotalMilliseconds, isErrorOrInterrupted);
}
}
public override Task SayHellos(HelloRequest request, IServerStreamWriter <HelloReply> responseStream, ServerCallContext context)
{
return(base.SayHellos(request, responseStream, context));
}
public async Task HalfDuplexCall(IAsyncStreamReader<StreamingOutputCallRequest> requestStream, IServerStreamWriter<StreamingOutputCallResponse> responseStream, ServerCallContext context)
{
throw new NotImplementedException();
}
public override async Task Details(DetailsReq request, IServerStreamWriter <DetailsResp> responseStream,
ServerCallContext context)
{
var options = request.Options;
var user = context.GetHttpContext().User;
var readOperation = ReadOperation;
if (user?.Identity?.Name != null)
{
readOperation =
readOperation.WithParameter(
Plugins.Authorization.Operations.Users.Parameters.User(user.Identity.Name));
}
if (!await _authorizationProvider.CheckAccessAsync(user, readOperation, context.CancellationToken).ConfigureAwait(false))
{
throw RpcExceptions.AccessDenied();
}
var detailsSource = new TaskCompletionSource <UserManagementMessage.UserData[]>();
var envelope = new CallbackEnvelope(OnMessage);
_publisher.Publish(string.IsNullOrWhiteSpace(options?.LoginName)
? (Message) new UserManagementMessage.GetAll(envelope, user)
: new UserManagementMessage.Get(envelope, user, options.LoginName));
var details = await detailsSource.Task.ConfigureAwait(false);
foreach (var detail in details)
{
await responseStream.WriteAsync(new DetailsResp {
UserDetails = new DetailsResp.Types.UserDetails {
Disabled = detail.Disabled,
Groups = { detail.Groups },
FullName = detail.FullName,
LoginName = detail.LoginName,
LastUpdated = detail.DateLastUpdated.HasValue
? new DetailsResp.Types.UserDetails.Types.DateTime
{
TicksSinceEpoch = detail.DateLastUpdated.Value.UtcDateTime.ToTicksSinceEpoch()
}
: null
}
}).ConfigureAwait(false);
}
void OnMessage(Message message)
{
if (HandleErrors(options?.LoginName, message, detailsSource))
{
return;
}
switch (message)
{
case UserManagementMessage.UserDetailsResult userDetails:
detailsSource.TrySetResult(new[] { userDetails.Data });
break;
case UserManagementMessage.AllUserDetailsResult allUserDetails:
detailsSource.TrySetResult(allUserDetails.Data);
break;
default:
detailsSource.TrySetException(RpcExceptions.UnknownError(1));
break;
}
}
}
public override Task CheckCustomer(IAsyncStreamReader <CheckCustomerRequest> requestStream, IServerStreamWriter <CheckCustomerResponse> responseStream, ServerCallContext context)
{
return(Task.CompletedTask);
}
public static async Task <string> WriteRecordAsync(IConnectionFactory connFactory, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
// debug
Logger.Debug($"Starting timer for {record.RecordId}");
var timer = Stopwatch.StartNew();
var conn = connFactory.GetConnection();
try
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
// debug
Logger.Debug(JsonConvert.SerializeObject(record, Formatting.Indented));
// semaphore
await WriteSemaphoreSlim.WaitAsync();
// call stored procedure
var querySb = new StringBuilder($"CALL {schema.Query}(");
foreach (var property in schema.Properties)
{
if (!recordMap.ContainsKey(property.Id))
{
throw new Exception($"{property.Id} is required by the stored procedure and is not mapped on the job.");
}
var rawValue = recordMap[property.Id];
if (rawValue == null || string.IsNullOrWhiteSpace(rawValue.ToString()))
{
querySb.Append("NULL,");
}
else
{
querySb.Append($"'{Utility.Utility.GetSafeString(Utility.Utility.GetSafeString(rawValue.ToString(), "'", "''"))}',");
}
}
querySb.Length--;
querySb.Append(")");
var query = querySb.ToString();
Logger.Debug($"WB querySb: {query}");
await conn.OpenAsync();
var cmd = connFactory.GetCommand(query, conn);
await cmd.ExecuteNonQueryAsync();
await conn.CloseAsync();
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Acknowledged Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return("");
}
catch (Exception e)
{
await conn.CloseAsync();
Logger.Error(e, $"Error writing record {e.Message}");
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = e.Message
};
await responseStream.WriteAsync(ack);
timer.Stop();
Logger.Debug($"Failed Record {record.RecordId} time: {timer.ElapsedMilliseconds}");
return(e.Message);
}
finally
{
WriteSemaphoreSlim.Release();
}
}
public override Task ListCustomers(CustomerSearch request, IServerStreamWriter <Customer> responseStream, ServerCallContext context)
{
return(Task.CompletedTask);
}
//Bidi Streaming API
public override async Task GreetEveryone(IAsyncStreamReader <GreetEveryoneRequest> requestStream, IServerStreamWriter <GreetEveryoneResponse> responseStream, ServerCallContext context)
{
while (await requestStream.MoveNext())
{
var result = String.Format("Hello {0} {1}",
requestStream.Current.Greeting.FirstName,
requestStream.Current.Greeting.LastName);
Console.WriteLine("Receieved: " + result);
await responseStream.WriteAsync(new GreetEveryoneResponse()
{
Result = result
});
}
}
public override async Task DivMany(IAsyncStreamReader<DivArgs> requestStream, IServerStreamWriter<DivReply> responseStream, ServerCallContext context)
{
await requestStream.ForEachAsync(async divArgs => await responseStream.WriteAsync(DivInternal(divArgs)));
}
/// <summary>
/// Writes records to CouchDB
/// </summary>
/// <param name="requestStream"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task WriteStream(IAsyncStreamReader <Record> requestStream,
IServerStreamWriter <RecordAck> responseStream, ServerCallContext context)
{
try
{
Logger.Info("Writing records to CouchDB...");
var schema = _server.WriteSettings.Schema;
var sla = _server.WriteSettings.CommitSLA;
var inCount = 0;
var outCount = 0;
// get next record to publish while connected and configured
//Logger.Info($"server is connected: {_server.Connected}, server is writeconfigured: {_server.WriteConfigured}");
while (await requestStream.MoveNext(context.CancellationToken) && _server.Connected &&
_server.WriteConfigured)
{
var record = requestStream.Current;
inCount++;
Logger.Debug($"Got record: {record.DataJson}");
if (_server.WriteSettings.IsReplication())
{
var config =
JsonConvert.DeserializeObject <ConfigureReplicationFormData>(_server.WriteSettings
.Replication.SettingsJson);
Logger.Info($"send record: {record.RecordId} to source system");
var task = Task.Run(() => Replication.WriteRecord(_client, schema, record, config));
if (task.Wait(TimeSpan.FromSeconds(sla)))
{
// send ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = task.Result
};
await responseStream.WriteAsync(ack);
if (String.IsNullOrEmpty(task.Result))
{
outCount++;
}
}
else
{
// send timeout ack
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = "timed out"
};
await responseStream.WriteAsync(ack);
}
}
else
{
throw new Exception("Only replication writeback are supported");
}
}
Logger.Info($"Wrote {outCount} of {inCount} records to CouchDB.");
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
throw;
}
}
/// <summary>
/// Generic streaming call handler.
/// </summary>
public async Task StreamingCall(IAsyncStreamReader<byte[]> requestStream, IServerStreamWriter<byte[]> responseStream, ServerCallContext context)
{
await requestStream.ForEachAsync(async request =>
{
await responseStream.WriteAsync(response);
});
}
public override Task ServerStreamingServerHandler <TRequest, TResponse>(TRequest request, IServerStreamWriter <TResponse> responseStream, ServerCallContext context, ServerStreamingServerMethod <TRequest, TResponse> continuation)
{
return(continuation(request, responseStream, context));
}
public async Task GetPersonList(PersonListRequest request, IServerStreamWriter<Person> responseStream, ServerCallContext context)
{
LogRpc(context);
foreach (Person person in _personRepository.GetAllPeople())
{
await responseStream.WriteAsync(person);
}
}
