private static bool SatisfyDataRequest(
[NotNull] DataVerificationResponse arg,
[NotNull] IVerificationDataProvider verificationDataProvider,
[NotNull] IClientStreamWriter <DataVerificationRequest> callRequestStream)
{
DataVerificationRequest result = new DataVerificationRequest();
try
{
if (arg.SchemaRequest != null)
{
result.Schema = verificationDataProvider.GetGdbSchema(arg.SchemaRequest);
}
else if (arg.DataRequest != null)
{
result.Data = verificationDataProvider
.GetData(arg.DataRequest).FirstOrDefault();
}
callRequestStream.WriteAsync(result);
return(true);
}
catch (Exception e)
{
_msg.Debug("Error handling data request", e);
// Send an empty message to make sure the server does not wait forever:
callRequestStream.WriteAsync(result);
throw;
}
}
private static async Task <DataVerificationRequest> RequestMoreDataAsync(
IAsyncStreamReader <DataVerificationRequest> requestStream,
IServerStreamWriter <DataVerificationResponse> responseStream,
ServerCallContext context,
DataVerificationResponse r)
{
DataVerificationRequest resultData = null;
Task responseReaderTask = Task.Run(
async() =>
{
while (resultData == null)
{
while (await requestStream.MoveNext().ConfigureAwait(false))
{
resultData = requestStream.Current;
break;
}
}
});
await responseStream.WriteAsync(r).ConfigureAwait(false);
await responseReaderTask.ConfigureAwait(false);
return(resultData);
}
public void InitializeSchema(ICollection <Dataset> datasets)
{
if (_dataRequestFunc == null)
{
return;
}
if (_virtualWorkspaces != null)
{
return;
}
var dataRequest = new DataVerificationResponse
{
SchemaRequest = new SchemaRequest()
};
dataRequest.SchemaRequest.DatasetIds.AddRange(datasets.Select(d => d.Id));
DataVerificationRequest dataResponse = _dataRequestFunc(dataRequest);
SetGdbSchema(ProtobufConversionUtils.CreateSchema(
dataResponse.Schema.ClassDefinitions,
dataResponse.Schema.RelclassDefinitions, _dataRequestFunc));
}
private async Task <bool> VerifyDataAsync(
[NotNull] QualityVerificationGrpc.QualityVerificationGrpcClient rpcClient,
[NotNull] CancellationTokenSource cancellationSource,
[CanBeNull] SchemaMsg schemaMsg = null)
{
using (var call = rpcClient.VerifyDataQuality())
{
var initialRequest =
new DataVerificationRequest
{
Request = VerificationRequest,
Schema = schemaMsg
};
await call.RequestStream.WriteAsync(initialRequest);
while (await call.ResponseStream.MoveNext(cancellationSource.Token))
{
var responseMsg = call.ResponseStream.Current;
if (responseMsg.SchemaRequest != null || responseMsg.DataRequest != null)
{
await ProvideDataToServer(responseMsg, call.RequestStream,
cancellationSource);
}
else
{
HandleProgressMsg(responseMsg.Response);
}
}
}
return(true);
}
public override IRow GetRow(int id)
{
Assert.True(_schema.HasOID, "The table {0} has no OID", DatasetUtils.GetName(_schema));
Assert.False(string.IsNullOrEmpty(_schema.OIDFieldName),
"The table {0} has no OID Field Name", DatasetUtils.GetName(_schema));
DataVerificationResponse response =
new DataVerificationResponse
{
DataRequest = new DataRequest
{
WhereClause = $"{_schema.OIDFieldName} = {id}"
}
};
if (_queryDefinition != null)
{
response.DataRequest.RelQueryDef = _queryDefinition;
}
else
{
response.DataRequest.ClassDef = Assert.NotNull(_classDefinition);
}
DataVerificationRequest moreData = GetData(response);
foreach (GdbObjectMsg gdbObjMsg in moreData.Data.GdbObjects)
{
return(ProtobufConversionUtils.FromGdbObjectMsg(gdbObjMsg, _schema));
}
// or better: COMException?
return(null);
}
public override async Task VerifyDataQuality(
IAsyncStreamReader <DataVerificationRequest> requestStream,
IServerStreamWriter <DataVerificationResponse> responseStream,
ServerCallContext context)
{
VerificationRequest request = null;
try
{
Assert.True(await requestStream.MoveNext(), "No request");
//while (await requestStream.MoveNext())
DataVerificationRequest initialrequest =
Assert.NotNull(requestStream.Current, "No request");
request = initialrequest.Request;
_msg.InfoFormat("Starting verification request from {0}", request);
_msg.DebugFormat("Request details: {0}", request);
if (Checkout3DAnalyst)
{
// It must be re-checked out (but somehow it's enough to do it
// on the calling thread-pool thread!?)
Ensure3dAnalyst();
}
Func <DataVerificationResponse, DataVerificationRequest> moreDataRequest =
delegate(DataVerificationResponse r)
{
return(Task.Run(async() =>
await RequestMoreDataAsync(
requestStream, responseStream, context, r)
.ConfigureAwait(false))
.Result);
};
Func <ITrackCancel, ServiceCallStatus> func =
trackCancel =>
VerifyDataQualityCore(initialrequest, moreDataRequest, responseStream,
trackCancel);
ServiceCallStatus result =
await GrpcServerUtils.ExecuteServiceCall(
func, context, _singleStaThreadScheduler);
_msg.InfoFormat("Verification {0}", result);
}
catch (Exception e)
{
_msg.Error($"Error verifying quality for request {request}", e);
SendFatalException(e, responseStream);
SetUnhealthy();
}
}
public override async Task VerifyDataQuality(
IAsyncStreamReader <DataVerificationRequest> requestStream,
IServerStreamWriter <DataVerificationResponse> responseStream,
ServerCallContext context)
{
VerificationRequest request = null;
try
{
Assert.True(await requestStream.MoveNext(), "No request");
DataVerificationRequest initialRequest =
Assert.NotNull(requestStream.Current, "No request");
request = initialRequest.Request;
await StartRequest(request);
Func <DataVerificationResponse, DataVerificationRequest> moreDataRequest =
delegate(DataVerificationResponse r)
{
return(Task.Run(async() =>
await RequestMoreDataAsync(
requestStream, responseStream, context, r)
.ConfigureAwait(false))
.Result);
};
Func <ITrackCancel, ServiceCallStatus> func =
trackCancel =>
VerifyDataQualityCore(initialRequest, moreDataRequest, responseStream,
trackCancel);
ServiceCallStatus result =
await GrpcServerUtils.ExecuteServiceCall(
func, context, _staThreadScheduler);
_msg.InfoFormat("Verification {0}", result);
}
catch (Exception e)
{
_msg.Error($"Error verifying quality for request {request}", e);
SendFatalException(e, responseStream);
SetUnhealthy();
}
finally
{
EndRequest();
}
}
private async Task SatisfyDataQueryAsync(
[NotNull] DataRequest dataRequest,
[NotNull] IClientStreamWriter <DataVerificationRequest> targetStream)
{
// Once the result messages are split up, this could be used for higher throughput
foreach (GdbData data in Assert.NotNull(VerificationDataProvider).GetData(dataRequest))
{
DataVerificationRequest r = new DataVerificationRequest
{
Data = data
};
await targetStream.WriteAsync(r);
}
}
private ITable GetRemoteQueryTable([NotNull] string relationshipClassName,
[NotNull] Model model,
[NotNull] IList <ITable> tables,
JoinType joinType,
[CanBeNull] string whereClause)
{
var dataRequest = new DataVerificationResponse
{
SchemaRequest = new SchemaRequest()
};
RelationshipClassQuery relClassQueryMsg =
new RelationshipClassQuery
{
RelationshipClassName = relationshipClassName,
WorkspaceHandle = model.Id,
JoinType = (int)joinType,
};
relClassQueryMsg.Tables.AddRange(tables.Select(DatasetUtils.GetName));
if (!string.IsNullOrEmpty(whereClause))
{
relClassQueryMsg.WhereClause = whereClause;
}
dataRequest.SchemaRequest.RelationshipClassQueries.Add(relClassQueryMsg);
DataVerificationRequest dataResponse = _dataRequestFunc(dataRequest);
GdbWorkspace gdbWorkspace =
Assert.NotNull(_virtualWorkspaces).First(w => w.WorkspaceHandle == model.Id);
ObjectClassMsg queryTableMsg = dataResponse.Schema.RelclassDefinitions.First();
Func <ITable, BackingDataset> createBackingDataset = null;
if (_dataRequestFunc != null)
{
createBackingDataset = (t) =>
new RemoteDataset(t, _dataRequestFunc, null, relClassQueryMsg);
}
// It is cached on the client side, in case various tests utilize the same definition.
// TODO: Test!
return(ProtobufConversionUtils.FromObjectClassMsg(queryTableMsg, gdbWorkspace,
createBackingDataset));
}
public override int GetRowCount(IQueryFilter filter)
{
DataRequest dataRequest = CreateDataRequest(filter);
dataRequest.CountOnly = true;
DataVerificationResponse response =
new DataVerificationResponse
{
DataRequest = dataRequest
};
DataVerificationRequest moreData = GetData(response);
if (moreData == null)
{
throw new IOException(
$"No data provided by the client for data request {response.DataRequest}");
}
return(moreData.Data.GdbObjectCount);
}
public override IEnumerable <IRow> Search(IQueryFilter filter, bool recycling)
{
DataRequest dataRequest = CreateDataRequest(filter);
DataVerificationResponse response =
new DataVerificationResponse
{
DataRequest = dataRequest
};
DataVerificationRequest moreData = GetData(response);
if (moreData == null)
{
throw new IOException(
$"No data provided by the client for data request {response.DataRequest}");
}
foreach (GdbObjectMsg gdbObjMsg in moreData.Data.GdbObjects)
{
yield return(ProtobufConversionUtils.FromGdbObjectMsg(gdbObjMsg, _schema));
}
}
private ServiceCallStatus VerifyDataQualityCore(
[NotNull] DataVerificationRequest initialRequest,
Func <DataVerificationResponse, DataVerificationRequest> moreDataRequest,
IServerStreamWriter <DataVerificationResponse> responseStream,
ITrackCancel trackCancel)
{
var request = initialRequest.Request;
SetupUserNameProvider(request);
void SendResponse(VerificationResponse r) => responseStream.WriteAsync(
new DataVerificationResponse {
Response = r
});
BackgroundVerificationService qaService = null;
List <GdbObjRefMsg> deletableAllowedErrorRefs = new List <GdbObjRefMsg>();
QualityVerification verification = null;
var issueCollection = new ConcurrentBag <IssueMsg>();
string cancellationMessage = null;
try
{
// TODO: Separate long-lived objects, such as datasetLookup, domainTransactions (add to this class) from
// short-term objects (request) -> add to background verification inputs
IBackgroundVerificationInputs backgroundVerificationInputs =
_verificationInputsFactoryMethod(request);
if (initialRequest.Schema != null)
{
backgroundVerificationInputs.SetGdbSchema(
ProtobufConversionUtils.CreateSchema(initialRequest.Schema.ClassDefinitions,
initialRequest
.Schema.RelclassDefinitions,
moreDataRequest));
}
else if (moreDataRequest != null)
{
backgroundVerificationInputs.SetRemoteDataAccess(moreDataRequest);
}
qaService = CreateVerificationService(backgroundVerificationInputs, issueCollection,
SendResponse, trackCancel);
verification = qaService.Verify(backgroundVerificationInputs, trackCancel);
deletableAllowedErrorRefs.AddRange(
GetDeletableAllowedErrorRefs(request.Parameters, qaService));
}
catch (Exception e)
{
_msg.Error($"Error checking quality for request {request}", e);
cancellationMessage = $"Server error: {e.Message}";
SetUnhealthy();
}
ServiceCallStatus result = SendFinalResponse(
verification, cancellationMessage ?? qaService.CancellationMessage, issueCollection,
deletableAllowedErrorRefs, qaService?.VerifiedPerimeter, SendResponse);
return(result);
}
