/// <summary>
/// Discovers schemas located in the users Zoho CRM instance
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns>Discovered schemas</returns>
public override async Task <DiscoverSchemasResponse> DiscoverSchemas(DiscoverSchemasRequest request,
ServerCallContext context)
{
Logger.SetLogPrefix("discover");
Logger.Info("Discovering Schemas...");
var sampleSize = checked ((int)request.SampleSize);
DiscoverSchemasResponse discoverSchemasResponse = new DiscoverSchemasResponse();
// only return requested schemas if refresh mode selected
if (request.Mode == DiscoverSchemasRequest.Types.Mode.All)
{
// get all schemas
try
{
var schemas = Discover.GetAllSchemas(_sessionFactory, sampleSize);
discoverSchemasResponse.Schemas.AddRange(await schemas.ToListAsync());
Logger.Info($"Schemas returned: {discoverSchemasResponse.Schemas.Count}");
return(discoverSchemasResponse);
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
return(new DiscoverSchemasResponse());
}
}
try
{
var refreshSchemas = request.ToRefresh;
Logger.Info($"Refresh schemas attempted: {refreshSchemas.Count}");
var schemas = Discover.GetRefreshSchemas(_sessionFactory, refreshSchemas, sampleSize);
discoverSchemasResponse.Schemas.AddRange(await schemas.ToListAsync());
// return all schemas
Logger.Info($"Schemas returned: {discoverSchemasResponse.Schemas.Count}");
return(discoverSchemasResponse);
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
return(new DiscoverSchemasResponse());
}
}
/// <summary>
/// Writes records to Cassandra
/// </summary>
/// <param name="requestStream"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task WriteStream(IAsyncStreamReader <Record> requestStream,
IServerStreamWriter <RecordAck> responseStream, ServerCallContext context)
{
try
{
Logger.Info("Writing records to Cassandra...");
var schema = _server.WriteSettings.Schema;
var inCount = 0;
// get next record to publish while connected and configured
while (await requestStream.MoveNext(context.CancellationToken) && _server.Connected &&
_server.WriteConfigured)
{
var record = requestStream.Current;
inCount++;
Logger.Debug($"Got record: {record.DataJson}");
if (_server.WriteSettings.IsReplication())
{
var config =
JsonConvert.DeserializeObject <ConfigureReplicationFormData>(_server.WriteSettings.Replication
.SettingsJson);
// send record to source system
// add await for unit testing
// removed to allow multiple to run at the same time
Task.Run(async() => await Replication.WriteRecord(_sessionFactory, schema, record, config, responseStream), context.CancellationToken);
}
else
{
// send record to source system
// add await for unit testing
// removed to allow multiple to run at the same time
Task.Run(async() =>
await Write.WriteRecordAsync(_sessionFactory, schema, record, responseStream),
context.CancellationToken);
}
}
Logger.Info($"Wrote {inCount} records to Cassandra.");
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
}
}
/// <summary>
/// Configures replication writebacks to Cassandra
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override Task <ConfigureReplicationResponse> ConfigureReplication(ConfigureReplicationRequest request,
ServerCallContext context)
{
Logger.SetLogPrefix("configure_replication");
Logger.Info($"Configuring write for schema name {request.Schema.Name}...");
var schemaJson = Replication.GetSchemaJson();
var uiJson = Replication.GetUIJson();
try
{
var errors = new List <string>();
if (!string.IsNullOrWhiteSpace(request.Form.DataJson))
{
// check for config errors
var replicationFormData = JsonConvert.DeserializeObject <ConfigureReplicationFormData>(request.Form.DataJson);
errors = replicationFormData.ValidateReplicationFormData();
}
return(Task.FromResult(new ConfigureReplicationResponse
{
Form = new ConfigurationFormResponse
{
DataJson = request.Form.DataJson,
Errors = { errors },
SchemaJson = schemaJson,
UiJson = uiJson,
StateJson = request.Form.StateJson
}
}));
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
return(Task.FromResult(new ConfigureReplicationResponse
{
Form = new ConfigurationFormResponse
{
DataJson = request.Form.DataJson,
Errors = { e.Message },
SchemaJson = schemaJson,
UiJson = uiJson,
StateJson = request.Form.StateJson
}
}));
}
}
/// <summary>
/// Handles disconnect requests from the agent
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override Task <DisconnectResponse> Disconnect(DisconnectRequest request, ServerCallContext context)
{
// clear connection
_server.Connected = false;
_server.Settings = null;
// alert connection session to close
if (_tcs != null)
{
_tcs.SetResult(true);
_tcs = null;
}
Logger.Info("Disconnected");
return(Task.FromResult(new DisconnectResponse()));
}
/// <summary>
/// Configures the plugin
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override Task <ConfigureResponse> Configure(ConfigureRequest request, ServerCallContext context)
{
Logger.Debug("Got configure request");
Logger.Debug(JsonConvert.SerializeObject(request, Formatting.Indented));
// ensure all directories are created
Directory.CreateDirectory(request.TemporaryDirectory);
Directory.CreateDirectory(request.PermanentDirectory);
Directory.CreateDirectory(request.LogDirectory);
// configure logger
Logger.SetLogLevel(request.LogLevel);
Logger.Init(request.LogDirectory);
_server.Config = request;
return(Task.FromResult(new ConfigureResponse()));
}
public override async Task ConnectSession(ConnectRequest request,
IServerStreamWriter <ConnectResponse> responseStream, ServerCallContext context)
{
Logger.SetLogPrefix("connect_session");
Logger.Info("Connecting session...");
// create task to wait for disconnect to be called
_tcs?.SetResult(true);
_tcs = new TaskCompletionSource <bool>();
// call connect method
var response = await Connect(request, context);
await responseStream.WriteAsync(response);
Logger.Info("Session connected.");
// wait for disconnect to be called
await _tcs.Task;
}
/// <summary>
/// Prepares writeback settings to write to Cassandra CQL3
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task <PrepareWriteResponse> PrepareWrite(PrepareWriteRequest request, ServerCallContext context)
{
// Logger.SetLogLevel(Logger.LogLevel.Debug);
Logger.SetLogPrefix(request.DataVersions.JobId);
Logger.Info("Preparing write...");
_server.WriteConfigured = false;
_server.WriteSettings = new WriteSettings
{
CommitSLA = request.CommitSlaSeconds,
Schema = request.Schema,
Replication = request.Replication,
DataVersions = request.DataVersions,
};
if (_server.WriteSettings.IsReplication())
{
// reconcile job
Logger.Info($"Starting to reconcile Replication Job {request.DataVersions.JobId}");
try
{
await Replication.ReconcileReplicationJobAsync(_sessionFactory, request);
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
return(new PrepareWriteResponse());
}
Logger.Info($"Finished reconciling Replication Job {request.DataVersions.JobId}");
}
_server.WriteConfigured = true;
Logger.Debug(JsonConvert.SerializeObject(_server.WriteSettings, Formatting.Indented));
Logger.Info("Write prepared.");
return(new PrepareWriteResponse());
}
/// <summary>
/// Publishes a stream of data for a given schema
/// </summary>
/// <param name="request"></param>
/// <param name="responseStream"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task ReadStream(ReadRequest request, IServerStreamWriter <Record> responseStream,
ServerCallContext context)
{
try
{
var schema = request.Schema;
var limit = request.Limit;
var limitFlag = request.Limit != 0;
var jobId = request.JobId;
var recordsCount = 0;
Logger.SetLogPrefix(jobId);
var records = Read.ReadRecords(_sessionFactory, schema);
await foreach (var record in records)
{
// stop publishing if the limit flag is enabled and the limit has been reached or the server is disconnected
if (limitFlag && recordsCount == limit || !_server.Connected)
{
break;
}
// publish record
await responseStream.WriteAsync(record);
recordsCount++;
}
Logger.Info($"Published {recordsCount} records");
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
}
}
/// <summary>
/// Establishes a connection with Cassandra.
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns>A message indicating connection success</returns>
public override async Task <ConnectResponse> Connect(ConnectRequest request, ServerCallContext context)
{
Logger.SetLogPrefix("connect");
// validate settings passed in
try
{
_server.Settings = JsonConvert.DeserializeObject <Settings>(request.SettingsJson);
_server.Settings.Validate();
}
catch (Exception e)
{
Logger.Error(e, e.Message);
return(new ConnectResponse
{
OauthStateJson = request.OauthStateJson,
ConnectionError = "",
OauthError = "",
SettingsError = e.Message
});
}
// initialize connection factory
try
{
_sessionFactory.Initialize(_server.Settings);
}
catch (Exception e)
{
Logger.Error(e, e.Message);
return(new ConnectResponse
{
OauthStateJson = request.OauthStateJson,
ConnectionError = e.Message,
OauthError = "",
SettingsError = ""
});
}
// test cluster factory
//var conn = _sessionFactory.GetConnection();
var session = Cluster.Builder()
.AddContactPoint(_server.Settings.Hostname).WithPort(Int32.Parse(_server.Settings.Port))
.WithCredentials(_server.Settings.Username, _server.Settings.Password)
.Build();
try
{
session.Connect();
}
catch (Exception e)
{
Logger.Error(e, e.Message);
return(new ConnectResponse
{
OauthStateJson = request.OauthStateJson,
ConnectionError = e.Message,
OauthError = "",
SettingsError = ""
});
}
finally
{
await session.ShutdownAsync();
}
// try
// {
// await conn.OpenAsync();
//
// if (!await conn.PingAsync())
// {
// return new ConnectResponse
// {
// OauthStateJson = request.OauthStateJson,
// ConnectionError = "Unable to ping target database.",
// OauthError = "",
// SettingsError = ""
// };
// }
// }
// catch (Exception e)
// {
// Logger.Error(e, e.Message, context);
//
// return new ConnectResponse
// {
// OauthStateJson = request.OauthStateJson,
// ConnectionError = e.Message,
// OauthError = "",
// SettingsError = ""
// };
// }
// finally
// {
// await conn.CloseAsync();
// }
_server.Connected = true;
return(new ConnectResponse
{
OauthStateJson = request.OauthStateJson,
ConnectionError = "",
OauthError = "",
SettingsError = ""
});
}
/// <summary>
/// Creates a form and handles form updates for write backs
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task <ConfigureWriteResponse> ConfigureWrite(ConfigureWriteRequest request,
ServerCallContext context)
{
Logger.Info("Configuring write...");
var storedProcedures = await Write.GetAllStoredProceduresAsync(_sessionFactory);
var schemaJson = Write.GetSchemaJson(storedProcedures);
var uiJson = Write.GetUIJson();
// if first call
if (string.IsNullOrWhiteSpace(request.Form.DataJson) || request.Form.DataJson == "{}")
{
return(new ConfigureWriteResponse
{
Form = new ConfigurationFormResponse
{
DataJson = "",
DataErrorsJson = "",
Errors = { },
SchemaJson = schemaJson,
UiJson = uiJson,
StateJson = ""
},
Schema = null
});
}
try
{
// get form data
var formData = JsonConvert.DeserializeObject <ConfigureWriteFormData>(request.Form.DataJson);
var storedProcedure = storedProcedures.Find(s => s.GetId() == formData.StoredProcedure);
// base schema to return
var schema = await Write.GetSchemaForStoredProcedureAsync(_sessionFactory, storedProcedure);
return(new ConfigureWriteResponse
{
Form = new ConfigurationFormResponse
{
DataJson = request.Form.DataJson,
Errors = { },
SchemaJson = schemaJson,
UiJson = uiJson,
StateJson = request.Form.StateJson
},
Schema = schema
});
}
catch (Exception e)
{
Logger.Error(e, e.Message, context);
return(new ConfigureWriteResponse
{
Form = new ConfigurationFormResponse
{
DataJson = request.Form.DataJson,
Errors = { e.Message },
SchemaJson = schemaJson,
UiJson = uiJson,
StateJson = request.Form.StateJson
},
Schema = null
});
}
}
public static async IAsyncEnumerable <Record> ReadRecords(ISessionFactory sessionFactory, Schema schema)
{
var session = sessionFactory.GetSession();
try
{
var query = schema.Query;
if (string.IsNullOrWhiteSpace(query))
{
query = $"SELECT * FROM {schema.Id}";
}
RowSet rows;
try
{
rows = await session.Execute(query);
}
catch (Exception e)
{
Logger.Error(e, e.Message);
yield break;
}
if (rows != null)
{
foreach (var row in rows)
{
var recordMap = new Dictionary <string, object>();
foreach (var property in schema.Properties)
{
try
{
switch (property.Type)
{
case PropertyType.String:
case PropertyType.Text:
case PropertyType.Decimal:
recordMap[property.Id] = row[property.Id].ToString();
break;
default:
recordMap[property.Id] = row[property.Id];
break;
}
}
catch (Exception e)
{
Logger.Error(e, $"No column or no data with property Id: {property.Id}");
Logger.Error(e, e.Message);
recordMap[property.Id] = null;
}
}
var record = new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(recordMap)
};
yield return(record);
}
// while (await reader.ReadAsync())
// {
// var recordMap = new Dictionary<string, object>();
//
// foreach (var property in schema.Properties)
// {
// try
// {
// switch (property.Type)
// {
// case PropertyType.String:
// case PropertyType.Text:
// case PropertyType.Decimal:
// recordMap[property.Id] = reader.GetValueById(property.Id, '`').ToString();
// break;
// default:
// recordMap[property.Id] = reader.GetValueById(property.Id, '`');
// break;
// }
// }
// catch (Exception e)
// {
// Logger.Error(e, $"No column with property Id: {property.Id}");
// Logger.Error(e, e.Message);
// recordMap[property.Id] = null;
// }
// }
//
// var record = new Record
// {
// Action = Record.Types.Action.Upsert,
// DataJson = JsonConvert.SerializeObject(recordMap)
// };
//
// yield return record;
// }
}
}
finally
{
//Noop
}
}
