/// <summary> /// Discovers schemas located in the users Zoho CRM instance /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns>Discovered schemas</returns> public override async Task <DiscoverSchemasResponse> DiscoverSchemas(DiscoverSchemasRequest request, ServerCallContext context) { Logger.SetLogPrefix("discover"); Logger.Info("Discovering Schemas..."); var sampleSize = checked ((int)request.SampleSize); DiscoverSchemasResponse discoverSchemasResponse = new DiscoverSchemasResponse(); // only return requested schemas if refresh mode selected if (request.Mode == DiscoverSchemasRequest.Types.Mode.All) { // get all schemas try { var schemas = Discover.GetAllSchemas(_sessionFactory, sampleSize); discoverSchemasResponse.Schemas.AddRange(await schemas.ToListAsync()); Logger.Info($"Schemas returned: {discoverSchemasResponse.Schemas.Count}"); return(discoverSchemasResponse); } catch (Exception e) { Logger.Error(e, e.Message, context); return(new DiscoverSchemasResponse()); } } try { var refreshSchemas = request.ToRefresh; Logger.Info($"Refresh schemas attempted: {refreshSchemas.Count}"); var schemas = Discover.GetRefreshSchemas(_sessionFactory, refreshSchemas, sampleSize); discoverSchemasResponse.Schemas.AddRange(await schemas.ToListAsync()); // return all schemas Logger.Info($"Schemas returned: {discoverSchemasResponse.Schemas.Count}"); return(discoverSchemasResponse); } catch (Exception e) { Logger.Error(e, e.Message, context); return(new DiscoverSchemasResponse()); } }
/// <summary> /// Writes records to Cassandra /// </summary> /// <param name="requestStream"></param> /// <param name="responseStream"></param> /// <param name="context"></param> /// <returns></returns> public override async Task WriteStream(IAsyncStreamReader <Record> requestStream, IServerStreamWriter <RecordAck> responseStream, ServerCallContext context) { try { Logger.Info("Writing records to Cassandra..."); var schema = _server.WriteSettings.Schema; var inCount = 0; // get next record to publish while connected and configured while (await requestStream.MoveNext(context.CancellationToken) && _server.Connected && _server.WriteConfigured) { var record = requestStream.Current; inCount++; Logger.Debug($"Got record: {record.DataJson}"); if (_server.WriteSettings.IsReplication()) { var config = JsonConvert.DeserializeObject <ConfigureReplicationFormData>(_server.WriteSettings.Replication .SettingsJson); // send record to source system // add await for unit testing // removed to allow multiple to run at the same time Task.Run(async() => await Replication.WriteRecord(_sessionFactory, schema, record, config, responseStream), context.CancellationToken); } else { // send record to source system // add await for unit testing // removed to allow multiple to run at the same time Task.Run(async() => await Write.WriteRecordAsync(_sessionFactory, schema, record, responseStream), context.CancellationToken); } } Logger.Info($"Wrote {inCount} records to Cassandra."); } catch (Exception e) { Logger.Error(e, e.Message, context); } }
/// <summary> /// Configures replication writebacks to Cassandra /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public override Task <ConfigureReplicationResponse> ConfigureReplication(ConfigureReplicationRequest request, ServerCallContext context) { Logger.SetLogPrefix("configure_replication"); Logger.Info($"Configuring write for schema name {request.Schema.Name}..."); var schemaJson = Replication.GetSchemaJson(); var uiJson = Replication.GetUIJson(); try { var errors = new List <string>(); if (!string.IsNullOrWhiteSpace(request.Form.DataJson)) { // check for config errors var replicationFormData = JsonConvert.DeserializeObject <ConfigureReplicationFormData>(request.Form.DataJson); errors = replicationFormData.ValidateReplicationFormData(); } return(Task.FromResult(new ConfigureReplicationResponse { Form = new ConfigurationFormResponse { DataJson = request.Form.DataJson, Errors = { errors }, SchemaJson = schemaJson, UiJson = uiJson, StateJson = request.Form.StateJson } })); } catch (Exception e) { Logger.Error(e, e.Message, context); return(Task.FromResult(new ConfigureReplicationResponse { Form = new ConfigurationFormResponse { DataJson = request.Form.DataJson, Errors = { e.Message }, SchemaJson = schemaJson, UiJson = uiJson, StateJson = request.Form.StateJson } })); } }
/// <summary> /// Handles disconnect requests from the agent /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public override Task <DisconnectResponse> Disconnect(DisconnectRequest request, ServerCallContext context) { // clear connection _server.Connected = false; _server.Settings = null; // alert connection session to close if (_tcs != null) { _tcs.SetResult(true); _tcs = null; } Logger.Info("Disconnected"); return(Task.FromResult(new DisconnectResponse())); }
/// <summary> /// Configures the plugin /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public override Task <ConfigureResponse> Configure(ConfigureRequest request, ServerCallContext context) { Logger.Debug("Got configure request"); Logger.Debug(JsonConvert.SerializeObject(request, Formatting.Indented)); // ensure all directories are created Directory.CreateDirectory(request.TemporaryDirectory); Directory.CreateDirectory(request.PermanentDirectory); Directory.CreateDirectory(request.LogDirectory); // configure logger Logger.SetLogLevel(request.LogLevel); Logger.Init(request.LogDirectory); _server.Config = request; return(Task.FromResult(new ConfigureResponse())); }
public override async Task ConnectSession(ConnectRequest request, IServerStreamWriter <ConnectResponse> responseStream, ServerCallContext context) { Logger.SetLogPrefix("connect_session"); Logger.Info("Connecting session..."); // create task to wait for disconnect to be called _tcs?.SetResult(true); _tcs = new TaskCompletionSource <bool>(); // call connect method var response = await Connect(request, context); await responseStream.WriteAsync(response); Logger.Info("Session connected."); // wait for disconnect to be called await _tcs.Task; }
/// <summary> /// Prepares writeback settings to write to Cassandra CQL3 /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public override async Task <PrepareWriteResponse> PrepareWrite(PrepareWriteRequest request, ServerCallContext context) { // Logger.SetLogLevel(Logger.LogLevel.Debug); Logger.SetLogPrefix(request.DataVersions.JobId); Logger.Info("Preparing write..."); _server.WriteConfigured = false; _server.WriteSettings = new WriteSettings { CommitSLA = request.CommitSlaSeconds, Schema = request.Schema, Replication = request.Replication, DataVersions = request.DataVersions, }; if (_server.WriteSettings.IsReplication()) { // reconcile job Logger.Info($"Starting to reconcile Replication Job {request.DataVersions.JobId}"); try { await Replication.ReconcileReplicationJobAsync(_sessionFactory, request); } catch (Exception e) { Logger.Error(e, e.Message, context); return(new PrepareWriteResponse()); } Logger.Info($"Finished reconciling Replication Job {request.DataVersions.JobId}"); } _server.WriteConfigured = true; Logger.Debug(JsonConvert.SerializeObject(_server.WriteSettings, Formatting.Indented)); Logger.Info("Write prepared."); return(new PrepareWriteResponse()); }
/// <summary> /// Publishes a stream of data for a given schema /// </summary> /// <param name="request"></param> /// <param name="responseStream"></param> /// <param name="context"></param> /// <returns></returns> public override async Task ReadStream(ReadRequest request, IServerStreamWriter <Record> responseStream, ServerCallContext context) { try { var schema = request.Schema; var limit = request.Limit; var limitFlag = request.Limit != 0; var jobId = request.JobId; var recordsCount = 0; Logger.SetLogPrefix(jobId); var records = Read.ReadRecords(_sessionFactory, schema); await foreach (var record in records) { // stop publishing if the limit flag is enabled and the limit has been reached or the server is disconnected if (limitFlag && recordsCount == limit || !_server.Connected) { break; } // publish record await responseStream.WriteAsync(record); recordsCount++; } Logger.Info($"Published {recordsCount} records"); } catch (Exception e) { Logger.Error(e, e.Message, context); } }
/// <summary> /// Establishes a connection with Cassandra. /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns>A message indicating connection success</returns> public override async Task <ConnectResponse> Connect(ConnectRequest request, ServerCallContext context) { Logger.SetLogPrefix("connect"); // validate settings passed in try { _server.Settings = JsonConvert.DeserializeObject <Settings>(request.SettingsJson); _server.Settings.Validate(); } catch (Exception e) { Logger.Error(e, e.Message); return(new ConnectResponse { OauthStateJson = request.OauthStateJson, ConnectionError = "", OauthError = "", SettingsError = e.Message }); } // initialize connection factory try { _sessionFactory.Initialize(_server.Settings); } catch (Exception e) { Logger.Error(e, e.Message); return(new ConnectResponse { OauthStateJson = request.OauthStateJson, ConnectionError = e.Message, OauthError = "", SettingsError = "" }); } // test cluster factory //var conn = _sessionFactory.GetConnection(); var session = Cluster.Builder() .AddContactPoint(_server.Settings.Hostname).WithPort(Int32.Parse(_server.Settings.Port)) .WithCredentials(_server.Settings.Username, _server.Settings.Password) .Build(); try { session.Connect(); } catch (Exception e) { Logger.Error(e, e.Message); return(new ConnectResponse { OauthStateJson = request.OauthStateJson, ConnectionError = e.Message, OauthError = "", SettingsError = "" }); } finally { await session.ShutdownAsync(); } // try // { // await conn.OpenAsync(); // // if (!await conn.PingAsync()) // { // return new ConnectResponse // { // OauthStateJson = request.OauthStateJson, // ConnectionError = "Unable to ping target database.", // OauthError = "", // SettingsError = "" // }; // } // } // catch (Exception e) // { // Logger.Error(e, e.Message, context); // // return new ConnectResponse // { // OauthStateJson = request.OauthStateJson, // ConnectionError = e.Message, // OauthError = "", // SettingsError = "" // }; // } // finally // { // await conn.CloseAsync(); // } _server.Connected = true; return(new ConnectResponse { OauthStateJson = request.OauthStateJson, ConnectionError = "", OauthError = "", SettingsError = "" }); }
/// <summary> /// Creates a form and handles form updates for write backs /// </summary> /// <param name="request"></param> /// <param name="context"></param> /// <returns></returns> public override async Task <ConfigureWriteResponse> ConfigureWrite(ConfigureWriteRequest request, ServerCallContext context) { Logger.Info("Configuring write..."); var storedProcedures = await Write.GetAllStoredProceduresAsync(_sessionFactory); var schemaJson = Write.GetSchemaJson(storedProcedures); var uiJson = Write.GetUIJson(); // if first call if (string.IsNullOrWhiteSpace(request.Form.DataJson) || request.Form.DataJson == "{}") { return(new ConfigureWriteResponse { Form = new ConfigurationFormResponse { DataJson = "", DataErrorsJson = "", Errors = { }, SchemaJson = schemaJson, UiJson = uiJson, StateJson = "" }, Schema = null }); } try { // get form data var formData = JsonConvert.DeserializeObject <ConfigureWriteFormData>(request.Form.DataJson); var storedProcedure = storedProcedures.Find(s => s.GetId() == formData.StoredProcedure); // base schema to return var schema = await Write.GetSchemaForStoredProcedureAsync(_sessionFactory, storedProcedure); return(new ConfigureWriteResponse { Form = new ConfigurationFormResponse { DataJson = request.Form.DataJson, Errors = { }, SchemaJson = schemaJson, UiJson = uiJson, StateJson = request.Form.StateJson }, Schema = schema }); } catch (Exception e) { Logger.Error(e, e.Message, context); return(new ConfigureWriteResponse { Form = new ConfigurationFormResponse { DataJson = request.Form.DataJson, Errors = { e.Message }, SchemaJson = schemaJson, UiJson = uiJson, StateJson = request.Form.StateJson }, Schema = null }); } }
public static async IAsyncEnumerable <Record> ReadRecords(ISessionFactory sessionFactory, Schema schema) { var session = sessionFactory.GetSession(); try { var query = schema.Query; if (string.IsNullOrWhiteSpace(query)) { query = $"SELECT * FROM {schema.Id}"; } RowSet rows; try { rows = await session.Execute(query); } catch (Exception e) { Logger.Error(e, e.Message); yield break; } if (rows != null) { foreach (var row in rows) { var recordMap = new Dictionary <string, object>(); foreach (var property in schema.Properties) { try { switch (property.Type) { case PropertyType.String: case PropertyType.Text: case PropertyType.Decimal: recordMap[property.Id] = row[property.Id].ToString(); break; default: recordMap[property.Id] = row[property.Id]; break; } } catch (Exception e) { Logger.Error(e, $"No column or no data with property Id: {property.Id}"); Logger.Error(e, e.Message); recordMap[property.Id] = null; } } var record = new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(recordMap) }; yield return(record); } // while (await reader.ReadAsync()) // { // var recordMap = new Dictionary<string, object>(); // // foreach (var property in schema.Properties) // { // try // { // switch (property.Type) // { // case PropertyType.String: // case PropertyType.Text: // case PropertyType.Decimal: // recordMap[property.Id] = reader.GetValueById(property.Id, '`').ToString(); // break; // default: // recordMap[property.Id] = reader.GetValueById(property.Id, '`'); // break; // } // } // catch (Exception e) // { // Logger.Error(e, $"No column with property Id: {property.Id}"); // Logger.Error(e, e.Message); // recordMap[property.Id] = null; // } // } // // var record = new Record // { // Action = Record.Types.Action.Upsert, // DataJson = JsonConvert.SerializeObject(recordMap) // }; // // yield return record; // } } } finally { //Noop } }