public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies { if (kafkaCrawlJobData == null) { throw new ArgumentNullException(nameof(kafkaCrawlJobData)); } if (client == null) { throw new ArgumentNullException(nameof(client)); } _log = log ?? throw new ArgumentNullException(nameof(log)); // TODO use info from kafkaCrawlJobData to instantiate the connection client.BaseUrl = new Uri(s_baseUri); client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString); }
public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies { if (kafkaCrawlJobData == null) { throw new ArgumentNullException(nameof(kafkaCrawlJobData)); } _kafkaCrawlJobData = kafkaCrawlJobData; if (client == null) { throw new ArgumentNullException(nameof(client)); } this.log = log ?? throw new ArgumentNullException(nameof(log)); this.client = client ?? throw new ArgumentNullException(nameof(client)); // TODO use info from kafkaCrawlJobData to instantiate the connection client.BaseUrl = new Uri(BaseUri); client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString); if (!string.IsNullOrEmpty(_kafkaCrawlJobData.KafkaServer)) //this means that Kafka stream is not configured, so we should use the API. If it's not empty/null, then it's configured so we should use it. { //Configure the consumer _consumerConfig = new ConsumerConfig { BootstrapServers = _kafkaCrawlJobData.KafkaServer, SecurityProtocol = SecurityProtocol.SaslSsl, SaslMechanism = SaslMechanism.Plain, SslCaLocation = @".\cacert.pem", SocketTimeoutMs = 60000, SessionTimeoutMs = 30000, SaslUsername = @"$ConnectionString", SaslPassword = _kafkaCrawlJobData.KafkaConnectionString, GroupId = @"$Default", AutoOffsetReset = AutoOffsetReset.Earliest }; } }
public override async Task <CrawlJobData> GetCrawlJobData( ProviderUpdateContext context, IDictionary <string, object> configuration, Guid organizationId, Guid userId, Guid providerDefinitionId) { if (configuration == null) { throw new ArgumentNullException(nameof(configuration)); } var kafkaCrawlJobData = new KafkaCrawlJobData(); if (configuration.ContainsKey(KafkaConstants.KeyName.ApiKey)) { kafkaCrawlJobData.ApiKey = configuration[KafkaConstants.KeyName.ApiKey].ToString(); } return(await Task.FromResult(kafkaCrawlJobData)); }