Beispiel #1
0
        public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
        {
            if (kafkaCrawlJobData == null)
            {
                throw new ArgumentNullException(nameof(kafkaCrawlJobData));
            }
            if (client == null)
            {
                throw new ArgumentNullException(nameof(client));
            }

            _log = log ?? throw new ArgumentNullException(nameof(log));

            // TODO use info from kafkaCrawlJobData to instantiate the connection
            client.BaseUrl = new Uri(s_baseUri);
            client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString);
        }
Beispiel #2
0
        public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
        {
            if (kafkaCrawlJobData == null)
            {
                throw new ArgumentNullException(nameof(kafkaCrawlJobData));
            }

            _kafkaCrawlJobData = kafkaCrawlJobData;

            if (client == null)
            {
                throw new ArgumentNullException(nameof(client));
            }

            this.log    = log ?? throw new ArgumentNullException(nameof(log));
            this.client = client ?? throw new ArgumentNullException(nameof(client));

            // TODO use info from kafkaCrawlJobData to instantiate the connection
            client.BaseUrl = new Uri(BaseUri);
            client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString);

            if (!string.IsNullOrEmpty(_kafkaCrawlJobData.KafkaServer)) //this means that Kafka stream is not configured, so we should use the API. If it's not empty/null, then it's configured so we should use it.
            {
                //Configure the consumer
                _consumerConfig = new ConsumerConfig
                {
                    BootstrapServers = _kafkaCrawlJobData.KafkaServer,

                    SecurityProtocol = SecurityProtocol.SaslSsl,
                    SaslMechanism    = SaslMechanism.Plain,
                    SslCaLocation    = @".\cacert.pem",
                    SocketTimeoutMs  = 60000,
                    SessionTimeoutMs = 30000,
                    SaslUsername     = @"$ConnectionString",

                    SaslPassword = _kafkaCrawlJobData.KafkaConnectionString,

                    GroupId         = @"$Default",
                    AutoOffsetReset = AutoOffsetReset.Earliest
                };
            }
        }
Beispiel #3
0
        public override async Task <CrawlJobData> GetCrawlJobData(
            ProviderUpdateContext context,
            IDictionary <string, object> configuration,
            Guid organizationId,
            Guid userId,
            Guid providerDefinitionId)
        {
            if (configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            var kafkaCrawlJobData = new KafkaCrawlJobData();

            if (configuration.ContainsKey(KafkaConstants.KeyName.ApiKey))
            {
                kafkaCrawlJobData.ApiKey = configuration[KafkaConstants.KeyName.ApiKey].ToString();
            }

            return(await Task.FromResult(kafkaCrawlJobData));
        }