public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
{
if (kafkaCrawlJobData == null)
{
throw new ArgumentNullException(nameof(kafkaCrawlJobData));
}
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
_log = log ?? throw new ArgumentNullException(nameof(log));
// TODO use info from kafkaCrawlJobData to instantiate the connection
client.BaseUrl = new Uri(s_baseUri);
client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString);
}
public KafkaClient(ILogger log, KafkaCrawlJobData kafkaCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
{
if (kafkaCrawlJobData == null)
{
throw new ArgumentNullException(nameof(kafkaCrawlJobData));
}
_kafkaCrawlJobData = kafkaCrawlJobData;
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
this.log = log ?? throw new ArgumentNullException(nameof(log));
this.client = client ?? throw new ArgumentNullException(nameof(client));
// TODO use info from kafkaCrawlJobData to instantiate the connection
client.BaseUrl = new Uri(BaseUri);
client.AddDefaultParameter("api_key", kafkaCrawlJobData.ApiKey, ParameterType.QueryString);
if (!string.IsNullOrEmpty(_kafkaCrawlJobData.KafkaServer)) //this means that Kafka stream is not configured, so we should use the API. If it's not empty/null, then it's configured so we should use it.
{
//Configure the consumer
_consumerConfig = new ConsumerConfig
{
BootstrapServers = _kafkaCrawlJobData.KafkaServer,
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanism = SaslMechanism.Plain,
SslCaLocation = @".\cacert.pem",
SocketTimeoutMs = 60000,
SessionTimeoutMs = 30000,
SaslUsername = @"$ConnectionString",
SaslPassword = _kafkaCrawlJobData.KafkaConnectionString,
GroupId = @"$Default",
AutoOffsetReset = AutoOffsetReset.Earliest
};
}
}
public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
var kafkaCrawlJobData = new KafkaCrawlJobData();
if (configuration.ContainsKey(KafkaConstants.KeyName.ApiKey))
{
kafkaCrawlJobData.ApiKey = configuration[KafkaConstants.KeyName.ApiKey].ToString();
}
return(await Task.FromResult(kafkaCrawlJobData));
}
