public async Task <ConfluentResponse <List <BinaryMessage> > > ConsumeAsBinaryAsync(
ConsumerInstance consumerInstance,
string topic,
CancellationToken cancellationToken)
{
return(await ConsumeAsBinaryAsync(consumerInstance, topic, null, cancellationToken).ConfigureAwait(false));
}
public async Task <ConfluentResponse <List <BinaryMessage> > > ConsumeAsBinaryAsync(
ConsumerInstance consumerInstance,
string topic,
int maxBytes)
{
return(await ConsumeAsBinaryAsync(consumerInstance, topic, maxBytes, CancellationToken.None).ConfigureAwait(false));
}
public async Task <ConfluentResponse <List <AvroMessage <TKey, TValue> > > > ConsumeAsAvroAsync <TKey, TValue>(
ConsumerInstance consumerInstance,
string topic)
where TKey : class
where TValue : class
{
return(await ConsumeAsAvroAsync <TKey, TValue>(consumerInstance, topic, null, CancellationToken.None).ConfigureAwait(false));
}
/// <summary>
/// This assumes that a topic has only one type. Jay Kreps recommends separating different event types
/// into different topics at http://www.confluent.io/blog/stream-data-platform-2/. Besides, heterogeneous-event
/// topics aren't directly supported by the driver.
/// </summary>
/// <returns></returns>
private static async Task <ConfluentResponse <List <AvroMessage <TK, TV> > > > ConsumeOnceAsync(
IConfluentClient confluentClient,
ConsumerInstance consumerInstance,
string topic)
{
return(await confluentClient.ConsumeAsAvroAsync <TK, TV>(consumerInstance, topic));
}
/// <summary>
/// At the moment this just listens to the stream
/// </summary>
//[Test]
//[Category("Integration")]
public void KafkaObservable_Should_Provide_An_Observable_Stream()
{
// Arrange
String instanceId = "0";
string consumerGroupName = "rxtestgroup";
string topic = "rxtest";
ConsumerInstance consumerInstance = null;
using (var client = Setup.CreateConfluentClient("http://199.204.211.42:8082"))
//using (var client = Setup.CreateConfluentClient("http://192.168.79.137:8082"))
{
try
{
// in production this should be written without blocking.
consumerInstance = Setup.ConsumerInstance(client, instanceId, consumerGroupName);
var consumer = new RxConsumer <String, RxConsumerTests.LogMessage>(client, consumerInstance, topic);
// Act
var observable = consumer.GetRecordStream(
TimeSpan.FromSeconds(5), ThreadPoolScheduler.Instance)
.Take(10)
.Subscribe(successResult =>
{
Console.WriteLine("Success: " + successResult.IsSuccess);
if (successResult.IsSuccess)
{
Console.WriteLine(successResult.Value.Key + "=" + successResult.Value.Value.Message);
}
else
{
Console.WriteLine("ERROR: " + successResult.Exception.Message);
}
},
() => Console.WriteLine("COMPLETED.") // not sure how to cancel this...
);
Thread.Sleep(15000);
Console.WriteLine("Disposing observer");
observable.Dispose();
}
finally
{
if (consumerInstance != null)
{
Console.WriteLine("Destroying instance");
client.DeleteConsumerAsync(consumerInstance);
}
}
}
// Assert
//Assert.Fail("Not Implemented Yet");
}
private async Task <ConfluentResponse <List <BinaryMessage> > > ConsumeAsBinaryAsync(
ConsumerInstance consumerInstance,
string topic,
int?maxBytes,
CancellationToken cancellationToken)
{
string requestUri = BuildConsumeRequestUri(topic, maxBytes);
HttpRequestMessage request = CreateRequestMessage(HttpMethod.Get, requestUri, consumerInstance.BaseUri)
.WithContentType(ContentTypeKafkaBinary)
.WithHostHeader(consumerInstance.BaseUri);
return(await ProcessRequest <List <BinaryMessage> >(request, cancellationToken).ConfigureAwait(false));
}
public async Task <ConfluentResponse> DeleteConsumerAsync(ConsumerInstance consumerInstance, CancellationToken cancellationToken)
{
HttpRequestMessage request = CreateRequestMessage(HttpMethod.Delete, "", consumerInstance.BaseUri)
.WithContentType(ContentTypeKafkaDefault)
.WithHostHeader(consumerInstance.BaseUri);
var response = await SendRequest(request, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return(ConfluentResponse.Success());
}
return(ConfluentResponse.Failed(await ReadResponseAs <Error>(response).ConfigureAwait(false)));
}
private async Task <ConfluentResponse <List <AvroMessage <TKey, TValue> > > > ConsumeAsAvroAsync <TKey, TValue>(
ConsumerInstance consumerInstance,
string topic,
int?maxBytes,
CancellationToken cancellationToken)
where TKey : class
where TValue : class
{
string requestUri = BuildConsumeRequestUri(topic, maxBytes);
HttpRequestMessage request = CreateRequestMessage(HttpMethod.Get, requestUri, consumerInstance.BaseUri)
.WithContentType(ContentTypeKafkaAvro)
.WithHostHeader(consumerInstance.BaseUri);
return(await ProcessRequest <List <AvroMessage <TKey, TValue> > >(request, cancellationToken).ConfigureAwait(false));
}
public async Task <ConfluentResponse> DeleteConsumerAsync(ConsumerInstance consumerInstance)
{
return(await DeleteConsumerAsync(consumerInstance, CancellationToken.None).ConfigureAwait(false));
}
public async Task <ConfluentResponse <List <ConsumerOffset> > > CommitOffsetAsync(ConsumerInstance consumerInstance, CancellationToken cancellationToken)
{
HttpRequestMessage request = CreateRequestMessage(HttpMethod.Post, "/offsets", consumerInstance.BaseUri)
.WithContentType(ContentTypeKafkaDefault)
.WithHostHeader(consumerInstance.BaseUri);
return(await ProcessRequest <List <ConsumerOffset> >(request, cancellationToken).ConfigureAwait(false));
}
public async Task <ConfluentResponse <List <ConsumerOffset> > > CommitOffsetAsync(ConsumerInstance consumerInstance)
{
return(await CommitOffsetAsync(consumerInstance, CancellationToken.None).ConfigureAwait(false));
}
//private readonly OffsetCommitter _committer;
public RxConsumer(IConfluentClient client, ConsumerInstance consumerInstance, String topic)
{
_client = client;
_consumerInstance = consumerInstance;
_topic = topic;
}
private static uint InitializeConsumers(List <string> AssemblyPaths, bool IgnoreBadFormat)
{
if (AssemblyPaths == null || AssemblyPaths.Count == 0)
{
return(0x2U); // ERROR_FILE_NOT_FOUND
}
var hostAssemblyPath = typeof(CalloutNative).Assembly.Location;
var proxyTypeName = typeof(DhcpServerCalloutConsumerProxy).FullName;
var instances = new List <ConsumerInstance>(AssemblyPaths.Count);
try
{
foreach (var assemblyPath in AssemblyPaths)
{
var domainSetup = new AppDomainSetup()
{
ApplicationBase = Path.GetDirectoryName(assemblyPath),
LoaderOptimization = LoaderOptimization.MultiDomain
};
var instance = new ConsumerInstance()
{
AssemblyPath = assemblyPath
};
instances.Add(instance);
instance.AppDomain = AppDomain.CreateDomain($"DhcpServerCalloutConsumer: {Path.GetFileName(assemblyPath)}", null, domainSetup);
var proxy = (DhcpServerCalloutConsumerProxy)instance.AppDomain.CreateInstanceFromAndUnwrap(hostAssemblyPath, proxyTypeName);
var initResult = proxy.TryInitialize(assemblyPath, ApiVersion);
if (IgnoreBadFormat && initResult == 0xBU) // ERROR_BAD_FORMAT
{
// skip assembly
instances.Remove(instance);
AppDomain.Unload(instance.AppDomain);
continue;
}
if (initResult != ERROR_SUCCESS)
{
throw new Win32Exception((int)initResult);
}
instance.Proxy = proxy;
instance.SupportFlags = proxy.GetSupportFlags();
}
}
catch (Exception)
{
// dispose and unload previously created instances
foreach (var instance in instances)
{
try
{
instance.Proxy?.Dispose();
instance.Proxy = null;
}
catch (Exception) { }
if (instance.AppDomain != null)
{
try
{
AppDomain.Unload(instance.AppDomain);
instance.AppDomain = null;
}
catch (Exception) { }
}
}
throw;
}
if (instances.Count == 0)
{
return(0x2U); // ERROR_FILE_NOT_FOUND
}
Consumers = instances.ToArray();
ConsumerRequirements = instances.Aggregate(
(CalloutConsumerSupportFlags)0, (a, i) => a | i.SupportFlags);
return(ERROR_SUCCESS);
}
private static void Listen(Options options)
{
// Arrange
String instanceId = options.InstanceId;
string consumerGroupName = options.ConsumerGroup;
string topic = options.Topic;
ConsumerInstance consumerInstance = null;
Log("Registering group/id: " + options.ConsumerGroup + "/" + options.InstanceId + "\r\n");
Log("Listening to topic: " + topic + "\r\n");
Log("Query interval: " + options.Sleep + "ms" + "\r\n");
using (var client = Setup.CreateConfluentClient(options.BaseUrl))
{
try
{
// in production this should be written without blocking.
consumerInstance = Setup.ConsumerInstance(client, instanceId, consumerGroupName);
var consumer = new RxConsumer <String, LogMessage>(client, consumerInstance, topic);
// Act
var observable = consumer.GetRecordStream(
TimeSpan.FromMilliseconds(options.Sleep),
ThreadPoolScheduler.Instance,
beforeCallAction: () => Log("."))
.Subscribe(
// OnSuccess
successResult =>
{
Log("Success: " + successResult.IsSuccess + "\r\n");
if (successResult.IsSuccess)
{
System.Console.WriteLine(successResult.Value.Key + "=" +
successResult.Value.Value.Message);
}
else
{
System.Console.WriteLine("ERROR: " + successResult.Exception.Message);
}
},
// OnCompleted
() => System.Console.WriteLine("COMPLETED.") // not sure how to cancel this...
);
System.Console.ReadLine();
System.Console.WriteLine("Disposing observer");
observable.Dispose();
}
catch (Exception ex)
{
System.Console.WriteLine(ex.Message);
if (ex.InnerException != null)
{
System.Console.WriteLine(ex.InnerException.Message);
}
}
finally
{
if (consumerInstance != null)
{
Log("Destroying Consumer Instance\r\n");
client.DeleteConsumerAsync(consumerInstance);
Log("Iterations: " + _iterations);
}
}
}
}
