static async Task Main(string[] args)
{
try
{
var program = new Program();
var cancellationTokenSource = new CancellationTokenSource();
var streamPipelines = new IStreamPipeline[] {
//new ReadFileStreamPipeline(path: $@"blabla.jpg")
//new ReadHttpClientStreamPipeline(url: $@"https://upload.wikimedia.org/wikipedia/commons/7/7d/01-W_Feb_20_2012_0310Z.jpg")
new ReadS3StreamPipeline(bucket: "alicebobbucket", key: "01-W_Feb_20_2012_0310Z.jpg")
, new WriteFileStreamPipeline(path: $@"C:\Users\mon_m\Pictures\wikipedia\tropical_depression.jpg")
};
var tasks = Task.WhenAll(streamPipelines
.Select(streamPipeline => streamPipeline
.Stream(program.Pipe, cancellationTokenSource))
.ToArray());
await tasks;
}
catch (Exception exception)
{
Console.WriteLine($"ERROR - {exception.Message}");
}
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
public static void ReadAndLink()
{
var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string readTopicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce
const string linkTopicName = "sample_out"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_out_announce
var readAdapter = new KafkaStreamAdapter(brokerList, "readConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
var writeAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
var stream = readAdapter.OpenStreamTopic(readTopicName); // Open the topic for streaming.
using (var reader = new Reader(dependencyUri, groupName, stream)) // Create a Reader to read from the stream
{
using (var outputTopic = writeAdapter.OpenOutputTopic(linkTopicName)) // Open the output topic, where you want to link the streamed input
{
using (var writer = new Writer(dependencyUri, AtlasConfiguration, GetDataFormat(), groupName, outputTopic)) // Create a Writer for the output topic and pass
{
const string outputFeedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
using (IStreamPipeline pipeline = reader.ReadAndLinkTSamples(GetParameterIds(), Models.TraceSamples, writer, outputFeedName)) // TelemetryDataHandler parameter can be used to handle the data read from the stream.
{
Thread.Sleep(5000); // NOTE: without this doesn't seem to work
Write(); //Write some data to have something to read while connection is open.
}
}
}
}
}
public StreamPipelineWrapper(IDisposable client, IStreamPipeline streamPipeline)
{
StreamPipeline = streamPipeline;
this.client = client;
}
