static async Task Main(string[] args) { try { var program = new Program(); var cancellationTokenSource = new CancellationTokenSource(); var streamPipelines = new IStreamPipeline[] { //new ReadFileStreamPipeline(path: $@"blabla.jpg") //new ReadHttpClientStreamPipeline(url: $@"https://upload.wikimedia.org/wikipedia/commons/7/7d/01-W_Feb_20_2012_0310Z.jpg") new ReadS3StreamPipeline(bucket: "alicebobbucket", key: "01-W_Feb_20_2012_0310Z.jpg") , new WriteFileStreamPipeline(path: $@"C:\Users\mon_m\Pictures\wikipedia\tropical_depression.jpg") }; var tasks = Task.WhenAll(streamPipelines .Select(streamPipeline => streamPipeline .Stream(program.Pipe, cancellationTokenSource)) .ToArray()); await tasks; } catch (Exception exception) { Console.WriteLine($"ERROR - {exception.Message}"); } Console.WriteLine("Press any key to continue..."); Console.ReadLine(); }
public static void ReadAndLink() { var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running const string groupName = "dev"; // The group name const string readTopicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce const string linkTopicName = "sample_out"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_out_announce var readAdapter = new KafkaStreamAdapter(brokerList, "readConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream. var writeAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream. var stream = readAdapter.OpenStreamTopic(readTopicName); // Open the topic for streaming. using (var reader = new Reader(dependencyUri, groupName, stream)) // Create a Reader to read from the stream { using (var outputTopic = writeAdapter.OpenOutputTopic(linkTopicName)) // Open the output topic, where you want to link the streamed input { using (var writer = new Writer(dependencyUri, AtlasConfiguration, GetDataFormat(), groupName, outputTopic)) // Create a Writer for the output topic and pass { const string outputFeedName = ""; // As sample DataFormat uses default feed, we will leave this empty. using (IStreamPipeline pipeline = reader.ReadAndLinkTSamples(GetParameterIds(), Models.TraceSamples, writer, outputFeedName)) // TelemetryDataHandler parameter can be used to handle the data read from the stream. { Thread.Sleep(5000); // NOTE: without this doesn't seem to work Write(); //Write some data to have something to read while connection is open. } } } } }
public StreamPipelineWrapper(IDisposable client, IStreamPipeline streamPipeline) { StreamPipeline = streamPipeline; this.client = client; }