public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(SqlAzureWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); //Component tasks expect output field names in TopologyBuilder topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields} }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(SqlAzureBolt).Name, //Set task name SqlAzureBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDBWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); //Store the incoming Vehicle records in DocumentDb topologyBuilder.SetBolt( typeof(DocumentDbBolt).Name, //Set task name DocumentDbBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Choose grouping //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
/// <summary> /// Defines and configures the topology /// </summary> /// <returns></returns> public ITopologyBuilder GetTopologyBuilder() { //Define a new topology named "EventTopology" TopologyBuilder topologyBuilder = new TopologyBuilder("EventTopology"); //The spout, which emits events topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"id", "event", "eventtime"}} }, 1); //A bolt that is used to look up previous events for the same session from HBase //If this is an end event, the time span between the start and end events is //emitted as 'duration' topologyBuilder.SetBolt( "Lookup", HBaseLookupBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"id", "event", "eventtime", "duration"}} }, 1).shuffleGrouping("Spout"); //A bolt that writes events to HBase topologyBuilder.SetBolt( "Store", HBaseBolt.Get, new Dictionary<string, List<string>>(), 1).shuffleGrouping("Lookup"); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForHBase).Name, //Set task name VehicleRecordGeneratorSpoutForHBase.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(HBaseLookupBolt).Name, //Set task name HBaseLookupBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForHBase).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TwitterTopology).Name + DateTime.Now.ToString("-yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(TwitterSpout).Name, TwitterSpout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"tweet"}} }, 1); topologyBuilder.SetBolt( typeof(TwitterBolt).Name, TwitterBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"count", "tweet"}} }, 1).shuffleGrouping(typeof(TwitterSpout).Name); topologyBuilder.SetBolt( typeof(SqlAzureBolt).Name, SqlAzureBolt.Get, new Dictionary<string, List<string>>(), 1).shuffleGrouping(typeof(TwitterBolt).Name); topologyBuilder.SetBolt( typeof(SignalRBroadcastBolt).Name, SignalRBroadcastBolt.Get, new Dictionary<string, List<string>>(), 1).shuffleGrouping(typeof(TwitterBolt).Name); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDBLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, true); topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name, //Set task name DocumentDbLookupBolt.Get, //Set task constructor delegate //Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, true). globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); // The java bolt "java_displayer" receives from the C# spout "generator" topologyBuilder.SetJavaBolt( "java_displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // The C# bolt "csharp-displayer" receive from the C# spout "generator" topologyBuilder.SetBolt( "csharp-displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TestEventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } } }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(GlobalCountBolt).Name, GlobalCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "timestamp", "totalCount" } } }, 1, true). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); // Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize // Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //We will use CreateFromClojureExpr method as we wish to pass in a complex Java object //The EventHubBolt takes a EventHubBoltConfig that we will create using clojure //NOTE: We need to escape the quotes for strings that need to be passes to clojure JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubFqnAddress, appConfig.EventHubEntityPath, "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); //Assuming a 4 'L' node cluster, we will have 16 worker slots available //We will half of those slots for this topology topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.workers", appConfig.EventHubPartitions.ToString()}, {"topology.max.spout.pending","4096"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields} }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); topologyBuilder.SetBolt( typeof(EventHubBolt).Name, //Set task name EventHubBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples partitionCount, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return topologyBuilder; }
public Constructor(string fullFilePath, Topology.Topology topology) { this.fullFilePath = fullFilePath ?? throw new NullReferenceException(); if (topology == null) { throw new NullReferenceException(); } InitializeComponent(); connection = ConnectionHelpers.OpenConnection(); crudHelper = new CrudHelper(connection); SetupSettings(); topologyBuilder = new TopologyBuilder(dgvField, topology); }
public void ShouldVisitTopologyMultipleTimesWithoutExceptions() { var factory = new DefaultConnectionFactory(); using (var connection = factory.Create(new Uri("amqp://localhost/integration"))) { var model = connection.CreateModel(); var topology = new Topology(); topology.DefineQueue("test-queue"); var builder = new TopologyBuilder(model); topology.Visit(builder); topology.Visit(builder); } }
public void ShouldVisitTopologyMultipleTimesExclusiveQueue() { var factory = new DefaultConnectionFactory(); using (var connection = factory.Create(new Uri("amqp://localhost/integration"))) { var model = connection.CreateModel(); var topology = new Topology(); topology.DefineQueue(); var builder = new TopologyBuilder(model); topology.Visit(builder); topology.Visit(builder); } }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDbReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, true); topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name, //Set task name DocumentDbLookupBolt.Get, //Set task constructor delegate //Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, //Or, new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, new List<string>() { "Vehicle" } }, 1, true). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); //Log the looked up records topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, true). globalGrouping(typeof(DocumentDbLookupBolt).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(HBaseLookupBolt).Name, //Set task name HBaseLookupBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); topologyBuilder.SetBolt( typeof(HBaseLookupBolt).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(HBaseLookupBolt).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor generatorConfig = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.GeneratorConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test") }); JavaComponentConstructor generator = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Generator", new List<Tuple<string, object>>() { Tuple.Create<string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig) }); topologyBuilder.SetJavaSpout( "generator", generator, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
/// <summary> /// Use Topology Specification API to describe the topology /// </summary> /// <returns></returns> public ITopologyBuilder GetTopologyBuilder() { // Use TopologyBuilder to define a Non-Tx topology // And define each spouts/bolts one by one TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld"); // Set a User customized config (Generator.config) for the Generator topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}} }, 1, "Generator.config"); topologyBuilder.SetBolt( "splitter", Splitter.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "firstLetterOfWord"}} }, 1).shuffleGrouping("generator"); // Use scp-field-group from Splitter to Counter, // and specify the second field in the Output schema of Splitter (Input schema of Counter) as the field grouping target // by passing the index array [1] (index start from 0) topologyBuilder.SetBolt( "counter", Counter.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}} }, 1).fieldsGrouping("splitter", new List<int>() {1}); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(SimpleHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); var javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); topologyBuilder.SetBolt( typeof(ThresholdBolt).Name, ThresholdBolt.Get, new Dictionary <string, List <string> >() { }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(VehicleRecordGeneratorSpout).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "5"); topologyBuilder.SetBolt( typeof(HBaseBolt).Name, //Set task name HBaseBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name). addConfigurations(boltConfig); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { // Start building a new topology TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology. Set parallelism hint to the number of partitions topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Create a config for the bolt. It's unused here var boltConfig = new StormConfig(); // Add the logbolt to the topology topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary<string, List<string>>(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"); // Create a configuration for the topology var topologyConfig = new StormConfig(); // Increase max pending for the spout topologyConfig.setMaxSpoutPending(8192); // Parallelism hint for the number of workers to match the number of EventHub partitions topologyConfig.setNumWorkers(eventHubPartitions); // Add the config and return the topology builder topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubsWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields } }, partitionCount / 2, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(EventHubBolt).Name, //Set task name EventHubBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples partitionCount, //Set number of tasks true //Set enableAck ). shuffleGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(partitionCount / 2); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List <object> constructorParams = new List <object>() { 100, "test", null }; List <string> paramTypes = new List <string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
/// <summary> /// Use Topology Specification API to describe the topology /// </summary> /// <returns></returns> public ITopologyBuilder GetTopologyBuilder() { // Use TopologyBuilder to define a Non-Tx topology // And define each spouts/bolts one by one TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorldHostModeMultiSpout"); // Set a User customized config (SentenceGenerator.config) for the SentenceGenerator topologyBuilder.SetSpout( "SentenceGenerator", SentenceGenerator.Get, new Dictionary<string, List<string>>() { {SentenceGenerator.STREAM_ID, new List<string>(){"sentence"}} }, 1, "SentenceGenerator.config"); topologyBuilder.SetSpout( "PersonGenerator", PersonGenerator.Get, new Dictionary<string, List<string>>() { {PersonGenerator.STREAM_ID, new List<string>(){"person"}} }, 1); topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1) .shuffleGrouping("SentenceGenerator", SentenceGenerator.STREAM_ID) .shuffleGrouping("PersonGenerator", PersonGenerator.STREAM_ID); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { string pluginName = string.Format("{0}.exe", GetType().Assembly.GetName().Name); TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld"); topologyBuilder.SetSpout( "generator", pluginName, new List<string>() { "generator" }, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}} }, 1); topologyBuilder.SetBolt( "splitter", pluginName, new List<string>() { "splitter" }, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "firstLetterOfWord"}} }, 1).shuffleGrouping("generator"); topologyBuilder.SetBolt( "counter", pluginName, new List<string>() { "counter" }, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}} }, 1).fieldsGrouping("splitter", new List<int>() { 1 }); topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public TopologyBuilderService(StatelessServiceContext context) : base(context) { this.baseLogString = $"{this.GetType()} [{this.GetHashCode()}] =>{Environment.NewLine}"; Logger.LogDebug($"{baseLogString} Ctor => Logger initialized"); try { this.topologyBuilder = new TopologyBuilder(); string infoMessage = $"{baseLogString} Ctor => Contract providers initialized."; Logger.LogInformation(infoMessage); ServiceEventSource.Current.ServiceMessage(this.Context, $"[TopologyBuilderService | Information] {infoMessage}"); } catch (Exception e) { string errorMessage = $"{baseLogString} Ctor => exception {e.Message}"; Logger.LogError(errorMessage, e); ServiceEventSource.Current.ServiceMessage(this.Context, $"[TopologyBuilderService | Error] {errorMessage}"); } }
/// <summary> /// Get a Storm topology builder /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubReader" + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods. // NOTE: These methods set the spout to read data in a String encoding. topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Add the logbolt to the topology // Use a serializer to understand data from the Java component topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary <string, List <string> >(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); return(topologyBuilder); }
/// <summary> /// Defines and configures the topology /// </summary> /// <returns></returns> public ITopologyBuilder GetTopologyBuilder() { //Define a new topology named "EventTopology" TopologyBuilder topologyBuilder = new TopologyBuilder("EventTopology"); //The spout, which emits events topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "id", "event", "eventtime" } } }, 1); //A bolt that is used to look up previous events for the same session from HBase //If this is an end event, the time span between the start and end events is //emitted as 'duration' topologyBuilder.SetBolt( "Lookup", HBaseLookupBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "id", "event", "eventtime", "duration" } } }, 1).shuffleGrouping("Spout"); //A bolt that writes events to HBase topologyBuilder.SetBolt( "Store", HBaseBolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping("Lookup"); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TwitterTopology).Name + DateTime.Now.ToString("-yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(TwitterSpout).Name, TwitterSpout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "tweet" } } }, 1); topologyBuilder.SetBolt( typeof(TwitterBolt).Name, TwitterBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "count", "tweet" } } }, 1).shuffleGrouping(typeof(TwitterSpout).Name); topologyBuilder.SetBolt( typeof(SqlAzureBolt).Name, SqlAzureBolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping(typeof(TwitterBolt).Name); topologyBuilder.SetBolt( typeof(SignalRBroadcastBolt).Name, SignalRBroadcastBolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping(typeof(TwitterBolt).Name); return(topologyBuilder); }
/// <summary> /// build delivery queues for each endpoint, where each delivery queue /// will correspond to a basic consumer /// </summary> private IEnumerable <IDeliveryQueue> BuildDeliveryQueues( IEnumerable <string> endpointNames, IModelReference modelReference, string messageType, IEndpointConfiguration configuration, ISubscriptionFailureStrategy subscriptionFailureStrategy) { return(endpointNames.Select((endpointName, i) => { var endpoint = new Endpoint(endpointName, messageType, i); var topologyBuilder = new TopologyBuilder(modelReference); var topology = configuration.BuildTopology(endpoint); topology.Visit(topologyBuilder); var subscribeQueue = topology.SubscribeQueue; return new DeliveryQueue( subscribeQueue, modelReference, subscriptionFailureStrategy); })); }
public IPublisher <TMessage> Create <TMessage>( IModelReference modelReference, IPublisherConfiguration <TMessage> configuration) { var messageType = typeof(TMessage).Name; var endpoint = new Endpoint(configuration.EndpointName ?? messageType, messageType, 0); var topology = configuration.BuildTopology(endpoint); if (configuration.ShouldBuildTopology) { var topologyBuilder = new TopologyBuilder(modelReference); topology.Visit(topologyBuilder); } var router = configuration.BuildRouter(); var faultStrategy = configuration.BuildFaultStrategy(); var messageSerializer = messageSerializers.FindOrDefault( configuration.ContentType); var publisher = configuration.ShouldConfirm ? new ConfirmingPublisher <TMessage>( modelReference, messageSerializer, topology.PublishExchange, router, faultStrategy) : new Publisher <TMessage>( modelReference, messageSerializer, topology.PublishExchange, router); publisher.Start(); Track(publisher); return(publisher); }
/// <summary> /// The resolve for. /// </summary> /// <param name="configuration"> /// The configuration. /// </param> /// <returns> /// The <see cref="Producer"/>. /// </returns> public Producer ResolveFor(ISenderConfiguration configuration) { Producer producer = this.TryResolverFor(configuration.Label); if (producer == null) { using (RabbitChannel channel = this._bus.OpenChannel()) { var topologyBuilder = new TopologyBuilder(channel); var builder = new RouteResolverBuilder(this._bus.Endpoint, topologyBuilder, configuration); Maybe <Func <IRouteResolverBuilder, IRouteResolver> > routeResolverBuilder = configuration.Options.GetRouteResolverBuilder(); Assumes.True(routeResolverBuilder.HasValue, "RouteResolverBuilder must be set for [{0}]", configuration.Label); IRouteResolver routeResolver = routeResolverBuilder.Value(builder); producer = new Producer(this._bus, configuration.Label, routeResolver, configuration.Options.IsConfirmationRequired()); producer.Failed += p => { { // lock (_producers) p.Dispose(); this._producers.Remove(p.Label); } }; // lock (_producers) this._producers.Add(configuration.Label, producer); } if (configuration.RequiresCallback) { producer.UseCallbackListener(this._bus.ListenerRegistry.ResolveFor(configuration.CallbackConfiguration)); } } return(producer); }
public Constructor(string fullFilePath, int cols, int rows) { this.fullFilePath = fullFilePath ?? throw new NullReferenceException(); if (cols < Topology.Topology.MinColsCount || cols > Topology.Topology.MaxColsCount) { throw new ArgumentOutOfRangeException(); } if (rows < Topology.Topology.MinRowsCount || rows > Topology.Topology.MaxRowsCount) { throw new ArgumentOutOfRangeException(); } InitializeComponent(); connection = ConnectionHelpers.OpenConnection(); crudHelper = new CrudHelper(connection); SetupSettings(); topologyBuilder = new TopologyBuilder(dgvField, cols, rows);; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("SensorStream" + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "count" } } }, 1); topologyBuilder.SetBolt( "Bolt", Bolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping("Spout"); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create <string, object>("int", 10), Tuple.Create <string, object>("int", 1024), Tuple.Create <string, object>("int", 1024 * eventHubPartitions), Tuple.Create <string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary <string, List <string> >(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024 * 1024) / 100); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Create a new topology named 'WordCount' TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount"); // Add the spout to the topology. // Name the component 'sentences' // Name the field that is emitted as 'sentence' topologyBuilder.SetSpout( "sentences", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sentence" } } }, 1); // Add the splitter bolt to the topology. // Name the component 'splitter' // Name the field that is emitted 'word' // Use suffleGrouping to distribute incoming tuples // from the 'sentences' spout across instances // of the splitter topologyBuilder.SetBolt( "splitter", Splitter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word" } } }, 1).shuffleGrouping("sentences"); // Add the counter bolt to the topology. // Name the component 'counter' // Name the fields that are emitted 'word' and 'count' // Use fieldsGrouping to ensure that tuples are routed // to counter instances based on the contents of field // position 0 (the word). This could also have been // List<string>(){"word"}. // This ensures that the word 'jumped', for example, will always // go to the same instance topologyBuilder.SetBolt( "counter", Counter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word", "count" } } }, 1).fieldsGrouping("splitter", new List <int>() { 0 }); //NOTE: This tends to work with parallelism hint set to 1. //If you increase this, you should pre-create the dataset //used by the bolt, as multiple instances of the bolt all //trying to create a new dataset will sometimes create //multiple datasets in Power BI, all with the same name. topologyBuilder.SetBolt( "PowerBI", PowerBiBolt.Get, new Dictionary <string, List <string> >(), 1).fieldsGrouping("counter", new List <int>() { 0 }); // Add topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Create a new topology named 'WordCount' TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount"); // Add the spout to the topology. // Name the component 'sentences' // Name the field that is emitted as 'sentence' topologyBuilder.SetSpout( "sentences", Spout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}} }, 1); // Add the splitter bolt to the topology. // Name the component 'splitter' // Name the field that is emitted 'word' // Use suffleGrouping to distribute incoming tuples // from the 'sentences' spout across instances // of the splitter topologyBuilder.SetBolt( "splitter", Splitter.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word"}}, // a default stream with a named tuple field {"cowbells", new List<string>(){"cowbell"}} // a named stream with a named tuple field }, 1).shuffleGrouping("sentences"); // Add the counter bolt to the topology. // Name the component 'counter' // Name the fields that are emitted 'word' and 'count' // Use fieldsGrouping to ensure that tuples are routed // to counter instances based on the contents of field // position 0 (the word). This could also have been // List<string>(){"word"}. // This ensures that the word 'jumped', for example, will always // go to the same instance topologyBuilder.SetBolt( "counter", Counter.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}} }, 1).fieldsGrouping("splitter", Constants.DEFAULT_STREAM_ID, new List<int>() { 0 }); // Add the cowbell bolt, which reads from the cowbell stream topologyBuilder.SetBolt( "cowbellbolt", CowBell.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"cowbellout"}} // emit a stream so we can see the cowbell }, 1).shuffleGrouping("splitter", "cowbells"); //subscribe to the stream named 'cowbell' // Add topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
private Producer BuildProducer(string url) { var reuseConnectionProperty = this.senderOptions.GetReuseConnection(); var reuseConnection = reuseConnectionProperty.HasValue && reuseConnectionProperty.Value; var source = new CancellationTokenSource(); var connection = this.connectionPool.Get(url, reuseConnection, source.Token); this.logger.Trace($"Using connection [{connection.Id}] at URL=[{url}] to resolve a producer"); using (var topologyBuilder = new TopologyBuilder(connection)) { var builder = new RouteResolverBuilder(this.bus.Endpoint, topologyBuilder, this.Configuration); var routeResolverBuilderFunc = this.Configuration.Options.GetRouteResolverBuilder(); Assumes.True( routeResolverBuilderFunc.HasValue, "RouteResolverBuilder must be set for [{0}]", this.Configuration.Label); var routeResolver = routeResolverBuilderFunc.Value(builder); var producer = new Producer( this.bus.Endpoint, connection, this.Configuration.Label, routeResolver, this.Configuration.Options.IsConfirmationRequired()); if (this.Configuration.RequiresCallback) { var callbackConfiguration = this.CreateCallbackReceiverConfiguration(url); var receiver = this.bus.RegisterReceiver(callbackConfiguration, true); this.logger.Trace( $"A sender of [{this.Configuration.Label}] requires a callback configuration; registering a receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}]"); this.logger.Trace( $"A new callback receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}] has been successfully registered, getting one of its listeners with URL=[{producer.BrokerUrl}]..."); var listener = receiver.GetListener(l => l.BrokerUrl == producer.BrokerUrl); if (listener == null) { throw new BusConfigurationException( $"Unable to find a suitable listener for receiver {receiver}"); } this.logger.Trace( $"A listener at URL=[{listener.BrokerUrl}] belonging to callback receiver of [{callbackConfiguration.Label}] acquired"); listener.StopOnChannelShutdown = true; producer.UseCallbackListener(listener); this.logger.Trace( $"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has registered a callback listener successfully"); } producer.StopOnChannelShutdown = true; producer.Stopped += (sender, args) => { this.OnProducerStopped(url, sender, args); }; return(producer); } }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create<string, object>("int", eventHubPartitions), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create<string, object>("int", 10), Tuple.Create<string, object>("int", 1024), Tuple.Create<string, object>("int", 1024*eventHubPartitions), Tuple.Create<string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List<Tuple<string, object>>() { Tuple.Create<string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary<string, List<string>>(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024*1024)/100); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var EventHubNamespace = ConfigurationManager.AppSettings["EventHubNamespace"]; if (String.IsNullOrWhiteSpace(EventHubNamespace)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubNamespace"); } var EventHubEntityPath = ConfigurationManager.AppSettings["EventHubEntityPath"]; if (String.IsNullOrWhiteSpace(EventHubEntityPath)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubEntityPath"); } var EventHubSharedAccessKeyName = ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKeyName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKeyName"); } var EventHubSharedAccessKey = ConfigurationManager.AppSettings["EventHubSharedAccessKey"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKey)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKey"); } var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); //You can use the new SetEventHubSpout method by providing EventHubSpoutConfig which will automatically create Java code to instantiate this spout //TODO: This method will not work if you do not include EventHub jar during publishing or deployment of this topology topologyBuilder.SetEventHubSpout( "EventHubSpout", //Set task name new EventHubSpoutConfig( EventHubSharedAccessKeyName, EventHubSharedAccessKey, EventHubNamespace, EventHubEntityPath, partitionCount), partitionCount ); //For a hybrid topology we need to declare a customize Java serializer that will serialize Java objects which will deseriailized in the bolt into C# objects topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). DeclareCustomizedJavaSerializer(new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" } ). globalGrouping("EventHubSpout"); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(4); //Set number of worker processes topologyConfig.setMaxSpoutPending(1024); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx1024m"); //Set Java Heap Size return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions), appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* topologyBuilder.SetBolt( typeof(EventRanker).Name, EventRanker.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions / 2 ). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1000); topologyConfig.setWorkerChildOps("-Xmx1024m"); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
/// <summary> /// Get a topology builder for this topology /// </summary> /// <returns></returns> public ITopologyBuilder GetTopologyBuilder() { // Create a new topology, with a name of "WordCount" and the current date TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount" + DateTime.Now.ToString("yyyyMMddHHmmss")); // Add the spout to the topology. // Name the component 'sentences' // Name the field that is emitted as 'sentence' topologyBuilder.SetSpout( "sentences", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sentence" } } }, 1); // Add the splitter bolt. // Name the component 'splitter' // Name the field that is emitted 'word' // Use shuffleGrouping to distribute incoming tuples // from the 'sentences' spout across instances of // the splitter topologyBuilder.SetBolt( "splitter", Splitter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word" } } }, 1).shuffleGrouping("sentences"); // Add the counter bolt. // Name the component 'counter'. // Name the fields that are emitted 'word' and 'count' // Use fieldsGrouping to ensure that tuples are routed // to counter instances based on the contents of field // position 0 ('word'). This could also be List<string>(){"word"} // Instead of position. // This ensures that words go to the same instance. For example, 'jumped' // will always go to the same instance, while 'cow' might always go to another instance. topologyBuilder.SetBolt( "counter", Counter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word", "count" } } }, 1).fieldsGrouping("splitter", new List <int>() { 0 }); // Return the builder return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions ); topologyBuilder.SetBolt( typeof(EventHubWriter).Name, EventHubWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1600); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(AlarmsOnAggregatesTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; //FLATT MESSAGES topologyBuilder.SetBolt( typeof(FlattBolt).Name, FlattBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sensor" } } }, eventHubPartitions, true ).DeclareCustomizedJavaSerializer(javaSerializerInfo) .shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"); //AGGREGATE MESSAGES var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "10"); topologyBuilder.SetBolt( typeof(AggregateBolt).Name, AggregateBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "aggregated" } } }, parallelismHint: 1, enableAck: false ) .globalGrouping(typeof(FlattBolt).Name) .addConfigurations(boltConfig); //ALARMS MESSAGES topologyBuilder.SetBolt( typeof(AlarmsBolt).Name, AlarmsBolt.Get, new Dictionary <string, List <string> >() { }, parallelismHint: 4, enableAck: false ).shuffleGrouping(typeof(AggregateBolt).Name); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.spout.EventHubSpout. (com.microsoft.eventhubs.spout.EventHubSpoutConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" {4} """"))", appConfig.EventHubUsername, appConfig.EventHubPassword, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions)); topologyBuilder.SetJavaSpout( "EventHubSpout", constructor, appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* * topologyBuilder.SetBolt( * typeof(EventRanker).Name, * EventRanker.Get, * new Dictionary<string, List<string>>() * { * {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } * }, * appConfig.EventHubPartitions / 2 * ). * fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary <string, List <string> >(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List <int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", "8" }, { "topology.max.spout.pending", "1000" }, { "topology.worker.childopts", @"""-Xmx1024m""" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Start building a new topology TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods. // NOTE: These methods set the spout to read data in a String encoding. /* * topologyBuilder.SetEventHubSpout( * "EventHubSpout", * new EventHubSpoutConfig( * ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], * ConfigurationManager.AppSettings["EventHubSharedAccessKey"], * ConfigurationManager.AppSettings["EventHubNamespace"], * ConfigurationManager.AppSettings["EventHubEntityPath"], * eventHubPartitions), * eventHubPartitions); */ // The following is an example of how to create the same spout using the JavaComponentConstructor, // which allows us to use UTF-8 encoding for reads. // NOTE!!!! This only works with the 9.5 version of the Event Hub components, which are located at // https://github.com/hdinsight/hdinsight-storm-examples/blob/master/lib/eventhubs/ // Create the UTF-8 data scheme var schemeConstructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.UnicodeEventDataScheme"); // Create the EventHubSpoutConfig var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>("com.microsoft.eventhubs.spout.IEventDataScheme", schemeConstructor) } ); // Create the spout var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); // Set the spout in the topology topologyBuilder.SetJavaSpout("EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Create a config for the bolt. It's unused here var boltConfig = new StormConfig(); // Add the logbolt to the topology // Use a serializer to understand data from the Java component topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary <string, List <string> >(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); // Create a configuration for the topology var topologyConfig = new StormConfig(); // Increase max pending for the spout topologyConfig.setMaxSpoutPending(8192); // Parallelism hint for the number of workers to match the number of EventHub partitions topologyConfig.setNumWorkers(eventHubPartitions); // Add the config and return the topology builder topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List<object> constructorParams = new List<object>() { 100, "test", null }; List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"}; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"}; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List<object> constructorParams = new List<object>() { 100, "test", string.Empty }; List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); JavaComponentConstructor constructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.EventHubSpout", new List <object>() { appConfig.EventHubUsername, appConfig.EventHubPassword, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions }, new List <string>() { "java.lang.String", "java.lang.String", "java.lang.String", "java.lang.String", "int" } ); topologyBuilder.SetJavaSpout( "com.microsoft.eventhubs.spout.EventHubSpout", constructor, appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(new Dictionary <string, string>() { { "topology.tick.tuple.freq.secs", "1" } }); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary <string, List <string> >(), 1). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(new Dictionary <string, string>() { { "topology.tick.tuple.freq.secs", "1" } }); topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", appConfig.EventHubPartitions.ToString() }, { "topology.max.spout.pending", "512" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { string pluginName = string.Format("{0}.exe", GetType().Assembly.GetName().Name); TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld"); topologyBuilder.SetSpout( "generator", pluginName, new List <string>() { "generator" }, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sentence" } } }, 1); topologyBuilder.SetBolt( "splitter", pluginName, new List <string>() { "splitter" }, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word", "firstLetterOfWord" } } }, 1).shuffleGrouping("generator"); topologyBuilder.SetBolt( "counter", pluginName, new List <string>() { "counter" }, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word", "count" } } }, 1).fieldsGrouping("splitter", new List <int>() { 1 }); topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions), appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* * topologyBuilder.SetBolt( * typeof(EventRanker).Name, * EventRanker.Get, * new Dictionary<string, List<string>>() * { * {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } * }, * appConfig.EventHubPartitions / 2 * ). * fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary <string, List <string> >(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List <int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1000); topologyConfig.setWorkerChildOps("-Xmx1024m"); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("AlertTopology"); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); topologyBuilder.SetBolt( typeof(ParserBolt).Name, ParserBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "temp", "createDate", "deviceId" } } }, eventHubPartitions, enableAck: true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(EmitAlertBolt).Name, EmitAlertBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "reason", "temp", "createDate", "deviceId" } } }, eventHubPartitions, enableAck: true ). shuffleGrouping(typeof(ParserBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
/// <summary> /// Builds a set of producers constructing one producer for each URL in the connection string /// </summary> private void BuildProducers() { var reuseConnectionProperty = this.senderOptions.GetReuseConnection(); var reuseConnection = reuseConnectionProperty.HasValue && reuseConnectionProperty.Value; this.logger.Trace( $"Building producers of [{this.Configuration.Label}]:\r\n\t{string.Join("\r\n\t", this.senderOptions.RabbitConnectionString.Select(url => $"Producer({this.Configuration.Label}): URL\t=>\t{url}"))}"); foreach (var url in this.senderOptions.RabbitConnectionString) { var source = new CancellationTokenSource(); var connection = this.connectionPool.Get(url, reuseConnection, source.Token); this.logger.Trace($"Using connection [{connection.Id}] at URL=[{url}] to resolve a producer"); var topologyBuilder = new TopologyBuilder(connection.OpenChannel()); var builder = new RouteResolverBuilder(this.bus.Endpoint, topologyBuilder, this.Configuration); var routeResolverBuilder = this.Configuration.Options.GetRouteResolverBuilder(); Assumes.True( routeResolverBuilder.HasValue, "RouteResolverBuilder must be set for [{0}]", this.Configuration.Label); var routeResolver = routeResolverBuilder.Value(builder); var producer = new Producer( this.bus.Endpoint, connection, this.Configuration.Label, routeResolver, this.Configuration.Options.IsConfirmationRequired()); producer.Failed += p => { // A failed producer will not be removed from the collection cause this failure should be treated as transient this.logger.Error($"Producer [{producer}] has failed"); }; if (this.Configuration.RequiresCallback) { var callbackConfiguration = this.Configuration.CallbackConfiguration; this.logger.Trace( $"A sender of [{this.Configuration.Label}] requires a callback configuration; registering a receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}]"); var receiver = this.bus.RegisterReceiver(this.Configuration.CallbackConfiguration); this.logger.Trace( $"A new callback receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}] has been successfully registered, getting one of its listeners with URL=[{producer.BrokerUrl}]..."); var listener = receiver.GetListener(l => l.BrokerUrl == producer.BrokerUrl); if (listener == null) { throw new BusConfigurationException( $"Unable to find a suitable listener for receiver {receiver}"); } this.logger.Trace( $"A listener at URL=[{listener.BrokerUrl}] belonging to callback receiver of [{callbackConfiguration.Label}] acquired"); producer.UseCallbackListener(listener); this.logger.Trace( $"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has registered a callback listener successfully"); } this.producers.Add(producer); this.logger.Trace( $"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has been added to the sender"); } }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(GlobalCountBolt).Name, GlobalCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "timestamp", "totalCount" } } }, 1, true). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Create a new topology named 'WordCount' TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount" + DateTime.Now.ToString("yyyyMMddHHmmss")); // Add the spout to the topology. // Name the component 'sentences' // Name the field that is emitted as 'sentence' topologyBuilder.SetSpout( "sentences", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sentence" } } }, 1); // Add the splitter bolt to the topology. // Name the component 'splitter' // Name the field that is emitted 'word' // Use suffleGrouping to distribute incoming tuples // from the 'sentences' spout across instances // of the splitter topologyBuilder.SetBolt( "splitter", Splitter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word" } } }, 1).shuffleGrouping("sentences"); // Add the counter bolt to the topology. // Name the component 'counter' // Name the fields that are emitted 'word' and 'count' // Use fieldsGrouping to ensure that tuples are routed // to counter instances based on the contents of field // position 0 (the word). This could also have been // List<string>(){"word"}. // This ensures that the word 'jumped', for example, will always // go to the same instance topologyBuilder.SetBolt( "counter", Counter.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "word", "count" } } }, 1).fieldsGrouping("splitter", new List <int>() { 0 }); // Add topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }