public ITopologyBuilder GetTopologyBuilder() { TransactionalTopologyBuilder topologyBuilder = new TransactionalTopologyBuilder(typeof(HybridTopologyTx_javaSpout_csharpBolt).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Demo how to use clojure code (in string) to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr("(microsoft.scp.example.HybridTopology.TxGenerator. 100 \"test\" nil)"); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", SCPTxBolt.SCP_TX_COMMIT_BOLT, TxDisplayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); // The java bolt "java_displayer" receives from the C# spout "generator" topologyBuilder.SetJavaBolt( "java_displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // The C# bolt "csharp-displayer" receive from the C# spout "generator" topologyBuilder.SetBolt( "csharp-displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); // Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize // Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //We will use CreateFromClojureExpr method as we wish to pass in a complex Java object //The EventHubBolt takes a EventHubBoltConfig that we will create using clojure //NOTE: We need to escape the quotes for strings that need to be passes to clojure JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubFqnAddress, appConfig.EventHubEntityPath, "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); //Assuming a 4 'L' node cluster, we will have 16 worker slots available //We will half of those slots for this topology topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", "8" }, { "topology.max.spout.pending", "1600" } }); return(topologyBuilder); }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor generatorConfig = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.GeneratorConfig", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test") }); JavaComponentConstructor generator = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Generator", new List <Tuple <string, object> >() { Tuple.Create <string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig) }); topologyBuilder.SetJavaSpout( "generator", generator, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { // Friendly name of the topology TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); // Number of partitions in Event Hub. Used for parallelization. int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); // Deserializer used to deserialize JSON data from C# components to java.lang.string List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; // Spout that emits randomly generated JSON data topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "event" } } }, partitionCount) .DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Java construcvtor for the Event Hub Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", ConfigurationManager.AppSettings["EventHubName"], "true")); // Set the bolt to subscribe to data from the spout topologyBuilder.SetJavaBolt( "eventhubbolt", constructor, partitionCount) .shuffleGrouping("Spout"); // Return the topology builder return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List <object> constructorParams = new List <object>() { 100, "test", string.Empty }; List <string> paramTypes = new List <string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor generatorConfig = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.GeneratorConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test") }); JavaComponentConstructor generator = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Generator", new List<Tuple<string, object>>() { Tuple.Create<string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig) }); topologyBuilder.SetJavaSpout( "generator", generator, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List <object> constructorParams = new List <object>() { 100, "test", null }; List <string> paramTypes = new List <string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TransactionalTopologyBuilder topologyBuilder = new TransactionalTopologyBuilder("HybridTopologyTx_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", TxGenerator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to use clojure code (from a text file) to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureFile("Clojure.txt"); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Start building a new topology TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods. // NOTE: These methods set the spout to read data in a String encoding. /* * topologyBuilder.SetEventHubSpout( * "EventHubSpout", * new EventHubSpoutConfig( * ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], * ConfigurationManager.AppSettings["EventHubSharedAccessKey"], * ConfigurationManager.AppSettings["EventHubNamespace"], * ConfigurationManager.AppSettings["EventHubEntityPath"], * eventHubPartitions), * eventHubPartitions); */ // The following is an example of how to create the same spout using the JavaComponentConstructor, // which allows us to use UTF-8 encoding for reads. // NOTE!!!! This only works with the 9.5 version of the Event Hub components, which are located at // https://github.com/hdinsight/hdinsight-storm-examples/blob/master/lib/eventhubs/ // Create the UTF-8 data scheme var schemeConstructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.UnicodeEventDataScheme"); // Create the EventHubSpoutConfig var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>("com.microsoft.eventhubs.spout.IEventDataScheme", schemeConstructor) } ); // Create the spout var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); // Set the spout in the topology topologyBuilder.SetJavaSpout("EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Create a config for the bolt. It's unused here var boltConfig = new StormConfig(); // Add the logbolt to the topology // Use a serializer to understand data from the Java component topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary <string, List <string> >(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); // Create a configuration for the topology var topologyConfig = new StormConfig(); // Increase max pending for the spout topologyConfig.setMaxSpoutPending(8192); // Parallelism hint for the number of workers to match the number of EventHub partitions topologyConfig.setNumWorkers(eventHubPartitions); // Add the config and return the topology builder topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create <string, object>("int", 10), Tuple.Create <string, object>("int", 1024), Tuple.Create <string, object>("int", 1024 * eventHubPartitions), Tuple.Create <string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary <string, List <string> >(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024 * 1024) / 100); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.spout.EventHubSpout. (com.microsoft.eventhubs.spout.EventHubSpoutConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" {4} """"))", appConfig.EventHubUsername, appConfig.EventHubPassword, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions)); topologyBuilder.SetJavaSpout( "EventHubSpout", constructor, appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* * topologyBuilder.SetBolt( * typeof(EventRanker).Name, * EventRanker.Get, * new Dictionary<string, List<string>>() * { * {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } * }, * appConfig.EventHubPartitions / 2 * ). * fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary <string, List <string> >(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List <int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", "8" }, { "topology.max.spout.pending", "1000" }, { "topology.worker.childopts", @"""-Xmx1024m""" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); JavaComponentConstructor constructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.EventHubSpout", new List <object>() { appConfig.EventHubUsername, appConfig.EventHubPassword, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions }, new List <string>() { "java.lang.String", "java.lang.String", "java.lang.String", "java.lang.String", "int" } ); topologyBuilder.SetJavaSpout( "com.microsoft.eventhubs.spout.EventHubSpout", constructor, appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(new Dictionary <string, string>() { { "topology.tick.tuple.freq.secs", "1" } }); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary <string, List <string> >(), 1). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(new Dictionary <string, string>() { { "topology.tick.tuple.freq.secs", "1" } }); topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", appConfig.EventHubPartitions.ToString() }, { "topology.max.spout.pending", "512" } }); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"}; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List<object> constructorParams = new List<object>() { 100, "test", string.Empty }; List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List<object> constructorParams = new List<object>() { 100, "test", null }; List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"}; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create<string, object>("int", eventHubPartitions), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create<string, object>("int", 10), Tuple.Create<string, object>("int", 1024), Tuple.Create<string, object>("int", 1024*eventHubPartitions), Tuple.Create<string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List<Tuple<string, object>>() { Tuple.Create<string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary<string, List<string>>(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024*1024)/100); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }