public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); // The java bolt "java_displayer" receives from the C# spout "generator" topologyBuilder.SetJavaBolt( "java_displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // The C# bolt "csharp-displayer" receive from the C# spout "generator" topologyBuilder.SetBolt( "csharp-displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); // Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize // Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //We will use CreateFromClojureExpr method as we wish to pass in a complex Java object //The EventHubBolt takes a EventHubBoltConfig that we will create using clojure //NOTE: We need to escape the quotes for strings that need to be passes to clojure JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubFqnAddress, appConfig.EventHubEntityPath, "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); //Assuming a 4 'L' node cluster, we will have 16 worker slots available //We will half of those slots for this topology topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.workers", "8" }, { "topology.max.spout.pending", "1600" } }); return(topologyBuilder); }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); // Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize // Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //We will use CreateFromClojureExpr method as we wish to pass in a complex Java object //The EventHubBolt takes a EventHubBoltConfig that we will create using clojure //NOTE: We need to escape the quotes for strings that need to be passes to clojure JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubFqnAddress, appConfig.EventHubEntityPath, "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); //Assuming a 4 'L' node cluster, we will have 16 worker slots available //We will half of those slots for this topology topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.workers", appConfig.EventHubPartitions.ToString()}, {"topology.max.spout.pending","4096"} }); return topologyBuilder; }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { // Friendly name of the topology TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); // Number of partitions in Event Hub. Used for parallelization. int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); // Deserializer used to deserialize JSON data from C# components to java.lang.string List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; // Spout that emits randomly generated JSON data topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "event" } } }, partitionCount) .DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Java construcvtor for the Event Hub Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", ConfigurationManager.AppSettings["EventHubName"], "true")); // Set the bolt to subscribe to data from the spout topologyBuilder.SetJavaBolt( "eventhubbolt", constructor, partitionCount) .shuffleGrouping("Spout"); // Return the topology builder return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List <object> constructorParams = new List <object>() { 100, "test", string.Empty }; List <string> paramTypes = new List <string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary <string, string>() { { "topology.kryo.register", "[\"[B\"]" } }); return(topologyBuilder); }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"}; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt List<object> constructorParams = new List<object>() { 100, "test", string.Empty }; List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" }; JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.kryo.register","[\"[B\"]"} }); return topologyBuilder; }