public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("TwitterStreaming" + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( "TwitterSpout", TwitterSpout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, TwitterSpout.OutputSchemaName } }, 1, true); var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "20"); topologyBuilder.SetBolt( "TopNTweetBolt", TopNTweetBolt.Get, new Dictionary <string, List <string> >() { { "TOPNTWEETS_STREAM", TopNTweetBolt.OutputSchemaName } }, 1, true) .shuffleGrouping("TwitterSpout") .addConfigurations(boltConfig); topologyBuilder.SetBolt( "AzureSqlBolt", AzureSqlBolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping("TopNTweetBolt", "TOPNTWEETS_STREAM"); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDBLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, true); topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name, //Set task name DocumentDbLookupBolt.Get, //Set task constructor delegate //Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, true). globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDBWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); //Store the incoming Vehicle records in DocumentDb topologyBuilder.SetBolt( typeof(DocumentDbBolt).Name, //Set task name DocumentDbBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Choose grouping //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForHBase).Name, //Set task name VehicleRecordGeneratorSpoutForHBase.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(HBaseLookupBolt).Name, //Set task name HBaseLookupBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForHBase).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, appConfig.EventHubPartitions ); topologyBuilder.SetBolt( typeof(EventHubWriter).Name, EventHubWriter.Get, new Dictionary <string, List <string> >(), appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1600); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(SqlAzureWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); //Component tasks expect output field names in TopologyBuilder topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields} }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(SqlAzureBolt).Name, //Set task name SqlAzureBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topoBuilder = new TopologyBuilder("TwitterSentimentAnalysis" + DateTime.Now.ToString("yyyyMMddHHmmss")); topoBuilder.SetSpout("TwitterReaderSpout", TwitterReaderSpout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, TwitterReaderSpout.OutputSchemaName } }, 1, true); // create a bolt with tick frequence of 10 secs to emit tuple var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "10"); topoBuilder.SetBolt("TweetRankBolt", TweetRankBolt.Get, new Dictionary <string, List <string> >() { { "TWEETRANK_STREAM", TweetRankBolt.OutputSchemaName } }, 1, true) .shuffleGrouping("TwitterReaderSpout") .addConfigurations(boltConfig); topoBuilder.SetBolt( "AzureSqlBolt", AzureSqlBolt.Get, new Dictionary <string, List <string> >(), 1).shuffleGrouping("TweetRankBolt", "TWEETRANK_STREAM"); return(topoBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TransactionalTopologyBuilder topologyBuilder = new TransactionalTopologyBuilder("HybridTopologyTx_javaSpout_csharpBolt"); // Demo how to use clojure code (in string) to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr("(microsoft.scp.example.HybridTopology.TxGenerator. 100 \"test\" nil)"); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", SCPTxBolt.SCP_TX_COMMIT_BOLT, TxDisplayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}} }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); // The java bolt "java_displayer" receives from the C# spout "generator" topologyBuilder.SetJavaBolt( "java_displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // The C# bolt "csharp-displayer" receive from the C# spout "generator" topologyBuilder.SetBolt( "csharp-displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TestEventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } } }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(GlobalCountBolt).Name, GlobalCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "timestamp", "totalCount" } } }, 1, true). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "Event" } } }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); topologyBuilder.SetBolt( typeof(EventHubBolt).Name, //Set task name EventHubBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples partitionCount, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt"); // Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt) // Here, fullname of the Java JSON Deserializer class and target deserialized class are required List <string> javaDeserializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" }; topologyBuilder.SetSpout( "generator", Generator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "person" } } }, 1, null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Displayer", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty) }); topologyBuilder.SetJavaBolt( "displayer", constructor, 1).shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setDebug(false); conf.setNumWorkers(1); conf.setStatsSampleRate(0.05); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor generatorConfig = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.GeneratorConfig", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test") }); JavaComponentConstructor generator = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Generator", new List <Tuple <string, object> >() { Tuple.Create <string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig) }); topologyBuilder.SetJavaSpout( "generator", generator, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(VehicleRecordGeneratorSpout).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "5"); topologyBuilder.SetBolt( typeof(HBaseBolt).Name, //Set task name HBaseBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name). addConfigurations(boltConfig); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields} }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); topologyBuilder.SetBolt( typeof(EventHubBolt).Name, //Set task name EventHubBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples partitionCount, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
/// <summary> /// Builds a topology that can be submitted to Storm on HDInsight /// </summary> /// <returns>A topology builder</returns> public ITopologyBuilder GetTopologyBuilder() { //The friendly name is 'EventHubWriter' TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss")); //Get the partition count int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]); //Create a deserializer for JSON to java.lang.String //so that Java components can consume data emitted by //C# components List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; //Set the spout topologyBuilder.SetSpout( "Spout", Spout.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, partitionCount). //Parallelism hint uses partition count DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream //Create constructor for the Java bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", ConfigurationManager.AppSettings["EventHubPolicyName"], ConfigurationManager.AppSettings["EventHubPolicyKey"], ConfigurationManager.AppSettings["EventHubNamespace"], "servicebus.windows.net", //suffix for servicebus fqdn ConfigurationManager.AppSettings["EventHubName"], "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, partitionCount). //Parallelism hint uses partition count shuffleGrouping("Spout"); //Consume data from spout StormConfig config = new StormConfig(); config.setNumWorkers(1); //Set the number of workers topologyBuilder.SetTopologyConfig(config); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDbReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, true); topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name, //Set task name DocumentDbLookupBolt.Get, //Set task constructor delegate //Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, //Or, new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, new List<string>() { "Vehicle" } }, 1, true). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); //Log the looked up records topologyBuilder.SetBolt( typeof(DocumentDbLookupBolt).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, true). globalGrouping(typeof(DocumentDbLookupBolt).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt"); // Demo how to set parameters to initialize the constructor of Java Spout/Bolt JavaComponentConstructor generatorConfig = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.GeneratorConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test") }); JavaComponentConstructor generator = new JavaComponentConstructor( "microsoft.scp.example.HybridTopology.Generator", new List<Tuple<string, object>>() { Tuple.Create<string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig) }); topologyBuilder.SetJavaSpout( "generator", generator, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", Displayer.Get, new Dictionary<string, List<string>>(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(SimpleHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); var javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); topologyBuilder.SetBolt( typeof(ThresholdBolt).Name, ThresholdBolt.Get, new Dictionary <string, List <string> >() { }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(SqlAzureWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); //Component tasks expect output field names in TopologyBuilder topologyBuilder.SetSpout( typeof(IISLogGeneratorSpout).Name, //Set task name IISLogGeneratorSpout.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); topologyBuilder.SetBolt( typeof(SqlAzureBolt).Name, //Set task name SqlAzureBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(IISLogGeneratorSpout).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpout).Name, //Set task name VehicleRecordGeneratorSpout.Get, //Set task constructor delegate new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(VehicleRecordGeneratorSpout).Name + typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name); var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "5"); topologyBuilder.SetBolt( typeof(HBaseBolt).Name, //Set task name HBaseBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpout).Name). addConfigurations(boltConfig); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { // Start building a new topology TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology. Set parallelism hint to the number of partitions topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Create a config for the bolt. It's unused here var boltConfig = new StormConfig(); // Add the logbolt to the topology topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary<string, List<string>>(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"); // Create a configuration for the topology var topologyConfig = new StormConfig(); // Increase max pending for the spout topologyConfig.setMaxSpoutPending(8192); // Parallelism hint for the number of workers to match the number of EventHub partitions topologyConfig.setNumWorkers(eventHubPartitions); // Add the config and return the topology builder topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(HBaseLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForHBase).Name, //Set task name VehicleRecordGeneratorSpoutForHBase.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); topologyBuilder.SetBolt( typeof(HBaseLookupBolt).Name, //Set task name HBaseLookupBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } }, 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForHBase).Name); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TransactionalTopologyBuilder topologyBuilder = new TransactionalTopologyBuilder("HybridTopologyTx_javaSpout_csharpBolt"); // Demo how to use clojure code (in string) to initialize the constructor of Java Spout/Bolt JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr("(microsoft.scp.example.HybridTopology.TxGenerator. 100 \"test\" nil)"); topologyBuilder.SetJavaSpout( "generator", constructor, 1); // Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( "displayer", SCPTxBolt.SCP_TX_COMMIT_BOLT, TxDisplayer.Get, new Dictionary <string, List <string> >(), 1). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("generator"); // Demo how to set topology config StormConfig conf = new StormConfig(); conf.setNumWorkers(1); conf.setWorkerChildOps("-Xmx1024m"); conf.Set("topology.kryo.register", "[\"[B\"]"); topologyBuilder.SetTopologyConfig(conf); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(DocumentDBWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); topologyBuilder.SetSpout( typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } }, 1, //Set number of tasks true //Set enableAck ); //Store the incoming Vehicle records in DocumentDb topologyBuilder.SetBolt( typeof(DocumentDbBolt).Name, //Set task name DocumentDbBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Choose grouping //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(1); //Set number of worker processes topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(GlobalCountBolt).Name, GlobalCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "timestamp", "totalCount" } } }, 1, true). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var EventHubNamespace = ConfigurationManager.AppSettings["EventHubNamespace"]; if (String.IsNullOrWhiteSpace(EventHubNamespace)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubNamespace"); } var EventHubEntityPath = ConfigurationManager.AppSettings["EventHubEntityPath"]; if (String.IsNullOrWhiteSpace(EventHubEntityPath)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubEntityPath"); } var EventHubSharedAccessKeyName = ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKeyName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKeyName"); } var EventHubSharedAccessKey = ConfigurationManager.AppSettings["EventHubSharedAccessKey"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKey)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKey"); } var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); //You can use the new SetEventHubSpout method by providing EventHubSpoutConfig which will automatically create Java code to instantiate this spout //TODO: This method will not work if you do not include EventHub jar during publishing or deployment of this topology topologyBuilder.SetEventHubSpout( "EventHubSpout", //Set task name new EventHubSpoutConfig( EventHubSharedAccessKeyName, EventHubSharedAccessKey, EventHubNamespace, EventHubEntityPath, partitionCount), partitionCount ); //For a hybrid topology we need to declare a customize Java serializer that will serialize Java objects which will deseriailized in the bolt into C# objects topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). DeclareCustomizedJavaSerializer(new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }). globalGrouping("EventHubSpout"); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(4); //Set number of worker processes topologyConfig.setMaxSpoutPending(1024); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx1024m"); //Set Java Heap Size return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder("AlertTopology"); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); topologyBuilder.SetBolt( typeof(ParserBolt).Name, ParserBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "temp", "createDate", "deviceId" } } }, eventHubPartitions, enableAck: true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(EmitAlertBolt).Name, EmitAlertBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "reason", "temp", "createDate", "deviceId" } } }, eventHubPartitions, enableAck: true ). shuffleGrouping(typeof(ParserBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { // Start building a new topology TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); // Get the number of partitions in EventHub var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); // Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods. // NOTE: These methods set the spout to read data in a String encoding. /* * topologyBuilder.SetEventHubSpout( * "EventHubSpout", * new EventHubSpoutConfig( * ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], * ConfigurationManager.AppSettings["EventHubSharedAccessKey"], * ConfigurationManager.AppSettings["EventHubNamespace"], * ConfigurationManager.AppSettings["EventHubEntityPath"], * eventHubPartitions), * eventHubPartitions); */ // The following is an example of how to create the same spout using the JavaComponentConstructor, // which allows us to use UTF-8 encoding for reads. // NOTE!!!! This only works with the 9.5 version of the Event Hub components, which are located at // https://github.com/hdinsight/hdinsight-storm-examples/blob/master/lib/eventhubs/ // Create the UTF-8 data scheme var schemeConstructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.UnicodeEventDataScheme"); // Create the EventHubSpoutConfig var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), //comment Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>("com.microsoft.eventhubs.spout.IEventDataScheme", schemeConstructor) } ); // Create the spout var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); // Set the spout in the topology topologyBuilder.SetJavaSpout("EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; // Create a config for the bolt. It's unused here var boltConfig = new StormConfig(); // Add the logbolt to the topology // Use a serializer to understand data from the Java component topologyBuilder.SetBolt( typeof(LogBolt).Name, LogBolt.Get, new Dictionary <string, List <string> >(), eventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); // Create a configuration for the topology var topologyConfig = new StormConfig(); // Increase max pending for the spout topologyConfig.setMaxSpoutPending(8192); // Parallelism hint for the number of workers to match the number of EventHub partitions topologyConfig.setNumWorkers(eventHubPartitions); // Add the config and return the topology builder topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List <Tuple <string, object> >() { Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create <string, object>("int", eventHubPartitions), Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create <string, object>("int", 10), Tuple.Create <string, object>("int", 1024), Tuple.Create <string, object>("int", 1024 * eventHubPartitions), Tuple.Create <string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List <Tuple <string, object> >() { Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary <string, List <string> >(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024 * 1024) / 100); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions), appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* * topologyBuilder.SetBolt( * typeof(EventRanker).Name, * EventRanker.Get, * new Dictionary<string, List<string>>() * { * {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } * }, * appConfig.EventHubPartitions / 2 * ). * fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary <string, List <string> >(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List <int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1000); topologyConfig.setWorkerChildOps("-Xmx1024m"); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(AlarmsOnAggregatesTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); topologyBuilder.SetEventHubSpout( "com.microsoft.eventhubs.spout.EventHubSpout", new EventHubSpoutConfig( ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"], ConfigurationManager.AppSettings["EventHubSharedAccessKey"], ConfigurationManager.AppSettings["EventHubNamespace"], ConfigurationManager.AppSettings["EventHubEntityPath"], eventHubPartitions), eventHubPartitions); List <string> javaSerializerInfo = new List <string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; //FLATT MESSAGES topologyBuilder.SetBolt( typeof(FlattBolt).Name, FlattBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "sensor" } } }, eventHubPartitions, true ).DeclareCustomizedJavaSerializer(javaSerializerInfo) .shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"); //AGGREGATE MESSAGES var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "10"); topologyBuilder.SetBolt( typeof(AggregateBolt).Name, AggregateBolt.Get, new Dictionary <string, List <string> >() { { Constants.DEFAULT_STREAM_ID, new List <string>() { "aggregated" } } }, parallelismHint: 1, enableAck: false ) .globalGrouping(typeof(FlattBolt).Name) .addConfigurations(boltConfig); //ALARMS MESSAGES topologyBuilder.SetBolt( typeof(AlarmsBolt).Name, AlarmsBolt.Get, new Dictionary <string, List <string> >() { }, parallelismHint: 4, enableAck: false ).shuffleGrouping(typeof(AggregateBolt).Name); var topologyConfig = new StormConfig(); topologyConfig.setMaxSpoutPending(8192); topologyConfig.setNumWorkers(eventHubPartitions); topologyBuilder.SetTopologyConfig(topologyConfig); return(topologyBuilder); }
public ITopologyBuilder GetTopologyBuilder() { var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]); TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]); var eventHubSpoutConfig = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpoutConfig", new List<Tuple<string, object>>() { Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]), Tuple.Create<string, object>("int", eventHubPartitions), Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""), Tuple.Create<string, object>("int", 10), Tuple.Create<string, object>("int", 1024), Tuple.Create<string, object>("int", 1024*eventHubPartitions), Tuple.Create<string, object>("long", 0), } ); var eventHubSpout = new JavaComponentConstructor( "com.microsoft.eventhubs.spout.EventHubSpout", new List<Tuple<string, object>>() { Tuple.Create<string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig) } ); topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, full name of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; var boltConfig = new StormConfig(); boltConfig.Set("topology.tick.tuple.freq.secs", "1"); topologyBuilder.SetBolt( typeof(PartialCountBolt).Name, PartialCountBolt.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } } }, eventHubPartitions, enableAck ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout"). addConfigurations(boltConfig); topologyBuilder.SetBolt( typeof(DBGlobalCountBolt).Name, DBGlobalCountBolt.Get, new Dictionary<string, List<string>>(), 1, enableAck). globalGrouping(typeof(PartialCountBolt).Name). addConfigurations(boltConfig); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(eventHubPartitions); if (enableAck) { topologyConfig.setNumAckers(eventHubPartitions); } else { topologyConfig.setNumAckers(0); } topologyConfig.setWorkerChildOps("-Xmx1g"); topologyConfig.setMaxSpoutPending((1024*1024)/100); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { var topologyBuilder = new TopologyBuilder(typeof(EventHubReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss")); var EventHubNamespace = ConfigurationManager.AppSettings["EventHubNamespace"]; if (String.IsNullOrWhiteSpace(EventHubNamespace)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubNamespace"); } var EventHubEntityPath = ConfigurationManager.AppSettings["EventHubEntityPath"]; if (String.IsNullOrWhiteSpace(EventHubEntityPath)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubEntityPath"); } var EventHubSharedAccessKeyName = ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKeyName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKeyName"); } var EventHubSharedAccessKey = ConfigurationManager.AppSettings["EventHubSharedAccessKey"]; if (String.IsNullOrWhiteSpace(EventHubSharedAccessKey)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKey"); } var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"]; if (String.IsNullOrWhiteSpace(EventHubPartitions)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions"); } var partitionCount = int.Parse(EventHubPartitions); //You can use the new SetEventHubSpout method by providing EventHubSpoutConfig which will automatically create Java code to instantiate this spout //TODO: This method will not work if you do not include EventHub jar during publishing or deployment of this topology topologyBuilder.SetEventHubSpout( "EventHubSpout", //Set task name new EventHubSpoutConfig( EventHubSharedAccessKeyName, EventHubSharedAccessKey, EventHubNamespace, EventHubEntityPath, partitionCount), partitionCount ); //For a hybrid topology we need to declare a customize Java serializer that will serialize Java objects which will deseriailized in the bolt into C# objects topologyBuilder.SetBolt( typeof(LoggerBolt).Name, //Set task name LoggerBolt.Get, //Set task constructor delegate new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples 1, //Set number of tasks true //Set enableAck ). DeclareCustomizedJavaSerializer(new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" } ). globalGrouping("EventHubSpout"); //Set the topology config var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(4); //Set number of worker processes topologyConfig.setMaxSpoutPending(1024); //Set maximum pending tuples from spout topologyConfig.setWorkerChildOps("-Xmx1024m"); //Set Java Heap Size return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions), appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* topologyBuilder.SetBolt( typeof(EventRanker).Name, EventRanker.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions / 2 ). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1000); topologyConfig.setWorkerChildOps("-Xmx1024m"); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions ); topologyBuilder.SetBolt( typeof(EventHubWriter).Name, EventHubWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1600); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }