public virtual void Initialize(Dictionary<string, Object> parms = null) { this.appConfig = new AppConfig(); this.PrimaryKey = this.appConfig.PrimaryKey; this.SecondaryKey = this.appConfig.SecondaryKey; if (String.IsNullOrWhiteSpace(this.PrimaryKey) || String.IsNullOrWhiteSpace(this.SecondaryKey)) { throw new Exception("PrimaryKey or SecondaryKey cannot be null."); } Context.Logger.Info("PrimaryKey = {0} SecondaryKey = {1}", this.PrimaryKey, this.SecondaryKey); Context.Logger.Info("primary.key = {0}", Context.Config.stormConf.ContainsKey("primary.key") ? Context.Config.stormConf["primary.key"] : "not found"); Context.Logger.Info("secondary.key = {0}", Context.Config.stormConf.ContainsKey("secondary.key") ? Context.Config.stormConf["secondary.key"] : "not found"); globalstopwatch = new Stopwatch(); globalstopwatch.Start(); emitstopwatch = new Stopwatch(); emitstopwatch.Start(); }
public EventGenerator(Context context, Dictionary<string, object> parms = null) { this.context = context; this.appConfig = new AppConfig(); Dictionary<string, List<Type>> outputSchema = new Dictionary<string, List<Type>>(); outputSchema.Add(Constants.DEFAULT_STREAM_ID, new List<Type>() { typeof(string) }); this.context.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema)); //This statement is used for Hybrid scenarios where you will add a customized serializer in C# spout //and a customized deserializer in your java bolt this.context.DeclareCustomizedSerializer(new CustomizedInteropJSONSerializer()); if (Context.Config.pluginConf.ContainsKey(Constants.NONTRANSACTIONAL_ENABLE_ACK)) { ackEnabled = (bool)(Context.Config.pluginConf[Constants.NONTRANSACTIONAL_ENABLE_ACK]); } globalstopwatch = new Stopwatch(); globalstopwatch.Start(); //To serialize WebRequestLog into a JSON to send into EventHub jsonserializer = JsonSerializer.Create(); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); // Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize // Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" }; topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //We will use CreateFromClojureExpr method as we wish to pass in a complex Java object //The EventHubBolt takes a EventHubBoltConfig that we will create using clojure //NOTE: We need to escape the quotes for strings that need to be passes to clojure JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr( String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " + @"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))", appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubFqnAddress, appConfig.EventHubEntityPath, "true")); topologyBuilder.SetJavaBolt( "EventHubBolt", constructor, appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); //Assuming a 4 'L' node cluster, we will have 16 worker slots available //We will half of those slots for this topology topologyBuilder.SetTopologyConfig(new Dictionary<string, string>() { {"topology.workers", appConfig.EventHubPartitions.ToString()}, {"topology.max.spout.pending","4096"} }); return topologyBuilder; }
/// <summary> /// /// </summary> /// <param name="context"></param> /// <param name="tablename"></param> public EventHubWriter(Context context, Dictionary<string, Object> parms = null) { this.context = context; this.appConfig = new AppConfig(); Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>(); inputSchema.Add(Constants.DEFAULT_STREAM_ID, new List<Type>() { typeof(string) }); this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null)); TopologyContext topologyContext = Context.TopologyContext; if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL) { Context.Logger.Info("EventHubWriter TopologyContext info:"); Context.Logger.Info("TaskId: {0}", topologyContext.GetThisTaskId()); var taskIndex = topologyContext.GetThisTaskIndex(); Context.Logger.Info("TaskIndex: {0}", taskIndex); string componentId = topologyContext.GetThisComponentId(); Context.Logger.Info("ComponentId: {0}", componentId); List<int> componentTasks = topologyContext.GetComponentTasks(componentId); Context.Logger.Info("ComponentTasks: {0}", componentTasks.Count); } InitializeEventHub(); if (Context.Config.pluginConf.ContainsKey(Constants.NONTRANSACTIONAL_ENABLE_ACK)) { ackEnabled = (bool)(Context.Config.pluginConf[Constants.NONTRANSACTIONAL_ENABLE_ACK]); } globalStopwatch = new Stopwatch(); globalStopwatch.Start(); emitStopwatch = new Stopwatch(); emitStopwatch.Start(); }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetEventHubSpout( "EventHubSpout", new EventHubSpoutConfig( appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey, appConfig.EventHubNamespace, appConfig.EventHubEntityPath, appConfig.EventHubPartitions), appConfig.EventHubPartitions); // Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string // Here, fullname of the Java JSON Serializer class is required List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" }; topologyBuilder.SetBolt( typeof(EventAggregator).Name, EventAggregator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions, true ). DeclareCustomizedJavaSerializer(javaSerializerInfo). shuffleGrouping("EventHubSpout"); //You can also setup a Ranker bolt to maintain top N records /* topologyBuilder.SetBolt( typeof(EventRanker).Name, EventRanker.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } } }, appConfig.EventHubPartitions / 2 ). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); */ topologyBuilder.SetBolt( typeof(EventHBaseWriter).Name, EventHBaseWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions / 4). fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 }); //Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology //The default JVM heap size for workers is 768m, we also increase that to 1024m //That helps the java spout have additional heap size at disposal. var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1000); topologyConfig.setWorkerChildOps("-Xmx1024m"); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }
public ITopologyBuilder GetTopologyBuilder() { appConfig = new AppConfig(); TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name); topologyBuilder.SetSpout( typeof(EventGenerator).Name, EventGenerator.Get, new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}} }, appConfig.EventHubPartitions ); topologyBuilder.SetBolt( typeof(EventHubWriter).Name, EventHubWriter.Get, new Dictionary<string, List<string>>(), appConfig.EventHubPartitions ). shuffleGrouping(typeof(EventGenerator).Name); var topologyConfig = new StormConfig(); topologyConfig.setNumWorkers(8); topologyConfig.setMaxSpoutPending(1600); topologyBuilder.SetTopologyConfig(topologyConfig); return topologyBuilder; }