public virtual void Initialize(Dictionary<string, Object> parms = null)
{
this.appConfig = new AppConfig();
this.PrimaryKey = this.appConfig.PrimaryKey;
this.SecondaryKey = this.appConfig.SecondaryKey;
if (String.IsNullOrWhiteSpace(this.PrimaryKey) || String.IsNullOrWhiteSpace(this.SecondaryKey))
{
throw new Exception("PrimaryKey or SecondaryKey cannot be null.");
}
Context.Logger.Info("PrimaryKey = {0} SecondaryKey = {1}", this.PrimaryKey, this.SecondaryKey);
Context.Logger.Info("primary.key = {0}",
Context.Config.stormConf.ContainsKey("primary.key") ? Context.Config.stormConf["primary.key"] : "not found");
Context.Logger.Info("secondary.key = {0}",
Context.Config.stormConf.ContainsKey("secondary.key") ? Context.Config.stormConf["secondary.key"] : "not found");
globalstopwatch = new Stopwatch();
globalstopwatch.Start();
emitstopwatch = new Stopwatch();
emitstopwatch.Start();
}
public EventGenerator(Context context, Dictionary<string, object> parms = null)
{
this.context = context;
this.appConfig = new AppConfig();
Dictionary<string, List<Type>> outputSchema = new Dictionary<string, List<Type>>();
outputSchema.Add(Constants.DEFAULT_STREAM_ID, new List<Type>() { typeof(string) });
this.context.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema));
//This statement is used for Hybrid scenarios where you will add a customized serializer in C# spout
//and a customized deserializer in your java bolt
this.context.DeclareCustomizedSerializer(new CustomizedInteropJSONSerializer());
if (Context.Config.pluginConf.ContainsKey(Constants.NONTRANSACTIONAL_ENABLE_ACK))
{
ackEnabled = (bool)(Context.Config.pluginConf[Constants.NONTRANSACTIONAL_ENABLE_ACK]);
}
globalstopwatch = new Stopwatch();
globalstopwatch.Start();
//To serialize WebRequestLog into a JSON to send into EventHub
jsonserializer = JsonSerializer.Create();
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
// Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize
// Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields
List<string> javaDeserializerInfo =
new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" };
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
//We will use CreateFromClojureExpr method as we wish to pass in a complex Java object
//The EventHubBolt takes a EventHubBoltConfig that we will create using clojure
//NOTE: We need to escape the quotes for strings that need to be passes to clojure
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace, appConfig.EventHubFqnAddress,
appConfig.EventHubEntityPath, "true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
//Assuming a 4 'L' node cluster, we will have 16 worker slots available
//We will half of those slots for this topology
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.workers", appConfig.EventHubPartitions.ToString()},
{"topology.max.spout.pending","4096"}
});
return topologyBuilder;
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
/// <param name="tablename"></param>
public EventHubWriter(Context context, Dictionary<string, Object> parms = null)
{
this.context = context;
this.appConfig = new AppConfig();
Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>();
inputSchema.Add(Constants.DEFAULT_STREAM_ID, new List<Type>() { typeof(string) });
this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null));
TopologyContext topologyContext = Context.TopologyContext;
if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL)
{
Context.Logger.Info("EventHubWriter TopologyContext info:");
Context.Logger.Info("TaskId: {0}", topologyContext.GetThisTaskId());
var taskIndex = topologyContext.GetThisTaskIndex();
Context.Logger.Info("TaskIndex: {0}", taskIndex);
string componentId = topologyContext.GetThisComponentId();
Context.Logger.Info("ComponentId: {0}", componentId);
List<int> componentTasks = topologyContext.GetComponentTasks(componentId);
Context.Logger.Info("ComponentTasks: {0}", componentTasks.Count);
}
InitializeEventHub();
if (Context.Config.pluginConf.ContainsKey(Constants.NONTRANSACTIONAL_ENABLE_ACK))
{
ackEnabled = (bool)(Context.Config.pluginConf[Constants.NONTRANSACTIONAL_ENABLE_ACK]);
}
globalStopwatch = new Stopwatch();
globalStopwatch.Start();
emitStopwatch = new Stopwatch();
emitStopwatch.Start();
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
topologyBuilder.SetEventHubSpout(
"EventHubSpout",
new EventHubSpoutConfig(
appConfig.EventHubSharedAccessKeyName,
appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace,
appConfig.EventHubEntityPath,
appConfig.EventHubPartitions),
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
topologyBuilder.SetBolt(
typeof(EventAggregator).Name,
EventAggregator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
//You can also setup a Ranker bolt to maintain top N records
/*
topologyBuilder.SetBolt(
typeof(EventRanker).Name,
EventRanker.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
},
appConfig.EventHubPartitions / 2
).
fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
*/
topologyBuilder.SetBolt(
typeof(EventHBaseWriter).Name,
EventHBaseWriter.Get,
new Dictionary<string, List<string>>(),
appConfig.EventHubPartitions / 4).
fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
//Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology
//The default JVM heap size for workers is 768m, we also increase that to 1024m
//That helps the java spout have additional heap size at disposal.
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(8);
topologyConfig.setMaxSpoutPending(1000);
topologyConfig.setWorkerChildOps("-Xmx1024m");
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
appConfig.EventHubPartitions
);
topologyBuilder.SetBolt(
typeof(EventHubWriter).Name,
EventHubWriter.Get,
new Dictionary<string, List<string>>(),
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(8);
topologyConfig.setMaxSpoutPending(1600);
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
