"")); //Last value is the zookeeper connection string - leave empty
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubReader");
topologyBuilder.SetJavaSpout(
"EventHubSpout",
constructor,
partitionCount);
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
"Bolt",
Bolt.Get,
new Dictionary <string, List <string> >(),
partitionCount,
true).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor generatorConfig = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.GeneratorConfig",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test")
});
JavaComponentConstructor generator = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Generator",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig)
});
topologyBuilder.SetJavaSpout(
"generator",
generator,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary <string, List <string> >(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setNumWorkers(1);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor generatorConfig = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.GeneratorConfig",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test")
});
JavaComponentConstructor generator = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Generator",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig)
});
topologyBuilder.SetJavaSpout(
"generator",
generator,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setNumWorkers(1);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List <object> constructorParams = new List <object>()
{
100, "test", null
};
List <string> paramTypes = new List <string>()
{
"int", "java.lang.String", "java.lang.String"
};
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes);
topologyBuilder.SetJavaSpout(
"generator",
constructor,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary <string, List <string> >(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List<object> constructorParams = new List<object>() { 100, "test", null };
List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" };
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes);
topologyBuilder.SetJavaSpout(
"generator",
constructor,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"};
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
// Start building a new topology
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Get the number of partitions in EventHub
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
// Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods.
// NOTE: These methods set the spout to read data in a String encoding.
/*
* topologyBuilder.SetEventHubSpout(
* "EventHubSpout",
* new EventHubSpoutConfig(
* ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
* ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
* ConfigurationManager.AppSettings["EventHubNamespace"],
* ConfigurationManager.AppSettings["EventHubEntityPath"],
* eventHubPartitions),
* eventHubPartitions);
*/
// The following is an example of how to create the same spout using the JavaComponentConstructor,
// which allows us to use UTF-8 encoding for reads.
// NOTE!!!! This only works with the 9.5 version of the Event Hub components, which are located at
// https://github.com/hdinsight/hdinsight-storm-examples/blob/master/lib/eventhubs/
// Create the UTF-8 data scheme
var schemeConstructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.UnicodeEventDataScheme");
// Create the EventHubSpoutConfig
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List <Tuple <string, object> >()
{
//comment
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
//comment
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create <string, object>("int", eventHubPartitions),
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.IEventDataScheme", schemeConstructor)
}
);
// Create the spout
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
// Set the spout in the topology
topologyBuilder.SetJavaSpout("EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
// Create a config for the bolt. It's unused here
var boltConfig = new StormConfig();
// Add the logbolt to the topology
// Use a serializer to understand data from the Java component
topologyBuilder.SetBolt(
typeof(LogBolt).Name,
LogBolt.Get,
new Dictionary <string, List <string> >(),
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
// Create a configuration for the topology
var topologyConfig = new StormConfig();
// Increase max pending for the spout
topologyConfig.setMaxSpoutPending(8192);
// Parallelism hint for the number of workers to match the number of EventHub partitions
topologyConfig.setNumWorkers(eventHubPartitions);
// Add the config and return the topology builder
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]);
TopologyBuilder topologyBuilder =
new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create <string, object>("int", eventHubPartitions),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""),
Tuple.Create <string, object>("int", 10),
Tuple.Create <string, object>("int", 1024),
Tuple.Create <string, object>("int", 1024 * eventHubPartitions),
Tuple.Create <string, object>("long", 0),
}
);
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"partialCount"
} }
},
eventHubPartitions,
enableAck
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary <string, List <string> >(),
1,
enableAck).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(eventHubPartitions);
if (enableAck)
{
topologyConfig.setNumAckers(eventHubPartitions);
}
else
{
topologyConfig.setNumAckers(0);
}
topologyConfig.setWorkerChildOps("-Xmx1g");
topologyConfig.setMaxSpoutPending((1024 * 1024) / 100);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]);
TopologyBuilder topologyBuilder =
new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create<string, object>("int", eventHubPartitions),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""),
Tuple.Create<string, object>("int", 10),
Tuple.Create<string, object>("int", 1024),
Tuple.Create<string, object>("int", 1024*eventHubPartitions),
Tuple.Create<string, object>("long", 0),
}
);
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() {
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } }
},
eventHubPartitions,
enableAck
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary<string, List<string>>(),
1,
enableAck).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(eventHubPartitions);
if (enableAck)
{
topologyConfig.setNumAckers(eventHubPartitions);
}
else
{
topologyConfig.setNumAckers(0);
}
topologyConfig.setWorkerChildOps("-Xmx1g");
topologyConfig.setMaxSpoutPending((1024*1024)/100);
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.spout.EventHubSpout. (com.microsoft.eventhubs.spout.EventHubSpoutConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" {4} """"))",
appConfig.EventHubUsername, appConfig.EventHubPassword,
appConfig.EventHubNamespace, appConfig.EventHubEntityPath,
appConfig.EventHubPartitions));
topologyBuilder.SetJavaSpout(
"EventHubSpout",
constructor,
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
typeof(EventAggregator).Name,
EventAggregator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
//You can also setup a Ranker bolt to maintain top N records
/*
* topologyBuilder.SetBolt(
* typeof(EventRanker).Name,
* EventRanker.Get,
* new Dictionary<string, List<string>>()
* {
* {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
* },
* appConfig.EventHubPartitions / 2
* ).
* fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
*/
topologyBuilder.SetBolt(
typeof(EventHBaseWriter).Name,
EventHBaseWriter.Get,
new Dictionary <string, List <string> >(),
appConfig.EventHubPartitions / 4).
fieldsGrouping(typeof(EventAggregator).Name, new List <int>()
{
0, 1, 2
});
//Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology
//The default JVM heap size for workers is 768m, we also increase that to 1024m
//That helps the java spout have additional heap size at disposal.
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.workers", "8" },
{ "topology.max.spout.pending", "1000" },
{ "topology.worker.childopts", @"""-Xmx1024m""" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
JavaComponentConstructor constructor =
new JavaComponentConstructor("com.microsoft.eventhubs.spout.EventHubSpout",
new List <object>()
{
appConfig.EventHubUsername,
appConfig.EventHubPassword,
appConfig.EventHubNamespace,
appConfig.EventHubEntityPath,
appConfig.EventHubPartitions
},
new List <string>()
{
"java.lang.String",
"java.lang.String",
"java.lang.String",
"java.lang.String",
"int"
}
);
topologyBuilder.SetJavaSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
constructor,
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"partialCount"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(new Dictionary <string, string>()
{
{ "topology.tick.tuple.freq.secs", "1" }
});
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary <string, List <string> >(),
1).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(new Dictionary <string, string>()
{
{ "topology.tick.tuple.freq.secs", "1" }
});
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.workers", appConfig.EventHubPartitions.ToString() },
{ "topology.max.spout.pending", "512" }
});
return(topologyBuilder);
}