public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(SqlAzureWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
//Component tasks expect output field names in TopologyBuilder
topologyBuilder.SetSpout(
typeof(IISLogGeneratorSpout).Name, //Set task name
IISLogGeneratorSpout.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() {
{Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields}
},
1, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(SqlAzureBolt).Name, //Set task name
SqlAzureBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(IISLogGeneratorSpout).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(DocumentDBWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name
VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() {
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields }
},
1, //Set number of tasks
true //Set enableAck
);
//Store the incoming Vehicle records in DocumentDb
topologyBuilder.SetBolt(
typeof(DocumentDbBolt).Name, //Set task name
DocumentDbBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name); //Choose grouping
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
/// <summary>
/// Defines and configures the topology
/// </summary>
/// <returns></returns>
public ITopologyBuilder GetTopologyBuilder()
{
//Define a new topology named "EventTopology"
TopologyBuilder topologyBuilder = new TopologyBuilder("EventTopology");
//The spout, which emits events
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"id", "event", "eventtime"}}
},
1);
//A bolt that is used to look up previous events for the same session from HBase
//If this is an end event, the time span between the start and end events is
//emitted as 'duration'
topologyBuilder.SetBolt(
"Lookup",
HBaseLookupBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"id", "event", "eventtime", "duration"}}
},
1).shuffleGrouping("Spout");
//A bolt that writes events to HBase
topologyBuilder.SetBolt(
"Store",
HBaseBolt.Get,
new Dictionary<string, List<string>>(),
1).shuffleGrouping("Lookup");
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(HBaseLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpoutForHBase).Name, //Set task name
VehicleRecordGeneratorSpoutForHBase.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() {
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } },
1, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(HBaseLookupBolt).Name, //Set task name
HBaseLookupBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() {
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForHBase.OutputFields } },
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(VehicleRecordGeneratorSpoutForHBase).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TwitterTopology).Name + DateTime.Now.ToString("-yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(TwitterSpout).Name,
TwitterSpout.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"tweet"}}
},
1);
topologyBuilder.SetBolt(
typeof(TwitterBolt).Name,
TwitterBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"count", "tweet"}}
},
1).shuffleGrouping(typeof(TwitterSpout).Name);
topologyBuilder.SetBolt(
typeof(SqlAzureBolt).Name,
SqlAzureBolt.Get,
new Dictionary<string, List<string>>(),
1).shuffleGrouping(typeof(TwitterBolt).Name);
topologyBuilder.SetBolt(
typeof(SignalRBroadcastBolt).Name,
SignalRBroadcastBolt.Get,
new Dictionary<string, List<string>>(),
1).shuffleGrouping(typeof(TwitterBolt).Name);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(DocumentDBLookupTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name, //Set task name
VehicleRecordGeneratorSpoutForDocumentDB.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } },
1,
true);
topologyBuilder.SetBolt(
typeof(DocumentDbLookupBolt).Name, //Set task name
DocumentDbLookupBolt.Get, //Set task constructor delegate
//Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field
new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpoutForDocumentDB.OutputFields } },
1,
true).
globalGrouping(typeof(VehicleRecordGeneratorSpoutForDocumentDB).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" };
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}}
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor constructor = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Displayer",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty)
});
// The java bolt "java_displayer" receives from the C# spout "generator"
topologyBuilder.SetJavaBolt(
"java_displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
// The C# bolt "csharp-displayer" receive from the C# spout "generator"
topologyBuilder.SetBolt(
"csharp-displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setDebug(false);
conf.setNumWorkers(1);
conf.setStatsSampleRate(0.05);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TestEventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
topologyBuilder.SetEventHubSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } }
},
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(GlobalCountBolt).Name,
GlobalCountBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "timestamp", "totalCount" } }
},
1,
true).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setMaxSpoutPending(8192);
topologyConfig.setNumWorkers(eventHubPartitions);
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
// Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize
// Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields
List<string> javaDeserializerInfo =
new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" };
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
//We will use CreateFromClojureExpr method as we wish to pass in a complex Java object
//The EventHubBolt takes a EventHubBoltConfig that we will create using clojure
//NOTE: We need to escape the quotes for strings that need to be passes to clojure
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace, appConfig.EventHubFqnAddress,
appConfig.EventHubEntityPath, "true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
//Assuming a 4 'L' node cluster, we will have 16 worker slots available
//We will half of those slots for this topology
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.workers", appConfig.EventHubPartitions.ToString()},
{"topology.max.spout.pending","4096"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(EventHubWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(IISLogGeneratorSpout).Name, //Set task name
IISLogGeneratorSpout.Get,
new Dictionary<string, List<string>>() { {Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields} },
1, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(LoggerBolt).Name, //Set task name
LoggerBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(IISLogGeneratorSpout).Name);
var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"];
if (String.IsNullOrWhiteSpace(EventHubPartitions))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions");
}
var partitionCount = int.Parse(EventHubPartitions);
topologyBuilder.SetBolt(
typeof(EventHubBolt).Name, //Set task name
EventHubBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
partitionCount, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(IISLogGeneratorSpout).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
/// <summary>
/// Builds a topology that can be submitted to Storm on HDInsight
/// </summary>
/// <returns>A topology builder</returns>
public ITopologyBuilder GetTopologyBuilder()
{
//The friendly name is 'EventHubWriter'
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss"));
//Get the partition count
int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]);
//Create a deserializer for JSON to java.lang.String
//so that Java components can consume data emitted by
//C# components
List<string> javaDeserializerInfo =
new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" };
//Set the spout
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
partitionCount). //Parallelism hint uses partition count
DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream
//Create constructor for the Java bolt
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
ConfigurationManager.AppSettings["EventHubPolicyName"],
ConfigurationManager.AppSettings["EventHubPolicyKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
"servicebus.windows.net", //suffix for servicebus fqdn
ConfigurationManager.AppSettings["EventHubName"],
"true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
partitionCount). //Parallelism hint uses partition count
shuffleGrouping("Spout"); //Consume data from spout
StormConfig config = new StormConfig();
config.setNumWorkers(1); //Set the number of workers
topologyBuilder.SetTopologyConfig(config);
return topologyBuilder;
}
public Constructor(string fullFilePath, Topology.Topology topology)
{
this.fullFilePath = fullFilePath ?? throw new NullReferenceException();
if (topology == null)
{
throw new NullReferenceException();
}
InitializeComponent();
connection = ConnectionHelpers.OpenConnection();
crudHelper = new CrudHelper(connection);
SetupSettings();
topologyBuilder = new TopologyBuilder(dgvField, topology);
}
public void ShouldVisitTopologyMultipleTimesWithoutExceptions()
{
var factory = new DefaultConnectionFactory();
using (var connection = factory.Create(new Uri("amqp://localhost/integration")))
{
var model = connection.CreateModel();
var topology = new Topology();
topology.DefineQueue("test-queue");
var builder = new TopologyBuilder(model);
topology.Visit(builder);
topology.Visit(builder);
}
}
public void ShouldVisitTopologyMultipleTimesExclusiveQueue()
{
var factory = new DefaultConnectionFactory();
using (var connection = factory.Create(new Uri("amqp://localhost/integration")))
{
var model = connection.CreateModel();
var topology = new Topology();
topology.DefineQueue();
var builder = new TopologyBuilder(model);
topology.Visit(builder);
topology.Visit(builder);
}
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(DocumentDbReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpout).Name, //Set task name
VehicleRecordGeneratorSpout.Get, //Set task constructor delegate
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields }
},
1,
true);
topologyBuilder.SetBolt(
typeof(DocumentDbLookupBolt).Name, //Set task name
DocumentDbLookupBolt.Get, //Set task constructor delegate
//Set the output field names - As DocumentDb return a JSON object, we will be expecting only 1 field
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields }
},
//Or, new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, new List<string>() { "Vehicle" } },
1,
true).
globalGrouping(typeof(VehicleRecordGeneratorSpout).Name);
//Log the looked up records
topologyBuilder.SetBolt(
typeof(DocumentDbLookupBolt).Name + typeof(LoggerBolt).Name, //Set task name
LoggerBolt.Get, //Set task constructor delegate
new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1,
true).
globalGrouping(typeof(DocumentDbLookupBolt).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(HBaseReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpout).Name, //Set task name
VehicleRecordGeneratorSpout.Get, //Set task constructor delegate
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields }
},
1, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(HBaseLookupBolt).Name, //Set task name
HBaseLookupBolt.Get, //Set task constructor delegate
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields }
},
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(VehicleRecordGeneratorSpout).Name);
topologyBuilder.SetBolt(
typeof(HBaseLookupBolt).Name + typeof(LoggerBolt).Name, //Set task name
LoggerBolt.Get, //Set task constructor delegate
new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(HBaseLookupBolt).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor generatorConfig = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.GeneratorConfig",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test")
});
JavaComponentConstructor generator = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Generator",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>("microsoft.scp.example.HybridTopology.GeneratorConfig", generatorConfig)
});
topologyBuilder.SetJavaSpout(
"generator",
generator,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setNumWorkers(1);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return topologyBuilder;
}
/// <summary>
/// Use Topology Specification API to describe the topology
/// </summary>
/// <returns></returns>
public ITopologyBuilder GetTopologyBuilder()
{
// Use TopologyBuilder to define a Non-Tx topology
// And define each spouts/bolts one by one
TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld");
// Set a User customized config (Generator.config) for the Generator
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}}
},
1,
"Generator.config");
topologyBuilder.SetBolt(
"splitter",
Splitter.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "firstLetterOfWord"}}
},
1).shuffleGrouping("generator");
// Use scp-field-group from Splitter to Counter,
// and specify the second field in the Output schema of Splitter (Input schema of Counter) as the field grouping target
// by passing the index array [1] (index start from 0)
topologyBuilder.SetBolt(
"counter",
Counter.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}}
},
1).fieldsGrouping("splitter", new List<int>() {1});
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(SimpleHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
topologyBuilder.SetEventHubSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
var javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
var boltConfig = new StormConfig();
topologyBuilder.SetBolt(
typeof(ThresholdBolt).Name,
ThresholdBolt.Get,
new Dictionary <string, List <string> >()
{
},
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setMaxSpoutPending(8192);
topologyConfig.setNumWorkers(eventHubPartitions);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(HBaseWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(VehicleRecordGeneratorSpout).Name, //Set task name
VehicleRecordGeneratorSpout.Get, //Set task constructor delegate
new Dictionary<string, List<string>>() { { Constants.DEFAULT_STREAM_ID, VehicleRecordGeneratorSpout.OutputFields } },
1, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(VehicleRecordGeneratorSpout).Name + typeof(LoggerBolt).Name, //Set task name
LoggerBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(VehicleRecordGeneratorSpout).Name);
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "5");
topologyBuilder.SetBolt(
typeof(HBaseBolt).Name, //Set task name
HBaseBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
globalGrouping(typeof(VehicleRecordGeneratorSpout).Name).
addConfigurations(boltConfig);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(1); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
// Start building a new topology
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Get the number of partitions in EventHub
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
// Add the EvetnHubSpout to the topology. Set parallelism hint to the number of partitions
topologyBuilder.SetEventHubSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
// Create a config for the bolt. It's unused here
var boltConfig = new StormConfig();
// Add the logbolt to the topology
topologyBuilder.SetBolt(
typeof(LogBolt).Name,
LogBolt.Get,
new Dictionary<string, List<string>>(),
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout");
// Create a configuration for the topology
var topologyConfig = new StormConfig();
// Increase max pending for the spout
topologyConfig.setMaxSpoutPending(8192);
// Parallelism hint for the number of workers to match the number of EventHub partitions
topologyConfig.setNumWorkers(eventHubPartitions);
// Add the config and return the topology builder
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(EventHubsWriterTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"];
if (String.IsNullOrWhiteSpace(EventHubPartitions))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions");
}
var partitionCount = int.Parse(EventHubPartitions);
topologyBuilder.SetSpout(
typeof(IISLogGeneratorSpout).Name, //Set task name
IISLogGeneratorSpout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, IISLogGeneratorSpout.OutputFields }
},
partitionCount / 2, //Set number of tasks
true //Set enableAck
);
topologyBuilder.SetBolt(
typeof(EventHubBolt).Name, //Set task name
EventHubBolt.Get, //Set task constructor delegate
new Dictionary <string, List <string> >(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
partitionCount, //Set number of tasks
true //Set enableAck
).
shuffleGrouping(typeof(IISLogGeneratorSpout).Name);
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(partitionCount / 2); //Set number of worker processes
topologyConfig.setMaxSpoutPending(512); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx768m"); //Set Java Heap Size
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List <object> constructorParams = new List <object>()
{
100, "test", null
};
List <string> paramTypes = new List <string>()
{
"int", "java.lang.String", "java.lang.String"
};
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes);
topologyBuilder.SetJavaSpout(
"generator",
constructor,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary <string, List <string> >(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}
/// <summary>
/// Use Topology Specification API to describe the topology
/// </summary>
/// <returns></returns>
public ITopologyBuilder GetTopologyBuilder()
{
// Use TopologyBuilder to define a Non-Tx topology
// And define each spouts/bolts one by one
TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorldHostModeMultiSpout");
// Set a User customized config (SentenceGenerator.config) for the SentenceGenerator
topologyBuilder.SetSpout(
"SentenceGenerator",
SentenceGenerator.Get,
new Dictionary<string, List<string>>()
{
{SentenceGenerator.STREAM_ID, new List<string>(){"sentence"}}
},
1,
"SentenceGenerator.config");
topologyBuilder.SetSpout(
"PersonGenerator",
PersonGenerator.Get,
new Dictionary<string, List<string>>()
{
{PersonGenerator.STREAM_ID, new List<string>(){"person"}}
},
1);
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1)
.shuffleGrouping("SentenceGenerator", SentenceGenerator.STREAM_ID)
.shuffleGrouping("PersonGenerator", PersonGenerator.STREAM_ID);
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
string pluginName = string.Format("{0}.exe", GetType().Assembly.GetName().Name);
TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld");
topologyBuilder.SetSpout(
"generator",
pluginName,
new List<string>() { "generator" },
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}}
},
1);
topologyBuilder.SetBolt(
"splitter",
pluginName,
new List<string>() { "splitter" },
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "firstLetterOfWord"}}
},
1).shuffleGrouping("generator");
topologyBuilder.SetBolt(
"counter",
pluginName,
new List<string>() { "counter" },
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}}
},
1).fieldsGrouping("splitter", new List<int>() { 1 });
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public TopologyBuilderService(StatelessServiceContext context)
: base(context)
{
this.baseLogString = $"{this.GetType()} [{this.GetHashCode()}] =>{Environment.NewLine}";
Logger.LogDebug($"{baseLogString} Ctor => Logger initialized");
try
{
this.topologyBuilder = new TopologyBuilder();
string infoMessage = $"{baseLogString} Ctor => Contract providers initialized.";
Logger.LogInformation(infoMessage);
ServiceEventSource.Current.ServiceMessage(this.Context, $"[TopologyBuilderService | Information] {infoMessage}");
}
catch (Exception e)
{
string errorMessage = $"{baseLogString} Ctor => exception {e.Message}";
Logger.LogError(errorMessage, e);
ServiceEventSource.Current.ServiceMessage(this.Context, $"[TopologyBuilderService | Error] {errorMessage}");
}
}
/// <summary>
/// Get a Storm topology builder
/// </summary>
/// <returns>A topology builder</returns>
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubReader" + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Get the number of partitions in EventHub
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
// Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods.
// NOTE: These methods set the spout to read data in a String encoding.
topologyBuilder.SetEventHubSpout(
"EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
// Add the logbolt to the topology
// Use a serializer to understand data from the Java component
topologyBuilder.SetBolt(
typeof(LogBolt).Name,
LogBolt.Get,
new Dictionary <string, List <string> >(),
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
return(topologyBuilder);
}
/// <summary>
/// Defines and configures the topology
/// </summary>
/// <returns></returns>
public ITopologyBuilder GetTopologyBuilder()
{
//Define a new topology named "EventTopology"
TopologyBuilder topologyBuilder = new TopologyBuilder("EventTopology");
//The spout, which emits events
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"id", "event", "eventtime"
} }
},
1);
//A bolt that is used to look up previous events for the same session from HBase
//If this is an end event, the time span between the start and end events is
//emitted as 'duration'
topologyBuilder.SetBolt(
"Lookup",
HBaseLookupBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"id", "event", "eventtime", "duration"
} }
},
1).shuffleGrouping("Spout");
//A bolt that writes events to HBase
topologyBuilder.SetBolt(
"Store",
HBaseBolt.Get,
new Dictionary <string, List <string> >(),
1).shuffleGrouping("Lookup");
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(TwitterTopology).Name + DateTime.Now.ToString("-yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
typeof(TwitterSpout).Name,
TwitterSpout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"tweet"
} }
},
1);
topologyBuilder.SetBolt(
typeof(TwitterBolt).Name,
TwitterBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"count", "tweet"
} }
},
1).shuffleGrouping(typeof(TwitterSpout).Name);
topologyBuilder.SetBolt(
typeof(SqlAzureBolt).Name,
SqlAzureBolt.Get,
new Dictionary <string, List <string> >(),
1).shuffleGrouping(typeof(TwitterBolt).Name);
topologyBuilder.SetBolt(
typeof(SignalRBroadcastBolt).Name,
SignalRBroadcastBolt.Get,
new Dictionary <string, List <string> >(),
1).shuffleGrouping(typeof(TwitterBolt).Name);
return(topologyBuilder);
}
/// <summary>
/// build delivery queues for each endpoint, where each delivery queue
/// will correspond to a basic consumer
/// </summary>
private IEnumerable <IDeliveryQueue> BuildDeliveryQueues(
IEnumerable <string> endpointNames,
IModelReference modelReference,
string messageType,
IEndpointConfiguration configuration,
ISubscriptionFailureStrategy subscriptionFailureStrategy)
{
return(endpointNames.Select((endpointName, i) =>
{
var endpoint = new Endpoint(endpointName, messageType, i);
var topologyBuilder = new TopologyBuilder(modelReference);
var topology = configuration.BuildTopology(endpoint);
topology.Visit(topologyBuilder);
var subscribeQueue = topology.SubscribeQueue;
return new DeliveryQueue(
subscribeQueue, modelReference, subscriptionFailureStrategy);
}));
}
public IPublisher <TMessage> Create <TMessage>(
IModelReference modelReference,
IPublisherConfiguration <TMessage> configuration)
{
var messageType = typeof(TMessage).Name;
var endpoint = new Endpoint(configuration.EndpointName ?? messageType, messageType, 0);
var topology = configuration.BuildTopology(endpoint);
if (configuration.ShouldBuildTopology)
{
var topologyBuilder = new TopologyBuilder(modelReference);
topology.Visit(topologyBuilder);
}
var router = configuration.BuildRouter();
var faultStrategy = configuration.BuildFaultStrategy();
var messageSerializer = messageSerializers.FindOrDefault(
configuration.ContentType);
var publisher = configuration.ShouldConfirm
? new ConfirmingPublisher <TMessage>(
modelReference,
messageSerializer,
topology.PublishExchange,
router,
faultStrategy)
: new Publisher <TMessage>(
modelReference,
messageSerializer,
topology.PublishExchange,
router);
publisher.Start();
Track(publisher);
return(publisher);
}
/// <summary>
/// The resolve for.
/// </summary>
/// <param name="configuration">
/// The configuration.
/// </param>
/// <returns>
/// The <see cref="Producer"/>.
/// </returns>
public Producer ResolveFor(ISenderConfiguration configuration)
{
Producer producer = this.TryResolverFor(configuration.Label);
if (producer == null)
{
using (RabbitChannel channel = this._bus.OpenChannel())
{
var topologyBuilder = new TopologyBuilder(channel);
var builder = new RouteResolverBuilder(this._bus.Endpoint, topologyBuilder, configuration);
Maybe <Func <IRouteResolverBuilder, IRouteResolver> > routeResolverBuilder = configuration.Options.GetRouteResolverBuilder();
Assumes.True(routeResolverBuilder.HasValue, "RouteResolverBuilder must be set for [{0}]", configuration.Label);
IRouteResolver routeResolver = routeResolverBuilder.Value(builder);
producer = new Producer(this._bus, configuration.Label, routeResolver, configuration.Options.IsConfirmationRequired());
producer.Failed += p =>
{
{
// lock (_producers)
p.Dispose();
this._producers.Remove(p.Label);
}
};
// lock (_producers)
this._producers.Add(configuration.Label, producer);
}
if (configuration.RequiresCallback)
{
producer.UseCallbackListener(this._bus.ListenerRegistry.ResolveFor(configuration.CallbackConfiguration));
}
}
return(producer);
}
public Constructor(string fullFilePath, int cols, int rows)
{
this.fullFilePath = fullFilePath ?? throw new NullReferenceException();
if (cols < Topology.Topology.MinColsCount ||
cols > Topology.Topology.MaxColsCount)
{
throw new ArgumentOutOfRangeException();
}
if (rows < Topology.Topology.MinRowsCount ||
rows > Topology.Topology.MaxRowsCount)
{
throw new ArgumentOutOfRangeException();
}
InitializeComponent();
connection = ConnectionHelpers.OpenConnection();
crudHelper = new CrudHelper(connection);
SetupSettings();
topologyBuilder = new TopologyBuilder(dgvField, cols, rows);;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("SensorStream" + DateTime.Now.ToString("yyyyMMddHHmmss"));
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"count"
} }
},
1);
topologyBuilder.SetBolt(
"Bolt",
Bolt.Get,
new Dictionary <string, List <string> >(),
1).shuffleGrouping("Spout");
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]);
TopologyBuilder topologyBuilder =
new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create <string, object>("int", eventHubPartitions),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""),
Tuple.Create <string, object>("int", 10),
Tuple.Create <string, object>("int", 1024),
Tuple.Create <string, object>("int", 1024 * eventHubPartitions),
Tuple.Create <string, object>("long", 0),
}
);
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"partialCount"
} }
},
eventHubPartitions,
enableAck
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary <string, List <string> >(),
1,
enableAck).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(eventHubPartitions);
if (enableAck)
{
topologyConfig.setNumAckers(eventHubPartitions);
}
else
{
topologyConfig.setNumAckers(0);
}
topologyConfig.setWorkerChildOps("-Xmx1g");
topologyConfig.setMaxSpoutPending((1024 * 1024) / 100);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
// Create a new topology named 'WordCount'
TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount");
// Add the spout to the topology.
// Name the component 'sentences'
// Name the field that is emitted as 'sentence'
topologyBuilder.SetSpout(
"sentences",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"sentence"
} }
},
1);
// Add the splitter bolt to the topology.
// Name the component 'splitter'
// Name the field that is emitted 'word'
// Use suffleGrouping to distribute incoming tuples
// from the 'sentences' spout across instances
// of the splitter
topologyBuilder.SetBolt(
"splitter",
Splitter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word"
} }
},
1).shuffleGrouping("sentences");
// Add the counter bolt to the topology.
// Name the component 'counter'
// Name the fields that are emitted 'word' and 'count'
// Use fieldsGrouping to ensure that tuples are routed
// to counter instances based on the contents of field
// position 0 (the word). This could also have been
// List<string>(){"word"}.
// This ensures that the word 'jumped', for example, will always
// go to the same instance
topologyBuilder.SetBolt(
"counter",
Counter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word", "count"
} }
},
1).fieldsGrouping("splitter", new List <int>()
{
0
});
//NOTE: This tends to work with parallelism hint set to 1.
//If you increase this, you should pre-create the dataset
//used by the bolt, as multiple instances of the bolt all
//trying to create a new dataset will sometimes create
//multiple datasets in Power BI, all with the same name.
topologyBuilder.SetBolt(
"PowerBI",
PowerBiBolt.Get,
new Dictionary <string, List <string> >(),
1).fieldsGrouping("counter", new List <int>()
{
0
});
// Add topology config
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
// Create a new topology named 'WordCount'
TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount");
// Add the spout to the topology.
// Name the component 'sentences'
// Name the field that is emitted as 'sentence'
topologyBuilder.SetSpout(
"sentences",
Spout.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"sentence"}}
},
1);
// Add the splitter bolt to the topology.
// Name the component 'splitter'
// Name the field that is emitted 'word'
// Use suffleGrouping to distribute incoming tuples
// from the 'sentences' spout across instances
// of the splitter
topologyBuilder.SetBolt(
"splitter",
Splitter.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word"}}, // a default stream with a named tuple field
{"cowbells", new List<string>(){"cowbell"}} // a named stream with a named tuple field
},
1).shuffleGrouping("sentences");
// Add the counter bolt to the topology.
// Name the component 'counter'
// Name the fields that are emitted 'word' and 'count'
// Use fieldsGrouping to ensure that tuples are routed
// to counter instances based on the contents of field
// position 0 (the word). This could also have been
// List<string>(){"word"}.
// This ensures that the word 'jumped', for example, will always
// go to the same instance
topologyBuilder.SetBolt(
"counter",
Counter.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"word", "count"}}
},
1).fieldsGrouping("splitter", Constants.DEFAULT_STREAM_ID, new List<int>() { 0 });
// Add the cowbell bolt, which reads from the cowbell stream
topologyBuilder.SetBolt(
"cowbellbolt",
CowBell.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"cowbellout"}} // emit a stream so we can see the cowbell
},
1).shuffleGrouping("splitter", "cowbells"); //subscribe to the stream named 'cowbell'
// Add topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
private Producer BuildProducer(string url)
{
var reuseConnectionProperty = this.senderOptions.GetReuseConnection();
var reuseConnection = reuseConnectionProperty.HasValue && reuseConnectionProperty.Value;
var source = new CancellationTokenSource();
var connection = this.connectionPool.Get(url, reuseConnection, source.Token);
this.logger.Trace($"Using connection [{connection.Id}] at URL=[{url}] to resolve a producer");
using (var topologyBuilder = new TopologyBuilder(connection))
{
var builder = new RouteResolverBuilder(this.bus.Endpoint, topologyBuilder, this.Configuration);
var routeResolverBuilderFunc = this.Configuration.Options.GetRouteResolverBuilder();
Assumes.True(
routeResolverBuilderFunc.HasValue,
"RouteResolverBuilder must be set for [{0}]",
this.Configuration.Label);
var routeResolver = routeResolverBuilderFunc.Value(builder);
var producer = new Producer(
this.bus.Endpoint,
connection,
this.Configuration.Label,
routeResolver,
this.Configuration.Options.IsConfirmationRequired());
if (this.Configuration.RequiresCallback)
{
var callbackConfiguration = this.CreateCallbackReceiverConfiguration(url);
var receiver = this.bus.RegisterReceiver(callbackConfiguration, true);
this.logger.Trace(
$"A sender of [{this.Configuration.Label}] requires a callback configuration; registering a receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}]");
this.logger.Trace(
$"A new callback receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}] has been successfully registered, getting one of its listeners with URL=[{producer.BrokerUrl}]...");
var listener = receiver.GetListener(l => l.BrokerUrl == producer.BrokerUrl);
if (listener == null)
{
throw new BusConfigurationException(
$"Unable to find a suitable listener for receiver {receiver}");
}
this.logger.Trace(
$"A listener at URL=[{listener.BrokerUrl}] belonging to callback receiver of [{callbackConfiguration.Label}] acquired");
listener.StopOnChannelShutdown = true;
producer.UseCallbackListener(listener);
this.logger.Trace(
$"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has registered a callback listener successfully");
}
producer.StopOnChannelShutdown = true;
producer.Stopped += (sender, args) =>
{
this.OnProducerStopped(url, sender, args);
};
return(producer);
}
}
public ITopologyBuilder GetTopologyBuilder()
{
var enableAck = bool.Parse(ConfigurationManager.AppSettings["EnableAck"]);
TopologyBuilder topologyBuilder =
new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create<string, object>("int", eventHubPartitions),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ""),
Tuple.Create<string, object>("int", 10),
Tuple.Create<string, object>("int", 1024),
Tuple.Create<string, object>("int", 1024*eventHubPartitions),
Tuple.Create<string, object>("long", 0),
}
);
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
topologyBuilder.SetJavaSpout("com.microsoft.eventhubs.spout.EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() {
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "partialCount" } }
},
eventHubPartitions,
enableAck
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary<string, List<string>>(),
1,
enableAck).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(eventHubPartitions);
if (enableAck)
{
topologyConfig.setNumAckers(eventHubPartitions);
}
else
{
topologyConfig.setNumAckers(0);
}
topologyConfig.setWorkerChildOps("-Xmx1g");
topologyConfig.setMaxSpoutPending((1024*1024)/100);
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
var topologyBuilder = new TopologyBuilder(typeof(EventHubReaderTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var EventHubNamespace = ConfigurationManager.AppSettings["EventHubNamespace"];
if (String.IsNullOrWhiteSpace(EventHubNamespace))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubNamespace");
}
var EventHubEntityPath = ConfigurationManager.AppSettings["EventHubEntityPath"];
if (String.IsNullOrWhiteSpace(EventHubEntityPath))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubEntityPath");
}
var EventHubSharedAccessKeyName = ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"];
if (String.IsNullOrWhiteSpace(EventHubSharedAccessKeyName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKeyName");
}
var EventHubSharedAccessKey = ConfigurationManager.AppSettings["EventHubSharedAccessKey"];
if (String.IsNullOrWhiteSpace(EventHubSharedAccessKey))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubSharedAccessKey");
}
var EventHubPartitions = ConfigurationManager.AppSettings["EventHubPartitions"];
if (String.IsNullOrWhiteSpace(EventHubPartitions))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "EventHubPartitions");
}
var partitionCount = int.Parse(EventHubPartitions);
//You can use the new SetEventHubSpout method by providing EventHubSpoutConfig which will automatically create Java code to instantiate this spout
//TODO: This method will not work if you do not include EventHub jar during publishing or deployment of this topology
topologyBuilder.SetEventHubSpout(
"EventHubSpout", //Set task name
new EventHubSpoutConfig(
EventHubSharedAccessKeyName,
EventHubSharedAccessKey,
EventHubNamespace,
EventHubEntityPath,
partitionCount),
partitionCount
);
//For a hybrid topology we need to declare a customize Java serializer that will serialize Java objects which will deseriailized in the bolt into C# objects
topologyBuilder.SetBolt(
typeof(LoggerBolt).Name, //Set task name
LoggerBolt.Get, //Set task constructor delegate
new Dictionary<string, List<string>>(), //Leave empty if the task has no outputSchema defined i.e. no outgoing tuples
1, //Set number of tasks
true //Set enableAck
).
DeclareCustomizedJavaSerializer(new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" } ).
globalGrouping("EventHubSpout");
//Set the topology config
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(4); //Set number of worker processes
topologyConfig.setMaxSpoutPending(1024); //Set maximum pending tuples from spout
topologyConfig.setWorkerChildOps("-Xmx1024m"); //Set Java Heap Size
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
topologyBuilder.SetEventHubSpout(
"EventHubSpout",
new EventHubSpoutConfig(
appConfig.EventHubSharedAccessKeyName,
appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace,
appConfig.EventHubEntityPath,
appConfig.EventHubPartitions),
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
topologyBuilder.SetBolt(
typeof(EventAggregator).Name,
EventAggregator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
//You can also setup a Ranker bolt to maintain top N records
/*
topologyBuilder.SetBolt(
typeof(EventRanker).Name,
EventRanker.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
},
appConfig.EventHubPartitions / 2
).
fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
*/
topologyBuilder.SetBolt(
typeof(EventHBaseWriter).Name,
EventHBaseWriter.Get,
new Dictionary<string, List<string>>(),
appConfig.EventHubPartitions / 4).
fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
//Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology
//The default JVM heap size for workers is 768m, we also increase that to 1024m
//That helps the java spout have additional heap size at disposal.
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(8);
topologyConfig.setMaxSpoutPending(1000);
topologyConfig.setWorkerChildOps("-Xmx1024m");
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
/// <summary>
/// Get a topology builder for this topology
/// </summary>
/// <returns></returns>
public ITopologyBuilder GetTopologyBuilder()
{
// Create a new topology, with a name of "WordCount" and the current date
TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount" + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Add the spout to the topology.
// Name the component 'sentences'
// Name the field that is emitted as 'sentence'
topologyBuilder.SetSpout(
"sentences",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"sentence"
} }
},
1);
// Add the splitter bolt.
// Name the component 'splitter'
// Name the field that is emitted 'word'
// Use shuffleGrouping to distribute incoming tuples
// from the 'sentences' spout across instances of
// the splitter
topologyBuilder.SetBolt(
"splitter",
Splitter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word"
} }
},
1).shuffleGrouping("sentences");
// Add the counter bolt.
// Name the component 'counter'.
// Name the fields that are emitted 'word' and 'count'
// Use fieldsGrouping to ensure that tuples are routed
// to counter instances based on the contents of field
// position 0 ('word'). This could also be List<string>(){"word"}
// Instead of position.
// This ensures that words go to the same instance. For example, 'jumped'
// will always go to the same instance, while 'cow' might always go to another instance.
topologyBuilder.SetBolt(
"counter",
Counter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word", "count"
} }
},
1).fieldsGrouping("splitter", new List <int>()
{
0
});
// Return the builder
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
appConfig.EventHubPartitions
);
topologyBuilder.SetBolt(
typeof(EventHubWriter).Name,
EventHubWriter.Get,
new Dictionary<string, List<string>>(),
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(8);
topologyConfig.setMaxSpoutPending(1600);
topologyBuilder.SetTopologyConfig(topologyConfig);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(AlarmsOnAggregatesTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
topologyBuilder.SetEventHubSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
//FLATT MESSAGES
topologyBuilder.SetBolt(
typeof(FlattBolt).Name,
FlattBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"sensor"
} }
},
eventHubPartitions,
true
).DeclareCustomizedJavaSerializer(javaSerializerInfo)
.shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout");
//AGGREGATE MESSAGES
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "10");
topologyBuilder.SetBolt(
typeof(AggregateBolt).Name,
AggregateBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"aggregated"
} }
},
parallelismHint: 1,
enableAck: false
)
.globalGrouping(typeof(FlattBolt).Name)
.addConfigurations(boltConfig);
//ALARMS MESSAGES
topologyBuilder.SetBolt(
typeof(AlarmsBolt).Name,
AlarmsBolt.Get,
new Dictionary <string, List <string> >()
{
},
parallelismHint: 4,
enableAck: false
).shuffleGrouping(typeof(AggregateBolt).Name);
var topologyConfig = new StormConfig();
topologyConfig.setMaxSpoutPending(8192);
topologyConfig.setNumWorkers(eventHubPartitions);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.spout.EventHubSpout. (com.microsoft.eventhubs.spout.EventHubSpoutConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" {4} """"))",
appConfig.EventHubUsername, appConfig.EventHubPassword,
appConfig.EventHubNamespace, appConfig.EventHubEntityPath,
appConfig.EventHubPartitions));
topologyBuilder.SetJavaSpout(
"EventHubSpout",
constructor,
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
typeof(EventAggregator).Name,
EventAggregator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
//You can also setup a Ranker bolt to maintain top N records
/*
* topologyBuilder.SetBolt(
* typeof(EventRanker).Name,
* EventRanker.Get,
* new Dictionary<string, List<string>>()
* {
* {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
* },
* appConfig.EventHubPartitions / 2
* ).
* fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
*/
topologyBuilder.SetBolt(
typeof(EventHBaseWriter).Name,
EventHBaseWriter.Get,
new Dictionary <string, List <string> >(),
appConfig.EventHubPartitions / 4).
fieldsGrouping(typeof(EventAggregator).Name, new List <int>()
{
0, 1, 2
});
//Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology
//The default JVM heap size for workers is 768m, we also increase that to 1024m
//That helps the java spout have additional heap size at disposal.
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.workers", "8" },
{ "topology.max.spout.pending", "1000" },
{ "topology.worker.childopts", @"""-Xmx1024m""" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
// Start building a new topology
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventHubReader).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Get the number of partitions in EventHub
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
// Add the EvetnHubSpout to the topology using the SetEventHubSpout and EventHubSpoutConfig helper methods.
// NOTE: These methods set the spout to read data in a String encoding.
/*
* topologyBuilder.SetEventHubSpout(
* "EventHubSpout",
* new EventHubSpoutConfig(
* ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
* ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
* ConfigurationManager.AppSettings["EventHubNamespace"],
* ConfigurationManager.AppSettings["EventHubEntityPath"],
* eventHubPartitions),
* eventHubPartitions);
*/
// The following is an example of how to create the same spout using the JavaComponentConstructor,
// which allows us to use UTF-8 encoding for reads.
// NOTE!!!! This only works with the 9.5 version of the Event Hub components, which are located at
// https://github.com/hdinsight/hdinsight-storm-examples/blob/master/lib/eventhubs/
// Create the UTF-8 data scheme
var schemeConstructor = new JavaComponentConstructor("com.microsoft.eventhubs.spout.UnicodeEventDataScheme");
// Create the EventHubSpoutConfig
var eventHubSpoutConfig = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpoutConfig",
new List <Tuple <string, object> >()
{
//comment
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"]),
//comment
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubSharedAccessKey"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubNamespace"]),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, ConfigurationManager.AppSettings["EventHubEntityPath"]),
Tuple.Create <string, object>("int", eventHubPartitions),
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.IEventDataScheme", schemeConstructor)
}
);
// Create the spout
var eventHubSpout = new JavaComponentConstructor(
"com.microsoft.eventhubs.spout.EventHubSpout",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>("com.microsoft.eventhubs.spout.EventHubSpoutConfig", eventHubSpoutConfig)
}
);
// Set the spout in the topology
topologyBuilder.SetJavaSpout("EventHubSpout", eventHubSpout, eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
// Create a config for the bolt. It's unused here
var boltConfig = new StormConfig();
// Add the logbolt to the topology
// Use a serializer to understand data from the Java component
topologyBuilder.SetBolt(
typeof(LogBolt).Name,
LogBolt.Get,
new Dictionary <string, List <string> >(),
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
// Create a configuration for the topology
var topologyConfig = new StormConfig();
// Increase max pending for the spout
topologyConfig.setMaxSpoutPending(8192);
// Parallelism hint for the number of workers to match the number of EventHub partitions
topologyConfig.setNumWorkers(eventHubPartitions);
// Add the config and return the topology builder
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_javaSpout_csharpBolt");
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List<object> constructorParams = new List<object>() { 100, "test", null };
List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" };
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Generator", constructorParams, paramTypes);
topologyBuilder.SetJavaSpout(
"generator",
constructor,
1);
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"};
topologyBuilder.SetBolt(
"displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List<string> javaDeserializerInfo = new List<string>()
{ "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"};
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}}
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List<object> constructorParams = new List<object>() { 100, "test", string.Empty };
List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" };
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes);
topologyBuilder.SetJavaBolt(
"displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
JavaComponentConstructor constructor =
new JavaComponentConstructor("com.microsoft.eventhubs.spout.EventHubSpout",
new List <object>()
{
appConfig.EventHubUsername,
appConfig.EventHubPassword,
appConfig.EventHubNamespace,
appConfig.EventHubEntityPath,
appConfig.EventHubPartitions
},
new List <string>()
{
"java.lang.String",
"java.lang.String",
"java.lang.String",
"java.lang.String",
"int"
}
);
topologyBuilder.SetJavaSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
constructor,
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"partialCount"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(new Dictionary <string, string>()
{
{ "topology.tick.tuple.freq.secs", "1" }
});
topologyBuilder.SetBolt(
typeof(DBGlobalCountBolt).Name,
DBGlobalCountBolt.Get,
new Dictionary <string, List <string> >(),
1).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(new Dictionary <string, string>()
{
{ "topology.tick.tuple.freq.secs", "1" }
});
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.workers", appConfig.EventHubPartitions.ToString() },
{ "topology.max.spout.pending", "512" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
string pluginName = string.Format("{0}.exe", GetType().Assembly.GetName().Name);
TopologyBuilder topologyBuilder = new TopologyBuilder("HelloWorld");
topologyBuilder.SetSpout(
"generator",
pluginName,
new List <string>()
{
"generator"
},
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"sentence"
} }
},
1);
topologyBuilder.SetBolt(
"splitter",
pluginName,
new List <string>()
{
"splitter"
},
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word", "firstLetterOfWord"
} }
},
1).shuffleGrouping("generator");
topologyBuilder.SetBolt(
"counter",
pluginName,
new List <string>()
{
"counter"
},
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word", "count"
} }
},
1).fieldsGrouping("splitter", new List <int>()
{
1
});
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
topologyBuilder.SetEventHubSpout(
"EventHubSpout",
new EventHubSpoutConfig(
appConfig.EventHubSharedAccessKeyName,
appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace,
appConfig.EventHubEntityPath,
appConfig.EventHubPartitions),
appConfig.EventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
topologyBuilder.SetBolt(
typeof(EventAggregator).Name,
EventAggregator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout");
//You can also setup a Ranker bolt to maintain top N records
/*
* topologyBuilder.SetBolt(
* typeof(EventRanker).Name,
* EventRanker.Get,
* new Dictionary<string, List<string>>()
* {
* {Constants.DEFAULT_STREAM_ID, new List<string>(){ "AggregationTimestamp", "PrimaryKey", "SecondaryKey", "AggregatedValue" } }
* },
* appConfig.EventHubPartitions / 2
* ).
* fieldsGrouping(typeof(EventAggregator).Name, new List<int>() { 0, 1, 2 });
*/
topologyBuilder.SetBolt(
typeof(EventHBaseWriter).Name,
EventHBaseWriter.Get,
new Dictionary <string, List <string> >(),
appConfig.EventHubPartitions / 4).
fieldsGrouping(typeof(EventAggregator).Name, new List <int>()
{
0, 1, 2
});
//Assuming a 4 'Large' node cluster we will use half of the worker slots for this topology
//The default JVM heap size for workers is 768m, we also increase that to 1024m
//That helps the java spout have additional heap size at disposal.
var topologyConfig = new StormConfig();
topologyConfig.setNumWorkers(8);
topologyConfig.setMaxSpoutPending(1000);
topologyConfig.setWorkerChildOps("-Xmx1024m");
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("AlertTopology");
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
topologyBuilder.SetEventHubSpout(
"EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
var boltConfig = new StormConfig();
topologyBuilder.SetBolt(
typeof(ParserBolt).Name,
ParserBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"temp", "createDate", "deviceId"
} }
},
eventHubPartitions,
enableAck: true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(EmitAlertBolt).Name,
EmitAlertBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"reason", "temp", "createDate", "deviceId"
} }
},
eventHubPartitions,
enableAck: true
).
shuffleGrouping(typeof(ParserBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setMaxSpoutPending(8192);
topologyConfig.setNumWorkers(eventHubPartitions);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
/// <summary>
/// Builds a set of producers constructing one producer for each URL in the connection string
/// </summary>
private void BuildProducers()
{
var reuseConnectionProperty = this.senderOptions.GetReuseConnection();
var reuseConnection = reuseConnectionProperty.HasValue && reuseConnectionProperty.Value;
this.logger.Trace(
$"Building producers of [{this.Configuration.Label}]:\r\n\t{string.Join("\r\n\t", this.senderOptions.RabbitConnectionString.Select(url => $"Producer({this.Configuration.Label}): URL\t=>\t{url}"))}");
foreach (var url in this.senderOptions.RabbitConnectionString)
{
var source = new CancellationTokenSource();
var connection = this.connectionPool.Get(url, reuseConnection, source.Token);
this.logger.Trace($"Using connection [{connection.Id}] at URL=[{url}] to resolve a producer");
var topologyBuilder = new TopologyBuilder(connection.OpenChannel());
var builder = new RouteResolverBuilder(this.bus.Endpoint, topologyBuilder, this.Configuration);
var routeResolverBuilder = this.Configuration.Options.GetRouteResolverBuilder();
Assumes.True(
routeResolverBuilder.HasValue,
"RouteResolverBuilder must be set for [{0}]",
this.Configuration.Label);
var routeResolver = routeResolverBuilder.Value(builder);
var producer = new Producer(
this.bus.Endpoint,
connection,
this.Configuration.Label,
routeResolver,
this.Configuration.Options.IsConfirmationRequired());
producer.Failed += p =>
{
// A failed producer will not be removed from the collection cause this failure should be treated as transient
this.logger.Error($"Producer [{producer}] has failed");
};
if (this.Configuration.RequiresCallback)
{
var callbackConfiguration = this.Configuration.CallbackConfiguration;
this.logger.Trace(
$"A sender of [{this.Configuration.Label}] requires a callback configuration; registering a receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}]");
var receiver = this.bus.RegisterReceiver(this.Configuration.CallbackConfiguration);
this.logger.Trace(
$"A new callback receiver of [{callbackConfiguration.Label}] with connection string [{callbackConfiguration.Options.GetConnectionString()}] has been successfully registered, getting one of its listeners with URL=[{producer.BrokerUrl}]...");
var listener = receiver.GetListener(l => l.BrokerUrl == producer.BrokerUrl);
if (listener == null)
{
throw new BusConfigurationException(
$"Unable to find a suitable listener for receiver {receiver}");
}
this.logger.Trace(
$"A listener at URL=[{listener.BrokerUrl}] belonging to callback receiver of [{callbackConfiguration.Label}] acquired");
producer.UseCallbackListener(listener);
this.logger.Trace(
$"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has registered a callback listener successfully");
}
this.producers.Add(producer);
this.logger.Trace(
$"A producer of [{producer.Label}] at URL=[{producer.BrokerUrl}] has been added to the sender");
}
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder(typeof(EventCountHybridTopology).Name + DateTime.Now.ToString("yyyyMMddHHmmss"));
var eventHubPartitions = int.Parse(ConfigurationManager.AppSettings["EventHubPartitions"]);
topologyBuilder.SetEventHubSpout(
"com.microsoft.eventhubs.spout.EventHubSpout",
new EventHubSpoutConfig(
ConfigurationManager.AppSettings["EventHubSharedAccessKeyName"],
ConfigurationManager.AppSettings["EventHubSharedAccessKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
ConfigurationManager.AppSettings["EventHubEntityPath"],
eventHubPartitions),
eventHubPartitions);
// Set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, full name of the Java JSON Serializer class is required
List <string> javaSerializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer"
};
var boltConfig = new StormConfig();
boltConfig.Set("topology.tick.tuple.freq.secs", "1");
topologyBuilder.SetBolt(
typeof(PartialCountBolt).Name,
PartialCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"partialCount"
} }
},
eventHubPartitions,
true
).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("com.microsoft.eventhubs.spout.EventHubSpout").
addConfigurations(boltConfig);
topologyBuilder.SetBolt(
typeof(GlobalCountBolt).Name,
GlobalCountBolt.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"timestamp", "totalCount"
} }
},
1,
true).
globalGrouping(typeof(PartialCountBolt).Name).
addConfigurations(boltConfig);
var topologyConfig = new StormConfig();
topologyConfig.setMaxSpoutPending(8192);
topologyConfig.setNumWorkers(eventHubPartitions);
topologyBuilder.SetTopologyConfig(topologyConfig);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
// Create a new topology named 'WordCount'
TopologyBuilder topologyBuilder = new TopologyBuilder("WordCount" + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Add the spout to the topology.
// Name the component 'sentences'
// Name the field that is emitted as 'sentence'
topologyBuilder.SetSpout(
"sentences",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"sentence"
} }
},
1);
// Add the splitter bolt to the topology.
// Name the component 'splitter'
// Name the field that is emitted 'word'
// Use suffleGrouping to distribute incoming tuples
// from the 'sentences' spout across instances
// of the splitter
topologyBuilder.SetBolt(
"splitter",
Splitter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word"
} }
},
1).shuffleGrouping("sentences");
// Add the counter bolt to the topology.
// Name the component 'counter'
// Name the fields that are emitted 'word' and 'count'
// Use fieldsGrouping to ensure that tuples are routed
// to counter instances based on the contents of field
// position 0 (the word). This could also have been
// List<string>(){"word"}.
// This ensures that the word 'jumped', for example, will always
// go to the same instance
topologyBuilder.SetBolt(
"counter",
Counter.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"word", "count"
} }
},
1).fieldsGrouping("splitter", new List <int>()
{
0
});
// Add topology config
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}