public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaCsharpBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List<string> javaDeserializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person" };
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}}
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor constructor = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Displayer",
new List<Tuple<string, object>>()
{
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"),
Tuple.Create<string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty)
});
// The java bolt "java_displayer" receives from the C# spout "generator"
topologyBuilder.SetJavaBolt(
"java_displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set a customized JSON Serializer to serialize a Java object (emitted by Java Spout) into JSON string
// Here, fullname of the Java JSON Serializer class is required
List<string> javaSerializerInfo = new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONSerializer" };
// The C# bolt "csharp-displayer" receive from the C# spout "generator"
topologyBuilder.SetBolt(
"csharp-displayer",
Displayer.Get,
new Dictionary<string, List<string>>(),
1).
DeclareCustomizedJavaSerializer(javaSerializerInfo).
shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setDebug(false);
conf.setNumWorkers(1);
conf.setStatsSampleRate(0.05);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
// Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize
// Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields
List <string> javaDeserializerInfo =
new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String"
};
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"Event"
} }
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
//We will use CreateFromClojureExpr method as we wish to pass in a complex Java object
//The EventHubBolt takes a EventHubBoltConfig that we will create using clojure
//NOTE: We need to escape the quotes for strings that need to be passes to clojure
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace, appConfig.EventHubFqnAddress,
appConfig.EventHubEntityPath, "true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
//Assuming a 4 'L' node cluster, we will have 16 worker slots available
//We will half of those slots for this topology
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.workers", "8" },
{ "topology.max.spout.pending", "1600" }
});
return(topologyBuilder);
}
/// <summary>
/// Builds a topology that can be submitted to Storm on HDInsight
/// </summary>
/// <returns>A topology builder</returns>
public ITopologyBuilder GetTopologyBuilder()
{
//The friendly name is 'EventHubWriter'
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss"));
//Get the partition count
int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]);
//Create a deserializer for JSON to java.lang.String
//so that Java components can consume data emitted by
//C# components
List <string> javaDeserializerInfo =
new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String"
};
//Set the spout
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"Event"
} }
},
partitionCount). //Parallelism hint uses partition count
DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream
//Create constructor for the Java bolt
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
ConfigurationManager.AppSettings["EventHubPolicyName"],
ConfigurationManager.AppSettings["EventHubPolicyKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
"servicebus.windows.net", //suffix for servicebus fqdn
ConfigurationManager.AppSettings["EventHubName"],
"true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
partitionCount). //Parallelism hint uses partition count
shuffleGrouping("Spout"); //Consume data from spout
StormConfig config = new StormConfig();
config.setNumWorkers(1); //Set the number of workers
topologyBuilder.SetTopologyConfig(config);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List <string> javaDeserializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"
};
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"person"
} }
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
JavaComponentConstructor constructor = new JavaComponentConstructor(
"microsoft.scp.example.HybridTopology.Displayer",
new List <Tuple <string, object> >()
{
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_PRIMITIVE_TYPE_INT, 100),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, "test"),
Tuple.Create <string, object>(JavaComponentConstructor.JAVA_LANG_STRING, string.Empty)
});
topologyBuilder.SetJavaBolt(
"displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set topology config
StormConfig conf = new StormConfig();
conf.setDebug(false);
conf.setNumWorkers(1);
conf.setStatsSampleRate(0.05);
conf.setWorkerChildOps("-Xmx1024m");
conf.Set("topology.kryo.register", "[\"[B\"]");
topologyBuilder.SetTopologyConfig(conf);
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
appConfig = new AppConfig();
TopologyBuilder topologyBuilder = new TopologyBuilder(this.GetType().Name);
// Set a customized JSON Deserializer to deserialize a C# object (emitted by C# Spout) into JSON string for Java to Deserialize
// Here, fullname of the Java JSON Deserializer class is required followed by the Java types for each of the fields
List<string> javaDeserializerInfo =
new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" };
topologyBuilder.SetSpout(
typeof(EventGenerator).Name,
EventGenerator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
appConfig.EventHubPartitions,
true
).
DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
//We will use CreateFromClojureExpr method as we wish to pass in a complex Java object
//The EventHubBolt takes a EventHubBoltConfig that we will create using clojure
//NOTE: We need to escape the quotes for strings that need to be passes to clojure
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
appConfig.EventHubSharedAccessKeyName, appConfig.EventHubSharedAccessKey,
appConfig.EventHubNamespace, appConfig.EventHubFqnAddress,
appConfig.EventHubEntityPath, "true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
appConfig.EventHubPartitions
).
shuffleGrouping(typeof(EventGenerator).Name);
//Assuming a 4 'L' node cluster, we will have 16 worker slots available
//We will half of those slots for this topology
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.workers", appConfig.EventHubPartitions.ToString()},
{"topology.max.spout.pending","4096"}
});
return topologyBuilder;
}
/// <summary>
/// Builds a topology that can be submitted to Storm on HDInsight
/// </summary>
/// <returns>A topology builder</returns>
public ITopologyBuilder GetTopologyBuilder()
{
// Friendly name of the topology
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss"));
// Number of partitions in Event Hub. Used for parallelization.
int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]);
// Deserializer used to deserialize JSON data from C# components to java.lang.string
List <string> javaDeserializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String"
};
// Spout that emits randomly generated JSON data
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"event"
} }
},
partitionCount)
.DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Java construcvtor for the Event Hub Bolt
JavaComponentConstructor constructor = JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(org.apache.storm.eventhubs.bolt.EventHubBolt. (org.apache.storm.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
ConfigurationManager.AppSettings["EventHubPolicyName"],
ConfigurationManager.AppSettings["EventHubPolicyKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
"servicebus.windows.net",
ConfigurationManager.AppSettings["EventHubName"],
"true"));
// Set the bolt to subscribe to data from the spout
topologyBuilder.SetJavaBolt(
"eventhubbolt",
constructor,
partitionCount)
.shuffleGrouping("Spout");
// Return the topology builder
return(topologyBuilder);
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List <string> javaDeserializerInfo = new List <string>()
{
"microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"
};
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary <string, List <string> >()
{
{ Constants.DEFAULT_STREAM_ID, new List <string>()
{
"person"
} }
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List <object> constructorParams = new List <object>()
{
100, "test", string.Empty
};
List <string> paramTypes = new List <string>()
{
"int", "java.lang.String", "java.lang.String"
};
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes);
topologyBuilder.SetJavaBolt(
"displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary <string, string>()
{
{ "topology.kryo.register", "[\"[B\"]" }
});
return(topologyBuilder);
}
/// <summary>
/// Builds a topology that can be submitted to Storm on HDInsight
/// </summary>
/// <returns>A topology builder</returns>
public ITopologyBuilder GetTopologyBuilder()
{
//The friendly name is 'EventHubWriter'
TopologyBuilder topologyBuilder = new TopologyBuilder("EventHubWriter" + DateTime.Now.ToString("yyyyMMddHHmmss"));
//Get the partition count
int partitionCount = int.Parse(ConfigurationManager.AppSettings["EventHubPartitionCount"]);
//Create a deserializer for JSON to java.lang.String
//so that Java components can consume data emitted by
//C# components
List<string> javaDeserializerInfo =
new List<string>() { "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "java.lang.String" };
//Set the spout
topologyBuilder.SetSpout(
"Spout",
Spout.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"Event"}}
},
partitionCount). //Parallelism hint uses partition count
DeclareCustomizedJavaDeserializer(javaDeserializerInfo); //Deserializer for the output stream
//Create constructor for the Java bolt
JavaComponentConstructor constructor =
JavaComponentConstructor.CreateFromClojureExpr(
String.Format(@"(com.microsoft.eventhubs.bolt.EventHubBolt. (com.microsoft.eventhubs.bolt.EventHubBoltConfig. " +
@"""{0}"" ""{1}"" ""{2}"" ""{3}"" ""{4}"" {5}))",
ConfigurationManager.AppSettings["EventHubPolicyName"],
ConfigurationManager.AppSettings["EventHubPolicyKey"],
ConfigurationManager.AppSettings["EventHubNamespace"],
"servicebus.windows.net", //suffix for servicebus fqdn
ConfigurationManager.AppSettings["EventHubName"],
"true"));
topologyBuilder.SetJavaBolt(
"EventHubBolt",
constructor,
partitionCount). //Parallelism hint uses partition count
shuffleGrouping("Spout"); //Consume data from spout
StormConfig config = new StormConfig();
config.setNumWorkers(1); //Set the number of workers
topologyBuilder.SetTopologyConfig(config);
return topologyBuilder;
}
public ITopologyBuilder GetTopologyBuilder()
{
TopologyBuilder topologyBuilder = new TopologyBuilder("HybridTopology_csharpSpout_javaBolt");
// Demo how to set a customized JSON Deserializer to deserialize a JSON string into Java object (to send to a Java Bolt)
// Here, fullname of the Java JSON Deserializer class and target deserialized class are required
List<string> javaDeserializerInfo = new List<string>()
{ "microsoft.scp.storm.multilang.CustomizedInteropJSONDeserializer", "microsoft.scp.example.HybridTopology.Person"};
topologyBuilder.SetSpout(
"generator",
Generator.Get,
new Dictionary<string, List<string>>()
{
{Constants.DEFAULT_STREAM_ID, new List<string>(){"person"}}
},
1,
null).DeclareCustomizedJavaDeserializer(javaDeserializerInfo);
// Demo how to set parameters to initialize the constructor of Java Spout/Bolt
List<object> constructorParams = new List<object>() { 100, "test", string.Empty };
List<string> paramTypes = new List<string>() { "int", "java.lang.String", "java.lang.String" };
JavaComponentConstructor constructor = new JavaComponentConstructor("microsoft.scp.example.HybridTopology.Displayer", constructorParams, paramTypes);
topologyBuilder.SetJavaBolt(
"displayer",
constructor,
1).shuffleGrouping("generator");
// Demo how to set topology config
topologyBuilder.SetTopologyConfig(new Dictionary<string, string>()
{
{"topology.kryo.register","[\"[B\"]"}
});
return topologyBuilder;
}