public KafkaSpout(Context ctx)
{
Context.Logger.Info("KafkaSpout constructor called");
this.ctx = ctx;
if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL)
{
zkAddr = GetZkAddr();
Console.Write("zookeeper address: " + zkAddr);
stateStore = StateStore.Get(statPath, zkAddr);
}
Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >();
outputSchema.Add("mydefault", new List <Type>()
{
typeof(KafkaMeta)
});
this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema));
}
public TxGenerator(Context ctx)
{
Context.Logger.Info("TxGenerator constructor called");
this.ctx = ctx;
// Declare Output schema
Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >();
outputSchema.Add("default", new List <Type>()
{
typeof(Person)
});
this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema));
this.ctx.DeclareCustomizedSerializer(new CustomizedInteropJSONSerializer());
// Demo how to use StateStore in Tx topology to persistence some states
// Get zookeeper address from storm conf, which is passed from java side
zkAddr = GetZkAddr();
zkRoot = GetZkRoot();
stateStore = StateStore.Get(zkRoot + "/" + statPath, zkAddr);
}
public Generator(Context ctx)
{
Context.Logger.Info("Generator constructor called");
this.ctx = ctx;
// Declare Output schema
Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >();
outputSchema.Add("default", new List <Type>()
{
typeof(string)
});
this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema));
// Demo how to use StateStore in Tx topology to persistence some states
if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL)
{
// Get zookeeper address from storm conf, which is passed from java side
zkAddr = GetZkAddr();
zkRoot = GetZkRoot();
stateStore = StateStore.Get(zkRoot + "/" + statPath, zkAddr);
}
}
public void StateStoreThreadTesting()
{
StateStore.Start("StateStoreTests");
Thread TestThread1 = new Thread(new ParameterizedThreadStart(SetValueForThreadTest));
Thread TestThread2 = new Thread(new ParameterizedThreadStart(SetValueForThreadTest));
TestThread1.Start("Test0");
TestThread2.Start("Test1");
TestThread1.Join();
TestThread2.Join();
StateStore.Save();
object Value1 = null;
object Value2 = null;
Thread TestThread3 = new Thread(() => { Value1 = StateStore.Get("TestValue-THREADING"); });
Thread TestThread4 = new Thread(() => { Value2 = StateStore.Get("TestValue-THREADING"); });
TestThread3.Start();
TestThread4.Start();
TestThread3.Join();
TestThread4.Join();
Assert.AreEqual(Value1, Value2);
Assert.IsTrue(Value1 != null);
}
public void BasicStateStoreTest()
{
const string Key = "TestValue-BASIC";
StateStore.Start("StateStoreTests");
StateStore.Set(Key, "TestValue-1");
StateStore.Set(Key, "TestValue-2");
StateStore.Set(Key, "TestValue-0");
StateStore.Set(Key, "TestValue-3");
StateStore.Set(Key, "TestValue-4");
StateStore.Set(Key, "TestValue-5");
StateStore.Set(Key, "TestValue-6");
StateStore.Set(Key, "TestValue-7");
StateStore.Set(Key, "TestValue-8");
StateStore.Set(Key, "TestValue-9");
StateStore.Set(Key, "TestValue-10");
Assert.AreEqual("TestValue-10", StateStore.Get(Key));
Assert.AreEqual(null, StateStore.Get("ThisValueIsNotInTheDictionary"));
StateStore.Save();
}
public void NextTx(out long seqId, Dictionary <string, Object> parms)
{
Context.Logger.Info("NextTx enter");
KafkaMeta meta = (KafkaMeta)parms[Constants.KAFKA_META];
// the previous kafka meta which get from zookeeper
KafkaMeta preMeta = null;
string regKey = "LastSend-" + meta.Topic;
var reg = stateStore.Get <Registry>(regName);
if (reg.ExistsKey(regKey))
{
preMeta = reg.GetKeyValue <KafkaMeta>(regKey);
}
else
{
reg.CreateKey(regKey);
}
bool isReady = false;
if (preMeta != null)
{
foreach (int i in preMeta.PartitionOffsets.Keys)
{
long preEndOffset = preMeta.PartitionOffsets[i].EndOffset;
// set current begin offset by previous endoffset from stored kafka meta
meta.PartitionOffsets[i].BeginOffset = preEndOffset;
long endOffset = meta.PartitionOffsets[i].EndOffset;
Context.Logger.Info(String.Format("For partition {0}, the begin offset {1}, the end offset {2}", i, preEndOffset, endOffset));
if (endOffset > preEndOffset)
{
isReady = true;
}
}
}
else
{
isReady = true;
Context.Logger.Info("preMeta is null");
}
if (isReady)
{
State state = stateStore.Create();
Context.Logger.Info("stateid in spout {0}", state.ID);
meta.StateId = BitConverter.GetBytes(state.ID);
// emit kafka meta
this.ctx.Emit("mydefault", new Values(meta));
// save kafka meta to zookeeper
reg.SetKeyValue(regKey, meta);
seqId = state.ID;
Context.Logger.Info("NextTx exit, seqId: {0}", seqId);
}
else
{
seqId = -1L;
}
System.Threading.Thread.Sleep(1000);
}