public KafkaSpout(Context ctx) { Context.Logger.Info("KafkaSpout constructor called"); this.ctx = ctx; if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL) { zkAddr = GetZkAddr(); Console.Write("zookeeper address: " + zkAddr); stateStore = StateStore.Get(statPath, zkAddr); } Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >(); outputSchema.Add("mydefault", new List <Type>() { typeof(KafkaMeta) }); this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema)); }
public TxGenerator(Context ctx) { Context.Logger.Info("TxGenerator constructor called"); this.ctx = ctx; // Declare Output schema Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >(); outputSchema.Add("default", new List <Type>() { typeof(Person) }); this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema)); this.ctx.DeclareCustomizedSerializer(new CustomizedInteropJSONSerializer()); // Demo how to use StateStore in Tx topology to persistence some states // Get zookeeper address from storm conf, which is passed from java side zkAddr = GetZkAddr(); zkRoot = GetZkRoot(); stateStore = StateStore.Get(zkRoot + "/" + statPath, zkAddr); }
public Generator(Context ctx) { Context.Logger.Info("Generator constructor called"); this.ctx = ctx; // Declare Output schema Dictionary <string, List <Type> > outputSchema = new Dictionary <string, List <Type> >(); outputSchema.Add("default", new List <Type>() { typeof(string) }); this.ctx.DeclareComponentSchema(new ComponentStreamSchema(null, outputSchema)); // Demo how to use StateStore in Tx topology to persistence some states if (Context.pluginType != SCPPluginType.SCP_NET_LOCAL) { // Get zookeeper address from storm conf, which is passed from java side zkAddr = GetZkAddr(); zkRoot = GetZkRoot(); stateStore = StateStore.Get(zkRoot + "/" + statPath, zkAddr); } }
public void StateStoreThreadTesting() { StateStore.Start("StateStoreTests"); Thread TestThread1 = new Thread(new ParameterizedThreadStart(SetValueForThreadTest)); Thread TestThread2 = new Thread(new ParameterizedThreadStart(SetValueForThreadTest)); TestThread1.Start("Test0"); TestThread2.Start("Test1"); TestThread1.Join(); TestThread2.Join(); StateStore.Save(); object Value1 = null; object Value2 = null; Thread TestThread3 = new Thread(() => { Value1 = StateStore.Get("TestValue-THREADING"); }); Thread TestThread4 = new Thread(() => { Value2 = StateStore.Get("TestValue-THREADING"); }); TestThread3.Start(); TestThread4.Start(); TestThread3.Join(); TestThread4.Join(); Assert.AreEqual(Value1, Value2); Assert.IsTrue(Value1 != null); }
public void BasicStateStoreTest() { const string Key = "TestValue-BASIC"; StateStore.Start("StateStoreTests"); StateStore.Set(Key, "TestValue-1"); StateStore.Set(Key, "TestValue-2"); StateStore.Set(Key, "TestValue-0"); StateStore.Set(Key, "TestValue-3"); StateStore.Set(Key, "TestValue-4"); StateStore.Set(Key, "TestValue-5"); StateStore.Set(Key, "TestValue-6"); StateStore.Set(Key, "TestValue-7"); StateStore.Set(Key, "TestValue-8"); StateStore.Set(Key, "TestValue-9"); StateStore.Set(Key, "TestValue-10"); Assert.AreEqual("TestValue-10", StateStore.Get(Key)); Assert.AreEqual(null, StateStore.Get("ThisValueIsNotInTheDictionary")); StateStore.Save(); }
public void NextTx(out long seqId, Dictionary <string, Object> parms) { Context.Logger.Info("NextTx enter"); KafkaMeta meta = (KafkaMeta)parms[Constants.KAFKA_META]; // the previous kafka meta which get from zookeeper KafkaMeta preMeta = null; string regKey = "LastSend-" + meta.Topic; var reg = stateStore.Get <Registry>(regName); if (reg.ExistsKey(regKey)) { preMeta = reg.GetKeyValue <KafkaMeta>(regKey); } else { reg.CreateKey(regKey); } bool isReady = false; if (preMeta != null) { foreach (int i in preMeta.PartitionOffsets.Keys) { long preEndOffset = preMeta.PartitionOffsets[i].EndOffset; // set current begin offset by previous endoffset from stored kafka meta meta.PartitionOffsets[i].BeginOffset = preEndOffset; long endOffset = meta.PartitionOffsets[i].EndOffset; Context.Logger.Info(String.Format("For partition {0}, the begin offset {1}, the end offset {2}", i, preEndOffset, endOffset)); if (endOffset > preEndOffset) { isReady = true; } } } else { isReady = true; Context.Logger.Info("preMeta is null"); } if (isReady) { State state = stateStore.Create(); Context.Logger.Info("stateid in spout {0}", state.ID); meta.StateId = BitConverter.GetBytes(state.ID); // emit kafka meta this.ctx.Emit("mydefault", new Values(meta)); // save kafka meta to zookeeper reg.SetKeyValue(regKey, meta); seqId = state.ID; Context.Logger.Info("NextTx exit, seqId: {0}", seqId); } else { seqId = -1L; } System.Threading.Thread.Sleep(1000); }