示例#1
0
        public async Task Process(MyType e, StreamSequenceToken sequenceToken) // Implements the Process method from IFilter
        {
            List <MyType> result = new List <MyType>();

            if (e.value == "watermark")
            {
                result.Add(e);
            }
            else
            {
                string Key = await jobManager.GetKey(this.GetPrimaryKey());

                string Value = await jobManager.GetValue(this.GetPrimaryKey());

                MyType new_e = NewEvent.CreateNewEvent(e, Key, Value);
                result = Apply(new_e);
            }
            List <Task> t       = new List <Task>();
            List <Guid> streams = await jobManager.GetPublish(this.GetPrimaryKey());

            foreach (var item1 in streams)
            {
                var stream = streamProvider.GetStream <MyType>(item1, null);
                foreach (var item2 in result)
                {
                    t.Add(stream.OnNextAsync(item2));
                }
            }
            await Task.WhenAll(t);
        }
示例#2
0
        private async Task Process(MyType e, StreamSequenceToken sequenceToken)
        {
            // Console.WriteLine($"Filter Grain receives: {e.key}, {e.value}, {e.timestamp.GetTimestamp()}");
            bool   emit_e;
            MyType ee;

            if (e.value == "watermark")
            {
                ee     = e;
                emit_e = true;
            }
            else
            {
                string Key = await jobManager.GetKey(this.GetPrimaryKey());

                string Value = await jobManager.GetValue(this.GetPrimaryKey());

                ee     = NewEvent.CreateNewEvent(e, Key, Value);
                emit_e = Apply(ee);
            }

            List <Task> t = new List <Task>();

            if (emit_e) // If the function returns true, send the element to SinkGrain
            {
                List <Guid> streams = await jobManager.GetPublish(this.GetPrimaryKey());

                foreach (var item in streams)
                {
                    var stream = streamProvider.GetStream <MyType>(item, null);
                    t.Add(stream.OnNextAsync(ee));
                }
            }
            await Task.WhenAll(t);
        }
示例#3
0
        private async Task Join(long end, List <MyType> input1, List <MyType> input2)
        {
            List <Task> t       = new List <Task>();
            List <Guid> streams = await jobManager.GetPublish(this.GetPrimaryKey());

            foreach (MyType r1 in input1)
            {
                foreach (MyType r2 in input2)
                {
                    //    Console.WriteLine($"Grain: {this.GetPrimaryKey()} r1.key:{r1.key} r1.value:{r1.value}  r2.key:{r2.key} r2.value{r2.value}");
                    if (r1.key == r2.key)
                    {
                        MyType r = new MyType(r1.key, r1.value + " " + r2.value, r1.timestamp);
                        //Console.WriteLine($"joinGrain emit join result: {r.key}, {r.value}, {r.timestamp.GetTimestamp()}");
                        foreach (var item in streams)
                        {
                            var stream = streamProvider.GetStream <MyType>(item, null);
                            t.Add(stream.OnNextAsync(r));
                        }
                    }
                }
            }
            await Task.WhenAll(t);

            // send a watermark after sending all result events for a window
            MyType watermark = new MyType("", "watermark", new Timestamp(end));

            foreach (var item in streams)
            {
                var stream = streamProvider.GetStream <MyType>(item, null);
                await stream.OnNextAsync(watermark);
            }
        }
示例#4
0
        private async Task Process(MyType e, StreamSequenceToken sequenceToken)
        {
            List <Guid> streams = await jobManager.GetPublish(this.GetPrimaryKey());

            List <Task> t = new List <Task>();

            Console.WriteLine($"AggregateGrain receives:{e.key}, {e.value}, {e.timestamp.GetTimestamp()}");

            MyType ee;

            if (e.value == "watermark")
            {
                ee = e;
            }
            else
            {
                string Key = await jobManager.GetKey(this.GetPrimaryKey());

                string Value = await jobManager.GetValue(this.GetPrimaryKey());

                ee = NewEvent.CreateNewEvent(e, Key, Value);
            }

            var r = func.FeedData(ee);   // r could be an empty dictionary

            if (r.Count > 0)
            {
                foreach (KeyValuePair <long, List <MyType> > records in r)
                {
                    MyType        watermark = new MyType("", "watermark", new Timestamp(records.Key));
                    List <MyType> result    = Apply(records.Value);
                    foreach (var item in streams)
                    {
                        var stream = streamProvider.GetStream <MyType>(item, null);
                        foreach (var record in result)
                        {
                            t.Add(stream.OnNextAsync(record));
                        }
                    }
                    await Task.WhenAll(t);

                    t = new List <Task>();
                    foreach (var item in streams)
                    {
                        var stream = streamProvider.GetStream <MyType>(item, null);
                        t.Add(stream.OnNextAsync(watermark));
                    }
                    await Task.WhenAll(t);
                }
                await Task.WhenAll(t);
            }
            else
            {
                await Task.CompletedTask;
            }
        }
示例#5
0
        // reference: https://github.com/dotnet/orleans/blob/master/src/Orleans.Core.Abstractions/Streams/Core/StreamSequenceToken.cs
        private async Task Process(string message, StreamSequenceToken sequenceToken)
        {
            //Console.WriteLine($"SourceGrain receives: {message}");
            string[] parts       = message.Split(" ");
            string   new_message = "";

            // new_message is without timestamp column
            for (int i = 0; i < parts.Length - 1; i++)
            {
                new_message += parts[i] + " ";
            }
            long      time      = Convert.ToInt64(parts[parts.Length - 1]);
            Timestamp timestamp = new Timestamp(time);

            // reference: https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/using-structs
            var    record         = new MyType("", new_message, timestamp);
            bool   emit_watermark = false;
            MyType watermark      = new MyType("", "watermark", new Timestamp(-1));

            // see if need to emit watermark
            if (time > currentMaxTimestamp)
            {
                currentMaxTimestamp = time;
                watermark.timestamp.SetTimestamp(currentMaxTimestamp - delay - 1);  // important !!!!!!
                emit_watermark = true;
            }

            // ask the Job Manager what is the guid for the stream, and publish the data to some streams
            List <Guid> streams = await jobManager.GetPublish(this.GetPrimaryKey());

            List <Task> t = new List <Task>();

            foreach (var item in streams)
            {
                var stream = streamProvider.GetStream <MyType>(item, null);
                t.Add(stream.OnNextAsync(record));
            }
            await Task.WhenAll(t);

            t = new List <Task>();
            if (emit_watermark)
            {
                foreach (var item in streams)
                {
                    var stream = streamProvider.GetStream <MyType>(item, null);
                    t.Add(stream.OnNextAsync(watermark));
                }
            }
            await Task.WhenAll(t);
        }