Ejemplo n.º 1
0
        KinesisRecord make_kinesis_record(int min_serialized_size = 0)
        {
            var kr = new KinesisRecord();
            int num_user_records = 1 + (new Random().Next() % 100);

            Func <UserRecord> make_ur = () =>
            {
                return(TestUtil.make_user_record(
                           new Random().Next().ToString(),
                           new Random().Next().ToString(),
                           new Random().Next().ToString(),
                           10000,
                           "myStream"));
            };

            for (int i = 0; i < num_user_records; i++)
            {
                kr.add(make_ur());
            }

            if (min_serialized_size > 0)
            {
                while ((int)kr.accurate_size() < min_serialized_size)
                {
                    kr.add(make_ur());
                }
            }

            return(kr);
        }
Ejemplo n.º 2
0
        public static void verify(List <UserRecord> original, KinesisRecord kr)
        {
            AggregatedRecord ar = null;

            verify_format(original, kr, ref ar);
            verify_content(original, ar);
        }
Ejemplo n.º 3
0
        public void KinesisRecordUnitTest_Clearing()
        {
            int N = 10;

            List <UserRecord> user_records = new List <UserRecord>();
            KinesisRecord     r            = new KinesisRecord();

            for (int i = 0; i < N; i++)
            {
                var ur = TestUtil.make_user_record();
                user_records.Add(ur);
                r.add(ur);
            }

            TestUtil.verify(user_records, r);

            for (int i = 0; i < 5; i++)
            {
                r.clear();
                user_records.Clear();

                for (int j = 0; j < N; j++)
                {
                    var ur = TestUtil.make_user_record();
                    user_records.Add(ur);
                    r.add(ur);
                }

                TestUtil.verify(user_records, r);
            }
        }
Ejemplo n.º 4
0
        public void KinesisRecordUnitTest_AccurateSize()
        {
            for (int i = 0; i < 100; i++)
            {
                KinesisRecord r           = new KinesisRecord();
                long          num_records = new Random().Next(1, 512);
                for (long j = 0; j < num_records; j++)
                {
                    int key_size  = new Random().Next(1, 256);
                    int data_size = new Random().Next(1, 64 * 1024);

                    var ur = TestUtil.make_user_record(new string('a', key_size),
                                                       new string('a', data_size),
                                                       new Random().Next() % 2 == 0 ? "" : "123");
                    r.add(ur);
                }
                int    predicted  = (int)r.accurate_size();
                string serialized = r.serialize();

                if (r.Items().Count > 1)
                {
                    Assert.AreEqual(predicted, serialized.Length - 16);
                }
                else
                {
                    Assert.AreEqual(predicted, serialized.Length);
                }
            }
        }
Ejemplo n.º 5
0
        public void ReducerUnitTest_CountLimit()
        {
            int limit           = 100;
            var reducer         = make_reducer(int.MaxValue, limit);
            List <UserRecord> v = new List <UserRecord>();

            for (int j = 0; j < 3; j++)
            {
                v.Clear();
                for (int i = 0; i < limit; i++)
                {
                    var ur = TestUtil.make_user_record();
                    v.Add(ur);
                    KinesisRecord result = reducer.add(ur) as KinesisRecord;
                    if (i < limit - 1)
                    {
                        Assert.IsNull(result);
                    }
                    else
                    {
                        Assert.IsNotNull(result);

                        TestUtil.verify(v, result);
                        Assert.AreEqual((int)reducer.size(), 0);
                    }
                }
            }
        }
Ejemplo n.º 6
0
        KinesisRecord make_kinesis_record(DateTime deadline, DateTime expiration)
        {
            var ur = TestUtil.make_user_record();

            ur.set_deadline(deadline);
            ur.set_expiration(expiration);
            ur.Predicted_shard(0);
            var kr = new KinesisRecord();

            kr.add(ur);
            return(kr);
        }
Ejemplo n.º 7
0
        public void KinesisRecordUnitTest_EstimatedSize()
        {
            for (int i = 0; i < 50; i++)
            {
                KinesisRecord r           = new KinesisRecord();
                int           num_records = new Random().Next(2, 512);

                // non repeated partition keys
                for (int j = 0; j < num_records; j++)
                {
                    int key_size  = new Random().Next(1, 256);
                    int data_size = new Random().Next(1, 64 * 1024);
                    var ur        = TestUtil.make_user_record(new string('a', key_size),
                                                              new string('a', data_size),
                                                              new Random().Next() % 2 == 0 ? "" : "123");
                    r.add(ur);
                }

                // repeated partition keys
                int key_size1 = new Random().Next(1, 256);
                for (long j = 0; j < num_records; j++)
                {
                    int data_size = new Random().Next(1, 64 * 1024);
                    var ur        = TestUtil.make_user_record(new string('a', key_size1),
                                                              new string('a', data_size),
                                                              new Random().Next() % 2 == 0 ? "" : "123");
                    r.add(ur);
                }

                // small keys small data
                for (long j = 0; j < num_records; j++)
                {
                    var ur = TestUtil.make_user_record(new string('a', 2),
                                                       new string('a', 2),
                                                       new Random().Next() % 2 == 0 ? "" : "123");
                    r.add(ur);
                }

                int    estimated  = (int)r.Estimated_size();
                string serialized = r.serialize();

                double diff            = (double)serialized.Length - estimated;
                double percentage_diff = diff / serialized.Length * 100;
                percentage_diff *= percentage_diff < 0 ? -1 : 1;

                StringBuilder ss = new StringBuilder();
                ss.Append("Estimated size should be within 1 percent or 32 bytes of actual ")
                .Append("size, estimate was ").Append(estimated).Append(", actual size was ")
                .Append(serialized.Length).Append(" (").Append(percentage_diff).Append("% difference)");
                //BOOST_CHECK_MESSAGE(percentage_diff < 1 || diff < 32, ss.ToString());
            }
        }
Ejemplo n.º 8
0
        public static void verify_unaggregated(UserRecord ur, KinesisRecord kr)
        {
            var serialized = kr.serialize();

            Assert.AreEqual(ur.Data().ToString(Encoding.Default), serialized);
            Assert.AreEqual(ur.Partition_key(), kr.partition_key());
            if (ur.explicit_hash_key().ToString() != "-1")
            {
                Assert.AreEqual(ur.explicit_hash_key().ToString(), kr.explicit_hash_key());
            }
            else
            {
                Assert.AreEqual(KPLNETInterface.Utils.GetDecimalHashKey(ur.Partition_key()).ToString(), kr.explicit_hash_key());
            }
        }
Ejemplo n.º 9
0
        public void ReducerUnitTest_NonEmpty()
        {
            KinesisRecord kr      = null;
            var           reducer = make_reducer(
                int.MaxValue,
                int.MaxValue,
                (result) =>
            {
                kr = result;
            });

            reducer.Flush();
            Thread.Sleep(100);
            Assert.IsNull(kr); //"Flush should not have produced a KinesisRecord because the Reducer is empty"
        }
Ejemplo n.º 10
0
        public void KinesisRecordUnitTest_Empty()
        {
            KinesisRecord r = new KinesisRecord();

            Assert.AreEqual(0, (int)r.accurate_size());
            Assert.AreEqual(0, (int)r.Estimated_size());

            try
            {
                r.serialize();
                Assert.Fail("Calling serialize on empty KinesisRecord should cause exception");
            }
            catch (Exception e)
            {
                // expected
            }
        }
Ejemplo n.º 11
0
        public void KinesisRecordUnitTest_SingleRecordTestMethod()
        {
            // No explicit hash key (ehk)
            {
                KinesisRecord r  = new KinesisRecord();
                var           ur = TestUtil.make_user_record();
                r.add(ur);
                TestUtil.verify_unaggregated(ur, r);
            }

            // With ehk
            {
                KinesisRecord r  = new KinesisRecord();
                var           ur = TestUtil.make_user_record("a", "a", "123");
                r.add(ur);
                TestUtil.verify_unaggregated(ur, r);
            }
        }
Ejemplo n.º 12
0
        public void ReducerUnitTest_Deadline()
        {
            KinesisRecord kr      = null;
            var           reducer = make_reducer(
                int.MaxValue,
                int.MaxValue,
                (result) => { kr = result; });

            List <UserRecord> v = new List <UserRecord>();

            for (int i = 0; i < 100; i++)
            {
                var ur = TestUtil.make_user_record(
                    "pk",
                    new Random().Next().ToString(),
                    "123",
                    5000 + i);
                v.Add(ur);
                reducer.add(ur);
            }

            // Should not flush after 1 second because we've set the deadlines to be 5
            Thread.Sleep(1000);
            Assert.AreEqual((int)reducer.size(), 100);

            // Now we're going to put a record with a deadline that's now to trigger the
            // flush.
            var ur1 = TestUtil.make_user_record(
                "pk",
                new Random().Next().ToString(),
                "123",
                0);

            reducer.add(ur1);
            // This record should be moved to the front when flushed because of its
            // deadline
            v.Insert(0, ur1);

            Thread.Sleep(300);
            Assert.AreEqual((int)reducer.size(), 0);
            Assert.IsNotNull(kr);
            TestUtil.verify(v, kr);
        }
Ejemplo n.º 13
0
        AwsKinesisResult make_prr_ctx(int num_kr, int num_ur_per_kr, string error, Amazon.Kinesis.Model.PutRecordsResponse outcome, DateTime start, DateTime end)
        {
            List <KinesisRecord> krs = new List <KinesisRecord>();

            for (int i = 0; i < num_kr; i++)
            {
                var kr = new KinesisRecord();
                for (int j = 0; j < num_ur_per_kr; j++)
                {
                    var ur = TestUtil.make_user_record();
                    ur.Predicted_shard(i);
                    kr.add(ur);
                }
                krs.Add(kr);
            }
            AwsKinesisResult result = new AwsKinesisResult(error, new AwsKinesisResponse(outcome), new PutRecordsRequest(), start, end);

            result.context <PutRecordsRequest>().Items().AddRange(krs);
            return(result);
        }
Ejemplo n.º 14
0
        public static void verify_format(List <UserRecord> original, KinesisRecord kr, ref AggregatedRecord container)
        {
            container = null;
            byte[] serialized = kr.SerializedAggregatedRecord;

            // verify magic number
            byte[] expected_magic = null;
            unchecked
            {
                expected_magic = new byte[] { (byte)-13, (byte)-119, (byte)-102, (byte)-62 };
            }
            int magic_len = expected_magic.Length;

            byte[] magic = serialized.Take(magic_len).ToArray();
            Assert.IsTrue(KPLNETInterface.Utils.AreArrayEqual(expected_magic, magic));

            // verify protobuf payload
            byte[] payload = serialized.Skip(expected_magic.Length).Take(serialized.Length - 16 - magic_len).ToArray();
            container = AggregatedRecord.Parser.ParseFrom(Google.Protobuf.ByteString.CopyFrom(payload));
            Assert.IsNotNull(container);

            // verify md5 checksum
            Assert.IsTrue(KPLNETInterface.Utils.AreArrayEqual(KPLNETInterface.Utils.GetMD5(payload), serialized.Skip(serialized.Length - 16).Take(16).ToArray()));

            // verify the explicit hash key set on the Kinesis record
            List <string> acceptable_hash_keys = new List <string>();

            foreach (var ur in original)
            {
                if (ur.explicit_hash_key() > -1)
                {
                    acceptable_hash_keys.Add(ur.explicit_hash_key().ToString());
                }
                else
                {
                    acceptable_hash_keys.Add(KPLNETInterface.Utils.GetDecimalHashKey(ur.Partition_key()).ToString());
                }
            }

            Assert.IsTrue(acceptable_hash_keys.Exists((i) => i == kr.explicit_hash_key()));
        }
Ejemplo n.º 15
0
        public void KinesisRecordUnitTest_SameEHK()
        {
            int N = 1000;
            int M = 100;

            List <UserRecord> user_records = new List <UserRecord>();
            KinesisRecord     r            = new KinesisRecord();

            for (int i = 0; i < N; i++)
            {
                var ur = TestUtil.make_user_record("pk", "data", "123");
                user_records.Add(ur);
                r.add(ur);
            }
            for (int i = 0; i < M; i++)
            {
                r.remove_last();
                user_records.RemoveAt(user_records.Count - 1);
            }

            TestUtil.verify(user_records, r);
        }
Ejemplo n.º 16
0
        public void KinesisRecordUnitTest_Deadlines()
        {
            // The nearest deadline should always be kept
            {
                KinesisRecord r     = new KinesisRecord();
                var           start = DateTime.Now;
                for (int i = 0; i < 10; i++)
                {
                    var ur = TestUtil.make_user_record();
                    ur.set_deadline(start.AddMilliseconds(i * 100));
                    ur.set_expiration(start.AddMilliseconds(i * 100));
                    r.add(ur);
                }
                Assert.IsTrue(r.Deadline() == start);
                Assert.IsTrue(r.Expiration() == start);
            }

            // If a nearer deadline comes in, it should override the previous
            {
                KinesisRecord r       = new KinesisRecord();
                var           earlier = DateTime.Now;
                var           later   = earlier.AddMilliseconds(500);
                {
                    var ur = TestUtil.make_user_record();
                    ur.set_deadline(later);
                    r.add(ur);
                    Assert.IsTrue(r.Deadline() == later);
                }
                {
                    var ur = TestUtil.make_user_record();
                    ur.set_deadline(earlier);
                    r.add(ur);
                    Assert.IsTrue(r.Deadline() == earlier);
                }
                // Removing the last added record should restore the previous deadline
                r.remove_last();
                Assert.IsTrue(r.Deadline() == later);
            }
        }
Ejemplo n.º 17
0
        public void KinesisRecordUnitTest_DifferentParitionKey()
        {
            int N = 1000;
            int M = 100;

            List <UserRecord> user_records = new List <UserRecord>();
            KinesisRecord     r            = new KinesisRecord();

            for (int i = 0; i < N; i++)
            {
                var ur = TestUtil.make_user_record(i.ToString());
                user_records.Add(ur);
                r.add(ur);
            }
            for (int i = 0; i < M; i++)
            {
                r.remove_last();
                user_records.RemoveAt(user_records.Count - 1);
            }

            TestUtil.verify(user_records, r);
        }
Ejemplo n.º 18
0
        public void AggregatorUnitTest_BasicTestMethod()
        {
            var aggregator = make_aggregator();

            for (int shard_id = 1; shard_id <= 3; shard_id++)
            {
                List <UserRecord> v  = new List <UserRecord>();
                KinesisRecord     kr = null;

                for (int i = 0; i < kCountLimit; i++)
                {
                    var ur = TestUtil.make_user_record("pk", TestUtil.random_string(new Random().Next(100)), TestUtil.get_hash_key(shard_id).ToString(), 10000 + i, "MyStream", 0);
                    kr = aggregator.put(ur);
                    v.Add(ur);

                    Assert.IsFalse(ur.Predicted_shard() == -1);
                    Assert.AreEqual(ur.Predicted_shard(), shard_id);
                }

                Assert.IsNotNull(kr);
                TestUtil.verify(v, kr);
            }
        }
Ejemplo n.º 19
0
        public void KinesisRecordUnitTest_MixedParitionKey()
        {
            List <string> keys = new List <string>()
            {
                "a", "b", "b", "a", "a", "c", "b", "a", "d", "e", "e", "d", "d", "d"
            };

            List <UserRecord> user_records = new List <UserRecord>();
            KinesisRecord     r            = new KinesisRecord();

            for (int i = 0; i < keys.Count; i++)
            {
                var ur = TestUtil.make_user_record(keys[i]);
                user_records.Add(ur);
                r.add(ur);
            }
            for (int i = 0; i < 3; i++)
            {
                r.remove_last();
                user_records.RemoveAt(user_records.Count - 1);
            }
            TestUtil.verify(user_records, r);
        }
Ejemplo n.º 20
0
        public void KinesisRecordUnitTest_MixedEHK()
        {
            List <string> keys = new List <string>()
            {
                "1", "2", "2", "1", "1", "3", "2", "1", "4", "5", "5", "4", "4", "4"
            };

            List <UserRecord> user_records = new List <UserRecord>();
            KinesisRecord     r            = new KinesisRecord();

            for (int i = 0; i < keys.Count; i++)
            {
                var ur = TestUtil.make_user_record("pk", "data", keys[i]);
                user_records.Add(ur);
                r.add(ur);
            }
            for (int i = 0; i < 3; i++)
            {
                r.remove_last();
                user_records.RemoveAt(user_records.Count - 1);
            }

            TestUtil.verify(user_records, r);
        }
Ejemplo n.º 21
0
        public void ReducerUnitTest_ResetTimeout()
        {
            KinesisRecord kr      = null;
            var           reducer = make_reducer(
                300,
                int.MaxValue,
                (result) =>
            {
                kr = result;
            });

            List <UserRecord> v = new List <UserRecord>();
            int k = 0;

            for (int i = 0; i < 10; i++)
            {
                var ur = TestUtil.make_user_record(
                    "pk",
                    new Random().Next().ToString(),
                    "123",
                    2000 + k++);
                v.Add(ur);
                reducer.add(ur);
            }

            // This record has a much closer deadline
            {
                var ur = TestUtil.make_user_record(
                    "pk",
                    new Random().Next().ToString(),
                    "123",
                    100);
                v.Insert(0, ur);
                reducer.add(ur);
            }

            // Put records until flush happens
            KinesisRecord kr2;

            {
                do
                {
                    // The other records have a further deadline.
                    var ur = TestUtil.make_user_record(
                        "pk",
                        new Random().Next().ToString(),
                        "123",
                        2000 + k++);
                    v.Add(ur);
                    kr2 = reducer.add(ur) as KinesisRecord;
                } while (kr2 == null);
            }

            // Put a few more to make sure the buffer is not empty
            for (int i = 0; i < 10; i++)
            {
                var ur = TestUtil.make_user_record(
                    "pk",
                    new Random().Next().ToString(),
                    "123",
                    2000 + k++);
                v.Add(ur);
                reducer.add(ur);
            }

            // No flush should happen in the next second even though we had a record with
            // a 100ms deadline - that record should've gotten flushed by the size limit,
            // and its deadline should no longer apply.
            Thread.Sleep(1000);
            Assert.IsNull(kr);

            // The timer should now be set to the min deadline of the remaining records.
            // It should go off after another second or so
            Thread.Sleep(1500);
            Assert.IsNotNull(kr);

            // Check data integrity
            Assert.AreEqual(kr.Size() + kr2.Size() + reducer.size(), (ulong)v.Count);
            for (int i = v.Count - 1; i >= (int)kr.Size() + (int)kr2.Size(); i--)
            {
                v.RemoveAt(i);
            }

            List <UserRecord> v2 = new List <UserRecord>();

            for (int i = (int)kr2.Size() - 1; i >= 0; i--)
            {
                v2.Insert(0, v[i]);
                v.RemoveAt(i);
            }
            TestUtil.verify(v, kr);
            TestUtil.verify(v2, kr2);
        }
Ejemplo n.º 22
0
        public void ReducerUnitTest_Concurrency()
        {
            int counter = 0;
            //LOG(info) << "Starting concurrency test. If this doesn't finish in 30 " << "seconds or so it probably means there's a deadlock.";
            ConcurrentBag <UserRecord>    put     = new ConcurrentBag <UserRecord>();
            ConcurrentBag <KinesisRecord> results = new ConcurrentBag <KinesisRecord>();

            var           reducer = make_reducer(5000, int.MaxValue, (result) => { results.Add(result); });
            List <Thread> threads = new List <Thread>();

            for (int i = 0; i < 16; i++)
            {
                var t = new Thread(() =>
                {
                    for (int j = 0; j < 32; j++)
                    {
                        KinesisRecord kr = null;
                        do
                        {
                            int count = Interlocked.Increment(ref counter);
                            var ur    = TestUtil.make_user_record("pk", count.ToString(), "123", count);
                            put.Add(ur);
                            kr = reducer.add(ur) as KinesisRecord;
                        } while (kr == null);

                        results.Add(kr);

                        // Call flush sometimes too to mix things up further
                        if (i % 8 == 0 && i == j)
                        {
                            reducer.Flush();
                        }
                    }
                });
                t.Start();
                threads.Add(t);
            }

            foreach (var t in threads)
            {
                t.Join();
            }

            while (reducer.size() > 0)
            {
                reducer.Flush();
                Thread.Sleep(1000);
            }

            Assert.AreEqual((int)reducer.size(), 0);

            //LOG(info) << "Finished putting data, " << counter << " records put. "
            //          << "Analyzing results...";

            // Check that all records made it out. Order is no longer guaranteed with
            // many workers, but we've put a monotonically increasing number in the
            // record data, so that will allow us to sort and match records up.
            List <UserRecord> put_v = new List <UserRecord>(put.ToArray());

            List <UserRecord> result_v = new List <UserRecord>();
            var resultsArray           = results.ToArray();

            for (int i = 0; i < results.Count; i++)
            {
                result_v.AddRange(resultsArray[i].Items());
            }
            Comparison <UserRecord> ckr = (a, b) => { return(string.Compare(a.Data().ToStringUtf8(), b.Data().ToStringUtf8())); };

            put_v.Sort(ckr);
            result_v.Sort(ckr);

            Assert.AreEqual(put_v.Count, result_v.Count);
            for (int i = 0; i < put_v.Count; i++)
            {
                Assert.AreEqual(put_v[i], result_v[i]);
            }
        }