public void testBloomFilter()
        {
            // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
            using (Stream file = File.OpenWrite(TestFilePath))
            {
                OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
                options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
                options.stripeSize(100000);
                options.compress(CompressionKind.ZLIB);
                options.bufferSize(10000);
                options.rowIndexStride(1000);
                options.bloomFilterColumns("S");
                using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
                {
                    Random r1 = new Random(1);
                    for (int i = 0; i < 21000; ++i)
                    {
                        writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
                            TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
                    }
                }
            }

            string outputFilename = "orc-file-dump-bloomfilter.out";
            using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
            {
                FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=3" });
            }

            TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
        }
Exemple #2
0
        public long Integer(long min = long.MinValue, long max = long.MaxValue)
        {
            ThrowIfValueHigherThan(nameof(min), min, max);

            var rand = new System.Random();

            return(rand.NextLong(min, max));
        }
Exemple #3
0
        public string Hexadecimal(long min = long.MinValue, long max = long.MaxValue)
        {
            ThrowIfValueHigherThan(nameof(min), min, max);

            var rand        = new System.Random();
            var randomValue = rand.NextLong(min, max);

            return(randomValue.ToString("X2"));
        }
Exemple #4
0
        public string AlphaNumeric(short minLength = 0, short maxLength = short.MaxValue)
        {
            ThrowIfValueHigherThan(nameof(minLength), minLength, maxLength);
            ThrowIfValueLowerThan(nameof(minLength), minLength, 0);

            var rand   = new System.Random();
            var length = (short)rand.NextLong(minLength, maxLength);

            return(faker.Random.AlphaNumeric(length));
        }
        public void testJsonDump()
        {
            ObjectInspector inspector;
            inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord));
            // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
            OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
                .inspector(inspector)
                .stripeSize(100000)
                .compress(CompressionKind.ZLIB)
                .bufferSize(10000)
                .rowIndexStride(1000)
                .bloomFilterColumns("s");
            using (Stream file = File.OpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
            {
                Random r1 = new Random(1);
                for (int i = 0; i < 21000; ++i)
                {
                    if (i % 100 == 0)
                    {
                        writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), null));
                    }
                    else
                    {
                        writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
                            TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
                    }
                }
            }

            const string outputFilename = "orc-file-dump.json";
            using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
            {
                FileDump.Main(new string[] { TestFilePath.ToString(), "-j", "-p", "--rowindex=3" });
            }

            TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
        }
Exemple #6
0
        public static object Next(this System.Random generator, Type desired)
        {
            switch (Type.GetTypeCode(desired))
            {
            case TypeCode.Boolean:
                return(generator.NextBool());

            case TypeCode.Byte:
                return(generator.NextByte());

            case TypeCode.Char:
                return(generator.NextChar());

            case TypeCode.DateTime:
                return(generator.NextDateTime());

            case TypeCode.Decimal:
                return(generator.NextDecimal());

            case TypeCode.Double:
                return(generator.NextDouble());

            case TypeCode.Int16:
                return(generator.NextShort());

            case TypeCode.Int32:
                return(generator.NextInt());

            case TypeCode.Int64:
                return(generator.NextLong());

            case TypeCode.SByte:
                return(generator.NextSByte());

            case TypeCode.Single:
                return(generator.NextFloat());

            case TypeCode.UInt16:
                return(generator.NextUShort());

            case TypeCode.UInt32:
                return(generator.NextUInt());

            case TypeCode.UInt64:
                return(generator.NextULong());

            default:
                throw new ArgumentOutOfRangeException("Cannot provide a random " + desired);
            }
        }
        private long FreeId()
        {
            long value;

            while (true)
            {
                value = random.NextLong(1, long.MaxValue);
                if (IsQueued(value))
                {
                    continue;
                }
                if (GetExecutingThread(value) != null)
                {
                    continue;
                }
                return(value);
            }
        }
Exemple #8
0
        public void ParseTest()
        {
            Random random = new Random();
            for (int i = 0; i != 100; ++i)
            {
                UInt48 expected = (UInt48)random.NextLong(UInt48.MaxValue + 1);

                UInt48 actual = UInt48.Parse(expected.ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture);
                Assert.AreEqual(expected, actual);

                actual = UInt48.Parse(expected.ToString(), NumberStyles.Integer);
                Assert.AreEqual(expected, actual);

                actual = UInt48.Parse(expected.ToString(), CultureInfo.InvariantCulture);
                Assert.AreEqual(expected, actual);

                actual = UInt48.Parse(expected.ToString());
                Assert.AreEqual(expected, actual);
            }
        }
Exemple #9
0
        public long Even(long min = long.MinValue, long max = long.MaxValue)
        {
            ThrowIfValueHigherThan(nameof(min), min, max);

            if (min == max && min % 2 != 0)
            {
                throw new ArgumentException($"Unable to get even number between {min} and {max}");
            }

            var  rand         = new System.Random();
            long randomNumber = rand.NextLong(min, max);

            if (randomNumber % 2 == 0)
            {
                return(randomNumber);
            }
            if (randomNumber + 1 <= max)
            {
                return(randomNumber + 1);
            }
            return(randomNumber - 1);
        }
Exemple #10
0
 private long RandomSeekPos(Random random, long size)
 {
     if (random == null || size <= 3L)
     {
         return 0L;
     }
     return (random.NextLong() & long.MaxValue) % (size / 3);
 }
Exemple #11
0
 /// <summary>
 /// Returns a random long over all possible values of long (except long.MaxValue, similar to
 /// random.Next())
 /// </summary>
 /// <param name="random">The given random instance</param>
 public static long NextLong(this System.Random random)
 {
     return(random.NextLong(long.MinValue, long.MaxValue));
 }
        public void testBitPack64Large()
        {
            ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(long));

            const int size = 1080832;
            long[] inp = new long[size];
            Random rand = new Random(1234);
            for (int i = 0; i < size; i++)
            {
                inp[i] = rand.NextLong();
            }
            List<long> input = inp.ToList();

            using (Stream file = File.OpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
                .inspector(inspector)
                .compress(CompressionKind.ZLIB)))
            {
                foreach (long l in input)
                {
                    writer.addRow(l);
                }
            }

            Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
            using (RecordReader rows = reader.rows())
            {
                int idx = 0;
                while (rows.hasNext())
                {
                    object row = rows.next();
                    Assert.Equal(input[idx++], ((long)row));
                }
            }
        }
Exemple #13
0
 public static Real Range(Real min, Real max)
 {
     return(Real.CreateFromRaw(_random.NextLong(min.RawValue, max.RawValue)));
 }
Exemple #14
0
 /// <summary>
 /// Returns a random long from 0 (inclusive) to max (exclusive)
 /// </summary>
 /// <param name="random">The given random instance</param>
 /// <param name="max">The exclusive maximum bound. Must be greater than 0</param>
 public static long NextLong(this System.Random random, long max)
 {
     return(random.NextLong(0, max));
 }
Exemple #15
0
 /// <summary>
 /// 0.0 以上 1.0 以下のランダムな浮動小数点数を返します。
 /// </summary>
 /// <returns>0.0 以上 1.0 以下の倍精度浮動小数点数。</returns>
 public static double UnitInterval(this Original.Random random)
 {
     return(random.NextLong() / (double)long.MaxValue);
 }
 public static long Random(this long value, long min, long max)
 {
     return(random_0.NextLong(min, max));
 }
        public void columnProjection()
        {
            ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(InnerStruct));

            int minInt = 0, maxInt = 0;
            string minStr = null, maxStr = null;
            Random r1 = null, r2 = null;
            using (Stream file = FileOpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
                    .inspector(inspector)
                    .stripeSize(1000)
                    .compress(CompressionKind.NONE)
                    .bufferSize(100)
                    .rowIndexStride(1000)))
            {
                r1 = new Random(1);
                r2 = new Random(2);
                int x;
                string y;
                for (int i = 0; i < 21000; ++i)
                {
                    x = r1.Next();
                    y = Long.toHexString(r2.NextLong());
                    if (i == 0 || x < minInt)
                    {
                        minInt = x;
                    }
                    if (i == 0 || x > maxInt)
                    {
                        maxInt = x;
                    }
                    if (i == 0 || y.CompareTo(minStr) < 0)
                    {
                        minStr = y;
                    }
                    if (i == 0 || y.CompareTo(maxStr) > 0)
                    {
                        maxStr = y;
                    }
                    writer.addRow(new InnerStruct(x, y));
                }
            }

            Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));

            // check out the statistics
            ColumnStatistics[] stats = reader.getStatistics();
            Assert.Equal(3, stats.Length);
            foreach (ColumnStatistics s in stats)
            {
                Assert.Equal(21000, s.getNumberOfValues());
                if (s is IntegerColumnStatistics)
                {
                    Assert.Equal(minInt, ((IntegerColumnStatistics)s).getMinimum());
                    Assert.Equal(maxInt, ((IntegerColumnStatistics)s).getMaximum());
                }
                else if (s is StringColumnStatistics)
                {
                    Assert.Equal(maxStr, ((StringColumnStatistics)s).getMaximum());
                    Assert.Equal(minStr, ((StringColumnStatistics)s).getMinimum());
                }
            }

            // check out the types
            IList<OrcProto.Type> types = reader.getTypes();
            Assert.Equal(3, types.Count);
            Assert.Equal(OrcProto.Type.Types.Kind.STRUCT, types[0].Kind);
            Assert.Equal(2, types[0].SubtypesCount);
            Assert.Equal(1, (int)types[0].GetSubtypes(0));
            Assert.Equal(2, (int)types[0].GetSubtypes(1));
            Assert.Equal(OrcProto.Type.Types.Kind.INT, types[1].Kind);
            Assert.Equal(0, types[1].SubtypesCount);
            Assert.Equal(OrcProto.Type.Types.Kind.STRING, types[2].Kind);
            Assert.Equal(0, types[2].SubtypesCount);

            // read the contents and make sure they match
            using (RecordReader rows1 = reader.rows(new bool[] { true, true, false }))
            using (RecordReader rows2 = reader.rows(new bool[] { true, false, true }))
            {
                r1 = new Random(1);
                r2 = new Random(2);
                for (int i = 0; i < 21000; ++i)
                {
                    Assert.Equal(true, rows1.hasNext());
                    Assert.Equal(true, rows2.hasNext());
                    OrcStruct row1 = (OrcStruct)rows1.next();
                    OrcStruct row2 = (OrcStruct)rows2.next();
                    Assert.Equal(r1.Next(), row1.getFieldValue(0));
                    Assert.Equal(Long.toHexString(r2.NextLong()),
                        row2.getFieldValue(1).ToString());
                }
                Assert.Equal(false, rows1.hasNext());
                Assert.Equal(false, rows2.hasNext());
            }
        }
        public void testZeroCopySeek()
        {
            ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(BigRow));

            const int COUNT = 32768;
            long[] intValues = new long[COUNT];
            double[] doubleValues = new double[COUNT];
            string[] stringValues = new string[COUNT];
            byte[][] byteValues = new byte[COUNT][];
            string[] words = new string[128];

            using (Stream file = FileOpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
                .inspector(inspector)
                .stripeSize(200000)
                .bufferSize(65536)
                .rowIndexStride(1000)))
            {
                Random rand = new Random(42);
                for (int i = 0; i < words.Length; ++i)
                {
                    words[i] = Integer.toHexString(rand.Next());
                }
                for (int i = 0; i < COUNT / 2; ++i)
                {
                    intValues[2 * i] = rand.NextLong();
                    intValues[2 * i + 1] = intValues[2 * i];
                    stringValues[2 * i] = words[rand.Next(words.Length)];
                    stringValues[2 * i + 1] = stringValues[2 * i];
                }
                for (int i = 0; i < COUNT; ++i)
                {
                    doubleValues[i] = rand.NextDouble();
                    byte[] buf = new byte[20];
                    rand.NextBytes(buf);
                    byteValues[i] = buf;
                }
                for (int i = 0; i < COUNT; ++i)
                {
                    writer.addRow(createRandomRow(intValues, doubleValues, stringValues,
                        byteValues, words, i));
                }
            }

            Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
            Assert.Equal(COUNT, reader.getNumberOfRows());
            /* enable zero copy record reader */
            #if false
            Configuration conf = new Configuration();
            HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_ZEROCOPY, true);
            #endif
            using (RecordReader rows = reader.rows())
            {
                /* all tests are identical to the other seek() tests */
                for (int i = COUNT - 1; i >= 0; --i)
                {
                    rows.seekToRow(i);
                    OrcStruct row = (OrcStruct)rows.next();
                    BigRow expected = createRandomRow(intValues, doubleValues,
                        stringValues, byteValues, words, i);
                    Assert.Equal(expected.boolean1, row.getFieldValue(0));
                    Assert.Equal(expected.byte1, row.getFieldValue(1));
                    Assert.Equal(expected.short1, row.getFieldValue(2));
                    Assert.Equal(expected.int1, row.getFieldValue(3));
                    Assert.Equal(expected.long1, row.getFieldValue(4));
                    Assert.Equal(expected.float1, (float)row.getFieldValue(5), 4);
                    Assert.Equal(expected.double1, (double)row.getFieldValue(6), 4);
                    Assert.Equal(expected.bytes1, row.getFieldValue(7));
                    Assert.Equal(expected.string1, row.getFieldValue(8));
                    List<InnerStruct> expectedList = expected.middle.list;
                    List<object> actualList =
                        (List<object>)((OrcStruct)row.getFieldValue(9)).getFieldValue(0);
                    compareList(expectedList, actualList);
                    compareList(expected.list, (List<object>)row.getFieldValue(10));
                }
            }

            IList<StripeInformation> stripes = reader.getStripes();
            long offsetOfStripe2 = 0;
            long offsetOfStripe4 = 0;
            long lastRowOfStripe2 = 0;
            for (int i = 0; i < 5; ++i)
            {
                StripeInformation stripe = stripes[i];
                if (i < 2)
                {
                    lastRowOfStripe2 += stripe.getNumberOfRows();
                }
                else if (i == 2)
                {
                    offsetOfStripe2 = stripe.getOffset();
                    lastRowOfStripe2 += stripe.getNumberOfRows() - 1;
                }
                else if (i == 4)
                {
                    offsetOfStripe4 = stripe.getOffset();
                }
            }
            bool[] columns = new bool[reader.getStatistics().Length];
            columns[5] = true; // long colulmn
            columns[9] = true; // text column
            /* use zero copy record reader */
            using (RecordReader rows = reader.rowsOptions(new RecordReaderOptions()
                .range(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2)
                .include(columns)))
            {
                rows.seekToRow(lastRowOfStripe2);
                for (int i = 0; i < 2; ++i)
                {
                    OrcStruct row = (OrcStruct)rows.next();
                    BigRow expected = createRandomRow(intValues, doubleValues,
                                                      stringValues, byteValues, words,
                                                      (int)(lastRowOfStripe2 + i));

                    Assert.Equal(expected.long1, row.getFieldValue(4));
                    Assert.Equal(expected.string1, row.getFieldValue(8));
                }
            }
        }
        public void testDictionaryThreshold()
        {
            // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
            // conf.setFloat(HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname, 0.49f);
            using (Stream file = File.OpenWrite(TestFilePath))
            {
                OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
                options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
                options.stripeSize(100000);
                options.compress(CompressionKind.ZLIB);
                options.bufferSize(10000);
                options.rowIndexStride(1000);
                using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
                {
                    Random r1 = new Random(1);
                    int nextInt = 0;
                    for (int i = 0; i < 21000; ++i)
                    {
                        // Write out the same string twice, this guarantees the fraction of rows with
                        // distinct strings is 0.5
                        if (i % 2 == 0)
                        {
                            nextInt = r1.Next(TestHelpers.words.Length);
                            // Append the value of i to the word, this guarantees when an index or word is repeated
                            // the actual string is unique.
                            TestHelpers.words[nextInt] += "-" + i;
                        }
                        writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[nextInt]));
                    }
                }
            }

            string outputFilename = "orc-file-dump-dictionary-threshold.out";
            using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
            {
                FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" });
            }

            TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
        }
        public void testSeek()
        {
            TypeDescription schema = createBigRowSchema();
            Random rand = new Random(42);
            const int COUNT = 32768;
            long[] intValues = new long[COUNT];
            double[] doubleValues = new double[COUNT];
            string[] stringValues = new string[COUNT];
            byte[][] byteValues = new byte[COUNT][];
            string[] words = new string[128];

            using (Stream file = File.OpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
                .setSchema(schema)
                .stripeSize(200000)
                .bufferSize(65536)
                .compress(CompressionKind.NONE) // Compression makes this very slow
                .rowIndexStride(1000)))
            {
                VectorizedRowBatch batch = schema.createRowBatch();
                for (int i = 0; i < words.Length; ++i)
                {
                    words[i] = Integer.toHexString(rand.Next());
                }
                for (int i = 0; i < COUNT / 2; ++i)
                {
                    intValues[2 * i] = rand.NextLong();
                    intValues[2 * i + 1] = intValues[2 * i];
                    stringValues[2 * i] = words[rand.Next(words.Length)];
                    stringValues[2 * i + 1] = stringValues[2 * i];
                }
                for (int i = 0; i < COUNT; ++i)
                {
                    doubleValues[i] = rand.NextDouble();
                    byte[] buf = new byte[20];
                    rand.NextBytes(buf);
                    byteValues[i] = buf;
                }
                for (int i = 0; i < COUNT; ++i)
                {
                    appendRandomRow(batch, intValues, doubleValues, stringValues,
                        byteValues, words, i);
                    if (batch.size == 1024)
                    {
                        writer.addRowBatch(batch);
                        batch.reset();
                    }
                }
                if (batch.size != 0)
                {
                    writer.addRowBatch(batch);
                }
            }

            Reader reader = OrcFile.createReader(TestFilePath,
                OrcFile.readerOptions(conf));
            Assert.Equal(COUNT, reader.getNumberOfRows());
            using (RecordReader rows = reader.rows())
            {
                // get the row index
                MetadataReader meta = ((RecordReaderImpl)rows).getMetadataReader();
                RecordReaderImpl.Index index =
                    meta.readRowIndex(reader.getStripes()[0], null, null, null, null,
                        null);
                // check the primitive columns to make sure they have the right number of
                // items in the first row group
                for (int c = 1; c < 9; ++c)
                {
                    OrcProto.RowIndex colIndex = index.getRowGroupIndex()[c];
                    Assert.Equal(1000U,
                        colIndex.GetEntry(0).Statistics.NumberOfValues);
                }

                OrcStruct row = null;
                for (int i = COUNT - 1; i >= 0; --i)
                {
                    rows.seekToRow(i);
                    row = (OrcStruct)rows.next();
                    BigRow expected = createRandomRow(intValues, doubleValues,
                        stringValues, byteValues, words, i);
                    Assert.Equal(expected.boolean1, row.getFieldValue(0));
                    Assert.Equal(expected.byte1, row.getFieldValue(1));
                    Assert.Equal(expected.short1, row.getFieldValue(2));
                    Assert.Equal(expected.int1, row.getFieldValue(3));
                    Assert.Equal(expected.long1, row.getFieldValue(4));
                    Assert.Equal(expected.float1, (float)row.getFieldValue(5), 4);
                    Assert.Equal(expected.double1, (double)row.getFieldValue(6), 4);
                    Assert.Equal(expected.bytes1, row.getFieldValue(7));
                    Assert.Equal(expected.string1, row.getFieldValue(8));
                    List<InnerStruct> expectedList = expected.middle.list;
                    IList<object> actualList =
                        (IList<object>)((OrcStruct)row.getFieldValue(9)).getFieldValue(0);
                    compareList(expectedList, actualList);
                    compareList(expected.list, (IList<object>)row.getFieldValue(10));
                }
            }

            long offsetOfStripe2 = 0;
            long offsetOfStripe4 = 0;
            long lastRowOfStripe2 = 0;
            for (int i = 0; i < 5; ++i)
            {
                StripeInformation stripe = reader.getStripes()[i];
                if (i < 2)
                {
                    lastRowOfStripe2 += stripe.getNumberOfRows();
                }
                else if (i == 2)
                {
                    offsetOfStripe2 = stripe.getOffset();
                    lastRowOfStripe2 += stripe.getNumberOfRows() - 1;
                }
                else if (i == 4)
                {
                    offsetOfStripe4 = stripe.getOffset();
                }
            }
            bool[] columns = new bool[reader.getStatistics().Length];
            columns[5] = true; // long colulmn
            columns[9] = true; // text column
            using (RecordReader rows = reader.rowsOptions(new RecordReaderOptions()
                .range(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2)
                .include(columns)))
            {
                rows.seekToRow(lastRowOfStripe2);
                for (int i = 0; i < 2; ++i)
                {
                    OrcStruct row = (OrcStruct)rows.next();
                    BigRow expected = createRandomRow(intValues, doubleValues,
                                                      stringValues, byteValues, words,
                                                      (int)(lastRowOfStripe2 + i));

                    Assert.Equal(expected.long1, row.getFieldValue(4));
                    Assert.Equal(expected.string1, row.getFieldValue(8));
                }
            }
        }
        public void testColumnProjection()
        {
            TypeDescription schema = createInnerSchema();
            Random r1 = new Random(1);
            Random r2 = new Random(2);
            int minInt = 0, maxInt = 0;
            string minStr = null, maxStr = null;

            using (Stream file = File.OpenWrite(TestFilePath))
            using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
                .setSchema(schema)
                .stripeSize(1000)
                .compress(CompressionKind.NONE)
                .bufferSize(100)
                .rowIndexStride(1000)))
            {
                VectorizedRowBatch batch = schema.createRowBatch();
                int x;
                string y;
                batch.size = 1000;
                bool first = true;
                for (int b = 0; b < 21; ++b)
                {
                    for (int r = 0; r < 1000; ++r)
                    {
                        x = r1.Next();
                        y = Long.toHexString(r2.NextLong());
                        if (first || x < minInt)
                        {
                            minInt = x;
                        }
                        if (first || x > maxInt)
                        {
                            maxInt = x;
                        }
                        if (first || y.CompareTo(minStr) < 0)
                        {
                            minStr = y;
                        }
                        if (first || y.CompareTo(maxStr) > 0)
                        {
                            maxStr = y;
                        }
                        first = false;
                        ((LongColumnVector)batch.cols[0]).vector[r] = x;
                        ((BytesColumnVector)batch.cols[1]).setVal(r, y.getBytes());
                    }
                    writer.addRowBatch(batch);
                }
            }

            Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));

            // check out the statistics
            ColumnStatistics[] stats = reader.getStatistics();
            Assert.Equal(3, stats.Length);
            foreach (ColumnStatistics s in stats)
            {
                Assert.Equal(21000, s.getNumberOfValues());
                if (s is IntegerColumnStatistics)
                {
                    Assert.Equal(minInt, ((IntegerColumnStatistics)s).getMinimum());
                    Assert.Equal(maxInt, ((IntegerColumnStatistics)s).getMaximum());
                }
                else if (s is StringColumnStatistics)
                {
                    Assert.Equal(maxStr, ((StringColumnStatistics)s).getMaximum());
                    Assert.Equal(minStr, ((StringColumnStatistics)s).getMinimum());
                }
            }

            // check out the types
            IList<OrcProto.Type> types = reader.getTypes();
            Assert.Equal(3, types.Count);
            Assert.Equal(OrcProto.Type.Types.Kind.STRUCT, types[0].Kind);
            Assert.Equal(2, types[0].SubtypesCount);
            Assert.Equal(1U, types[0].GetSubtypes(0));
            Assert.Equal(2U, types[0].GetSubtypes(1));
            Assert.Equal(OrcProto.Type.Types.Kind.INT, types[1].Kind);
            Assert.Equal(0, types[1].SubtypesCount);
            Assert.Equal(OrcProto.Type.Types.Kind.STRING, types[2].Kind);
            Assert.Equal(0, types[2].SubtypesCount);

            // read the contents and make sure they match
            using (RecordReader rows1 = reader.rows(new bool[] { true, true, false }))
            using (RecordReader rows2 = reader.rows(new bool[] { true, false, true }))
            {
                r1 = new Random(1);
                r2 = new Random(2);
                for (int i = 0; i < 21000; ++i)
                {
                    Assert.Equal(true, rows1.hasNext());
                    Assert.Equal(true, rows2.hasNext());
                    OrcStruct row1 = (OrcStruct)rows1.next();
                    OrcStruct row2 = (OrcStruct)rows2.next();
                    Assert.Equal(r1.Next(), row1.getFieldValue(0));
                    Assert.Equal(Long.toHexString(r2.NextLong()),
                        row2.getFieldValue(1).ToString());
                }
                Assert.Equal(false, rows1.hasNext());
                Assert.Equal(false, rows2.hasNext());
            }
        }
        private long RandomLong(Random random)
        {
            long val;
            switch (random.Next(4))
            {
                case 0:
                    val = 1L << (random.Next(63)); //  patterns like 0x000000100000 (-1 yields patterns like 0x0000fff)
                    break;

                case 1:
                    val = -1L << (random.Next(63)); // patterns like 0xfffff00000
                    break;

                default:
                    val = random.NextLong();
                    break;
            }

            val += random.Next(5) - 2;

            if (random.NextBoolean())
            {
                if (random.NextBoolean())
                {
                    val += random.Next(100) - 50;
                }
                if (random.NextBoolean())
                {
                    val = ~val;
                }
                if (random.NextBoolean())
                {
                    val = val << 1;
                }
                if (random.NextBoolean())
                {
                    val = (long)((ulong)val >> 1);
                }
            }

            return val;
        }
        public IList<Entity> GenerateFakeEntities(string entityName, int howMany)
        {
            var metadataProvider = _MetadataProvider;
            var metadata = metadataProvider.GetEntityMetadata(entityName);
            Random rand = new Random();

            List<Entity> results = new List<Entity>();

            // Used for generating random dates.
            DateTime minCrmDate = new DateTime(1900, 1, 1);
            int crmDayRange = (DateTime.Today - minCrmDate).Days;

            for (int i = 0; i < howMany; i++)
            {
                var ent = new Microsoft.Xrm.Sdk.Entity(entityName);
                ent.Id = Guid.NewGuid();

                int stateCode = rand.Next(0, 1);
                int statusCode = stateCode + 1;

                foreach (var a in metadata.Attributes)
                {
                    switch (a.AttributeType.Value)
                    {
                        case AttributeTypeCode.BigInt:
                            var randomBigInt = (long)rand.NextLong(0, Int64.MaxValue);
                            ent[a.LogicalName] = randomBigInt;
                            break;
                        case AttributeTypeCode.Boolean:
                            int randomBoolInt = rand.Next(0, 1);
                            ent[a.LogicalName] = randomBoolInt == 1;
                            break;
                        case AttributeTypeCode.CalendarRules:
                            break;
                        case AttributeTypeCode.Customer:
                            int randomCustomerInt = rand.Next(0, 1);
                            string customerentity = "contact";
                            Guid customerId = Guid.NewGuid();
                            if (randomCustomerInt == 1)
                            {
                                customerentity = "account";
                            }
                            EntityReference customerRef = new EntityReference(customerentity, customerId);
                            ent[a.LogicalName] = customerRef;
                            break;
                        case AttributeTypeCode.DateTime:
                            DateTime randomDate = rand.NextCrmDate(minCrmDate, crmDayRange);
                            ent[a.LogicalName] = randomDate;
                            break;
                        case AttributeTypeCode.Decimal:
                            var decAtt = (DecimalAttributeInfo)a;
                            var scale = decAtt.NumericScale;
                            byte byteScale = (byte)scale;
                            var randomDecimal = rand.NextDecimal(byteScale);
                            ent[a.LogicalName] = randomDecimal;
                            break;
                        case AttributeTypeCode.Double:
                            var doubleAtt = (DoubleAttributeInfo)a;
                            var doubleScale = doubleAtt.NumericScale;
                            byte byteDoubleScale = (byte)doubleScale;
                            // todo apply precision / scale
                            var randomDouble = rand.NextDouble();
                            ent[a.LogicalName] = randomDouble;
                            break;
                        case AttributeTypeCode.EntityName:
                            break;
                        case AttributeTypeCode.Integer:
                            ent[a.LogicalName] = rand.Next();
                            break;
                        case AttributeTypeCode.Lookup:
                            break;
                        case AttributeTypeCode.ManagedProperty:
                            break;
                        case AttributeTypeCode.Memo:
                            var randomMemoString = string.Format("Test Memo String {0}", DateTime.UtcNow.Ticks.ToString());
                            ent[a.LogicalName] = randomMemoString;
                            break;
                        case AttributeTypeCode.Money:
                            var moneyAtt = (MoneyAttributeInfo)a;
                            var mscale = moneyAtt.NumericScale;
                            byte bytemScale = (byte)mscale;
                            var randomMoneyDecimal = rand.NextDecimal(bytemScale);
                            var randMoney = new Money(randomMoneyDecimal);
                            ent[a.LogicalName] = randMoney;
                            break;
                        case AttributeTypeCode.Owner:
                            EntityReference ownerRef = new EntityReference("systemuser", Guid.NewGuid());
                            ent[a.LogicalName] = ownerRef;
                            break;
                        case AttributeTypeCode.PartyList:
                            break;
                        case AttributeTypeCode.Picklist:
                            OptionSetValue optValue = new OptionSetValue(rand.Next());
                            ent[a.LogicalName] = optValue;
                            break;
                        case AttributeTypeCode.State:
                            // todo randomise active and inactive.
                            var stateCodeOpt = new OptionSetValue(stateCode);
                            ent[a.LogicalName] = stateCodeOpt;
                            break;
                        case AttributeTypeCode.Status:
                            var statusCodeOpt = new OptionSetValue(statusCode);
                            // todo randomise active and inactive.
                            ent[a.LogicalName] = statusCodeOpt;
                            break;
                        case AttributeTypeCode.String:
                            var randomString = string.Format("TestString{0}", DateTime.UtcNow.Ticks.ToString());
                            ent[a.LogicalName] = randomString;
                            break;
                        case AttributeTypeCode.Uniqueidentifier:
                            ent[a.LogicalName] = Guid.NewGuid();
                            break;
                        case AttributeTypeCode.Virtual:
                            break;
                        default:
                            break;
                    }

                }

                results.Add(ent);
            }

            return results;
        }