コード例 #1
0
 private static RandomDatum[] Generate(int count)
 {
     Log.Info("generating " + count + " records in memory");
     RandomDatum[]         data      = new RandomDatum[count];
     RandomDatum.Generator generator = new RandomDatum.Generator();
     for (int i = 0; i < count; i++)
     {
         generator.Next();
         data[i] = generator.GetValue();
     }
     Log.Info("sorting " + count + " records");
     Arrays.Sort(data);
     return(data);
 }
コード例 #2
0
ファイル: TestArrayFile.cs プロジェクト: orf53975/hadoop.net
 private static RandomDatum[] Generate(int count)
 {
     if (Log.IsDebugEnabled())
     {
         Log.Debug("generating " + count + " records in debug");
     }
     RandomDatum[]         data      = new RandomDatum[count];
     RandomDatum.Generator generator = new RandomDatum.Generator();
     for (int i = 0; i < count; i++)
     {
         generator.Next();
         data[i] = generator.GetValue();
     }
     return(data);
 }
コード例 #3
0
 /// <exception cref="System.IO.IOException"/>
 private void WriteMetadataTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType
                                compressionType, CompressionCodec codec, SequenceFile.Metadata metadata)
 {
     fs.Delete(file, true);
     Log.Info("creating " + count + " records with metadata and with " + compressionType
              + " compression");
     SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(RandomDatum
                                                                                   ), typeof(RandomDatum), compressionType, codec, null, metadata);
     RandomDatum.Generator generator = new RandomDatum.Generator(seed);
     for (int i = 0; i < count; i++)
     {
         generator.Next();
         RandomDatum key   = generator.GetKey();
         RandomDatum value = generator.GetValue();
         writer.Append(key, value);
     }
     writer.Close();
 }
コード例 #4
0
        /// <exception cref="System.IO.IOException"/>
        private static void ReadTest(FileSystem fs, RandomDatum[] data, string file)
        {
            RandomDatum v      = new RandomDatum();
            int         sample = (int)Math.Sqrt(data.Length);
            Random      random = new Random();

            Log.Info("reading " + sample + " records");
            SetFile.Reader reader = new SetFile.Reader(fs, file, conf);
            for (int i = 0; i < sample; i++)
            {
                if (!reader.Seek(data[random.Next(data.Length)]))
                {
                    throw new RuntimeException("wrong value at " + i);
                }
            }
            reader.Close();
            Log.Info("done reading " + data.Length);
        }
コード例 #5
0
        /// <exception cref="System.IO.IOException"/>
        private void CheckSort(FileSystem fs, int count, int seed, Path file)
        {
            Log.Info("sorting " + count + " records in memory for debug");
            RandomDatum.Generator generator = new RandomDatum.Generator(seed);
            SortedDictionary <RandomDatum, RandomDatum> map = new SortedDictionary <RandomDatum
                                                                                    , RandomDatum>();

            for (int i = 0; i < count; i++)
            {
                generator.Next();
                RandomDatum key   = generator.GetKey();
                RandomDatum value = generator.GetValue();
                map[key] = value;
            }
            Log.Debug("checking order of " + count + " records");
            RandomDatum k = new RandomDatum();
            RandomDatum v = new RandomDatum();
            IEnumerator <KeyValuePair <RandomDatum, RandomDatum> > iterator = map.GetEnumerator(
                );

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, file.Suffix(".sorted"),
                                                                 conf);
            for (int i_1 = 0; i_1 < count; i_1++)
            {
                KeyValuePair <RandomDatum, RandomDatum> entry = iterator.Next();
                RandomDatum key   = entry.Key;
                RandomDatum value = entry.Value;
                reader.Next(k, v);
                if (!k.Equals(key))
                {
                    throw new RuntimeException("wrong key at " + i_1);
                }
                if (!v.Equals(value))
                {
                    throw new RuntimeException("wrong value at " + i_1);
                }
            }
            reader.Close();
            Log.Debug("sucessfully checked " + count + " records");
        }
コード例 #6
0
ファイル: TestArrayFile.cs プロジェクト: orf53975/hadoop.net
        /// <exception cref="System.IO.IOException"/>
        private static void ReadTest(FileSystem fs, RandomDatum[] data, string file, Configuration
                                     conf)
        {
            RandomDatum v = new RandomDatum();

            if (Log.IsDebugEnabled())
            {
                Log.Debug("reading " + data.Length + " debug");
            }
            ArrayFile.Reader reader = new ArrayFile.Reader(fs, file, conf);
            try
            {
                for (int i = 0; i < data.Length; i++)
                {
                    // try forwards
                    reader.Get(i, v);
                    if (!v.Equals(data[i]))
                    {
                        throw new RuntimeException("wrong value at " + i);
                    }
                }
                for (int i_1 = data.Length - 1; i_1 >= 0; i_1--)
                {
                    // then backwards
                    reader.Get(i_1, v);
                    if (!v.Equals(data[i_1]))
                    {
                        throw new RuntimeException("wrong value at " + i_1);
                    }
                }
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("done reading " + data.Length + " debug");
                }
            }
            finally
            {
                reader.Close();
            }
        }
コード例 #7
0
 /// <exception cref="System.IO.IOException"/>
 private void MergeTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType
                        compressionType, bool fast, int factor, int megabytes)
 {
     Log.Debug("creating " + factor + " files with " + count / factor + " records");
     SequenceFile.Writer[] writers = new SequenceFile.Writer[factor];
     Path[] names       = new Path[factor];
     Path[] sortedNames = new Path[factor];
     for (int i = 0; i < factor; i++)
     {
         names[i]       = file.Suffix("." + i);
         sortedNames[i] = names[i].Suffix(".sorted");
         fs.Delete(names[i], true);
         fs.Delete(sortedNames[i], true);
         writers[i] = SequenceFile.CreateWriter(fs, conf, names[i], typeof(RandomDatum), typeof(
                                                    RandomDatum), compressionType);
     }
     RandomDatum.Generator generator = new RandomDatum.Generator(seed);
     for (int i_1 = 0; i_1 < count; i_1++)
     {
         generator.Next();
         RandomDatum key   = generator.GetKey();
         RandomDatum value = generator.GetValue();
         writers[i_1 % factor].Append(key, value);
     }
     for (int i_2 = 0; i_2 < factor; i_2++)
     {
         writers[i_2].Close();
     }
     for (int i_3 = 0; i_3 < factor; i_3++)
     {
         Log.Debug("sorting file " + i_3 + " with " + count / factor + " records");
         NewSorter(fs, fast, megabytes, factor).Sort(names[i_3], sortedNames[i_3]);
     }
     Log.Info("merging " + factor + " files with " + count / factor + " debug");
     fs.Delete(new Path(file + ".sorted"), true);
     NewSorter(fs, fast, megabytes, factor).Merge(sortedNames, file.Suffix(".sorted"));
 }
コード例 #8
0
        /// <exception cref="System.IO.IOException"/>
        private void ReadTest(FileSystem fs, int count, int seed, Path file)
        {
            Log.Debug("reading " + count + " records");
            SequenceFile.Reader   reader    = new SequenceFile.Reader(fs, file, conf);
            RandomDatum.Generator generator = new RandomDatum.Generator(seed);
            RandomDatum           k         = new RandomDatum();
            RandomDatum           v         = new RandomDatum();
            DataOutputBuffer      rawKey    = new DataOutputBuffer();

            SequenceFile.ValueBytes rawValue = reader.CreateValueBytes();
            for (int i = 0; i < count; i++)
            {
                generator.Next();
                RandomDatum key   = generator.GetKey();
                RandomDatum value = generator.GetValue();
                try
                {
                    if ((i % 5) == 0)
                    {
                        // Testing 'raw' apis
                        rawKey.Reset();
                        reader.NextRaw(rawKey, rawValue);
                    }
                    else
                    {
                        // Testing 'non-raw' apis
                        if ((i % 2) == 0)
                        {
                            reader.Next(k);
                            reader.GetCurrentValue(v);
                        }
                        else
                        {
                            reader.Next(k, v);
                        }
                        // Check
                        if (!k.Equals(key))
                        {
                            throw new RuntimeException("wrong key at " + i);
                        }
                        if (!v.Equals(value))
                        {
                            throw new RuntimeException("wrong value at " + i);
                        }
                    }
                }
                catch (IOException ioe)
                {
                    Log.Info("Problem on row " + i);
                    Log.Info("Expected key = " + key);
                    Log.Info("Expected len = " + key.GetLength());
                    Log.Info("Actual key = " + k);
                    Log.Info("Actual len = " + k.GetLength());
                    Log.Info("Expected value = " + value);
                    Log.Info("Expected len = " + value.GetLength());
                    Log.Info("Actual value = " + v);
                    Log.Info("Actual len = " + v.GetLength());
                    Log.Info("Key equals: " + k.Equals(key));
                    Log.Info("value equals: " + v.Equals(value));
                    throw;
                }
            }
            reader.Close();
        }
コード例 #9
0
 public virtual void Next()
 {
     key   = new RandomDatum(random);
     value = new RandomDatum(random);
 }