Beispiel #1
0
        public static void encodeParity(this ReedSolomon reedSolomon, IByteBuffer[] shards, int offset, int byteCount)
        {
            // Check arguments.
            reedSolomon.checkBuffersAndSizes(shards, offset, byteCount);

            // Build the array of output buffers.
            IByteBuffer[] outputs = new IByteBuffer [reedSolomon.ParityShardCount];
            Array.Copy(shards, reedSolomon.DataShardCount, outputs, 0, reedSolomon.ParityShardCount);
//        System.arraycopy(shards, dataShardCount, outputs, 0, parityShardCount);

            // Do the coding.
            LOOP.codeSomeShards(
                reedSolomon.ParityRows,
                shards, reedSolomon.DataShardCount,
                outputs, reedSolomon.ParityShardCount,
                offset, byteCount);
        }
Beispiel #2
0
        /**
         * Encodes a set of data shards, and then tries decoding
         * using all possible subsets of the encoded shards.
         *
         * Uses 5+5 coding, so there must be 5 input data shards.
         */
        private void runEncodeDecode(int dataCount, int parityCount, byte[][] dataShards)
        {
            int totalCount  = dataCount + parityCount;
            int shardLength = dataShards[0].Length;

            // Make the list of data and parity shards.
//        assertEquals(dataCount, dataShards.length);
            int dataLength = dataShards[0].Length;

            byte [] [] allShards = new byte [totalCount] [];
            for (int i = 0; i < dataCount; i++)
            {
                byte[] temp = new byte[dataLength];
                Array.Copy(dataShards[i], 0, temp, 0, dataLength);
                allShards[i] = temp;
            }
            for (int i = dataCount; i < totalCount; i++)
            {
                allShards[i] = new byte [dataLength];
            }

            // Encode.
            ReedSolomon codec = ReedSolomon.create(dataCount, parityCount);

            codec.encodeParity(allShards, 0, dataLength);

            // Make a copy to decode with.
            byte [] [] testShards   = new byte [totalCount] [];
            bool []    shardPresent = new bool [totalCount];
            for (int i = 0; i < totalCount; i++)
            {
                byte[] temp = new byte[shardLength];
                Array.Copy(allShards[i], 0, temp, 0, shardLength);
                testShards[i]   = temp;
                shardPresent[i] = true;
            }

            // Decode with 0, 1, ..., 5 shards missing.
            for (int numberMissing = 0; numberMissing < parityCount + 1; numberMissing++)
            {
                tryAllSubsetsMissing(codec, allShards, testShards, shardPresent, numberMissing);
            }
        }
Beispiel #3
0
        /**
         * Checks the consistency of arguments passed to public methods.
         */
        private static void checkBuffersAndSizes(this ReedSolomon reedSolomon, IByteBuffer[] shards, int offset,
                                                 int byteCount)
        {
            // The number of buffers should be equal to the number of
            // data shards plus the number of parity shards.
            if (shards.Length != reedSolomon.TotalShardCount)
            {
                throw new Exception("wrong number of shards: " + shards.Length);
            }

            // All of the shard buffers should be the same length.
            int shardLength = shards[0].ReadableBytes;

            for (int i = 1; i < shards.Length; i++)
            {
                if (shards[i].ReadableBytes != shardLength)
                {
                    throw new Exception("Shards are different sizes");
                }
            }

            // The offset and byteCount must be non-negative and fit in the buffers.
            if (offset < 0)
            {
                throw new Exception("offset is negative: " + offset);
            }

            if (byteCount < 0)
            {
                throw new Exception("byteCount is negative: " + byteCount);
            }

            if (shardLength < offset + byteCount)
            {
                throw new Exception("buffers to small: " + byteCount + offset);
            }
        }
        private Measurement doOneCheckMeasurement(ReedSolomon codec, BufferSet[] bufferSets, byte[] tempBuffer)
        {
            long passesCompleted = 0;
            long bytesChecked    = 0;
            long checkingTime    = 0;

            while (checkingTime < MEASUREMENT_DURATION)
            {
                BufferSet bufferSet = bufferSets[nextBuffer];
                nextBuffer = (nextBuffer + 1) % bufferSets.Length;
                byte[][] shards = bufferSet.buffers;

                Stopwatch stopwatch  =  new Stopwatch();
                stopwatch.Start();

                if (!codec.isParityCorrect(shards, 0, BUFFER_SIZE, tempBuffer))
                {
                    // if the parity is not correct, it will throw off the
                    // benchmarking because it may return early.
                    throw new Exception("parity not correct");
                }

                stopwatch.Stop();  //  停止监视
                    TimeSpan timespan  =  stopwatch.Elapsed;  
                    checkingTime      += (long)timespan.TotalMilliseconds;
                bytesChecked    += BUFFER_SIZE * DATA_COUNT;
                passesCompleted += 1;
            }

            double      seconds   = ((double)checkingTime) / 1000.0;
            double      megabytes = ((double)bytesChecked) / 1000000.0;
            Measurement result    = new Measurement(megabytes, seconds);

            Console.WriteLine("        {0} passes, {1}", passesCompleted, result);
            return(result);
        }
        public void run()
        {
            Console.WriteLine("preparing...");
            BufferSet[] bufferSets = new BufferSet [NUMBER_OF_BUFFER_SETS];
            for (int iBufferSet = 0; iBufferSet < NUMBER_OF_BUFFER_SETS; iBufferSet++)
            {
                bufferSets[iBufferSet] = new BufferSet();
            }

            byte[] tempBuffer = new byte [BUFFER_SIZE];

            List <String> summaryLines = new List <String>();
            StringBuilder csv          = new StringBuilder();

            csv.Append("Outer,Middle,Inner,Multiply,Encode,Check\n");
            foreach (var codingLoop in ALL_CODING_LOOPS)
            {
                Measurement encodeAverage = new Measurement();
                {
                    String testName = codingLoop.GetType().Name + " encodeParity";


                    Console.WriteLine("\nTEST: " + testName);
                    ReedSolomon codec = new ReedSolomon(DATA_COUNT, PARITY_COUNT, codingLoop);
                    Console.WriteLine("    warm up...");
                    doOneEncodeMeasurement(codec, bufferSets);
                    doOneEncodeMeasurement(codec, bufferSets);
                    Console.WriteLine("    testing...");
                    for (int iMeasurement = 0; iMeasurement < 10; iMeasurement++)
                    {
                        encodeAverage.add(doOneEncodeMeasurement(codec, bufferSets));
                    }

                    Console.WriteLine("AVERAGE: {0}", encodeAverage);
                    summaryLines.Add(testName + " " + encodeAverage);
                }
                // The encoding test should have filled all of the buffers with
                // correct parity, so we can benchmark parity checking.
                Measurement checkAverage = new Measurement();
                {
                    String testName = codingLoop.GetType().Name + " isParityCorrect";
                    Console.WriteLine("\nTEST: " + testName);
                    ReedSolomon codec = new ReedSolomon(DATA_COUNT, PARITY_COUNT, codingLoop);
                    Console.WriteLine("    warm up...");
                    doOneEncodeMeasurement(codec, bufferSets);
                    doOneEncodeMeasurement(codec, bufferSets);
                    Console.WriteLine("    testing...");
                    for (int iMeasurement = 0; iMeasurement < 10; iMeasurement++)
                    {
                        checkAverage.add(doOneCheckMeasurement(codec, bufferSets, tempBuffer));
                    }

                    Console.WriteLine("AVERAGE: {0}", checkAverage);
                    summaryLines.Add(testName + " " + checkAverage);
                }
                csv.Append(codingLoopNameToCsvPrefix(codingLoop.GetType().Name));
                csv.Append((int)encodeAverage.getRate());
                csv.Append(",");
                csv.Append((int)checkAverage.getRate());
                csv.Append("\n");
            }

            Console.WriteLine("\n");
            Console.WriteLine(csv.ToString());

            Console.WriteLine("\nSummary:\n");
            foreach (var line in summaryLines)
            {
                Console.WriteLine(line);
            }
        }
Beispiel #6
0
        public static void decodeMissing(this ReedSolomon reedSolomon, IByteBuffer[] shards,
                                         bool[] shardPresent,
                                         int offset,
                                         int byteCount)
        {
            // Check arguments.
            reedSolomon.checkBuffersAndSizes(shards, offset, byteCount);

            // Quick check: are all of the shards present?  If so, there's
            // nothing to do.
            int numberPresent = 0;

            for (int i = 0; i < reedSolomon.TotalShardCount; i++)
            {
                if (shardPresent[i])
                {
                    numberPresent += 1;
                }
            }

            if (numberPresent == reedSolomon.TotalShardCount)
            {
                // Cool.  All of the shards data data.  We don't
                // need to do anything.
                return;
            }

            // More complete sanity check
            if (numberPresent < reedSolomon.DataShardCount)
            {
                throw new Exception("Not enough shards present");
            }

            // Pull out the rows of the matrix that correspond to the
            // shards that we have and build a square matrix.  This
            // matrix could be used to generate the shards that we have
            // from the original data.
            //
            // Also, pull out an array holding just the shards that
            // correspond to the rows of the submatrix.  These shards
            // will be the input to the decoding process that re-creates
            // the missing data shards.
            Matrix subMatrix = new Matrix(reedSolomon.DataShardCount, reedSolomon.DataShardCount);

            IByteBuffer[] subShards = new IByteBuffer[reedSolomon.DataShardCount];
            {
                int subMatrixRow = 0;
                for (int matrixRow = 0;
                     matrixRow < reedSolomon.TotalShardCount && subMatrixRow < reedSolomon.DataShardCount;
                     matrixRow++)
                {
                    if (shardPresent[matrixRow])
                    {
                        for (int c = 0; c < reedSolomon.DataShardCount; c++)
                        {
                            subMatrix.set(subMatrixRow, c, reedSolomon.Matrix.get(matrixRow, c));
                        }

                        subShards[subMatrixRow] = shards[matrixRow];
                        subMatrixRow           += 1;
                    }
                }
            }

            // Invert the matrix, so we can go from the encoded shards
            // back to the original data.  Then pull out the row that
            // generates the shard that we want to decode.  Note that
            // since this matrix maps back to the orginal data, it can
            // be used to create a data shard, but not a parity shard.
            Matrix dataDecodeMatrix = subMatrix.invert();

            // Re-create any data shards that were missing.
            //
            // The input to the coding is all of the shards we actually
            // have, and the output is the missing data shards.  The computation
            // is done using the special decode matrix we just built.
            IByteBuffer[] outputs     = new IByteBuffer[reedSolomon.ParityShardCount];
            byte[][]      matrixRows  = new byte [reedSolomon.ParityShardCount][];
            int           outputCount = 0;

            for (int iShard = 0; iShard < reedSolomon.DataShardCount; iShard++)
            {
                if (!shardPresent[iShard])
                {
                    outputs[outputCount]    = shards[iShard];
                    matrixRows[outputCount] = dataDecodeMatrix.getRow(iShard);
                    outputCount            += 1;
                }
            }

            LOOP.codeSomeShards(
                matrixRows,
                subShards, reedSolomon.DataShardCount,
                outputs, outputCount,
                offset, byteCount);

            // Now that we have all of the data shards intact, we can
            // compute any of the parity that is missing.
            //
            // The input to the coding is ALL of the data shards, including
            // any that we just calculated.  The output is whichever of the
            // data shards were missing.
            outputCount = 0;
            for (int iShard = reedSolomon.DataShardCount; iShard < reedSolomon.TotalShardCount; iShard++)
            {
                if (!shardPresent[iShard])
                {
                    outputs[outputCount]    = shards[iShard];
                    matrixRows[outputCount] = reedSolomon.ParityRows[iShard - reedSolomon.DataShardCount];
                    outputCount            += 1;
                }
            }

            LOOP.codeSomeShards(
                matrixRows,
                shards, reedSolomon.DataShardCount,
                outputs, outputCount,
                offset, byteCount);
        }