/// <exception cref="System.IO.IOException"/> public virtual void Map(WritableComparable key, Writable value, OutputCollector <IntWritable , SortValidator.RecordStatsChecker.RecordStatsWritable> output, Reporter reporter ) { // Set up rawKey and rawValue on the first call to 'map' if (recordId == -1) { rawKey = CreateRaw(key.GetType()); rawValue = CreateRaw(value.GetType()); } ++recordId; if (this.key == sortOutput) { // Check if keys are 'sorted' if this // record is from sort's output if (prevKey == null) { prevKey = key; keyClass = prevKey.GetType(); } else { // Sanity check if (keyClass != key.GetType()) { throw new IOException("Type mismatch in key: expected " + keyClass.FullName + ", received " + key.GetType().FullName); } // Check if they were sorted correctly if (prevKey.CompareTo(key) > 0) { throw new IOException("The 'map-reduce' framework wrongly" + " classifed (" + prevKey + ") > (" + key + ") " + "for record# " + recordId); } prevKey = key; } // Check if the sorted output is 'partitioned' right int keyPartition = partitioner.GetPartition(key, value, noSortReducers); if (partition != keyPartition) { throw new IOException("Partitions do not match for record# " + recordId + " ! - '" + partition + "' v/s '" + keyPartition + "'"); } } // Construct the record-stats and output (this.key, record-stats) byte[] keyBytes = rawKey.GetRawBytes(key); int keyBytesLen = rawKey.GetRawBytesLength(key); byte[] valueBytes = rawValue.GetRawBytes(value); int valueBytesLen = rawValue.GetRawBytesLength(value); int keyValueChecksum = (WritableComparator.HashBytes(keyBytes, keyBytesLen) ^ WritableComparator .HashBytes(valueBytes, valueBytesLen)); output.Collect(this.key, new SortValidator.RecordStatsChecker.RecordStatsWritable ((keyBytesLen + valueBytesLen), 1, keyValueChecksum)); }
/// <summary> /// Use (the specified slice of the array returned by) /// <see cref="Org.Apache.Hadoop.IO.BinaryComparable.GetBytes()"/> /// to partition. /// </summary> public override int GetPartition(BinaryComparable key, V value, int numPartitions ) { int length = key.GetLength(); int leftIndex = (leftOffset + length) % length; int rightIndex = (rightOffset + length) % length; int hash = WritableComparator.HashBytes(key.GetBytes(), leftIndex, rightIndex - leftIndex + 1); return((hash & int.MaxValue) % numPartitions); }
public override int GetHashCode() { return(WritableComparator.HashBytes(bytes, length)); }
public override int GetHashCode() { return(WritableComparator.HashBytes(identifier, identifier.Length)); }