public override int Compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2)
{
int n1 = WritableUtils.DecodeVIntSize(b1[s1]);
int n2 = WritableUtils.DecodeVIntSize(b2[s2]);
IList <KeyFieldHelper.KeyDescription> allKeySpecs = keyFieldHelper.KeySpecs();
if (allKeySpecs.Count == 0)
{
return(CompareBytes(b1, s1 + n1, l1 - n1, b2, s2 + n2, l2 - n2));
}
int[] lengthIndicesFirst = keyFieldHelper.GetWordLengths(b1, s1 + n1, s1 + l1);
int[] lengthIndicesSecond = keyFieldHelper.GetWordLengths(b2, s2 + n2, s2 + l2);
foreach (KeyFieldHelper.KeyDescription keySpec in allKeySpecs)
{
int startCharFirst = keyFieldHelper.GetStartOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst
, keySpec);
int endCharFirst = keyFieldHelper.GetEndOffset(b1, s1 + n1, s1 + l1, lengthIndicesFirst
, keySpec);
int startCharSecond = keyFieldHelper.GetStartOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond
, keySpec);
int endCharSecond = keyFieldHelper.GetEndOffset(b2, s2 + n2, s2 + l2, lengthIndicesSecond
, keySpec);
int result;
if ((result = CompareByteSequence(b1, startCharFirst, endCharFirst, b2, startCharSecond
, endCharSecond, keySpec)) != 0)
{
return(result);
}
}
return(0);
}
private void TestKeySpecs(string input, string expectedOutput, KeyFieldHelper helper
, int s1, int e1)
{
Log.Info("input : " + input);
string keySpecs = helper.KeySpecs()[0].ToString();
Log.Info("keyspecs : " + keySpecs);
byte[] inputBytes = Sharpen.Runtime.GetBytesForString(input);
// get the input bytes
if (e1 == -1)
{
e1 = inputBytes.Length;
}
Log.Info("length : " + e1);
// get the word lengths
int[] indices = helper.GetWordLengths(inputBytes, s1, e1);
// get the start index
int start = helper.GetStartOffset(inputBytes, s1, e1, indices, helper.KeySpecs()[
0]);
Log.Info("start : " + start);
if (expectedOutput == null)
{
NUnit.Framework.Assert.AreEqual("Expected -1 when the start index is invalid", -1
, start);
return;
}
// get the end index
int end = helper.GetEndOffset(inputBytes, s1, e1, indices, helper.KeySpecs()[0]);
Log.Info("end : " + end);
//my fix
end = (end >= inputBytes.Length) ? inputBytes.Length - 1 : end;
int length = end + 1 - start;
Log.Info("length : " + length);
byte[] outputBytes = new byte[length];
System.Array.Copy(inputBytes, start, outputBytes, 0, length);
string output = Sharpen.Runtime.GetStringForBytes(outputBytes);
Log.Info("output : " + output);
Log.Info("expected-output : " + expectedOutput);
NUnit.Framework.Assert.AreEqual(keySpecs + " failed on input '" + input + "'", expectedOutput
, output);
}
public override int GetPartition(K2 key, V2 value, int numReduceTasks)
{
byte[] keyBytes;
IList <KeyFieldHelper.KeyDescription> allKeySpecs = keyFieldHelper.KeySpecs();
if (allKeySpecs.Count == 0)
{
return(GetPartition(key.ToString().GetHashCode(), numReduceTasks));
}
try
{
keyBytes = Sharpen.Runtime.GetBytesForString(key.ToString(), "UTF-8");
}
catch (UnsupportedEncodingException e)
{
throw new RuntimeException("The current system does not " + "support UTF-8 encoding!"
, e);
}
// return 0 if the key is empty
if (keyBytes.Length == 0)
{
return(0);
}
int[] lengthIndicesFirst = keyFieldHelper.GetWordLengths(keyBytes, 0, keyBytes.Length
);
int currentHash = 0;
foreach (KeyFieldHelper.KeyDescription keySpec in allKeySpecs)
{
int startChar = keyFieldHelper.GetStartOffset(keyBytes, 0, keyBytes.Length, lengthIndicesFirst
, keySpec);
// no key found! continue
if (startChar < 0)
{
continue;
}
int endChar = keyFieldHelper.GetEndOffset(keyBytes, 0, keyBytes.Length, lengthIndicesFirst
, keySpec);
currentHash = HashCode(keyBytes, startChar, endChar, currentHash);
}
return(GetPartition(currentHash, numReduceTasks));
}
/// <summary>Test is key-field-helper's parse option.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestparseOption()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
string keySpecs = "-k1.2,3.4";
string eKeySpecs = keySpecs;
helper.ParseOption(keySpecs);
string actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
// test -k a.b
keySpecs = "-k 1.2";
eKeySpecs = "-k1.2,0.0";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4n";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4r";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4 -k5.6,7.8n -k9.10,11.12r -k13.14,15.16nr";
//1st
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
// 2nd
eKeySpecs = "-k5.6,7.8n";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
//3rd
eKeySpecs = "-k9.10,11.12r";
actKeySpecs = helper.KeySpecs()[2].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
//4th
eKeySpecs = "-k13.14,15.16nr";
actKeySpecs = helper.KeySpecs()[3].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2n,3.4";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2r,3.4";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2nr,3.4";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4n";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4r";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4nr";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8nr";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-n -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8n";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-r -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8r";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4n -k5.6,7.8";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4r -k5.6,7.8";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4nr -k5.6,7.8";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-n";
eKeySpecs = "-k1.1,0.0n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-r";
eKeySpecs = "-k1.1,0.0r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr";
eKeySpecs = "-k1.1,0.0nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
}