/// <exception cref="System.Exception"/>
public virtual void TestCoding()
{
string before = "Bad \t encoding \t testcase";
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(before);
string after = text.ToString();
Assert.True(before.Equals(after));
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// test string to utf8
ByteBuffer bb = Org.Apache.Hadoop.IO.Text.Encode(before);
byte[] utf8Text = ((byte[])bb.Array());
byte[] utf8Java = Runtime.GetBytesForString(before, "UTF-8");
Assert.Equal(0, WritableComparator.CompareBytes(utf8Text, 0, bb
.Limit(), utf8Java, 0, utf8Java.Length));
// test utf8 to string
after = Org.Apache.Hadoop.IO.Text.Decode(utf8Java);
Assert.True(before.Equals(after));
}
}
/// <summary>
/// test
/// <c>Text</c>
/// readFields/write operations
/// </summary>
public virtual void TestReadWriteOperations()
{
string line = "adsawseeeeegqewgasddga";
byte[] inputBytes = Runtime.GetBytesForString(line);
inputBytes = Bytes.Concat(new byte[] { unchecked ((byte)22) }, inputBytes);
DataInputBuffer @in = new DataInputBuffer();
DataOutputBuffer @out = new DataOutputBuffer();
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(line);
try
{
@in.Reset(inputBytes, inputBytes.Length);
text.ReadFields(@in);
}
catch (Exception)
{
Fail("testReadFields error !!!");
}
try
{
text.Write(@out);
}
catch (IOException)
{
}
catch (Exception)
{
Fail("testReadWriteOperations error !!!");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestWithWritable()
{
conf.Set("io.serializations", "org.apache.hadoop.io.serializer.WritableSerialization"
);
Log.Info("Testing DefaultStringifier with Text");
Random random = new Random();
//test with a Text
for (int i = 0; i < 10; i++)
{
//generate a random string
StringBuilder builder = new StringBuilder();
int strLen = random.Next(40);
for (int j = 0; j < strLen; j++)
{
builder.Append(alphabet[random.Next(alphabet.Length)]);
}
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(builder.ToString()
);
DefaultStringifier <Org.Apache.Hadoop.IO.Text> stringifier = new DefaultStringifier
<Org.Apache.Hadoop.IO.Text>(conf, typeof(Org.Apache.Hadoop.IO.Text));
string str = stringifier.ToString(text);
Org.Apache.Hadoop.IO.Text claimedText = stringifier.FromString(str);
Log.Info("Object: " + text);
Log.Info("String representation of the object: " + str);
Assert.Equal(text, claimedText);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFindAfterUpdatingContents()
{
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text("abcd");
text.Set(Runtime.GetBytesForString("a"));
Assert.Equal(text.GetLength(), 1);
Assert.Equal(text.Find("a"), 0);
Assert.Equal(text.Find("b"), -1);
}
/// <exception cref="System.Exception"/>
public virtual void TestValidate()
{
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text("abcd\u20acbdcd\u20ac"
);
byte[] utf8 = text.GetBytes();
int length = text.GetLength();
Org.Apache.Hadoop.IO.Text.ValidateUTF8(utf8, 0, length);
}
/// <exception cref="System.Exception"/>
public virtual void TestFind()
{
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text("abcd\u20acbdcd\u20ac"
);
Assert.True(text.Find("abd") == -1);
Assert.True(text.Find("ac") == -1);
Assert.True(text.Find("\u20ac") == 4);
Assert.True(text.Find("\u20ac", 5) == 11);
}
public virtual void TestCharAt()
{
string line = "adsawseeeeegqewgasddga";
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(line);
for (int i = 0; i < line.Length; i++)
{
Assert.True("testCharAt error1 !!!", text.CharAt(i) == line[i]);
}
Assert.Equal("testCharAt error2 !!!", -1, text.CharAt(-1));
Assert.Equal("testCharAt error3 !!!", -1, text.CharAt(100));
}
/// <exception cref="CharacterCodingException"/>
public virtual void TestTextText()
{
Org.Apache.Hadoop.IO.Text a = new Org.Apache.Hadoop.IO.Text("abc");
Org.Apache.Hadoop.IO.Text b = new Org.Apache.Hadoop.IO.Text("a");
b.Set(a);
Assert.Equal("abc", b.ToString());
a.Append(Runtime.GetBytesForString("xdefgxxx"), 1, 4);
Assert.Equal("modified aliased string", "abc", b.ToString());
Assert.Equal("appended string incorrectly", "abcdefg", a.ToString
());
// add an extra byte so that capacity = 14 and length = 8
a.Append(new byte[] { (byte)('d') }, 0, 1);
Assert.Equal(14, a.GetBytes().Length);
Assert.Equal(8, a.CopyBytes().Length);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestStoreLoad()
{
Log.Info("Testing DefaultStringifier#store() and #load()");
conf.Set("io.serializations", "org.apache.hadoop.io.serializer.WritableSerialization"
);
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text("uninteresting test string"
);
string keyName = "test.defaultstringifier.key1";
DefaultStringifier.Store(conf, text, keyName);
Org.Apache.Hadoop.IO.Text claimedText = DefaultStringifier.Load <Org.Apache.Hadoop.IO.Text
>(conf, keyName);
Assert.Equal("DefaultStringifier#load() or #store() might be flawed"
, text, claimedText);
}
/// <exception cref="System.Exception"/>
public virtual void TestCompare()
{
DataOutputBuffer out1 = new DataOutputBuffer();
DataOutputBuffer out2 = new DataOutputBuffer();
DataOutputBuffer out3 = new DataOutputBuffer();
Text.Comparator comparator = new Text.Comparator();
for (int i = 0; i < NumIterations; i++)
{
// reset output buffer
out1.Reset();
out2.Reset();
out3.Reset();
// generate two random strings
string str1 = GetTestString();
string str2 = GetTestString();
if (i == 0)
{
str1 = GetLongString();
str2 = GetLongString();
}
else
{
str1 = GetTestString();
str2 = GetTestString();
}
// convert to texts
Org.Apache.Hadoop.IO.Text txt1 = new Org.Apache.Hadoop.IO.Text(str1);
Org.Apache.Hadoop.IO.Text txt2 = new Org.Apache.Hadoop.IO.Text(str2);
Org.Apache.Hadoop.IO.Text txt3 = new Org.Apache.Hadoop.IO.Text(str1);
// serialize them
txt1.Write(out1);
txt2.Write(out2);
txt3.Write(out3);
// compare two strings by looking at their binary formats
int ret1 = comparator.Compare(out1.GetData(), 0, out1.GetLength(), out2.GetData()
, 0, out2.GetLength());
// compare two strings
int ret2 = txt1.CompareTo(txt2);
Assert.Equal(ret1, ret2);
Assert.Equal("Equivalence of different txt objects, same content"
, 0, txt1.CompareTo(txt3));
Assert.Equal("Equvalence of data output buffers", 0, comparator
.Compare(out1.GetData(), 0, out3.GetLength(), out3.GetData(), 0, out3.GetLength(
)));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestReadWithKnownLength()
{
string line = "hello world";
byte[] inputBytes = Runtime.GetBytesForString(line, Charsets.Utf8);
DataInputBuffer @in = new DataInputBuffer();
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text();
@in.Reset(inputBytes, inputBytes.Length);
text.ReadWithKnownLength(@in, 5);
Assert.Equal("hello", text.ToString());
// Read longer length, make sure it lengthens
@in.Reset(inputBytes, inputBytes.Length);
text.ReadWithKnownLength(@in, 7);
Assert.Equal("hello w", text.ToString());
// Read shorter length, make sure it shortens
@in.Reset(inputBytes, inputBytes.Length);
text.ReadWithKnownLength(@in, 2);
Assert.Equal("he", text.ToString());
}
/// <exception cref="System.Exception"/>
public virtual void TestClear()
{
// Test lengths on an empty text object
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text();
Assert.Equal("Actual string on an empty text object must be an empty string"
, string.Empty, text.ToString());
Assert.Equal("Underlying byte array length must be zero", 0, text
.GetBytes().Length);
Assert.Equal("String's length must be zero", 0, text.GetLength
());
// Test if clear works as intended
text = new Org.Apache.Hadoop.IO.Text("abcd\u20acbdcd\u20ac");
int len = text.GetLength();
text.Clear();
Assert.Equal("String must be empty after clear()", string.Empty
, text.ToString());
Assert.True("Length of the byte array must not decrease after clear()"
, text.GetBytes().Length >= len);
Assert.Equal("Length of the string must be reset to 0 after clear()"
, 0, text.GetLength());
}
/// <exception cref="System.IO.IOException"/>
internal static long WriteBench(JobConf conf)
{
// OutputFormat instantiation
long filelen = conf.GetLong("filebench.file.bytes", 5 * 1024 * 1024 * 1024);
Org.Apache.Hadoop.IO.Text key = new Org.Apache.Hadoop.IO.Text();
Org.Apache.Hadoop.IO.Text val = new Org.Apache.Hadoop.IO.Text();
string fn = conf.Get("test.filebench.name", string.Empty);
Path outd = FileOutputFormat.GetOutputPath(conf);
conf.Set("mapred.work.output.dir", outd.ToString());
OutputFormat outf = conf.GetOutputFormat();
RecordWriter <Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> rw = outf.GetRecordWriter
(outd.GetFileSystem(conf), conf, fn, Reporter.Null);
try
{
long acc = 0L;
DateTime start = new DateTime();
for (int i = 0; acc < filelen; ++i)
{
i %= keys.Length;
key.Set(keys[i]);
val.Set(values[i]);
rw.Write(key, val);
acc += keys[i].Length;
acc += values[i].Length;
}
DateTime end = new DateTime();
return(end.GetTime() - start.GetTime());
}
finally
{
rw.Close(Reporter.Null);
}
}
