public void testDump()
{
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
using (Stream file = File.OpenWrite(TestFilePath))
{
OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
options.stripeSize(100000);
options.compress(CompressionKind.ZLIB);
options.bufferSize(10000);
options.rowIndexStride(1000);
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
for (int i = 0; i < 21000; ++i)
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
}
}
}
string outputFilename = "orc-file-dump.out";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void TestErrors(TestFilePath testFilePath)
{
var inputName = testFilePath.FilePath;
var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath));
var inputFile = Path.Combine(baseDir, inputName);
var inputText = File.ReadAllText(inputFile);
var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension);
Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]");
var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8);
var result = string.Empty;
try
{
Json.Validate(inputText);
}
catch (JsonException exception)
{
result = exception.ToString();
}
Console.Write(result);
TextAssert.AreEqual(expectedOutputText, result);
}
public void Test(TestFilePath testFilePath)
{
var inputName = testFilePath.FilePath;
var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath));
var inputFile = Path.Combine(baseDir, inputName);
var inputText = File.ReadAllText(inputFile);
var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension);
Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]");
var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8);
var template = Template.Parse(inputText, "text");
var result = string.Empty;
if (template.HasErrors)
{
for (int i = 0; i < template.Messages.Count; i++)
{
var message = template.Messages[i];
if (i > 0)
{
result += "\n";
}
result += message;
}
}
else
{
Assert.NotNull(template.Page);
try
{
result = template.Render();
}
catch (ScriptRuntimeException exception)
{
result = GetReason(exception);
}
}
Console.WriteLine("Result");
Console.WriteLine("======================================");
Console.WriteLine(result);
Console.WriteLine("Expected");
Console.WriteLine("======================================");
Console.WriteLine(expectedOutputText);
TextAssert.AreEqual(expectedOutputText, result);
}
public void testDictionaryThreshold()
{
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
// conf.setFloat(HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname, 0.49f);
using (Stream file = File.OpenWrite(TestFilePath))
{
OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
options.stripeSize(100000);
options.compress(CompressionKind.ZLIB);
options.bufferSize(10000);
options.rowIndexStride(1000);
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
int nextInt = 0;
for (int i = 0; i < 21000; ++i)
{
// Write out the same string twice, this guarantees the fraction of rows with
// distinct strings is 0.5
if (i % 2 == 0)
{
nextInt = r1.Next(TestHelpers.words.Length);
// Append the value of i to the word, this guarantees when an index or word is repeated
// the actual string is unique.
TestHelpers.words[nextInt] += "-" + i;
}
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[nextInt]));
}
}
}
string outputFilename = "orc-file-dump-dictionary-threshold.out";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void testJsonDump()
{
ObjectInspector inspector;
inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord));
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.ZLIB)
.bufferSize(10000)
.rowIndexStride(1000)
.bloomFilterColumns("s");
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
for (int i = 0; i < 21000; ++i)
{
if (i % 100 == 0)
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), null));
}
else
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
}
}
}
const string outputFilename = "orc-file-dump.json";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "-j", "-p", "--rowindex=3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void TestErrors(TestFilePath testFilePath)
{
var inputName = testFilePath.FilePath;
var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath));
var inputFile = Path.Combine(baseDir, inputName);
var inputText = File.ReadAllText(inputFile);
var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension);
Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]");
var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8);
var result = string.Empty;
try
{
Json.Validate(inputText);
}
catch (JsonException exception)
{
result = exception.ToString();
}
Console.Write(result);
TextAssert.AreEqual(expectedOutputText, result);
}
public void Test(TestFilePath testFilePath)
{
var inputName = testFilePath.FilePath;
var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath));
var inputFile = Path.Combine(baseDir, inputName);
var inputText = File.ReadAllText(inputFile);
var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension);
Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]");
var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8);
var template = Template.Parse(inputText, "text");
var result = string.Empty;
if (template.HasErrors)
{
for (int i = 0; i < template.Messages.Count; i++)
{
var message = template.Messages[i];
if (i > 0)
{
result += "\n";
}
result += message;
}
}
else
{
Assert.NotNull(template.Page);
try
{
result = template.Render();
}
catch (ScriptRuntimeException exception)
{
result = exception.ToString();
}
}
Console.WriteLine("Result");
Console.WriteLine("======================================");
Console.WriteLine(result);
Console.WriteLine("Expected");
Console.WriteLine("======================================");
Console.WriteLine(expectedOutputText);
TextAssert.AreEqual(expectedOutputText, result);
}
