public void testDump() { // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION"); using (Stream file = File.OpenWrite(TestFilePath)) { OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf); options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord))); options.stripeSize(100000); options.compress(CompressionKind.ZLIB); options.bufferSize(10000); options.rowIndexStride(1000); using (Writer writer = OrcFile.createWriter(TestFilePath, file, options)) { Random r1 = new Random(1); for (int i = 0; i < 21000; ++i) { writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[r1.Next(TestHelpers.words.Length)])); } } } string outputFilename = "orc-file-dump.out"; using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename))) { FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" }); } TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename)); }
public void TestErrors(TestFilePath testFilePath) { var inputName = testFilePath.FilePath; var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath)); var inputFile = Path.Combine(baseDir, inputName); var inputText = File.ReadAllText(inputFile); var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension); Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]"); var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8); var result = string.Empty; try { Json.Validate(inputText); } catch (JsonException exception) { result = exception.ToString(); } Console.Write(result); TextAssert.AreEqual(expectedOutputText, result); }
public void Test(TestFilePath testFilePath) { var inputName = testFilePath.FilePath; var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath)); var inputFile = Path.Combine(baseDir, inputName); var inputText = File.ReadAllText(inputFile); var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension); Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]"); var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8); var template = Template.Parse(inputText, "text"); var result = string.Empty; if (template.HasErrors) { for (int i = 0; i < template.Messages.Count; i++) { var message = template.Messages[i]; if (i > 0) { result += "\n"; } result += message; } } else { Assert.NotNull(template.Page); try { result = template.Render(); } catch (ScriptRuntimeException exception) { result = GetReason(exception); } } Console.WriteLine("Result"); Console.WriteLine("======================================"); Console.WriteLine(result); Console.WriteLine("Expected"); Console.WriteLine("======================================"); Console.WriteLine(expectedOutputText); TextAssert.AreEqual(expectedOutputText, result); }
public void testDictionaryThreshold() { // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION"); // conf.setFloat(HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname, 0.49f); using (Stream file = File.OpenWrite(TestFilePath)) { OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf); options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord))); options.stripeSize(100000); options.compress(CompressionKind.ZLIB); options.bufferSize(10000); options.rowIndexStride(1000); using (Writer writer = OrcFile.createWriter(TestFilePath, file, options)) { Random r1 = new Random(1); int nextInt = 0; for (int i = 0; i < 21000; ++i) { // Write out the same string twice, this guarantees the fraction of rows with // distinct strings is 0.5 if (i % 2 == 0) { nextInt = r1.Next(TestHelpers.words.Length); // Append the value of i to the word, this guarantees when an index or word is repeated // the actual string is unique. TestHelpers.words[nextInt] += "-" + i; } writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[nextInt])); } } } string outputFilename = "orc-file-dump-dictionary-threshold.out"; using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename))) { FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" }); } TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename)); }
public void testJsonDump() { ObjectInspector inspector; inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)); // conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION"); OrcFile.WriterOptions options = OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(100000) .compress(CompressionKind.ZLIB) .bufferSize(10000) .rowIndexStride(1000) .bloomFilterColumns("s"); using (Stream file = File.OpenWrite(TestFilePath)) using (Writer writer = OrcFile.createWriter(TestFilePath, file, options)) { Random r1 = new Random(1); for (int i = 0; i < 21000; ++i) { if (i % 100 == 0) { writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), null)); } else { writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[r1.Next(TestHelpers.words.Length)])); } } } const string outputFilename = "orc-file-dump.json"; using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename))) { FileDump.Main(new string[] { TestFilePath.ToString(), "-j", "-p", "--rowindex=3" }); } TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename)); }
public void Test(TestFilePath testFilePath) { var inputName = testFilePath.FilePath; var baseDir = Path.GetFullPath(Path.Combine(BaseDirectory, RelativeBasePath)); var inputFile = Path.Combine(baseDir, inputName); var inputText = File.ReadAllText(inputFile); var expectedOutputFile = Path.ChangeExtension(inputFile, OutputEndFileExtension); Assert.True(File.Exists(expectedOutputFile), $"Expecting output result file [{expectedOutputFile}] for input file [{inputName}]"); var expectedOutputText = File.ReadAllText(expectedOutputFile, Encoding.UTF8); var template = Template.Parse(inputText, "text"); var result = string.Empty; if (template.HasErrors) { for (int i = 0; i < template.Messages.Count; i++) { var message = template.Messages[i]; if (i > 0) { result += "\n"; } result += message; } } else { Assert.NotNull(template.Page); try { result = template.Render(); } catch (ScriptRuntimeException exception) { result = exception.ToString(); } } Console.WriteLine("Result"); Console.WriteLine("======================================"); Console.WriteLine(result); Console.WriteLine("Expected"); Console.WriteLine("======================================"); Console.WriteLine(expectedOutputText); TextAssert.AreEqual(expectedOutputText, result); }