/// <summary>
/// Loads project from file
/// </summary>
/// <param name="path">Path to .brain/.brainz file</param>
public void OpenProject(string path)
{
MyLog.INFO.WriteLine("Loading project: " + path);
string content;
try
{
string newProjectName = MyProject.MakeNameFromPath(path);
content = ProjectLoader.LoadProject(path,
MyMemoryBlockSerializer.GetTempStorage(newProjectName));
using (MyMemoryManager.Backup backup = MyMemoryManager.GetBackup())
{
Project = MyProject.Deserialize(content, Path.GetDirectoryName(path));
backup.Forget();
}
Project.FileName = path;
}
catch (Exception e)
{
MyLog.ERROR.WriteLine("Project loading failed: " + e.Message);
throw;
}
}
public void testMemoryManagementV12()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(InnerStruct));
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
using (Stream file = FileOpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.compress(CompressionKind.NONE)
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
.version(OrcFile.Version.V_0_12)))
{
Assert.Equal(TestFilePath, memory.path);
for (int i = 0; i < 2500; ++i)
{
writer.addRow(new InnerStruct(i * 300, Integer.toHexString(10 * i)));
}
writer.close();
Assert.Equal(null, memory.path);
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
int j = 0;
foreach (StripeInformation stripe in reader.getStripes())
{
j++;
Assert.True(stripe.getDataLength() < 5000,
"stripe " + j + " is too long at " + stripe.getDataLength());
}
// with HIVE-7832, the dictionaries will be disabled after writing the first
// stripe as there are too many distinct values. Hence only 3 stripes as
// compared to 25 stripes in version 0.11 (above test case)
Assert.Equal(3, j);
Assert.Equal(2500, reader.getNumberOfRows());
}
public void testMemoryManagementV12()
{
TypeDescription schema = createInnerSchema();
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.setSchema(schema)
.compress(CompressionKind.NONE)
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
.version(OrcFile.Version.V_0_12)))
{
VectorizedRowBatch batch = schema.createRowBatch();
Assert.Equal(TestFilePath, memory.path);
batch.size = 1;
for (int i = 0; i < 2500; ++i)
{
((LongColumnVector)batch.cols[0]).vector[0] = i * 300;
((BytesColumnVector)batch.cols[1]).setVal(0,
Integer.toHexString(10 * i).getBytes());
writer.addRowBatch(batch);
}
}
Assert.Null(memory.path);
Reader reader = OrcFile.createReader(TestFilePath,
OrcFile.readerOptions(conf));
int idx = 0;
foreach (StripeInformation stripe in reader.getStripes())
{
idx += 1;
Assert.True(stripe.getDataLength() < 5000,
"stripe " + idx + " is too long at " + stripe.getDataLength());
}
// with HIVE-7832, the dictionaries will be disabled after writing the first
// stripe as there are too many distinct values. Hence only 3 stripes as
// compared to 25 stripes in version 0.11 (above test case)
Assert.Equal(3, idx);
Assert.Equal(2500, reader.getNumberOfRows());
}
public void testMemoryManagementV11()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(InnerStruct));
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
using (Stream file = FileOpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.compress(CompressionKind.NONE)
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
.version(OrcFile.Version.V_0_11)))
{
Assert.Equal(TestFilePath, memory.path);
for (int i = 0; i < 2500; ++i)
{
writer.addRow(new InnerStruct(i * 300, Integer.toHexString(10 * i)));
}
writer.close();
Assert.Equal(null, memory.path);
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
int j = 0;
foreach (StripeInformation stripe in reader.getStripes())
{
j++;
Assert.True(stripe.getDataLength() < 5000,
"stripe " + j + " is too long at " + stripe.getDataLength());
}
Assert.Equal(25, j);
Assert.Equal(2500, reader.getNumberOfRows());
}
public void testMemoryManagementV11()
{
TypeDescription schema = createInnerSchema();
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.setSchema(schema)
.compress(CompressionKind.NONE)
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
.memory(memory)
.version(OrcFile.Version.V_0_11)))
{
Assert.Equal(TestFilePath, memory.path);
VectorizedRowBatch batch = schema.createRowBatch();
batch.size = 1;
for (int i = 0; i < 2500; ++i)
{
((LongColumnVector)batch.cols[0]).vector[0] = i * 300;
((BytesColumnVector)batch.cols[1]).setVal(0,
Integer.toHexString(10 * i).getBytes());
writer.addRowBatch(batch);
}
}
Assert.Null(memory.path);
Reader reader = OrcFile.createReader(TestFilePath,
OrcFile.readerOptions(conf));
int idx = 0;
foreach (StripeInformation stripe in reader.getStripes())
{
idx += 1;
Assert.True(stripe.getDataLength() < 5000,
"stripe " + idx + " is too long at " + stripe.getDataLength());
}
Assert.Equal(25, idx);
Assert.Equal(2500, reader.getNumberOfRows());
}
