public void testCallback()
{
Configuration conf = new Configuration();
MemoryManager mgr = new MemoryManager(configuredPoolSize);
long pool = mgr.getTotalMemoryPool();
LoggingCallback[] calls = new LoggingCallback[20];
for (int i = 0; i < calls.Length; ++i)
{
calls[i] = new LoggingCallback();
mgr.addWriter(i.ToString(), pool / 4, calls[i]);
}
// add enough rows to get the memory manager to check the limits
for (int i = 0; i < 10000; ++i)
{
mgr.addedRow(1);
}
for (int call = 0; call < calls.Length; ++call)
{
Assert.Equal(2, calls[call].LogLength);
foreach (double argument in calls[call].Log)
{
Assert.Equal(0.2, argument, ERROR);
}
}
}
public void testBasics()
{
MemoryManager mgr = new MemoryManager(configuredPoolSize);
NullCallback callback = new NullCallback();
long poolSize = mgr.getTotalMemoryPool();
Assert.Equal(configuredPoolSize, poolSize);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p1", 1000, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p1", poolSize / 2, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p2", poolSize / 2, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p3", poolSize / 2, callback);
Assert.Equal(0.6666667, mgr.getAllocationScale(), 5);
mgr.addWriter("p4", poolSize / 2, callback);
Assert.Equal(0.5, mgr.getAllocationScale(), 6);
mgr.addWriter("p4", 3 * poolSize / 2, callback);
Assert.Equal(0.3333333, mgr.getAllocationScale(), 6);
mgr.removeWriter("p1");
mgr.removeWriter("p2");
Assert.Equal(0.5, mgr.getAllocationScale(), 5);
mgr.removeWriter("p4");
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
}
public WriterImpl(
Stream stream,
string path,
OrcFile.WriterOptions options,
ObjectInspector inspector,
TypeDescription schema,
long stripeSize,
CompressionKind compress,
int bufferSize,
int rowIndexStride,
MemoryManager memoryManager,
bool addBlockPadding,
OrcFile.Version version,
OrcFile.WriterCallback callback,
OrcFile.EncodingStrategy encodingStrategy,
OrcFile.CompressionStrategy compressionStrategy,
double paddingTolerance,
long blockSizeValue,
string bloomFilterColumnNames,
double bloomFilterFpp)
{
this.baseStream = stream;
this.streamFactory = new StreamFactory(this);
this.path = path;
this.options = options;
this.callback = callback;
this.schema = schema;
this.adjustedStripeSize = stripeSize;
this.defaultStripeSize = stripeSize;
this.version = version;
this.encodingStrategy = encodingStrategy;
this.compressionStrategy = compressionStrategy;
this.addBlockPadding = addBlockPadding;
this.blockSize = blockSizeValue;
this.paddingTolerance = paddingTolerance;
this.compress = compress;
this.rowIndexStride = rowIndexStride;
this.memoryManager = memoryManager;
buildIndex = rowIndexStride > 0;
codec = createCodec(compress);
int numColumns = schema.getMaximumId() + 1;
this.bufferSize = getEstimatedBufferSize(defaultStripeSize, numColumns, bufferSize);
if (version == OrcFile.Version.V_0_11)
{
/* do not write bloom filters for ORC v11 */
this.bloomFilterColumns = new bool[schema.getMaximumId() + 1];
}
else
{
this.bloomFilterColumns =
OrcUtils.includeColumns(bloomFilterColumnNames, schema);
}
this.bloomFilterFpp = bloomFilterFpp;
treeWriter = createTreeWriter(inspector, schema, streamFactory, false);
if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE)
{
throw new ArgumentException("Row stride must be at least " +
MIN_ROW_INDEX_STRIDE);
}
// ensure that we are able to handle callbacks before we register ourselves
memoryManager.addWriter(path, stripeSize, this);
}
public override void addWriter(string path, long requestedAllocation, MemoryManager.Callback callback)
{
this.path = path;
this.lastAllocation = requestedAllocation;
this.callback = callback;
}
public void testConfig()
{
Configuration conf = new Configuration();
conf.set("hive.exec.orc.memory.pool", "0.9");
MemoryManager mgr = new MemoryManager(conf);
long mem = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
System.Console.WriteLine("Memory = " + mem);
long pool = mgr.getTotalMemoryPool();
Assert.True("Pool too small: " + pool, mem * 0.899 < pool);
Assert.True("Pool too big: " + pool, pool < mem * 0.901);
}
/**
* A package local option to set the memory manager.
*/
public WriterOptions memory(MemoryManager value)
{
memoryManagerValue = value;
return this;
}
public WriterOptions(Properties tableProperties, Configuration conf)
{
configuration = conf;
memoryManagerValue = getMemoryManager(conf);
stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
rowIndexStrideValue =
(int)OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);
bufferSizeValue = (int)OrcConf.BUFFER_SIZE.getLong(tableProperties,
conf);
blockPaddingValue =
OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
compressValue = (CompressionKind)Enum.Parse(
typeof(CompressionKind),
OrcConf.COMPRESS.getString(tableProperties, conf),
true);
string versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
conf);
versionValue = VersionHelper.byName(versionName);
string enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,
conf);
_encodingStrategy = (EncodingStrategy)Enum.Parse(typeof(EncodingStrategy), enString, true);
string compString =
OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);
compressionStrategy = (CompressionStrategy)Enum.Parse(typeof(CompressionStrategy), compString, true);
_paddingTolerance =
OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);
_bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,
conf);
_bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,
conf);
timeZone = TimeZoneInfo.Local.Id;
}
