public void testCallback()
{
Configuration conf = new Configuration();
MemoryManager mgr = new MemoryManager(configuredPoolSize);
long pool = mgr.getTotalMemoryPool();
LoggingCallback[] calls = new LoggingCallback[20];
for (int i = 0; i < calls.Length; ++i)
{
calls[i] = new LoggingCallback();
mgr.addWriter(i.ToString(), pool / 4, calls[i]);
}
// add enough rows to get the memory manager to check the limits
for (int i = 0; i < 10000; ++i)
{
mgr.addedRow(1);
}
for (int call = 0; call < calls.Length; ++call)
{
Assert.Equal(2, calls[call].LogLength);
foreach (double argument in calls[call].Log)
{
Assert.Equal(0.2, argument, ERROR);
}
}
}
public void testBasics()
{
MemoryManager mgr = new MemoryManager(configuredPoolSize);
NullCallback callback = new NullCallback();
long poolSize = mgr.getTotalMemoryPool();
Assert.Equal(configuredPoolSize, poolSize);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p1", 1000, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p1", poolSize / 2, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p2", poolSize / 2, callback);
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
mgr.addWriter("p3", poolSize / 2, callback);
Assert.Equal(0.6666667, mgr.getAllocationScale(), 5);
mgr.addWriter("p4", poolSize / 2, callback);
Assert.Equal(0.5, mgr.getAllocationScale(), 6);
mgr.addWriter("p4", 3 * poolSize / 2, callback);
Assert.Equal(0.3333333, mgr.getAllocationScale(), 6);
mgr.removeWriter("p1");
mgr.removeWriter("p2");
Assert.Equal(0.5, mgr.getAllocationScale(), 5);
mgr.removeWriter("p4");
Assert.Equal(1.0, mgr.getAllocationScale(), 5);
}
public WriterImpl(
Stream stream,
string path,
OrcFile.WriterOptions options,
ObjectInspector inspector,
TypeDescription schema,
long stripeSize,
CompressionKind compress,
int bufferSize,
int rowIndexStride,
MemoryManager memoryManager,
bool addBlockPadding,
OrcFile.Version version,
OrcFile.WriterCallback callback,
OrcFile.EncodingStrategy encodingStrategy,
OrcFile.CompressionStrategy compressionStrategy,
double paddingTolerance,
long blockSizeValue,
string bloomFilterColumnNames,
double bloomFilterFpp)
{
this.baseStream = stream;
this.streamFactory = new StreamFactory(this);
this.path = path;
this.options = options;
this.callback = callback;
this.schema = schema;
this.adjustedStripeSize = stripeSize;
this.defaultStripeSize = stripeSize;
this.version = version;
this.encodingStrategy = encodingStrategy;
this.compressionStrategy = compressionStrategy;
this.addBlockPadding = addBlockPadding;
this.blockSize = blockSizeValue;
this.paddingTolerance = paddingTolerance;
this.compress = compress;
this.rowIndexStride = rowIndexStride;
this.memoryManager = memoryManager;
buildIndex = rowIndexStride > 0;
codec = createCodec(compress);
int numColumns = schema.getMaximumId() + 1;
this.bufferSize = getEstimatedBufferSize(defaultStripeSize, numColumns, bufferSize);
if (version == OrcFile.Version.V_0_11)
{
/* do not write bloom filters for ORC v11 */
this.bloomFilterColumns = new bool[schema.getMaximumId() + 1];
}
else
{
this.bloomFilterColumns =
OrcUtils.includeColumns(bloomFilterColumnNames, schema);
}
this.bloomFilterFpp = bloomFilterFpp;
treeWriter = createTreeWriter(inspector, schema, streamFactory, false);
if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE)
{
throw new ArgumentException("Row stride must be at least " +
MIN_ROW_INDEX_STRIDE);
}
// ensure that we are able to handle callbacks before we register ourselves
memoryManager.addWriter(path, stripeSize, this);
}