protected AbstractTask(TaskId id, IEnumerable <TopicPartition> partition, ProcessorTopology topology, IConsumer <byte[], byte[]> consumer, IStreamConfig config, IChangelogRegister changelogRegister)
{
log = Logger.GetLogger(GetType());
logPrefix = $"stream-task[{id.Id}|{id.Partition}] ";
Partition = partition;
Id = id;
Topology = topology;
this.consumer = consumer;
configuration = config;
var offsetCheckpointMngt = config.OffsetCheckpointManager
?? new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}"));
offsetCheckpointMngt.Configure(config, id);
stateMgr = new ProcessorStateManager(
id,
partition,
topology.StoresToTopics,
changelogRegister,
offsetCheckpointMngt);
}
public void Begin()
{
Random rd = new Random();
config = new StreamConfig();
config.ApplicationId = $"RocksDbOptionsTests";
config.UseRandomRocksDbConfigForTest();
config.RocksDbConfigHandler = (name, options) =>
{
options
.EnableStatistics()
.IncreaseParallelism(parallelism)
.OptimizeForPointLookup(blockCacheSizeMb)
.OptimizeLevelStyleCompaction(memTableMemoryBudget)
.OptimizeUniversalStyleCompaction(memtableMemoryBudget2)
.PrepareForBulkLoad()
.SetAccessHintOnCompactionStart(accessHintOnCompactionStart)
.SetAdviseRandomOnOpen(adviseRandomOnOpen)
.SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite)
.SetAllowMmapReads(false)
.SetAllowMmapWrites(false)
.SetArenaBlockSize(arenaBlockSize)
.SetBaseBackgroundCompactions(baseBackgroundCompactions)
.SetBloomLocality(bloomLocality)
.SetBytesPerSync(bytesPerSync)
// .SetCompactionFilter(IntPtr.Zero)
// .SetCompactionFilterFactory(IntPtr.Zero)
.SetCompactionReadaheadSize(1200)
.SetCompactionStyle(RocksDbSharp.Compaction.Level)
//.SetComparator(IntPtr.Zero)
.SetCompression(RocksDbSharp.Compression.Lz4)
.SetCompressionOptions(1, 2, 3, 4)
.SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1)
.SetCreateIfMissing()
.SetCreateMissingColumnFamilies()
.SetDbLogDir("test")
.SetDbWriteBufferSize(1000)
.SetDeleteObsoleteFilesPeriodMicros(50)
.SetDisableAutoCompactions(1)
.SetEnableWriteThreadAdaptiveYield(true)
//.SetEnv(IntPtr.Zero)
.SetErrorIfExists()
//.SetFifoCompactionOptions(IntPtr.Zero)
.SetHardPendingCompactionBytesLimit(1)
.SetHardRateLimit(40597)
.SetHashLinkListRep(12)
.SetHashSkipListRep(56, 4, 2)
//.SetInfoLog(IntPtr.Zero)
.SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG)
.SetInplaceUpdateNumLocks(134)
.SetIsFdCloseOnExec(false)
.SetKeepLogFileNum(1)
.SetLevel0FileNumCompactionTrigger(14)
.SetLevel0SlowdownWritesTrigger(144)
.SetInplaceUpdateSupport(true)
.SetLevel0StopWritesTrigger(24)
.SetLevelCompactionDynamicLevelBytes(true)
.SetLogFileTimeToRoll(154)
.SetManifestPreallocationSize(153)
.SetMaxBackgroundFlushes(3)
.SetMaxBytesForLevelBase(1453)
.SetMaxBytesForLevelMultiplier(2)
.SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1)
.SetMaxCompactionBytes(345678)
.SetMaxFileOpeningThreads(2)
.SetMaxLogFileSize(1)
.SetMaxManifestFileSize(131)
.SetMaxMemCompactionLevel(12)
.SetMaxOpenFiles(20)
.SetMaxSequentialSkipInIterations(13)
.SetMaxSuccessiveMerges(12)
.SetMaxTotalWalSize(12)
.SetMaxWriteBufferNumber(15543)
.SetMaxWriteBufferNumberToMaintain(126)
.SetMemtableHugePageSize(64317)
.SetMemtableHugePageSize(64317)
.SetMemtablePrefixBloomSizeRatio(12)
//.SetMergeOperator(IntPtr.Zero)
.SetMinLevelToCompress(1)
.SetMemtableVectorRep()
.SetMinWriteBufferNumberToMerge(1312)
.SetNumLevels(1)
.SetOptimizeFiltersForHits(56)
.SetParanoidChecks()
.SetSkipLogErrorOnRecovery(true)
.SetSoftPendingCompactionBytesLimit(134)
.SetSoftRateLimit(131)
.SetStatsDumpPeriodSec(532)
.SetTableCacheNumShardbits(12)
.SetTableCacheRemoveScanCountLimit(66)
.SetTargetFileSizeBase(6)
.SetTargetFileSizeMultiplier(2)
.SetUint64addMergeOperator()
//.SetUniversalCompactionOptions(IntPtr.Zero)
.SetUseAdaptiveMutex(false)
.SetUseDirectIoForFlushAndCompaction(true)
.SetUseDirectReads(true)
.SetUseFsync(1)
.SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords)
.SetWALSizeLimitMB(40)
.SetWALTtlSeconds(1454151413)
.SetWriteBufferSize(45678976543)
.SetPrefixExtractor(SliceTransform.CreateNoOp())
.SetPurgeRedundantKvsWhileFlush(false)
.SetRateLimitDelayMaxMilliseconds(1762)
.SetRecycleLogFileNum(1)
.SetReportBgIoStats(true)
.SetPlainTableFactory(1, 23, 4, 2)
.SetMaxBackgroundCompactions(1);
};
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}
public void Register(TopicPartition topicPartition, ProcessorStateManager processorStateManager)
{
restoringPartitions.Add(topicPartition);
}
