public ChunkManagerConfig(string basePath,
IFileNamingStrategy fileNamingStrategy,
int chunkDataSize,
int chunkDataUnitSize,
int chunkDataCount,
int flushChunkIntervalMilliseconds,
bool enableCache,
bool syncFlush,
int chunkReaderCount,
int maxLogRecordSize,
int chunkWriteBuffer,
int chunkReadBuffer,
int chunkCacheMaxPercent,
int chunkCacheMinPercent,
int preCacheChunkCount,
int chunkInactiveTimeMaxSeconds,
int chunkLocalCacheSize,
bool enableChunkWriteStatistic,
bool enableChunkReadStatistic)
{
Ensure.NotNullOrEmpty(basePath, "basePath");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Nonnegative(chunkDataSize, "chunkDataSize");
Ensure.Nonnegative(chunkDataUnitSize, "chunkDataUnitSize");
Ensure.Nonnegative(chunkDataCount, "chunkDataCount");
Ensure.Positive(flushChunkIntervalMilliseconds, "flushChunkIntervalMilliseconds");
Ensure.Positive(maxLogRecordSize, "maxLogRecordSize");
Ensure.Positive(chunkWriteBuffer, "chunkWriteBuffer");
Ensure.Positive(chunkReadBuffer, "chunkReadBuffer");
Ensure.Positive(chunkCacheMaxPercent, "chunkCacheMaxPercent");
Ensure.Positive(chunkCacheMinPercent, "chunkCacheMinPercent");
Ensure.Nonnegative(preCacheChunkCount, "preCacheChunkCount");
Ensure.Nonnegative(chunkInactiveTimeMaxSeconds, "chunkInactiveTimeMaxSeconds");
Ensure.Positive(chunkLocalCacheSize, "chunkLocalCacheSize");
if (chunkDataSize <= 0 && (chunkDataUnitSize <= 0 || chunkDataCount <= 0))
{
throw new ArgumentException(string.Format("Invalid chunk data size arugment. chunkDataSize: {0}, chunkDataUnitSize: {1}, chunkDataCount: {2}", chunkDataSize, chunkDataUnitSize, chunkDataCount));
}
BasePath = basePath;
FileNamingStrategy = fileNamingStrategy;
ChunkDataSize = chunkDataSize;
ChunkDataUnitSize = chunkDataUnitSize;
ChunkDataCount = chunkDataCount;
FlushChunkIntervalMilliseconds = flushChunkIntervalMilliseconds;
EnableCache = enableCache;
SyncFlush = syncFlush;
ChunkReaderCount = chunkReaderCount;
MaxLogRecordSize = maxLogRecordSize;
ChunkWriteBuffer = chunkWriteBuffer;
ChunkReadBuffer = chunkReadBuffer;
ChunkCacheMaxPercent = chunkCacheMaxPercent;
ChunkCacheMinPercent = chunkCacheMinPercent;
PreCacheChunkCount = preCacheChunkCount;
ChunkInactiveTimeMaxSeconds = chunkInactiveTimeMaxSeconds;
ChunkLocalCacheSize = chunkLocalCacheSize;
EnableChunkWriteStatistic = enableChunkWriteStatistic;
EnableChunkReadStatistic = enableChunkReadStatistic;
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
long maxChunksCacheSize,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
ICheckpoint epochCheckpoint,
ICheckpoint truncateCheckpoint,
bool inMemDb = false)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(maxChunksCacheSize, "maxChunksCacheSize");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(epochCheckpoint, "epochCheckpoint");
Ensure.NotNull(truncateCheckpoint, "truncateCheckpoint");
Path = path;
ChunkSize = chunkSize;
MaxChunksCacheSize = maxChunksCacheSize;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
EpochCheckpoint = epochCheckpoint;
TruncateCheckpoint = truncateCheckpoint;
FileNamingStrategy = fileNamingStrategy;
InMemDb = inMemDb;
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
long maxChunksCacheSize,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
ICheckpoint epochCheckpoint,
ICheckpoint truncateCheckpoint,
bool inMemDb = false)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(maxChunksCacheSize, "maxChunksCacheSize");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(epochCheckpoint, "epochCheckpoint");
Ensure.NotNull(truncateCheckpoint, "truncateCheckpoint");
Path = path;
ChunkSize = chunkSize;
MaxChunksCacheSize = maxChunksCacheSize;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
EpochCheckpoint = epochCheckpoint;
TruncateCheckpoint = truncateCheckpoint;
FileNamingStrategy = fileNamingStrategy;
InMemDb = inMemDb;
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
int cachedChunkCount,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
params ICheckpoint[] namedCheckpoints)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(cachedChunkCount, "cachedChunkCount");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(namedCheckpoints, "namedCheckpoints");
// if ((chunkSize & (chunkSize-1)) != 0)
// throw new ArgumentException("Segment size should be the power of 2.", "chunkSize");
Path = path;
ChunkSize = chunkSize;
CachedChunkCount = cachedChunkCount;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
FileNamingStrategy = fileNamingStrategy;
_namedCheckpoints = namedCheckpoints.ToDictionary(x => x.Name);
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
int cachedChunkCount,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
params ICheckpoint[] namedCheckpoints)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(cachedChunkCount, "cachedChunkCount");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(namedCheckpoints, "namedCheckpoints");
// if ((chunkSize & (chunkSize-1)) != 0)
// throw new ArgumentException("Segment size should be the power of 2.", "chunkSize");
Path = path;
ChunkSize = chunkSize;
CachedChunkCount = cachedChunkCount;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
FileNamingStrategy = fileNamingStrategy;
_namedCheckpoints = namedCheckpoints.ToDictionary(x => x.Name);
}
//TODO GFY add fluent builder for this
public TransactionFileDatabaseConfig(string path,
string filePrefix,
long segmentSize,
ICheckpoint writerCheckpoint,
IEnumerable<ICheckpoint> namedCheckpoints)
{
if (path == null)
throw new ArgumentNullException("path");
if (filePrefix == null)
throw new ArgumentNullException("filePrefix");
if (segmentSize <= 0)
throw new ArgumentOutOfRangeException("segmentSize");
if (writerCheckpoint == null)
throw new ArgumentNullException("writerCheckpoint");
if (namedCheckpoints == null)
throw new ArgumentNullException("namedCheckpoints");
// if ((segmentSize & (segmentSize-1)) != 0)
// throw new ArgumentException("Segment size should be the power of 2.", "segmentSize");
Path = path;
FilePrefix = filePrefix;
FileNamingStrategy = new PrefixFileNamingStrategy(path, filePrefix);
SegmentSize = segmentSize;
WriterCheckpoint = writerCheckpoint;
_namedCheckpoints = namedCheckpoints.ToDictionary(x => x.Name);
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
long maxChunksCacheSize,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
ICheckpoint epochCheckpoint,
ICheckpoint proposalCheckpoint,
ICheckpoint truncateCheckpoint,
ICheckpoint replicationCheckpoint,
ICheckpoint indexCheckpoint,
ICheckpoint streamExistenceFilterCheckpoint,
int initialReaderCount,
int maxReaderCount,
bool inMemDb = false,
bool unbuffered = false,
bool writethrough = false,
bool optimizeReadSideCache = false,
bool reduceFileCachePressure = false,
long maxTruncation = 256 * 1024 * 1024)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(maxChunksCacheSize, "maxChunksCacheSize");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(epochCheckpoint, "epochCheckpoint");
Ensure.NotNull(proposalCheckpoint, "proposalCheckpoint");
Ensure.NotNull(truncateCheckpoint, "truncateCheckpoint");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
Ensure.NotNull(indexCheckpoint, "indexCheckpoint");
Ensure.NotNull(streamExistenceFilterCheckpoint, "streamExistenceFilterCheckpoint");
Ensure.Positive(initialReaderCount, "initialReaderCount");
Ensure.Positive(maxReaderCount, "maxReaderCount");
Path = path;
ChunkSize = chunkSize;
MaxChunksCacheSize = maxChunksCacheSize;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
EpochCheckpoint = epochCheckpoint;
ProposalCheckpoint = proposalCheckpoint;
TruncateCheckpoint = truncateCheckpoint;
ReplicationCheckpoint = replicationCheckpoint;
IndexCheckpoint = indexCheckpoint;
StreamExistenceFilterCheckpoint = streamExistenceFilterCheckpoint;
FileNamingStrategy = fileNamingStrategy;
InMemDb = inMemDb;
Unbuffered = unbuffered;
WriteThrough = writethrough;
InitialReaderCount = initialReaderCount;
MaxReaderCount = maxReaderCount;
OptimizeReadSideCache = optimizeReadSideCache;
ReduceFileCachePressure = reduceFileCachePressure;
MaxTruncation = maxTruncation;
}
public SFTPFilesDownloader(
ILogger <SFTPFilesDownloader> logger,
IFileNamingStrategy namingStrategy,
IOptions <SFtpOptions> options,
IFilesManager manager)
{
_logger = logger;
_namingStrategy = namingStrategy;
_manager = manager;
_options = options;
}
/// <summary>
/// Создает менеджера страниц данных.
/// </summary>
public DataPageManager(DataPageManagerConfig config)
{
_config = config;
if (!Directory.Exists(_config.Directory))
{
Directory.CreateDirectory(_config.Directory);
}
Name = config.Name;
_dataPageFileNamingStrategy = new HierarchyFileNamingStrategy(_config.Directory);
_dataPages = new ConcurrentDictionary <int, DataPage>();
_dataRecordIndexStorage = new DataRecordIndexStorage(_config.Directory, new DataRecordIndexStoreConfig(TimeSpan.FromMilliseconds(500)));
LoadMetaData();
var lastRecordIndex = _dataRecordIndexStorage.AsEnumerable().Reverse().FirstOrDefault();
if (lastRecordIndex != default)
{
_currentDataRecordId = lastRecordIndex.DataRecordId;
}
}
public TFChunkDbConfig(string path,
IFileNamingStrategy fileNamingStrategy,
int chunkSize,
long maxChunksCacheSize,
ICheckpoint writerCheckpoint,
ICheckpoint chaserCheckpoint,
ICheckpoint epochCheckpoint,
ICheckpoint truncateCheckpoint,
ICheckpoint replicationCheckpoint,
int initialReaderCount,
bool inMemDb = false,
bool unbuffered = false,
bool writethrough = false)
{
Ensure.NotNullOrEmpty(path, "path");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Positive(chunkSize, "chunkSize");
Ensure.Nonnegative(maxChunksCacheSize, "maxChunksCacheSize");
Ensure.NotNull(writerCheckpoint, "writerCheckpoint");
Ensure.NotNull(chaserCheckpoint, "chaserCheckpoint");
Ensure.NotNull(epochCheckpoint, "epochCheckpoint");
Ensure.NotNull(truncateCheckpoint, "truncateCheckpoint");
Ensure.NotNull(replicationCheckpoint, "replicationCheckpoint");
Ensure.Positive(initialReaderCount, "initialReaderCount");
Path = path;
ChunkSize = chunkSize;
MaxChunksCacheSize = maxChunksCacheSize;
WriterCheckpoint = writerCheckpoint;
ChaserCheckpoint = chaserCheckpoint;
EpochCheckpoint = epochCheckpoint;
TruncateCheckpoint = truncateCheckpoint;
ReplicationCheckpoint = replicationCheckpoint;
FileNamingStrategy = fileNamingStrategy;
InMemDb = inMemDb;
Unbuffered = unbuffered;
WriteThrough = writethrough;
InitialReaderCount = initialReaderCount;
}
//TODO GFY add fluent builder for this
public TransactionFileDatabaseConfig(string path,
string filePrefix,
long segmentSize,
ICheckpoint writerCheckpoint,
IEnumerable <ICheckpoint> namedCheckpoints)
{
if (path == null)
{
throw new ArgumentNullException("path");
}
if (filePrefix == null)
{
throw new ArgumentNullException("filePrefix");
}
if (segmentSize <= 0)
{
throw new ArgumentOutOfRangeException("segmentSize");
}
if (writerCheckpoint == null)
{
throw new ArgumentNullException("writerCheckpoint");
}
if (namedCheckpoints == null)
{
throw new ArgumentNullException("namedCheckpoints");
}
// if ((segmentSize & (segmentSize-1)) != 0)
// throw new ArgumentException("Segment size should be the power of 2.", "segmentSize");
Path = path;
FilePrefix = filePrefix;
FileNamingStrategy = new PrefixFileNamingStrategy(path, filePrefix);
SegmentSize = segmentSize;
WriterCheckpoint = writerCheckpoint;
_namedCheckpoints = namedCheckpoints.ToDictionary(x => x.Name);
}
public ChunkManagerConfig(string basePath,
IFileNamingStrategy fileNamingStrategy,
int chunkDataSize,
int chunkDataUnitSize,
int chunkDataCount,
int flushChunkIntervalMilliseconds,
bool enableCache,
bool syncFlush,
int chunkReaderCount,
int maxLogRecordSize,
int chunkWriteBuffer,
int chunkReadBuffer,
int chunkCacheMaxPercent,
int chunkCacheMinPercent,
int preCacheChunkCount,
int chunkInactiveTimeMaxSeconds,
int chunkLocalCacheSize,
bool enableChunkWriteStatistic,
bool enableChunkReadStatistic)
{
basePath.CheckNotNullOrEmpty("basePath");
fileNamingStrategy.CheckNotNull("fileNamingStrategy");
chunkDataSize.CheckGreaterThan("chunkDataSize", 0, true);
chunkDataUnitSize.CheckGreaterThan("chunkDataUnitSize", 0, true);
chunkDataCount.CheckGreaterThan("chunkDataCount", 0, true);
flushChunkIntervalMilliseconds.CheckGreaterThan("flushChunkIntervalMilliseconds", 0);
maxLogRecordSize.CheckGreaterThan("maxLogRecordSize", 0);
chunkWriteBuffer.CheckGreaterThan("chunkWriteBuffer", 0);
chunkReadBuffer.CheckGreaterThan("chunkReadBuffer", 0);
chunkCacheMaxPercent.CheckGreaterThan("chunkCacheMaxPercent", 0);
chunkCacheMinPercent.CheckGreaterThan("chunkCacheMinPercent", 0);
preCacheChunkCount.CheckGreaterThan("preCacheChunkCount", 0, true);
chunkInactiveTimeMaxSeconds.CheckGreaterThan("chunkInactiveTimeMaxSeconds", 0, true);
chunkLocalCacheSize.CheckGreaterThan("chunkLocalCacheSize", 0);
if (chunkDataSize <= 0 && (chunkDataUnitSize <= 0 || chunkDataCount <= 0))
{
throw new ArgumentException(string.Format("Invalid chunk data size arugment. chunkDataSize: {0}, chunkDataUnitSize: {1}, chunkDataCount: {2}", chunkDataSize, chunkDataUnitSize, chunkDataCount));
}
BasePath = basePath;
FileNamingStrategy = fileNamingStrategy;
ChunkDataSize = chunkDataSize;
ChunkDataUnitSize = chunkDataUnitSize;
ChunkDataCount = chunkDataCount;
FlushChunkIntervalMilliseconds = flushChunkIntervalMilliseconds;
EnableCache = enableCache;
SyncFlush = syncFlush;
ChunkReaderCount = chunkReaderCount;
MaxLogRecordSize = maxLogRecordSize;
ChunkWriteBuffer = chunkWriteBuffer;
ChunkReadBuffer = chunkReadBuffer;
ChunkCacheMaxPercent = chunkCacheMaxPercent;
ChunkCacheMinPercent = chunkCacheMinPercent;
PreCacheChunkCount = preCacheChunkCount;
ChunkInactiveTimeMaxSeconds = chunkInactiveTimeMaxSeconds;
ChunkLocalCacheSize = chunkLocalCacheSize;
EnableChunkWriteStatistic = enableChunkWriteStatistic;
EnableChunkReadStatistic = enableChunkReadStatistic;
if (GetChunkDataSize() > 1024 * 1024 * 1024)
{
throw new ArgumentException("Chunk data size cannot bigger than 1G");
}
}
public FileSink(IMetricsSerializer serializer, IFileNamingStrategy fileNamingStrategy)
{
_serializer = serializer;
_fileNamingStrategy = fileNamingStrategy;
}
public ChunkManagerConfig(string basePath,
IFileNamingStrategy fileNamingStrategy,
int chunkDataSize,
int chunkDataUnitSize,
int chunkDataCount,
int flushChunkIntervalMilliseconds,
bool enableCache,
bool syncFlush,
int chunkReaderCount,
int maxLogRecordSize,
int chunkWriteBuffer,
int chunkReadBuffer,
int chunkCacheMaxPercent,
int chunkCacheMinPercent,
int preCacheChunkCount,
int chunkInactiveTimeMaxSeconds,
int chunkLocalCacheSize,
bool enableChunkWriteStatistic,
bool enableChunkReadStatistic)
{
Ensure.NotNullOrEmpty(basePath, "basePath");
Ensure.NotNull(fileNamingStrategy, "fileNamingStrategy");
Ensure.Nonnegative(chunkDataSize, "chunkDataSize");
Ensure.Nonnegative(chunkDataUnitSize, "chunkDataUnitSize");
Ensure.Nonnegative(chunkDataCount, "chunkDataCount");
Ensure.Positive(flushChunkIntervalMilliseconds, "flushChunkIntervalMilliseconds");
Ensure.Positive(maxLogRecordSize, "maxLogRecordSize");
Ensure.Positive(chunkWriteBuffer, "chunkWriteBuffer");
Ensure.Positive(chunkReadBuffer, "chunkReadBuffer");
Ensure.Positive(chunkCacheMaxPercent, "chunkCacheMaxPercent");
Ensure.Positive(chunkCacheMinPercent, "chunkCacheMinPercent");
Ensure.Nonnegative(preCacheChunkCount, "preCacheChunkCount");
Ensure.Nonnegative(chunkInactiveTimeMaxSeconds, "chunkInactiveTimeMaxSeconds");
Ensure.Positive(chunkLocalCacheSize, "chunkLocalCacheSize");
if (chunkDataSize <= 0 && (chunkDataUnitSize <= 0 || chunkDataCount <= 0))
{
throw new ArgumentException(string.Format("Invalid chunk data size arugment. chunkDataSize: {0}, chunkDataUnitSize: {1}, chunkDataCount: {2}", chunkDataSize, chunkDataUnitSize, chunkDataCount));
}
BasePath = basePath;
FileNamingStrategy = fileNamingStrategy;
ChunkDataSize = chunkDataSize;
ChunkDataUnitSize = chunkDataUnitSize;
ChunkDataCount = chunkDataCount;
FlushChunkIntervalMilliseconds = flushChunkIntervalMilliseconds;
EnableCache = enableCache;
SyncFlush = syncFlush;
ChunkReaderCount = chunkReaderCount;
MaxLogRecordSize = maxLogRecordSize;
ChunkWriteBuffer = chunkWriteBuffer;
ChunkReadBuffer = chunkReadBuffer;
ChunkCacheMaxPercent = chunkCacheMaxPercent;
ChunkCacheMinPercent = chunkCacheMinPercent;
PreCacheChunkCount = preCacheChunkCount;
ChunkInactiveTimeMaxSeconds = chunkInactiveTimeMaxSeconds;
ChunkLocalCacheSize = chunkLocalCacheSize;
EnableChunkWriteStatistic = enableChunkWriteStatistic;
EnableChunkReadStatistic = enableChunkReadStatistic;
if (GetChunkDataSize() > 1024 * 1024 * 1024)
{
throw new ArgumentException("Chunk data size cannot bigger than 1G");
}
}