private static JournalMetadata CreateMetadata <T>(EPartitionType configValue, string workingDir, params string[] symbols) { var jconf = new JournalElement { DefaultPath = workingDir, PartitionType = configValue }; if (symbols != null) { foreach (string sym in symbols) { var se = new SymbolElement { AvgSize = 20, HintDistinctCount = 100, MaxSize = 120, Name = sym }; jconf.Columns.Add(se); } } var meta = JournalBuilder.CreateNewJournalMetadata(jconf, typeof(T)); return(meta); }
public PartitionDate(DateTime date, int version, EPartitionType partitionType) : this() { PartitionType = partitionType; switch (PartitionType) { case EPartitionType.Day: Date = date.Date; break; case EPartitionType.Month: Date = new DateTime(date.Year, date.Month, 1); break; case EPartitionType.Year: Date = new DateTime(date.Year, 1, 1); break; default: Date = DateTime.MinValue; break; } Version = version; }
public JournalSettings(string defaultPath, string timestampColumn, string keySymbol, EPartitionType partitionType, int openPartitionTtl, int maxOpenPartitions, int lagHours, int recordHit = -1, int avgSize = -1 ) { _defaultPath = defaultPath; _timestampColumn = timestampColumn; _keySymbol = keySymbol; _partitionType = partitionType; _openPartitionTTL = openPartitionTtl; _maxOpenPartitions = maxOpenPartitions; _lagHours = lagHours; _recordHit = recordHit; if (_recordHit < 0) { _recordHit = MetadataConstants.DEFAULT_RECORD_HINT; } _avgSize = avgSize; if (_avgSize < 0) { _avgSize = MetadataConstants.AVG_RECORD_SIZE; } // Size of read chunks. Rounded to power of 2. _bitHint = (int)Math.Ceiling(Math.Log(checked (_avgSize * _recordHit), 2)); }
public string ShouldParseDirectoryNames(string directoryName, EPartitionType partitionType) { var date = PartitionManagerUtils.ParseDateFromDirName(directoryName, partitionType); return(date.Value.Date.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); }
private IJournal <PocoType> WriteJournal(EPartitionType type, TimeSpan increment, int days, AsyncJournalServer server = null) { Utils.ClearJournal <PocoType>(FOLDER_PATH); IJournal <PocoType> qj = OpenJournal(EFileAccess.ReadWrite, type, server); AppendRecords(qj, START_DATE, increment, days); return(qj); }
public PartitionDate(DateTime date, int version, EPartitionType partitionType, string name) : this() { PartitionType = partitionType; Date = date; Version = version; _name = name; }
public EPartitionType Should_parse_partition_type(EPartitionType configValue, string partitionString) { using (var tempDir = new DisposableTempDir()) { var meta = CreateMetadata <Quote>(configValue, tempDir.DirName); var pm = CreatePartitionManager(meta, partitionString); var jc = new JournalCore(meta, pm); return(meta.Settings.PartitionType); } }
private static IJournal <PocoType> OpenJournal(EFileAccess access, EPartitionType type, AsyncJournalServer server) { _directoryPath = Path.Combine(Utils.FindJournalsPath(), FOLDER_PATH); return(new JournalBuilder() .WithRecordCountHint((int)1E6) .WithPartitionBy(type) .WithLocation(_directoryPath) .WithSymbolColumn("Sym", 20, 5, 5) .WithTimestampColumn("Timestamp") .WithAccess(access) .WithSerializerFactoryName(MetadataConstants.POCO_SERIALIZER_NAME) .WithJournalServer(server) .ToJournal <PocoType>()); }
public string Should_save_value_in_partition_type_file(EPartitionType configValue, string partitionString) { using (var tempDir = new DisposableTempDir()) { var meta = CreateMetadata <Quote>(configValue, tempDir.DirName); var pm = CreatePartitionManager(meta, partitionString); var jc = new JournalCore(meta, pm); var defPath = meta.Settings.DefaultPath; var pfile = Path.Combine(defPath, MetadataConstants.PARTITION_TYPE_FILENAME); return(File.ReadAllText(pfile)); } }
public static PartitionDate?ParseDateFromDirName(string subDir, EPartitionType partitionType) { string dateString = subDir; int versionSepIndex = dateString.LastIndexOf(MetadataConstants.PARTITION_VERSION_SEPARATOR); int version = 0; if (versionSepIndex >= 0) { if (dateString.Length > versionSepIndex + 1) { string versionPart = dateString.Substring(versionSepIndex + 1); if (!int.TryParse(versionPart, out version)) { return(null); } } dateString = dateString.Substring(0, versionSepIndex); } switch (partitionType) { case EPartitionType.Day: break; case EPartitionType.Month: dateString += "-01"; break; case EPartitionType.Year: dateString += "-01-01"; break; default: if (MetadataConstants.DEFAULT_PARTITION_DIR .Equals(dateString, StringComparison.OrdinalIgnoreCase)) { return(new PartitionDate(DateTime.MinValue, version, partitionType, subDir)); } return(null); } DateTime date; if (!DateTime.TryParseExact(dateString, MetadataConstants.PARTITION_DATE_FORMAT, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out date)) { return(null); } return(new PartitionDate(date.Date, version, partitionType, subDir)); }
internal JournalSettings(JournalElement jconf, IEnumerable <ColumnMetadata> actualColumns) { _defaultPath = jconf.DefaultPath; _timestampColumn = jconf.TimestampColumn; _keySymbol = jconf.Key; _partitionType = jconf.PartitionType; _openPartitionTtl = jconf.OpenPartitionTtl; _maxOpenPartitions = jconf.MaxOpenPartitions; _lagHours = jconf.LagHours; _columns = actualColumns.ToArray(); _recordHint = jconf.RecordHint; if (_recordHint <= 0) { _recordHint = MetadataConstants.DEFAULT_RECORD_HINT; } }
public static string GetPartitionDirName(DateTime timestamp, EPartitionType partitionType) { switch (partitionType) { case EPartitionType.Day: return(timestamp.ToString("yyyy-MM-dd")); case EPartitionType.Month: return(timestamp.ToString("yyyy-MM")); case EPartitionType.Year: return(timestamp.ToString("yyyy")); default: return(MetadataConstants.DEFAULT_PARTITION_DIR); } }
private PartitionManager CreatePartitionManager <T>(EPartitionType pariPartitionType, DisposableTempDir dir, ICompositeFileFactory compositeFileFactory, EFileAccess access, string[] paritions, params string[] symbols) { CreateSubDirs(paritions, dir.DirName); JournalMetadata meta = CreateMetadata <T>(pariPartitionType, dir.DirName, symbols); var txLog = new Mock <ITxLog>(); txLog.Setup(s => s.Get()).Returns(new TxRec { JournalMaxRowID = RowIDUtil.ToRowID(1, 10) }); var part = new PartitionManager(meta, access, compositeFileFactory, new AsyncJournalServer(TimeSpan.FromSeconds(1)), txLog.Object); return(part); }
public static DateTime GetPartitionEndDate(DateTime startDate, EPartitionType partitionType) { DebugCheckDate(startDate); switch (partitionType) { case EPartitionType.Day: return(startDate.AddDays(1)); case EPartitionType.Month: return(startDate.AddMonths(1)); case EPartitionType.Year: return(startDate.AddYears(1)); default: return(DateTime.MaxValue); } }
public static DateTime GetPartitionStartDate(DateTime timestamp, EPartitionType partitionType) { switch (partitionType) { case EPartitionType.Day: return(timestamp.Date); case EPartitionType.Month: return(new DateTime(timestamp.Year, timestamp.Month, 1)); case EPartitionType.Year: return(new DateTime(timestamp.Year, 1, 1)); ; default: return(DateTime.MinValue); } }
public void OverridePartitionType(EPartitionType value) { _partitionType = value; }
private void WritePartitionType(EPartitionType value) { var path = Path.Combine(_settings.DefaultPath, MetadataConstants.PARTITION_TYPE_FILENAME); File.WriteAllText(path, value.ToString().ToUpper()); }
public JournalBuilder WithPartitionBy(EPartitionType partitionType) { _config.PartitionType = partitionType; return(this); }