private TsdbTableEntity Convert(BinaryWriter writer, MemoryStream stream, TEntry entry, string partitionKey)
{
var entity = new TsdbTableEntity();
entity.RowKey = AtsKeyCalculator.CalculateRowKey(entry.GetTimestamp());
entity.PartitionKey = partitionKey;
AtsSerializer.SerializeEntry <TKey, TEntry>(writer, entry);
entity.P0 = stream.ToArray();
return(entity);
}
private string CreatePartitionFilter(TKey key)
{
var id = _keyConverter.Convert(key);
var fromPartitionKey = AtsKeyCalculator.CalculateMaxPartitionKey(id, key, _partitioningProvider); // 0000
var toPartitionKey = AtsKeyCalculator.CalculateMinPartitionKey(id, key, _partitioningProvider); // 9999
return(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, fromPartitionKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, toPartitionKey)));
}
private string CreateSpecificPartitionFilter(TKey key, DateTime from, DateTime to, string partitionKeyRange)
{
var id = _keyConverter.Convert(key);
var fromRowKey = AtsKeyCalculator.CalculateRowKey(from);
var toRowKey = AtsKeyCalculator.CalculateRowKey(to);
var partitionKey = AtsKeyCalculator.CalculatePartitionKey(id, partitionKeyRange);
return(TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, fromRowKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, toRowKey))));
}
private IEnumerable <KeyValuePair <AtsTablePartition, List <TEntry> > > IterateByPartition(IEnumerable <ISerie <TKey, TEntry> > series)
{
Dictionary <AtsTablePartition, List <TEntry> > lookup = new Dictionary <AtsTablePartition, List <TEntry> >();
var hashkeys = new HashSet <EntryKey <TKey> >();
foreach (var serie in series)
{
var key = serie.GetKey();
var id = _keyConverter.Convert(key);
foreach (var entry in serie.GetEntries())
{
var timestamp = entry.GetTimestamp();
var hashkey = new EntryKey <TKey>(key, timestamp);
if (!hashkeys.Contains(hashkey))
{
var table = _tableProvider.GetTable(timestamp);
var pk = AtsKeyCalculator.CalculatePartitionKey(id, key, timestamp, _partitioningProvider);
var tpk = new AtsTablePartition(table, pk);
List <TEntry> items;
if (!lookup.TryGetValue(tpk, out items))
{
items = new List <TEntry>();
lookup.Add(tpk, items);
}
items.Add(entry);
if (items.Count == 100)
{
lookup.Remove(tpk);
yield return(new KeyValuePair <AtsTablePartition, List <TEntry> >(tpk, items));
}
hashkeys.Add(hashkey);
}
}
}
foreach (var kvp in lookup)
{
yield return(kvp);
}
}
private string CreateGeneralFilter(TKey key, DateTime from, DateTime to)
{
var id = _keyConverter.Convert(key);
var fromRowKey = AtsKeyCalculator.CalculateRowKey(from);
var toRowKey = AtsKeyCalculator.CalculateRowKey(to);
var fromPartitionKey = AtsKeyCalculator.CalculatePartitionKey(id, key, from, _partitioningProvider);
var toPartitionKey = AtsKeyCalculator.CalculatePartitionKey(id, key, to.AddTicks(-1), _partitioningProvider); // -1 tick because it is an approximation value and we use gte operation
return(TableQuery.CombineFilters(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThanOrEqual, fromPartitionKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, toPartitionKey)),
TableOperators.And,
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, fromRowKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, toRowKey))));
}
private List <TsdbTableEntity> CreateTableEntitiesFor(TKey key, List <TEntry> entries)
{
List <TsdbTableEntity> tableEntities = new List <TsdbTableEntity>();
var id = _keyConverter.Convert(key);
var results = AtsSerializer.Serialize <TKey, TEntry>(entries, TsdbTableEntity.MaxByteCapacity);
foreach (var result in results)
{
var entity = new TsdbTableEntity();
entity.SetData(result.Data);
entity.RowKey = AtsKeyCalculator.CalculateRowKey(result.From);
entity.PartitionKey = AtsKeyCalculator.CalculatePartitionKey(id, key, result.From, _partitioningProvider);
tableEntities.Add(entity);
}
return(tableEntities);
}
private IEnumerable <EntrySplitResult <TKey, TEntry> > IterateByPartition(IEnumerable <ISerie <TKey, TEntry> > series)
{
Dictionary <string, EntrySplitResult <TKey, TEntry> > lookup = new Dictionary <string, EntrySplitResult <TKey, TEntry> >();
var hashkeys = new HashSet <EntryKey <TKey> >();
foreach (var serie in series)
{
var key = serie.GetKey();
var id = _keyConverter.Convert(key);
foreach (var entry in serie.GetEntries())
{
var timestamp = entry.GetTimestamp();
var hashkey = new EntryKey <TKey>(key, timestamp);
if (!hashkeys.Contains(hashkey))
{
var pk = AtsKeyCalculator.CalculatePartitionKey(id, key, timestamp, _partitioningProvider);
EntrySplitResult <TKey, TEntry> items;
if (!lookup.TryGetValue(pk, out items))
{
items = new EntrySplitResult <TKey, TEntry>(key, pk);
lookup.Add(pk, items);
}
items.Insert(entry);
hashkeys.Add(hashkey);
}
}
}
foreach (var result in lookup)
{
result.Value.Sort(Sort.Descending);
yield return(result.Value);
}
}
