protected override async Task OnPublished(IEnumerable <ISerie <TKey, TEntry> > series, PublicationType publish)
{
await WaitWhileDisconnectedAsync().ConfigureAwait(false);
var tasks = new List <Task>();
if (publish.HasFlag(PublicationType.LatestPerCollection))
{
var latest = FindLatestForEachId(series);
foreach (var serie in latest)
{
if (serie.Entries.Count > 0)
{
var key = serie.GetKey();
var entry = serie.GetEntries().First();
tasks.Add(_connection.PublishLatestAsync <TKey, TEntry>(_keyConverter.Convert(key), entry));
}
}
}
if (publish.HasFlag(PublicationType.AllFromCollections))
{
foreach (var serie in series)
{
if (serie.GetEntries().Count > 0)
{
var key = serie.GetKey();
tasks.Add(_connection.PublishAllAsync <TKey, TEntry>(_keyConverter.Convert(key), serie.GetEntries()));
}
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
private async Task StoreForAll(IEnumerable <ISerie <TKey, TEntry> > series)
{
await CreateTable().ConfigureAwait(false);
using (await _cc.WriteAsync().ConfigureAwait(false))
{
using (var connection = new SqlConnection(_connectionString))
{
await connection.OpenAsync().ConfigureAwait(false);
var keys = new HashSet <EntryKey <TKey> >();
List <SqlDataRecord> records = new List <SqlDataRecord>();
foreach (var serie in series)
{
var key = serie.GetKey();
SqlSerializer.Serialize <TKey, TEntry>(serie.GetEntries(), (entry, data) =>
{
var timestamp = entry.GetTimestamp();
var hashkey = new EntryKey <TKey>(key, timestamp);
if (!keys.Contains(hashkey))
{
SqlDataRecord record = new SqlDataRecord(Sql.InsertParameterMetadata);
record.SetString(0, _keyConverter.Convert(key));
record.SetDateTime(1, timestamp);
record.SetSqlBinary(2, new SqlBinary(data));
records.Add(record);
keys.Add(hashkey);
}
});
}
if (records.Count > 0)
{
using (var tx = connection.BeginTransaction(IsolationLevel.ReadCommitted))
{
using (var command = connection.CreateCommand())
{
command.CommandText = Sql.GetInsertProcedureName(_tableName);
command.CommandType = CommandType.StoredProcedure;
command.Transaction = tx;
var parameter = command.Parameters.AddWithValue("@Inserts", records);
parameter.SqlDbType = SqlDbType.Structured;
parameter.TypeName = Sql.GetInsertParameterType(_tableName);
await command.ExecuteNonQueryAsync().ConfigureAwait(false);
}
tx.Commit();
}
}
}
}
}
private List <TsdbTableEntity> CreateTableEntitiesFor(TKey key, List <TEntry> entries)
{
List <TsdbTableEntity> tableEntities = new List <TsdbTableEntity>();
var id = _keyConverter.Convert(key);
var results = AtsSerializer.Serialize <TKey, TEntry>(entries, TsdbTableEntity.MaxByteCapacity);
foreach (var result in results)
{
var entity = new TsdbTableEntity();
entity.SetData(result.Data);
entity.RowKey = AtsKeyCalculator.CalculateRowKey(result.From);
entity.PartitionKey = AtsKeyCalculator.CalculatePartitionKey(id, key, result.From, _partitioningProvider);
tableEntities.Add(entity);
}
return(tableEntities);
}
private IEnumerable <KeyValuePair <AtsTablePartition, List <TEntry> > > IterateByPartition(IEnumerable <ISerie <TKey, TEntry> > series)
{
Dictionary <AtsTablePartition, List <TEntry> > lookup = new Dictionary <AtsTablePartition, List <TEntry> >();
var hashkeys = new HashSet <EntryKey <TKey> >();
foreach (var serie in series)
{
var key = serie.GetKey();
var id = _keyConverter.Convert(key);
foreach (var entry in serie.GetEntries())
{
var timestamp = entry.GetTimestamp();
var hashkey = new EntryKey <TKey>(key, timestamp);
if (!hashkeys.Contains(hashkey))
{
var table = _tableProvider.GetTable(timestamp);
var pk = AtsKeyCalculator.CalculatePartitionKey(id, key, timestamp, _partitioningProvider);
var tpk = new AtsTablePartition(table, pk);
List <TEntry> items;
if (!lookup.TryGetValue(tpk, out items))
{
items = new List <TEntry>();
lookup.Add(tpk, items);
}
items.Add(entry);
if (items.Count == 100)
{
lookup.Remove(tpk);
yield return(new KeyValuePair <AtsTablePartition, List <TEntry> >(tpk, items));
}
hashkeys.Add(hashkey);
}
}
}
foreach (var kvp in lookup)
{
yield return(kvp);
}
}
public Task WriteAsync(IEnumerable <ISerie <TKey, TEntry> > series)
{
_sem.Wait();
try
{
if (_currentSize > _maxStorageSize)
{
throw new TsdbException("Could not write any of the entries to the temporary file storage because it exceeds the maximum storage size.");
}
// Open the current or a new file
FileInfo fileInfo = GetOrCreateCurrentFile();
FileStream fileStream = fileInfo.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite);
long startFileSize = fileStream.Length;
// Seek to the end of the file
fileStream.Seek(0, SeekOrigin.End);
// Create the writer
BinaryWriter writer = new BinaryWriter(fileStream, Encoding.ASCII, true);
bool disposed = false;
long beforeLength = startFileSize;
long afterLength = beforeLength;
foreach (var serie in series)
{
var key = serie.GetKey();
var id = _keyConverter.Convert(key);
foreach (var entry in serie.GetEntries())
{
beforeLength = afterLength;
if (beforeLength > _maxFileSize)
{
// flush to file
writer.Flush();
// increment storage size
_currentSize += (beforeLength - startFileSize);
// close the current file stream
writer.Dispose();
fileStream.Dispose();
// create a new file reference...
fileInfo = CreateCurrentFile();
fileStream = fileInfo.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite);
writer = new BinaryWriter(fileStream, Encoding.ASCII, true);
startFileSize = fileStream.Length;
beforeLength = startFileSize;
afterLength = beforeLength;
}
long storageSize = _currentSize + (beforeLength - startFileSize);
if (storageSize > _maxStorageSize)
{
// flush to file
writer.Flush();
// increment storage size
_currentSize += (beforeLength - startFileSize);
// close the current file stream
writer.Dispose();
fileStream.Dispose();
disposed = true;
// throw exception indicating which entries could not be inserted (with skip)
throw new TsdbException("Could not write all the entries to the temporary file storage because it exceeds the maximum storage size.");
}
writer.Write(id);
writer.Write(entry.GetTimestamp().Ticks);
entry.Write(writer);
afterLength = fileStream.Length;
}
}
writer.Flush();
writer.Dispose();
fileStream.Dispose();
}
finally
{
_sem.Release();
}
return(_completed);
}
private IEnumerable <InfluxEntryAdapter <TEntry> > Convert(IEnumerable <ISerie <TKey, TEntry> > series)
{
var keys = new HashSet <EntryKey <TKey> >();
foreach (var serie in series)
{
var key = serie.GetKey();
var id = _keyConverter.Convert(key);
foreach (var entry in serie.GetEntries())
{
var hashkey = new EntryKey <TKey>(key, ((IEntry)entry).GetTimestamp());
if (!keys.Contains(hashkey))
{
yield return(new InfluxEntryAdapter <TEntry>(id, entry));
keys.Add(hashkey);
}
}
}
}
private string CreateDefaultTagFilter(TKey key)
{
return($"\"{ReservedNames.UniqueId}\" = '{_keyConverter.Convert( key )}'");
}