public async Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived)
{
//var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
CloudStorageAccount account = null;
if (!String.IsNullOrWhiteSpace(shardKeyArrived.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(shardKeyArrived.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
try
{
account = new CloudStorageAccount(accountSas, shardKeyArrived.Source.AccountName, endpointSuffix: "", useHttps: true);
}
catch (Exception ex)
{
TheTrace.TraceError(ex.ToString());
}
}
else
{
account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
}
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
return(await table.ExecuteQueryAsync(new TableQuery().Where(TableQuery.GenerateFilterCondition("PartitionKey", "eq", shardKeyArrived.ShardKey))).ConfigureAwait(false));
}
public void DateTimeOffsetForModulo10MinuteShard()
{
var shardKeyArrived = new ShardKeyArrived()
{
ShardKey = "0000000000000000006___0636403734870000000"
};
Assert.Equal("201709070931", shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm"));
}
public void DateTimeOffset_GetsReturnedCorrectly()
{
var shardKeyArrived = new ShardKeyArrived()
{
ShardKey = "0635901169200000000"
};
Assert.Equal("201602031722", shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm"));
}
public Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived)
{
var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
return(Task.FromResult(table.ExecuteQuery(new TableQuery().Where(
TableQuery.GenerateFilterCondition("PartitionKey", "eq", shardKeyArrived.ShardKey)))));
}
public static DateTimeOffset GetDateTimeOffset(this ShardKeyArrived shardKeyArrived)
{
long shardKey;
var shardKeyArr = shardKeyArrived.ShardKey.Split('_');
if (shardKeyArr.Length == 4)
{
long.TryParse(shardKeyArr[3], out shardKey);
}
else
{
long.TryParse(shardKeyArrived.ShardKey, out shardKey);
}
return(new DateTimeOffset(new DateTime(shardKey), TimeSpan.Zero));
}
private IEnumerable <DynamicTableEntity> PreprocessEntities(IEnumerable <DynamicTableEntity> entities, ShardKeyArrived shardKeyArrived, string shardKeyTime)
{
var minDateTime = DateTimeOffset.MaxValue;
var n = 0;
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning(
"SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
yield return(entity);
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (n > 0)
{
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
}
public static DateTimeOffset GetDateTimeOffset(this ShardKeyArrived shardKeyArrived)
{
return(new DateTimeOffset(new DateTime(Convert.ToInt64(shardKeyArrived.ShardKey)), TimeSpan.Zero));
}
public async Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived)
{
var shard = shardKeyArrived.ShardKey; // this is ticks of that minute
var thatMinute = new DateTime(long.Parse(shard));
var tableName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.TableName);
var shardFieldName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardFieldName);
var idFieldName =
(string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.IdFieldName);
var timestampFieldName =
(string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.TimestampFieldName);
var entities = new List <DynamicTableEntity>();
var connection = new SqlConnection(shardKeyArrived.Source.ConnectionString);
connection.Open();
var fieldIndices = new Dictionary <string, int>();
using (connection)
{
var sql = string.Format("SELECT * FROM {0} WHERE {1} = '{2}'", tableName, shardFieldName, thatMinute);
TheTrace.TraceInformation("SQL is {0}", sql);
var reader = await connection.ExecuteReaderAsync(sql);
for (int i = 0; i < reader.FieldCount; i++)
{
fieldIndices.Add(reader.GetName(i), i);
}
int count = 0;
while (reader.Read())
{
var data = new Dictionary <string, EntityProperty>();
for (int i = 0; i < reader.FieldCount; i++)
{
data.Add(reader.GetName(i), EntityProperty.CreateEntityPropertyFromObject(reader[i]));
}
foreach (var kv in data)
{
TheTrace.TraceInformation("Key=>'{0}' value=>'{1}'", kv.Key, kv.Value);
}
TheTrace.TraceInformation("timestampFieldName => '{0}'", data[timestampFieldName].ToString());
var entity = new DynamicTableEntity(data[shardFieldName].DateTimeOffsetValue.Value.ToString("yyyyMMddHHmmss"), data[idFieldName].GuidValue.ToString());
entity.Timestamp = data[timestampFieldName].DateTimeOffsetValue.Value;
for (int i = 0; i < reader.FieldCount; i++)
{
entity.Properties.Add(reader.GetName(i), data[reader.GetName(i)]);
}
entities.Add(entity);
count++;
}
TheTrace.TraceInformation("Added this many row: " + count);
}
return(entities);
}
