public async Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived) { //var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString); CloudStorageAccount account = null; if (!String.IsNullOrWhiteSpace(shardKeyArrived.Source.AccountSasKey)) { // Create new storage credentials using the SAS token. var accountSas = new StorageCredentials(shardKeyArrived.Source.AccountSasKey); // Use these credentials and the account name to create a Blob service client. try { account = new CloudStorageAccount(accountSas, shardKeyArrived.Source.AccountName, endpointSuffix: "", useHttps: true); } catch (Exception ex) { TheTrace.TraceError(ex.ToString()); } } else { account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString); } var client = account.CreateCloudTableClient(); var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString()); return(await table.ExecuteQueryAsync(new TableQuery().Where(TableQuery.GenerateFilterCondition("PartitionKey", "eq", shardKeyArrived.ShardKey))).ConfigureAwait(false)); }
public void DateTimeOffsetForModulo10MinuteShard() { var shardKeyArrived = new ShardKeyArrived() { ShardKey = "0000000000000000006___0636403734870000000" }; Assert.Equal("201709070931", shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm")); }
public void DateTimeOffset_GetsReturnedCorrectly() { var shardKeyArrived = new ShardKeyArrived() { ShardKey = "0635901169200000000" }; Assert.Equal("201602031722", shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm")); }
public Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived) { var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString); var client = account.CreateCloudTableClient(); var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString()); return(Task.FromResult(table.ExecuteQuery(new TableQuery().Where( TableQuery.GenerateFilterCondition("PartitionKey", "eq", shardKeyArrived.ShardKey))))); }
public static DateTimeOffset GetDateTimeOffset(this ShardKeyArrived shardKeyArrived) { long shardKey; var shardKeyArr = shardKeyArrived.ShardKey.Split('_'); if (shardKeyArr.Length == 4) { long.TryParse(shardKeyArr[3], out shardKey); } else { long.TryParse(shardKeyArrived.ShardKey, out shardKey); } return(new DateTimeOffset(new DateTime(shardKey), TimeSpan.Zero)); }
private IEnumerable <DynamicTableEntity> PreprocessEntities(IEnumerable <DynamicTableEntity> entities, ShardKeyArrived shardKeyArrived, string shardKeyTime) { var minDateTime = DateTimeOffset.MaxValue; var n = 0; foreach (var entity in entities) { var eventDateTimeOffset = entity.GetEventDateTimeOffset(); var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds; if (delayInSeconds >= _shardKeyDelayWarning) { TheTrace.TraceWarning( "SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}", delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime); } entity.Timestamp = eventDateTimeOffset; yield return(entity); minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime; n++; } TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime); if (n > 0) { _telemetryProvider.WriteTelemetry( "ShardKeyArrivedActor log delay duration", (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds, shardKeyArrived.Source.TypeName); } }
public static DateTimeOffset GetDateTimeOffset(this ShardKeyArrived shardKeyArrived) { return(new DateTimeOffset(new DateTime(Convert.ToInt64(shardKeyArrived.ShardKey)), TimeSpan.Zero)); }
public async Task <IEnumerable <DynamicTableEntity> > QueryAsync(ShardKeyArrived shardKeyArrived) { var shard = shardKeyArrived.ShardKey; // this is ticks of that minute var thatMinute = new DateTime(long.Parse(shard)); var tableName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.TableName); var shardFieldName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardFieldName); var idFieldName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.IdFieldName); var timestampFieldName = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.TimestampFieldName); var entities = new List <DynamicTableEntity>(); var connection = new SqlConnection(shardKeyArrived.Source.ConnectionString); connection.Open(); var fieldIndices = new Dictionary <string, int>(); using (connection) { var sql = string.Format("SELECT * FROM {0} WHERE {1} = '{2}'", tableName, shardFieldName, thatMinute); TheTrace.TraceInformation("SQL is {0}", sql); var reader = await connection.ExecuteReaderAsync(sql); for (int i = 0; i < reader.FieldCount; i++) { fieldIndices.Add(reader.GetName(i), i); } int count = 0; while (reader.Read()) { var data = new Dictionary <string, EntityProperty>(); for (int i = 0; i < reader.FieldCount; i++) { data.Add(reader.GetName(i), EntityProperty.CreateEntityPropertyFromObject(reader[i])); } foreach (var kv in data) { TheTrace.TraceInformation("Key=>'{0}' value=>'{1}'", kv.Key, kv.Value); } TheTrace.TraceInformation("timestampFieldName => '{0}'", data[timestampFieldName].ToString()); var entity = new DynamicTableEntity(data[shardFieldName].DateTimeOffsetValue.Value.ToString("yyyyMMddHHmmss"), data[idFieldName].GuidValue.ToString()); entity.Timestamp = data[timestampFieldName].DateTimeOffsetValue.Value; for (int i = 0; i < reader.FieldCount; i++) { entity.Properties.Add(reader.GetName(i), data[reader.GetName(i)]); } entities.Add(entity); count++; } TheTrace.TraceInformation("Added this many row: " + count); } return(entities); }