public static EntityKey GetDateEntityKey(DateTimeOffset buildDate, BuildId buildId, string identifier)
{
identifier = AzureUtil.NormalizeKey(identifier, '_');
return(new EntityKey(
DateTimeKey.GetDateKey(buildDate),
$"{new BuildKey(buildId).Key}-{identifier}"));
}
public BuildStateMessage(DateTimeKey buildStateKey, BoundBuildId buildId)
{
BuildStateKeyRaw = buildStateKey.Key;
HostRaw = buildId.Host.ToString();
JobName = buildId.JobName;
BuildNumber = buildId.Number;
}
private static async Task TestFailureYesterday(int days = -1)
{
var account = GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
var table = tableClient.GetTableReference(TableNames.BuildState);
var date = DateTimeOffset.UtcNow.AddDays(days);
var query = TableQueryUtil.And(
TableQueryUtil.PartitionKey(DateTimeKey.GetDateKey(date)),
TableQueryUtil.Column(nameof(BuildStateEntity.IsBuildFinished), true),
TableQueryUtil.Column(nameof(BuildStateEntity.IsDataComplete), false));
var list = await AzureUtil.QueryAsync <BuildStateEntity>(table, query);
foreach (var entity in list)
{
var populator = new BuildTablePopulator(tableClient, CounterUtilFactory, CreateClient(entity.BoundBuildId), TextWriter.Null);
try
{
Console.Write($"{entity.BuildId} ... ");
await populator.PopulateBuild(entity.BoundBuildId);
Console.WriteLine("good");
}
catch (Exception ex)
{
Console.WriteLine("ERRROR");
Console.WriteLine(ex);
}
}
}
public BuildStateMessage(DateTimeKey buildStateKey, BoundBuildId buildId)
{
BuildStateKeyRaw = buildStateKey.Key;
HostRaw = buildId.Host.ToString();
JobName = buildId.JobName;
BuildNumber = buildId.Number;
}
private static async Task MigrateDateKeyCore <T>(string tableName)
where T : ITableEntity, new()
{
Console.WriteLine($"Processing {tableName}");
var account = GetStorageAccount();
var table = account.CreateCloudTableClient().GetTableReference(tableName);
var startKey = new DateKey(DateKey.StartDate);
var endKey = new DateKey(DateTimeOffset.UtcNow);
var query = TableQueryUtil.And(
TableQueryUtil.PartitionKey(startKey.Key, ColumnOperator.GreaterThanOrEqual),
TableQueryUtil.PartitionKey(endKey.Key, ColumnOperator.LessThanOrEqual));
var list = await AzureUtil.QueryAsync <T>(table, query);
Console.WriteLine($"Processing {list.Count} entities");
var deleteList = new List <EntityKey>();
foreach (var entity in list)
{
deleteList.Add(entity.GetEntityKey());
var dateKey = DateKey.Parse(entity.PartitionKey);
var dateTimeKey = new DateTimeKey(dateKey.Date, DateTimeKeyFlags.Date);
entity.PartitionKey = dateTimeKey.Key;
}
Console.WriteLine("Writing new values");
await AzureUtil.InsertBatchUnordered(table, list);
Console.WriteLine("Deleting old values");
await AzureUtil.DeleteBatchUnordered(table, deleteList);
}
public void DateTimeSimple()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00AM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160915T0100", key.Key);
}
public void DateTimeWidth()
{
var date = DateTimeOffset.Parse("2016/1/2 1:01PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160102T1301", key.Key);
}
public void DateTimeMilitary()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160915T1300", key.Key);
}
/// <summary>
/// Get or create the build state partition key for the build id.
/// </summary>
/// <remarks>
/// This has to take into account that builds take place across days. Which day wins doesn't matter
/// so long as we don't duplicate the data.
/// </remarks>
internal async Task<DateTimeKey> GetOrCreateBuildStateKey(BoundBuildId buildId)
{
// TODO: do this correctly
var key = new DateTimeKey(DateTimeOffset.UtcNow, DateTimeKeyFlags.Date);
var task = new Task<DateTimeKey>(() => key);
task.Start();
return await task;
}
public BuildStateEntity(DateTimeKey key, BoundBuildId buildId, bool isBuildFinished)
{
PartitionKey = key.Key;
RowKey = GetRowKey(buildId);
HostRaw = buildId.Host.ToString();
BuildNumber = buildId.Number;
JobName = buildId.JobName;
IsBuildFinished = isBuildFinished;
}
public BuildStateEntity(DateTimeKey key, BoundBuildId buildId, bool isBuildFinished)
{
PartitionKey = key.Key;
RowKey = GetRowKey(buildId);
HostRaw = buildId.Host.ToString();
BuildNumber = buildId.Number;
JobName = buildId.JobName;
IsBuildFinished = isBuildFinished;
}
/// <summary>
/// Get or create the build state partition key for the build id.
/// </summary>
/// <remarks>
/// This has to take into account that builds take place across days. Which day wins doesn't matter
/// so long as we don't duplicate the data.
/// </remarks>
internal async Task <DateTimeKey> GetOrCreateBuildStateKey(BoundBuildId buildId)
{
// TODO: do this correctly
var key = new DateTimeKey(DateTimeOffset.UtcNow, DateTimeKeyFlags.Date);
var task = new Task <DateTimeKey>(() => key);
task.Start();
return(await task);
}
public void DateComparisonYears()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.Date);
for (var i = 1; i < 1000; i++)
{
var newKey = new DateTimeKey(date.AddYears(i), key.Flags);
Assert.True(string.CompareOrdinal(newKey.Key, key.Key) > 0);
}
}
public void DateComparisonDays()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.Date);
for (var i = 1; i < 1000; i++)
{
var newKey = new DateTimeKey(date.AddDays(i), key.Flags);
Assert.True(string.CompareOrdinal(newKey.Key, key.Key) > 0);
}
}
public void Create_Throws_If_Key_Type_Not_Supported()
{
InMemoryDb db = new InMemoryDb();
InMemoryDbTable <DateTime, DateTimeKey> table = db.GetTable <DateTime, DateTimeKey>();
Repository <DateTime, DateTimeKey> repo = new Repository <DateTime, DateTimeKey>(table);
DateTimeKey entity = new DateTimeKey();
// create
repo.Add(entity);
}
public List <string> GetViewNames(DateTimeOffset startDate)
{
var key = DateTimeKey.GetDateKey(startDate);
var filter = TableQueryUtil.PartitionKey(key, ColumnOperator.GreaterThanOrEqual);
var query = new TableQuery <ViewNameEntity>().Where(filter);
var viewNameList = _viewNameDateTable.ExecuteQuery(query);
var list = new List <string>();
list.Add(AzureUtil.ViewNameAll);
list.AddRange(viewNameList.Select(x => x.ViewName).Distinct());
return(list);
}
public async Task QuerySimple()
{
var date = DateTimeOffset.Parse("2016/01/02");
var key = DateTimeKey.GetDateKey(date);
var entity = new TestEntity()
{
PartitionKey = key,
RowKey = Guid.NewGuid().ToString("N"),
Count = 42
};
await Table.ExecuteAsync(TableOperation.Insert(entity));
var list = await CounterUtil.QueryAsync(date, date);
Assert.Equal(1, list.Count);
Assert.Equal(42, list[0].Count);
}
private static async Task EnqueueCore(CloudQueue queue, DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan?delay, CancellationToken cancellationToken)
{
// Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that
// we don't unnecessarily ask Jenkins for information.
var buildMessage = new BuildStateMessage()
{
BuildStateKeyRaw = buildStateKey.Key,
BuildNumber = buildId.Number,
HostRaw = buildId.Host.ToString(),
JobName = buildId.JobName
};
var queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(buildMessage));
await queue.AddMessageAsync(
queueMessage,
timeToLive : null,
initialVisibilityDelay : delay,
options : null,
operationContext : null,
cancellationToken : cancellationToken);
}
private async Task EnqueueProcessBuild(DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan?delay, CancellationToken cancellationToken)
{
await EnqueueCore(_processBuildQueue, buildStateKey, buildId, delay, cancellationToken);
}
public static EntityKey GetEntityKey(DateTimeKey key, BoundBuildId buildId) => new EntityKey(key.Key, GetRowKey(buildId));
private async Task EnqueueEmailBuild(DateTimeKey buildStateKey, BoundBuildId buildId, CancellationToken cancellationToken)
{
await EnqueueCore(_emailBuildQueue, buildStateKey, buildId, null, cancellationToken);
}
public static string GetPartitionKey(DateTimeOffset dateTime) => DateTimeKey.GetKey(dateTime, Flags);
public static string GetPartitionKey(DateTimeKey key) => key.Key;
public void Time()
{
var dateTime = DateTimeKey.ParseDateTime("20160102T0102", DateTimeKeyFlags.DateTime);
Assert.Equal(DateTimeOffset.Parse("2016/01/02 1:02AM"), dateTime);
}
private static string FilterSinceDate(DateTimeOffset startDate)
{
var key = DateTimeKey.GetDateKey(startDate);
return(TableQueryUtil.Column(ColumnName.PartitionKey, key, ColumnOperator.GreaterThanOrEqual));
}
public static string GetPartitionKey(DateTimeKey key) => key.Key;
public static EntityKey GetEntityKey(DateTimeKey key, BoundBuildId buildId) => new EntityKey(key.Key, GetRowKey(buildId));
public void DateTimeSimple()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00AM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160915T0100", key.Key);
}
public void DateTimeWidth()
{
var date = DateTimeOffset.Parse("2016/1/2 1:01PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160102T1301", key.Key);
}
public void DateTimeMilitary()
{
var date = DateTimeOffset.Parse("2016/09/15 1:00PM");
var key = new DateTimeKey(date, DateTimeKeyFlags.DateTime);
Assert.Equal("20160915T1300", key.Key);
}
private async Task EnqueueProcessBuild(DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan? delay, CancellationToken cancellationToken)
{
await EnqueueCore(_processBuildQueue, buildStateKey, buildId, delay, cancellationToken);
}
private static async Task EnqueueCore(CloudQueue queue, DateTimeKey buildStateKey, BoundBuildId buildId, TimeSpan? delay, CancellationToken cancellationToken)
{
// Enqueue a message to process the build. Insert a delay if the build isn't finished yet so that
// we don't unnecessarily ask Jenkins for information.
var buildMessage = new BuildStateMessage()
{
BuildStateKeyRaw = buildStateKey.Key,
BuildNumber = buildId.Number,
HostRaw = buildId.Host.ToString(),
JobName = buildId.JobName
};
var queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(buildMessage));
await queue.AddMessageAsync(
queueMessage,
timeToLive: null,
initialVisibilityDelay: delay,
options: null,
operationContext: null,
cancellationToken: cancellationToken);
}
public static EntityKey GetDateEntityKey(DateTimeOffset buildDate, BuildId buildId)
{
return(new EntityKey(
DateTimeKey.GetDateKey(buildDate),
new BuildKey(buildId).Key));
}
private async Task EnqueueEmailBuild(DateTimeKey buildStateKey, BoundBuildId buildId, CancellationToken cancellationToken)
{
await EnqueueCore(_emailBuildQueue, buildStateKey, buildId, null, cancellationToken);
}
