public static EntityKey GetDateEntityKey(DateTimeOffset buildDate, BuildId buildId, string identifier)
{
identifier = AzureUtil.NormalizeKey(identifier, '_');
return(new EntityKey(
DateTimeKey.GetDateKey(buildDate),
$"{new BuildKey(buildId).Key}-{identifier}"));
}
private static async Task TestFailureYesterday(int days = -1)
{
var account = GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
var table = tableClient.GetTableReference(TableNames.BuildState);
var date = DateTimeOffset.UtcNow.AddDays(days);
var query = TableQueryUtil.And(
TableQueryUtil.PartitionKey(DateTimeKey.GetDateKey(date)),
TableQueryUtil.Column(nameof(BuildStateEntity.IsBuildFinished), true),
TableQueryUtil.Column(nameof(BuildStateEntity.IsDataComplete), false));
var list = await AzureUtil.QueryAsync <BuildStateEntity>(table, query);
foreach (var entity in list)
{
var populator = new BuildTablePopulator(tableClient, CounterUtilFactory, CreateClient(entity.BoundBuildId), TextWriter.Null);
try
{
Console.Write($"{entity.BuildId} ... ");
await populator.PopulateBuild(entity.BoundBuildId);
Console.WriteLine("good");
}
catch (Exception ex)
{
Console.WriteLine("ERRROR");
Console.WriteLine(ex);
}
}
}
public async Task <ActionResult> Status(bool all = false, bool error = false, DateTimeOffset?startDate = null)
{
var startDateValue = startDate ?? DateTimeOffset.UtcNow;
var key = BuildStateEntity.GetPartitionKey(startDateValue);
var table = _storageAccount.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.BuildState);
var filter = TableQueryUtil.Column(nameof(TableEntity.PartitionKey), key, ColumnOperator.GreaterThanOrEqual);
if (error)
{
filter = TableQueryUtil.And(
filter,
TableQueryUtil.Column(nameof(BuildStateEntity.Error), (string)null, ColumnOperator.NotEqual));
}
else if (!all)
{
filter = TableQueryUtil.And(
filter,
TableQueryUtil.Column(nameof(BuildStateEntity.IsDataComplete), false));
}
var list = await AzureUtil.QueryAsync <BuildStateEntity>(table, filter);
var model = new BuildStatusModel(all, error, startDateValue, list);
return(View(viewName: "Status", model: model));
}
private static async Task MigrateCounter()
{
var account = GetStorageAccount();
var tableClient = account.CreateCloudTableClient();
var tableNames = new[]
{
AzureConstants.TableNames.TestCacheCounter,
AzureConstants.TableNames.TestRunCounter,
AzureConstants.TableNames.UnitTestQueryCounter
};
foreach (var tableName in tableNames)
{
var table = tableClient.GetTableReference(tableName);
var query = new TableQuery <DynamicTableEntity>().Select(new[] { "PartitionKey", "RowKey" });
var list = new List <DynamicTableEntity>();
foreach (var entity in table.ExecuteQuery(query))
{
DateTime dateTime;
if (!DateTime.TryParseExact(entity.PartitionKey, "yyyy-MM-dd", CultureInfo.CurrentCulture, DateTimeStyles.None, out dateTime))
{
continue;
}
list.Add(entity);
}
await AzureUtil.DeleteBatchUnordered(table, list);
}
}
public BuildResultEntity(
BoundBuildId buildId,
DateTimeOffset buildDateTime,
TimeSpan duration,
string jobKind,
string machineName,
BuildResultClassification classification,
PullRequestInfo prInfo)
{
JobName = buildId.JobId.Name;
JobKind = jobKind;
ViewName = AzureUtil.GetViewName(BuildId.JobId);
BuildNumber = buildId.Number;
HostRaw = buildId.Host.ToString();
ClassificationKindRaw = classification.Kind.ToString();
ClassificationName = classification.Name;
BuildDateTime = buildDateTime.UtcDateTime;
MachineName = machineName;
IsPullRequest = JobUtil.IsPullRequestJobName(buildId.JobId);
DurationSeconds = (int)duration.TotalSeconds;
if (prInfo != null)
{
PullRequestId = prInfo.Id;
PullRequestAuthor = prInfo.Author;
PullRequestAuthorEmail = prInfo.AuthorEmail;
PullRequestUrl = prInfo.PullUrl;
PullRequestSha1 = prInfo.Sha1;
Debug.Assert(HasPullRequestInfo);
Debug.Assert(PullRequestInfo != null);
}
Debug.Assert(BuildDateTime.Kind == DateTimeKind.Utc);
}
private static async Task MigrateDateKeyCore <T>(string tableName)
where T : ITableEntity, new()
{
Console.WriteLine($"Processing {tableName}");
var account = GetStorageAccount();
var table = account.CreateCloudTableClient().GetTableReference(tableName);
var startKey = new DateKey(DateKey.StartDate);
var endKey = new DateKey(DateTimeOffset.UtcNow);
var query = TableQueryUtil.And(
TableQueryUtil.PartitionKey(startKey.Key, ColumnOperator.GreaterThanOrEqual),
TableQueryUtil.PartitionKey(endKey.Key, ColumnOperator.LessThanOrEqual));
var list = await AzureUtil.QueryAsync <T>(table, query);
Console.WriteLine($"Processing {list.Count} entities");
var deleteList = new List <EntityKey>();
foreach (var entity in list)
{
deleteList.Add(entity.GetEntityKey());
var dateKey = DateKey.Parse(entity.PartitionKey);
var dateTimeKey = new DateTimeKey(dateKey.Date, DateTimeKeyFlags.Date);
entity.PartitionKey = dateTimeKey.Key;
}
Console.WriteLine("Writing new values");
await AzureUtil.InsertBatchUnordered(table, list);
Console.WriteLine("Deleting old values");
await AzureUtil.DeleteBatchUnordered(table, deleteList);
}
// TODO: Consider using the host name here as part of the key. Need to understand what happens when
// jenkins sends multiple events with same BuildId from different hosts (because it picks one of the
// several host names we have for the server).
public static EntityKey GetExactEntityKey(BuildId buildId)
{
var partitionKey = AzureUtil.NormalizeKey(buildId.JobId.Name, '_');
var rowKey = buildId.Number.ToString("0000000000");
return(new EntityKey(partitionKey, rowKey));
}
public async Task TaoFailure()
{
var buildId = new BuildId(4, JobId.ParseName("test"));
_restClient.AddJson(
buildId: buildId,
buildResultJson: TestResources.Tao1BuildResult,
buildInfoJson: TestResources.Tao1BuildInfo,
failureInfoJson: TestResources.Tao1FailureInfo,
testReportJson: TestResources.Tao1TestResult);
var entity = await _populator.PopulateBuild(buildId);
var filter = FilterUtil
.Column(nameof(BuildFailureEntity.JobName), buildId.JobName)
.Filter;
var list = AzureUtil.Query <BuildFailureEntity>(_buildFailureExactTable, filter).ToList();
Assert.Equal(2, list.Count);
foreach (var item in list)
{
Assert.Equal(BuildFailureKind.TestCase, item.BuildFailureKind);
Assert.Equal(buildId, item.BuildId);
}
}
public static async Task EmailFailedBuild(
[QueueTrigger(QueueNames.EmailBuild)] string messageJson,
[Table(TableNames.BuildState)] CloudTable buildStateTable,
TextWriter logger,
CancellationToken cancellationToken)
{
var buildMessage = JsonConvert.DeserializeObject <BuildStateMessage>(messageJson);
var entityKey = BuildStateEntity.GetEntityKey(buildMessage.BuildStateKey, buildMessage.BoundBuildId);
var entity = await AzureUtil.QueryAsync <BuildStateEntity>(buildStateTable, entityKey, cancellationToken);
var textBuilder = new StringBuilder();
var htmlBuilder = new StringBuilder();
AppendEmailText(entity, textBuilder, htmlBuilder);
var message = new SendGridMessage()
{
Html = htmlBuilder.ToString(),
Text = textBuilder.ToString(),
};
message.AddTo("*****@*****.**");
message.AddTo("*****@*****.**");
message.From = new MailAddress("*****@*****.**");
message.Subject = $"Build process error {entity.JobName}";
var key = CloudConfigurationManager.GetSetting(SharedConstants.SendGridApiKeySettingName);
var web = new Web(apiKey: key);
await web.DeliverAsync(message);
}
/// <summary>
/// Populate the <see cref="BuildResultEntity"/> structures for a build overwriting any data
/// that existed before. Returns the entity if enough information was there to process the value.
/// </summary>
public async Task PopulateBuild(BoundBuildId buildId)
{
var data = await GetPopulateData(buildId);
var result = data.Result;
await PopulateViewName(buildId.JobId, result.BuildDateTimeOffset);
await _buildResultDateTable.ExecuteAsync(TableOperation.InsertOrReplace(result.CopyDate()));
await _buildResultExactTable.ExecuteAsync(TableOperation.InsertOrReplace(result.CopyExact()));
var failures = data.Failures;
if (failures.Count > 0)
{
// Important to use InsertOrReplace here. It's possible for a populate job to be cut off in the
// middle when the BuildFailure table is updated but not yet the BuildProcessed table. Hence
// we'll up here again doing a batch insert.
await AzureUtil.ExecuteBatchUnordered(_buildFailureExactTable, TableOperationType.InsertOrReplace, failures.Select(x => x.CopyExact()));
await AzureUtil.ExecuteBatchUnordered(_buildFailureDateTable, TableOperationType.InsertOrReplace, failures.Select(x => x.CopyDate()));
}
await PopulateCounters(result);
}
private static async Task Random()
{
/*
* var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_corefx/job/master/job/fedora23_debug_tst/134/");
* var buildId = boundBuildId.BuildId;
* var client = CreateClient(uri: boundBuildId.HostUri, auth: true);
* var buildInfo = await client.GetBuildInfoAsync(buildId);
* var buildResult = await client.GetBuildResultAsync(buildInfo);
* var test = await client.GetFailedTestCasesAsync(buildId);
* var prInfo = await client.GetPullRequestInfoAsync(buildId);
*/
var testboundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/release_1.0.0/job/x64_release_rhel7.2_pri1_flow/30/");
var testbuildId = testboundBuildId.BuildId;
var client = CreateClient(uri: testboundBuildId.HostUri, auth: true);
var elapsedTimeObj = client.GetBuildInfo(testbuildId).Duration;
Console.WriteLine($"\tET: {elapsedTimeObj.TotalMilliseconds}");
var account = GetStorageAccount();
var dateKey = new DateKey(DateTimeOffset.UtcNow - TimeSpan.FromDays(1));
var table = account.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.BuildResultDate);
var query = new TableQuery <BuildResultEntity>()
.Where(FilterUtil
.Column(ColumnNames.PartitionKey, dateKey, ColumnOperator.GreaterThanOrEqual)
.And(FilterUtil.Column("MachineName", "Azure0602081822")));
var all = await AzureUtil.QueryAsync(table, query);
foreach (var entity in all)
{
var boundBuildId = new BoundBuildId(SharedConstants.DotnetJenkinsUri.Host, entity.BuildId);
Console.WriteLine(boundBuildId.Uri);
}
}
/// <summary>
/// Has this entity been completely processed at this point.
/// </summary>
private async Task <bool> HasPopulatedData(BuildId buildId, CancellationToken cancellationToken)
{
var key = BuildResultEntity.GetExactEntityKey(buildId);
var entity = await AzureUtil.QueryAsync <BuildResultEntity>(_buildResultExact, key, cancellationToken);
return(entity != null);
}
/// <summary>
/// Populate the given build and update the unprocessed table accordingly. If there is no
/// existing entity in the unprocessed table, this won't add one. It will only update existing
/// ones.
/// </summary>
internal async Task Populate(BuildStateMessage message, BuildTablePopulator populator, CancellationToken cancellationToken)
{
var buildId = message.BuildId;
var entityKey = BuildStateEntity.GetEntityKey(message.BuildStateKey, message.BoundBuildId);
var entity = await AzureUtil.QueryAsync <BuildStateEntity>(_buildStateTable, entityKey, cancellationToken);
var completed = await PopulateCore(entity, populator, cancellationToken);
// Unable to complete the build, consider this is a 404 missing that we need to handle.
if (!completed && entity.BuildMissingCount > MissingBuildLimit)
{
completed = await PopulateMissing(entity, populator, cancellationToken);
}
if (completed)
{
return;
}
var isDone = (DateTimeOffset.UtcNow - entity.BuildStateKey.DateTime).TotalDays > DayWindow;
if (isDone)
{
await EnqueueEmailBuild(entity.BuildStateKey, entity.BoundBuildId, cancellationToken);
}
else
{
// Wait an hour to retry. Hope that a bug fix is uploaded or jenkins gets back into a good state.
await EnqueueProcessBuild(entity.BuildStateKey, entity.BoundBuildId, TimeSpan.FromHours(1), cancellationToken);
}
}
private static async Task ViewNameMigration()
{
var account = GetStorageAccount();
var client = account.CreateCloudTableClient();
var viewNameTable = client.GetTableReference(AzureConstants.TableNames.ViewNameDate);
var buildResultTable = client.GetTableReference(AzureConstants.TableNames.BuildResultDate);
var startDate = DateTimeOffset.UtcNow - TimeSpan.FromDays(14);
var query = new TableQuery <DynamicTableEntity>()
.Where(FilterUtil.SinceDate(ColumnNames.PartitionKey, startDate))
.Select(new[] { "PartitionKey", nameof(BuildResultEntity.ViewName) });
var all = await AzureUtil.QueryAsync(buildResultTable, query);
var set = new HashSet <Tuple <DateKey, string> >();
var list = new List <ViewNameEntity>();
foreach (var entity in all)
{
var dateKey = DateKey.Parse(entity.PartitionKey);
var viewName = entity.Properties[nameof(BuildResultEntity.ViewName)].StringValue;
var tuple = Tuple.Create(dateKey, viewName);
if (set.Add(tuple))
{
list.Add(new ViewNameEntity(dateKey, viewName));
}
}
await AzureUtil.InsertBatchUnordered(viewNameTable, list);
}
internal static async Task <int> Go()
{
try
{
var client = SharedUtil.CreateGitHubClient();
var storageAccount = SharedUtil.CreateStorageAccount();
AzureUtil.EnsureAzureResources(storageAccount);
// await DumpHooks(client);
// await DumpMilestones(client);
// await PrintRateLimits(client);
// await TestRateLimits(client, storageAccount);
await PopulateSince();
// await FixNulls(storageAccount);
// await DumpSince(client);
// await InitRepo(client, storageAccount, SharedUtil.RepoId);
// await Misc(client, storageAccount);
return(0);
}
catch (Exception ex)
{
Console.Write($"{ex.Message}");
return(1);
}
}
/// <summary>
/// Is this build alreadiy fully populated.
/// </summary>
public async Task <bool> IsPopulated(BuildId buildId, CancellationToken cancellationToken = default(CancellationToken))
{
var key = BuildResultEntity.GetExactEntityKey(buildId);
var entity = await AzureUtil.QueryAsync <DynamicTableEntity>(_buildResultExactTable, key, cancellationToken);
return(entity != null);
}
/// <summary>
/// Milestone information still does not come down as a part of events. This function catches these types of
/// update by doing a 'since' query on GitHub and bulk updating all of the changed values.
/// </summary>
public static async Task GithubPopulateIssuesSince(
[TimerTrigger("0 0/1 * * * *")] TimerInfo timerInfo,
[Table(TableNames.RoachIssueTable)] CloudTable issueTable,
[Table(TableNames.RoachMilestoneTable)] CloudTable milestoneTable,
[Table(TableNames.RoachStatusTable)] CloudTable statusTable,
TextWriter logger,
CancellationToken cancellationToken)
{
var client = SharedUtil.CreateGitHubClient();
var storagePopulator = new StoragePopulator(client, issueTable, milestoneTable);
// TODO: Need to make this adaptable to all repos, not just dotnet/roslyn
var allRepos = new[] { SharedUtil.RepoId };
foreach (var repo in allRepos)
{
var statusEntity = await AzureUtil.QueryAsync <RoachStatusEntity>(statusTable, RoachStatusEntity.GetEntityKey(repo), cancellationToken);
if (statusEntity == null || statusEntity.LastBulkUpdate.Value == null)
{
logger.WriteLine($"Repo {repo.Owner}/{repo.Name} does not have a status entry. Cannot do a since update.");
return;
}
var before = DateTimeOffset.UtcNow;
await storagePopulator.PopulateIssuesSince(repo, statusEntity.LastBulkUpdate.Value, cancellationToken);
// Given there are no events for milestones need to do a bulk update here.
await storagePopulator.PopulateMilestones(repo, cancellationToken);
statusEntity.SetLastBulkUpdate(before);
await statusTable.ExecuteAsync(TableOperation.Replace(statusEntity), cancellationToken);
}
}
public void Assert_Xml_Deadlock_Report_BuildSignature()
{
var time = DateTime.Parse("2019-01-01 00:00:00").ToUniversalTime();
var message = JsonConvert.SerializeObject(CreateDeadlockXmlDiagnosticsMessageFromJson());
var buildSignature = AzureUtil.BuildSignature(message.ToString(), SharedKey, CustomerId, time.ToString("r"));
buildSignature.Should().Be("SharedKey 5a57b610-6006-4ed5-848e-2a8324262c28:VngkZ+DskKUMSHVEMfTP2lraIWxllcjl2TsrND9PynU=");
}
private DotNetQueryUtil CreateForServer(DevOpsServer server)
{
// https://github.com/jaredpar/devops-util/issues/19
// Consider using a cache here
var azureUtil = new AzureUtil(server);
return(new DotNetQueryUtil(server, azureUtil));
}
public void JobNameInFolder()
{
var jobId = JobId.ParseName("job/cat/job/dog");
var buildId = new BuildId(42, jobId);
var buildKey = new BuildKey(buildId);
Assert.False(AzureUtil.IsIllegalKey(buildKey.Key));
}
internal static CloudStorageAccount GetStorageAccount()
{
// This is using the storage emulator account. Make sure to run the following before starting
// "C:\Program Files (x86)\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe" start
var account = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["StorageConnectionString"]);
AzureUtil.EnsureAzureResources(account);
return(account);
}
public void Simple()
{
var jobId = JobId.ParseName("dog");
var buildId = new BuildId(42, jobId);
var buildKey = new BuildKey(buildId);
Assert.False(AzureUtil.IsIllegalKey(buildKey.Key));
Assert.Equal("42-dog", buildKey.Key);
}
private static async Task FixNulls(CloudStorageAccount storageAccount)
{
var table = storageAccount.CreateCloudTableClient().GetTableReference(AzureConstants.TableNames.RoachIssueTable);
var filter = FilterUtil.PartitionKey(EntityKeyUtil.ToKey(SharedUtil.RepoId));
var list = await AzureUtil.QueryAsync <RoachIssueEntity>(table, filter);
var bad = list.Where(x => x.Assignee == null).ToList();
await AzureUtil.DeleteBatchUnordered(table, bad);
}
internal async Task Update(CloudQueue processBuildQueue, CancellationToken cancellationToken)
{
var query = new TableQuery <UnprocessedBuildEntity>();
await AzureUtil.QueryAsync(
_unprocessedBuildTable,
query,
e => UpdateEntity(e, processBuildQueue, cancellationToken),
cancellationToken);
}
public void Configuration(IAppBuilder app)
{
ConfigureAuth(app);
var connectionString = CloudConfigurationManager.GetSetting(SharedConstants.StorageConnectionStringName);
var storage = new DashboardStorage(connectionString);
AzureUtil.EnsureAzureResources(storage.StorageAccount);
}
public void ComplexJobKey()
{
var jobId = JobId.ParseName("job/cat/job/dog");
var buildId = new BuildId(42, jobId);
var entityKey = BuildResultEntity.GetExactEntityKey(buildId);
Assert.False(AzureUtil.IsIllegalKey(entityKey.PartitionKey));
Assert.False(AzureUtil.IsIllegalKey(entityKey.RowKey));
}
public void ComplexJobKeyDate()
{
var jobId = JobId.ParseName("job/cat/job/dog");
var buildId = new BuildId(42, jobId);
var entityKey = BuildFailureEntity.GetDateEntityKey(DateTimeOffset.UtcNow, buildId, "terrible/blah");
Assert.False(AzureUtil.IsIllegalKey(entityKey.PartitionKey));
Assert.False(AzureUtil.IsIllegalKey(entityKey.RowKey));
}
public async Task Assert_Xml_Deadlock_Report_PostData_Fails()
{
var time = DateTime.UtcNow;
var message = JsonConvert.SerializeObject(CreateDeadlockXmlDiagnosticsMessageFromJson());
var buildSignature = AzureUtil.BuildSignature(message.ToString(), SharedKey, CustomerId, time.ToString("r"));
var result = await AzureUtil.PostData(buildSignature, time.ToString("r"), message, CustomerId);
result.Should().Be("API Post Exception : No such host is known.");
}
public async Task InsertSameRowKey()
{
var key = "test";
var count = AzureUtil.MaxBatchCount * 2;
var list = GetSameRowList(count, key);
await AzureUtil.InsertBatchUnordered(_table, list);
var found = await AzureUtil.QueryAsync <DynamicTableEntity>(_table, TableQueryUtil.RowKey(key));
Assert.Equal(count, found.Count);
}
internal async Task <SendGridMessage> Clean(CancellationToken cancellationToken)
{
var limit = DateTimeOffset.UtcNow - TimeSpan.FromHours(12);
var filter = FilterUtil.Column(
nameof(UnprocessedBuildEntity.LastUpdate),
limit,
ColumnOperator.LessThanOrEqual);
var query = new TableQuery <UnprocessedBuildEntity>().Where(filter);
var list = await AzureUtil.QueryAsync(_unprocessedBuildTable, query, cancellationToken);
if (list.Count == 0)
{
return(null);
}
var textBuilder = new StringBuilder();
var htmlBuilder = new StringBuilder();
foreach (var entity in list)
{
var boundBuildId = entity.BoundBuildId;
var buildId = boundBuildId.BuildId;
// GC Stress jobs can correctly execute for up to 3 days. This is a bit of an outlier but one we
// need to handle;
if (JobUtil.IsGCStressJob(buildId.JobId))
{
var stressLimit = DateTimeOffset.UtcNow - TimeSpan.FromDays(3);
if (entity.LastUpdate >= stressLimit)
{
continue;
}
}
_logger.WriteLine($"Deleting stale data {boundBuildId.Uri}");
textBuilder.Append($"Deleting stale data: {boundBuildId.Uri}");
textBuilder.Append($"Eror: {entity.StatusText}");
htmlBuilder.Append($@"<div>");
htmlBuilder.Append($@"<div>Build <a href=""{boundBuildId.Uri}"">{buildId.JobName} {buildId.Number}</a></div>");
htmlBuilder.Append($@"<div>Error: {WebUtility.HtmlEncode(entity.StatusText)}</div>");
htmlBuilder.Append($@"</div>");
}
await AzureUtil.DeleteBatchUnordered(_unprocessedBuildTable, list);
return(new SendGridMessage()
{
Text = textBuilder.ToString(),
Html = htmlBuilder.ToString()
});
}
