private void IngestData(List <TeamProjectReference> projects)
{
List <VssProjectEntity> entities = new List <VssProjectEntity>();
foreach (TeamProjectReference project in projects)
{
VssProjectEntity entity = new VssProjectEntity
{
Organization = this.vssClient.OrganizationName,
ProjectId = project.Id,
Name = project.Name,
LastUpdateTime = project.LastUpdateTime,
Revision = project.Revision,
State = project.State.ToString(),
Visibility = project.Visibility.ToString(),
Url = project.Url,
};
entities.Add(entity);
}
this.logger.LogInformation("Start ingesting projects data...");
using VssDbContext context = new VssDbContext(logger, this.sqlConnectionString);
using IDbContextTransaction transaction = context.Database.BeginTransaction();
context.BulkDelete(context.VssProjectEntities.Where(v => v.Organization == this.vssClient.OrganizationName));
int insertedResult = context.BulkInsert(entities);
transaction.Commit();
this.logger.LogInformation($"Done ingesting {insertedResult} records");
}
private void IngestPullRequests(List <GitPullRequest> pullRequests, TeamProjectReference project)
{
List <VssPullRequestEntity> entities = new List <VssPullRequestEntity>();
foreach (GitPullRequest pullRequest in pullRequests)
{
VssPullRequestEntity entity = new VssPullRequestEntity
{
AuthorEmail = pullRequest.CreatedBy.UniqueName,
ClosedDate = pullRequest.ClosedDate,
CreationDate = pullRequest.CreationDate,
PullRequestId = pullRequest.PullRequestId,
LastMergeCommitID = pullRequest.LastMergeCommit?.CommitId,
LastMergeTargetCommitId = pullRequest.LastMergeTargetCommit?.CommitId,
SourceBranch = pullRequest.SourceRefName,
Status = pullRequest.Status.ToString(),
TargetBranch = pullRequest.TargetRefName,
Title = pullRequest.Title,
ProjectId = pullRequest.Repository.ProjectReference.Id,
RepositoryId = pullRequest.Repository.Id,
Organization = this.vssClient.OrganizationName,
RowUpdatedDate = Helper.UtcNow,
ProjectName = project.Name,
};
entities.Add(entity);
}
using VssDbContext context = new VssDbContext(logger, this.sqlConnectionString);
context.BulkInsertOrUpdate(entities);
}
// Clean up any stale data since this is a snapshot of data ingestion
private void Cleanup()
{
this.logger.LogInformation($"Deleting up stale data for {this.vssClient.OrganizationName}...");
using VssDbContext dbContext = new VssDbContext(logger, this.sqlServerConnectionString);
using IDbContextTransaction transaction = dbContext.Database.BeginTransaction();
int deletedBuildDefinitionResult = dbContext.BulkDelete(dbContext.VssBuildDefinitionEntities.Where(v => v.Organization == this.vssClient.OrganizationName && v.RowUpdatedDate < Helper.UtcNow));
int deletedBuildDefinitionStepResult = dbContext.BulkDelete(dbContext.VssBuildDefinitionStepEntities.Where(v => v.Organization == this.vssClient.OrganizationName && v.RowUpdatedDate < Helper.UtcNow).ToList());
transaction.Commit();
this.logger.LogInformation($"Done deleting {deletedBuildDefinitionResult} build definitions and {deletedBuildDefinitionStepResult} build definition steps");
}
private DateTime GetPullRequestWatermark(TeamProjectReference project, PullRequestStatus status)
{
// Default by going back to 1 month if no data has been ingested for this repo
DateTime mostRecentDate = DateTime.UtcNow.AddMonths(-1);
// Get latest ingested date for pull request from pull request watermark table
using VssDbContext context = new VssDbContext(logger, this.sqlConnectionString);
VssPullRequestWatermarkEntity latestWatermark = context.VssPullRequestWatermarkEntities.Where(v => v.ProjectId == project.Id && v.PullRequestStatus == status.ToString()).FirstOrDefault();
if (latestWatermark != null)
{
mostRecentDate = latestWatermark.RowUpdatedDate;
}
return(mostRecentDate);
}
private void UpdatePullRequestWatermark(PullRequestStatus status, TeamProjectReference project)
{
VssPullRequestWatermarkEntity vssPullRequestWatermarkEntity = new VssPullRequestWatermarkEntity
{
RowUpdatedDate = Helper.UtcNow,
Organization = this.vssClient.OrganizationName,
ProjectId = project.Id,
ProjectName = project.Name,
PullRequestStatus = status.ToString(),
};
this.logger.LogInformation("Start ingesting pull request data...");
using VssDbContext context = new VssDbContext(logger, this.sqlConnectionString);
int ingestedResult = context.BulkInsertOrUpdate(new List <VssPullRequestWatermarkEntity> {
vssPullRequestWatermarkEntity
});
this.logger.LogInformation($"Done ingesting {ingestedResult} records");
}
private void IngestData(List <BuildDefinition> buildDefinitions, TeamProjectReference project)
{
List <VssBuildDefinitionEntity> buildDefinitionEntities = new List <VssBuildDefinitionEntity>();
List <VssBuildDefinitionStepEntity> buildDefinitionStepEntities = new List <VssBuildDefinitionStepEntity>();
foreach (BuildDefinition buildDefinition in buildDefinitions)
{
// Parse build definitions
VssBuildDefinitionEntity buildDefinitionEntity = new VssBuildDefinitionEntity
{
Id = buildDefinition.Id,
Name = buildDefinition.Name,
Path = buildDefinition.Path,
ProjectName = buildDefinition.Project.Name,
ProjectId = buildDefinition.Project.Id,
PoolName = buildDefinition.Queue?.Pool?.Name,
PoolId = buildDefinition.Queue?.Pool?.Id,
IsHosted = buildDefinition.Queue?.Pool?.IsHosted,
QueueName = buildDefinition.Queue?.Name,
QueueId = buildDefinition.Queue?.Id,
CreatedDate = buildDefinition.CreatedDate,
UniqueName = buildDefinition.AuthoredBy.UniqueName,
Process = buildDefinition.Process.GetType().Name,
WebLink = (buildDefinition.Links.Links["web"] as ReferenceLink)?.Href,
RepositoryName = buildDefinition.Repository?.Name,
RepositoryId = buildDefinition.Repository?.Id,
Organization = this.vssClient.OrganizationName,
GZipCompressedJsonData = Helper.Compress(buildDefinition),
};
buildDefinitionEntities.Add(buildDefinitionEntity);
// Parse build definition steps for designer process
if (buildDefinition.Process.GetType() == typeof(DesignerProcess))
{
DesignerProcess process = (DesignerProcess)buildDefinition.Process;
int stepNumber = 1;
foreach (Phase phase in process.Phases)
{
foreach (BuildDefinitionStep buildDefinitionStep in phase.Steps)
{
VssBuildDefinitionStepEntity buildDefinitionStepEntity = new VssBuildDefinitionStepEntity
{
StepNumber = stepNumber,
// Build definition reference
BuildDefinitionId = buildDefinition.Id,
ProjectId = buildDefinition.Project.Id,
ProjectName = buildDefinition.Project.Name,
// Phase reference
PhaseType = this.GetPhaseType(phase),
PhaseRefName = phase.RefName,
PhaseName = phase.Name,
PhaseQueueId = this.GetPhaseQueueId(phase),
// Build definition steps
DisplayName = buildDefinitionStep.DisplayName,
Enabled = buildDefinitionStep.Enabled,
TaskDefinitionId = buildDefinitionStep.TaskDefinition.Id,
TaskVersionSpec = buildDefinitionStep.TaskDefinition.VersionSpec,
Condition = buildDefinitionStep.Condition,
// Extra data context
Organization = this.vssClient.OrganizationName,
//Data = Helper.SerializeObject(phase.Steps),
};
buildDefinitionStepEntities.Add(buildDefinitionStepEntity);
stepNumber++;
}
}
}
// todo: Need to parse yaml file here
}
this.logger.LogInformation("Ingesting build definition data...");
using VssDbContext dbContext = new VssDbContext(logger, this.sqlServerConnectionString);
using IDbContextTransaction transaction = dbContext.Database.BeginTransaction();
int buildDefinitionEntitiesResult = dbContext.BulkInsertOrUpdate(buildDefinitionEntities);
int buildDefinitionStepEntitiesResult = dbContext.BulkInsertOrUpdate(buildDefinitionStepEntities);
transaction.Commit();
this.logger.LogInformation($"Done ingesting {buildDefinitionEntitiesResult} build definitions, {buildDefinitionStepEntitiesResult} build definition steps");
}
