public async Task <HttpResponseMessage> ProcessWebHook([HttpTrigger(AuthorizationLevel.Function, "POST")] HttpRequestMessage request,
[DurableClient] IDurableOrchestrationClient orchestrationClient,
ExecutionContext executionContext)
{
DateTime startTime = DateTime.UtcNow;
string requestBody = await request.Content.ReadAsStringAsync().ConfigureAwait(false);
request.Headers.TryGetValues("X-GitHub-Delivery", out IEnumerable <string> deliveryGuidValues);
if (deliveryGuidValues == null || !deliveryGuidValues.Any())
{
string errorMessage = $"WebHookProcessor requires 'X-GitHub-Event' header.";
this.telemetryClient.TrackTrace(errorMessage, SeverityLevel.Error);
throw new FatalException(errorMessage);
}
string sessionId = deliveryGuidValues.First();
request.Headers.TryGetValues("X-GitHub-Event", out IEnumerable <string> eventTypeValues);
if (eventTypeValues == null || !eventTypeValues.Any())
{
string errorMessage = $"WebHookProcessor endpoint requires 'X-GitHub-Event' header.";
this.telemetryClient.TrackTrace(errorMessage, SeverityLevel.Error);
throw new FatalException(errorMessage);
}
string eventType = eventTypeValues.First();
// The request does not have to go through the LogicApp. This might happen e.g., for local testing. Therefore, assume that Logic-App-related headers are optional:
// X-LogicApp-Timestamp
// X-Ms-Workflow-Run-Id
request.Headers.TryGetValues("X-LogicApp-Timestamp", out IEnumerable <string> logicAppStartDateValues);
string logicAppStartDate = string.Empty;
if (logicAppStartDateValues != null && logicAppStartDateValues.Any())
{
logicAppStartDate = logicAppStartDateValues.First();
}
request.Headers.TryGetValues("X-Ms-Workflow-Run-Id", out IEnumerable <string> logicAppRunIdValues);
string logicAppRunId = string.Empty;
if (logicAppRunIdValues != null && logicAppRunIdValues.Any())
{
logicAppRunId = logicAppRunIdValues.First();
}
OrchestrationContext context = new OrchestrationContext()
{
RequestBody = requestBody,
CollectorType = CollectorType.Main.ToString(),
SessionId = sessionId,
EventType = eventType,
FunctionStartDate = startTime,
LogicAppStartDate = logicAppStartDate,
LogicAppRunId = logicAppRunId,
InvocationId = executionContext.InvocationId.ToString(),
};
ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context);
telemetryClient.TrackEvent("SessionStart", GetMainCollectorSessionStartEventProperties(context, identifier: eventType, logicAppRunId));
string instanceId = await orchestrationClient.StartNewAsync("ProcessWebHookOrchestration", context).ConfigureAwait(false);
return(orchestrationClient.CreateCheckStatusResponse(request, instanceId, returnInternalServerErrorOnFailure: true));
}
private async Task ExecuteTrafficCollector(ExecutionContext executionContext, ILogger logger)
{
DateTime functionStartDate = DateTime.UtcNow;
string sessionId = Guid.NewGuid().ToString();
string identifier = "TrafficTimer";
CloudQueue trafficCloudQueue = await AzureHelpers.GetStorageQueueAsync("traffic").ConfigureAwait(false);
IQueue trafficQueue = new CloudQueueWrapper(trafficCloudQueue);
FunctionContext context = new FunctionContext()
{
CollectorType = CollectorType.TrafficTimer.ToString(),
FunctionStartDate = functionStartDate,
SessionId = sessionId,
InvocationId = executionContext.InvocationId.ToString(),
};
StatsTracker statsTracker = null;
bool success = false;
ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger);
try
{
telemetryClient.TrackEvent("SessionStart", GetCollectorCommonSessionStartEventProperties(context, identifier));
ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
await rateLimiterCache.InitializeAsync().ConfigureAwait(false);
ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
await requestsCache.InitializeAsync().ConfigureAwait(false);
string organizations = await AzureHelpers.GetBlobContentAsync("github-settings", "organizations.json").ConfigureAwait(false);
JArray organizationsArray = JArray.Parse(organizations);
foreach (JToken organizationToken in organizationsArray)
{
JObject organization = (JObject)organizationToken;
string organizationLogin = organization.SelectToken("$.OrganizationLogin").Value <string>();
long organizationId = organization.SelectToken("$.OrganizationId").Value <long>();
IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 80.0, this.apiDomain);
GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);
statsTracker = new StatsTracker(telemetryClient, httpClient, StatsTrackerRefreshFrequency);
IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.TrafficTimer, httpClient, organizationLogin, this.apiDomain);
CollectorBase <GitHubCollectionNode> collector = new GitHubCollector(httpClient, authentication, telemetryClient, new List <IRecordWriter>());
GitHubCollectionNode repositoriesNode = new GitHubCollectionNode()
{
RecordType = DataContract.RepositoryInstanceRecordType,
ApiName = DataContract.RepositoriesApiName,
GetInitialUrl = additionalMetadata => OnboardingProcessor.InitialRepositoriesUrl(organizationLogin, this.apiDomain),
ProcessRecordAsync = async record =>
{
string repositoryName = record.SelectToken("$.name").Value <string>();
long repositoryId = record.SelectToken("$.id").Value <long>();
Repository repository = new Repository(organizationId, repositoryId, organizationLogin, repositoryName);
await trafficQueue.PutObjectAsJsonStringAsync(repository, TimeSpan.MaxValue).ConfigureAwait(false);
return(new List <RecordWithContext>());
},
};
await collector.ProcessAsync(repositoriesNode).ConfigureAwait(false);
}
success = true;
}
catch (Exception exception) when(!(exception is FatalException))
{
telemetryClient.TrackException(exception, "TrafficTimer failed.");
throw exception;
}
finally
{
SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths: string.Empty, GetCollectorCommonSessionStartEventProperties(context, identifier), success);
statsTracker?.Stop();
}
}
public async Task Traffic([QueueTrigger("traffic")] string queueItem, ExecutionContext executionContext, ILogger logger)
{
DateTime functionStartDate = DateTime.UtcNow;
string sessionId = Guid.NewGuid().ToString();
JsonSerializerSettings serializerSettings = new JsonSerializerSettings()
{
TypeNameHandling = TypeNameHandling.None
};
Repository repositoryDetails = JsonConvert.DeserializeObject <Repository>(queueItem, serializerSettings);
FunctionContextWriter <FunctionContext> contextWriter = new FunctionContextWriter <FunctionContext>();
string identifier = $"Traffic";
FunctionContext context = new FunctionContext()
{
CollectorType = CollectorType.Traffic.ToString(),
FunctionStartDate = functionStartDate,
SessionId = sessionId,
InvocationId = executionContext.InvocationId.ToString(),
};
StatsTracker statsTracker = null;
string outputPaths = string.Empty;
bool success = false;
ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger);
try
{
telemetryClient.TrackEvent("SessionStart", GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails));
ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
await rateLimiterCache.InitializeAsync().ConfigureAwait(false);
IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? repositoryDetails.OrganizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 50.0, this.apiDomain);
ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
await requestsCache.InitializeAsync().ConfigureAwait(false);
GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);
IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.Traffic, httpClient, repositoryDetails.OrganizationLogin, this.apiDomain);
StorageManager storageManager;
List <IRecordWriter> recordWriters;
using (storageManager = this.configManager.GetStorageManager(context.CollectorType, telemetryClient))
{
recordWriters = storageManager.InitializeRecordWriters(identifier, context, contextWriter, this.adlsClient.AdlsClient);
IRecordStatsTracker recordStatsTracker = null;
foreach (IRecordWriter recordWriter in recordWriters)
{
recordWriter.SetOutputPathPrefix($"{repositoryDetails.OrganizationId}/{repositoryDetails.RepositoryId}");
if (recordStatsTracker == null)
{
recordStatsTracker = recordWriter;
}
}
statsTracker = new StatsTracker(telemetryClient, httpClient, recordStatsTracker, StatsTrackerRefreshFrequency);
TrafficProcessor processor = new TrafficProcessor(authentication, recordWriters, httpClient, telemetryClient, this.apiDomain);
await processor.ProcessAsync(repositoryDetails).ConfigureAwait(false);
}
await storageManager.FinalizeRecordWritersAsync().ConfigureAwait(false);
outputPaths = RecordWriterExtensions.GetOutputPaths(recordWriters);
success = true;
}
catch (Exception exception) when(!(exception is FatalException))
{
telemetryClient.TrackException(exception, "Traffic failed.");
throw exception;
}
finally
{
SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths, GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails), success);
statsTracker?.Stop();
}
}
public async Task ProcessWebHookActivity([ActivityTrigger] IDurableActivityContext durableContext)
{
OrchestrationContext context = durableContext.GetInput <OrchestrationContext>();
string requestBody = context.RequestBody;
WebhookProcessorContext functionContext = context.Downgrade();
WebhookProcessorContextWriter contextWriter = new WebhookProcessorContextWriter();
JObject record = JObject.Parse(requestBody);
string organizationName;
Dictionary <string, string> additionalTelemetryProperties = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> property in GetMainCollectorSessionStartEventProperties(functionContext, identifier: functionContext.EventType, functionContext.LogicAppRunId))
{
additionalTelemetryProperties.Add(property.Key, property.Value);
}
string outputPaths = string.Empty;
bool success = false;
ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, functionContext);
try
{
// Not all payloads have a "repository" attribute e.g., membership, organization, project, project_card, etc. paylods. Look under $organization attribute first, if available.
JToken organizationNameToken = record.SelectToken("$.organization.login");
if (organizationNameToken == null)
{
organizationNameToken = record.SelectToken("$.repository.owner.login");
}
if (organizationNameToken == null)
{
// NOTE: BELOW RELEVANT IF WE ARE RELYING ON APP WEBHOOKS
// In theory, all event payloads will have repository.owner.login (see https://developer.github.com/webhooks/event-payloads/#webhook-payload-object-common-properties)
// However, there are certain app-specific webhooks (e.g. new_permissions_accepted, which is not documented/can't be found by a search engine) that lack this.
// note: other example app-specific webhooks - app installed into repo, app uninstalled from repo, etc.
throw new FatalTerminalException("Could not find organization name in webhook payload.");
}
organizationName = organizationNameToken.Value <string>();
IEventsBookkeeper eventsBookkeeper = new EventsBookkeeper(telemetryClient);
await eventsBookkeeper.InitializeAsync().ConfigureAwait(false);
ICache <RecordTableEntity> recordsCache = new AzureTableCache <RecordTableEntity>(telemetryClient, "records");
await recordsCache.InitializeAsync().ConfigureAwait(false);
ICache <RepositoryItemTableEntity> collectorCache = new AzureTableCache <RepositoryItemTableEntity>(telemetryClient, "github");
await collectorCache.InitializeAsync().ConfigureAwait(false);
ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
await rateLimiterCache.InitializeAsync().ConfigureAwait(false);
ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
await requestsCache.InitializeAsync().ConfigureAwait(false);
IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationName : "*", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 90.0, this.apiDomain);
GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);
IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.Main, httpClient, organizationName, this.apiDomain);
StorageManager storageManager;
List <IRecordWriter> recordWriters;
using (storageManager = this.configManager.GetStorageManager(context.CollectorType, telemetryClient))
{
recordWriters = storageManager.InitializeRecordWriters(identifier: functionContext.EventType, functionContext, contextWriter, this.adlsClient.AdlsClient);
WebHookProcessor processor = new WebHookProcessor(requestBody, functionContext, authentication, httpClient, recordWriters, eventsBookkeeper, recordsCache, collectorCache, telemetryClient, this.apiDomain);
additionalTelemetryProperties = await processor.ProcessAsync().ConfigureAwait(false);
foreach (KeyValuePair <string, string> property in GetMainCollectorSessionStartEventProperties(functionContext, identifier: functionContext.EventType, functionContext.LogicAppRunId))
{
additionalTelemetryProperties.Add(property.Key, property.Value);
}
}
await storageManager.FinalizeRecordWritersAsync().ConfigureAwait(false);
outputPaths = RecordWriterExtensions.GetOutputPaths(recordWriters);
success = true;
}
catch (Exception exception) when(!(exception is FatalException))
{
telemetryClient.TrackException(exception, "ProcessWebHookActivity failed.");
throw exception;
}
finally
{
SendSessionEndEvent(telemetryClient, functionContext.FunctionStartDate, outputPaths, additionalTelemetryProperties, success);
}
}