Esempio n. 1
0
        public async Task <HttpResponseMessage> ProcessWebHook([HttpTrigger(AuthorizationLevel.Function, "POST")] HttpRequestMessage request,
                                                               [DurableClient] IDurableOrchestrationClient orchestrationClient,
                                                               ExecutionContext executionContext)
        {
            DateTime startTime   = DateTime.UtcNow;
            string   requestBody = await request.Content.ReadAsStringAsync().ConfigureAwait(false);

            request.Headers.TryGetValues("X-GitHub-Delivery", out IEnumerable <string> deliveryGuidValues);
            if (deliveryGuidValues == null || !deliveryGuidValues.Any())
            {
                string errorMessage = $"WebHookProcessor requires 'X-GitHub-Event' header.";
                this.telemetryClient.TrackTrace(errorMessage, SeverityLevel.Error);
                throw new FatalException(errorMessage);
            }
            string sessionId = deliveryGuidValues.First();

            request.Headers.TryGetValues("X-GitHub-Event", out IEnumerable <string> eventTypeValues);
            if (eventTypeValues == null || !eventTypeValues.Any())
            {
                string errorMessage = $"WebHookProcessor endpoint requires 'X-GitHub-Event' header.";
                this.telemetryClient.TrackTrace(errorMessage, SeverityLevel.Error);
                throw new FatalException(errorMessage);
            }
            string eventType = eventTypeValues.First();

            // The request does not have to go through the LogicApp. This might happen e.g., for local testing. Therefore, assume that Logic-App-related headers are optional:
            // X-LogicApp-Timestamp
            // X-Ms-Workflow-Run-Id
            request.Headers.TryGetValues("X-LogicApp-Timestamp", out IEnumerable <string> logicAppStartDateValues);
            string logicAppStartDate = string.Empty;

            if (logicAppStartDateValues != null && logicAppStartDateValues.Any())
            {
                logicAppStartDate = logicAppStartDateValues.First();
            }

            request.Headers.TryGetValues("X-Ms-Workflow-Run-Id", out IEnumerable <string> logicAppRunIdValues);
            string logicAppRunId = string.Empty;

            if (logicAppRunIdValues != null && logicAppRunIdValues.Any())
            {
                logicAppRunId = logicAppRunIdValues.First();
            }

            OrchestrationContext context = new OrchestrationContext()
            {
                RequestBody       = requestBody,
                CollectorType     = CollectorType.Main.ToString(),
                SessionId         = sessionId,
                EventType         = eventType,
                FunctionStartDate = startTime,
                LogicAppStartDate = logicAppStartDate,
                LogicAppRunId     = logicAppRunId,
                InvocationId      = executionContext.InvocationId.ToString(),
            };

            ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context);

            telemetryClient.TrackEvent("SessionStart", GetMainCollectorSessionStartEventProperties(context, identifier: eventType, logicAppRunId));

            string instanceId = await orchestrationClient.StartNewAsync("ProcessWebHookOrchestration", context).ConfigureAwait(false);

            return(orchestrationClient.CreateCheckStatusResponse(request, instanceId, returnInternalServerErrorOnFailure: true));
        }
Esempio n. 2
0
        private async Task ExecuteTrafficCollector(ExecutionContext executionContext, ILogger logger)
        {
            DateTime functionStartDate = DateTime.UtcNow;
            string   sessionId         = Guid.NewGuid().ToString();
            string   identifier        = "TrafficTimer";

            CloudQueue trafficCloudQueue = await AzureHelpers.GetStorageQueueAsync("traffic").ConfigureAwait(false);

            IQueue trafficQueue = new CloudQueueWrapper(trafficCloudQueue);

            FunctionContext context = new FunctionContext()
            {
                CollectorType     = CollectorType.TrafficTimer.ToString(),
                FunctionStartDate = functionStartDate,
                SessionId         = sessionId,
                InvocationId      = executionContext.InvocationId.ToString(),
            };

            StatsTracker     statsTracker    = null;
            bool             success         = false;
            ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger);

            try
            {
                telemetryClient.TrackEvent("SessionStart", GetCollectorCommonSessionStartEventProperties(context, identifier));

                ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
                await rateLimiterCache.InitializeAsync().ConfigureAwait(false);

                ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
                await requestsCache.InitializeAsync().ConfigureAwait(false);

                string organizations = await AzureHelpers.GetBlobContentAsync("github-settings", "organizations.json").ConfigureAwait(false);

                JArray organizationsArray = JArray.Parse(organizations);
                foreach (JToken organizationToken in organizationsArray)
                {
                    JObject organization      = (JObject)organizationToken;
                    string  organizationLogin = organization.SelectToken("$.OrganizationLogin").Value <string>();
                    long    organizationId    = organization.SelectToken("$.OrganizationId").Value <long>();

                    IRateLimiter     rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 80.0, this.apiDomain);
                    GitHubHttpClient httpClient  = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);

                    statsTracker = new StatsTracker(telemetryClient, httpClient, StatsTrackerRefreshFrequency);

                    IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.TrafficTimer, httpClient, organizationLogin, this.apiDomain);
                    CollectorBase <GitHubCollectionNode> collector = new GitHubCollector(httpClient, authentication, telemetryClient, new List <IRecordWriter>());

                    GitHubCollectionNode repositoriesNode = new GitHubCollectionNode()
                    {
                        RecordType         = DataContract.RepositoryInstanceRecordType,
                        ApiName            = DataContract.RepositoriesApiName,
                        GetInitialUrl      = additionalMetadata => OnboardingProcessor.InitialRepositoriesUrl(organizationLogin, this.apiDomain),
                        ProcessRecordAsync = async record =>
                        {
                            string repositoryName = record.SelectToken("$.name").Value <string>();
                            long   repositoryId   = record.SelectToken("$.id").Value <long>();

                            Repository repository = new Repository(organizationId, repositoryId, organizationLogin, repositoryName);
                            await trafficQueue.PutObjectAsJsonStringAsync(repository, TimeSpan.MaxValue).ConfigureAwait(false);

                            return(new List <RecordWithContext>());
                        },
                    };

                    await collector.ProcessAsync(repositoriesNode).ConfigureAwait(false);
                }

                success = true;
            }
            catch (Exception exception) when(!(exception is FatalException))
            {
                telemetryClient.TrackException(exception, "TrafficTimer failed.");
                throw exception;
            }
            finally
            {
                SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths: string.Empty, GetCollectorCommonSessionStartEventProperties(context, identifier), success);
                statsTracker?.Stop();
            }
        }
Esempio n. 3
0
        public async Task Traffic([QueueTrigger("traffic")] string queueItem, ExecutionContext executionContext, ILogger logger)
        {
            DateTime functionStartDate = DateTime.UtcNow;
            string   sessionId         = Guid.NewGuid().ToString();

            JsonSerializerSettings serializerSettings = new JsonSerializerSettings()
            {
                TypeNameHandling = TypeNameHandling.None
            };
            Repository repositoryDetails = JsonConvert.DeserializeObject <Repository>(queueItem, serializerSettings);
            FunctionContextWriter <FunctionContext> contextWriter = new FunctionContextWriter <FunctionContext>();
            string identifier = $"Traffic";

            FunctionContext context = new FunctionContext()
            {
                CollectorType     = CollectorType.Traffic.ToString(),
                FunctionStartDate = functionStartDate,
                SessionId         = sessionId,
                InvocationId      = executionContext.InvocationId.ToString(),
            };

            StatsTracker     statsTracker    = null;
            string           outputPaths     = string.Empty;
            bool             success         = false;
            ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger);

            try
            {
                telemetryClient.TrackEvent("SessionStart", GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails));

                ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
                await rateLimiterCache.InitializeAsync().ConfigureAwait(false);

                IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? repositoryDetails.OrganizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 50.0, this.apiDomain);
                ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
                await requestsCache.InitializeAsync().ConfigureAwait(false);

                GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);

                IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.Traffic, httpClient, repositoryDetails.OrganizationLogin, this.apiDomain);

                StorageManager       storageManager;
                List <IRecordWriter> recordWriters;
                using (storageManager = this.configManager.GetStorageManager(context.CollectorType, telemetryClient))
                {
                    recordWriters = storageManager.InitializeRecordWriters(identifier, context, contextWriter, this.adlsClient.AdlsClient);
                    IRecordStatsTracker recordStatsTracker = null;

                    foreach (IRecordWriter recordWriter in recordWriters)
                    {
                        recordWriter.SetOutputPathPrefix($"{repositoryDetails.OrganizationId}/{repositoryDetails.RepositoryId}");
                        if (recordStatsTracker == null)
                        {
                            recordStatsTracker = recordWriter;
                        }
                    }

                    statsTracker = new StatsTracker(telemetryClient, httpClient, recordStatsTracker, StatsTrackerRefreshFrequency);

                    TrafficProcessor processor = new TrafficProcessor(authentication, recordWriters, httpClient, telemetryClient, this.apiDomain);
                    await processor.ProcessAsync(repositoryDetails).ConfigureAwait(false);
                }

                await storageManager.FinalizeRecordWritersAsync().ConfigureAwait(false);

                outputPaths = RecordWriterExtensions.GetOutputPaths(recordWriters);
                success     = true;
            }
            catch (Exception exception) when(!(exception is FatalException))
            {
                telemetryClient.TrackException(exception, "Traffic failed.");
                throw exception;
            }
            finally
            {
                SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths, GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails), success);
                statsTracker?.Stop();
            }
        }
Esempio n. 4
0
        public async Task ProcessWebHookActivity([ActivityTrigger] IDurableActivityContext durableContext)
        {
            OrchestrationContext context = durableContext.GetInput <OrchestrationContext>();
            string requestBody           = context.RequestBody;
            WebhookProcessorContext       functionContext = context.Downgrade();
            WebhookProcessorContextWriter contextWriter   = new WebhookProcessorContextWriter();

            JObject record = JObject.Parse(requestBody);
            string  organizationName;

            Dictionary <string, string> additionalTelemetryProperties = new Dictionary <string, string>();

            foreach (KeyValuePair <string, string> property in GetMainCollectorSessionStartEventProperties(functionContext, identifier: functionContext.EventType, functionContext.LogicAppRunId))
            {
                additionalTelemetryProperties.Add(property.Key, property.Value);
            }
            string           outputPaths     = string.Empty;
            bool             success         = false;
            ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, functionContext);

            try
            {
                // Not all payloads have a "repository" attribute e.g., membership, organization, project, project_card, etc. paylods. Look under $organization attribute first, if available.
                JToken organizationNameToken = record.SelectToken("$.organization.login");
                if (organizationNameToken == null)
                {
                    organizationNameToken = record.SelectToken("$.repository.owner.login");
                }
                if (organizationNameToken == null)
                {
                    // NOTE: BELOW RELEVANT IF WE ARE RELYING ON APP WEBHOOKS
                    // In theory, all event payloads will have repository.owner.login (see https://developer.github.com/webhooks/event-payloads/#webhook-payload-object-common-properties)
                    // However, there are certain app-specific webhooks (e.g. new_permissions_accepted, which is not documented/can't be found by a search engine) that lack this.
                    // note: other example app-specific webhooks - app installed into repo, app uninstalled from repo, etc.
                    throw new FatalTerminalException("Could not find organization name in webhook payload.");
                }
                organizationName = organizationNameToken.Value <string>();

                IEventsBookkeeper eventsBookkeeper = new EventsBookkeeper(telemetryClient);
                await eventsBookkeeper.InitializeAsync().ConfigureAwait(false);

                ICache <RecordTableEntity> recordsCache = new AzureTableCache <RecordTableEntity>(telemetryClient, "records");
                await recordsCache.InitializeAsync().ConfigureAwait(false);

                ICache <RepositoryItemTableEntity> collectorCache = new AzureTableCache <RepositoryItemTableEntity>(telemetryClient, "github");
                await collectorCache.InitializeAsync().ConfigureAwait(false);

                ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
                await rateLimiterCache.InitializeAsync().ConfigureAwait(false);

                ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
                await requestsCache.InitializeAsync().ConfigureAwait(false);

                IRateLimiter     rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationName : "*", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 90.0, this.apiDomain);
                GitHubHttpClient httpClient  = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);

                IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.Main, httpClient, organizationName, this.apiDomain);

                StorageManager       storageManager;
                List <IRecordWriter> recordWriters;
                using (storageManager = this.configManager.GetStorageManager(context.CollectorType, telemetryClient))
                {
                    recordWriters = storageManager.InitializeRecordWriters(identifier: functionContext.EventType, functionContext, contextWriter, this.adlsClient.AdlsClient);
                    WebHookProcessor processor = new WebHookProcessor(requestBody, functionContext, authentication, httpClient, recordWriters, eventsBookkeeper, recordsCache, collectorCache, telemetryClient, this.apiDomain);
                    additionalTelemetryProperties = await processor.ProcessAsync().ConfigureAwait(false);

                    foreach (KeyValuePair <string, string> property in GetMainCollectorSessionStartEventProperties(functionContext, identifier: functionContext.EventType, functionContext.LogicAppRunId))
                    {
                        additionalTelemetryProperties.Add(property.Key, property.Value);
                    }
                }

                await storageManager.FinalizeRecordWritersAsync().ConfigureAwait(false);

                outputPaths = RecordWriterExtensions.GetOutputPaths(recordWriters);
                success     = true;
            }
            catch (Exception exception) when(!(exception is FatalException))
            {
                telemetryClient.TrackException(exception, "ProcessWebHookActivity failed.");
                throw exception;
            }
            finally
            {
                SendSessionEndEvent(telemetryClient, functionContext.FunctionStartDate, outputPaths, additionalTelemetryProperties, success);
            }
        }