public async Task ProcessAsync_Success_JObject() { string url = "SomeUrl"; string response = @" { ""ProjectId"": ""ProjectId1"" } "; this.httpClient.AddResponse(url, HttpStatusCode.OK, response); GitHubCollectionNode collectionNode = new GitHubCollectionNode() { ApiName = "ApiName", RecordType = "RecordType", GetInitialUrl = metadata => url, ResponseType = typeof(JObject) }; await this.collector.ProcessAsync(collectionNode).ConfigureAwait(false); List <Tuple <JObject, RecordContext> > records = this.recordWriter.GetRecords(); Assert.AreEqual(1, records.Count); Assert.AreEqual("ProjectId1", records[0].Item1.SelectToken("$.ProjectId").Value <string>()); RecordContext recordContext1 = records[0].Item2; Assert.AreEqual(url, recordContext1.AdditionalMetadata["OriginatingUrl"]); Assert.AreEqual("RecordType", recordContext1.RecordType); }
public async Task ProcessAsync(Repository repository) { Dictionary <string, JToken> additionalMetadata = new Dictionary <string, JToken>() { { "OrganizationId", repository.OrganizationId }, { "OrganizationLogin", repository.OrganizationLogin }, { "RepositoryId", repository.RepositoryId }, { "RepositoryName", repository.RepositoryName }, }; // Referrers GitHubCollectionNode referrersNode = new GitHubCollectionNode() { RecordType = DataContract.ReferrerInstanceRecordType, ApiName = DataContract.ReferrersApiName, GetInitialUrl = additionalMetadata => InitialReferrersUrl(repository, this.apiDomain), AdditionalMetadata = additionalMetadata, }; foreach (IRecordWriter recordWriter in this.recordWriters) { await recordWriter.NewOutputAsync(DataContract.ReferrerInstanceRecordType).ConfigureAwait(false); } await this.collector.ProcessAsync(referrersNode).ConfigureAwait(false); // Views GitHubCollectionNode viewsNode = new GitHubCollectionNode() { RecordType = DataContract.ViewInstanceRecordType, ApiName = DataContract.ViewsApiName, GetInitialUrl = additionalMetadata => InitialViewsUrl(repository, this.apiDomain), AdditionalMetadata = additionalMetadata, ResponseType = typeof(JObject), }; foreach (IRecordWriter recordWriter in this.recordWriters) { await recordWriter.NewOutputAsync(DataContract.ViewInstanceRecordType).ConfigureAwait(false); } await this.collector.ProcessAsync(viewsNode).ConfigureAwait(false); // Clones GitHubCollectionNode clonesNode = new GitHubCollectionNode() { RecordType = DataContract.CloneInstanceRecordType, ApiName = DataContract.ClonesApiName, GetInitialUrl = additionalMetadata => InitialClonesUrl(repository, this.apiDomain), AdditionalMetadata = additionalMetadata, ResponseType = typeof(JObject), }; foreach (IRecordWriter recordWriter in this.recordWriters) { await recordWriter.NewOutputAsync(DataContract.CloneInstanceRecordType).ConfigureAwait(false); } await this.collector.ProcessAsync(clonesNode).ConfigureAwait(false); // Paths GitHubCollectionNode pathsNode = new GitHubCollectionNode() { RecordType = DataContract.PathInstanceRecordType, ApiName = DataContract.PathsApiName, GetInitialUrl = additionalMetadata => InitialPathsUrl(repository, this.apiDomain), AdditionalMetadata = additionalMetadata, }; foreach (IRecordWriter recordWriter in this.recordWriters) { await recordWriter.NewOutputAsync(DataContract.PathInstanceRecordType).ConfigureAwait(false); } await this.collector.ProcessAsync(pathsNode).ConfigureAwait(false); }
private async Task ExecuteTrafficCollector(ExecutionContext executionContext, ILogger logger) { DateTime functionStartDate = DateTime.UtcNow; string sessionId = Guid.NewGuid().ToString(); string identifier = "TrafficTimer"; CloudQueue trafficCloudQueue = await AzureHelpers.GetStorageQueueAsync("traffic").ConfigureAwait(false); IQueue trafficQueue = new CloudQueueWrapper(trafficCloudQueue); FunctionContext context = new FunctionContext() { CollectorType = CollectorType.TrafficTimer.ToString(), FunctionStartDate = functionStartDate, SessionId = sessionId, InvocationId = executionContext.InvocationId.ToString(), }; StatsTracker statsTracker = null; bool success = false; ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger); try { telemetryClient.TrackEvent("SessionStart", GetCollectorCommonSessionStartEventProperties(context, identifier)); ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter"); await rateLimiterCache.InitializeAsync().ConfigureAwait(false); ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests"); await requestsCache.InitializeAsync().ConfigureAwait(false); string organizations = await AzureHelpers.GetBlobContentAsync("github-settings", "organizations.json").ConfigureAwait(false); JArray organizationsArray = JArray.Parse(organizations); foreach (JToken organizationToken in organizationsArray) { JObject organization = (JObject)organizationToken; string organizationLogin = organization.SelectToken("$.OrganizationLogin").Value <string>(); long organizationId = organization.SelectToken("$.OrganizationId").Value <long>(); IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 80.0, this.apiDomain); GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient); statsTracker = new StatsTracker(telemetryClient, httpClient, StatsTrackerRefreshFrequency); IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.TrafficTimer, httpClient, organizationLogin, this.apiDomain); CollectorBase <GitHubCollectionNode> collector = new GitHubCollector(httpClient, authentication, telemetryClient, new List <IRecordWriter>()); GitHubCollectionNode repositoriesNode = new GitHubCollectionNode() { RecordType = DataContract.RepositoryInstanceRecordType, ApiName = DataContract.RepositoriesApiName, GetInitialUrl = additionalMetadata => OnboardingProcessor.InitialRepositoriesUrl(organizationLogin, this.apiDomain), ProcessRecordAsync = async record => { string repositoryName = record.SelectToken("$.name").Value <string>(); long repositoryId = record.SelectToken("$.id").Value <long>(); Repository repository = new Repository(organizationId, repositoryId, organizationLogin, repositoryName); await trafficQueue.PutObjectAsJsonStringAsync(repository, TimeSpan.MaxValue).ConfigureAwait(false); return(new List <RecordWithContext>()); }, }; await collector.ProcessAsync(repositoriesNode).ConfigureAwait(false); } success = true; } catch (Exception exception) when(!(exception is FatalException)) { telemetryClient.TrackException(exception, "TrafficTimer failed."); throw exception; } finally { SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths: string.Empty, GetCollectorCommonSessionStartEventProperties(context, identifier), success); statsTracker?.Stop(); } }