public static async Task Main(string[] args)
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.Enrich.FromLogContext()
.WriteTo.Console()
.CreateLogger();
var configuration = new ConfigurationBuilder()
.AddJsonFile("defaultSettings.json")
.AddCommandLine(args)
.AddEnvironmentVariables()
.Build();
// Print out all config data
foreach (var child in configuration.GetChildren())
{
Log.Logger.Information($"{child.Path} ({child.Key}) = {child.Value ?? "(null)"}");
}
var config = new RunTimeConfiguration();
configuration.Bind(config);
Log.Logger.Information("Config is {@Config}", config);
if (config.InitialDelayDuration > 0)
{
System.Threading.Thread.Sleep(1000 * config.InitialDelayDuration);
}
if (string.IsNullOrWhiteSpace(config.Url))
{
Log.Logger.Error("You must specify a url for the Repository Analytics API");
return;
}
if (string.IsNullOrWhiteSpace(config.User) && string.IsNullOrWhiteSpace(config.Organization))
{
Log.Logger.Error("You must specify a user or an origanization");
return;
}
else if (!string.IsNullOrWhiteSpace(config.User) && !string.IsNullOrWhiteSpace(config.Organization))
{
Log.Logger.Error("You can not specify both a user and an origanization");
return;
}
var inDockerEnvVar = Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER");
var runningInContainer = inDockerEnvVar == null || inDockerEnvVar == "true";
var repositoryAnalyticsOrchestrator = new RepositoryAnalysisOrchestrator(Log.Logger, new RestClient(), runningInContainer);
await repositoryAnalyticsOrchestrator.OrchestrateAsync(config);
return;
}
public async Task OrchestrateAsync(RunTimeConfiguration config)
{
bool moreRepostoriesToRead = false;
var sourceRepositoriesRead = 0;
var sourceRepositoriesAnalyzed = 0;
var repositoryAnalysisErrors = new List <(string repoName, string errorMessage, string errorStackTrace)>();
try
{
var userOrOranizationNameQueryStringKey = string.Empty;
var userOrOganziationNameQueryStringValue = string.Empty;
if (!string.IsNullOrWhiteSpace(config.User))
{
userOrOranizationNameQueryStringKey = "user";
userOrOganziationNameQueryStringValue = config.User;
}
else
{
userOrOranizationNameQueryStringKey = "organization";
userOrOganziationNameQueryStringValue = config.Organization;
}
restClient.BaseUrl = new Uri(config.Url);
string endCursor = null;
var stopWatch = Stopwatch.StartNew();
do
{
logger.Information($"Reading next batch of {config.BatchSize} repositories for login {config.User ?? config.Organization}");
CursorPagedResults <RepositorySummary> results = null;
if (endCursor != null)
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
request.AddQueryParameter("endCursor", endCursor);
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
else
{
var request = new RestRequest("/api/repositorysource/repositories");
request.AddQueryParameter("owner", userOrOganziationNameQueryStringValue);
request.AddQueryParameter("take", config.BatchSize.ToString());
var response = await restClient.ExecuteTaskAsync <CursorPagedResults <RepositorySummary> >(request);
if (!response.IsSuccessful)
{
throw new ArgumentException($"{response.StatusDescription} - {response.ErrorMessage}");
}
results = response.Data;
}
sourceRepositoriesRead += results.Results.Count();
endCursor = results.EndCursor;
moreRepostoriesToRead = results.MoreToRead;
var repositoryAnalysisTasks = new List <Task>();
using (var semaphore = new SemaphoreSlim(config.Concurrency))
{
foreach (var result in results.Results)
{
// await here until there is a room for this task
await semaphore.WaitAsync();
repositoryAnalysisTasks.Add(SendAnalysisRequest(semaphore, result));
}
await Task.WhenAll(repositoryAnalysisTasks);
}
logger.Information($"Finished analyizing batch of {config.BatchSize} repositories. {sourceRepositoriesAnalyzed} respositories analyzed thus far");
} while (moreRepostoriesToRead);
stopWatch.Stop();
logger.Information($"\nAnalyized {sourceRepositoriesAnalyzed} out of {sourceRepositoriesRead} repositories in {stopWatch.Elapsed.TotalMinutes} minutes");
logger.Information($"\nThere were {repositoryAnalysisErrors.Count} analyisis errors");
foreach (var repositoryAnalysisError in repositoryAnalysisErrors)
{
logger.Error($"{repositoryAnalysisError.repoName} - {repositoryAnalysisError.errorMessage}");
}
Console.WriteLine("\nExecution complete!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
catch (Exception ex)
{
logger.Error($"FATAL EXCEPTION OCCURRED! {ex.Message}");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
}
async Task SendAnalysisRequest(SemaphoreSlim semaphore, RepositorySummary repositorySummary)
{
try
{
if (config.AsOf.HasValue)
{
logger.Information($"Starting analysis of {repositorySummary.Url} as of {config.AsOf.Value.ToString("F")}");
}
else
{
logger.Information($"Starting analysis of {repositorySummary.Url}");
}
var repositoryAnalysis = new RepositoryAnalysis
{
ForceCompleteRefresh = config.RefreshAll,
RepositoryLastUpdatedOn = repositorySummary.UpdatedAt,
RepositoryId = repositorySummary.Url,
AsOf = config.AsOf
};
var request = new RestRequest("/api/repositoryanalysis/", Method.POST);
request.AddJsonBody(repositoryAnalysis);
if (sourceRepositoriesAnalyzed == 0)
{
// If no requests have been completed then set the timeout to be higher as if an organization
// is being targeted then the reading of all the team information can take a few minutes.
request.Timeout = config.FirstApiCallTimeout;
}
var response = await restClient.ExecuteTaskAsync(request);
if (!response.IsSuccessful)
{
if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("No connection could be made because the target machine actively refused it"))
{
logger.Error($"UNABLE TO REACH API!!");
if (!runningInContainer)
{
Console.WriteLine("\nPress any key to exit");
Console.ReadKey();
}
return;
}
else
{
logger.Error(response?.ErrorMessage);
repositoryAnalysisErrors.Add((repositorySummary.Url, response.StatusDescription, null));
}
}
}
catch (Exception ex)
{
repositoryAnalysisErrors.Add((repositorySummary.Url, ex.Message, ex.StackTrace));
}
finally
{
semaphore.Release();
Interlocked.Increment(ref sourceRepositoriesAnalyzed);
}
}
}