private static CosmosClient CreateCosmosClient(string connectionString, CosmosOptions cosmosOptions)
{
GuardEmptyString(connectionString, "cosmosDbConnectionString");
cosmosOptions.Guard();
var builder = new CosmosClientBuilder(connectionString);
// ConnectionPolicy のDefault値は、SDKのConnectionPolicy.csで設定されています。
// ## Connection Mode について
// Default で ConnectionMode.Direct/Protocol.Tcp で接続されます。
// もしGateway(ConnectionMode.Gateway/Protocol.Https) を使いたければ、以下メソッドを呼ぶ
// builder.UseConnectionModeGateway(maxConnectionLimit: ??);
// Default: CamelCase Serialize/Deserialize and ignore Readonly property
// TODO: 設定変更用のconfigは未実装
//var settings = JsonSerializerSettingsFactory.CreateForReadonlyIgnoreAndCamelCase();
var settings = JsonSerializerSettingsFactory.CreateForCamelCase();
builder.UseCustomJsonSerializer(new CustomizableCaseJsonSerializer(settings));
if (cosmosOptions.ThrottlingRetryOptions != null)
{
builder.UseThrottlingRetryOptions(cosmosOptions.ThrottlingRetryOptions.MaxRetryWaitTimeOnThrottledRequests,
cosmosOptions.ThrottlingRetryOptions.MaxRetryAttemptsOnThrottledRequests);
}
// multi-master support
if (!string.IsNullOrEmpty(cosmosOptions.CurrentRegion))
{
builder.UseCurrentRegion(cosmosOptions.CurrentRegion);
}
return(builder.Build());
}
private static int RunDownloadCodeChurnFromCosmosDbToJsonFiles(DownloadFromCosmosDbCommandLineArgs a)
{
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var jsonOutputProcessor = new JsonOutputProcessor(new DataConverter(), new FileStreamFactory(), logger);
var environment = new EnvironmentImpl();
var endDate = a.EndDate ?? (a.EndDate = environment.GetCurrentDateTime());
switch (a.DocumentType)
{
case DocumentType.BugDatabase:
{
var data = cosmosOutputProcessor.GetDocumentsInDateRange <WorkItem>(a.StartDate.Value, endDate.Value);
jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data);
break;
}
case DocumentType.CodeChurn:
{
var data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, endDate.Value);
jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data);
break;
}
default:
throw new ArgumentOutOfRangeException();
}
return(0);
}
/// <summary>
/// Initializes a new instance of the <see cref="SearchCommand" /> class.
/// </summary>
/// <param name="maxRunsText">The maximum runs text.</param>
/// <param name="city">The city.</param>
/// <param name="state">The state.</param>
/// <param name="zip">The zip.</param>
/// <param name="namesFilePath">The names text file path.</param>
/// <param name="options">The options.</param>
public SearchCommand(string maxRunsText, string city, string state, string zip, string namesFilePath, CommandLineOptions options)
{
int.TryParse(maxRunsText, out int maxRuns);
_maxRuns = maxRuns;
_city = city;
_state = state;
_zip = zip;
_namesFilePath = namesFilePath;
_resultOutputPath = Program.SearchResultsDirectory;
_options = options;
this.Repository = Program.Repository;
this.Configuration = Program.Configuration;
this.FindPersonController = new FindPersonController(this.Configuration);
this.Import = new Import();
this.Export = new Export();
this.Mapper = MapperFactory.Get();
this.SerializerSettings = JsonSerializerSettingsFactory.Get();
//Default Value
this.SearchWaitMs = 60000;
var waitMs = Configuration.GetValue <string>("SearchSettings:WaitMs");
int.TryParse(waitMs, out this.SearchWaitMs);
this.PeopleSearch = new PeopleSearch(Repository, FindPersonController, SerializerSettings, Mapper, Export, _resultOutputPath, SearchWaitMs);
}
/// <summary>
/// Initializes a new instance of the <see cref="ImportExportTests"/> class.
/// </summary>
public ImportExportTests()
{
this.MockRepository = MockRepositoryFactory.Get();
this.SerializerSettings = JsonSerializerSettingsFactory.Get();
this.Mapper = MapperFactory.Get();
this.MockImport = MockImportFactory.Get();
this.MockExport = MockExportFactory.Get();
this.ImportExport = new ImportExport(MockRepository.Object, this.SerializerSettings, Mapper, MockImport.Object, MockExport.Object);
}
public void WhenCreateDefaultSerializerSettingsForCosmosDBShouldSerializerHaveExpectedValuesSet()
{
var settings = JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB();
Assert.Equal(DateFormatHandling.IsoDateFormat, settings.DateFormatHandling);
Assert.Equal(DateTimeZoneHandling.RoundtripKind, settings.DateTimeZoneHandling);
Assert.Equal(DateFormatHandling.IsoDateFormat, settings.DateFormatHandling);
Assert.All(settings.Converters, x => Assert.IsAssignableFrom <StringEnumConverter>(x));
}
/// <summary>
/// Initializes a new instance of the <see cref="PeopleSearchHelper_ShouldCreatePersonSearchResult"/> class.
/// </summary>
public PersonSearchResultHelperTests()
{
MockRepository = MockRepositoryFactory.Get();
var serializerSettings = JsonSerializerSettingsFactory.Get();
var mapper = MapperFactory.Get();
this.PersonSearchResultHelper = new PersonSearchResultHelper(MockRepository.Object, serializerSettings, mapper);
}
/// <summary>
/// Initializes a new instance of the <see cref="PeopleSearchHelper_ShouldCreatePersonSearchResult"/> class.
/// </summary>
public PersonSearchRequestHelperTests()
{
MockRepository = MockRepositoryFactory.Get();
MockFindPersonController = MockFindPersonControllerFactory.Get();
MockExport = MockExportFactory.Get();
var serializerSettings = JsonSerializerSettingsFactory.Get();
var mapper = MapperFactory.Get();
var resultOutputPath = "";
this.PersonSearchRequestHelper = new PersonSearchRequestHelper(MockRepository.Object, MockFindPersonController.Object, serializerSettings, mapper, MockExport.Object, resultOutputPath);
}
/// <summary>
/// Initializes a new instance of the <see cref="PeopleSearchTests"/> class.
/// </summary>
public PeopleSearchTests()
{
this.MockRepository = MockRepositoryFactory.Get();
this.MockFindPersonController = MockFindPersonControllerFactory.Get();
this.SerializerSettings = JsonSerializerSettingsFactory.Get();
this.Mapper = MapperFactory.Get();
this.MockExport = MockExportFactory.Get();
var resultOutputPath = "";
var searchWaitMs = 60000;
this.PeopleSearch = new PeopleSearch(MockRepository.Object, MockFindPersonController.Object, SerializerSettings, Mapper, MockExport.Object, resultOutputPath, searchWaitMs);
}
/// <summary>
/// Initializes a new instance of the <see cref="ImportSearchesCommand" /> class.
/// </summary>
/// <param name="path">The path.</param>
/// <param name="options">The options.</param>
public ImportSearchesCommand(string path, CommandLineOptions options)
{
_path = path;
_options = options;
this.Repository = Program.Repository;
this.Import = new Import();
this.Export = new Export();
this.Mapper = MapperFactory.Get();
this.SerializerSettings = JsonSerializerSettingsFactory.Get();
this.ImportExport = new ImportExport(Repository, SerializerSettings, Mapper, Import, Export);
}
public static async Task <T> Handle <T>(HttpResponseMessage response)
{
var json = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
var error = JsonConvert.DeserializeObject <ErrorResponse>(json, JsonSerializerSettingsFactory.Create()) ?? ErrorResponse.NewEmpty();
throw new GiphyApiClientException($"Error with request. HTTP response code: '{response.StatusCode}'. Message: '{error.Message}'.", (int)response.StatusCode);
}
return(JsonConvert.DeserializeObject <T>(json, JsonSerializerSettingsFactory.Create()));
}
/// <summary>
/// Initializes a new instance of the <see cref="ExportPeopleCommand" /> class.
/// </summary>
/// <param name="options">The options.</param>
public ExportPeopleCommand(CommandLineOptions options)
{
_options = options;
this.Repository = Program.Repository;
this.Import = new Import();
this.Export = new Export();
this.Mapper = MapperFactory.Get();
this.SerializerSettings = JsonSerializerSettingsFactory.Get();
_fullPath = Path.Combine(Program.ExportDirectory, $"PeopleExport-{DateTime.Now}.csv");
this.ImportExport = new ImportExport(Repository, SerializerSettings, Mapper, Import, Export);
}
public static void Register(HttpConfiguration config, IContainer container)
{
// Web API configuration and services
// Web API routes
config.MapHttpAttributeRoutes();
config.Routes.MapHttpRoute("DefaultApi", "api/{controller}/{id}", new { id = RouteParameter.Optional });
config.Formatters.XmlFormatter.SupportedMediaTypes.Clear();
config.Formatters.JsonFormatter.SerializerSettings = JsonSerializerSettingsFactory.Create(container.Resolve <IComponentContext>());
config.Services.Replace(typeof(IExceptionHandler), new GlobalExceptionHandler());
config.Services.Add(typeof(IExceptionLogger), new GlobalErrorLogger());
config.EnableCors(new EnableCorsAttribute("*", "*", "GET, POST, OPTIONS, PUT, DELETE"));
}
private static int RunSonarGenericMetricsFromCosmosDb(SonarGenericMetricsCosmosDbCommandLineArgs a)
{
var logger = new ConsoleLoggerWithTimestamp();
var jsonParser = new JsonListParser <DailyCodeChurn>(new FileStreamFactory());
var jsonExporter = new JsonExporter(new FileStreamFactory());
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var environment = new EnvironmentImpl();
var converters = new MeasureConverterListBuilder(environment).Build(a);
Dictionary <DateTime, Dictionary <string, DailyCodeChurn> > data;
if (a.StartDate == null && a.EndDate == null)
{
data = cosmosOutputProcessor.GetAllDocumentsByProjectNameAndDocumentType <DailyCodeChurn>();
}
else
{
if (a.StartDate == null)
{
throw new ArgumentNullException(nameof(a.StartDate));
}
if (a.EndDate == null)
{
a.EndDate = environment.GetCurrentDateTime();
}
data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, a.EndDate.Value);
}
var processor = new SonarGenericMetricsProcessor(jsonParser, converters, jsonExporter,
new ConsoleLoggerWithTimestamp(), new DataConverter());
processor.Process(a, data);
return(0);
}
private static int RunPerforceToCosmosDbCodeChurnProcessor(P4ExtractToCosmosDbCommandLineArgs a)
{
var processWrapper = new ProcessWrapper();
var changesParser = new ChangesParser();
var describeParser = new DescribeParser();
var commandLineParser = new CommandLineParser();
var logger = new ConsoleLoggerWithTimestamp();
var stopWatch = new StopWatchWrapper();
var bugDatabaseFactory = new BugDatabaseFactory();
var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory);
var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory));
var fileSystem = new FileSystem();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var bugDatabaseProcessor = new CosmosDbBugDatabaseProcessor(bugDatabaseDllLoader, fileSystem, webRequest, logger, dataDocumentRepository, a.CosmosProjectName);
var processor = new PerforceCodeChurnProcessor(processWrapper, changesParser, describeParser, commandLineParser, bugDatabaseProcessor, logger, stopWatch, cosmosOutputProcessor, a);
processor.QueryBugDatabase();
return(processor.Extract());
}
private static int RunCodeChurnFromFiles(DataFromFilesToCosmosDbCommandLineArgs a)
{
var fileSystem = new FileSystem();
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
IDataFromFileToCosmosDb codeChurnFromFileToCosmosDb;
if (a.DocumentType == DocumentType.CodeChurn)
{
codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <DailyCodeChurn>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <DailyCodeChurn>(new FileStreamFactory()), a.Path, a.CosmosProjectName);
}
else
{
codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <WorkItem>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <WorkItem>(new FileStreamFactory()), a.Path, a.CosmosProjectName);
}
var output = codeChurnFromFileToCosmosDb.Extract();
return(output);
}
private static int RunGitToCosmosDbCodeChurnProcessor(GitExtractToCosmosDbCommandLineArgs a)
{
var processWrapper = new ProcessWrapper();
var commandLineParser = new CommandLineParser();
var gitLogParser = new GitLogParser();
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var bugDatabaseFactory = new BugDatabaseFactory();
var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory);
var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory));
var fileSystem = new FileSystem();
var jsonParser = new JsonListParser <WorkItem>(new FileStreamFactory());
var bugDatabaseProcessor = new BugDatabaseProcessor(bugDatabaseDllLoader, webRequest, fileSystem, jsonParser, logger, string.Empty);
//, a.BugDatabaseOutputFile
var processor = new GitCodeChurnProcessor(commandLineParser, processWrapper, gitLogParser, cosmosOutputProcessor, bugDatabaseProcessor, logger, a);
processor.QueryBugDatabase();
return(processor.Extract());
}
protected virtual JsonSerializerSettings CreateJsonSerializerSettings() => JsonSerializerSettingsFactory?.Invoke();
/// <summary>
/// Stores a list of statements to the LRS
/// </summary>
/// <param name="statements">The list of statements to store</param>
public async Task SendStatementAsync(IEnumerable <Statement> statements)
{
using (var content = new StringContent(JsonConvert.SerializeObject(statements.ToList(), JsonSerializerSettingsFactory.CreateSettings()), Encoding.UTF8, "application/json"))
{
using (var response = await HttpClient.PostAsync($"{_statementEndPoint}", content))
{
if (!response.IsSuccessStatusCode)
{
var responseContent = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Could not send statements.\r\nLRS responded with: \r\n{(int)response.StatusCode} {response.ReasonPhrase}\r\n{responseContent}");
}
}
}
}
/// <summary>
/// Main application entry point.
/// </summary>
/// <param name="args">The arguments.</param>
/// <returns></returns>
public static int Main(string[] args)
{
#region Test args for debugging
//args = new string[2] { "exportpeople" };
//args = new string[2] { "importsearches", @"C:\Users\dunca\Desktop\RaleighSwahiliNamesSearch\export.csv" };
//args = new string[7] { "search", "", "NC", "", @"C:\Users\dunca\Desktop\RaleighSwahiliNamesSearch\SwahiliNames.A.csv", "100" };
#endregion
#region Configure Logging
var logsFolder = Path.Combine(Environment.CurrentDirectory, "logs");
Directory.CreateDirectory(logsFolder);
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.Enrich.FromLogContext()
.WriteTo.RollingFile("logs/namesearch-{Date}.log")
.WriteTo.Console()
.CreateLogger();
#endregion Configure Logging
var log = Log.Logger;
try
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json");
Configuration = builder.Build();
#region Create Directories
ExportDirectory = Path.Combine(Environment.CurrentDirectory, "Exports");
Directory.CreateDirectory(ExportDirectory);
SearchResultsDirectory = Path.Combine(Environment.CurrentDirectory, "SearchResults");
Directory.CreateDirectory(SearchResultsDirectory);
#endregion
#region Dependency Injection Container
// use Dependency injection in console app https://andrewlock.net/using-dependency-injection-in-a-net-core-console-application/
// add the framework services
var services = new ServiceCollection()
.AddLogging(loggingBuilder => loggingBuilder.AddSerilog(dispose: true))
.AddDbContext <ApplicationDbContext>(optionsBuilder => optionsBuilder.UseSqlite("Data Source=blog.db"))
.AddTransient <IMapper, IMapper>((ctx) =>
{
return(MapperFactory.Get());
})
.AddTransient <JsonSerializerSettings, JsonSerializerSettings>((ctx) =>
{
return(JsonSerializerSettingsFactory.Get());
})
.AddScoped <IEntityFrameworkRepository, EntityFrameworkRepository>();
services.AddMvc();
// add StructureMap
var container = new Container();
container.Configure(config =>
{
// Register stuff in container, using the StructureMap APIs...
config.Scan(_ =>
{
_.AssemblyContainingType(typeof(Program));
_.WithDefaultConventions();
});
// Populate the container using the service collection
config.Populate(services);
});
//Set static instances
Repository = container.GetInstance <IEntityFrameworkRepository>();
Mapper = container.GetInstance <IMapper>();
SerializerSettings = container.GetInstance <JsonSerializerSettings>();
#endregion Dependency Injection Container
var options = CommandLineOptions.Parse(args);
log.InformationEvent("Main", "Run command {command} with arguments {args}", options?.Command, args);
if (options?.Command == null)
{
// RootCommand will have printed help
return(1);
}
var result = options.Command.Run();
return(result);
}
catch (Exception ex)
{
log.FatalEvent(ex, "Main", "Fatal Application Failure with message {message}", ex.Message);
throw;
}
finally
{
log.InformationEvent("Main", "Application Ending");
Log.CloseAndFlush();
// Allow users to see output. Prevent console closing on its own
Console.ReadLine();
}
}
/// <summary>
/// Creates a JSON representation of the Actor based on the Experience API
/// specification.
/// </summary>
/// <param name="prettyPrint">If set to true, the json will have indentation for easier readability.</param>
/// <returns>The JSON representation of the Actor</returns>
public string ToJson(bool prettyPrint = false)
{
return(JsonConvert.SerializeObject(this, prettyPrint ? Formatting.Indented : Formatting.None, JsonSerializerSettingsFactory.CreateSettings()));
}
