public static async Task ProvisionResources(this IBlobClient client, IHost host, CancellationToken token = default)
{
var configuration = host.Services.GetRequiredService <IConfiguration>();
switch (client)
{
case S3BlobClient _:
if (Environment.GetEnvironmentVariable("MINIO_SERVER") != null)
{
var s3Client = host.Services.GetRequiredService <AmazonS3Client>();
var s3Options = new S3BlobClientOptions();
configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
var buckets = await s3Client.ListBucketsAsync(token);
if (!buckets.Buckets.Exists(bucket => bucket.BucketName == s3Options.Buckets[WellknownBuckets.ImportLegacyBucket]))
{
await s3Client.PutBucketAsync(s3Options.Buckets[WellknownBuckets.ImportLegacyBucket], token);
}
}
break;
case FileBlobClient _:
var fileOptions = new FileBlobClientOptions();
configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
if (!Directory.Exists(fileOptions.Directory))
{
Directory.CreateDirectory(fileOptions.Directory);
}
break;
}
}
private static async Task Main(string[] args)
{
AppDomain.CurrentDomain.FirstChanceException += (sender, eventArgs) =>
Log.Debug(eventArgs.Exception, "FirstChanceException event raised in {AppDomain}.", AppDomain.CurrentDomain.FriendlyName);
AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) =>
Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program.");
var host = new HostBuilder()
.ConfigureHostConfiguration(builder => {
builder
.AddEnvironmentVariables("DOTNET_")
.AddEnvironmentVariables("ASPNETCORE_");
})
.ConfigureAppConfiguration((hostContext, builder) =>
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
if (hostContext.HostingEnvironment.IsProduction())
{
builder
.SetBasePath(Directory.GetCurrentDirectory());
}
builder
.AddJsonFile("appsettings.json", true, false)
.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName.ToLowerInvariant()}.json", true, false)
.AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", true, false)
.AddEnvironmentVariables()
.AddCommandLine(args);
})
.ConfigureLogging((hostContext, builder) =>
{
Serilog.Debugging.SelfLog.Enable(Console.WriteLine);
var loggerConfiguration = new LoggerConfiguration()
.ReadFrom.Configuration(hostContext.Configuration)
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.Enrich.WithThreadId()
.Enrich.WithEnvironmentUserName();
Log.Logger = loggerConfiguration.CreateLogger();
builder.AddSerilog(Log.Logger);
})
.ConfigureServices((hostContext, builder) =>
{
var blobOptions = new BlobClientOptions();
hostContext.Configuration.Bind(blobOptions);
switch (blobOptions.BlobClientType)
{
case nameof(S3BlobClient):
var s3Options = new S3BlobClientOptions();
hostContext.Configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
// Use MINIO
if (hostContext.Configuration.GetValue <string>("MINIO_SERVER") != null)
{
if (hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY") == null)
{
throw new Exception("The MINIO_ACCESS_KEY configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY") == null)
{
throw new Exception("The MINIO_SECRET_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY"),
hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY")),
new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1, // minio's default region
ServiceURL = hostContext.Configuration.GetValue <string>("MINIO_SERVER"),
ForcePathStyle = true
}
)
);
}
else // Use AWS
{
if (hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID") == null)
{
throw new Exception("The AWS_ACCESS_KEY_ID configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY") == null)
{
throw new Exception("The AWS_SECRET_ACCESS_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID"),
hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY"))
)
);
}
builder.AddSingleton <IBlobClient>(sp =>
new S3BlobClient(
sp.GetService <AmazonS3Client>(),
s3Options.Buckets[WellknownBuckets.ImportLegacyBucket]
)
);
break;
case nameof(FileBlobClient):
var fileOptions = new FileBlobClientOptions();
hostContext.Configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
builder.AddSingleton <IBlobClient>(sp =>
new FileBlobClient(
new DirectoryInfo(fileOptions.Directory)
)
);
break;
default:
throw new Exception(blobOptions.BlobClientType + " is not a supported blob client type.");
}
builder
.AddSingleton <WellKnownBinaryReader>()
.AddSingleton <RecyclableMemoryStreamManager>()
.AddSingleton <IClock>(SystemClock.Instance)
.AddSingleton <IEventReader, LegacyEventReader>()
.AddSingleton <LegacyStreamArchiveWriter>()
.AddSingleton(
new SqlConnection(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.Legacy)
)
);
})
.Build();
var configuration = host.Services.GetRequiredService <IConfiguration>();
var logger = host.Services.GetRequiredService <ILogger <Program> >();
var reader = host.Services.GetRequiredService <IEventReader>();
var writer = host.Services.GetRequiredService <LegacyStreamArchiveWriter>();
var blobClient = host.Services.GetRequiredService <IBlobClient>();
var blobClientOptions = new BlobClientOptions();
configuration.Bind(blobClientOptions);
try
{
await WaitFor.SeqToBecomeAvailable(configuration);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.Legacy);
logger.LogBlobClientCredentials(blobClientOptions);
await WaitFor.SqlServerToBecomeAvailable(
new SqlConnectionStringBuilder(configuration.GetConnectionString(WellknownConnectionNames.Legacy))
, logger);
await OptimizeDatabasePerformance(
new SqlConnectionStringBuilder(
configuration.GetConnectionString(WellknownConnectionNames.Legacy)), logger);
await blobClient.ProvisionResources(host);
using (var connection = host.Services.GetRequiredService <SqlConnection>())
{
await connection.OpenAsync();
await writer.WriteAsync(reader.ReadEvents(connection));
}
}
catch (Exception exception)
{
logger.LogCritical(exception, "Encountered a fatal exception, exiting program.");
}
finally
{
Log.CloseAndFlush();
}
}
public static async Task Main(string[] args)
{
Console.WriteLine("Starting RoadRegistry.Product.ProjectionHost");
AppDomain.CurrentDomain.FirstChanceException += (sender, eventArgs) =>
Log.Debug(eventArgs.Exception, "FirstChanceException event raised in {AppDomain}.", AppDomain.CurrentDomain.FriendlyName);
AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) =>
Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program.");
var host = new HostBuilder()
.ConfigureHostConfiguration(builder => {
builder
.AddEnvironmentVariables("DOTNET_")
.AddEnvironmentVariables("ASPNETCORE_");
})
.ConfigureAppConfiguration((hostContext, builder) =>
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
if (hostContext.HostingEnvironment.IsProduction())
{
builder
.SetBasePath(Directory.GetCurrentDirectory());
}
builder
.AddJsonFile("appsettings.json", true, false)
.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName.ToLowerInvariant()}.json", true, false)
.AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", true, false)
.AddEnvironmentVariables()
.AddCommandLine(args);
})
.ConfigureLogging((hostContext, builder) =>
{
Serilog.Debugging.SelfLog.Enable(Console.WriteLine);
var loggerConfiguration = new LoggerConfiguration()
.ReadFrom.Configuration(hostContext.Configuration)
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.Enrich.WithThreadId()
.Enrich.WithEnvironmentUserName();
Log.Logger = loggerConfiguration.CreateLogger();
builder.AddSerilog(Log.Logger);
})
.ConfigureServices((hostContext, builder) =>
{
var blobOptions = new BlobClientOptions();
hostContext.Configuration.Bind(blobOptions);
switch (blobOptions.BlobClientType)
{
case nameof(S3BlobClient):
var s3Options = new S3BlobClientOptions();
hostContext.Configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
// Use MINIO
if (hostContext.Configuration.GetValue <string>("MINIO_SERVER") != null)
{
if (hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY") == null)
{
throw new Exception("The MINIO_ACCESS_KEY configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY") == null)
{
throw new Exception("The MINIO_SECRET_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY"),
hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY")),
new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1, // minio's default region
ServiceURL = hostContext.Configuration.GetValue <string>("MINIO_SERVER"),
ForcePathStyle = true
}
)
);
}
else // Use AWS
{
if (hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID") == null)
{
throw new Exception("The AWS_ACCESS_KEY_ID configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY") == null)
{
throw new Exception("The AWS_SECRET_ACCESS_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID"),
hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY"))
)
);
}
builder.AddSingleton <IBlobClient>(sp =>
new S3BlobClient(
sp.GetService <AmazonS3Client>(),
s3Options.Buckets[WellknownBuckets.UploadsBucket]
)
);
break;
case nameof(FileBlobClient):
var fileOptions = new FileBlobClientOptions();
hostContext.Configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
builder.AddSingleton <IBlobClient>(sp =>
new FileBlobClient(
new DirectoryInfo(fileOptions.Directory)
)
);
break;
default:
throw new Exception(blobOptions.BlobClientType + " is not a supported blob client type.");
}
builder
.AddSingleton <IClock>(SystemClock.Instance)
.AddSingleton <Scheduler>()
.AddHostedService <EventProcessor>()
.AddSingleton(new RecyclableMemoryStreamManager())
.AddSingleton(new EnvelopeFactory(
EventProcessor.EventMapping,
new EventDeserializer((eventData, eventType) =>
JsonConvert.DeserializeObject(eventData, eventType, EventProcessor.SerializerSettings)))
)
.AddSingleton <Func <ProductContext> >(
() =>
new ProductContext(
new DbContextOptionsBuilder <ProductContext>()
.UseSqlServer(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.ProductProjections),
options => options.EnableRetryOnFailure()
).Options)
)
.AddSingleton(sp => new ConnectedProjection <ProductContext>[]
{
new OrganizationRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new GradeSeparatedJunctionRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadNetworkInfoProjection(),
new RoadNodeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentEuropeanRoadAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentLaneAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentNationalRoadAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentNumberedRoadAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentSurfaceAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding),
new RoadSegmentWidthAttributeRecordProjection(sp.GetRequiredService <RecyclableMemoryStreamManager>(), WindowsAnsiEncoding)
})
.AddSingleton(sp =>
Be.Vlaanderen.Basisregisters.ProjectionHandling.Connector.Resolve
.WhenEqualToHandlerMessageType(
sp.GetRequiredService <ConnectedProjection <ProductContext>[]>()
.SelectMany(projection => projection.Handlers)
.ToArray()
)
)
.AddSingleton(sp => AcceptStreamMessage.WhenEqualToMessageType(sp.GetRequiredService <ConnectedProjection <ProductContext>[]>(), EventProcessor.EventMapping))
.AddSingleton <IStreamStore>(sp =>
new MsSqlStreamStore(
new MsSqlStreamStoreSettings(
sp
.GetService <IConfiguration>()
.GetConnectionString(WellknownConnectionNames.Events)
)
{
Schema = WellknownSchemas.EventSchema
}))
.AddSingleton <IRunnerDbContextMigratorFactory>(new ProductContextMigrationFactory());
})
.Build();
var migratorFactory = host.Services.GetRequiredService <IRunnerDbContextMigratorFactory>();
var configuration = host.Services.GetRequiredService <IConfiguration>();
var streamStore = host.Services.GetRequiredService <IStreamStore>();
var loggerFactory = host.Services.GetRequiredService <ILoggerFactory>();
var logger = host.Services.GetRequiredService <ILogger <Program> >();
var blobClientOptions = new BlobClientOptions();
configuration.Bind(blobClientOptions);
try
{
await WaitFor.SeqToBecomeAvailable(configuration).ConfigureAwait(false);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.Events);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.ProductProjections);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.ProductProjectionsAdmin);
logger.LogBlobClientCredentials(blobClientOptions);
await DistributedLock <Program> .RunAsync(async() =>
{
await WaitFor.SqlStreamStoreToBecomeAvailable(streamStore, logger).ConfigureAwait(false);
await migratorFactory.CreateMigrator(configuration, loggerFactory)
.MigrateAsync(CancellationToken.None).ConfigureAwait(false);
await host.RunAsync().ConfigureAwait(false);
},
DistributedLockOptions.LoadFromConfiguration(configuration),
logger)
.ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogCritical(e, "Encountered a fatal exception, exiting program.");
}
finally
{
Log.CloseAndFlush();
}
}
private static async Task Main(string[] args)
{
AppDomain.CurrentDomain.FirstChanceException += (sender, eventArgs) =>
Log.Debug(eventArgs.Exception, "FirstChanceException event raised in {AppDomain}.", AppDomain.CurrentDomain.FriendlyName);
AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) =>
Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program.");
var host = new HostBuilder()
.ConfigureHostConfiguration(builder => {
builder
.AddEnvironmentVariables("DOTNET_")
.AddEnvironmentVariables("ASPNETCORE_");
})
.ConfigureAppConfiguration((hostContext, builder) =>
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
if (hostContext.HostingEnvironment.IsProduction())
{
builder
.SetBasePath(Directory.GetCurrentDirectory());
}
builder
.AddJsonFile("appsettings.json", true, false)
.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName.ToLowerInvariant()}.json", true, false)
.AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", true, false)
.AddEnvironmentVariables()
.AddCommandLine(args)
.Build();
})
.ConfigureLogging((hostContext, builder) =>
{
Serilog.Debugging.SelfLog.Enable(Console.WriteLine);
var loggerConfiguration = new LoggerConfiguration()
.ReadFrom.Configuration(hostContext.Configuration)
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.Enrich.WithThreadId()
.Enrich.WithEnvironmentUserName();
Log.Logger = loggerConfiguration.CreateLogger();
builder.AddSerilog(Log.Logger);
})
.ConfigureServices((hostContext, builder) =>
{
var blobOptions = new BlobClientOptions();
hostContext.Configuration.Bind(blobOptions);
switch (blobOptions.BlobClientType)
{
case nameof(S3BlobClient):
var s3Options = new S3BlobClientOptions();
hostContext.Configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
// Use MINIO
if (hostContext.Configuration.GetValue <string>("MINIO_SERVER") != null)
{
if (hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY") == null)
{
throw new Exception("The MINIO_ACCESS_KEY configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY") == null)
{
throw new Exception("The MINIO_SECRET_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY"),
hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY")),
new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1, // minio's default region
ServiceURL = hostContext.Configuration.GetValue <string>("MINIO_SERVER"),
ForcePathStyle = true
}
)
);
}
else // Use AWS
{
if (hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID") == null)
{
throw new Exception("The AWS_ACCESS_KEY_ID configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY") == null)
{
throw new Exception("The AWS_SECRET_ACCESS_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID"),
hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY"))
)
);
}
builder.AddSingleton <IBlobClient>(sp =>
new S3BlobClient(
sp.GetService <AmazonS3Client>(),
s3Options.Buckets[WellknownBuckets.ImportLegacyBucket]
)
);
break;
case nameof(FileBlobClient):
var fileOptions = new FileBlobClientOptions();
hostContext.Configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
builder.AddSingleton <IBlobClient>(sp =>
new FileBlobClient(
new DirectoryInfo(fileOptions.Directory)
)
);
break;
default:
throw new Exception(blobOptions.BlobClientType + " is not a supported blob client type.");
}
var legacyStreamArchiveReader = new LegacyStreamArchiveReader(
new JsonSerializerSettings
{
Formatting = Formatting.Indented,
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Unspecified,
DateParseHandling = DateParseHandling.DateTime,
DefaultValueHandling = DefaultValueHandling.Ignore
}
);
builder
.AddSingleton(legacyStreamArchiveReader)
.AddSingleton(
new SqlConnection(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.Events)
)
)
.AddSingleton <IStreamStore>(new MsSqlStreamStore(
new MsSqlStreamStoreSettings(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.Events)
)
{
Schema = WellknownSchemas.EventSchema
}))
.AddSingleton <LegacyStreamEventsWriter>();
})
.Build();
var configuration = host.Services.GetRequiredService <IConfiguration>();
var logger = host.Services.GetRequiredService <ILogger <Program> >();
var reader = host.Services.GetRequiredService <LegacyStreamArchiveReader>();
var writer = host.Services.GetRequiredService <LegacyStreamEventsWriter>();
var client = host.Services.GetRequiredService <IBlobClient>();
var streamStore = host.Services.GetRequiredService <IStreamStore>();
var blobClientOptions = new BlobClientOptions();
configuration.Bind(blobClientOptions);
try
{
await WaitFor.SeqToBecomeAvailable(configuration).ConfigureAwait(false);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.Events);
logger.LogBlobClientCredentials(blobClientOptions);
await DistributedLock <Program> .RunAsync(async() =>
{
var eventsConnectionStringBuilder =
new SqlConnectionStringBuilder(
configuration.GetConnectionString(WellknownConnectionNames.Events));
var masterConnectionStringBuilder =
new SqlConnectionStringBuilder(eventsConnectionStringBuilder.ConnectionString)
{
InitialCatalog = "master"
};
await WaitFor.SqlServerToBecomeAvailable(masterConnectionStringBuilder, logger).ConfigureAwait(false);
await WaitFor.SqlServerDatabaseToBecomeAvailable(
masterConnectionStringBuilder,
eventsConnectionStringBuilder,
logger
).ConfigureAwait(false);
if (streamStore is MsSqlStreamStore sqlStreamStore)
{
await sqlStreamStore.CreateSchema().ConfigureAwait(false);
}
var page = await streamStore
.ReadStreamForwards(RoadNetworks.Stream, StreamVersion.Start, 1)
.ConfigureAwait(false);
if (page.Status == PageReadStatus.StreamNotFound)
{
var blob = await client
.GetBlobAsync(new BlobName("import-streams.zip"), CancellationToken.None)
.ConfigureAwait(false);
var watch = Stopwatch.StartNew();
using (var blobStream = await blob.OpenAsync().ConfigureAwait(false))
{
await writer
.WriteAsync(reader.Read(blobStream))
.ConfigureAwait(false);
}
logger.LogInformation("Total append took {0}ms", watch.ElapsedMilliseconds);
}
else
{
logger.LogWarning(
"The road network was previously imported. This can only be performed once.");
}
},
DistributedLockOptions.LoadFromConfiguration(configuration),
logger)
.ConfigureAwait(false);
}
catch (Exception exception)
{
logger.LogCritical(exception, "Encountered a fatal exception, exiting program.");
}
finally
{
Log.CloseAndFlush();
}
}
public static IWebHostBuilder CreateWebHostBuilder(string[] args)
=> new WebHostBuilder()
.UseDefaultForApi <Startup>(
new ProgramOptions
{
Hosting =
{
HttpPort = 5000
},
Logging =
{
WriteTextToConsole = false,
WriteJsonToConsole = false
},
Runtime =
{
CommandLineArgs = args
}
})
.ConfigureServices((hostContext, builder) =>
{
var blobOptions = new BlobClientOptions();
hostContext.Configuration.Bind(blobOptions);
switch (blobOptions.BlobClientType)
{
case nameof(S3BlobClient):
var s3Options = new S3BlobClientOptions();
hostContext.Configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
// Use MINIO
if (Environment.GetEnvironmentVariable("MINIO_SERVER") != null)
{
if (Environment.GetEnvironmentVariable("MINIO_ACCESS_KEY") == null)
{
throw new Exception("The MINIO_ACCESS_KEY environment variable was not set.");
}
if (Environment.GetEnvironmentVariable("MINIO_SECRET_KEY") == null)
{
throw new Exception("The MINIO_SECRET_KEY environment variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
Environment.GetEnvironmentVariable("MINIO_ACCESS_KEY"),
Environment.GetEnvironmentVariable("MINIO_SECRET_KEY")),
new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1, // minio's default region
ServiceURL = Environment.GetEnvironmentVariable("MINIO_SERVER"),
ForcePathStyle = true
}
)
);
}
else // Use AWS
{
if (Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID") == null)
{
throw new Exception("The AWS_ACCESS_KEY_ID environment variable was not set.");
}
if (Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY") == null)
{
throw new Exception("The AWS_SECRET_ACCESS_KEY environment variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID"),
Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY"))
)
);
}
builder.AddSingleton <IBlobClient>(sp =>
new S3BlobClient(
sp.GetService <AmazonS3Client>(),
s3Options.Buckets[WellknownBuckets.UploadsBucket]
)
);
break;
case nameof(FileBlobClient):
var fileOptions = new FileBlobClientOptions();
hostContext.Configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
builder.AddSingleton <IBlobClient>(sp =>
new FileBlobClient(
new DirectoryInfo(fileOptions.Directory)
)
);
break;
default:
throw new Exception(blobOptions.BlobClientType + " is not a supported blob client type.");
}
builder
.AddSingleton <IStreamStore>(sp =>
new MsSqlStreamStore(
new MsSqlStreamStoreSettings(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.Events))
{
Schema = WellknownSchemas.EventSchema
}))
.AddSingleton <IClock>(SystemClock.Instance)
.AddSingleton(new RecyclableMemoryStreamManager())
.AddSingleton(sp => new RoadNetworkSnapshotReaderWriter(
new SqlBlobClient(
new SqlConnectionStringBuilder(
hostContext.Configuration.GetConnectionString(WellknownConnectionNames.Snapshots)),
WellknownSchemas.SnapshotSchema),
sp.GetService <RecyclableMemoryStreamManager>()))
.AddSingleton <IRoadNetworkSnapshotReader>(sp =>
sp.GetRequiredService <RoadNetworkSnapshotReaderWriter>())
.AddSingleton <IRoadNetworkSnapshotWriter>(sp =>
sp.GetRequiredService <RoadNetworkSnapshotReaderWriter>())
.AddSingleton(sp => Dispatch.Using(Resolve.WhenEqualToMessage(
new CommandHandlerModule[]
{
new RoadNetworkChangesArchiveCommandModule(
sp.GetService <IBlobClient>(),
sp.GetService <IStreamStore>(),
sp.GetService <IRoadNetworkSnapshotReader>(),
new ZipArchiveValidator(Encoding.GetEncoding(1252)),
sp.GetService <IClock>()
),
new RoadNetworkCommandModule(
sp.GetService <IStreamStore>(),
sp.GetService <IRoadNetworkSnapshotReader>(),
sp.GetService <IClock>()
)
})))
.AddScoped(sp => new TraceDbConnection <BackOfficeContext>(
new SqlConnection(sp.GetRequiredService <IConfiguration>().GetConnectionString(WellknownConnectionNames.BackOfficeProjections)),
sp.GetRequiredService <IConfiguration>()["DataDog:ServiceName"]))
.AddDbContext <BackOfficeContext>((sp, options) => options
.UseLoggerFactory(sp.GetService <ILoggerFactory>())
.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking)
.UseSqlServer(
sp.GetRequiredService <TraceDbConnection <BackOfficeContext> >(),
sql => sql.EnableRetryOnFailure())
);
});
public static async Task Main(string[] args)
{
Console.WriteLine("Starting RoadRegistry.BackOffice.EventHost");
AppDomain.CurrentDomain.FirstChanceException += (sender, eventArgs) =>
Log.Debug(eventArgs.Exception, "FirstChanceException event raised in {AppDomain}.", AppDomain.CurrentDomain.FriendlyName);
AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) =>
Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program.");
var host = new HostBuilder()
.ConfigureHostConfiguration(builder => {
builder
.AddEnvironmentVariables("DOTNET_")
.AddEnvironmentVariables("ASPNETCORE_");
})
.ConfigureAppConfiguration((hostContext, builder) =>
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
if (hostContext.HostingEnvironment.IsProduction())
{
builder
.SetBasePath(Directory.GetCurrentDirectory());
}
builder
.AddJsonFile("appsettings.json", true, false)
.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName.ToLowerInvariant()}.json", true, false)
.AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", true, false)
.AddEnvironmentVariables()
.AddCommandLine(args);
})
.ConfigureLogging((hostContext, builder) =>
{
Serilog.Debugging.SelfLog.Enable(Console.WriteLine);
var loggerConfiguration = new LoggerConfiguration()
.ReadFrom.Configuration(hostContext.Configuration)
.Enrich.FromLogContext()
.Enrich.WithMachineName()
.Enrich.WithThreadId()
.Enrich.WithEnvironmentUserName();
Log.Logger = loggerConfiguration.CreateLogger();
builder.AddSerilog(Log.Logger);
})
.ConfigureServices((hostContext, builder) =>
{
var blobOptions = new BlobClientOptions();
hostContext.Configuration.Bind(blobOptions);
switch (blobOptions.BlobClientType)
{
case nameof(S3BlobClient):
var s3Options = new S3BlobClientOptions();
hostContext.Configuration.GetSection(nameof(S3BlobClientOptions)).Bind(s3Options);
// Use MINIO
if (hostContext.Configuration.GetValue <string>("MINIO_SERVER") != null)
{
if (hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY") == null)
{
throw new Exception("The MINIO_ACCESS_KEY configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY") == null)
{
throw new Exception("The MINIO_SECRET_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("MINIO_ACCESS_KEY"),
hostContext.Configuration.GetValue <string>("MINIO_SECRET_KEY")),
new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.USEast1, // minio's default region
ServiceURL = hostContext.Configuration.GetValue <string>("MINIO_SERVER"),
ForcePathStyle = true
}
)
);
}
else // Use AWS
{
if (hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID") == null)
{
throw new Exception("The AWS_ACCESS_KEY_ID configuration variable was not set.");
}
if (hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY") == null)
{
throw new Exception("The AWS_SECRET_ACCESS_KEY configuration variable was not set.");
}
builder.AddSingleton(new AmazonS3Client(
new BasicAWSCredentials(
hostContext.Configuration.GetValue <string>("AWS_ACCESS_KEY_ID"),
hostContext.Configuration.GetValue <string>("AWS_SECRET_ACCESS_KEY"))
)
);
}
builder.AddSingleton <IBlobClient>(sp =>
new S3BlobClient(
sp.GetService <AmazonS3Client>(),
s3Options.Buckets[WellknownBuckets.UploadsBucket]
)
);
break;
case nameof(FileBlobClient):
var fileOptions = new FileBlobClientOptions();
hostContext.Configuration.GetSection(nameof(FileBlobClientOptions)).Bind(fileOptions);
builder.AddSingleton <IBlobClient>(sp =>
new FileBlobClient(
new DirectoryInfo(fileOptions.Directory)
)
);
break;
default:
throw new Exception(blobOptions.BlobClientType + " is not a supported blob client type.");
}
builder
.AddSingleton <Scheduler>()
.AddHostedService <EventProcessor>()
.AddSingleton <IEventProcessorPositionStore>(sp =>
new SqlEventProcessorPositionStore(
new SqlConnectionStringBuilder(
sp.GetService <IConfiguration>().GetConnectionString(WellknownConnectionNames.EventHost)
),
WellknownSchemas.EventHostSchema))
.AddSingleton <IStreamStore>(sp =>
new MsSqlStreamStore(
new MsSqlStreamStoreSettings(
sp
.GetService <IConfiguration>()
.GetConnectionString(WellknownConnectionNames.Events)
)
{
Schema = WellknownSchemas.EventSchema
}))
.AddSingleton <IClock>(SystemClock.Instance)
.AddSingleton(new RecyclableMemoryStreamManager())
.AddSingleton(sp => new RoadNetworkSnapshotReaderWriter(
new SqlBlobClient(
new SqlConnectionStringBuilder(
sp
.GetService <IConfiguration>()
.GetConnectionString(WellknownConnectionNames.Snapshots)
), WellknownSchemas.SnapshotSchema),
sp.GetService <RecyclableMemoryStreamManager>()))
.AddSingleton <IRoadNetworkSnapshotReader>(sp => sp.GetRequiredService <RoadNetworkSnapshotReaderWriter>())
.AddSingleton <IRoadNetworkSnapshotWriter>(sp => sp.GetRequiredService <RoadNetworkSnapshotReaderWriter>())
.AddSingleton(sp => new EventHandlerModule[]
{
new RoadNetworkChangesArchiveEventModule(
sp.GetService <IBlobClient>(),
new ZipArchiveTranslator(Encoding.UTF8),
sp.GetService <IStreamStore>()
),
new RoadNetworkEventModule(
sp.GetService <IStreamStore>(),
sp.GetService <IRoadNetworkSnapshotReader>(),
sp.GetService <IRoadNetworkSnapshotWriter>(),
sp.GetService <IClock>())
})
.AddSingleton(sp => AcceptStreamMessage.WhenEqualToMessageType(sp.GetRequiredService <EventHandlerModule[]>(), EventProcessor.EventMapping))
.AddSingleton(sp => Dispatch.Using(Resolve.WhenEqualToMessage(sp.GetRequiredService <EventHandlerModule[]>())));
})
.Build();
var configuration = host.Services.GetRequiredService <IConfiguration>();
var streamStore = host.Services.GetRequiredService <IStreamStore>();
var logger = host.Services.GetRequiredService <ILogger <Program> >();
var blobClient = host.Services.GetRequiredService <IBlobClient>();
var blobClientOptions = new BlobClientOptions();
configuration.Bind(blobClientOptions);
try
{
await WaitFor.SeqToBecomeAvailable(configuration).ConfigureAwait(false);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.Events);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.EventHost);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.EventHostAdmin);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.Snapshots);
logger.LogSqlServerConnectionString(configuration, WellknownConnectionNames.SnapshotsAdmin);
logger.LogBlobClientCredentials(blobClientOptions);
await DistributedLock <Program> .RunAsync(async() =>
{
await WaitFor.SqlStreamStoreToBecomeAvailable(streamStore, logger).ConfigureAwait(false);
await
new SqlBlobSchema(
new SqlConnectionStringBuilder(configuration.GetConnectionString(WellknownConnectionNames.SnapshotsAdmin))
).CreateSchemaIfNotExists(WellknownSchemas.SnapshotSchema).ConfigureAwait(false);
await
new SqlEventProcessorPositionStoreSchema(
new SqlConnectionStringBuilder(configuration.GetConnectionString(WellknownConnectionNames.EventHostAdmin))
).CreateSchemaIfNotExists(WellknownSchemas.EventHostSchema).ConfigureAwait(false);
await blobClient.ProvisionResources(host).ConfigureAwait(false);
await host.RunAsync().ConfigureAwait(false);
},
DistributedLockOptions.LoadFromConfiguration(configuration),
logger)
.ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogCritical(e, "Encountered a fatal exception, exiting program.");
}
finally
{
Log.CloseAndFlush();
}
}
