public void Sort(string fromLabel, string toLabel, IComparer <T> sort, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var streams = GetAllSegments(fromLabel);
var data = new Queue <Stream>(streams);
for (var i = 0; i < data.Count; i++)
{
using (var stream = data.Dequeue())
{
using var sw = new StreamWriter(CreateSegment(toLabel, i));
var items = Read(stream, sort, logger, metrics, cancellationToken);
foreach (var item in items)
{
var line = _serialize(item);
sw.WriteLine(line);
}
sw.Flush();
}
DeleteSegment(fromLabel, i);
}
}
public DocumentDbBackgroundTaskStore(IDocumentDbRepository <BackgroundTaskDocument> repository,
IServerTimestampService timestamps, ISafeLogger <DocumentDbBackgroundTaskStore> logger)
{
_repository = repository;
_timestamps = timestamps;
_logger = logger;
}
public DocumentDbRepository(string slot, IOptionsMonitor <DocumentDbOptions> options,
ISafeLogger <DocumentDbRepository <T> > logger)
{
_reads = ReadAccessor.Create(typeof(T));
_writes = WriteAccessor.Create(typeof(T));
_slot = slot;
_options = options;
_logger = logger;
var defaultSettings = new JsonSerializerSettings();
var documentDbOptions = options.Get(_slot);
_client = new DocumentClient(EndpointUri, documentDbOptions.AccountKey, defaultSettings);
_client2 = new CosmosClient(EndpointUri.ToString(), documentDbOptions.AccountKey,
new CosmosClientOptions
{
AllowBulkExecution = true,
ConnectionMode = ConnectionMode.Gateway,
MaxRetryAttemptsOnRateLimitedRequests = 100
});
CreateDatabaseIfNotExistsAsync().Wait();
CreateCollectionIfNotExistsAsync().Wait();
}
public static IServiceCollection AddSecurityPolicies(this IServiceCollection services,
IConfiguration securityConfig,
IConfiguration superUserConfig,
ISafeLogger logger)
{
return(AddSecurityPolicies(services, securityConfig.FastBind, superUserConfig.FastBind, logger));
}
public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
unsafe
{
var pendingLength = 0;
var buffer = Constants.Buffer;
return(ReadOrCountLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));
void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding x)
{
var target = new Span <byte>(buffer, pendingLength, length);
var segment = new ReadOnlySpan <byte>(start, length);
segment.CopyTo(target);
if (partial)
{
pendingLength = length;
}
else
{
var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength);
onNewLine?.Invoke(lineNumber, encoding.GetString(line));
pendingLength = 0;
}
}
}
}
private static void AddHttps(this IServiceCollection services, ISafeLogger logger, SecurityOptions options)
{
if (options.Https.Enabled)
{
logger?.Trace(() => "HTTPS enabled.");
services.AddHttpsRedirection(o =>
{
o.HttpsPort = null;
o.RedirectStatusCode = options.Https.Hsts.Enabled ? 307 : 301;
});
if (options.Https.Hsts.Enabled)
{
logger?.Trace(() => "HSTS enabled.");
services.AddHsts(o =>
{
o.MaxAge = options.Https.Hsts.HstsMaxAge;
o.IncludeSubDomains = options.Https.Hsts.IncludeSubdomains;
o.Preload = options.Https.Hsts.Preload;
});
}
}
}
public BackgroundTaskHost(
IServiceProvider backgroundServices,
IBackgroundTaskStore store,
IBackgroundTaskSerializer serializer,
ITypeResolver typeResolver,
IOptionsMonitor <BackgroundTaskOptions> options,
ISafeLogger <BackgroundTaskHost> logger)
{
_backgroundServices = backgroundServices;
Store = store;
Serializer = serializer;
_typeResolver = typeResolver;
_options = options;
_logger = logger;
options.OnChange(OnSettingsChanged);
_schedulers = new ConcurrentDictionary <int, TaskScheduler>();
_factories = new ConcurrentDictionary <TaskScheduler, TaskFactory>();
_pending = new ConcurrentDictionary <object, HandlerHooks>();
_cancel = new CancellationTokenSource();
// dispatch thread
_background = new PushQueue <IEnumerable <BackgroundTask> >();
_background.Attach(WithPendingTasks);
_background.AttachBacklog(WithOverflowTasks);
_background.AttachUndeliverable(WithFailedTasks);
// maintenance thread
_maintenance = new PushQueue <IEnumerable <BackgroundTask> >();
_maintenance.Attach(WithHangingTasks);
_maintenance.AttachBacklog(WithHangingTasks);
_maintenance.AttachUndeliverable(WithFailedTasks);
}
public CosmosBackgroundTaskStore(ICosmosRepository repository,
Func <DateTimeOffset> timestamps, ISafeLogger <CosmosBackgroundTaskStore> logger)
{
_repository = repository;
_timestamps = timestamps;
_logger = logger;
}
private static void AddSuperUser(this IServiceCollection services, ISafeLogger logger, SuperUserOptions options)
{
if (options.Enabled)
{
logger?.Trace(() => "SuperUser enabled.");
services.AddDefaultAuthorization(Constants.Security.Policies.SuperUserOnly, ClaimValues.SuperUser);
}
}
public SchemaDiscoveryService(SchemaService service, IHostEnvironment environment,
IOptionsMonitor <SchemaOptions> options, ISafeLogger <SchemaDiscoveryService> logger)
{
_service = service;
_environment = environment;
_options = options;
_logger = logger;
}
public SqliteBackgroundTaskStore(IServiceProvider serviceProvider,
Func <DateTimeOffset> timestamps,
IOptionsMonitor <BackgroundTaskOptions> options,
string tablePrefix = nameof(BackgroundTask),
ISafeLogger <SqliteBackgroundTaskStore> logger = null)
{
_serviceProvider = serviceProvider;
_timestamps = timestamps;
_options = options;
_tablePrefix = tablePrefix;
_logger = logger;
}
public TokenController(
Func <DateTimeOffset> timestamps,
IHttpContextAccessor http,
ISignInService <TUser> signInService,
ITokenFabricator <TKey> fabricator,
ISafeLogger <TokenController <TUser, TTenant, TApplication, TKey> > logger)
{
_timestamps = timestamps;
_http = http;
_signInService = signInService;
_fabricator = fabricator;
_logger = logger;
}
public IEnumerable <T> Read(Stream stream, IComparer <T> sort, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var list = new List <T>();
LineReader.ReadLines(stream, Encoding.UTF8, (lineNumber, line) =>
{
var t = _deserialize(line);
list.Add(t);
}, logger, metrics, cancellationToken);
list.Sort(sort);
return(list);
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLineAsString onNewLine,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
unsafe
{
void NewLine(long n, bool p, byte *s, int l, Encoding e)
{
onNewLine?.Invoke(n, e.GetString(s, l));
}
return(ReadOrCountLines(stream, encoding, buffer, NewLine, logger, metrics, cancellationToken));
}
}
public void AddCloudMetricsPublisher(IMetricsBuilder builder, ISafeLogger logger, AzureOptions options)
{
logger.Info(() => "Adding Application Insights Metrics & Health Checks Reporting");
builder.PushToApplicationInsights(p =>
{
p.MetricsSampleEventName = Constants.Events.MetricsSample;
p.HealthCheckEventName = Constants.Events.HealthCheck;
p.PublishHealthChecks = true;
p.PublishHealthy = false;
p.PublishMetrics = true;
});
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, string separator,
NewValueAsSpan onNewValue, ISafeLogger logger = null, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
unsafe
{
return(ReadLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));
void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding e)
{
LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue);
}
}
}
private static void TryAddDefaultPolicy(this IServiceCollection services, ISafeLogger logger,
AuthorizationOptions x, string scheme)
{
if (x.DefaultPolicy?.AuthenticationSchemes.Count != 0)
{
logger?.Info(() =>
$"Skipping default policy build; '{string.Join(",", x.DefaultPolicy?.AuthenticationSchemes ?? Enumerable.Empty<string>())}' already registered.");
return;
}
logger?.Info(() => $"Registering default policy with scheme '{scheme}'.");
x.DefaultPolicy = new AuthorizationPolicyBuilder(scheme)
.RequireAuthenticatedUserExtended(services)
.Build();
}
private static void ScanForGeneratedObjects(this IServiceCollection services, string backendType,
IConfiguration security, ISafeLogger logger, string rootPath, Assembly assembly)
{
foreach (var type in assembly.GetExportedTypes())
{
if (!type.IsAbstract || !type.IsSealed)
{
continue;
}
if (type.Name != "ServiceCollectionExtensions")
{
continue;
}
var method = type.GetMethod("AddGenerated",
new[] { typeof(IServiceCollection), typeof(IConfiguration), typeof(string) });
if (method == null)
{
continue;
}
Type batchOptionsType;
switch (backendType)
{
case nameof(DocumentDb):
batchOptionsType = typeof(DocumentDbBatchOptions);
break;
case nameof(SqlServer):
batchOptionsType = typeof(SqlServerBatchOptions);
break;
case nameof(Sqlite):
batchOptionsType = typeof(SqliteBatchOptions);
break;
default:
throw new ArgumentOutOfRangeException();
}
logger.Info(() => "Found generated objects API in {AssemblyName}", assembly.GetName().Name);
method.MakeGenericMethod(batchOptionsType)
.Invoke(null, new object[] { services, security, rootPath });
}
}
private static void AddCors(this IServiceCollection services, ISafeLogger logger, CorsOptions cors)
{
if (!cors.Enabled)
{
return;
}
logger?.Trace(() => "CORS enabled.");
services.AddRouting(o => { });
services.AddCors(o =>
{
o.AddPolicy(Constants.Security.Policies.CorsPolicy, builder =>
{
builder
.WithOrigins(cors.Origins ?? new[] { "*" })
.WithMethods(cors.Methods ?? new[] { "*" })
.WithHeaders(cors.Headers ?? new[] { "*" })
.WithExposedHeaders(cors.ExposedHeaders ?? new string[0]);
if (cors.AllowCredentials && cors.Origins?.Length > 0 && cors.Origins[0] != "*")
{
builder.AllowCredentials();
}
else
{
builder.DisallowCredentials();
}
if (cors.AllowOriginWildcards)
{
builder.SetIsOriginAllowedToAllowWildcardSubdomains();
}
if (cors.PreflightMaxAgeSeconds.HasValue)
{
builder.SetPreflightMaxAge(TimeSpan.FromSeconds(cors.PreflightMaxAgeSeconds.Value));
}
});
});
}
public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
var count = 0L;
var segments = 0;
var sw = new StreamWriter(CreateSegment(label, segments));
try
{
foreach (var item in stream)
{
var line = _serialize(item);
BuiltInMetrics.LineLength <T>(metrics, line.Length);
sw.WriteLine(line);
count++;
if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes)
{
continue;
}
sw.Flush();
sw.Close();
segments++;
sw = new StreamWriter(CreateSegment(label, segments));
}
}
finally
{
sw.Flush();
sw.Close();
}
return(new SegmentStats
{
RecordCount = count,
RecordLength = (int)BuiltInMetrics.GetMeanLineLength <T>(metrics) + 1,
SegmentCount = segments
});
}
private static void AddAuthentication(this IServiceCollection services, ISafeLogger logger,
SecurityOptions security,
SuperUserOptions superUser)
{
var tokens = security.Tokens;
var cookies = security.Cookies;
var claims = security.Claims;
if (tokens.Enabled || cookies.Enabled || superUser.Enabled)
{
if (!tokens.Enabled && !cookies.Enabled && superUser.Enabled)
{
logger?.Trace(() => "Authentication enabled for super user only.");
}
else
{
logger?.Trace(() => "Authentication enabled.");
}
services.AddAuthentication(security, superUser, tokens, cookies, claims);
}
if (tokens.Enabled || superUser.Enabled)
{
services.AddAuthorization(x =>
{
TryAddDefaultPolicy(services, logger, x, tokens.Scheme);
});
}
if (cookies.Enabled)
{
services.AddAuthorization(x =>
{
TryAddDefaultPolicy(services, logger, x, cookies.Scheme);
});
}
}
public static long CountLines(Stream stream, Encoding encoding, byte[] workingBuffer, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadOrCountLines(stream, encoding, workingBuffer, null, logger, metrics, cancellationToken));
}
public static long CountLines(Stream stream, Encoding encoding, ISafeLogger logger = null,
IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(CountLines(stream, encoding, Constants.Buffer, logger, metrics, cancellationToken));
}
// Derived from MimeKit's MimeParser
private static long ReadOrCountLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine,
ISafeLogger logger, IMetricsHost metrics, CancellationToken cancellationToken)
{
var count = 0L;
var offset = stream.CanSeek ? stream.Position : 0L;
var from = Constants.ReadAheadSize;
var to = Constants.ReadAheadSize;
var endOfStream = false;
var preamble = encoding.GetPreambleBuffer();
unsafe
{
fixed(byte *buffer = workingBuffer)
{
if (stream.CanSeek && stream.Position != offset)
{
stream.Seek(offset, SeekOrigin.Begin);
}
if (!ReadPreamble(stream, preamble, buffer, workingBuffer, ref from, ref to, ref endOfStream,
cancellationToken))
{
throw new FormatException(ErrorStrings.UnexpectedEndOfStream);
}
do
{
if (ReadAhead(stream, workingBuffer, Constants.ReadAheadSize, 2, ref from, ref to,
ref endOfStream, cancellationToken) <= 0)
{
break;
}
var position = buffer + from;
var end = buffer + to;
var startIndex = from;
*end = (byte)'\n';
while (position < end)
{
var alignment = (startIndex + 3) & ~3;
var aligned = buffer + alignment;
var start = position;
var c = *aligned;
*aligned = Constants.LineFeed;
while (*position != Constants.LineFeed)
{
position++;
}
*aligned = c;
if (position == aligned && c != Constants.LineFeed)
{
var dword = (uint *)position;
uint mask;
do
{
mask = *dword++ ^ 0x0A0A0A0A;
mask = (mask - 0x01010101) & ~mask & 0x80808080;
} while (mask == 0);
position = (byte *)(dword - 1);
while (*position != Constants.LineFeed)
{
position++;
}
}
var length = (int)(position - start);
BuiltInMetrics.BytesPerSecond(metrics, length);
if (position < end)
{
length++;
position++;
count++;
onNewLine?.Invoke(count, false, start, length, encoding);
}
else if (count == 0 && position == end)
{
onNewLine?.Invoke(count, false, start, length, encoding);
return(1);
}
else
{
// line spans across the read-ahead buffer
onNewLine?.Invoke(count, true, start, length, encoding);
}
startIndex += length;
}
from = startIndex;
} while (true);
}
}
return(count);
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] buffer,
int maxWorkingMemoryBytes = 0, ISafeLogger logger = null, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
buffer ??= Constants.Buffer;
var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());
logger?.Debug(() => "Started streaming lines");
var task = new Task(() =>
{
if (!Task.CurrentId.HasValue)
{
throw new ArgumentNullException(nameof(buffer), "Could not find current thread ID");
}
var id = Task.CurrentId.Value;
long count = 0;
var pendingLength = 0;
try
{
unsafe
{
count = ReadLines(stream, encoding, PendingStreams[id],
(lineNumber, partial, start, length, e) =>
{
var line = new ReadOnlySpan <byte>(start, length);
if (partial)
{
pendingLength = length;
}
else
{
var memory = new byte[length];
line.CopyTo(memory);
var ctor = new LineConstructor
{
lineNumber = lineNumber, length = length + pendingLength, buffer = memory
};
if (maxWorkingMemoryBytes > 0)
{
var usedBytes =
queue.Count * (PendingStreams[id].Length + sizeof(long) + sizeof(int));
while (usedBytes > maxWorkingMemoryBytes)
{
Task.Delay(10, cancellationToken).Wait(cancellationToken);
}
}
queue.Add(ctor, cancellationToken);
pendingLength = 0;
}
}, logger, metrics, cancellationToken);
}
}
catch (Exception ex)
{
logger?.Error(() => "Error streaming lines", ex);
}
finally
{
logger?.Debug(() => "Finished streaming {Count} lines", count);
queue.CompleteAdding();
if (!PendingStreams.TryRemove(id, out _))
{
throw new InvalidOperationException("Line stream failed trying to clean up");
}
}
}, cancellationToken);
if (!PendingStreams.TryAdd(task.Id, buffer))
{
throw new InvalidOperationException("Line stream failed trying to start up");
}
task.Start();
return(queue.GetConsumingEnumerable(cancellationToken));
}
public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding,
int maxWorkingMemoryBytes = 0, ISafeLogger logger = null, IMetricsHost metrics = null,
CancellationToken cancellationToken = default)
{
return(StreamLines(stream, encoding, Constants.Buffer, maxWorkingMemoryBytes, logger, metrics,
cancellationToken));
}
public static long ReadLines(Stream stream, Encoding encoding, string separator, NewValueAsSpan onNewValue,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadLines(stream, encoding, Constants.Buffer, separator, onNewValue, logger, metrics,
cancellationToken));
}
public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLine onNewLine,
ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
{
return(ReadOrCountLines(stream, encoding, buffer, onNewLine, logger, metrics, cancellationToken));
}
public static IMetricsBuilder AddCloudMetricsPublisher <T>(this IMetricsBuilder builder, ISafeLogger logger,
ICloudOptions options)
where T : ICloudOptions
{
foreach (var module in ScanForTypesImplementing <ICloudMetricsPublisher <T> >())
{
var method =
typeof(ICloudMetricsPublisher <T>).GetMethod(nameof(ICloudMetricsPublisher <T>
.AddCloudMetricsPublisher));
method?.Invoke(module, new object[] { builder, logger, options });
}
return(builder);
}
public static IServiceCollection AddCloudTelemetry <T>(this IServiceCollection services, ISafeLogger logger,
ICloudOptions options)
where T : ICloudOptions
{
foreach (var module in ScanForTypesImplementing <ICloudTelemetry <T> >())
{
var method = typeof(ICloudTelemetry <T>).GetMethod(nameof(ICloudTelemetry <T> .AddCloudTelemetry));
method?.Invoke(module, new object[] { services, logger, options });
}
return(services);
}
