Esempio n. 1
0
        public void Sort(string fromLabel, string toLabel, IComparer <T> sort, ISafeLogger logger = null,
                         IMetricsHost metrics = null, CancellationToken cancellationToken = default)
        {
            var streams = GetAllSegments(fromLabel);
            var data    = new Queue <Stream>(streams);

            for (var i = 0; i < data.Count; i++)
            {
                using (var stream = data.Dequeue())
                {
                    using var sw = new StreamWriter(CreateSegment(toLabel, i));

                    var items = Read(stream, sort, logger, metrics, cancellationToken);
                    foreach (var item in items)
                    {
                        var line = _serialize(item);
                        sw.WriteLine(line);
                    }

                    sw.Flush();
                }

                DeleteSegment(fromLabel, i);
            }
        }
 public DocumentDbBackgroundTaskStore(IDocumentDbRepository <BackgroundTaskDocument> repository,
                                      IServerTimestampService timestamps, ISafeLogger <DocumentDbBackgroundTaskStore> logger)
 {
     _repository = repository;
     _timestamps = timestamps;
     _logger     = logger;
 }
Esempio n. 3
0
        public DocumentDbRepository(string slot, IOptionsMonitor <DocumentDbOptions> options,
                                    ISafeLogger <DocumentDbRepository <T> > logger)
        {
            _reads   = ReadAccessor.Create(typeof(T));
            _writes  = WriteAccessor.Create(typeof(T));
            _slot    = slot;
            _options = options;
            _logger  = logger;

            var defaultSettings   = new JsonSerializerSettings();
            var documentDbOptions = options.Get(_slot);

            _client = new DocumentClient(EndpointUri, documentDbOptions.AccountKey, defaultSettings);


            _client2 = new CosmosClient(EndpointUri.ToString(), documentDbOptions.AccountKey,
                                        new CosmosClientOptions
            {
                AllowBulkExecution = true,
                ConnectionMode     = ConnectionMode.Gateway,
                MaxRetryAttemptsOnRateLimitedRequests = 100
            });

            CreateDatabaseIfNotExistsAsync().Wait();
            CreateCollectionIfNotExistsAsync().Wait();
        }
Esempio n. 4
0
File: Add.cs Progetto: qiqi545/HQ
 public static IServiceCollection AddSecurityPolicies(this IServiceCollection services,
                                                      IConfiguration securityConfig,
                                                      IConfiguration superUserConfig,
                                                      ISafeLogger logger)
 {
     return(AddSecurityPolicies(services, securityConfig.FastBind, superUserConfig.FastBind, logger));
 }
Esempio n. 5
0
        public static long ReadLines(Stream stream, Encoding encoding, NewLineAsString onNewLine,
                                     ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
        {
            unsafe
            {
                var pendingLength = 0;
                var buffer        = Constants.Buffer;

                return(ReadOrCountLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));

                void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding x)
                {
                    var target  = new Span <byte>(buffer, pendingLength, length);
                    var segment = new ReadOnlySpan <byte>(start, length);

                    segment.CopyTo(target);

                    if (partial)
                    {
                        pendingLength = length;
                    }
                    else
                    {
                        var line = new ReadOnlySpan <byte>(buffer, 0, length + pendingLength);
                        onNewLine?.Invoke(lineNumber, encoding.GetString(line));
                        pendingLength = 0;
                    }
                }
            }
        }
Esempio n. 6
0
File: Add.cs Progetto: qiqi545/HQ
        private static void AddHttps(this IServiceCollection services, ISafeLogger logger, SecurityOptions options)
        {
            if (options.Https.Enabled)
            {
                logger?.Trace(() => "HTTPS enabled.");

                services.AddHttpsRedirection(o =>
                {
                    o.HttpsPort          = null;
                    o.RedirectStatusCode = options.Https.Hsts.Enabled ? 307 : 301;
                });

                if (options.Https.Hsts.Enabled)
                {
                    logger?.Trace(() => "HSTS enabled.");

                    services.AddHsts(o =>
                    {
                        o.MaxAge            = options.Https.Hsts.HstsMaxAge;
                        o.IncludeSubDomains = options.Https.Hsts.IncludeSubdomains;
                        o.Preload           = options.Https.Hsts.Preload;
                    });
                }
            }
        }
Esempio n. 7
0
        public BackgroundTaskHost(
            IServiceProvider backgroundServices,
            IBackgroundTaskStore store,
            IBackgroundTaskSerializer serializer,
            ITypeResolver typeResolver,
            IOptionsMonitor <BackgroundTaskOptions> options,
            ISafeLogger <BackgroundTaskHost> logger)
        {
            _backgroundServices = backgroundServices;
            Store         = store;
            Serializer    = serializer;
            _typeResolver = typeResolver;
            _options      = options;
            _logger       = logger;
            options.OnChange(OnSettingsChanged);

            _schedulers = new ConcurrentDictionary <int, TaskScheduler>();
            _factories  = new ConcurrentDictionary <TaskScheduler, TaskFactory>();
            _pending    = new ConcurrentDictionary <object, HandlerHooks>();
            _cancel     = new CancellationTokenSource();

            // dispatch thread
            _background = new PushQueue <IEnumerable <BackgroundTask> >();
            _background.Attach(WithPendingTasks);
            _background.AttachBacklog(WithOverflowTasks);
            _background.AttachUndeliverable(WithFailedTasks);

            // maintenance thread
            _maintenance = new PushQueue <IEnumerable <BackgroundTask> >();
            _maintenance.Attach(WithHangingTasks);
            _maintenance.AttachBacklog(WithHangingTasks);
            _maintenance.AttachUndeliverable(WithFailedTasks);
        }
 public CosmosBackgroundTaskStore(ICosmosRepository repository,
                                  Func <DateTimeOffset> timestamps, ISafeLogger <CosmosBackgroundTaskStore> logger)
 {
     _repository = repository;
     _timestamps = timestamps;
     _logger     = logger;
 }
Esempio n. 9
0
File: Add.cs Progetto: qiqi545/HQ
 private static void AddSuperUser(this IServiceCollection services, ISafeLogger logger, SuperUserOptions options)
 {
     if (options.Enabled)
     {
         logger?.Trace(() => "SuperUser enabled.");
         services.AddDefaultAuthorization(Constants.Security.Policies.SuperUserOnly, ClaimValues.SuperUser);
     }
 }
Esempio n. 10
0
 public SchemaDiscoveryService(SchemaService service, IHostEnvironment environment,
                               IOptionsMonitor <SchemaOptions> options, ISafeLogger <SchemaDiscoveryService> logger)
 {
     _service     = service;
     _environment = environment;
     _options     = options;
     _logger      = logger;
 }
Esempio n. 11
0
 public SqliteBackgroundTaskStore(IServiceProvider serviceProvider,
                                  Func <DateTimeOffset> timestamps,
                                  IOptionsMonitor <BackgroundTaskOptions> options,
                                  string tablePrefix = nameof(BackgroundTask),
                                  ISafeLogger <SqliteBackgroundTaskStore> logger = null)
 {
     _serviceProvider = serviceProvider;
     _timestamps      = timestamps;
     _options         = options;
     _tablePrefix     = tablePrefix;
     _logger          = logger;
 }
Esempio n. 12
0
 public TokenController(
     Func <DateTimeOffset> timestamps,
     IHttpContextAccessor http,
     ISignInService <TUser> signInService,
     ITokenFabricator <TKey> fabricator,
     ISafeLogger <TokenController <TUser, TTenant, TApplication, TKey> > logger)
 {
     _timestamps    = timestamps;
     _http          = http;
     _signInService = signInService;
     _fabricator    = fabricator;
     _logger        = logger;
 }
Esempio n. 13
0
        public IEnumerable <T> Read(Stream stream, IComparer <T> sort, ISafeLogger logger = null,
                                    IMetricsHost metrics = null, CancellationToken cancellationToken = default)
        {
            var list = new List <T>();

            LineReader.ReadLines(stream, Encoding.UTF8, (lineNumber, line) =>
            {
                var t = _deserialize(line);
                list.Add(t);
            }, logger, metrics, cancellationToken);
            list.Sort(sort);
            return(list);
        }
Esempio n. 14
0
        public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLineAsString onNewLine,
                                     ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
        {
            unsafe
            {
                void NewLine(long n, bool p, byte *s, int l, Encoding e)
                {
                    onNewLine?.Invoke(n, e.GetString(s, l));
                }

                return(ReadOrCountLines(stream, encoding, buffer, NewLine, logger, metrics, cancellationToken));
            }
        }
Esempio n. 15
0
        public void AddCloudMetricsPublisher(IMetricsBuilder builder, ISafeLogger logger, AzureOptions options)
        {
            logger.Info(() => "Adding Application Insights Metrics & Health Checks Reporting");

            builder.PushToApplicationInsights(p =>
            {
                p.MetricsSampleEventName = Constants.Events.MetricsSample;
                p.HealthCheckEventName   = Constants.Events.HealthCheck;
                p.PublishHealthChecks    = true;
                p.PublishHealthy         = false;
                p.PublishMetrics         = true;
            });
        }
Esempio n. 16
0
        public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, string separator,
                                     NewValueAsSpan onNewValue, ISafeLogger logger = null, IMetricsHost metrics = null,
                                     CancellationToken cancellationToken           = default)
        {
            unsafe
            {
                return(ReadLines(stream, encoding, buffer, OnNewLine, logger, metrics, cancellationToken));

                void OnNewLine(long lineNumber, bool partial, byte *start, int length, Encoding e)
                {
                    LineValuesReader.ReadValues(lineNumber, start, length, e, separator, onNewValue);
                }
            }
        }
Esempio n. 17
0
File: Add.cs Progetto: qiqi545/HQ
        private static void TryAddDefaultPolicy(this IServiceCollection services, ISafeLogger logger,
                                                AuthorizationOptions x, string scheme)
        {
            if (x.DefaultPolicy?.AuthenticationSchemes.Count != 0)
            {
                logger?.Info(() =>
                             $"Skipping default policy build; '{string.Join(",", x.DefaultPolicy?.AuthenticationSchemes ?? Enumerable.Empty<string>())}' already registered.");
                return;
            }

            logger?.Info(() => $"Registering default policy with scheme '{scheme}'.");
            x.DefaultPolicy = new AuthorizationPolicyBuilder(scheme)
                              .RequireAuthenticatedUserExtended(services)
                              .Build();
        }
Esempio n. 18
0
File: Add.cs Progetto: qiqi545/HQ
        private static void ScanForGeneratedObjects(this IServiceCollection services, string backendType,
                                                    IConfiguration security, ISafeLogger logger, string rootPath, Assembly assembly)
        {
            foreach (var type in assembly.GetExportedTypes())
            {
                if (!type.IsAbstract || !type.IsSealed)
                {
                    continue;
                }

                if (type.Name != "ServiceCollectionExtensions")
                {
                    continue;
                }

                var method = type.GetMethod("AddGenerated",
                                            new[] { typeof(IServiceCollection), typeof(IConfiguration), typeof(string) });
                if (method == null)
                {
                    continue;
                }

                Type batchOptionsType;
                switch (backendType)
                {
                case nameof(DocumentDb):
                    batchOptionsType = typeof(DocumentDbBatchOptions);
                    break;

                case nameof(SqlServer):
                    batchOptionsType = typeof(SqlServerBatchOptions);
                    break;

                case nameof(Sqlite):
                    batchOptionsType = typeof(SqliteBatchOptions);
                    break;

                default:
                    throw new ArgumentOutOfRangeException();
                }

                logger.Info(() => "Found generated objects API in {AssemblyName}", assembly.GetName().Name);
                method.MakeGenericMethod(batchOptionsType)
                .Invoke(null, new object[] { services, security, rootPath });
            }
        }
Esempio n. 19
0
File: Add.cs Progetto: qiqi545/HQ
        private static void AddCors(this IServiceCollection services, ISafeLogger logger, CorsOptions cors)
        {
            if (!cors.Enabled)
            {
                return;
            }

            logger?.Trace(() => "CORS enabled.");

            services.AddRouting(o => { });

            services.AddCors(o =>
            {
                o.AddPolicy(Constants.Security.Policies.CorsPolicy, builder =>
                {
                    builder
                    .WithOrigins(cors.Origins ?? new[] { "*" })
                    .WithMethods(cors.Methods ?? new[] { "*" })
                    .WithHeaders(cors.Headers ?? new[] { "*" })
                    .WithExposedHeaders(cors.ExposedHeaders ?? new string[0]);

                    if (cors.AllowCredentials && cors.Origins?.Length > 0 && cors.Origins[0] != "*")
                    {
                        builder.AllowCredentials();
                    }
                    else
                    {
                        builder.DisallowCredentials();
                    }

                    if (cors.AllowOriginWildcards)
                    {
                        builder.SetIsOriginAllowedToAllowWildcardSubdomains();
                    }

                    if (cors.PreflightMaxAgeSeconds.HasValue)
                    {
                        builder.SetPreflightMaxAge(TimeSpan.FromSeconds(cors.PreflightMaxAgeSeconds.Value));
                    }
                });
            });
        }
Esempio n. 20
0
        public SegmentStats Segment(string label, IEnumerable <T> stream, int maxWorkingMemoryBytes = 0,
                                    ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
        {
            var count    = 0L;
            var segments = 0;
            var sw       = new StreamWriter(CreateSegment(label, segments));

            try
            {
                foreach (var item in stream)
                {
                    var line = _serialize(item);
                    BuiltInMetrics.LineLength <T>(metrics, line.Length);

                    sw.WriteLine(line);
                    count++;

                    if (maxWorkingMemoryBytes == 0 || sw.BaseStream.Length < maxWorkingMemoryBytes)
                    {
                        continue;
                    }

                    sw.Flush();
                    sw.Close();
                    segments++;
                    sw = new StreamWriter(CreateSegment(label, segments));
                }
            }
            finally
            {
                sw.Flush();
                sw.Close();
            }

            return(new SegmentStats
            {
                RecordCount = count,
                RecordLength = (int)BuiltInMetrics.GetMeanLineLength <T>(metrics) + 1,
                SegmentCount = segments
            });
        }
Esempio n. 21
0
File: Add.cs Progetto: qiqi545/HQ
        private static void AddAuthentication(this IServiceCollection services, ISafeLogger logger,
                                              SecurityOptions security,
                                              SuperUserOptions superUser)
        {
            var tokens  = security.Tokens;
            var cookies = security.Cookies;
            var claims  = security.Claims;

            if (tokens.Enabled || cookies.Enabled || superUser.Enabled)
            {
                if (!tokens.Enabled && !cookies.Enabled && superUser.Enabled)
                {
                    logger?.Trace(() => "Authentication enabled for super user only.");
                }
                else
                {
                    logger?.Trace(() => "Authentication enabled.");
                }

                services.AddAuthentication(security, superUser, tokens, cookies, claims);
            }

            if (tokens.Enabled || superUser.Enabled)
            {
                services.AddAuthorization(x =>
                {
                    TryAddDefaultPolicy(services, logger, x, tokens.Scheme);
                });
            }

            if (cookies.Enabled)
            {
                services.AddAuthorization(x =>
                {
                    TryAddDefaultPolicy(services, logger, x, cookies.Scheme);
                });
            }
        }
Esempio n. 22
0
 public static long CountLines(Stream stream, Encoding encoding, byte[] workingBuffer, ISafeLogger logger = null,
                               IMetricsHost metrics = null, CancellationToken cancellationToken = default)
 {
     return(ReadOrCountLines(stream, encoding, workingBuffer, null, logger, metrics, cancellationToken));
 }
Esempio n. 23
0
 public static long CountLines(Stream stream, Encoding encoding, ISafeLogger logger = null,
                               IMetricsHost metrics = null, CancellationToken cancellationToken = default)
 {
     return(CountLines(stream, encoding, Constants.Buffer, logger, metrics, cancellationToken));
 }
Esempio n. 24
0
        // Derived from MimeKit's MimeParser
        private static long ReadOrCountLines(Stream stream, Encoding encoding, byte[] workingBuffer, NewLine onNewLine,
                                             ISafeLogger logger, IMetricsHost metrics, CancellationToken cancellationToken)
        {
            var count       = 0L;
            var offset      = stream.CanSeek ? stream.Position : 0L;
            var from        = Constants.ReadAheadSize;
            var to          = Constants.ReadAheadSize;
            var endOfStream = false;

            var preamble = encoding.GetPreambleBuffer();

            unsafe
            {
                fixed(byte *buffer = workingBuffer)
                {
                    if (stream.CanSeek && stream.Position != offset)
                    {
                        stream.Seek(offset, SeekOrigin.Begin);
                    }

                    if (!ReadPreamble(stream, preamble, buffer, workingBuffer, ref from, ref to, ref endOfStream,
                                      cancellationToken))
                    {
                        throw new FormatException(ErrorStrings.UnexpectedEndOfStream);
                    }

                    do
                    {
                        if (ReadAhead(stream, workingBuffer, Constants.ReadAheadSize, 2, ref from, ref to,
                                      ref endOfStream, cancellationToken) <= 0)
                        {
                            break;
                        }

                        var position   = buffer + from;
                        var end        = buffer + to;
                        var startIndex = from;

                        *end = (byte)'\n';

                        while (position < end)
                        {
                            var alignment = (startIndex + 3) & ~3;
                            var aligned   = buffer + alignment;
                            var start     = position;
                            var c         = *aligned;

                            *aligned = Constants.LineFeed;
                            while (*position != Constants.LineFeed)
                            {
                                position++;
                            }

                            *aligned = c;

                            if (position == aligned && c != Constants.LineFeed)
                            {
                                var  dword = (uint *)position;
                                uint mask;
                                do
                                {
                                    mask = *dword++ ^ 0x0A0A0A0A;
                                    mask = (mask - 0x01010101) & ~mask & 0x80808080;
                                } while (mask == 0);

                                position = (byte *)(dword - 1);
                                while (*position != Constants.LineFeed)
                                {
                                    position++;
                                }
                            }

                            var length = (int)(position - start);

                            BuiltInMetrics.BytesPerSecond(metrics, length);

                            if (position < end)
                            {
                                length++;
                                position++;
                                count++;

                                onNewLine?.Invoke(count, false, start, length, encoding);
                            }
                            else if (count == 0 && position == end)
                            {
                                onNewLine?.Invoke(count, false, start, length, encoding);
                                return(1);
                            }
                            else
                            {
                                // line spans across the read-ahead buffer
                                onNewLine?.Invoke(count, true, start, length, encoding);
                            }

                            startIndex += length;
                        }

                        from = startIndex;
                    } while (true);
                }
            }

            return(count);
        }
Esempio n. 25
0
        public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding, byte[] buffer,
                                                                int maxWorkingMemoryBytes           = 0, ISafeLogger logger = null, IMetricsHost metrics = null,
                                                                CancellationToken cancellationToken = default)
        {
            buffer ??= Constants.Buffer;

            var queue = new BlockingCollection <LineConstructor>(new ConcurrentQueue <LineConstructor>());

            logger?.Debug(() => "Started streaming lines");

            var task = new Task(() =>
            {
                if (!Task.CurrentId.HasValue)
                {
                    throw new ArgumentNullException(nameof(buffer), "Could not find current thread ID");
                }

                var id = Task.CurrentId.Value;

                long count        = 0;
                var pendingLength = 0;
                try
                {
                    unsafe
                    {
                        count = ReadLines(stream, encoding, PendingStreams[id],
                                          (lineNumber, partial, start, length, e) =>
                        {
                            var line = new ReadOnlySpan <byte>(start, length);
                            if (partial)
                            {
                                pendingLength = length;
                            }
                            else
                            {
                                var memory = new byte[length];
                                line.CopyTo(memory);

                                var ctor = new LineConstructor
                                {
                                    lineNumber = lineNumber, length = length + pendingLength, buffer = memory
                                };

                                if (maxWorkingMemoryBytes > 0)
                                {
                                    var usedBytes =
                                        queue.Count * (PendingStreams[id].Length + sizeof(long) + sizeof(int));
                                    while (usedBytes > maxWorkingMemoryBytes)
                                    {
                                        Task.Delay(10, cancellationToken).Wait(cancellationToken);
                                    }
                                }

                                queue.Add(ctor, cancellationToken);
                                pendingLength = 0;
                            }
                        }, logger, metrics, cancellationToken);
                    }
                }
                catch (Exception ex)
                {
                    logger?.Error(() => "Error streaming lines", ex);
                }
                finally
                {
                    logger?.Debug(() => "Finished streaming {Count} lines", count);
                    queue.CompleteAdding();

                    if (!PendingStreams.TryRemove(id, out _))
                    {
                        throw new InvalidOperationException("Line stream failed trying to clean up");
                    }
                }
            }, cancellationToken);

            if (!PendingStreams.TryAdd(task.Id, buffer))
            {
                throw new InvalidOperationException("Line stream failed trying to start up");
            }

            task.Start();

            return(queue.GetConsumingEnumerable(cancellationToken));
        }
Esempio n. 26
0
 public static IEnumerable <LineConstructor> StreamLines(Stream stream, Encoding encoding,
                                                         int maxWorkingMemoryBytes           = 0, ISafeLogger logger = null, IMetricsHost metrics = null,
                                                         CancellationToken cancellationToken = default)
 {
     return(StreamLines(stream, encoding, Constants.Buffer, maxWorkingMemoryBytes, logger, metrics,
                        cancellationToken));
 }
Esempio n. 27
0
 public static long ReadLines(Stream stream, Encoding encoding, string separator, NewValueAsSpan onNewValue,
                              ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
 {
     return(ReadLines(stream, encoding, Constants.Buffer, separator, onNewValue, logger, metrics,
                      cancellationToken));
 }
Esempio n. 28
0
 public static long ReadLines(Stream stream, Encoding encoding, byte[] buffer, NewLine onNewLine,
                              ISafeLogger logger = null, IMetricsHost metrics = null, CancellationToken cancellationToken = default)
 {
     return(ReadOrCountLines(stream, encoding, buffer, onNewLine, logger, metrics, cancellationToken));
 }
Esempio n. 29
0
        public static IMetricsBuilder AddCloudMetricsPublisher <T>(this IMetricsBuilder builder, ISafeLogger logger,
                                                                   ICloudOptions options)
            where T : ICloudOptions
        {
            foreach (var module in ScanForTypesImplementing <ICloudMetricsPublisher <T> >())
            {
                var method =
                    typeof(ICloudMetricsPublisher <T>).GetMethod(nameof(ICloudMetricsPublisher <T>
                                                                        .AddCloudMetricsPublisher));
                method?.Invoke(module, new object[] { builder, logger, options });
            }

            return(builder);
        }
Esempio n. 30
0
        public static IServiceCollection AddCloudTelemetry <T>(this IServiceCollection services, ISafeLogger logger,
                                                               ICloudOptions options)
            where T : ICloudOptions
        {
            foreach (var module in ScanForTypesImplementing <ICloudTelemetry <T> >())
            {
                var method = typeof(ICloudTelemetry <T>).GetMethod(nameof(ICloudTelemetry <T> .AddCloudTelemetry));
                method?.Invoke(module, new object[] { services, logger, options });
            }

            return(services);
        }