public SiloConnectionListener(
IServiceProvider serviceProvider,
IOptions <ConnectionOptions> connectionOptions,
IOptions <SiloConnectionOptions> siloConnectionOptions,
MessageCenter messageCenter,
IOptions <EndpointOptions> endpointOptions,
ILocalSiloDetails localSiloDetails,
ConnectionManager connectionManager,
ConnectionCommon connectionShared,
ProbeRequestMonitor probeRequestMonitor)
: base(serviceProvider.GetRequiredServiceByKey <object, IConnectionListenerFactory>(ServicesKey), connectionOptions, connectionManager, connectionShared)
{
this.siloConnectionOptions = siloConnectionOptions.Value;
this.messageCenter = messageCenter;
this.localSiloDetails = localSiloDetails;
this.connectionManager = connectionManager;
this.connectionShared = connectionShared;
this.probeRequestMonitor = probeRequestMonitor;
this.endpointOptions = endpointOptions.Value;
}
public SiloConnectionListener(
IServiceProvider serviceProvider,
IOptions <ConnectionOptions> connectionOptions,
IOptions <SiloConnectionOptions> siloConnectionOptions,
MessageCenter messageCenter,
MessageFactory messageFactory,
INetworkingTrace trace,
IOptions <EndpointOptions> endpointOptions,
ILocalSiloDetails localSiloDetails,
ConnectionManager connectionManager)
: base(serviceProvider, serviceProvider.GetRequiredServiceByKey <object, IConnectionListenerFactory>(ServicesKey), connectionOptions, connectionManager, trace)
{
this.siloConnectionOptions = siloConnectionOptions.Value;
this.messageCenter = messageCenter;
this.messageFactory = messageFactory;
this.trace = trace;
this.localSiloDetails = localSiloDetails;
this.connectionManager = connectionManager;
this.endpointOptions = endpointOptions.Value;
}
public GatewayConnectionListener(
IServiceProvider serviceProvider,
IOptions <ConnectionOptions> connectionOptions,
IOptions <SiloConnectionOptions> siloConnectionOptions,
OverloadDetector overloadDetector,
ILocalSiloDetails localSiloDetails,
IOptions <EndpointOptions> endpointOptions,
MessageCenter messageCenter,
ConnectionManager connectionManager,
ConnectionCommon connectionShared)
: base(serviceProvider.GetRequiredServiceByKey <object, IConnectionListenerFactory>(ServicesKey), connectionOptions, connectionManager, connectionShared)
{
this.siloConnectionOptions = siloConnectionOptions.Value;
this.overloadDetector = overloadDetector;
this.gateway = messageCenter.Gateway;
this.localSiloDetails = localSiloDetails;
this.messageCenter = messageCenter;
this.connectionShared = connectionShared;
this.endpointOptions = endpointOptions.Value;
}
public SiloConnectionListener(
IServiceProvider serviceProvider,
IOptions <ConnectionOptions> connectionOptions,
IConnectionListenerFactory listenerFactory,
MessageCenter messageCenter,
MessageFactory messageFactory,
INetworkingTrace trace,
IOptions <EndpointOptions> endpointOptions,
ILocalSiloDetails localSiloDetails,
ISiloStatusOracle siloStatusOracle,
ConnectionManager connectionManager)
: base(serviceProvider, listenerFactory, connectionOptions, trace)
{
this.messageCenter = messageCenter;
this.messageFactory = messageFactory;
this.trace = trace;
this.localSiloDetails = localSiloDetails;
this.siloStatusOracle = siloStatusOracle;
this.connectionManager = connectionManager;
this.endpointOptions = endpointOptions.Value;
}
public void PostConfigure(string name, EndpointOptions options)
{
// Use PostConfigure to give the developer an opportunity to set SiloPort and GatewayPort using regular
// Configure methods without needing to worry about ordering with respect to the UseKubernetesHosting call.
if (options.AdvertisedIPAddress is null)
{
var hostingOptions = _serviceProvider.GetRequiredService <IOptions <KubernetesHostingOptions> >().Value;
var podIp = IPAddress.Parse(hostingOptions.PodIP);
options.AdvertisedIPAddress = podIp;
}
if (options.SiloListeningEndpoint is null)
{
options.SiloListeningEndpoint = new IPEndPoint(IPAddress.Any, options.SiloPort);
}
if (options.GatewayListeningEndpoint is null && options.GatewayPort > 0)
{
options.GatewayListeningEndpoint = new IPEndPoint(IPAddress.Any, options.GatewayPort);
}
}
public void EndpointSettingsWithPrefix()
{
var services = new ServiceCollection();
services.AddTransient <ISettingsProvider, ResolvingSettingsProvider>();
var mockProvider = new Mock <ISettingsProvider <EndpointSettings> >(MockBehavior.Strict);
EndpointSettings tempSettings = null;
mockProvider
.Setup(_ => _.TryGetSettings(out tempSettings))
.Returns(() => false);
var prefix = Guid.NewGuid().ToString();
tempSettings = new EndpointOptions();
mockProvider
.Setup(_ => _.TryGetSettings(prefix, out tempSettings))
.Returns(() => true);
services.AddSingleton(mockProvider.Object);
var serviceProvider = services.BuildServiceProvider();
using ((IDisposable)serviceProvider)
{
var provider = serviceProvider.GetRequiredService <ISettingsProvider>();
var exists = provider.TryGetSettings(prefix, out EndpointSettings settings);
Assert.True(exists);
Assert.Same(tempSettings, settings);
exists = provider.TryGetSettings(out settings);
Assert.False(exists);
}
mockProvider.Verify();
}
public GatewayConnectionListener(
IServiceProvider serviceProvider,
IOptions <ConnectionOptions> connectionOptions,
IConnectionListenerFactory listenerFactory,
MessageFactory messageFactory,
OverloadDetector overloadDetector,
Gateway gateway,
INetworkingTrace trace,
ILocalSiloDetails localSiloDetails,
IOptions <MultiClusterOptions> multiClusterOptions,
IOptions <EndpointOptions> endpointOptions,
MessageCenter messageCenter)
: base(serviceProvider, listenerFactory, connectionOptions, trace)
{
this.messageFactory = messageFactory;
this.overloadDetector = overloadDetector;
this.gateway = gateway;
this.trace = trace;
this.localSiloDetails = localSiloDetails;
this.multiClusterOptions = multiClusterOptions;
this.messageCenter = messageCenter;
this.endpointOptions = endpointOptions.Value;
}
private string PostToString(Post post, Uri siteUri, EndpointOptions endpointOptions)
{
var imageUri = endpointOptions.AddImageHtml.HasValue && endpointOptions.AddImageHtml == true && post.Files != null && post.Files.Any()
? new UriBuilder(siteUri)
{
Path = post.Files.First().Path
}.Uri + Environment.NewLine
: string.Empty;
var postUri = new UriBuilder(siteUri)
{
Path = $"{endpointOptions.BoardId}/res/{post.Parent}.html", Fragment = post.Num.ToString()
}.Uri + Environment.NewLine;
var subjectDecoded = post.SubjectDecoded();
var postSubject = !string.IsNullOrWhiteSpace(subjectDecoded) && !CommentStartsWithSubject(post.Comment, subjectDecoded)
? $"[{subjectDecoded}]" + Environment.NewLine
: string.Empty;
return(imageUri
+ postUri
+ postSubject
+ post.Comment);
}
private static void ElasticContainerServiceEndpointSettings(
EndpointOptions endpointOptions,
string ecsContainerMetadataUri)
{
var responseBody = string.Empty;
if (string.IsNullOrEmpty(ecsContainerMetadataUri))
{
throw new Exception("ECS container metadata URI cannot be null.");
}
var response = new HttpClient().GetAsync(ecsContainerMetadataUri).Result;
if (response.StatusCode == HttpStatusCode.OK)
{
responseBody = response.Content.ReadAsStringAsync().Result;
}
var ecsMetadata = JsonSerializer.Deserialize <EcsMetadata>(responseBody);
var ip = EC2InstanceMetadata.PrivateIpAddress;
var siloPort = ecsMetadata?.Ports?.FirstOrDefault(x => x.ContainerPort == DefaultSiloPort)?.HostPort ?? 0;
var gatewayPort =
ecsMetadata?.Ports?.FirstOrDefault(x => x.ContainerPort == DefaultGatewayPort)?.HostPort ?? 0;
if (ip == default || siloPort == default || gatewayPort == default)
{
throw new Exception(
$"ECS metadata retrieval failed. Values received: Ip='{ip}', SiloPort='{siloPort}', GatewayPort='{gatewayPort}'.");
}
endpointOptions.AdvertisedIPAddress = IPAddress.Parse(ip);
endpointOptions.SiloPort = siloPort;
endpointOptions.GatewayPort = gatewayPort;
endpointOptions.SiloListeningEndpoint = new IPEndPoint(IPAddress.Any, DefaultSiloPort);
endpointOptions.GatewayListeningEndpoint = new IPEndPoint(IPAddress.Any, DefaultGatewayPort);
}
private string ThreadToString(Thread thread, Uri siteUri, EndpointOptions endpointOptions)
{
var imageUri = endpointOptions.AddImageHtml.HasValue && endpointOptions.AddImageHtml == true && thread.Files != null && thread.Files.Any()
? new UriBuilder(siteUri)
{
Path = thread.Files.First().Path
}.Uri + Environment.NewLine
: string.Empty;
var threadUri = new UriBuilder(siteUri)
{
Path = $"{endpointOptions.BoardId}/res/{thread.Num}.html"
}.Uri + Environment.NewLine;
var subjectDecoded = thread.SubjectDecoded();
var threadSubject = !string.IsNullOrWhiteSpace(subjectDecoded) && !CommentStartsWithSubject(thread.Comment, subjectDecoded)
? $"[{subjectDecoded}]" + Environment.NewLine
: string.Empty;
return(imageUri
+ threadUri
+ threadSubject
+ thread.Comment);
}
public List <string> RenderAsPlainText(IEnumerable <Thread> threads, Uri siteUri, EndpointOptions endpointOptions)
{
var resultItems = threads
.Select(p => ThreadToString(p, siteUri, endpointOptions))
.ToList();
return(resultItems);
}
public async Task <List <Dto.ThreadDto.Post> > ExtractAsync(IDvachApi api, State state, EndpointOptions endpointOptions, CancellationToken cancellationToken)
{
var dvachBoard = await api.GetBoard(endpointOptions.BoardId, cancellationToken);
var extractedThreads = ExtractThreads(dvachBoard);
_logger.LogInformation($"{extractedThreads.Count} threads total in {endpointOptions.BoardId}");
var filteredThreads = FilterThreads(extractedThreads, state, endpointOptions);
_logger.LogInformation($"{filteredThreads.Count} relevant threads in {endpointOptions.BoardId}");
var extractedPosts = await ExtractPostsAsync(api, endpointOptions, filteredThreads.Select(p => p.Num).ToList(), cancellationToken);
_logger.LogInformation($"{extractedPosts.Count} posts extracted from {endpointOptions.BoardId} threads");
return(extractedPosts);
}
private List <Dto.BoardDto.Thread> FilterThreads(List <Dto.BoardDto.Thread> extractedItems, State state, EndpointOptions endpointOptions)
{
return(extractedItems
.Where(ri =>
(endpointOptions.ThreadIsSticky == null || (endpointOptions.ThreadIsSticky.Value == false && ri.Sticky == 0) || (endpointOptions.ThreadIsSticky.Value == true && ri.Sticky > 0)) &&
(endpointOptions.ThreadSubjectIncludedPatterns == null || !endpointOptions.ThreadSubjectIncludedPatterns.Any() || endpointOptions.ThreadSubjectIncludedPatterns.Any(pattern => Regex.IsMatch(ri.Subject, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))) &&
(endpointOptions.ThreadSubjectExcludedPatterns == null || !endpointOptions.ThreadSubjectExcludedPatterns.Any() || !endpointOptions.ThreadSubjectExcludedPatterns.Any(pattern => Regex.IsMatch(ri.Subject, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))) &&
(endpointOptions.ThreadCommentIncludedPatterns == null || !endpointOptions.ThreadCommentIncludedPatterns.Any() || endpointOptions.ThreadCommentIncludedPatterns.Any(pattern => Regex.IsMatch(ri.Comment, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))) &&
(endpointOptions.ThreadCommentExcludedPatterns == null || !endpointOptions.ThreadCommentExcludedPatterns.Any() || !endpointOptions.ThreadCommentExcludedPatterns.Any(pattern => Regex.IsMatch(ri.Comment, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))))
.ToList());
}
public List <ResponseItem> Filter(VkNewsfeedResult vkNewsfeedResult, State state, EndpointOptions endpointOptions)
{
var filteredItems = vkNewsfeedResult.ResponseItems.AsQueryable();
if (state.LastRecordCreatedUtc != null)
{
filteredItems = filteredItems.Where(ri => DateTimeUtils.TimestampToUtcDateTime(ri.Date) > state.LastRecordCreatedUtc.Value);
}
filteredItems = filteredItems
.Where(ri =>
(ri.MarkedAsAds == null || ri.MarkedAsAds == 0) &&
!string.IsNullOrEmpty(ri.Text) &&
(endpointOptions.IsFemale == null || ri.SignerId == null || vkNewsfeedResult.Profiles.All(p => p.Id != ri.SignerId) || IsSexMatch(vkNewsfeedResult.Profiles.First(p => p.Id == ri.SignerId), endpointOptions.IsFemale.Value)) &&
(endpointOptions.IncludedPatterns == null || !endpointOptions.IncludedPatterns.Any() || endpointOptions.IncludedPatterns.Any(pattern => Regex.IsMatch(ri.Text, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))) &&
(endpointOptions.ExcludedPatterns == null || !endpointOptions.ExcludedPatterns.Any() || !endpointOptions.ExcludedPatterns.Any(pattern => Regex.IsMatch(ri.Text, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))));
return(filteredItems
.ToList());
}
public StudentsController(StudentRepository studentRepository, IOptions <EndpointOptions> endpointOptions)
{
this.studentRepository = studentRepository;
this.endpointOptions = endpointOptions.Value;
}
public List <string> RenderAsPlainText(IEnumerable <Post> posts, Uri siteUri, EndpointOptions endpointOptions)
{
var resultItems = posts
.Select(p => PostToString(p, siteUri, endpointOptions))
.ToList();
return(resultItems);
}
public ErrorHandler(EndpointOptions options)
{
this.options = options;
}
public HomeController(IOptions <EndpointOptions> optionsAccessor)
{
this._endpointOptions = optionsAccessor.Value;
}
public BettingEndpointConnector(IOptions <EndpointOptions> endpointOptions)
{
this.endpointOptions = endpointOptions.Value;
}
public async Task <VkNewsfeedResult> ExtractAsync(ILogger logger, IVkApi api, ProviderSettings providerSettings, State state, EndpointOptions endpointOptions, CancellationToken cancellationToken)
{
VkNewsFeedGetResponse apiResult;
string startFrom = null;
var extractedProfiles = new List <Profile>();
var extractedItems = new List <ResponseItem>();
do
{
apiResult = await GetVkFeedAsync(api, providerSettings.AccessToken, startFrom, cancellationToken);
logger.LogDebug($"{nameof(GetVkFeedAsync)} result: {apiResult.Response.Items.Count} items, NextFrom = {apiResult.Response.NextFrom}");
startFrom = apiResult.Response.NextFrom;
extractedProfiles.AddRange(apiResult.Response.Profiles);
extractedItems.AddRange(apiResult.Response.Items);
} while (!string.IsNullOrWhiteSpace(startFrom) &&
(state.LastRecordCreatedUtc == null || apiResult.AreAllItemsNewerThan(state.LastRecordCreatedUtc.Value)) &&
(endpointOptions.MaxDaysFromNow == null || endpointOptions.MaxDaysFromNow == 0 || apiResult.AreAllItemsNewerThan(DateTime.UtcNow.AddDays(endpointOptions.MaxDaysFromNow.Value * -1))));
return(new VkNewsfeedResult()
{
Profiles = extractedProfiles,
ResponseItems = extractedItems.OrderBy(ri => ri.Date).ToList()
});
}
public static IServiceCollection AddEndpoints(this IServiceCollection services, EndpointOptions endpoints)
{
var map = new Dictionary <string, Type>();
if (endpoints.EnableTokenEndpoint)
{
map.Add(Constants.RoutePaths.Oidc.Token, typeof(TokenEndpoint));
}
if (endpoints.EnableDiscoveryEndpoint)
{
map.Add(Constants.RoutePaths.Oidc.DiscoveryConfiguration, typeof(DiscoveryEndpoint));
}
if (endpoints.EnableUserInfoEndpoint)
{
map.Add(Constants.RoutePaths.Oidc.UserInfo, typeof(UserInfoEndpoint));
}
if (endpoints.EnableIntrospectionEndpoint)
{
map.Add(Constants.RoutePaths.Oidc.Introspection, typeof(IntrospectionEndpoint));
}
if (endpoints.EnableAuthorizeEndpoint)
{
map.Add(Constants.RoutePaths.Oidc.Authorize, typeof(AuthorizeEndpoint));
}
services.AddInstance <IEndpointRouter>(new EndpointRouter(map));
foreach (var item in map)
{
services.AddTransient(item.Value);
}
return(services);
}
public EndpointService(IOptions <EndpointOptions> endpointOptions, ILogger <EndpointService> log)
{
_endpointOptions = endpointOptions.Value;
_log = log;
}
public VersionMiddleware(RequestDelegate next, EndpointOptions endpointOptions)
{
Next = next;
EndpointOptions = endpointOptions;
}
private async Task <List <Dto.ThreadDto.Post> > ExtractPostsAsync(IDvachApi api, EndpointOptions endpointOptions, IList <string> threadIdentifiers, CancellationToken cancellationToken)
{
var result = new List <Dto.ThreadDto.Post>();
foreach (var threadId in threadIdentifiers)
{
var threadDto = await api.GetThread(endpointOptions.BoardId, threadId, cancellationToken);
var posts = threadDto?.Threads?.FirstOrDefault()?.Posts ??
new List <Dto.ThreadDto.Post>();
result.AddRange(posts);
}
result.ForEach(p => p.Comment = StripHtml(p.Comment));
return(result.OrderBy(p => p.Timestamp).ToList());
}
public ConsoleTitleService(IOptions <EndpointOptions> endpointOptions, IOptions <OutkeepHttpApiServerOptions> httpApiOptions)
{
_endpointOptions = endpointOptions?.Value ?? throw new ArgumentNullException(nameof(endpointOptions));
_httpApiOptions = httpApiOptions?.Value ?? throw new ArgumentNullException(nameof(httpApiOptions));
}
public List <Dto.ThreadDto.Post> Filter(List <Dto.ThreadDto.Post> extractedItems, State state, EndpointOptions endpointOptions)
{
var filteredItems = extractedItems.AsQueryable();
if (state.LastRecordCreatedUtc != null)
{
filteredItems = filteredItems.Where(t => DateTimeUtils.TimestampToUtcDateTime(t.Timestamp) > state.LastRecordCreatedUtc);
}
return(filteredItems
.Where(ri =>
(endpointOptions.PostIncludedPatterns == null || !endpointOptions.PostIncludedPatterns.Any() || endpointOptions.PostIncludedPatterns.Any(pattern => Regex.IsMatch(ri.Comment, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))) &&
(endpointOptions.PostExcludedPatterns == null || !endpointOptions.PostExcludedPatterns.Any() || !endpointOptions.PostExcludedPatterns.Any(pattern => Regex.IsMatch(ri.Comment, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline))))
.ToList());
}
public SubjectsController(SubjectRepository subjectRepository, IOptions <EndpointOptions> endpointOptions, ILogger <SubjectsController> logger)
{
this.subjectRepository = subjectRepository;
this.endpointOptions = endpointOptions.Value;
this.logger = logger;
}