private void TriggerQuickfolioNewsDownloader() // called at OnConnected and also periodically. Separete for each clients.
{
try
{
// we only send all news to the newly connected p_connId, and not All clients.
// each client has its own private m_newsDownloader, and own timer.
// this is a waste of resources, because we download things separately for each clients
// TODO Daya:
// in the future, we might store news in memory and feed the users from that, and we don't download news multiple times.
// m_newsDownloader.GetCommonNewsAndSendToClient(DashboardClient.g_clients);
m_newsDownloader.GetCommonNewsAndSendToClient(this);
m_newsDownloader.GetStockNewsAndSendToClient(this); // with 13 tickers, it can take 13 * 2 = 26seconds
nExceptionsTriggerQuickfolioNewsDownloader = 0;
}
catch
{
// It is expected that DownloadStringWithRetryAsync() throws exceptions sometimes and download fails. Around midnight.
// The request was canceled due to the configured HttpClient.Timeout of 100 seconds elapsing.
// We only inform HealthMonitor if it happened 4*4=16 times. (about 4 hours)
nExceptionsTriggerQuickfolioNewsDownloader++;
}
if (nExceptionsTriggerQuickfolioNewsDownloader >= 16)
{
Utils.Logger.Error($"TriggerQuickfolioNewsDownloader() nExceptionsTriggerQuickfolioNewsDownloader: {nExceptionsTriggerQuickfolioNewsDownloader}");
string msg = $"SqCore.TriggerQuickfolioNewsDownloader() failed {nExceptionsTriggerQuickfolioNewsDownloader}x. See log files.";
HealthMonitorMessage.SendAsync(msg, HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
nExceptionsTriggerQuickfolioNewsDownloader = 0;
}
}
private static void AppDomain_BckgThrds_UnhandledException(object p_sender, UnhandledExceptionEventArgs p_e)
{
Exception exception = (p_e.ExceptionObject as Exception) ?? new SqException($"Unhandled exception doesn't derive from System.Exception: {p_e.ToString() ?? "Null ExceptionObject"}");
Utils.Logger.Error(exception, $"AppDomain_BckgThrds_UnhandledException(). Terminating '{p_e?.IsTerminating.ToString() ?? "Null ExceptionObject"}'. Exception: '{ exception.ToStringWithShortenedStackTrace(2000)}'");
// isSendable check is not required. This background thread crash will terminate the main app. We should surely notify HealthMonitor.
string msg = $"App 'SqCore.Website' is terminated because exception in background thread. C#.AppDomain_BckgThrds_UnhandledException(). See log files.";
HealthMonitorMessage.SendAsync(msg, HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}
private void ReconnectToGatewaysTimer_Elapsed(object?p_stateObj) // Timer is coming on a ThreadPool thread
{
Utils.Logger.Info("GatewaysWatcher:ReconnectToGatewaysTimer_Elapsed() BEGIN");
try
{
bool isMainGatewayConnectedBefore = m_mainGateway != null && m_mainGateway.IsConnected;
// IB API is not async. Thread waits until the connection is established.
// Task.Run() uses threads from the thread pool, so it executes those connections parallel in the background. Then wait for them.
var reconnectTasks = m_gateways.Where(l => !l.IsConnected).Select(r => Task.Run(() => r.Reconnect()));
Task.WhenAll(reconnectTasks).TurnAsyncToSyncTask(); // "await Task.WhenAll()" has to be waited properly
Utils.Logger.Info("GatewaysWatcher:ReconnectToGateways() reconnectTasks ended.");
foreach (var gateway in m_gateways)
{
Utils.Logger.Info($"GatewayId: '{gateway.GatewayId}' IsConnected: {gateway.IsConnected}");
}
bool isMainGatewayConnectedNow = m_mainGateway != null && m_mainGateway.IsConnected;
if (!isMainGatewayConnectedBefore && isMainGatewayConnectedNow) // if this is the first time mainGateway connected after being dead
{
MainGatewayJustConnected();
}
}
catch (Exception e)
{
Utils.Logger.Info("GatewaysWatcher:TryReconnectToGateways() in catching exception (it is expected on MTS that TWS is not running, so it cannot connect): " + e.ToStringWithShortenedStackTrace(400));
}
// Without all the IB connections (isAllConnected), we can choose to crash the App, but we do NOT do that, because we may be able to recover them later.
// It is a strategic (safety vs. conveniency) decision: in that case if not all IBGW is connected, (it can be an 'expected error'), VBroker runs further and try connecting every 10 min.
// on ManualTrader server failed connection is expected. Don't send Error. However, on AutoTraderServer, it is unexpected (at the moment), because IBGateways and VBrokers restarts every day.
var notConnectedGateways = String.Join(",", m_gateways.Where(l => !l.IsConnected).Select(r => r.GatewayId + "/"));
if (!String.IsNullOrEmpty(notConnectedGateways))
{
if (IgnoreErrorsBasedOnMarketTradingTime(offsetToOpenMin: -60)) // ignore errors only before 8:30, instead of 9:30 OpenTime
{
return; // skip processing the error further. Don't send it to HealthMonitor.
}
// It can happen if somebody manually closed TWS on MTS and restarted it.
// But don't ignore for all gateways. It can be important for 'some' gateways, because SqCore server can do live trading.
// Also in the future: check IsCriticalTradingTime() usage AND
// write a service that runs at every CriticalTradingTime starts, and checks that the TradeableGatewayIds are connected
// and That should send the HealthMonitor warning, not this
HealthMonitorMessage.SendAsync($"ReconnectToGatewaysTimer() tried to connect to not connected gateways (3x, 10sec sleep). Still not connected gateways {notConnectedGateways}", HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}
Utils.Logger.Info("GatewaysWatcher:ReconnectToGatewaysTimer_Elapsed() END");
}
// Polling for changes 3x every day
// historical data can partially come from our Redis-Sql DB or partially from YF
static async Task <CompactFinTimeSeries <DateOnly, uint, float, uint>?> CreateDailyHist(Db p_db, User[] p_users, AssetsCache p_assetCache)
{
Utils.Logger.Info("ReloadHistoricalDataAndSetTimer() START");
Console.Write("*MemDb.DailyHist Download from YF: ");
DateTime etNow = DateTime.UtcNow.FromUtcToEt();
try
{
Dictionary <AssetId32Bits, List <Split> > potentialMissingYfSplits = await GetPotentianMissingYfSplits(p_db, p_assetCache);
var assetsDates = new Dictionary <uint, DateOnly[]>();
var assetsAdjustedCloses = new Dictionary <uint, float[]>();
foreach (var asset in p_assetCache.Assets)
{
(DateOnly[] dates, float[] adjCloses) = await GetDatesAndAdjCloses(asset, potentialMissingYfSplits);
if (adjCloses.Length != 0)
{
assetsDates[asset.AssetId] = dates;
assetsAdjustedCloses[asset.AssetId] = adjCloses;
Console.Write($"{asset.Symbol}, ");
Debug.Write($"{asset.SqTicker}, first: DateTime: {dates.First()}, Close: {adjCloses.First()}, last: DateTime: {dates.Last()}, Close: {adjCloses.Last()}"); // only writes to Console in Debug mode in vscode 'Debug Console'
}
}
// NAV assets should be grouped by user, because we create a synthetic new aggregatedNAV. This aggregate should add up the RAW UnadjustedNAV (not adding up the adjustedNAV), so we have to create it at MemDbReload.
var navAssets = p_assetCache.Assets.Where(r => r.AssetId.AssetTypeID == AssetType.BrokerNAV).Select(r => (BrokerNav)r);
var navAssetsByUser = navAssets.ToLookup(r => r.User); // ToLookup() uses User.Equals()
int nVirtualAggNavAssets = 0;
foreach (IGrouping <User?, BrokerNav>?navAssetsOfUser in navAssetsByUser)
{
AddNavAssetsOfUserToAdjCloses(navAssetsOfUser, p_assetCache, p_db, ref nVirtualAggNavAssets, assetsDates, assetsAdjustedCloses);
} // NAVs per user
List <DateOnly> mergedDates = UnionAllDates(assetsDates);
DateOnly[] mergedDatesArr = mergedDates.ToArray();
var values = MergeCloses(assetsDates, assetsAdjustedCloses, mergedDatesArr);
return(new CompactFinTimeSeries <DateOnly, uint, float, uint>(mergedDatesArr, values));
}
catch (Exception e)
{
Utils.Logger.Error(e, "Exception in ReloadHistoricalDataAndSetTimer()");
await HealthMonitorMessage.SendAsync($"Exception in SqCoreWebsite.C#.MemDb. Exception: '{ e.ToStringWithShortenedStackTrace(1600)}'", HealthMonitorMessageID.SqCoreWebCsError);
}
return(null);
}
// Called by the GC.FinalizerThread. Occurs when a faulted task's unobserved exception is about to trigger exception which, by default, would terminate the process.
private static void TaskScheduler_UnobservedTaskException(object?p_sender, UnobservedTaskExceptionEventArgs p_e)
{
gLogger.Error(p_e.Exception, $"TaskScheduler_UnobservedTaskException()");
string msg = $"Exception in SqCore.Website.C#.TaskScheduler_UnobservedTaskException. Exception: '{ p_e.Exception.ToStringWithShortenedStackTrace(1600)}'. ";
msg += Utils.TaskScheduler_UnobservedTaskExceptionMsg(p_sender, p_e);
gLogger.Warn(msg);
p_e.SetObserved(); // preventing it from triggering exception escalation policy which, by default, terminates the process.
bool isSendable = SqFirewallMiddlewarePreAuthLogger.IsSendableToHealthMonitorForEmailing(p_e.Exception);
if (isSendable)
{
HealthMonitorMessage.SendAsync(msg, HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}
}
public async Task <ActionResult> Index()
{
string jsLogMessage = string.Empty;
using (var reader = new StreamReader(Request.Body))
{
// example: '{"message":"A simple error() test message to NGXLogger","additional":[],"level":5,"timestamp":"2020-01-18T00:46:47.740Z","fileName":"ExampleJsClientGet.js","lineNumber":"52"}'
jsLogMessage = await reader.ReadToEndAsync();
}
// 1. just log the event to our file log
var clientIP = WsUtils.GetRequestIPv6(this.HttpContext);
var clientUserEmail = WsUtils.GetRequestUser(this.HttpContext);
if (clientUserEmail == null)
{
clientUserEmail = "*****@*****.**";
}
string jsLogMsgWithOrigin = $"Javascript Logger /JsLogController was called by '{clientUserEmail}' from '{clientIP}'. Received JS log: '{jsLogMessage}'";
Utils.Logger.Info(jsLogMsgWithOrigin);
// 2. interpret the log and if it is an error, notify HealthMonitor
try
{
var jsLogObj = JsonSerializer.Deserialize <NGXLogInterface>(jsLogMessage);
if (jsLogObj == null || jsLogObj.level == NgxLoggerLevel.ERROR || jsLogObj.level == NgxLoggerLevel.FATAL)
{ // notify HealthMonitor to send an email
await HealthMonitorMessage.SendAsync(jsLogMsgWithOrigin, HealthMonitorMessageID.SqCoreWebJsError);
}
}
catch (Exception e)
{
Utils.Logger.Error(e, "JsLogController(). Probably serialization problem. JsLogMessage: " + jsLogMessage);
throw; // if we don't rethrow it, Kestrel will not send HealthMonitor message. Although we should fix this error.
}
return(NoContent()); // The common use case is to return 204 (NoContent) as a result of a PUT request, updating a resource
}
public async Task Invoke(HttpContext httpContext)
{
// 0. This is the first entry point of ASP middleware. It is the first in the chain. Before that, there is only settings, like Rewrite() and Https redirection.
if (httpContext == null)
{
throw new ArgumentNullException(nameof(httpContext));
}
// 1. Do whitelist check first. That will sort out the most number of bad requests. Always do that filter First that results the most refusing. Try to consume less resources, so don't log it to file.
if (!IsHttpRequestOnWhitelist(httpContext))
{
// Console.WriteLine($"SqFirewall: request '{httpContext.Request.Host}' '{httpContext.Request.Path}' is not on whitelist.");
// return Unauthorized(); // can only be used in Controller. https://github.com/aspnet/Mvc/blob/rel/1.1.1/src/Microsoft.AspNetCore.Mvc.Core/ControllerBase.cs
httpContext.Response.StatusCode = StatusCodes.Status410Gone; // '410 Gone' is better than '404 Not Found'. Client will not request it later. See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
await httpContext.Response.WriteAsync($"SqFirewall: request '{httpContext.Request.Host}' '{httpContext.Request.Path}' is not on whitelist.", Encoding.UTF8);
return;
}
// 2. Do blacklists too, because whitelist might check only for prefixes. Don't push it to the next Middleware if the path or IP is on the blacklist. In the future, implement a whitelist too, and only allow requests explicitely on the whitelist.
if (IsClientIpOrPathOnBlacklist(httpContext))
{
// silently log it and stop processing
// string msg = String.Format($"{DateTime.UtcNow.ToString("HH':'mm':'ss.f")}#Blacklisted request is terminated: {httpContext.Request.Method} '{httpContext.Request.Path}' from {WsUtils.GetRequestIPv6(httpContext)}");
// Console.WriteLine(msg);
// gLogger.Info(msg);
httpContext.Response.StatusCode = StatusCodes.Status410Gone; // '410 Gone' is better than '404 Not Found'. Client will not request it later. See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
await httpContext.Response.WriteAsync($"SqFirewall: request '{httpContext.Request.Path}' is on blacklist.", Encoding.UTF8);
return;
}
Exception?exception = null;
DateTime startTime = DateTime.UtcNow;
var sw = Stopwatch.StartNew();
try
{
// If crashing in query "/signin-google", then see comment in Startup.cs:OnRemoteFailure
// if (httpContext.Request.Path.ToString().StartsWith("/signin-google"))
// Utils.Logger.Info("SqFirewallMiddlewarePreAuthLogger._next() will be called to check Google Authentication.");
await _next(httpContext); // continue in middleware app.UseAuthentication();
}
catch (Exception e)
{
// when NullReference exception was raised in TestHealthMonitorEmailByRaisingException(), The exception didn't fall to here. if
// It was handled already and I got a nice Error page to the client. So, here, we don't have the exceptions and exception messages and the stack trace.
exception = e;
Utils.Logger.Error(e, "SqFirewallMiddlewarePreAuthLogger._next() middleware");
if (e.InnerException != null)
{
Utils.Logger.Error(e, "SqFirewallMiddlewarePreAuthLogger._next() middleware. InnerException.");
}
throw;
}
finally
{
sw.Stop(); // Kestrel measures about 50ms more overhead than this measurement. Add 50ms more to estimate reaction time.
var statusCode = httpContext.Response?.StatusCode; // it may be null if there was an Exception
var level = statusCode > 499 ? Microsoft.Extensions.Logging.LogLevel.Error : Microsoft.Extensions.Logging.LogLevel.Information;
var clientIP = WsUtils.GetRequestIPv6(httpContext);
var clientUserEmail = WsUtils.GetRequestUser(httpContext);
var requestLog = new HttpRequestLog()
{
StartTime = DateTime.UtcNow,
IsHttps = httpContext.Request.IsHttps,
Method = httpContext.Request.Method,
Host = httpContext.Request.Host, // "sqcore.net" for main, "dashboard.sqcore.net" for sub-domain queries
Path = httpContext.Request.Path,
QueryString = httpContext.Request.QueryString.ToString(),
ClientIP = clientIP,
ClientUserEmail = clientUserEmail,
StatusCode = statusCode,
TotalMilliseconds = sw.Elapsed.TotalMilliseconds,
IsError = exception != null || (level == Microsoft.Extensions.Logging.LogLevel.Error),
Exception = exception
};
lock (Program.g_webAppGlobals.HttpRequestLogs) // prepare for multiple threads
{
Program.g_webAppGlobals.HttpRequestLogs.Enqueue(requestLog);
while (Program.g_webAppGlobals.HttpRequestLogs.Count > 50 * 10) // 2018-02-19: MaxHttpRequestLogs was 50, but changed to 500, because RTP (RealTimePrice) rolls 50 items out after 2 hours otherwise. 500 items will last for 20 hours.
{
Program.g_webAppGlobals.HttpRequestLogs.Dequeue();
}
}
// $"{DateTime.UtcNow.ToString("MMdd'T'HH':'mm':'ss.fff")}#
// string.Format("Value is {0}", someValue) which will check for a null reference and replace it with an empty string. It will however throw an exception if you actually pass null like this string.Format("Value is {0}", null)
string msg = String.Format("PreAuth.Postprocess: Returning {0}#{1}{2} {3} '{4} {5}' from {6} (u: {7}) ret: {8} in {9:0.00}ms", requestLog.StartTime.ToString("HH':'mm':'ss.f"), requestLog.IsError ? "ERROR in " : string.Empty, requestLog.IsHttps ? "HTTPS" : "HTTP", requestLog.Method, requestLog.Host, requestLog.Path, requestLog.ClientIP, requestLog.ClientUserEmail, requestLog.StatusCode, requestLog.TotalMilliseconds);
// string shortMsg = String.Format("{0}#{1} {2} '{3} {4}' from {5} ({6}) in {7:0.00}ms", requestLog.StartTime.ToString("HH':'mm':'ss.f"), requestLog.IsError ? "ERROR in " : string.Empty, requestLog.Method, requestLog.Host, requestLog.Path, requestLog.ClientIP, requestLog.ClientUserEmail, requestLog.TotalMilliseconds);
// Console.WriteLine(shortMsg);
gLogger.Info(msg);
if (requestLog.IsError)
{
LogDetailedContextForError(httpContext, requestLog);
}
// at the moment, send only raised Exceptions to HealthMonitor, not general IsErrors, like wrong statusCodes
if (requestLog.Exception != null && IsSendableToHealthMonitorForEmailing(requestLog.Exception))
{
StringBuilder sb = new StringBuilder("Exception in SqCore.Website.C#.SqFirewallMiddlewarePreAuthLogger. \r\n");
var requestLogStr = String.Format("{0}#{1}{2} {3} '{4}' from {5} (u: {6}) ret: {7} in {8:0.00}ms", requestLog.StartTime.ToString("HH':'mm':'ss.f"), requestLog.IsError ? "ERROR in " : string.Empty, requestLog.IsHttps ? "HTTPS" : "HTTP", requestLog.Method, requestLog.Path + (String.IsNullOrEmpty(requestLog.QueryString) ? string.Empty : requestLog.QueryString), requestLog.ClientIP, requestLog.ClientUserEmail, requestLog.StatusCode, requestLog.TotalMilliseconds);
sb.Append("Request: " + requestLogStr + "\r\n");
sb.Append("Exception: '" + requestLog.Exception.ToStringWithShortenedStackTrace(1600) + "'\r\n");
HealthMonitorMessage.SendAsync(sb.ToString(), HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}
}
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
// Asp.Net DependenciInjection (DI) of Kestrel policy for separating the creation of dependencies (IWebHostEnvironment, Options, Logger) from its actual usage in Controllers.
// That way Controllers are light. And if there is 100 Controller classes, repeating the creation of Dependent objects (IWebHostEnvironment) is not in their source code. So, the source code of Controllers are light.
// DI is not necessary. DotNet core bases classes doesn't use that for logging or anything. However, Kestrel uses it, which we can honour. It also helps in unit-test.
// But it is perfectly fine to do the Creation of dependencies (Logger, like nLog) to do it in the Controller.
// Transient objects are always different; a new instance is provided to every controller and every service.
// Scoped objects are the same within a request, but different across different requests
// Singleton objects are the same for every object and every request(regardless of whether an instance is provided in ConfigureServices)
services.AddSingleton(_ => Utils.Configuration); // this is the proper DependenciInjection (DI) way of pushing it as a service to Controllers. So you don't have to manage the creation or disposal of instances.
services.AddSingleton(_ => Program.g_webAppGlobals);
services.AddHttpsRedirection(options =>
{
options.RedirectStatusCode = StatusCodes.Status307TemporaryRedirect;
options.HttpsPort = 5001;
});
// https://docs.microsoft.com/en-us/aspnet/core/performance/caching/response?view=aspnetcore-3.0
services.AddResponseCaching(); // DI: these services could be used in MVC control/Razor pages (either as [Attributes], or in code)
services.AddMvc(options => // AddMvc() equals AddControllersWithViews() + AddRazorPages()
{ // this CashProfile is given once here, and if it changes, we only have to change here, not in all Controllers.
options.CacheProfiles.Add("NoCache",
new CacheProfile()
{
Duration = 0,
Location = ResponseCacheLocation.None,
NoStore = true
});
options.CacheProfiles.Add("DefaultShortDuration",
new CacheProfile()
{
Duration = 60 * 1 // 1 min for real-time price data
});
options.CacheProfiles.Add("DefaultMidDuration",
new CacheProfile()
{
//Duration = (int)TimeSpan.FromHours(12).TotalSeconds
Duration = 100000 // 100,000 seconds = 27 hours
});
}).SetCompatibilityVersion(CompatibilityVersion.Version_3_0);
//services.AddControllersWithViews(); // AddMvc() equals AddControllersWithViews() + AddRazorPages(), so we don't use Razor pages now.
// In production, the Angular files (index.html) will be served from this directory, but actually we don't use UseSpaStaticFiles(), so we don't need this here.
services.AddSpaStaticFiles(configuration =>
{
configuration.RootPath = "Angular/dist";
});
// https://docs.microsoft.com/en-us/aspnet/core/performance/response-compression
services.AddResponseCompression(options =>
{
options.Providers.Add <BrotliCompressionProvider>();
options.Providers.Add <GzipCompressionProvider>();
// Default Mime types: application/javascript, application/json, application/xml, text/css, text/html, text/json, text/plain, text/xml
options.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat(new[] { "image/svg+xml" });
});
services.Configure <BrotliCompressionProviderOptions>(options =>
{
options.Level = CompressionLevel.Fastest;
});
string googleClientId = Utils.Configuration["Google:ClientId"];
string googleClientSecret = Utils.Configuration["Google:ClientSecret"];
if (!String.IsNullOrEmpty(googleClientId) && !String.IsNullOrEmpty(googleClientSecret))
{
// The reason you have BOTH google and cookies Auth is because you're using google for identity information but using cookies for storage of the identity for only asking Google once.
//So AddIdentity() is not required, but Cookies Yes.
services.AddAuthentication(options =>
{
// If you don't want the cookie to be automatically authenticated and assigned to HttpContext.User,
// remove the CookieAuthenticationDefaults.AuthenticationScheme parameter passed to AddAuthentication.
options.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme; // For anything else (sign in, sign out, authenticate, forbid), use the cookies scheme
options.DefaultChallengeScheme = GoogleDefaults.AuthenticationScheme; // For challenges, use the google scheme. If not, "InvalidOperationException: No authenticationScheme was specified"
options.DefaultAuthenticateScheme = CookieAuthenticationDefaults.AuthenticationScheme;
options.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme;
})
.AddCookie(o => { // CookieAuth will be the default from the two, GoogleAuth is used only for Challenge
o.LoginPath = "/UserAccount/login";
o.LogoutPath = "/UserAccount/logout";
// 2020-05-30: WARN|Microsoft.AspNetCore.Authentication.Google.GoogleHandler: '.AspNetCore.Correlation.Google.bzb7A4oxoS_pz_xQk0N4WngqgL0nyLUiT0k5QSPsD_M' cookie not found.
// "Exception: Correlation failed.".
// Maybe because SameSite cookies policy changed.
// I suspect Bunny used an old Chrome or FFox or Edge.
// "AspNetCore as a rule does not implement browser sniffing for you because User-Agents values are highly unstable"
// However, if updating browser of the user to the latest Chrome doesn't solve it, we may implement these changes:
// https://github.com/dotnet/aspnetcore/issues/14996
// https://docs.microsoft.com/en-us/aspnet/core/security/samesite?view=aspnetcore-3.1
// "Cookies without SameSite header are treated as SameSite=Lax by default.
// SameSite=None must be used to allow cross-site cookie use.
// Cookies that assert SameSite=None must also be marked as Secure. (requires HTTPS)"
// 2020-01: 'Correlation failed.' is a Browser Cache problem. 2020-06-03: JMC could log in. Error email 'correlation failed' arrived. When I used F12 in Chrome, disabled cache; then login went OK.
// 2020-08: Chrome implements this default behavior as of version 84. (2020-08). Edge doesn't restrict that yet.
// without any intervention, http://localhost/login returns this to the browser: ""Set-Cookie: .AspNetCore.Correlation.Google._AcFoUd0-sbBMoGfefWKA2WlqpVJwD2bGYTYs6axoBU=N; expires=Fri, 14 Aug 2020 14:45:30 GMT; path=/signin-google; samesite=none; httponly"
// and Chrome throws an Error to JsConsole: "A cookie associated with a resource at http://localhost/ was set with `SameSite=None` but without `Secure`. It has been blocked"
// disable this feature by going to "chrome://flags" and disabling "Cookies without SameSite must be secure", but it is good for development only
// So, from now on, because we want to use Chrome84+, if we want login, we have to develop in HTTPS mode, not HTTP. We can completely forget HTTP. Just use HTTPS, even in DEV.
// >GoogleAuth Login system uses cookie (.AspNetCore.Correlation.Google). From 2020-08, Chrome blocks a SameSite=None, which is not Secure.
// But Secure means it is running on HTTPS. So, local development will also need to be done with HTTPS urls.
// >Specify SameSite=None and Secure if the cookie should be sent in cross-site requests. This enables third-party use.
// Specify SameSite=Strict or SameSite=Lax if the cookie should not be sent in cross-site requests.
// But even in this case, if we use Both HTTP, HTTPS at development, Login problems arise on HTTP.
// >Chrome debug: cookie HTTP://".AspNetCore.Cookies": "This set-cookie was blocked because it was not sent over a secure connection and would have overwritten a cookie with a secure attribute.",
// but then that Secure HTTPS cookie with the same name is not sent to the non-secure HTTP request. (It is only sent to the HTTPS request).
// Therefore, we should use only the HTTPS protocol, even in local development. (except if AWS CloudFront cannot handle HTTPS to HTTPS conversions)
// See cookies: Facebook and Google logins only work in HTTPS (even locally), and because we want in Local development the same experience as is production, we eliminate HTTP in local development
// Cookies are shared between ports. So, https://localhost:5001/ and https://localhost:443 share the same cookie (login info), but http://localhost:5000/ cannot overwrite that cookie in Chrome
// https://docs.microsoft.com/en-us/aspnet/core/security/samesite?view=aspnetcore-3.1
o.Cookie.SameSite = SameSiteMode.Lax; // sets the cookie ".AspNetCore.Cookies"
o.Cookie.SecurePolicy = CookieSecurePolicy.Always; // Note this will also require you to be running on HTTPS. Local development will also need to be done with HTTPS urls.
// o.Cookie.SecurePolicy = CookieSecurePolicy.SameAsRequest; // this is the default BTW, so no need to set.
// problem: if Cookie storage works in https://localhost:5001/UserAccount/login but not in HTTP: http://localhost:5000/UserAccount/login
// "Note that the http page cannot set an insecure cookie of the same name as the secure cookie."
// Solution: Manually delete the cookie from Chrome. see here. https://bugs.chromium.org/p/chromium/issues/detail?id=843371
// in Production, only HTTPS is allowed anyway, so it will work. Best is not mix development in both HTTP/HTTPS (just stick to one of them).
// stick to HTTPS. Although Chrome browser-caching will not work in HTTPS (because of insecure cert), it is better to test HTTPS, because that will be the production.
// Controls how much time the authentication ticket stored in the cookie will remain valid
// This is separate from the value of Microsoft.AspNetCore.Http.CookieOptions.Expires, which specifies how long the browser will keep the cookie. We will set that in OnTicketReceived()
o.ExpireTimeSpan = TimeSpan.FromDays(350); // allow 1 year expiration.
})
.AddGoogle("Google", options =>
{
options.ClientId = googleClientId;
options.ClientSecret = googleClientSecret;
options.CorrelationCookie.SameSite = SameSiteMode.Lax; // sets the cookie ".AspNetCore.Correlation.Google.*"
options.CorrelationCookie.SecurePolicy = CookieSecurePolicy.Always;
// Note: Once logged in to Google Ecosystem (and once allowed Sqcore website), the Google login prompt (offering different users) does not even display.
// Do we want it displayed? Probably NOT. Because this is good and fast:
// "the Google login prompt does not even display. From the app I get redirected to Google,
// and because I am already signed in with a user with that domain, Google immediately returns that as the authenticated user to your app."
// If you really want to logout that Guser from SqCore: sign out of your Google account (in Gmail, GDrive or any G.app), or open an Incognito browser window.
// >e.g. go do GoogleDrive: log-out as user. After that SqCore will ask the user login user only once. But that login will login to ALL Google services.
// Which is actually fine. That is what I want. Once user logged in to his Gmail, he can enjoy SqCore without logging in again.
// So, the same way, why GoogleDrive doesn't re-ask the password every time, the same applies here too.
options.Events = new OAuthEvents
{
// https://www.jerriepelser.com/blog/forcing-users-sign-in-gsuite-domain-account/
OnRedirectToAuthorizationEndpoint = context =>
{
Utils.Logger.Info("GoogleAuth.OnRedirectToAuthorizationEndpoint()");
//context.Response.Redirect(context.RedirectUri + "&hd=" + System.Net.WebUtility.UrlEncode("jerriepelser.com"));
context.Response.Redirect(context.RedirectUri, false); // Temporary redirect response (HTTP 302)
return(Task.CompletedTask);
},
OnCreatingTicket = context =>
{
Utils.Logger.Info("GoogleAuth.OnCreatingTicket(), User: "******"[Authorize] attribute forced Google auth. Email:'{email ?? "null"}', RedirectUri: '{context.Properties.RedirectUri ?? "null"}'");
// if (!Utils.IsAuthorizedGoogleUsers(Utils.Configuration, email))
// throw new Exception($"Google Authorization Is Required. Your Google account: '{ email }' is not accepted. Logout this Google user and login with another one.");
//string domain = context.User.Value<string>("domain");
//if (domain != "jerriepelser.com")
// throw new GoogleAuthenticationException("You must sign in with a jerriepelser.com email address");
return(Task.CompletedTask);
},
OnTicketReceived = context =>
{
Utils.Logger.Info("GoogleAuth.OnTicketReceived()");
// if this is not set, then the cookie in the browser expires, even though the validation-info in the cookie is still valid. By default, cookies expire: "When the browsing session ends" Expires: 'session'
// https://www.jerriepelser.com/blog/managing-session-lifetime-aspnet-core-oauth-providers/
context.Properties.IsPersistent = true;
context.Properties.ExpiresUtc = DateTimeOffset.UtcNow.AddDays(25);
return(Task.FromResult(0));
},
OnRemoteFailure = remoteFailureContext =>
{
Utils.Logger.Error("GoogleAuth.OnRemoteFailure()");
Console.WriteLine("Error! GoogleAuth.OnRemoteFailure(");
// 2021-07-06: Daya had login problems on localhost:5001 only.
// "/signin-google?...<signin-token>" crashed in SqFirewallMiddlewarePreAuthLogger: _await _next(httpContext);
// "Microsoft.AspNetCore.Authentication.Google.GoogleHandler: Information: Error from RemoteAuthentication: A task was canceled.."
// StackTrace:
// at System.Net.Http.HttpConnectionPool.GetHttpConnectionAsync()
// ...
// at System.Net.Http.HttpClient.SendAsyncCore()
// ..
// at Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context)
// at SqCoreWeb.SqFirewallMiddlewarePreAuthLogger.Invoke(HttpContext httpContext)
// It seems that when Google returns our token in "/signin-google?...<signin-token>", ASP.Net core downloads something from GoogleServer with HttpClient
// That failed. We don't know the reason yet.
// It was quickly solved by that Daya changed his internet service provider to BackupInternet2 (maybe mobile-internet). With that internet connection this HttpClient download didn't fail.
// Some people in the forums complain about the same things and said that they solved it by setting up a Proxy.
// So, something is weird with Daya's Main internet service provider. He will have a new fiber optic internet in a week.
// https://github.com/googleapis/google-api-dotnet-client/issues/1394
// https://github.com/Clancey/SimpleAuth/issues/41 " if there is no [proper, secure] internet connection ... it results in a TaskCanceledException"
HealthMonitorMessage.SendAsync("GoogleAuth.OnRemoteFailure(). See comments in code.", HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
return(Task.FromResult(0));
}
};
});
}
else
{
Console.WriteLine("A_G_CId and A_G_CSe from Config has NOT been found. Cannot initialize GoogelAuthentication.");
//Utils.Logger.Warn("A_G_CId and A_G_CSe from Config has NOT been found. Cannot initialize GoogelAuthentication.");
}
}
public static void Main(string[] args) // entry point Main cannot be flagged as async, because at first await, Main thread would go back to Threadpool, but that terminates the Console app
{
string appName = System.Reflection.MethodBase.GetCurrentMethod()?.ReflectedType?.Namespace ?? "UnknownNamespace";
string systemEnvStr = $"(v1.0.15,{Utils.RuntimeConfig() /* Debug | Release */},CLR:{System.Environment.Version},{System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription},OS:{System.Environment.OSVersion},usr:{System.Environment.UserName},CPU:{System.Environment.ProcessorCount},ThId-{Thread.CurrentThread.ManagedThreadId})";
Console.WriteLine($"Hi {appName}.{systemEnvStr}");
gLogger.Info($"********** Main() START {systemEnvStr}");
// Setting Console.Title
// on Linux use it only in GUI mode. It works with graphical Xterm in VNC, but with 'screen' or with Putty it is buggy and after this, the next 200 characters are not written to console.
// Future work if needed: bring a flag to use it in string[] args, but by default, don't set Title on Linux
if (Utils.RunningPlatform() != Platform.Linux) // https://stackoverflow.com/questions/47059468/get-or-set-the-console-title-in-linux-and-macosx-with-net-core
{
Console.Title = $"{appName} v1.0.15"; // "SqCoreWeb v1.0.15"
}
gHeartbeatTimer = new System.Threading.Timer((e) => // Heartbeat log is useful to find out when VM was shut down, or when the App crashed
{
Utils.Logger.Info($"**g_nHeartbeat: {gNheartbeat} (at every {cHeartbeatTimerFrequencyMinutes} minutes)");
gNheartbeat++;
}, null, TimeSpan.FromMinutes(0.5), TimeSpan.FromMinutes(cHeartbeatTimerFrequencyMinutes));
string sensitiveConfigFullPath = Utils.SensitiveConfigFolderPath() + $"SqCore.WebServer.{appName}.NoGitHub.json";
string systemEnvStr2 = $"Current working directory of the app: '{Directory.GetCurrentDirectory()}',{Environment.NewLine}SensitiveConfigFullPath: '{sensitiveConfigFullPath}'";
gLogger.Info(systemEnvStr2);
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory()) // GetCurrentDirectory() is the folder of the '*.csproj'.
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) // no need to copy appsettings.json to the sub-directory of the EXE.
.AddJsonFile(sensitiveConfigFullPath, optional: true, reloadOnChange: true);
//.AddUserSecrets<Program>() // Used mostly in Development only, not in Production. Stored in a JSON configuration file in a system-protected user profile folder on the local machine. (e.g. user's %APPDATA%\Microsoft\UserSecrets\), the secret values aren't encrypted, but could be in the future.
// do we need it?: No. Sensitive files are in separate folders, not up on GitHub. If server is not hacked, we don't care if somebody who runs the code can read the settings file. Also, scrambling secret file makes it more difficult to change it realtime.
//.AddEnvironmentVariables(); // not needed in general. We dont' want to clutter op.sys. environment variables with app specific values.
Utils.Configuration = builder.Build();
Utils.MainThreadIsExiting = new ManualResetEventSlim(false);
// HealthMonitorMessage.InitGlobals(ServerIp.HealthMonitorPublicIp, ServerIp.DefaultHealthMonitorServerPort); // until HealthMonitor runs on the same Server, "localhost" is OK
Email.SenderName = Utils.Configuration["Emails:HQServer"];
Email.SenderPwd = Utils.Configuration["Emails:HQServerPwd"];
PhoneCall.TwilioSid = Utils.Configuration["PhoneCall:TwilioSid"];
PhoneCall.TwilioToken = Utils.Configuration["PhoneCall:TwilioToken"];
PhoneCall.PhoneNumbers[Caller.Gyantal] = Utils.Configuration["PhoneCall:PhoneNumberGyantal"];
PhoneCall.PhoneNumbers[Caller.Charmat0] = Utils.Configuration["PhoneCall:PhoneNumberCharmat0"];
StrongAssert.g_strongAssertEvent += StrongAssertMessageSendingEventHandler;
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_BckgThrds_UnhandledException);
TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException; // Occurs when a faulted task's unobserved exception is about to trigger exception which, by default, would terminate the process.
try
{
// 1. PreInit services. They might add callbacks to MemDb's events.
DashboardClient.PreInit(); // services add handlers to the MemDb.EvMemDbInitialized event.
// 2. Init services
var redisConnString = (Utils.RunningPlatform() == Platform.Windows) ? Utils.Configuration["ConnectionStrings:RedisDefault"] : Utils.Configuration["ConnectionStrings:RedisLinuxLocalhost"];
int redisDbIndex = 0; // DB-0 is ProductionDB. DB-1+ can be used for Development when changing database schema, so the Production system can still work on the ProductionDB
var db = new Db(redisConnString, redisDbIndex, null); // mid-level DB wrapper above low-level DB
MemDb.gMemDb.Init(db); // high level DB used by functionalities
BrokersWatcher.gWatcher.Init();
Caretaker.gCaretaker.Init("SqCoreServer", Utils.Configuration["Emails:ServiceSupervisors"], p_needDailyMaintenance: true, TimeSpan.FromHours(2));
SqTaskScheduler.gTaskScheduler.Init();
Services_Init();
// 3. Run services.
// Create a dedicated thread for a single task that is running for the lifetime of my application.
KestrelWebServer_Run(args);
string userInput = string.Empty;
do
{
userInput = DisplayMenuAndExecute().Result; // we cannot 'await' it, because Main thread would terminate, which would close the whole Console app.
} while (userInput != "UserChosenExit" && userInput != "ConsoleIsForcedToShutDown");
}
catch (Exception e)
{
gLogger.Error(e, $"CreateHostBuilder(args).Build().Run() exception.");
if (e is System.Net.Sockets.SocketException)
{
gLogger.Error("Linux. See 'Allow non-root process to bind to port under 1024.txt'. If Dotnet.exe was updated, it lost privilaged port. Try 'whereis dotnet','sudo setcap 'cap_net_bind_service=+ep' /usr/share/dotnet/dotnet'.");
}
HealthMonitorMessage.SendAsync($"Exception in SqCoreWebsite.C#.MainThread. Exception: '{ e.ToStringWithShortenedStackTrace(1600)}'", HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}
Utils.MainThreadIsExiting.Set(); // broadcast main thread shutdown
// 4. Try to gracefully stop services.
KestrelWebServer_Stop();
int timeBeforeExitingSec = 2;
Console.WriteLine($"Exiting in {timeBeforeExitingSec}sec...");
Thread.Sleep(TimeSpan.FromSeconds(timeBeforeExitingSec)); // give some seconds for long running background threads to quit
// 5. Dispose service resources
KestrelWebServer_Exit();
Services_Exit();
SqTaskScheduler.gTaskScheduler.Exit();
Caretaker.gCaretaker.Exit();
BrokersWatcher.gWatcher.Exit();
MemDb.gMemDb.Exit();
gLogger.Info("****** Main() END");
NLog.LogManager.Shutdown();
}
internal static void StrongAssertMessageSendingEventHandler(StrongAssertMessage p_msg)
{
gLogger.Info("StrongAssertEmailSendingEventHandler()");
HealthMonitorMessage.SendAsync($"Msg from SqCore.Website.C#.StrongAssert. StrongAssert Warning (if Severity is NoException, it is just a mild Warning. If Severity is ThrowException, that exception triggers a separate message to HealthMonitor as an Error). Severity: {p_msg.Severity}, Message: { p_msg.Message}, StackTrace: { p_msg.StackTrace.ToStringWithShortenedStackTrace(1600)}", HealthMonitorMessageID.SqCoreWebCsError).TurnAsyncToSyncTask();
}