private void SetupNotRepeatingDailyReportTimer()
{
bool isMarketTradingDay;
DateTime openTimeUtc, closeTimeUtc;
bool isTradingHoursOK = Utils.DetermineUsaMarketTradingHours(DateTime.UtcNow, out isMarketTradingDay, out openTimeUtc, out closeTimeUtc, TimeSpan.FromDays(3));
if (!isTradingHoursOK)
{
Utils.Logger.Warn("DetermineUsaMarketTradingHours() was not ok.");
// SetupNotRepeatingDailyReportTimer() happens once per day, if DetermineUsaMarketTradingHours() fails, send email only once per day.
StrongAssert.Fail(Severity.NoException, $"DetermineUsaMarketTradingHours() failed. They probably changed their website. They do it annually. Daily HealthMonitor email at market close will not be sent.");
}
else
{
DateTime dailyReportStartTime = closeTimeUtc.AddMinutes(1.0); // run it 1 minute after close, when VBrokers finished
if (isMarketTradingDay && (DateTime.UtcNow.AddSeconds(10) < dailyReportStartTime)) // if it is only 10 seconds or less until close, don't start it.
{
Utils.Logger.Info("SetupNotRepeatingDailyReportTimer for UTC time as: " + (DateTime)dailyReportStartTime);
if (m_dailyReportTimer != null)
{
m_dailyReportTimer.Change((DateTime)dailyReportStartTime - DateTime.UtcNow, Timeout.InfiniteTimeSpan);
}
}
else
{
Utils.Logger.Info("SetupNotRepeatingDailyReportTimer not set. It is not a market day or market is closed already. ReportTimer will be set by MarketOpenTimer on the next trading day.");
}
}
}
private void CheckSpIndexChanges()
{
string url = "https://www.spglobal.com/spdji/en/indices/equity/sp-500/#news-research";
string?webpage = Utils.DownloadStringWithRetryAsync(url).TurnAsyncToSyncTask();
StrongAssert.True(!String.IsNullOrEmpty(webpage), Severity.ThrowException, "Error in Overmind.CheckSpIndexChanges(). DownloadStringWithRetry()");
if (webpage !.Length < 20000) // usually, it is 270K. If it is less than 50K, maybe an error message: "504 ERROR...The request could not be satisfied...CloudFront attempted to establish a connection with the origin"
// once per month rarely we receive "<head><title>502 Bad Gateway</title></head>"
// they have to restart their server so for 5-10 minutes, it is not available even in Chrome clients.
// in this case, sleep for 10 min, then retry
{
Utils.Logger.Warn($"CheckSpIndexChanges(). Page size is unexpectedly small: '{webpage}'");
Thread.Sleep(TimeSpan.FromMinutes(10));
webpage = Utils.DownloadStringWithRetryAsync(url).TurnAsyncToSyncTask();
StrongAssert.True(!String.IsNullOrEmpty(webpage), Severity.ThrowException, "Error in Overmind.CheckSpIndexChanges(). 2x DownloadStringWithRetry()");
}
// "<li class=\"meta-data-date\">Mar 24, 2021</li>\n <li class=\"meta-data-date\">5:15 PM</li>\n"
// Skip the first split and assume every second <li> is a Date, every second <li> is a time for that day.
// It is enough to check the first entry.
// inefficient code doing string allocations
// string[] split = webpage.Split("<li class=\"meta-data-date\">", StringSplitOptions.RemoveEmptyEntries);
// string firstDateTime = split[1].Split("</li>\n")[0].Trim() + " " + split[2].Split("</li>\n")[0].Trim();
int pos1 = webpage !.IndexOf("<li class=\"meta-data-date\">") + "<li class=\"meta-data-date\">".Length;
int pos2 = webpage.IndexOf("</li>\n", pos1);
int pos3 = webpage.IndexOf("<li class=\"meta-data-date\">", pos2 + "</li>\n".Length) + "<li class=\"meta-data-date\">".Length;
int pos4 = webpage.IndexOf("</li>\n", pos3);
ReadOnlySpan <char> pageSpan1 = webpage.AsSpan().Slice(pos1, pos2 - pos1); // 'ReadOnlySpan<char>' cannot be declared in async methods or lambda expressions.
ReadOnlySpan <char> pageSpan2 = webpage.AsSpan().Slice(pos3, pos4 - pos3);
string firstDateTimeStr = String.Concat(pageSpan1, " ", pageSpan2);
Utils.Logger.Info($"CheckSpIndexChanges() firstDateTime: '{firstDateTimeStr}'");
DateTime firstDateET = ParseSpglobalDateStr(firstDateTimeStr);
bool isLatestNewsTodayAfterClose = firstDateET.Date == DateTime.UtcNow.FromUtcToEt().Date&& firstDateET.Hour >= 16;
if (isLatestNewsTodayAfterClose)
{
string subject = "SqCore: Potential S&P500 index change is detected!";
StringBuilder sb = new StringBuilder(Email.g_htmlEmailStart);
sb.Append(@"<span class=""sqImportantOK"">Potential <strong>S&P500 index change</strong> is detected!</span><br/><br/>");
sb.Append($"The official S&P website has a <strong>new post</strong> published today in the last 5-10 minutes.<br/>");
sb.Append($"The news items are often PDF files, English sentences, which cannot be processed by a program.<br/>");
sb.Append($"A human reader is needed to understand if it is relevant.<br/><br/>");
sb.Append($"<strong>Action: </strong><br/> Go to <a href=\"{url}\">the site</a> and read the latest news.");
sb.Append(Email.g_htmlEmailEnd);
string emailHtmlBody = sb.ToString();
new Email {
ToAddresses = String.Concat(Utils.Configuration["Emails:Gyant"], ";", Utils.Configuration["Emails:Charm0"]), Subject = subject, Body = emailHtmlBody, IsBodyHtml = true
}.Send();
}
}
public ActionResult TestHealthMonitorByRaisingStrongAssert()
{
StrongAssert.Fail(Severity.NoException, "Testing TestHealthMonitorByRaisingStrongAssert() with NoException. ThrowException version of StrongAssert can survive if it is catched.");
return(Content(@"<HTML><body>TestHealthMonitorByRaisingStrongAssert() finished OK. HealthMonitor should have received the message. </body></HTML>", "text/html"));
}
internal string?GetAmazonApiResponse(string p_actionWithParams) // converted from Python code
{
string method = "GET";
string service = "ec2";
string host = "ec2.amazonaws.com";
string region = "us-east-1";
string endpoint = "https://ec2.amazonaws.com";
string access_key = Utils.Configuration["AmazonAws:AccessKey"];
string secret_key = Utils.Configuration["AmazonAws:SecretKey"];
StrongAssert.NotEmpty(access_key, Severity.ThrowException, "AmazonAwsAccessKey is not found");
StrongAssert.NotEmpty(secret_key, Severity.ThrowException, "AmazonAwsSecretKey is not found");
//Assert.False(reader.ReadBoolean());
//Assert.True(reader.ReadBoolean());
//Assert.Throws<ArgumentNullException>(() => new BlobReader(null, 1));
//Assert.Equal(0, new BlobReader(null, 0).Length); // this is valid
//Assert.Throws<BadImageFormatException>(() => new BlobReader(null, 0).ReadByte()); // but can't read anything non-empty from it...
//Assert.Same(string.Empty, new BlobReader(null, 0).ReadUtf8NullTerminated()); // can read empty string.
// Create a date for headers and the credential string
DateTime t = DateTime.UtcNow;
string amz_date = t.ToString("yyyyMMdd'T'HHmmss'Z'", CultureInfo.InvariantCulture); //.strftime('%Y%m%dT%H%M%SZ') # Format date as YYYYMMDD'T'HHMMSS'Z'
string datestamp = t.ToString("yyyyMMdd", CultureInfo.InvariantCulture); //# Date w/o time, used in credential scope
// ************* TASK 1: CREATE A CANONICAL REQUEST *************
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// Because almost all information is being passed in the query string,
// the order of these steps is slightly different than examples that
// use an authorization header.
// Step 1: Define the verb (GET, POST, etc.)--already done.
// Step 2: Create canonical URI--the part of the URI from domain to query
// string (use '/' if no path)
string canonical_uri = "/";
// Step 3: Create the canonical headers and signed headers. Header names
// and value must be trimmed and lowercase, and sorted in ASCII order.
// Note trailing \n in canonical_headers.
// signed_headers is the list of headers that are being included
// as part of the signing process. For requests that use query strings,
// only "host" is included in the signed headers.
string canonical_headers = "host:" + host + "\n";
string signed_headers = "host";
// Match the algorithm to the hashing algorithm you use, either SHA-1 or
// SHA-256 (recommended)
string algorithm = "AWS4-HMAC-SHA256";
string credential_scope = datestamp + "/" + region + "/" + service + "/" + "aws4_request";
// Step 4: Create the canonical query string. In this example, request
// parameters are in the query string. Query string values must
// be URL-encoded (space=%20). The parameters must be sorted by name.
string canonical_querystring = "Action=" + p_actionWithParams + "&Version=2015-10-01";
canonical_querystring += "&X-Amz-Algorithm=AWS4-HMAC-SHA256";
string accKeyPlusScope = access_key + "/" + credential_scope;
var encoder = System.Text.Encodings.Web.UrlEncoder.Default;
string encAccKeyPlusScope = encoder.Encode(accKeyPlusScope);
canonical_querystring += "&X-Amz-Credential=" + encAccKeyPlusScope;
canonical_querystring += "&X-Amz-Date=" + amz_date;
canonical_querystring += "&X-Amz-Expires=604800";
canonical_querystring += "&X-Amz-SignedHeaders=" + signed_headers;
// Step 5: Create payload hash. For GET requests, the payload is an
// empty string ("").
HashAlgorithm hashAlg = SHA256.Create();
string payload_hash = BitConverter.ToString(hashAlg.ComputeHash(Encoding.UTF8.GetBytes(""))).Replace("-", string.Empty).ToLower();
// Step 6: Combine elements to create create canonical request
string canonical_request = method + "\n" + canonical_uri + "\n" + canonical_querystring + "\n" + canonical_headers + "\n" + signed_headers + "\n" + payload_hash;
// ************* TASK 2: CREATE THE STRING TO SIGN*************
string string_to_sign = algorithm + "\n" + amz_date + "\n" + credential_scope + "\n" + BitConverter.ToString(hashAlg.ComputeHash(Encoding.UTF8.GetBytes(canonical_request))).Replace("-", string.Empty).ToLower();
// ************* TASK 3: CALCULATE THE SIGNATURE *************
// Create the signing key
byte[] signing_key = getSignatureKey(secret_key, datestamp, region, service);
// Sign the string_to_sign using the signing_key
HMACSHA256 hmac = new HMACSHA256(signing_key);
var computedHexDigest = BitConverter.ToString(hmac.ComputeHash(Encoding.UTF8.GetBytes(string_to_sign))).Replace("-", string.Empty).ToLower();
string signature = computedHexDigest;
//signature = hmac.new(signing_key, (string_to_sign).encode("utf-8"), hashlib.sha256).hexdigest()
// ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
// The auth information can be either in a query string
// value or in a header named Authorization. This code shows how to put
// everything into a query string.
canonical_querystring += "&X-Amz-Signature=" + signature;
// ************* SEND THE REQUEST *************
// The 'host' header is added automatically by the Python 'request' lib. But it
// must exist as a header in the request.
string request_url = endpoint + "?" + canonical_querystring;
//Utils.Logger.Info("request_url:" + request_url);
string?hmWebsiteStr = Utils.DownloadStringWithRetryAsync(request_url, 5, TimeSpan.FromSeconds(5), true).TurnAsyncToSyncTask(); // caller will handle Exceptions.
return(hmWebsiteStr);
}
void ProcessTcpClient(TcpClient p_tcpClient)
{
Utils.Logger.Info($"ProcessTcpClient() START");
TcpMessage?message = null;
try
{
BinaryReader br = new BinaryReader(p_tcpClient.GetStream());
message = (new TcpMessage()).DeserializeFrom(br);
if (message == null)
{
Console.WriteLine("<Tcp:>" + DateTime.UtcNow.ToString("MM-dd HH:mm:ss") + $" Msg: NULL"); // user can quickly check from Console the messages
Utils.Logger.Info($"<Tcp:>ProcessTcpClient: Message: NULL");
return;
}
Console.WriteLine("<Tcp:>" + DateTime.UtcNow.ToString("MM-dd HH:mm:ss") + $" Msg.ID:{message.ID}, Param:{(String.IsNullOrEmpty(message.ParamStr)?"NULL": message.ParamStr)}"); // user can quickly check from Console the messages
Utils.Logger.Info($"<Tcp:>ProcessTcpClient: Message ID:\"{ message.ID}\", ParamStr: \"{(String.IsNullOrEmpty(message.ParamStr)?"NULL": message.ParamStr)}\", ResponseFormat: \"{message.ResponseFormat}\"");
}
catch (Exception e) // Background thread can crash application. A background thread does not keep the managed execution environment running.
{
Console.WriteLine($"<Tcp:>Expected Exception. We don't rethrow it. Occurs daily when client VBroker VM server reboots or it is a second message when VBroker crashes dead. ReadTcpClientStream(BckgTh:{Thread.CurrentThread.IsBackground}). {e.Message}, InnerException: " + ((e.InnerException != null) ? e.InnerException.Message : "null"));
Utils.Logger.Info($"<Tcp:>Expected Exception. We don't rethrow it. Occurs daily when client VBroker VM server reboots or it is a second message when VBroker crashes dead. ReadTcpClientStream(BckgTh:{Thread.CurrentThread.IsBackground}). {e.Message}, InnerException: " + ((e.InnerException != null) ? e.InnerException.Message : "null"));
if (e is System.IO.EndOfStreamException) // in this case, there is no message data.
{
// If VBroker crashes totally, it sends a proper Message ID:"ReportErrorFromVirtualBroker" msg. once,
// but next it may initiate a second message, but it cannot pump the data through, because it is already crashed and all its threads are stopped.
// However, don't worry, because the first VBroker message is already under processing. So, second message can be ignored.
// the BinaryReader couldn't read the stream, so there is no message, so we dont'n know whether message.ID = VBroker or not. It is unknown. In that case, swallow the error and return, but don't crash HealthMonitor.
Utils.Logger.Info($"ProcessTcpClient: System.IO.EndOfStreamException was detected. Return without crashing HealthMonitor thread.");
return; // there is no point processing as we don't know the data. However, we still don't want to Crash Healthmonitor. So, just swallow the error.
}
else
{
// we may have message.ID and data and we may process it.
}
}
if (message == null)
{
Console.WriteLine("<Tcp:>" + DateTime.UtcNow.ToString("MM-dd HH:mm:ss") + $" Msg: NULL"); // user can quickly check from Console the messages
Utils.Logger.Info($"<Tcp:>ProcessTcpClient: Message: NULL");
return;
}
if (message.ResponseFormat == TcpMessageResponseFormat.None) // if not required to answer message, then dispose tcpClient quickly to release resources
{
Utils.TcpClientDispose(p_tcpClient);
}
Utils.Logger.Info($"ProcessTcpClient. Processing messageID {message.ID}.");
switch ((HealthMonitorMessageID)message.ID)
{
case HealthMonitorMessageID.Ping:
ServePingRequest(p_tcpClient, message);
break;
case HealthMonitorMessageID.TestHardCrash:
throw new Exception("Testing Hard Crash by Throwing this Exception");
case HealthMonitorMessageID.ReportErrorFromVirtualBroker:
case HealthMonitorMessageID.ReportWarningFromVirtualBroker:
case HealthMonitorMessageID.ReportOkFromVirtualBroker:
MessageFromVirtualBroker(p_tcpClient, message);
break;
case HealthMonitorMessageID.GetHealthMonitorCurrentStateToHealthMonitorWebsite:
CurrentStateToHealthMonitorWebsite(p_tcpClient, message);
break;
case HealthMonitorMessageID.ReportErrorFromSQLabWebsite:
case HealthMonitorMessageID.SqCoreWebCsError:
case HealthMonitorMessageID.SqCoreWebJsError:
ErrorFromWebsite(p_tcpClient, message);
break;
default:
StrongAssert.Fail(Severity.NoException, $"<Tcp:>ProcessTcpClient: Message ID:'{ message.ID}' is unexpected, unhandled. This probably means a serious error.");
break;
}
if (message.ResponseFormat != TcpMessageResponseFormat.None) // if Processing needed Response to Client, we dispose here. otherwise, it was disposed before putting into processing queue
{
Utils.TcpClientDispose(p_tcpClient);
}
Utils.Logger.Info($"ProcessTcpClient() END");
}
