private async Task GetComputerInfoAsync(CancellationToken token)
{
var sb = new StringBuilder();
int logicalProcessorCount = Environment.ProcessorCount;
try
{
OSInfo osInfo = await OperatingSystemInfoProvider.Instance.GetOSInfoAsync(token);
this.osStatus = osInfo.Status;
// Active, bound ports.
int activePorts = OperatingSystemInfoProvider.Instance.GetActivePortCount();
// Active, ephemeral ports.
int activeEphemeralPorts = OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount();
(int lowPortOS, int highPortOS) = OperatingSystemInfoProvider.Instance.TupleGetDynamicPortRange();
string osEphemeralPortRange = string.Empty;
string fabricAppPortRange = string.Empty;
string clusterManifestXml = IsTestRun ? File.ReadAllText(
TestManifestPath) : await FabricClientInstance.ClusterManager.GetClusterManifestAsync(
AsyncClusterOperationTimeoutSeconds, Token).ConfigureAwait(false);
(int lowPortApp, int highPortApp) =
NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(
FabricServiceContext.NodeContext.NodeType,
clusterManifestXml);
int firewalls = NetworkUsage.GetActiveFirewallRulesCount();
// OS info.
_ = sb.AppendLine("OS Information:\r\n");
_ = sb.AppendLine($"Name: {osInfo.Name}");
_ = sb.AppendLine($"Version: {osInfo.Version}");
if (string.IsNullOrEmpty(osInfo.InstallDate))
{
_ = sb.AppendLine($"InstallDate: {osInfo.InstallDate}");
}
_ = sb.AppendLine($"LastBootUpTime*: {osInfo.LastBootUpTime}");
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// WU AutoUpdate - Download enabled.
// If the config setting EnableWindowsAutoUpdateCheck is set to false, then don't add this info to sb.
if (this.isWUADSettingEnabled)
{
string auMessage = "WindowsUpdateAutoDownloadEnabled: ";
if (this.auStateUnknown)
{
auMessage += "Unknown";
}
else
{
auMessage += this.isWindowsUpdateAutoDownloadEnabled;
}
_ = sb.AppendLine(auMessage);
}
// Not supported for Linux.
_ = sb.AppendLine($"OSLanguage: {osInfo.Language}");
_ = sb.AppendLine($"OSHealthStatus*: {osInfo.Status}");
}
_ = sb.AppendLine($"NumberOfProcesses*: {osInfo.NumberOfProcesses}");
if (lowPortOS > -1)
{
osEphemeralPortRange = $"{lowPortOS} - {highPortOS}";
_ = sb.AppendLine($"OSEphemeralTCPPortRange: {osEphemeralPortRange} (Active*: {activeEphemeralPorts})");
}
if (lowPortApp > -1)
{
fabricAppPortRange = $"{lowPortApp} - {highPortApp}";
_ = sb.AppendLine($"FabricApplicationTCPPortRange: {fabricAppPortRange}");
}
if (firewalls > -1)
{
_ = sb.AppendLine($"ActiveFirewallRules*: {firewalls}");
}
if (activePorts > -1)
{
_ = sb.AppendLine($"TotalActiveTCPPorts*: {activePorts}");
}
// Hardware info.
// Proc/Mem
_ = sb.AppendLine($"{Environment.NewLine}Hardware Information:{Environment.NewLine}");
_ = sb.AppendLine($"LogicalProcessorCount: {logicalProcessorCount}");
if (osInfo.TotalVirtualMemorySizeKB > 0)
{
_ = sb.AppendLine($"TotalVirtualMemorySize: {osInfo.TotalVirtualMemorySizeKB / 1048576} GB");
}
if (osInfo.TotalVisibleMemorySizeKB > 0)
{
_ = sb.AppendLine($"TotalVisibleMemorySize: {osInfo.TotalVisibleMemorySizeKB / 1048576} GB");
}
_ = sb.AppendLine($"FreePhysicalMemory*: {Math.Round(osInfo.AvailableMemoryKB / 1048576.0, 2)} GB");
_ = sb.AppendLine($"FreeVirtualMemory*: {Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2)} GB");
// Disk
var drivesInformationTuple = DiskUsage.GetCurrentDiskSpaceTotalAndUsedPercentAllDrives(SizeUnit.Gigabytes);
var logicalDriveCount = drivesInformationTuple.Count;
string driveInfo = string.Empty;
_ = sb.AppendLine($"LogicalDriveCount: {logicalDriveCount}");
foreach (var(driveName, diskSize, percentConsumed) in drivesInformationTuple)
{
string drvSize;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
string systemDrv = "Data";
if (string.Equals(Environment.SystemDirectory.Substring(0, 1), driveName.Substring(0, 1), StringComparison.OrdinalIgnoreCase))
{
systemDrv = "System";
}
drvSize = $"Drive {driveName} ({systemDrv}) Size: {diskSize} GB, Consumed*: {percentConsumed}%";
}
else
{
drvSize = $"Mount point: {driveName}, Size: {diskSize} GB, Consumed*: {percentConsumed}%";
}
_ = sb.AppendLine(drvSize);
driveInfo += $"{drvSize}{Environment.NewLine}";
}
string osHotFixes = string.Empty;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
osHotFixes = GetWindowsHotFixes(token);
}
if (!string.IsNullOrEmpty(osHotFixes))
{
_ = sb.AppendLine($"\nWindows Patches/Hot Fixes*:\n\n{osHotFixes}");
}
// Dynamic info qualifier (*)
_ = sb.AppendLine($"\n* Dynamic data.");
this.osReport = sb.ToString();
string hotFixes = string.Empty;
// ETW.
if (IsEtwEnabled)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
hotFixes = GetWindowsHotFixes(token, generateUrl: false).Replace("\r\n", ", ").TrimEnd(',');
}
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
HealthState = "Ok",
Node = NodeName,
Observer = ObserverName,
OS = osInfo.Name,
OSVersion = osInfo.Version,
OSInstallDate = osInfo.InstallDate,
AutoUpdateEnabled = this.auStateUnknown ? "Unknown" : this.isWindowsUpdateAutoDownloadEnabled.ToString(),
osInfo.LastBootUpTime,
WindowsAutoUpdateEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = (int)(osInfo.TotalVisibleMemorySizeKB / 1048576),
AvailablePhysicalMemoryGB = Math.Round(osInfo.FreePhysicalMemoryKB / 1048576.0, 2),
AvailableVirtualMemoryGB = Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = osInfo.NumberOfProcesses,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = hotFixes,
});
}
// Telemetry
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
if (string.IsNullOrEmpty(hotFixes) && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
hotFixes = GetWindowsHotFixes(token, generateUrl: false).Replace("\r\n", ", ").TrimEnd(',');
}
TelemetryClient?.ReportMetricAsync(
new MachineTelemetryData
{
HealthState = "Ok",
Node = NodeName,
Observer = ObserverName,
OS = osInfo.Name,
OSVersion = osInfo.Version,
OSInstallDate = osInfo.InstallDate,
LastBootUpTime = osInfo.LastBootUpTime,
WindowsUpdateAutoDownloadEnabled = this.isWindowsUpdateAutoDownloadEnabled,
TotalMemorySizeGB = (int)osInfo.TotalVisibleMemorySizeKB / 1048576,
AvailablePhysicalMemoryGB = Math.Round(osInfo.FreePhysicalMemoryKB / 1048576.0, 2),
AvailableVirtualMemoryGB = Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2),
LogicalProcessorCount = logicalProcessorCount,
LogicalDriveCount = logicalDriveCount,
DriveInfo = driveInfo,
NumberOfRunningProcesses = osInfo.NumberOfProcesses,
ActiveFirewallRules = firewalls,
ActivePorts = activePorts,
ActiveEphemeralPorts = activeEphemeralPorts,
WindowsDynamicPortRange = osEphemeralPortRange,
FabricAppPortRange = fabricAppPortRange,
HotFixes = hotFixes,
}, Token);
}
}
catch (Exception e) when(e is FabricException || e is OperationCanceledException || e is TaskCanceledException || e is InvalidComObjectException)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
$"Handled Exception processing OS information:{Environment.NewLine}{e}");
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Error,
$"Unhandled Exception processing OS information:{Environment.NewLine}{e}");
throw;
}
}
public async Task CanInitializeAsync()
{
Assert.Throws <ArgumentNullException>(() => new TotServer(null));
var serverEndPoint = new IPEndPoint(IPAddress.Loopback, new Random().Next(5000, 5500));
var server = new TotServer(serverEndPoint);
await server.StopAsync(); // make sure calling stop doesn't throw exception
var tcpClient = new TcpClient();
var tcpClient2 = new TcpClient();
var tcpClient3 = new TcpClient();
var tcpClient4 = new TcpClient();
try
{
await server.StartAsync();
Assert.Throws <ConnectionException>(() => new TotClient(tcpClient));
await tcpClient.ConnectAsync(serverEndPoint.Address, serverEndPoint.Port);
await tcpClient3.ConnectAsync(serverEndPoint.Address, serverEndPoint.Port);
await tcpClient4.ConnectAsync(serverEndPoint.Address, serverEndPoint.Port);
var totClient3 = new TotClient(tcpClient3);
var totClient4 = new TotClient(tcpClient4);
await totClient4.StartAsync();
await totClient4.StopAsync();
await server.StopAsync();
// tcpClient doesn't know if the server has stopped, so it will work
var totClient = new TotClient(tcpClient);
await totClient3.StopAsync(); // make sure it doesn't throw exception
Assert.Throws <ConnectionException>(() => totClient3 = new TotClient(tcpClient3));
await totClient.StartAsync(); // Start will not fail, but rather retry periodically
totClient.Disconnected += TotClient_Disconnected_CanInitializeAsync;
while (0 == Interlocked.Read(ref _totClient_Disconnected_CanInitializeAsyncCalled))
{
await Task.Delay(10);
}
totClient.Disconnected -= TotClient_Disconnected_CanInitializeAsync;
await totClient.StopAsync();
var thrownSocketExceptionFactoryExtendedSocketException = false;
try
{
await tcpClient2.ConnectAsync(serverEndPoint.Address, serverEndPoint.Port);
}
// this will be the uncatchable SocketExceptionFactory+ExtendedSocketException
catch (Exception ex) when(ex.Message.StartsWith("No connection could be made because the target machine actively refused it"))
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
thrownSocketExceptionFactoryExtendedSocketException = true;
}
}
catch (Exception ex) when(ex.Message.StartsWith("Connection refused"))
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
thrownSocketExceptionFactoryExtendedSocketException = true;
}
}
Assert.True(thrownSocketExceptionFactoryExtendedSocketException);
server = new TotServer(serverEndPoint);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
await server.StartAsync();
await tcpClient2.ConnectAsync(serverEndPoint.Address, serverEndPoint.Port);
var totClient2 = new TotClient(tcpClient2);
await totClient2.StartAsync();
await totClient2.StopAsync();
}
else // on non-windows platforms this is not possible (address already in use
{
await Assert.ThrowsAsync <SocketException>(async() => await server.StartAsync());
}
}
finally
{
await server.StopAsync();
}
}
public KafkaConfluentConsumerDeclareTests(ITestOutputHelper output)
{
string SupplyCertificateLocation()
{
//For different platforms, we have to figure out how to get the connection right
//see: https://docs.confluent.io/platform/current/tutorials/examples/clients/docs/csharp.html
return(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "/usr/local/etc/[email protected]/cert.pem" : null);
}
// -- Confluent supply these values, see their .NET examples for your account
// You need to set those values as environment variables, which we then read, in order
// to run these tests
const string groupId = "Kafka Message Producer Send Test";
string bootStrapServer = Environment.GetEnvironmentVariable("CONFLUENT_BOOSTRAP_SERVER");
string userName = Environment.GetEnvironmentVariable("CONFLUENT_SASL_USERNAME");
string password = Environment.GetEnvironmentVariable("CONFLUENT_SASL_PASSWORD");
_output = output;
_producer = new KafkaMessageProducerFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { bootStrapServer },
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanisms = SaslMechanism.Plain,
SaslUsername = userName,
SaslPassword = password,
SslCaLocation = SupplyCertificateLocation()
},
new KafkaPublication
{
Topic = new RoutingKey(_topic),
NumPartitions = 1,
ReplicationFactor = 3,
//These timeouts support running on a container using the same host as the tests,
//your production values ought to be lower
MessageTimeoutMs = 10000,
RequestTimeoutMs = 10000,
MakeChannels = OnMissingChannel.Assume //This will not make the topic
}).Create();
//This should force creation of the topic - will fail if no topic creation code
_consumer = new KafkaMessageConsumerFactory(
new KafkaMessagingGatewayConfiguration
{
Name = "Kafka Producer Send Test",
BootStrapServers = new[] { bootStrapServer },
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanisms = SaslMechanism.Plain,
SaslUsername = userName,
SaslPassword = password,
SslCaLocation = SupplyCertificateLocation()
})
.Create(new KafkaSubscription <MyCommand>(
channelName: new ChannelName(_queueName),
routingKey: new RoutingKey(_topic),
groupId: groupId,
numOfPartitions: 1,
replicationFactor: 3,
makeChannels: OnMissingChannel.Create
)
);
}
public static bool IsLinux() => RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
public void Start(bool ensureRunning, string dataDir)
{
if (TorSocks5EndPoint is null)
{
return;
}
new Thread(delegate() // Don't ask. This is the only way it worked on Win10/Ubuntu18.04/Manjuro(1 processor VM)/Fedora(1 processor VM)
{
try
{
// 1. Is it already running?
// 2. Can I simply run it from output directory?
// 3. Can I copy and unzip it from assets?
// 4. Throw exception.
try
{
if (IsTorRunningAsync(TorSocks5EndPoint).GetAwaiter().GetResult())
{
Logger.LogInfo <TorProcessManager>("Tor is already running.");
return;
}
var fullBaseDirectory = Path.GetFullPath(AppContext.BaseDirectory);
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (!fullBaseDirectory.StartsWith('/'))
{
fullBaseDirectory.Insert(0, "/");
}
}
var torDir = Path.Combine(dataDir, "tor");
var torPath = "";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
torPath = $@"{torDir}\Tor\tor.exe";
}
else // Linux or OSX
{
torPath = $@"{torDir}/Tor/tor";
}
if (!File.Exists(torPath))
{
Logger.LogInfo <TorProcessManager>($"Tor instance NOT found at {torPath}. Attempting to acquire it...");
InstallTor(fullBaseDirectory, torDir);
}
else if (!IoHelpers.CheckExpectedHash(torPath, Path.Combine(fullBaseDirectory, "TorDaemons")))
{
Logger.LogInfo <TorProcessManager>($"Updating Tor...");
string backupTorDir = $"{torDir}_backup";
if (Directory.Exists(backupTorDir))
{
Directory.Delete(backupTorDir, true);
}
Directory.Move(torDir, backupTorDir);
InstallTor(fullBaseDirectory, torDir);
}
else
{
Logger.LogInfo <TorProcessManager>($"Tor instance found at {torPath}.");
}
string torArguments = $"--SOCKSPort {TorSocks5EndPoint}";
if (!string.IsNullOrEmpty(LogFile))
{
IoHelpers.EnsureContainingDirectoryExists(LogFile);
var logFileFullPath = Path.GetFullPath(LogFile);
torArguments += $" --Log \"notice file {logFileFullPath}\"";
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
TorProcess = Process.Start(new ProcessStartInfo {
FileName = torPath,
Arguments = torArguments,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true
});
Logger.LogInfo <TorProcessManager>($"Starting Tor process with Process.Start.");
}
else // Linux and OSX
{
string runTorCmd = $"LD_LIBRARY_PATH=$LD_LIBRARY_PATH:={torDir}/Tor && export LD_LIBRARY_PATH && cd {torDir}/Tor && ./tor {torArguments}";
EnvironmentHelpers.ShellExec(runTorCmd, false);
Logger.LogInfo <TorProcessManager>($"Started Tor process with shell command: {runTorCmd}.");
}
if (ensureRunning)
{
Task.Delay(3000).ConfigureAwait(false).GetAwaiter().GetResult(); // dotnet brainfart, ConfigureAwait(false) IS NEEDED HERE otherwise (only on) Manjuro Linux fails, WTF?!!
if (!IsTorRunningAsync(TorSocks5EndPoint).GetAwaiter().GetResult())
{
throw new TorException("Attempted to start Tor, but it is not running.");
}
Logger.LogInfo <TorProcessManager>("Tor is running.");
}
}
catch (Exception ex)
{
throw new TorException("Could not automatically start Tor. Try running Tor manually.", ex);
}
}
catch (Exception ex)
{
Logger.LogError <TorProcessManager>(ex);
}
}).Start();
}
private void GetProcessInfo(string procName)
{
// This is to support differences between Linux and Windows dotnet process naming pattern.
// Default value is what Windows expects for proc name. In linux, the procname is an argument (typically) dotnet command.
string dotnetArg = procName;
Process[] processes = null;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && procName.Contains("dotnet "))
{
dotnetArg = $"{procName.Replace("dotnet ", string.Empty)}";
processes = GetDotnetProcessesByFirstArgument(dotnetArg);
}
else
{
processes = Process.GetProcessesByName(procName);
}
if (processes.Length == 0)
{
return;
}
Stopwatch timer = new Stopwatch();
foreach (var process in processes)
{
CpuUsage cpuUsage = new CpuUsage();
try
{
Token.ThrowIfCancellationRequested();
// Warm up the perf counters.
_ = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(process.Id);
// Ports
int activePortCount = OperatingSystemInfoProvider.Instance.GetActivePortCount(process.Id, FabricServiceContext);
TotalActivePortCountAllSystemServices += activePortCount;
int activeEphemeralPortCount = OperatingSystemInfoProvider.Instance.GetActiveEphemeralPortCount(process.Id, FabricServiceContext);
TotalActiveEphemeralPortCountAllSystemServices += activeEphemeralPortCount;
this.allActiveTcpPortData.FirstOrDefault(x => x.Id == dotnetArg).Data.Add(activePortCount);
this.allEphemeralTcpPortData.FirstOrDefault(x => x.Id == dotnetArg).Data.Add(activeEphemeralPortCount);
TimeSpan duration = TimeSpan.FromSeconds(5);
if (MonitorDuration > TimeSpan.MinValue)
{
duration = MonitorDuration;
}
timer.Start();
while (!process.HasExited && timer.Elapsed <= duration)
{
Token.ThrowIfCancellationRequested();
try
{
// CPU Time for service process.
int cpu = (int)cpuUsage.GetCpuUsagePercentageProcess(process);
this.allCpuData.FirstOrDefault(x => x.Id == dotnetArg).Data.Add(cpu);
// Private Working Set for service process.
float mem = ProcessInfoProvider.Instance.GetProcessPrivateWorkingSetInMB(process.Id);
this.allMemData.FirstOrDefault(x => x.Id == dotnetArg).Data.Add(mem);
Thread.Sleep(250);
}
catch (Exception e)
{
WriteToLogWithLevel(
ObserverName,
$"Can't observe {process} details:{Environment.NewLine}{e}",
LogLevel.Warning);
throw;
}
}
}
catch (Win32Exception)
{
// This will always be the case if FabricObserver.exe is not running as Admin or LocalSystem.
// It's OK. Just means that the elevated process (like FabricHost.exe) won't be observed.
WriteToLogWithLevel(
ObserverName,
$"Can't observe {process.ProcessName} due to it's privilege level. FabricObserver must be running as System or Admin for this specific task.",
LogLevel.Information);
break;
}
finally
{
process?.Dispose();
}
timer.Stop();
timer.Reset();
}
}
private async Task HandleOutputStreamAsync()
{
this.hasWork = false;
await Task.Run(async() =>
{
// Only do calculation related to transfer window when the file contains multiple chunks.
if (!this.EnableOneChunkFileOptimization)
{
// We do check point consistancy validation in reader, and directly use it in writer.
if ((null != this.TransferJob.CheckPoint.TransferWindow) &&
this.TransferJob.CheckPoint.TransferWindow.Any())
{
this.TransferJob.CheckPoint.TransferWindow.Sort();
this.expectOffset = this.TransferJob.CheckPoint.TransferWindow[0];
}
else
{
this.expectOffset = this.TransferJob.CheckPoint.EntryTransferOffset;
}
}
if (TransferLocationType.Stream == this.TransferJob.Destination.Type)
{
Stream streamInDestination = (this.TransferJob.Destination as StreamLocation).Stream;
if (!streamInDestination.CanWrite)
{
throw new NotSupportedException(string.Format(
CultureInfo.CurrentCulture,
Resources.StreamMustSupportWriteException,
"outputStream"));
}
if (!streamInDestination.CanSeek)
{
throw new NotSupportedException(string.Format(
CultureInfo.CurrentCulture,
Resources.StreamMustSupportSeekException,
"outputStream"));
}
this.outputStream = streamInDestination;
}
else
{
this.filePath = (this.TransferJob.Destination as FileLocation).FilePath;
if (!this.Controller.IsForceOverwrite)
{
await this.Controller.CheckOverwriteAsync(
LongPathFile.Exists(this.filePath),
this.TransferJob.Source.Instance,
this.filePath);
}
this.Controller.CheckCancellation();
try
{
FileMode fileMode = 0 == this.expectOffset ? FileMode.OpenOrCreate : FileMode.Open;
string longFilePath = filePath;
//#if DOTNET5_4
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Attempt to open the file first so that we throw an exception before getting into the async work
this.outputStream = new FileStream(
longFilePath,
fileMode,
FileAccess.ReadWrite,
FileShare.None);
//#else
}
else
{
if (Interop.CrossPlatformHelpers.IsWindows)
{
longFilePath = LongPath.ToUncPath(longFilePath);
}
this.outputStream = LongPathFile.Open(
filePath,
fileMode,
FileAccess.ReadWrite,
FileShare.None);
//#endif
}
this.ownsStream = true;
}
catch (Exception ex)
{
string exceptionMessage = string.Format(
CultureInfo.CurrentCulture,
Resources.FailedToOpenFileException,
filePath,
ex.Message);
throw new TransferException(
TransferErrorCode.OpenFileFailed,
exceptionMessage,
ex);
}
}
this.outputStream.SetLength(this.SharedTransferData.TotalLength);
this.md5HashStream = new MD5HashStream(
this.outputStream,
this.expectOffset,
!this.SharedTransferData.DisableContentMD5Validation);
if (this.md5HashStream.FinishedSeparateMd5Calculator)
{
this.state = State.Write;
}
else
{
this.state = State.CalculateMD5;
}
this.PreProcessed = true;
// Switch state internal for one chunk small file.
if (this.EnableOneChunkFileOptimization &&
State.Write == this.state &&
((this.SharedTransferData.TotalLength == this.expectOffset) || this.SharedTransferData.AvailableData.ContainsKey(this.expectOffset)))
{
this.isStateSwitchedInternal = true;
await this.WriteChunkDataAsync().ConfigureAwait(false);
}
else
{
this.hasWork = true;
}
});
}
/// <summary>
/// Determines whether the ServiceCollector can run or not.
/// </summary>
/// <returns>True on Windows, Linux, Mac OS</returns>
public override bool CanRunOnPlatform()
{
return(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux));
}
private SeleniumStandaloneServer()
{
var port = FindAvailablePort();
Uri = new UriBuilder("http", "localhost", port, "/wd/hub").Uri;
var psi = new ProcessStartInfo
{
FileName = "npm",
Arguments = $"run selenium-standalone start -- -- -port {port}",
RedirectStandardOutput = true,
RedirectStandardError = true,
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
psi.FileName = "cmd";
psi.Arguments = $"/c npm {psi.Arguments}";
}
var process = Process.Start(psi);
var builder = new StringBuilder();
process.OutputDataReceived += LogOutput;
process.ErrorDataReceived += LogOutput;
process.BeginOutputReadLine();
process.BeginErrorReadLine();
// The Selenium sever has to be up for the entirety of the tests and is only shutdown when the application (i.e. the test) exits.
AppDomain.CurrentDomain.ProcessExit += (sender, e) =>
{
if (!process.HasExited)
{
process.KillTree(TimeSpan.FromSeconds(10));
process.Dispose();
}
};
void LogOutput(object sender, DataReceivedEventArgs e)
{
lock (builder)
{
builder.AppendLine(e.Data);
}
}
var waitForStart = Task.Run(async() =>
{
var httpClient = new HttpClient
{
Timeout = TimeSpan.FromSeconds(1),
};
while (true)
{
try
{
var responseTask = httpClient.GetAsync(Uri);
var response = await responseTask;
if (response.StatusCode == HttpStatusCode.OK)
{
return;
}
}
catch (OperationCanceledException)
{
}
await Task.Delay(1000);
}
});
try
{
// Wait in intervals instead of indefinitely to prevent thread starvation.
while (!waitForStart.TimeoutAfter(Timeout).Wait(1000))
{
}
}
catch (Exception ex)
{
string output;
lock (builder)
{
output = builder.ToString();
}
throw new InvalidOperationException($"Failed to start selenium sever. {System.Environment.NewLine}{output}", ex.GetBaseException());
}
}
public async Task InitializeAsync()
{
if (!string.IsNullOrEmpty(_functionsWorkerLanguage))
{
Environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, _functionsWorkerLanguage);
}
IConfiguration configuration = TestHelpers.GetTestConfiguration();
string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
QueueClient = storageAccount.CreateCloudQueueClient();
BlobClient = storageAccount.CreateCloudBlobClient();
TableStorageAccount tableStorageAccount = TableStorageAccount.Parse(connectionString);
TableClient = tableStorageAccount.CreateCloudTableClient();
await CreateTestStorageEntities();
// ApiHubTestHelper.SetDefaultConnectionFactory();
//ILoggerProviderFactory loggerProviderFactory = new TestLoggerProviderFactory(LoggerProvider);
// Reset the timer logs first, since one of the tests will
// be checking them
TestHelpers.ClearFunctionLogs("TimerTrigger");
TestHelpers.ClearFunctionLogs("ListenerStartupException");
Host = new HostBuilder()
.ConfigureDefaultTestWebScriptHost(webjobsBuilder =>
{
webjobsBuilder.AddAzureStorage();
// This needs to added manually at the ScriptHost level, as although FunctionMetadataManager is available through WebHost,
// it needs to change the services during its lifetime.
webjobsBuilder.Services.AddSingleton <IFunctionMetadataManager, FunctionMetadataManager>();
},
o =>
{
o.ScriptPath = _rootPath;
o.LogPath = TestHelpers.GetHostLogFileDirectory().Parent.FullName;
},
runStartupHostedServices: true)
.ConfigureServices(services =>
{
services.Configure <ScriptJobHostOptions>(o =>
{
o.FileLoggingMode = FileLoggingMode.Always;
if (_functions != null)
{
o.Functions = _functions;
}
});
if (_proxyClient != null)
{
services.AddSingleton <ProxyClientExecutor>(_proxyClient);
}
// Shared memory data transfer
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
services.AddSingleton <IMemoryMappedFileAccessor, MemoryMappedFileAccessorWindows>();
}
else
{
services.AddSingleton <IMemoryMappedFileAccessor, MemoryMappedFileAccessorUnix>();
}
services.AddSingleton <ISharedMemoryManager, SharedMemoryManager>();
ConfigureServices(services);
})
.ConfigureLogging(b =>
{
b.AddProvider(LoggerProvider);
})
.Build();
JobHost = Host.GetScriptHost();
if (_startHost)
{
JobHost.HostStarted += (s, e) => _hostStartedEvent.Set();
await Host.StartAsync();
_hostStartedEvent.Wait(TimeSpan.FromSeconds(30));
}
}
private static (string, string) ResolveCommand(string fileName, IEnumerable <string> searchPaths, bool waitForExit)
{
string appName = Path.ChangeExtension(fileName, null);
string command = null;
string commandArgs = null;
foreach (string searchPath in searchPaths)
{
string testPath = Path.GetFullPath(Path.Combine(searchPath, appName));
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
string windowsTestPath = Path.ChangeExtension(testPath, "exe");
if (File.Exists(windowsTestPath))
{
command = windowsTestPath;
break;
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
string macTestPath = Path.ChangeExtension(testPath, "app");
if (waitForExit)
{
// If waitForExit, get the executable out of the app bundle.
macTestPath = Path.Combine(macTestPath, "Contents", "MacOS", appName);
if (File.Exists(macTestPath))
{
command = macTestPath;
break;
}
}
else
{
// Otherwise use the .app itself.
if (Directory.Exists(macTestPath))
{
command = "open";
commandArgs = $"-n \"{macTestPath}\" --args";
break;
}
}
}
else
{
string linuxTestPath = Path.ChangeExtension(testPath, null);
if (File.Exists(linuxTestPath))
{
command = linuxTestPath;
break;
}
}
string dotnetTestPath = Path.ChangeExtension(testPath, "dll");
if (File.Exists(dotnetTestPath))
{
command = DotNetMuxer.MuxerPathOrDefault();
commandArgs = $"\"{dotnetTestPath}\"";
break;
}
}
if (command == null)
{
throw new FileNotFoundException($"{appName} is not in the search path!");
}
return(command, commandArgs);
}
public static IList <string> GetMeadowSerialPortsForWindows(ILogger?logger = null)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) == false)
{
throw new PlatformNotSupportedException("This method is only supported on Windows");
}
logger ??= NullLogger.Instance;
try
{
const string WildernessLabsPnpDeviceIDPrefix = @"USB\VID_" + MeadowCLI.Constants.WILDERNESS_LABS_USB_VID;
// Win32_PnPEntity lives in root\CIMV2
const string wmiScope = "root\\CIMV2";
// escape special characters in the device id prefix
string escapedPrefix = WildernessLabsPnpDeviceIDPrefix.Replace("\\", "\\\\").Replace("_", "[_]");
// our query for all ports that have a PnP device id starting with Wilderness Labs' USB VID.
string query = @$ "SELECT Name, Caption, PNPDeviceID FROM Win32_PnPEntity WHERE PNPClass = 'Ports' AND PNPDeviceID like '{escapedPrefix}%'";
logger.LogDebug("Running WMI Query: {query}", query);
List <string> results = new();
// build the searcher for the query
using ManagementObjectSearcher searcher = new(wmiScope, query);
// get the query results
foreach (ManagementObject moResult in searcher.Get())
{
// Try Caption and if not Name, they both seems to contain the COM port
string portLongName = moResult["Caption"].ToString();
if (string.IsNullOrEmpty(portLongName))
{
portLongName = moResult["Name"].ToString();
}
string pnpDeviceId = moResult["PNPDeviceID"].ToString();
// we could collect and return a fair bit of other info from the query:
//string description = moResult["Description"].ToString();
//string service = moResult["Service"].ToString();
//string manufacturer = moResult["Manufacturer"].ToString();
var comIndex = portLongName.IndexOf("(COM") + 1;
var copyLength = portLongName.IndexOf(")") - comIndex;
var port = portLongName.Substring(comIndex, copyLength);
// the meadow serial is in the device id, after
// the characters: USB\VID_XXXX&PID_XXXX\
// so we'll just split is on \ and grab the 3rd element as the format is standard, but the length may vary.
var splits = pnpDeviceId.Split('\\');
var serialNumber = splits[2];
logger.LogDebug($"Found Wilderness Labs device at `{port}` with serial `{serialNumber}`");
results.Add($"{port}"); // removed serial number for consistency and will break fallback ({serialNumber})");
}
return(results.ToArray());
}
catch (ApplicationException aex)
{
// eat it for now
logger.LogDebug(aex, "This error can be safely ignored.");
// Since WMI Failed fall back to using SerialPort
var ports = SerialPort.GetPortNames();
//hack to skip COM1
ports = ports.Where((source, index) => source != "COM1").Distinct().ToArray();
return(ports);
}
}
public static IList <string> GetMeadowSerialPortsForOsx(ILogger?logger = null)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) == false)
{
throw new PlatformNotSupportedException("This method is only supported on macOS");
}
logger ??= NullLogger.Instance;
logger.LogDebug("Get Meadow Serial ports");
var ports = new List <string>();
var psi = new ProcessStartInfo
{
FileName = "/usr/sbin/ioreg",
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = "-r -c IOUSBHostDevice -l"
};
string output = string.Empty;
using (var p = Process.Start(psi))
{
if (p != null)
{
output = p.StandardOutput.ReadToEnd();
p.WaitForExit();
}
}
//split into lines
var lines = output.Split("\n\r".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
var foundMeadow = false;
foreach (var line in lines)
{
if (line.Contains("Meadow F7 Micro"))
{
foundMeadow = true;
}
else if (line.IndexOf("+-o", StringComparison.Ordinal) == 0)
{
foundMeadow = false;
}
//now find the IODialinDevice entry which contains the serial port name
if (foundMeadow && line.Contains("IODialinDevice"))
{
int startIndex = line.IndexOf("/");
int endIndex = line.IndexOf("\"", startIndex + 1);
var port = line.Substring(startIndex, endIndex - startIndex);
logger.LogDebug($"Found Meadow at {port}", port);
ports.Add(port);
foundMeadow = false;
}
}
logger.LogDebug("Found {count} ports", ports.Count);
return(ports);
}
public override Task ReportAsync(CancellationToken token)
{
try
{
token.ThrowIfCancellationRequested();
// OS Health.
if (this.osStatus != null && !string.Equals(this.osStatus, "OK", StringComparison.OrdinalIgnoreCase))
{
string healthMessage = $"OS reporting unhealthy: {this.osStatus}";
var healthReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = healthMessage,
State = HealthState.Error,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
// This means this observer created a Warning or Error SF Health Report
HasActiveFabricErrorOrWarning = true;
// Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example.).
if (IsTelemetryProviderEnabled && IsObserverTelemetryEnabled)
{
_ = TelemetryClient?.ReportHealthAsync(
HealthScope.Application,
FabricRuntime.GetActivationContext().ApplicationName,
HealthState.Error,
$"{NodeName} - OS reporting unhealthy: {this.osStatus}",
ObserverName,
Token);
}
}
else if (HasActiveFabricErrorOrWarning && string.Equals(this.osStatus, "OK", StringComparison.OrdinalIgnoreCase))
{
// Clear Error or Warning with an OK Health Report.
string healthMessage = $"OS reporting healthy: {this.osStatus}";
var healthReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = healthMessage,
State = HealthState.Ok,
HealthReportTimeToLive = default(TimeSpan),
};
HealthReporter.ReportHealthToServiceFabric(healthReport);
// Reset internal health state.
HasActiveFabricErrorOrWarning = false;
}
if (ObserverManager.ObserverWebAppDeployed)
{
var logPath = Path.Combine(ObserverLogger.LogFolderBasePath, "SysInfo.txt");
// This file is used by the web application (log reader.).
if (!ObserverLogger.TryWriteLogFile(logPath, $"Last updated on {DateTime.UtcNow.ToString("M/d/yyyy HH:mm:ss")} UTC<br/>{this.osReport}"))
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
"Unable to create SysInfo.txt file.");
}
}
var report = new HealthReport
{
Observer = ObserverName,
HealthMessage = this.osReport,
State = HealthState.Ok,
NodeName = NodeName,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(report);
// Windows Update automatic download enabled?
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
this.isWindowsUpdateAutoDownloadEnabled)
{
string linkText =
$"{Environment.NewLine}For clusters of Silver durability or above, " +
$"please consider <a href=\"https://docs.microsoft.com/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-automatic-upgrade\" target=\"blank\">" +
$"enabling VMSS automatic OS image upgrades</a> to prevent unexpected VM reboots. " +
$"For Bronze durability clusters, please consider deploying the " +
$"<a href=\"https://docs.microsoft.com/azure/service-fabric/service-fabric-patch-orchestration-application\" target=\"blank\">Patch Orchestration Service</a>.";
string auServiceEnabledMessage = $"Windows Update Automatic Download is enabled.{linkText}";
report = new HealthReport
{
Observer = ObserverName,
Property = "OSConfiguration",
HealthMessage = auServiceEnabledMessage,
State = HealthState.Warning,
NodeName = NodeName,
HealthReportTimeToLive = SetHealthReportTimeToLive(),
};
HealthReporter.ReportHealthToServiceFabric(report);
if (IsTelemetryProviderEnabled &&
IsObserverTelemetryEnabled &&
RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Send Health Report as Telemetry (perhaps it signals an Alert from App Insights, for example.).
var telemetryData = new TelemetryData(FabricClientInstance, token)
{
HealthEventDescription = auServiceEnabledMessage,
HealthState = "Warning",
Metric = "WUAutoDownloadEnabled",
Value = this.isWindowsUpdateAutoDownloadEnabled,
NodeName = NodeName,
ObserverName = ObserverName,
Source = ObserverConstants.FabricObserverName,
};
_ = TelemetryClient?.ReportMetricAsync(
telemetryData,
Token);
}
// ETW.
if (IsEtwEnabled && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Logger.EtwLogger?.Write(
ObserverConstants.FabricObserverETWEventName,
new
{
HealthState = "Warning",
HealthEventDescription = auServiceEnabledMessage,
ObserverName,
Metric = "WUAutoDownloadEnabled",
Value = this.isWindowsUpdateAutoDownloadEnabled,
NodeName,
});
}
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// reset au globals for fresh detection during next observer run.
this.isWindowsUpdateAutoDownloadEnabled = false;
this.auStateUnknown = false;
this.isWUADSettingEnabled = false;
}
return(Task.CompletedTask);
}
catch (Exception e)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Error,
$"Unhandled exception processing OS information:{Environment.NewLine}{e}");
throw;
}
}
private void Initialize()
{
if (this.stopwatch == null)
{
this.stopwatch = new Stopwatch();
}
Token.ThrowIfCancellationRequested();
this.stopwatch.Start();
SetThresholdSFromConfiguration();
// CPU data
if (this.allCpuData == null)
{
this.allCpuData = new List <FabricResourceUsageData <int> >(this.processWatchList.Count);
foreach (var proc in this.processWatchList)
{
this.allCpuData.Add(
new FabricResourceUsageData <int>(
ErrorWarningProperty.TotalCpuTime,
proc,
DataCapacity,
UseCircularBuffer));
}
}
// Memory data
if (this.allMemData == null)
{
this.allMemData = new List <FabricResourceUsageData <float> >(this.processWatchList.Count);
foreach (var proc in this.processWatchList)
{
this.allMemData.Add(
new FabricResourceUsageData <float>(
ErrorWarningProperty.TotalMemoryConsumptionMb,
proc,
DataCapacity,
UseCircularBuffer));
}
}
// Ports
if (this.allActiveTcpPortData == null)
{
this.allActiveTcpPortData = new List <FabricResourceUsageData <int> >(this.processWatchList.Count);
foreach (var proc in this.processWatchList)
{
this.allActiveTcpPortData.Add(
new FabricResourceUsageData <int>(
ErrorWarningProperty.TotalActivePorts,
proc,
DataCapacity,
UseCircularBuffer));
}
}
if (this.allEphemeralTcpPortData == null)
{
this.allEphemeralTcpPortData = new List <FabricResourceUsageData <int> >(this.processWatchList.Count);
foreach (var proc in this.processWatchList)
{
this.allEphemeralTcpPortData.Add(
new FabricResourceUsageData <int>(
ErrorWarningProperty.TotalEphemeralPorts,
proc,
DataCapacity,
UseCircularBuffer));
}
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && this.monitorWinEventLog &&
this.evtRecordList == null)
{
this.evtRecordList = new List <EventRecord>();
}
}
private static async Task <int> Main(string[] args)
{
var isService = false;
var app = new CommandLineApplication();
var service = app.Option("-s|--service", "Launch in service mode", CommandOptionType.NoValue);
var help = app.Option("-? | -h | --help", "Show help information", CommandOptionType.NoValue);
app.Execute(args);
if (help.Value() != null)
{
app.ShowHelp();
return(0);
}
if (service.Value() != null)
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Console.WriteLine("Running as a service only available on Windows");
}
else
{
isService = true;
}
}
var workingDirectory = Directory.GetCurrentDirectory();
if (isService)
{
var path = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
Directory.SetCurrentDirectory(path);
workingDirectory = path;
}
var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
var config = new ConfigurationBuilder();
config.SetBasePath(workingDirectory)
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{env}.json", optional: true, reloadOnChange: true);
config.AddEnvironmentVariables();
var Configuration = config.Build();
var services = new ServiceCollection();
services.AddLogging(logging =>
{
var loggingConfiguration = Configuration.GetSection("Logging");
logging.AddConfiguration(loggingConfiguration);
logging.AddConsole();
logging.AddDebug();
if (env != EnvironmentName.Development)
{
logging.AddFile(loggingConfiguration);
}
});
IServiceProvider serviceProvider;
try
{
services.AddSystemChecker(Configuration);
serviceProvider = services.BuildServiceProvider();
}
catch (Exception e)
{
var serviceCollection = new ServiceCollection();
serviceCollection.AddLogging(builder =>
{
builder
.AddFile("logs/systemchecker-critical-{Date}.log")
.AddDebug()
.AddConsole();
});
using (var logProvider = serviceCollection.BuildServiceProvider())
{
var critLogger = logProvider.GetRequiredService <ILogger <Program> >();
critLogger.LogInformation($"Current directory: {Directory.GetCurrentDirectory()}");
critLogger.LogError(e, "Critical failure in host build");
}
return(1);
}
var logger = serviceProvider.GetRequiredService <ILogger <Program> >();
try
{
if (isService)
{
logger.LogInformation("Starting as service");
RunAsService(serviceProvider);
}
else
{
logger.LogInformation("Starting with scheduler");
await Run(serviceProvider);
}
}
catch (Exception e)
{
logger.LogError(e, "Fatal error");
Thread.Sleep(1000); // allow time to flush logs in the event of a crash
return(1);
}
return(0);
}
private void SetThresholdSFromConfiguration()
{
/* Error thresholds */
Token.ThrowIfCancellationRequested();
var cpuError = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverCpuErrorLimitPct);
if (!string.IsNullOrEmpty(cpuError))
{
_ = int.TryParse(cpuError, out int threshold);
if (threshold > 100 || threshold < 0)
{
throw new ArgumentException($"{threshold}% is not a meaningful threshold value for {ObserverConstants.FabricSystemObserverCpuErrorLimitPct}.");
}
CpuErrorUsageThresholdPct = threshold;
}
var memError = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverMemoryErrorLimitMb);
if (!string.IsNullOrEmpty(memError))
{
_ = int.TryParse(memError, out int threshold);
if (threshold < 0)
{
throw new ArgumentException($"{threshold} is not a meaningful threshold value for {ObserverConstants.FabricSystemObserverMemoryErrorLimitMb}.");
}
MemErrorUsageThresholdMb = threshold;
}
// Ports
var activeTcpPortsError = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverNetworkErrorActivePorts);
if (!string.IsNullOrEmpty(activeTcpPortsError))
{
_ = int.TryParse(activeTcpPortsError, out int threshold);
ActiveTcpPortCountError = threshold;
}
var activeEphemeralPortsError = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverNetworkErrorEphemeralPorts);
if (!string.IsNullOrEmpty(activeEphemeralPortsError))
{
_ = int.TryParse(activeEphemeralPortsError, out int threshold);
ActiveEphemeralPortCountError = threshold;
}
/* Warning thresholds */
Token.ThrowIfCancellationRequested();
var cpuWarn = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverCpuWarningLimitPct);
if (!string.IsNullOrEmpty(cpuWarn))
{
_ = int.TryParse(cpuWarn, out int threshold);
if (threshold > 100 || threshold < 0)
{
throw new ArgumentException($"{threshold}% is not a meaningful threshold value for {ObserverConstants.FabricSystemObserverCpuWarningLimitPct}.");
}
CpuWarnUsageThresholdPct = threshold;
}
var memWarn = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverMemoryWarningLimitMb);
if (!string.IsNullOrEmpty(memWarn))
{
_ = int.TryParse(memWarn, out int threshold);
if (threshold < 0)
{
throw new ArgumentException($"{threshold} MB is not a meaningful threshold value for {ObserverConstants.FabricSystemObserverMemoryWarningLimitMb}.");
}
MemWarnUsageThresholdMb = threshold;
}
// Ports
var activeTcpPortsWarning = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverNetworkWarningActivePorts);
if (!string.IsNullOrEmpty(activeTcpPortsWarning))
{
_ = int.TryParse(activeTcpPortsWarning, out int threshold);
ActiveTcpPortCountWarning = threshold;
}
var activeEphemeralPortsWarning = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverNetworkWarningEphemeralPorts);
if (!string.IsNullOrEmpty(activeEphemeralPortsWarning))
{
_ = int.TryParse(activeEphemeralPortsWarning, out int threshold);
ActiveEphemeralPortCountWarning = threshold;
}
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
// Monitor Windows event log for SF and System Error/Critical events?
// This can be noisy. Use wisely.
var watchEvtLog = GetSettingParameterValue(
ObserverConstants.FabricSystemObserverConfigurationSectionName,
ObserverConstants.FabricSystemObserverMonitorWindowsEventLog);
if (!string.IsNullOrEmpty(watchEvtLog) && bool.TryParse(watchEvtLog, out bool watchEl))
{
this.monitorWinEventLog = watchEl;
}
}
public App()
{
Button helloWorldButton = new Button();
helloWorldButton.Text = "Hello world";
Button planarSubdivisionButton = new Button();
planarSubdivisionButton.Text = "Planar Subdivision";
Button faceDetectionButton = new Button();
faceDetectionButton.Text = "Face Detection (CascadeClassifier)";
Button faceLandmarkDetectionButton = new Button();
faceLandmarkDetectionButton.Text = "Face Landmark Detection (DNN Module)";
Button featureDetectionButton = new Button();
featureDetectionButton.Text = "Feature Matching";
Button shapeDetectionButton = new Button();
shapeDetectionButton.Text = "Shape Detection";
Button pedestrianDetectionButton = new Button();
pedestrianDetectionButton.Text = "Pedestrian Detection";
Button ocrButton = new Button();
ocrButton.Text = "OCR";
Button maskRcnnButton = new Button();
maskRcnnButton.Text = "Mask RCNN (DNN module)";
Button yoloButton = new Button();
yoloButton.Text = "Yolo (DNN module)";
Button stopSignDetectionButton = new Button();
stopSignDetectionButton.Text = "Stop Sign Detection (DNN module)";
List <View> buttonList = new List <View>()
{
helloWorldButton,
planarSubdivisionButton,
faceDetectionButton,
faceLandmarkDetectionButton,
featureDetectionButton,
shapeDetectionButton,
pedestrianDetectionButton,
ocrButton,
maskRcnnButton,
stopSignDetectionButton,
yoloButton,
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Emgu.Util.Platform.ClrType != Emgu.Util.Platform.Clr.NetFxCore)
{
Button viz3dButton = new Button();
viz3dButton.Text = "Simple 3D reconstruction";
buttonList.Add(viz3dButton);
viz3dButton.Clicked += (sender, args) =>
{
Mat left = CvInvoke.Imread("imL.png", ImreadModes.Color);
Mat right = CvInvoke.Imread("imR.png", ImreadModes.Color);
Viz3d v = Simple3DReconstruct.GetViz3d(left, right);
v.Spin();
};
}
StackLayout buttonsLayout = new StackLayout
{
VerticalOptions = LayoutOptions.Start,
};
foreach (View b in buttonList)
{
buttonsLayout.Children.Add(b);
}
// The root page of your application
ContentPage page =
new ContentPage()
{
Content = new ScrollView()
{
Content = buttonsLayout,
}
};
String aboutIcon = null;
/*
* String aboutIcon;
* if (Emgu.Util.Platform.OperationSystem == Emgu.Util.Platform.OS.IOS)
* {
* aboutIcon = null;
* }
* else if (Emgu.Util.Platform.ClrType == Emgu.Util.Platform.Clr.NetFxCore)
* aboutIcon = null;
* else
* aboutIcon = "questionmark.png";*/
MainPage =
new NavigationPage(
page
);
ToolbarItem aboutItem = new ToolbarItem("About", aboutIcon,
() =>
{
MainPage.Navigation.PushAsync(new AboutPage());
//page.DisplayAlert("Emgu CV Examples", "App version: ...", "Ok");
}
);
page.ToolbarItems.Add(aboutItem);
helloWorldButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new HelloWorldPage());
};
planarSubdivisionButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new PlanarSubdivisionPage());
};
faceDetectionButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new FaceDetectionPage());
};
shapeDetectionButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new ShapeDetectionPage());
};
pedestrianDetectionButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new PedestrianDetectionPage());
};
featureDetectionButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new FeatureMatchingPage());
};
if (Emgu.Util.Platform.ClrType == Emgu.Util.Platform.Clr.NetFxCore)
{
//No DNN module for UWP apps
maskRcnnButton.IsVisible = false;
faceLandmarkDetectionButton.IsVisible = false;
stopSignDetectionButton.IsVisible = false;
yoloButton.IsVisible = false;
}
else
{
maskRcnnButton.Clicked += (sender, args) => { MainPage.Navigation.PushAsync(new MaskRcnnPage()); };
faceLandmarkDetectionButton.Clicked += (sender, args) => { MainPage.Navigation.PushAsync(new FaceLandmarkDetectionPage()); };
stopSignDetectionButton.Clicked += (sender, args) =>
{
MaskRcnnPage stopSignDetectionPage = new MaskRcnnPage();
stopSignDetectionPage.DefaultImage = "stop-sign.jpg";
stopSignDetectionPage.ObjectsOfInterest = new string[] { "stop sign" };
MainPage.Navigation.PushAsync(stopSignDetectionPage);
};
yoloButton.Clicked += (sender, args) => { MainPage.Navigation.PushAsync(new YoloPage()); };
}
ocrButton.Clicked += (sender, args) =>
{
MainPage.Navigation.PushAsync(new OcrPage());
};
}
public static void Main(string[] args)
{
CommandLineOptions commandLineOptions = new CommandLineOptions(args);
// Force-load prefs.
var prefs = EditorPreferences.Instance;
OpenTKWindow window = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? (OpenTKWindow)new DedicatedThreadWindow(960, 540, WindowState.Maximized)
: new SameThreadWindow(960, 540, WindowState.Maximized);
window.Title = "ge.Editor";
Game game = new Game();
GraphicsBackEndPreference backEndPref = commandLineOptions.PreferOpenGL ? GraphicsBackEndPreference.OpenGL : GraphicsBackEndPreference.None;
GraphicsSystem gs = new GraphicsSystem(window, prefs.RenderQuality, backEndPref);
gs.Context.ResourceFactory.AddShaderLoader(new EmbeddedResourceShaderLoader(typeof(Program).GetTypeInfo().Assembly));
game.SystemRegistry.Register(gs);
game.LimitFrameRate = false;
InputSystem inputSystem = new InputSystem(window);
inputSystem.RegisterCallback((input) =>
{
if (input.GetKeyDown(Key.F4) && (input.GetKey(Key.AltLeft) || input.GetKey(Key.AltRight)))
{
game.Exit();
}
});
game.SystemRegistry.Register(inputSystem);
ImGuiRenderer imGuiRenderer = new ImGuiRenderer(gs.Context, window.NativeWindow, inputSystem);
gs.SetImGuiRenderer(imGuiRenderer);
var als = new AssemblyLoadSystem();
game.SystemRegistry.Register(als);
AssetSystem assetSystem = new EditorAssetSystem(Path.Combine(AppContext.BaseDirectory, "Assets"), als.Binder);
game.SystemRegistry.Register(assetSystem);
EditorSceneLoaderSystem esls = new EditorSceneLoaderSystem(game, game.SystemRegistry.GetSystem<GameObjectQuerySystem>());
game.SystemRegistry.Register<SceneLoaderSystem>(esls);
esls.AfterSceneLoaded += () => game.ResetDeltaTime();
CommandLineOptions.AudioEnginePreference? audioPreference = commandLineOptions.AudioPreference;
AudioEngineOptions audioEngineOptions =
!audioPreference.HasValue ? AudioEngineOptions.Default
: audioPreference == CommandLineOptions.AudioEnginePreference.None ? AudioEngineOptions.UseNullAudio
: AudioEngineOptions.UseOpenAL;
AudioSystem audioSystem = new AudioSystem(audioEngineOptions);
game.SystemRegistry.Register(audioSystem);
BehaviorUpdateSystem bus = new BehaviorUpdateSystem(game.SystemRegistry);
game.SystemRegistry.Register(bus);
bus.Register(imGuiRenderer);
PhysicsSystem ps = new PhysicsSystem(PhysicsLayersDescription.Default);
game.SystemRegistry.Register(ps);
ConsoleCommandSystem ccs = new ConsoleCommandSystem(game.SystemRegistry);
game.SystemRegistry.Register(ccs);
game.SystemRegistry.Register(new SynchronizationHelperSystem());
window.Closed += game.Exit;
var editorSystem = new EditorSystem(game.SystemRegistry, commandLineOptions, imGuiRenderer);
editorSystem.DiscoverComponentsFromAssembly(typeof(Program).GetTypeInfo().Assembly);
// Editor system registers itself.
game.RunMainLoop();
window.NativeWindow.Dispose();
EditorPreferences.Instance.Save();
}
static void Main(string[] args)
{
var SO = "WIN";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
SO = "LINUX";
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
SO = "WIN";
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine("MAC not supported"); System.Environment.Exit(0);
}
/* -------------------------------------------------------------------- */
/* Read config file . */
/* -------------------------------------------------------------------- */
var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
var builder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true, true)
.AddJsonFile($"appsettings.{SO}.json", true, true)
.AddJsonFile($"appsettings.{env}.json", true, true)
.AddEnvironmentVariables();
config = builder.Build();
logger.Debug($"Configuration: {config.GetDebugView()}");
// https://blog.bitscry.com/2017/11/14/reading-lists-from-appsettings-json/
//List<string> PolygonsLayers = config.GetSection("CATCHMENTS_LAYERS").Get<List<string>>();
var datapath = config["DATA_PATH"];
if (!Directory.Exists(datapath))
{
logger.Error($"No se ha encontrado la ruta de datos {datapath}");
}
/* -------------------------------------------------------------------- */
/* Configure GDal driver(s). */
/* -------------------------------------------------------------------- */
try
{
Gdal.PushErrorHandler(GdalUtils.GDalErrorHandler);
GdalUtils.Configure();
Gdal.UseExceptions();
}
catch (Exception ex)
{
logger.Error(ex, ex.StackTrace + " " + Gdal.GetLastErrorMsg());
}
// Lectura de datos de AEMet de GNavarra (radares individuales) y traducción a TIFF
if (false)
{
var TarsDir = @"C:\XXX\GeoTiffTests\data\GNavarra_AEMet.tar\Radar\RAD_ZAR.2021030.00.tar\";
foreach (var f in Directory.GetFiles(TarsDir, "*.tar"))
{
ReadAEMetRadarFile(f, datapath);
}
System.Environment.Exit(1);
}
// Lectura de datos de AEMet (composición radar) y traducción a GeoTIFF
if (false)
{
var GZsDir = @"C:\Users\Administrador.000\Desktop\Nueva carpeta\";
foreach (var f in Directory.GetFiles(GZsDir, "ACUM-RAD-*.gz"))
{
Console.WriteLine(f);
UncompressFiles(f, GZsDir);
}
foreach (var f in Directory.GetFiles(GZsDir, "AREA????"))
{
var output = Path.ChangeExtension(f, ".tiff");
if (File.Exists(output))
{
File.Delete(output);
}
object raster_metedata = new {
type = "AEMet_radar",
ogirin = $"{f}",
creation_time_utc = DateTime.Now.ToUniversalTime().ToString("yyyyMMddHHmmss")
};
AREAnnnToGTiff(f, output, JsonConvert.SerializeObject(raster_metedata), JsonConvert.SerializeObject(new {}));
logger.Info($"Creado {output} desde AREAnnnn ({f})");
}
System.Environment.Exit(1);
}
// Lectura de ficheros CRR-Ph, obtención de la banda de acumulación de lluvia y traducción a GeoTIFF
if (false)
{
var BAND_NAME = "crrph_accum";
foreach (var netcdf in Directory.GetFiles("C:/XXX/Navarra/smb-gnavarra-crr/", "*.nc"))
{
logger.Info($"CRR-Ph NetCDf to GeoTIFF {netcdf}");
object raster_metedata = new {
type = "CRR-Ph",
ogirin = $"{netcdf}",
creation_time_utc = DateTime.Now.ToUniversalTime().ToString("yyyyMMddHHmmss")
};
using (Dataset ds = Gdal.Open(netcdf, Access.GA_ReadOnly))
{
var list = ds.GetMetadata("SUBDATASETS").Cast <string>().ToList();
//Console.WriteLine($" {string.Join(" \r\n", list)}");
foreach (var subdataset in list.Where(s => s.Contains("_NAME=") && s.Contains(BAND_NAME, StringComparison.OrdinalIgnoreCase)))
{
logger.Info($"Subdataset: {subdataset}");
Console.WriteLine($" {subdataset}");
var InputFileName = subdataset.Split("=")[1];
var OutputFileName = $"C:/XXX/Navarra/smb-gnavarra-crr/{Path.GetFileNameWithoutExtension(netcdf)}_{subdataset.Split("//")[1]}.tiff";
CrrPhNetCDFToGTiff(InputFileName, OutputFileName, JsonConvert.SerializeObject(raster_metedata), JsonConvert.SerializeObject(new {}));
}
}
}
;
//var netcdf = "C:/XXX/Navarra/smb-gnavarra-crr/S_NWC_CRR-Ph_MSG4_AEMET-VISIR_20210310T230000Z.nc";
//-2854883.8 3000.4033 0 4979169.5 0 -3000.4033
}
// Equal rasters sum
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("===========================SUM rasters (--)==================================");
var InputRaster = Path.Combine(datapath, "2021036.120000.RAD_ZAR - copia.tiff");
GdalUtils.SumRasters(Directory.GetFiles(datapath, "2021036.??0000.RAD_ZAR.tiff"), Path.Combine(datapath, "sumaParalelo.tiff"));
}
// Coordinates reprojection
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("===========================REPROJECTION coordinate testing (OK)==================================");
try
{
var ret = GdalUtils.ReprojectCoordinates(23030, 4326, 85530d, 446100d, 0d);
}
catch (Exception ex)
{
logger.Error(ex, ex.StackTrace + " " + Gdal.GetLastErrorMsg());
}
}
// Gdal info: información sobre la carga de GDAL
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("===========================GDAL INFO==================================");
try
{
GdalUtils.GetGdalInfo();
}
catch (Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// Create TIFF and adding bands
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================Crear GTiff y añadir bandas =====================================");
try
{
var output = Path.Combine(datapath, "CreateRasterNew.tiff");
if (File.Exists(output))
{
File.Delete(output);
}
int NRows = 200;
int Ncols = 100;
double MinX = 55;
double MinY = 45;
double CellSize = 0.1;
string src_wkt = GdalUtils.EPSG2WKT(4326);
var valores = new List <float[]>();
for (var band = 0; band < 5; band++)
{
var buffer = new float [NRows * Ncols];
for (int i = 0; i < Ncols; i++)
{
for (int j = 0; j < NRows; j++)
{
buffer[i * Ncols + j] = (float)(i * 256 / Ncols) * band;
}
}
valores.Add(buffer);
}
var GeoTrans = new[] { MinX, CellSize, 0, MinY, 0, CellSize };
GdalUtils.CreateRaster("GTiff", output, NRows, Ncols, MinX, MinY, CellSize, src_wkt, GeoTrans, valores, null, null);
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// Raster reprojection
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================Raster reprojection =====================================");
try
{
int OutEpsg = 23030;
var input = Path.Combine(datapath, "CreateRaster.tiff");
var output = Path.Combine(datapath, $"CreateRasterNew{OutEpsg}.tiff");
if (File.Exists(output))
{
File.Delete(output);
}
GdalUtils.RasterReprojection(input, output, OutEpsg);
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// GDAL Translate GRIB2 => GTiff
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================Translate GRIB2 => GTiff (OK) =====================================");
try
{
// Funciona pero tarda mucho:
var input = Path.Combine(datapath, "pluviometrosIDW.tiff");
var output = Path.Combine(datapath, "pluviometrosIDW.asc");
if (File.Exists(output))
{
File.Delete(output);
}
GdalUtils.TranslateRasterFormat(input, output, "AAIGrid");
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// IDW with gradient correction
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================IDW con gradiente (OK)=====================================");
try
{
double CellSize = 10000;
double xMin = 360000;
double yMax = 4830000;
int NumCols = 59;
int NumRows = 39;
double yMin = yMax - (NumRows * CellSize);
double xMax = xMin + (NumCols * CellSize);
Random random = new Random();
double GetRandomNumber(double minimum, double maximum)
{
return(random.NextDouble() * (maximum - minimum) + minimum);
}
int NumTermometros = 350;
var Points = new List <OSGeo.OGR.Geometry>();
// Add more points
for (int w = 1; w < NumTermometros; w++)
{
var pnew = new Geometry(wkbGeometryType.wkbPoint);
pnew.AddPointZM(
GetRandomNumber(xMin, xMax),
GetRandomNumber(yMin, yMax),
GetRandomNumber(100, 300),
GetRandomNumber(0, 10));
Points.Add(pnew);
}
SurfaceInterpolations.IdwTemperaturesWithElevationCorrection(Path.Combine(datapath, $"IdwTemperaturesWithElevationCorrection_{Points.Count}.tiff"), Points);
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// IDW with NearestNeighbour
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================IDW NN (OK)=====================================");
try
{
SurfaceInterpolations.IDWwithNearestNeighbour(Path.Combine(datapath, "pluviometros_23030.shp"), Path.Combine(datapath, "pluviometrosIDW.tiff"));
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// Create contour
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("========================Contour (OK)=====================================");
try
{
var input = Path.Combine(datapath, "pluviometrosIDW.tiff");
var output = Path.Combine(datapath, "contour.shp");
if (File.Exists(input))
{
File.Delete(input);
}
if (File.Exists(output))
{
File.Delete(output);
}
GdalUtils.Contour(input, output, 1d, 0d);
}
catch (System.Exception ex)
{
logger.Error(ex, Gdal.GetLastErrorMsg());
}
}
// Raster info: Geotiff multiband
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("=============================Info GEOTIFF Multiband======================================");
GDALInfo.Info(Path.Combine(datapath, "CHEBROe00.20201125.tif"), false);
}
// Raster info: GRIB2 multiband
if (false)
{
Console.WriteLine("===================================================================");
Console.WriteLine("==============================Info GRIB2=====================================");
GDALInfo.Info(Path.Combine(datapath, "CHEBROe00.20201125.grib2"), true);
}
}
public static bool IsWindows() => RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
private async Task ThrowWhenHandlePositionIsChanged(bool useAsync)
{
string fileName = GetTestFilePath();
using (FileStream fs = new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 0x100, useAsync))
{
// write some data to move the position, flush to ensure OS position is updated
fs.Write(TestBuffer, 0, TestBuffer.Length);
fs.Flush();
if (fs.SafeFileHandle.IsInvalid)
{
// nothing to test
return;
}
using (FileStream fsr = new FileStream(fs.SafeFileHandle, FileAccess.Read, TestBuffer.Length, useAsync))
{
Assert.Equal(TestBuffer.Length, fs.Position);
Assert.Equal(TestBuffer.Length, fsr.Position);
// Operations on original filestream will fail if data is in buffer and position changes.
// Put data in FS write buffer and update position from FSR
fs.WriteByte(0);
fsr.Position = 0;
Assert.Throws <IOException>(() => fs.Position);
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.Read(new byte[1], 0, 1));
fs.WriteByte(0);
fsr.Position++;
if (useAsync && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Async I/O behaviors differ due to kernel-based implementation on Windows
{
Assert.Throws <IOException>(() => FSAssert.CompletesSynchronously(fs.ReadAsync(new byte[1], 0, 1)));
}
else
{
await Assert.ThrowsAsync <IOException>(() => fs.ReadAsync(new byte[1], 0, 1));
}
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.ReadByte());
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.Seek(0, SeekOrigin.End));
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.SetLength(2));
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.Flush());
fs.WriteByte(0);
fsr.Position++;
Assert.Throws <IOException>(() => fs.Dispose());
}
}
}
public override void Bad()
{
string data;
while (true)
{
data = ""; /* Initialize data */
/* Read data from a database */
{
try
{
/* setup the connection */
using (SqlConnection connection = IO.GetDBConnection())
{
connection.Open();
/* prepare and execute a (hardcoded) query */
using (SqlCommand command = new SqlCommand(null, connection))
{
command.CommandText = "select name from users where id=0";
command.Prepare();
using (SqlDataReader dr = command.ExecuteReader())
{
/* POTENTIAL FLAW: Read data from a database query SqlDataReader */
data = dr.GetString(1);
}
}
}
}
catch (SqlException exceptSql)
{
IO.Logger.Log(NLog.LogLevel.Warn, exceptSql, "Error with SQL statement");
}
}
break;
}
while (true)
{
string xmlFile = null;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
/* running on Windows */
xmlFile = "..\\..\\CWE643_Xpath_Injection__Helper.xml";
}
else
{
/* running on non-Windows */
xmlFile = "../../CWE643_Xpath_Injection__Helper.xml";
}
if (data != null)
{
/* assume username||password as source */
string[] tokens = data.Split("||".ToCharArray());
if (tokens.Length < 2)
{
return;
}
string username = tokens[0];
string password = tokens[1];
/* build xpath */
XPathDocument inputXml = new XPathDocument(xmlFile);
XPathNavigator xPath = inputXml.CreateNavigator();
/* INCIDENTAL: CWE180 Incorrect Behavior Order: Validate Before Canonicalize
* The user input should be canonicalized before validation. */
/* POTENTIAL FLAW: user input is used without validate */
string query = "//users/user[name/text()='" + username +
"' and pass/text()='" + password + "']" +
"/secret/text()";
string secret = (string)xPath.Evaluate(query);
}
break;
}
}
/// <summary>
/// Gets the command line arguments for ffmpeg.
/// </summary>
/// <param name="outputPath">The output path of the file.</param>
/// <param name="state">The <see cref="StreamState"/>.</param>
/// <returns>The command line arguments as a string.</returns>
private string GetCommandLineArguments(string outputPath, StreamState state)
{
var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions);
var threads = EncodingHelper.GetNumberOfThreads(state, _encodingOptions, videoCodec); // GetNumberOfThreads is static.
var inputModifier = _encodingHelper.GetInputModifier(state, _encodingOptions);
var mapArgs = state.IsOutputVideo ? _encodingHelper.GetMapArgs(state) : string.Empty;
var directory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath));
var outputFileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputPath);
var outputPrefix = Path.Combine(directory, outputFileNameWithoutExtension);
var outputExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer);
var outputTsArg = outputPrefix + "%d" + outputExtension;
var segmentFormat = outputExtension.TrimStart('.');
if (string.Equals(segmentFormat, "ts", StringComparison.OrdinalIgnoreCase))
{
segmentFormat = "mpegts";
}
else if (string.Equals(segmentFormat, "mp4", StringComparison.OrdinalIgnoreCase))
{
var outputFmp4HeaderArg = string.Empty;
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
if (isWindows)
{
// on Windows, the path of fmp4 header file needs to be configured
outputFmp4HeaderArg = " -hls_fmp4_init_filename \"" + outputPrefix + "-1" + outputExtension + "\"";
}
else
{
// on Linux/Unix, ffmpeg generate fmp4 header file to m3u8 output folder
outputFmp4HeaderArg = " -hls_fmp4_init_filename \"" + outputFileNameWithoutExtension + "-1" + outputExtension + "\"";
}
segmentFormat = "fmp4" + outputFmp4HeaderArg;
}
else
{
_logger.LogError("Invalid HLS segment container: {SegmentFormat}", segmentFormat);
}
var maxMuxingQueueSize = _encodingOptions.MaxMuxingQueueSize > 128
? _encodingOptions.MaxMuxingQueueSize.ToString(CultureInfo.InvariantCulture)
: "128";
var baseUrlParam = string.Format(
CultureInfo.InvariantCulture,
"\"hls/{0}/\"",
Path.GetFileNameWithoutExtension(outputPath));
return(string.Format(
CultureInfo.InvariantCulture,
"{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number 0 -hls_base_url {9} -hls_playlist_type event -hls_segment_filename \"{10}\" -y \"{11}\"",
inputModifier,
_encodingHelper.GetInputArgument(state, _encodingOptions),
threads,
mapArgs,
GetVideoArguments(state),
GetAudioArguments(state),
maxMuxingQueueSize,
state.SegmentLength.ToString(CultureInfo.InvariantCulture),
segmentFormat,
baseUrlParam,
outputTsArg,
outputPath).Trim());
}
static void Main(string[] args)
{
if (!string.IsNullOrWhiteSpace(GetArg("-remote")))
{
new MyWindow(GetArg("-remote"));
return;
}
if (!string.IsNullOrWhiteSpace(GetArg("-child")) || !string.IsNullOrWhiteSpace(GetArg("-parent")))
{
string ch = GetArg("-child");
string parent = GetArg("-parent");
Process.GetProcessById(int.Parse(parent)).WaitForExit();
Process.GetProcessById(int.Parse(ch)).Kill();
Process.GetProcessById(int.Parse(ch)).WaitForExit();
return;
}
IPGlobalProperties ipGlobalProperties = IPGlobalProperties.GetIPGlobalProperties();
TcpConnectionInformation[] tcpConnInfoArray = ipGlobalProperties.GetActiveTcpConnections();
//look for a port that is not in use
port = new Random().Next(50000, 60000);
foreach (TcpConnectionInformation tcpi in tcpConnInfoArray)
{
if (tcpi.LocalEndPoint.Port == port)
{
portfound = false;
Console.WriteLine("console port was already in use");
break;
}
}
if (portfound)
{
Console.WriteLine("Loading");
List <string> cmdargs = new List <string>();
string command = string.Empty;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Console.WriteLine("Platform windows");
if (string.IsNullOrEmpty(GetArg("-gamelocation")))
{
Console.WriteLine("Starting game server...");
string file = AppDomain.CurrentDomain.BaseDirectory + "/SCP_ET.exe";
Console.WriteLine(file);
command = file;
cmdargs.Add("-consoleport");
cmdargs.Add(port.ToString());
cmdargs.Add("-logfile");
cmdargs.Add(AppDomain.CurrentDomain.BaseDirectory + "/logs/SCP-ETServerLog-" + DateTime.UtcNow.Ticks + ".txt");
}
else
{
Console.WriteLine("Starting game server...");
string file = GetArg("-gamelocation") + "/SCP_ET.exe";
Console.WriteLine(file);
command = file;
cmdargs.Add("-consoleport");
cmdargs.Add(port.ToString());
cmdargs.Add("-logfile");
cmdargs.Add(GetArg("-gamelocation") + "/logs/SCP-ETServerLog-" + DateTime.UtcNow.Ticks + ".txt");
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
Console.WriteLine("Platform linux");
if (string.IsNullOrEmpty(GetArg("-gamelocation")))
{
Console.WriteLine("Starting game server...");
string file = AppDomain.CurrentDomain.BaseDirectory + "/scp_et.x86_64";
Console.WriteLine(file);
command = file;
cmdargs.Add("-consoleport");
cmdargs.Add(port.ToString());
cmdargs.Add("-logfile");
cmdargs.Add(AppDomain.CurrentDomain.BaseDirectory + "/logs/SCP-ETServerLog-" + DateTime.UtcNow.Ticks + ".txt");
}
else
{
Console.WriteLine("Starting game server...");
string file = GetArg("-gamelocation") + "/scp_et.x86_64";
Console.WriteLine(file);
command = file;
cmdargs.Add("-consoleport");
cmdargs.Add(port.ToString());
cmdargs.Add("-logfile");
cmdargs.Add(GetArg("-gamelocation") + "/logs/SCP-ETServerLog-" + DateTime.UtcNow.Ticks + ".txt");
}
}
else
{
Console.WriteLine($"Unable to start game server: Platform {RuntimeInformation.OSDescription} not supported.");
}
// ChildProcessStartInfo info = new ChildProcessStartInfo(Path.GetFullPath(command), cmdargs.ToArray());
ProcessStartInfo info2 = new ProcessStartInfo(Path.GetFullPath(command), string.Join(' ', cmdargs));
using (Process cmd = Process.Start(info2))
{
ProcessStartInfo currentInfo = new ProcessStartInfo(Environment.GetCommandLineArgs()[0], $"-child {cmd.Id} -parent {Process.GetCurrentProcess().Id}");
Process current = Process.Start(currentInfo);
console = new TcpConsoleClient();
console.ConnectToTcpServer();
gameServer = new GameTcpServer(int.TryParse(GetArg("-remoteport"), out int rconport) ? rconport : 8701);
AppDomain.CurrentDomain.DomainUnload += (s, e) => { cmd.Kill(); cmd.WaitForExit(); };
AppDomain.CurrentDomain.ProcessExit += (s, e) => { cmd.Kill(); cmd.WaitForExit(); };
AppDomain.CurrentDomain.UnhandledException += (s, e) => { cmd.Kill(); cmd.WaitForExit(); };
listeninput();
}
}
}
public override async Task ObserveAsync(CancellationToken token)
{
// If set, this observer will only run during the supplied interval.
// See Settings.xml, CertificateObserverConfiguration section, RunInterval parameter for an example.
if (RunInterval > TimeSpan.MinValue &&
DateTime.Now.Subtract(LastRunDateTime) < RunInterval)
{
return;
}
Token = token;
if (Token.IsCancellationRequested)
{
return;
}
Initialize();
try
{
foreach (var procName in this.processWatchList)
{
Token.ThrowIfCancellationRequested();
string dotnet = string.Empty;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && procName.EndsWith(".dll"))
{
dotnet = "dotnet ";
}
GetProcessInfo($"{dotnet}{procName}");
}
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
WriteToLogWithLevel(
ObserverName,
"Unhandled exception in ObserveAsync. Failed to observe CPU and Memory usage of " + string.Join(",", this.processWatchList) + ": " + e,
LogLevel.Error);
}
throw;
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
ObserverManager.ObserverWebAppDeployed &&
this.monitorWinEventLog)
{
ReadServiceFabricWindowsEventLog();
}
// Set TTL.
this.stopwatch.Stop();
RunDuration = this.stopwatch.Elapsed;
this.stopwatch.Reset();
await ReportAsync(token).ConfigureAwait(true);
LastRunDateTime = DateTime.Now;
}
public override void Verify()
{
// verify that aaguid is 16 empty bytes (note: required by fido2 conformance testing, could not find this in spec?)
if (0 != AuthData.AttestedCredentialData.AaGuid.CompareTo(Guid.Empty))
{
throw new VerificationException("Aaguid was not empty parsing fido-u2f atttestation statement");
}
// 1. Verify that attStmt is valid CBOR conforming to the syntax defined above and perform CBOR decoding on it to extract the contained fields.
if (null == X5c || CBORType.Array != X5c.Type || X5c.Count != 1)
{
throw new VerificationException("Malformed x5c in fido - u2f attestation");
}
// 2a. the attestation certificate attestnCert MUST be the first element in the array
if (null == X5c.Values || 0 == X5c.Values.Count ||
CBORType.ByteString != X5c.Values.First().Type ||
0 == X5c.Values.First().GetByteString().Length)
{
throw new VerificationException("Malformed x5c in fido-u2f attestation");
}
var cert = new X509Certificate2(X5c.Values.First().GetByteString());
// TODO : Check why this variable isn't used. Remove it or use it.
var u2ftransports = U2FTransportsFromAttnCert(cert.Extensions);
var aaguid = AaguidFromAttnCertExts(cert.Extensions);
if (null != _metadataService && null != aaguid)
{
var guidAaguid = AttestedCredentialData.FromBigEndian(aaguid);
var entry = _metadataService.GetEntry(guidAaguid);
if (null != entry && null != entry.MetadataStatement)
{
if (entry.Hash != entry.MetadataStatement.Hash)
{
throw new VerificationException("Authenticator metadata statement has invalid hash");
}
var root = new X509Certificate2(Convert.FromBase64String(entry.MetadataStatement.AttestationRootCertificates.FirstOrDefault()));
var chain = new X509Chain();
chain.ChainPolicy.ExtraStore.Add(root);
chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
var valid = chain.Build(cert);
if (// the root cert has exactly one status listed against it
chain.ChainElements[chain.ChainElements.Count - 1].ChainElementStatus.Length == 1 &&
// and that that status is a status of exactly UntrustedRoot
chain.ChainElements[chain.ChainElements.Count - 1].ChainElementStatus[0].Status == X509ChainStatusFlags.UntrustedRoot)
{
valid = true;
}
if (false == valid)
{
throw new VerificationException("Invalid certificate chain in U2F attestation");
}
}
}
// 2b. If certificate public key is not an Elliptic Curve (EC) public key over the P-256 curve, terminate this algorithm and return an appropriate error
var pubKey = cert.GetECDsaPublicKey();
var keyParams = pubKey.ExportParameters(false);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (!keyParams.Curve.Oid.FriendlyName.Equals(ECCurve.NamedCurves.nistP256.Oid.FriendlyName))
{
throw new VerificationException("Attestation certificate public key is not an Elliptic Curve (EC) public key over the P-256 curve");
}
}
else
{
if (!keyParams.Curve.Oid.Value.Equals(ECCurve.NamedCurves.nistP256.Oid.Value))
{
throw new VerificationException("Attestation certificate public key is not an Elliptic Curve (EC) public key over the P-256 curve");
}
}
// 3. Extract the claimed rpIdHash from authenticatorData, and the claimed credentialId and credentialPublicKey from authenticatorData
// see rpIdHash, credentialId, and credentialPublicKey variables
// 4. Convert the COSE_KEY formatted credentialPublicKey (see Section 7 of [RFC8152]) to CTAP1/U2F public Key format
var x = CredentialPublicKey[CBORObject.FromObject(COSE.KeyTypeParameter.X)].GetByteString();
var y = CredentialPublicKey[CBORObject.FromObject(COSE.KeyTypeParameter.Y)].GetByteString();
var publicKeyU2F = new byte[1] {
0x4
}.Concat(x).Concat(y).ToArray();
// 5. Let verificationData be the concatenation of (0x00 || rpIdHash || clientDataHash || credentialId || publicKeyU2F)
var verificationData = new byte[1] {
0x00
};
verificationData = verificationData
.Concat(AuthData.RpIdHash)
.Concat(clientDataHash)
.Concat(AuthData.AttestedCredentialData.CredentialID)
.Concat(publicKeyU2F.ToArray())
.ToArray();
// 6. Verify the sig using verificationData and certificate public key
if (null == Sig || CBORType.ByteString != Sig.Type || 0 == Sig.GetByteString().Length)
{
throw new VerificationException("Invalid fido-u2f attestation signature");
}
var ecsig = CryptoUtils.SigFromEcDsaSig(Sig.GetByteString(), pubKey.KeySize);
if (null == ecsig)
{
throw new VerificationException("Failed to decode fido-u2f attestation signature from ASN.1 encoded form");
}
var coseAlg = CredentialPublicKey[CBORObject.FromObject(COSE.KeyCommonParameter.Alg)].AsInt32();
var hashAlg = CryptoUtils.algMap[coseAlg];
if (true != pubKey.VerifyData(verificationData, ecsig, hashAlg))
{
throw new VerificationException("Invalid fido-u2f attestation signature");
}
}
public override Task ReportAsync(CancellationToken token)
{
Token.ThrowIfCancellationRequested();
// Informational report. For now, Linux is where we pay close attention to memory use by Fabric system services as there are still a few issues in that realm..
var timeToLiveWarning = SetHealthReportTimeToLive();
var portInformationReport = new HealthReport
{
Observer = ObserverName,
NodeName = NodeName,
HealthMessage = $"Number of ports in use by Fabric services: {TotalActivePortCountAllSystemServices}{Environment.NewLine}" +
$"Number of ephemeral ports in use by Fabric services: {TotalActiveEphemeralPortCountAllSystemServices}{Environment.NewLine}" +
$"Fabric memory use MB: {this.allMemData.Where(x => x.Id == "Fabric")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" +
(RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ?
$"FabricGateway memory use MB: {this.allMemData.Where(x => x.Id == "FabricGateway.exe")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" +
$"FabricHost memory use MB: {this.allMemData.Where(x => x.Id == "FabricHost")?.FirstOrDefault()?.AverageDataValue}{Environment.NewLine}" : string.Empty),
State = HealthState.Ok,
HealthReportTimeToLive = timeToLiveWarning,
};
HealthReporter.ReportHealthToServiceFabric(portInformationReport);
// Reset ports counters.
TotalActivePortCountAllSystemServices = 0;
TotalActiveEphemeralPortCountAllSystemServices = 0;
// CPU
ProcessResourceDataList(
this.allCpuData,
CpuErrorUsageThresholdPct,
CpuWarnUsageThresholdPct);
// Memory
ProcessResourceDataList(
this.allMemData,
MemErrorUsageThresholdMb,
MemWarnUsageThresholdMb);
// Ports - Active TCP
ProcessResourceDataList(
this.allActiveTcpPortData,
ActiveTcpPortCountError,
ActiveTcpPortCountWarning);
// Ports - Ephemeral
ProcessResourceDataList(
this.allEphemeralTcpPortData,
ActiveEphemeralPortCountError,
ActiveEphemeralPortCountWarning);
// Windows Event Log
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && ObserverManager.ObserverWebAppDeployed &&
this.monitorWinEventLog)
{
// SF Eventlog Errors?
// Write this out to a new file, for use by the web front end log viewer.
// Format = HTML.
int count = this.evtRecordList.Count();
var logPath = Path.Combine(ObserverLogger.LogFolderBasePath, "EventVwrErrors.txt");
// Remove existing file.
if (File.Exists(logPath))
{
try
{
File.Delete(logPath);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
if (count >= 10)
{
var sb = new StringBuilder();
_ = sb.AppendLine("<br/><div><strong>" +
"<a href='javascript:toggle(\"evtContainer\")'>" +
"<div id=\"plus\" style=\"display: inline; font-size: 25px;\">+</div> " + count +
" Error Events in ServiceFabric and System</a> " +
"Event logs</strong>.<br/></div>");
_ = sb.AppendLine("<div id='evtContainer' style=\"display: none;\">");
foreach (var evt in this.evtRecordList.Distinct())
{
token.ThrowIfCancellationRequested();
try
{
// Access event properties:
_ = sb.AppendLine("<div>" + evt.LogName + "</div>");
_ = sb.AppendLine("<div>" + evt.LevelDisplayName + "</div>");
if (evt.TimeCreated.HasValue)
{
_ = sb.AppendLine("<div>" + evt.TimeCreated.Value.ToShortDateString() + "</div>");
}
foreach (var prop in evt.Properties)
{
if (prop.Value != null && Convert.ToString(prop.Value).Length > 0)
{
_ = sb.AppendLine("<div>" + prop.Value + "</div>");
}
}
}
catch (EventLogException)
{
}
}
_ = sb.AppendLine("</div>");
_ = ObserverLogger.TryWriteLogFile(logPath, sb.ToString());
_ = sb.Clear();
}
// Clean up.
if (count > 0)
{
this.evtRecordList.Clear();
}
}
ClearDataContainers();
return(Task.CompletedTask);
}
public bool Compile()
{
List <MetadataReference> _ref = new List <MetadataReference>();
if (TargetType == TargetType.NETSTANDARD)
{
FileInfo fileInfo = new FileInfo(Assembly.GetAssembly(typeof(CoreCompiler)).Location);
var dire = fileInfo.Directory;
var str = "netstandard_";
switch (NetStandardVersion)
{
case NetStandardVersion.V2_0_0:
str += "2.0.0.dll";
break;
case NetStandardVersion.V2_0_3:
str += "2.0.3.dll";
break;
default:
break;
}
PreDLLRefs.Add(new FileInfo(Path.Combine(dire.FullName, "NetStandardLib", str)));
DLLRefs.Add(Path.Combine(dire.FullName, "NetStandardLib", str));
}
else
{
var dir = (new FileInfo(typeof(object).Assembly.Location).Directory);
foreach (var item in dir.EnumerateFiles("*.dll"))
{
PreDLLRefs.Add(item);
if (item.Name == "System.Runtime.dll")
{
DLLRefs.Add(item.FullName);
}
//else if (item.Name == "System.Private.CoreLib.dll")
//{
// DLLRefs.Add(item.FullName);
//}
}
{
_ref.Add(MetadataReference.CreateFromFile(typeof(object).Assembly.Location));
_ref.Add(MetadataReference.CreateFromFile(typeof(Console).Assembly.Location));
_ref.Add(MetadataReference.CreateFromFile(typeof(string).Assembly.Location));
_ref.Add(MetadataReference.CreateFromFile(typeof(System.Collections.IEnumerable).Assembly.Location));
_ref.Add(MetadataReference.CreateFromFile(typeof(File).Assembly.Location));
}
}
//SourceFileResolver sourceFileResolver=new SourceFileResolver()
var c = CSharpCompilation.Create(TargetName).WithOptions(new CSharpCompilationOptions(
TargetType != TargetType.EXE ? OutputKind.DynamicallyLinkedLibrary : OutputKind.ConsoleApplication,
optimizationLevel: optimizationLevel,
allowUnsafe: AllowUnsafe,
platform: Platform,
mainTypeName: MainClass == "" ? null : MainClass,
warningLevel: 4,
xmlReferenceResolver: null
));
List <SyntaxTree> syntaxTrees = new List <SyntaxTree>();
List <string> Usings = new List <string>();
Console.WriteLine(Language.GetString("General", "ResSrcFile", "Resolving Source Files..."));
foreach (var item in files)
{
var s = CSharpSyntaxTree.ParseText(File.ReadAllText(item), new CSharpParseOptions(LanguageVersion, kind: SourceCodeKind.Regular));
foreach (var Syntax in s.GetRoot().ChildNodes())
{
if (Syntax.IsKind(SyntaxKind.UsingDirective))
{
String @using = Syntax.ChildNodes().First().ToString();
if (!Usings.Contains(@using))
{
Usings.Add(@using);
}
}
}
syntaxTrees.Add(s);
}
foreach (var item in Usings)
{
foreach (var dlls in PreDLLRefs)
{
//if (dlls.Name.StartsWith(item))
//{
// DLLRefs.Add(dlls.FullName);
//}
if (dlls.Name == (item + ".dll"))
{
_ref.Add(MetadataReference.CreateFromFile(dlls.FullName));
//DLLRefs.Add(dlls.FullName);
}
}
}
foreach (var item in DLLRefs)
{
_ref.Add(MetadataReference.CreateFromFile(item));
}
c = c.AddReferences(_ref)
.AddSyntaxTrees(syntaxTrees);
string realFile = TargetFile;
if (realFile.ToLower().EndsWith(".exe"))
{
realFile = realFile.Remove(realFile.Length - 4) + ".dll";
}
else if (!realFile.ToLower().EndsWith(".dll"))
{
realFile = realFile + ".dll";
}
var result = c.Emit(realFile);
foreach (var item in result.Diagnostics)
{
Console.WriteLine(item.ToString());
}
if (result.Success == true)
{
DirectoryInfo TargetDir = new FileInfo(TargetFile).Directory;
if (CopyLibs == true)
{
foreach (var item in DLLRefs)
{
var fi = new FileInfo(item);
fi.CopyTo(Path.Combine(TargetDir.FullName, fi.Name), true);
}
}
if (result.Success)
{
if (TargetType == TargetType.EXE)
{
{
var fi = new FileInfo(Assembly.GetAssembly(this.GetType()).Location);
var di = fi.Directory;
var runtimeConfig = new FileInfo(Path.Combine(di.FullName, "scsc.runtimeconfig.json"));
runtimeConfig.CopyTo(Path.Combine(TargetDir.FullName, new FileInfo(realFile).Name.Remove(new FileInfo(realFile).Name.Length - 4) + ".runtimeconfig.json"), true);
}
{
var fi = new FileInfo(Assembly.GetAssembly(this.GetType()).Location);
var di = fi.Directory;
var runtimeConfig = new FileInfo(Path.Combine(di.FullName, "scsc.runtimeconfig.json"));
runtimeConfig.CopyTo(Path.Combine(TargetDir.FullName, "SimpleCSharpCompiler.Host.runtimeconfig.json"), true);
}
{
var fi = new FileInfo(Assembly.GetAssembly(this.GetType()).Location);
var di = fi.Directory;
var runtimeConfig = new FileInfo(Path.Combine(di.FullName, "SimpleCSharpCompiler.Host.dll"));
runtimeConfig.CopyTo(Path.Combine(TargetDir.FullName, "SimpleCSharpCompiler.Host.dll"), true);
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var fi = new FileInfo(Assembly.GetAssembly(this.GetType()).Location);
var di = fi.Directory;
if (Platform == Platform.X64)
{
var HostFile = new FileInfo(Path.Combine(di.FullName, "PreBuilt", "Host.Win32.x64.exe"));
HostFile.CopyTo(TargetFile, true);
}
else
if (Platform == Platform.X86)
{
var HostFile = new FileInfo(Path.Combine(di.FullName, "PreBuilt", "Host.Win32.x86.exe"));
HostFile.CopyTo(TargetFile, true);
}
}
else
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
var fi = new FileInfo(Assembly.GetAssembly(this.GetType()).Location);
var di = fi.Directory;
var HostFile = new FileInfo(Path.Combine(di.FullName, "PreBuilt", "Host.Linux64"));
HostFile.CopyTo(TargetFile, true);
}
{
File.WriteAllText(Path.Combine(TargetDir.FullName, "Runtime.Config"), new FileInfo(realFile).Name);
}
}
}
}
return(result.Success);
}
public MiniClusterNode(
string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp,
IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds,
ISubsystem[] subsystems = null, int?chunkSize = null, int?cachedChunkSize = null,
bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000,
bool inMemDb = true, bool disableFlushToDisk = false, bool readOnlyReplica = false)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport",
true); //TODO JPB Remove this sadness when dotnet core supports kestrel + http2 on macOS
}
RunningTime.Start();
RunCount += 1;
DebugIndex = debugIndex;
_dbPath = Path.Combine(
pathname,
string.Format(
"mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port));
Directory.CreateDirectory(_dbPath);
FileStreamExtensions.ConfigureFlush(disableFlushToDisk);
Db =
new TFChunkDb(
CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb));
InternalTcpEndPoint = internalTcp;
InternalTcpSecEndPoint = internalTcpSec;
InternalHttpEndPoint = internalHttp;
ExternalTcpEndPoint = externalTcp;
ExternalTcpSecEndPoint = externalTcpSec;
ExternalHttpEndPoint = externalHttp;
var certificate = ssl_connections.GetCertificate();
var disableInternalTls = !ssl_connections.IsValidCertificate(certificate); //use internal TLS only if CA certificate is installed
var singleVNodeSettings = new ClusterVNodeSettings(
Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint,
ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint,
new Data.GossipAdvertiseInfo(InternalTcpEndPoint, InternalTcpSecEndPoint,
ExternalTcpEndPoint, ExternalTcpSecEndPoint,
InternalHttpEndPoint, ExternalHttpEndPoint,
null, null, 0, 0), enableTrustedAuth,
certificate, 1, false,
"", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10),
disableInternalTls, false, TimeSpan.FromHours(1), StatsStorage.None, 0,
new InternalAuthenticationProviderFactory(), new LegacyAuthorizationProviderFactory(), disableScavengeMerging: true, scavengeHistoryMaxAge: 30,
adminOnPublic: true,
statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(2),
gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(3),
extTcpHeartbeatTimeout: TimeSpan.FromSeconds(2), extTcpHeartbeatInterval: TimeSpan.FromSeconds(2),
intTcpHeartbeatTimeout: TimeSpan.FromSeconds(2), intTcpHeartbeatInterval: TimeSpan.FromSeconds(2),
deadMemberRemovalPeriod: TimeSpan.FromSeconds(1800),
verifyDbHash: false, maxMemtableEntryCount: memTableSize,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
startStandardProjections: false, disableHTTPCaching: false, logHttpRequests: false,
connectionPendingSendBytesThreshold: Opts.ConnectionPendingSendBytesThresholdDefault,
connectionQueueSizeThreshold: Opts.ConnectionQueueSizeThresholdDefault,
readOnlyReplica: readOnlyReplica,
ptableMaxReaderCount: Constants.PTableMaxReaderCountDefault,
enableExternalTCP: true,
createHttpMessageHandler: () => new SocketsHttpHandler {
SslOptions = new SslClientAuthenticationOptions {
RemoteCertificateValidationCallback = delegate { return(true); }
}
},
gossipOverHttps: UseHttpsInternally());
_isReadOnlyReplica = readOnlyReplica;
Log.Information(
"\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n"
+ "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n",
"ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp,
"OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(),
Marshal.SizeOf(typeof(IntPtr)) * 8, "GC:",
GC.MaxGeneration == 0
? "NON-GENERATION (PROBABLY BOEHM)"
: string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:",
ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:",
ExternalHttpEndPoint);
Node = new ClusterVNode(Db, singleVNodeSettings,
infoController: new InfoController(null, ProjectionType.None), subsystems: subsystems,
gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds));
Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue));
_host = new WebHostBuilder()
.UseKestrel(o => {
o.Listen(InternalHttpEndPoint, options => {
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
options.Protocols = HttpProtocols.Http2;
}
else
{
options.UseHttps();
}
});
o.Listen(ExternalHttpEndPoint);
})
.UseStartup(Node.Startup)
.Build();
_kestrelTestServer = new TestServer(new WebHostBuilder()
.UseKestrel()
.UseStartup(Node.Startup));
}
