public FileLogger(string path, LogLevel logLevel = LogLevel.Debug) : base(logLevel)
{
try
{
if (!File.Exists(path))
{
stream = new StreamWriter(File.Open(path, FileMode.Create, FileAccess.Write, FileShare.Read));
}
else
{
stream = new StreamWriter(File.Open(path, FileMode.Truncate, FileAccess.Write, FileShare.Read));
}
queue = new ConcurrentQueue <string>();
writerTask = new Task(() =>
{
bool needFlush = false;
while (true)
{
if (queue.TryDequeue(out string msg))
{
stream?.WriteLineAsync(msg).Wait();
needFlush = true;
}
else
{
if (needFlush)
{
stream?.FlushAsync().Wait();
needFlush = false;
}
if (isTerminating)
{
return;
}
Thread.Sleep(10);
}
}
});
writerTask.Start();
writer = GetFileWriter(stream, queue);
Terminate = () =>
{
isTerminating = true;
writerTask?.Wait();
};
}
catch
{
stream = null;
}
}
private static async Task SendAsync(StreamWriter writer, string messageToSend)
{
ConfiguredTaskAwaitable?writeLineAwaitable = writer?.WriteLineAsync(messageToSend).ConfigureAwait(false);
if (writeLineAwaitable.HasValue)
{
await writeLineAwaitable.Value;
}
ConfiguredTaskAwaitable?flushAwaitable = writer?.FlushAsync().ConfigureAwait(false);
if (flushAwaitable.HasValue)
{
await flushAwaitable.Value;
}
}
public async Task SendFlightPlanAsync(string callsign, bool isIFR, string type, string registration, string title,
string departure, string arrival, string route, int?speed, int altitude, TimeSpan?enroute, string pilotRemarks)
{
var ifrs = isIFR ? "I" : "V";
var alternate = "NONE";
// NOTE: this is not needed right now due to lack of data
//var remarks = $"Aircraft = {title.Replace(":", "_")}. Registration = {registration.Replace(":", "_")}";
var remarks = string.IsNullOrWhiteSpace(pilotRemarks) ? string.Empty : pilotRemarks.Replace("\n", " ").Replace(":", "_");
var fp = $"$FP{callsign}:*A:{ifrs}:{type.Replace(":", "_")}:{speed}:{departure}:::{altitude}:{arrival}:::{(enroute == null ? ":" : $"{enroute.Value.Hours:00}:{enroute.Value.Minutes:00}")}:{alternate}:{remarks}:{route}:";
await writer?.WriteLineAsync(fp);
await writer?.FlushAsync();
logger.LogInformation("Sent Flight Plan: " + fp);
}
private async Task <T> GetKeyObjectFromFile <T>(string name, IKeyJsonOps <T> keyOp)
{
string keyPath = GetFunctionSecretsFilePath(name);
string key = null;
if (!FileSystemHelpers.FileExists(keyPath))
{
FileSystemHelpers.EnsureDirectory(Path.GetDirectoryName(keyPath));
try
{
using (var fileStream = FileSystemHelpers.OpenFile(keyPath, FileMode.CreateNew, FileAccess.Write, FileShare.None))
// will fail if file exists, prevent reading prematurely
// getting the lock early so no redundant work is being done
{
string jsonContent = keyOp.GenerateKeyJson(SecurityUtility.GenerateSecretStringsKeyPair(keyOp.NumberOfKeysInDefaultFormat), FunctionSiteExtensionVersion, out key);
using (var sw = new StringWriter())
using (var sr = new System.IO.StringReader(jsonContent))
{
new JsonTextWriter(sw)
{
Formatting = Formatting.Indented
}.WriteToken(new JsonTextReader(sr));
// if lock acquire lock return false, I wait until write finishes and read keyPath
using (var streamWriter = new StreamWriter(fileStream))
{
await streamWriter.WriteAsync(sw.ToString());
await streamWriter.FlushAsync();
}
}
}
return(keyOp.GenerateKeyObject(key, name));
}
catch (IOException)
{
// don't throw exception if the file already existed
// fallback to read key files
}
}
string jsonStr = null;
int timeOut = 5;
while (true)
{
try
{
jsonStr = await FileSystemHelpers.ReadAllTextFromFileAsync(keyPath);
break;
}
catch (Exception)
{
if (timeOut == 0)
{
throw new TimeoutException($"Fail to read {keyPath}, the file is being held by another process");
}
timeOut--;
await Task.Delay(250);
}
}
bool isEncrypted;
key = keyOp.GetKeyValueFromJson(jsonStr, out isEncrypted);
if (isEncrypted)
{
key = SecurityUtility.DecryptSecretString(key);
}
return(keyOp.GenerateKeyObject(key, name));
}
public async Task WriteableFilesAreWriteable()
{
const string TestName = nameof(WriteableFilesAreWriteable);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
ICacheSession session = await cache.CreateSessionAsync().SuccessAsync();
string origionalFileContents = "foo";
CasHash hash;
string filePath = FileUtilities.GetTempFileName();
try
{
using (StreamWriter sw = new StreamWriter(filePath))
{
await sw.WriteAsync(origionalFileContents);
await sw.FlushAsync();
sw.Close();
}
hash = await session.AddToCasAsync(filePath, FileState.Writeable).SuccessAsync();
// Great, so it added.
using (StreamWriter sw = new StreamWriter(filePath, true))
{
await sw.WriteLineAsync("Bar");
}
// Get the origional file and read it to ensure it didn't change.
StreamReader sr = new StreamReader(await session.GetStreamAsync(hash).SuccessAsync());
string cacheContents = await sr.ReadToEndAsync();
XAssert.AreEqual(origionalFileContents, cacheContents, "File content in the cache should not have been modified by a write to a file added as writeable.");
}
finally
{
File.Delete(filePath);
}
// Now, materialize the file and ensure we can write to it.
filePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
string placedPath = await session.ProduceFileAsync(hash, filePath, FileState.Writeable).SuccessAsync();
try
{
XAssert.AreEqual(filePath, placedPath, "File should be placed where asked...");
using (StreamWriter sw = new StreamWriter(filePath, true))
{
await sw.WriteLineAsync("Bar");
}
}
finally
{
File.Delete(placedPath);
}
await session.CloseAsync().SuccessAsync();
await ShutdownCacheAsync(cache, testCacheId);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILiveApis liveApis, ISharedSettings sharedSettings, ILogger <HttpService> logger)
{
// if (env.IsDevelopment())
// {
// app.UseDeveloperExceptionPage();
// }
// only allow requests from the original web site.
app.UseCors(builder =>
{
builder.AllowAnyOrigin()
.AllowAnyMethod()
.AllowAnyHeader()
// .AllowCredentials()
.WithHeaders()
.WithMethods();
// .WithOrigins(streams.OriginUrl);
// .WithOrigins();
});
// var rand = EncryptString.GenerateRandomKey();
app.Run(async(context) =>
{
try
{
async Task SendFailedResponse(ReturnValue returnValue)
{
logger.LogError(returnValue.Exception,
$"Path: {context.Request.Path}, Message: {returnValue.Message}");
context.Response.StatusCode = 400;
context.Response.ContentType = "application/json";
await using (var writer = new StreamWriter(context.Response.Body))
{
await writer.WriteAsync(returnValue.Serialize());
await writer.FlushAsync().ConfigureAwait(false);
}
}
async Task SendInvalidPath()
{
var returnValue = new ReturnValue(false, $"Invalid url path. {context.Request.Path}.", null);
await SendFailedResponse(returnValue);
}
context.Features.Get <IHttpMaxRequestBodySizeFeature>().MaxRequestBodySize = 1_000_000_000;
var path = context.Request.Path;
var segments = path.Value.Split('/');
if (segments.Length < 1)
{
await SendInvalidPath();
return;
}
switch (segments[1])
{
case "ping":
context.Response.StatusCode = 200;
context.Response.ContentType = "application/json";
await using (var writer = new StreamWriter(context.Response.Body))
{
await writer.WriteAsync("{ \"status\": \"alive\"}");
await writer.FlushAsync().ConfigureAwait(false);
}
break;
case "setRaw":
if (segments.Length < 3)
{
await SendInvalidPath();
return;
}
try
{
var key = segments[2];
var value = segments[3];
sharedSettings.SetCacheItem(key + "-raw", value);
}
catch (Exception e)
{
var returnValue = new ReturnValue(false, "Set raw call failed: " + e.Message, e);
await SendFailedResponse(returnValue);
}
break;
case "api":
if (segments.Length < 2)
{
await SendInvalidPath();
return;
}
try
{
var key1 = HttpUtility.UrlDecode(segments[2]);
if (segments.Length > 3 && segments[3] == "ping")
{
var ping = liveApis.Ping(key1);
await using (var writer = new StreamWriter(context.Response.Body))
{
await writer.WriteAsync(ping.Serialize());
await writer.FlushAsync().ConfigureAwait(false);
}
return;
}
var action = "";
if (segments.Length > 3)
{
action = segments[3];
}
var parameters = context.Request.QueryString.Value;
var ipAddress = context.Request.HttpContext.Connection.RemoteIpAddress;
var data = await liveApis.Query(key1, action, parameters, ipAddress.ToString());
context.Response.StatusCode = 200;
context.Response.ContentType = "application/json";
await using (var writer = new StreamWriter(context.Response.Body))
{
await writer.WriteAsync(data);
await writer.FlushAsync().ConfigureAwait(false);
}
}
catch (Exception e)
{
var returnValue = new ReturnValue(false, "API call failed: " + e.Message, e);
await SendFailedResponse(returnValue);
}
break;
case "download":
if (segments.Length < 2)
{
await SendInvalidPath();
return;
}
try
{
var key = segments[2];
using (var downloadStream = await sharedSettings.GetCacheItem <DownloadStream>(key))
{
if (downloadStream == null)
{
throw new RemoteException(
"Remote agent call failed, the response key was not found.");
}
switch (downloadStream.Type)
{
case "file":
context.Response.ContentType = "application/octet-stream";
break;
case "csv":
context.Response.ContentType = "text/csv";
break;
case "json":
context.Response.ContentType = "application/json";
break;
default:
throw new ArgumentOutOfRangeException(
$"The type {downloadStream.Type} was not recognized.");
}
context.Response.StatusCode = downloadStream.IsError ? 400 : 200;
if (!string.IsNullOrEmpty(downloadStream.FileName))
{
context.Response.Headers.Add("Content-Disposition",
"attachment; filename=\"" + downloadStream.FileName + "\"");
}
await downloadStream.Stream.CopyToAsync(context.Response.Body,
context.RequestAborted);
}
}
catch (OperationCanceledException)
{
}
catch (Exception e)
{
var returnValue = new ReturnValue(false, "Remote agent error: " + e.Message, e);
await SendFailedResponse(returnValue);
}
break;
case "upload":
if (segments.Length < 2)
{
await SendInvalidPath();
return;
}
try
{
var files = context.Request.Form.Files;
if (files.Count >= 1)
{
var key2 = segments[2];
var uploadStream = await sharedSettings.GetCacheItem <Func <Stream, Task> >(key2);
await uploadStream.Invoke(files[0].OpenReadStream());
}
else
{
throw new Exception("The file upload only supports one file.");
}
}
catch (Exception e)
{
var returnValue = new ReturnValue(false, "Upload data failed: " + e.Message, e);
await SendFailedResponse(returnValue);
}
break;
}
}
catch (Exception ex)
{
logger.LogError(ex, $"Path: {context.Request.Path}, Message: {ex.Message}");
}
});
}
//public virtual async Task<bool> ReadFile()
//{
// long position = 0;
// try
// {
// return await this.ReadFile(position);
// }
// catch (Exception exception1)
// {
// return false;
// }
//}
//public virtual async Task<bool> ReadFile(long position)
//{
// this.m_strings = new List<string>();
// if (!(await FileExistsAsync(this.m_path)))
// {
// return false;
// }
// FileStream stream = null;
// try
// {
// stream = new FileStream(this.m_path, FileMode.Open, FileAccess.Read, (FileShare.Delete | FileShare.ReadWrite));
// }
// catch (Exception obj1)
// {
// return false;
// }
// StreamReader reader = null;
// try
// {
// stream.Seek(position, SeekOrigin.Begin);
// if (position == 0)
// {
// reader = new StreamReader(stream, true);
// }
// else
// {
// reader = new StreamReader(stream, this.m_encoding);
// }
// }
// catch (Exception obj2)
// {
// stream.Close();
// return false;
// }
// bool flag = false;
// string item = string.Empty;
// try
// {
// do
// {
// item = await reader.ReadLineAsync();
// if (item != null)
// {
// this.m_strings.Add(item);
// }
// else
// {
// break;
// }
// } while (true);
// if (position == 0)
// {
// this.m_encoding = reader.CurrentEncoding;
// }
// position = stream.Position;
// flag = true;
// }
// catch (Exception exception1)
// {
// }
// reader.Close();
// stream.Close();
// return flag;
//}
public async Task <bool> WriteFile()
{
bool Ret = false;
Stopwatch SW = new Stopwatch();
SW.Start();
try
{
bool Locked = await this.LockFile(FileShare.Read, FileSystemRights.CreateFiles& FileSystemRights.Write, FileAccess.Write, FileMode.OpenOrCreate);
if (Locked)
{
StreamWriter writer = null;
bool Err = false;
try
{
writer = new StreamWriter(this.m_fileStream, this.m_encoding); //With {.AutoFlush = True}
}
catch (Exception obj2)
{
Console.WriteLine("Error creating streamwriter: " + obj2.Message);
if (writer != null)
{
writer.Close();
}
this.m_fileStream.Close();
Err = true;
}
if (Err == true)
{
return(Ret);
}
long position = 0;
try
{
position = this.m_fileStream.Length;
this.m_fileStream.Seek(0, SeekOrigin.End);
ClsLogDetailItem OutQueueItem = new ClsLogDetailItem();
//if (position == 0)
//{
// //Convert the class to tab delimited format - HEADER:
// string OutCsv = null;
// if (this.m_ProcessAsCSV)
// {
// OutCsv = ClsToCSV(OutQueueItem, true);
// //write header
// await writer.WriteLineAsync(OutCsv);
// }
// else
// {
// if (OutQueueItem.Message.Length > 0) //for some reason we are getting an empty line?
// {
// OutCsv = OutQueueItem.Message;
// //write header
// await writer.WriteLineAsync(OutCsv);
// }
// }
//}
while (this.WriteQueue.TryDequeue(out OutQueueItem))
{
//update time in queue
OutQueueItem.QueueWaitMS = (DateTime.UtcNow - OutQueueItem.TimeUTC).TotalMilliseconds;
//Convert the class to tab delimited format:
string OutCsv = null;
//if (this.m_ProcessAsCSV)
//{
// OutCsv = ClsToCSV(OutQueueItem, false);
//}
//else
//{
OutCsv = OutQueueItem.Message;
//}
await writer.WriteLineAsync(OutCsv);
}
await writer.FlushAsync();
Ret = true;
}
catch (Exception exception1)
{
Console.WriteLine("Error writing to streamwriter: " + exception1.Message);
}
try
{
if (position > 0 && this.m_WasLocked)
{
this.m_fileStream.Unlock(position, 0xFFFF);
}
}
catch (IOException exception)
{
Ret = false;
Console.WriteLine("Error unlocking file: " + exception.Message); //already unlocked
}
writer.Close();
}
else
{
Ret = false;
}
}
catch (Exception ex)
{
Console.WriteLine("Error: " + ex.Message);
}
finally
{
if (Ret)
{
this.LastLogWriteTime = DateTime.Now;
}
}
return(Ret);
}
public async Task AppendAllLinesAsync(IEnumerable <string> lines, CancellationToken cancellationToken = default)
{
if (lines is null || !lines.Any())
{
return;
}
IoHelpers.EnsureContainingDirectoryExists(NewFilePath);
if (File.Exists(NewFilePath))
{
File.Delete(NewFilePath);
}
var byteArrayBuilder = new ByteArrayBuilder();
var linesArray = lines.ToArray();
var linesIndex = 0;
using (var sr = OpenText())
using (var fs = File.OpenWrite(NewFilePath))
using (var sw = new StreamWriter(fs, Encoding.ASCII, Constants.BigFileReadWriteBufferSize))
{
// 1. First copy.
if (!sr.EndOfStream)
{
var lineTask = sr.ReadLineAsync();
Task wTask = Task.CompletedTask;
string line = null;
while (lineTask != null)
{
if (line is null)
{
line = await lineTask;
}
lineTask = sr.EndOfStream ? null : sr.ReadLineAsync();
if (linesArray[linesIndex] == line) // If the line is a line we want to write, then we know that someone else have worked into the file.
{
linesIndex++;
continue;
}
await wTask;
wTask = sw.WriteLineAsync(line);
ContinueBuildHash(byteArrayBuilder, line);
cancellationToken.ThrowIfCancellationRequested();
line = null;
}
await wTask;
}
await sw.FlushAsync();
// 2. Then append.
foreach (var line in linesArray)
{
await sw.WriteLineAsync(line);
ContinueBuildHash(byteArrayBuilder, line);
cancellationToken.ThrowIfCancellationRequested();
}
await sw.FlushAsync();
}
var res = await WorkWithHashAsync(byteArrayBuilder, cancellationToken);
if (res.same)
{
return;
}
SafeMoveNewToOriginal();
await WriteOutHashAsync(res.hash);
}
private async Task CopyModelAsCsvToStreamAsync(NameValueCollection requestParameters, Stream responseStream, Func <bool> isCancelled, Func <Task> flushResponseAsync = null)
{
SecurityProviderCache.ValidateCurrentProvider();
string modelName = requestParameters["ModelName"];
string hubName = requestParameters["HubName"];
string filterText = requestParameters["FilterText"];
string sortField = requestParameters["SortField"];
bool sortAscending = requestParameters["SortAscending"].ParseBoolean();
bool showDeleted = requestParameters["ShowDeleted"].ParseBoolean();
string[] parentKeys = requestParameters["ParentKeys"].Split(',');
const int PageSize = 250;
if (string.IsNullOrEmpty(modelName))
{
throw new ArgumentNullException(nameof(modelName), "Cannot download CSV data: no model type name was specified.");
}
if (string.IsNullOrEmpty(hubName))
{
throw new ArgumentNullException(nameof(hubName), "Cannot download CSV data: no hub type name was specified.");
}
Type modelType = AssemblyInfo.FindType(modelName);
if ((object)modelType == null)
{
throw new InvalidOperationException($"Cannot download CSV data: failed to find model type \"{modelName}\" in loaded assemblies.");
}
Type hubType = AssemblyInfo.FindType(hubName);
if ((object)hubType == null)
{
throw new InvalidOperationException($"Cannot download CSV data: failed to find hub type \"{hubName}\" in loaded assemblies.");
}
IRecordOperationsHub hub;
// Record operation tuple defines method name and allowed roles
Tuple <string, string> queryRecordCountOperation;
Tuple <string, string> queryRecordsOperation;
string queryRoles;
try
{
hub = Activator.CreateInstance(hubType) as IRecordOperationsHub;
if ((object)hub == null)
{
throw new SecurityException($"Cannot download CSV data: hub type \"{hubName}\" is not a IRecordOperationsHub, access cannot be validated.");
}
Tuple <string, string>[] recordOperations;
try
{
// Get any authorized query roles as defined in hub records operations for modeled table, default to read allowed for query
recordOperations = hub.RecordOperationsCache.GetRecordOperations(modelType);
if ((object)recordOperations == null)
{
throw new NullReferenceException();
}
}
catch (KeyNotFoundException ex)
{
throw new SecurityException($"Cannot download CSV data: hub type \"{hubName}\" does not define record operations for \"{modelName}\", access cannot be validated.", ex);
}
// Get record operation for querying record count
queryRecordCountOperation = recordOperations[(int)RecordOperation.QueryRecordCount];
if ((object)queryRecordCountOperation == null)
{
throw new NullReferenceException();
}
// Get record operation for querying records
queryRecordsOperation = recordOperations[(int)RecordOperation.QueryRecords];
if ((object)queryRecordsOperation == null)
{
throw new NullReferenceException();
}
// Get any defined role restrictions for record query operation - access to CSV download will based on these roles
queryRoles = string.IsNullOrEmpty(queryRecordsOperation.Item1) ? "*" : queryRecordsOperation.Item2 ?? "*";
}
catch (Exception ex)
{
throw new SecurityException($"Cannot download CSV data: failed to instantiate hub type \"{hubName}\" or access record operations, access cannot be validated.", ex);
}
using (DataContext dataContext = new DataContext())
using (StreamWriter writer = new StreamWriter(responseStream))
{
// Validate current user has access to requested data
if (!dataContext.UserIsInRole(queryRoles))
{
throw new SecurityException($"Cannot download CSV data: access is denied for user \"{Thread.CurrentPrincipal.Identity?.Name ?? "Undefined"}\", minimum required roles = {queryRoles.ToDelimitedString(", ")}.");
}
AdoDataConnection connection = dataContext.Connection;
ITableOperations table = dataContext.Table(modelType);
string[] fieldNames = table.GetFieldNames(false);
Func <Task> flushAsync = async() =>
{
// ReSharper disable once AccessToDisposedClosure
await writer.FlushAsync();
if ((object)flushResponseAsync != null)
{
await flushResponseAsync();
}
};
// Write column headers
await writer.WriteLineAsync(string.Join(",", fieldNames.Select(fieldName => connection.EscapeIdentifier(fieldName, true))));
await flushAsync();
// See if modeled table has a flag field that represents a deleted row
bool hasDeletedField = !string.IsNullOrEmpty(dataContext.GetIsDeletedFlag(modelType));
// Get query operation methods
MethodInfo queryRecordCount = hubType.GetMethod(queryRecordCountOperation.Item1);
MethodInfo queryRecords = hubType.GetMethod(queryRecordsOperation.Item1);
// Setup query parameters
List <object> queryRecordCountParameters = new List <object>();
List <object> queryRecordsParameters = new List <object>();
// Add current show deleted state parameter, if model defines a show deleted field
if (hasDeletedField)
{
queryRecordCountParameters.Add(showDeleted);
}
// Add any parent key restriction parameters
if (parentKeys.Length > 0 && parentKeys[0].Length > 0)
{
queryRecordCountParameters.AddRange(parentKeys);
}
// Add parameters for query records from query record count parameters - they match up to this point
queryRecordsParameters.AddRange(queryRecordCountParameters);
// Add sort field parameter
queryRecordsParameters.Add(sortField);
// Add ascending sort order parameter
queryRecordsParameters.Add(sortAscending);
// Track parameter index for current page to query
int pageParameterIndex = queryRecordsParameters.Count;
// Add page index parameter
queryRecordsParameters.Add(0);
// Add page size parameter
queryRecordsParameters.Add(PageSize);
// Add filter text parameter
queryRecordCountParameters.Add(filterText);
queryRecordsParameters.Add(filterText);
// Read queried records in page sets so there is not a memory burden and long initial query delay on very large data sets
int recordCount = (int)queryRecordCount.Invoke(hub, queryRecordCountParameters.ToArray());
int totalPages = Math.Max((int)Math.Ceiling(recordCount / (double)PageSize), 1);
// Write data pages
for (int page = 0; page < totalPages && !isCancelled(); page++)
{
// Update desired page to query
queryRecordsParameters[pageParameterIndex] = page + 1;
// Query page records
IEnumerable records = queryRecords.Invoke(hub, queryRecordsParameters.ToArray()) as IEnumerable ?? Enumerable.Empty <object>();
int exportCount = 0;
// Export page records
foreach (object record in records)
{
// Periodically check for client cancellation
if (exportCount++ % (PageSize / 4) == 0 && isCancelled())
{
break;
}
await writer.WriteLineAsync(string.Join(",", fieldNames.Select(fieldName => $"\"{table.GetFieldValue(record, fieldName)}\"")));
}
await flushAsync();
}
}
}
/// <summary>
/// Creates a TCP connection to server
/// </summary>
/// <param name="bufferSize"></param>
/// <param name="connectionTimeOutSeconds"></param>
/// <param name="remoteHostName"></param>
/// <param name="httpCmd"></param>
/// <param name="httpVersion"></param>
/// <param name="isHttps"></param>
/// <param name="remotePort"></param>
/// <param name="supportedSslProtocols"></param>
/// <param name="remoteCertificateValidationCallback"></param>
/// <param name="localCertificateSelectionCallback"></param>
/// <param name="externalHttpProxy"></param>
/// <param name="externalHttpsProxy"></param>
/// <param name="clientStream"></param>
/// <returns></returns>
internal async Task <TcpConnection> CreateClient(int bufferSize, int connectionTimeOutSeconds,
string remoteHostName, int remotePort, Version httpVersion,
bool isHttps, SslProtocols supportedSslProtocols,
RemoteCertificateValidationCallback remoteCertificateValidationCallback, LocalCertificateSelectionCallback localCertificateSelectionCallback,
ExternalProxy externalHttpProxy, ExternalProxy externalHttpsProxy,
Stream clientStream)
{
TcpClient client;
Stream stream;
if (isHttps)
{
SslStream sslStream = null;
//If this proxy uses another external proxy then create a tunnel request for HTTPS connections
if (externalHttpsProxy != null && externalHttpsProxy.HostName != remoteHostName)
{
client = new TcpClient(externalHttpsProxy.HostName, externalHttpsProxy.Port);
stream = client.GetStream();
using (var writer = new StreamWriter(stream, Encoding.ASCII, bufferSize, true))
{
await writer.WriteLineAsync($"CONNECT {remoteHostName}:{remotePort} HTTP/{httpVersion}");
await writer.WriteLineAsync($"Host: {remoteHostName}:{remotePort}");
await writer.WriteLineAsync("Connection: Keep-Alive");
if (!string.IsNullOrEmpty(externalHttpsProxy.UserName) && externalHttpsProxy.Password != null)
{
await writer.WriteLineAsync("Proxy-Connection: keep-alive");
await writer.WriteLineAsync("Proxy-Authorization" + ": Basic " + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(externalHttpsProxy.UserName + ":" + externalHttpsProxy.Password)));
}
await writer.WriteLineAsync();
await writer.FlushAsync();
writer.Close();
}
using (var reader = new CustomBinaryReader(stream))
{
var result = await reader.ReadLineAsync();
if (!new string[] { "200 OK", "connection established" }.Any(s => result.ToLower().Contains(s.ToLower())))
{
throw new Exception("Upstream proxy failed to create a secure tunnel");
}
await reader.ReadAllLinesAsync();
}
}
else
{
client = new TcpClient(remoteHostName, remotePort);
stream = client.GetStream();
}
try
{
sslStream = new SslStream(stream, true, remoteCertificateValidationCallback,
localCertificateSelectionCallback);
await sslStream.AuthenticateAsClientAsync(remoteHostName, null, supportedSslProtocols, false);
stream = sslStream;
}
catch
{
sslStream?.Dispose();
throw;
}
}
else
{
if (externalHttpProxy != null && externalHttpProxy.HostName != remoteHostName)
{
client = new TcpClient(externalHttpProxy.HostName, externalHttpProxy.Port);
stream = client.GetStream();
}
else
{
client = new TcpClient(remoteHostName, remotePort);
stream = client.GetStream();
}
}
client.ReceiveTimeout = connectionTimeOutSeconds * 1000;
client.SendTimeout = connectionTimeOutSeconds * 1000;
stream.ReadTimeout = connectionTimeOutSeconds * 1000;
stream.WriteTimeout = connectionTimeOutSeconds * 1000;
return(new TcpConnection()
{
UpStreamHttpProxy = externalHttpProxy,
UpStreamHttpsProxy = externalHttpsProxy,
HostName = remoteHostName,
Port = remotePort,
IsHttps = isHttps,
TcpClient = client,
StreamReader = new CustomBinaryReader(stream),
Stream = stream,
Version = httpVersion
});
}
/// <summary>
/// Gets the labels from the extensions of the provided raw service manifest.
/// </summary>
private async Task <IDictionary <string, string> > ExtractLabelsAsync(
string rawServiceManifest,
string targetServiceTypeName,
CancellationToken cancellationToken)
{
using (var reader = XmlReader.Create(new StringReader(rawServiceManifest),
CreateSafeXmlSetting(1024 * 1024, 1024)))
{
XDocument parsedManifest;
try
{
parsedManifest = await XDocument.LoadAsync(reader, LoadOptions.None, cancellationToken);
}
catch (System.Xml.XmlException ex)
{
// TODO: we don't know if the service wants to use the gateway yet, so not sure if this classifies as config error (considering it will escalate into a bad health report)
throw new ConfigException("Failed to parse service manifest XML.", ex);
}
var elements = parsedManifest
.Elements(XNSServiceManifest + "ServiceManifest")
.Elements(XNSServiceManifest + "ServiceTypes")
.Elements().Where(s => (string)s.Attribute("ServiceTypeName") == targetServiceTypeName)
.Elements(XNSServiceManifest + "Extensions")
.Elements(XNSServiceManifest + "Extension").Where(s => (string)s.Attribute("Name") == ConfigurationValues.ExtensionName)
.Elements(XNSFabricNoSchema + "Service");
if (!elements.Any())
{
return(new Dictionary <string, string>());
}
await using (var stream = new MemoryStream())
{
await using (var sw = new StreamWriter(stream))
{
using (var writer = new XmlNoNamespaceWriter(sw, new XmlWriterSettings {
CloseOutput = false
}))
{
foreach (var element in elements)
{
element.Save(writer);
}
writer.Flush();
await sw.FlushAsync();
var labels = XmlStreamToDictionaryParser.Parse(stream, (options) =>
{
options.KeyDelimiter = ".";
options.Parents = new List <string> {
"Service"
};
options.IsIndexAttribute = (attribute, stack) =>
{
switch (stack.FirstOrDefault())
{
case "Endpoint": return(string.Equals(attribute, "Id", StringComparison.OrdinalIgnoreCase));
case "Route": return(string.Equals(attribute, "Id", StringComparison.OrdinalIgnoreCase));
}
return(false);
};
});
return(labels);
}
}
}
}
}
public async Task DnsCertBridge()
{
AssertOnlyInSetupMode();
var action = GetQueryStringValueAndAssertIfSingleAndNotEmpty("action"); // Action can be: claim | user-domains | check-availability
using (var reader = new StreamReader(RequestBodyStream()))
{
var payload = await reader.ReadToEndAsync();
var content = new StringContent(payload, Encoding.UTF8, "application/json");
try
{
string error = null;
object result = null;
string responseString = null;
string errorMessage = null;
try
{
var response = await ApiHttpClient.Instance.PostAsync("/api/v1/dns-n-cert/" + action, content).ConfigureAwait(false);
HttpContext.Response.StatusCode = (int)response.StatusCode;
responseString = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
if ((int)response.StatusCode >= 500 && (int)response.StatusCode <= 599)
{
error = responseString;
errorMessage = GeneralDomainRegistrationError;
}
else
{
result = JsonConvert.DeserializeObject <JObject>(responseString);
if (result != null)
{
if (((JObject)result).TryGetValue(nameof(ExceptionDispatcher.ExceptionSchema.Error), out var err))
{
error = err.ToString();
}
if (((JObject)result).TryGetValue(nameof(ExceptionDispatcher.ExceptionSchema.Message), out var msg))
{
errorMessage = msg.ToString();
}
}
}
}
catch (Exception e)
{
result = responseString;
HttpContext.Response.StatusCode = (int)HttpStatusCode.InternalServerError;
error = e.ToString();
errorMessage = DomainRegistrationServiceUnreachableError;
}
await using (var streamWriter = new StreamWriter(ResponseBodyStream()))
{
if (error != null)
{
new JsonSerializer().Serialize(streamWriter, new
{
Message = errorMessage,
Response = result,
Error = error,
Type = typeof(RavenException).FullName
});
await streamWriter.FlushAsync();
}
else
{
await streamWriter.WriteAsync(responseString);
}
await streamWriter.FlushAsync();
}
}
catch (Exception e)
{
throw new InvalidOperationException(GeneralDomainRegistrationError, e);
}
}
}
/// <summary>
/// Flush the write buffers to the stream.
/// </summary>
/// <returns>A task that represents the asynchronous flush operation.</returns>
public Task FlushAsync()
{
return(_writer.FlushAsync());
}
public async Task UserDomains()
{
AssertOnlyInSetupMode();
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "license activation");
var licenseInfo = JsonDeserializationServer.LicenseInfo(json);
var content = new StringContent(JsonConvert.SerializeObject(licenseInfo), Encoding.UTF8, "application/json");
try
{
string error = null;
object result = null;
string responseString = null;
string errorMessage = null;
try
{
var response = await ApiHttpClient.Instance.PostAsync("/api/v1/dns-n-cert/user-domains", content).ConfigureAwait(false);
HttpContext.Response.StatusCode = (int)response.StatusCode;
responseString = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
if (response.IsSuccessStatusCode == false)
{
error = responseString;
errorMessage = GeneralDomainRegistrationError;
}
else
{
result = JsonConvert.DeserializeObject <JObject>(responseString);
}
}
catch (Exception e)
{
result = responseString;
HttpContext.Response.StatusCode = (int)HttpStatusCode.InternalServerError;
error = e.ToString();
errorMessage = DomainRegistrationServiceUnreachableError;
}
if (error != null)
{
JToken errorJToken = null;
if (responseString != null)
{
JsonConvert.DeserializeObject <JObject>(responseString).TryGetValue("Error", out errorJToken);
}
await using (var streamWriter = new StreamWriter(ResponseBodyStream()))
{
new JsonSerializer().Serialize(streamWriter, new
{
Message = errorMessage,
Response = result,
Error = errorJToken ?? error
});
await streamWriter.FlushAsync();
}
return;
}
var results = JsonConvert.DeserializeObject <UserDomainsResult>(responseString);
var fullResult = new UserDomainsAndLicenseInfo
{
UserDomainsWithIps = new UserDomainsWithIps
{
Emails = results.Emails,
RootDomains = results.RootDomains,
Domains = new Dictionary <string, List <SubDomainAndIps> >()
}
};
foreach (var domain in results.Domains)
{
var list = new List <SubDomainAndIps>();
foreach (var subDomain in domain.Value)
{
try
{
list.Add(new SubDomainAndIps
{
SubDomain = subDomain,
// The ip list will be populated on the next call (/setup/populate-ips), when we know which root domain the user selected
});
}
catch (Exception)
{
continue;
}
}
fullResult.UserDomainsWithIps.Domains.Add(domain.Key, list);
}
var licenseStatus = await SetupManager
.GetUpdatedLicenseStatus(ServerStore, licenseInfo.License)
.ConfigureAwait(false);
fullResult.MaxClusterSize = licenseStatus.MaxClusterSize;
fullResult.LicenseType = licenseStatus.Type;
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
var blittable = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.ToBlittable(fullResult, context);
context.Write(writer, blittable);
}
}
catch (LicenseExpiredException)
{
throw;
}
catch (Exception e)
{
throw new InvalidOperationException(GeneralDomainRegistrationError, e);
}
}
}
public async Task RunAsyncMethod(bool isValid,
bool isValidDevice,
string region,
string verificationPayload,
string platform,
bool isChaffRequest,
HttpStatusCode expectedStatusCode
)
{
// preparation
var config = new Mock <IConfiguration>();
config.Setup(_ => _["SupportRegions"]).Returns("Region1,Region2");
var diagnosisRepo = new Mock <IDiagnosisRepository>();
diagnosisRepo.Setup(_ => _.SubmitDiagnosisAsync(It.IsAny <string>(),
It.IsAny <DateTimeOffset>(),
It.IsAny <string>(),
It.IsAny <TemporaryExposureKeyModel[]>()))
.ReturnsAsync(new DiagnosisModel());
var tekRepo = new Mock <ITemporaryExposureKeyRepository>();
var validationServer = new Mock <IValidationServerService>();
validationServer.Setup(_ => _.Validate(It.IsAny <HttpRequest>())).Returns(IValidationServerService.ValidateResult.Success);
var deviceCheck = new Mock <IDeviceValidationService>();
deviceCheck.Setup(_ => _.Validation(It.IsAny <string>(), It.IsAny <V3DiagnosisSubmissionParameter>(), It.IsAny <DateTimeOffset>())).ReturnsAsync(isValidDevice);
var verification = new Mock <IVerificationService>();
var temporaryExposureKeyValidationService = new Mock <ITemporaryExposureKeyValidationService>();
temporaryExposureKeyValidationService.Setup(x => x.Validate(It.IsAny <bool>(), It.IsAny <V3DiagnosisSubmissionParameter.Key>())).Returns(true);
var logger = new Mock.LoggerMock <V3DiagnosisApi>();
var diagnosisApi = new V3DiagnosisApi(config.Object,
tekRepo.Object,
deviceCheck.Object,
verification.Object,
validationServer.Object,
temporaryExposureKeyValidationService.Object,
logger);
var context = new Mock <HttpContext>();
var keydata = new byte[KEY_LENGTH];
RandomNumberGenerator.Create().GetBytes(keydata);
var keyDataString = Convert.ToBase64String(keydata);
var dateTime = DateTime.UtcNow.AddDays(-7).Date;
var bodyJson = new V3DiagnosisSubmissionParameter()
{
HasSymptom = true,
OnsetOfSymptomOrTestDate = dateTime.ToString(Constants.FORMAT_TIMESTAMP),
VerificationPayload = verificationPayload,
Regions = new[] { region },
Platform = platform,
DeviceVerificationPayload = "DeviceVerificationPayload",
AppPackageName = "Covid19Radar",
Keys = new V3DiagnosisSubmissionParameter.Key[] {
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-8).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-7).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-6).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-5).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-4).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-3).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-2).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(-1).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(0).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(1).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(2).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(3).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(4).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(5).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(6).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(7).ToRollingStartNumber(), ReportType = 7
},
new V3DiagnosisSubmissionParameter.Key()
{
KeyData = keyDataString, RollingPeriod = 144, RollingStartNumber = dateTime.AddDays(8).ToRollingStartNumber(), ReportType = 7
},
}
};
var bodyString = JsonConvert.SerializeObject(bodyJson);
using var stream = new MemoryStream();
using (var writer = new StreamWriter(stream, leaveOpen: true))
{
await writer.WriteAsync(bodyString);
await writer.FlushAsync();
}
stream.Seek(0, SeekOrigin.Begin);
context.Setup(_ => _.Request.Body).Returns(stream);
if (isChaffRequest)
{
IHeaderDictionary headers = new HeaderDictionary()
{
{ "X-Chaff", "Foo" /* Server will check X-Chaff header existence, content no matter. */ }
};
context.Setup(_ => _.Request.Headers).Returns(headers);
}
// action
var result = await diagnosisApi.RunAsync(context.Object.Request);
// assert
if (result is OkObjectResult okObjectResult)
{
Assert.AreEqual(((int)expectedStatusCode), okObjectResult.StatusCode);
}
else if (result is BadRequestErrorMessageResult)
{
Assert.AreEqual(expectedStatusCode, HttpStatusCode.BadRequest);
}
}
public async Task <ActionResult> DumpSuiteJobs(
[FromRoute] FlowSnake suiteId,
[FromServices] RurikawaDb db)
{
var suite = await dbService.GetTestSuite(suiteId);
if (suite == null)
{
return(NotFound());
}
var columns = suite.TestGroups
.SelectMany(group => group.Value.Select(value => value.Name))
.ToList();
var ptr = db.Jobs.FromSqlInterpolated($@"
select
distinct on (account)
*
from jobs
where test_suite = {suiteId.Num}
order by account, id desc
").AsAsyncEnumerable();
Response.StatusCode = 200;
Response.ContentType = "application/csv";
const int flushInterval = 100;
// write to body of response
var sw = new StreamWriter(Response.Body);
var csvWriter = new CsvWriter(sw);
csvWriter.QuoteAllFields = true;
csvWriter.WriteField("id");
csvWriter.WriteField("account");
csvWriter.WriteField("repo");
csvWriter.WriteField("revision");
csvWriter.WriteField("stage");
csvWriter.WriteField("result_kind");
foreach (var col in columns)
{
csvWriter.WriteField(col);
}
csvWriter.NextRecord();
int counter = 0;
await foreach (var val in ptr)
{
if (counter % flushInterval == 0)
{
await sw.FlushAsync();
}
WriteJobInfo(csvWriter, val, columns);
counter++;
}
await sw.FlushAsync();
return(new EmptyResult());
}
public override async Task Run(string[] args)
{
if (args.Length < 3)
{
Console.WriteLine("Usage: PortTunneler client [server] [protocol] [ip] [port]");
return;
}
var server = IPEndPoint.Parse(args[0]);
if (server.Port == 0)
{
server.Port = 2020;
}
if (!Protocol.TryParse(args[1], out var protocol))
{
Console.WriteLine("Invalid protocol was used.");
return;
}
if (!IPEndPoint.TryParse(args[2], out var ip))
{
Console.WriteLine("Invalid ip endpoint for the local server.");
return;
}
int port;
if (args.Length == 3)
{
port = ip.Port;
if (port == 0)
{
Console.WriteLine("A port needs to be specified in the IP or as an argument.");
return;
}
}
else
{
if (!int.TryParse(args[3], out port))
{
Console.WriteLine("Invalid port specified.");
return;
}
}
if (port < IPEndPoint.MinPort || port > IPEndPoint.MaxPort)
{
throw new ArgumentOutOfRangeException(nameof(port), port,
"Port is too " + (port < IPEndPoint.MinPort ? "small" : "big"));
}
try
{
var connectionClient = new TcpClient();
await connectionClient.ConnectAsync(server.Address, server.Port);
var stream = connectionClient.GetStream();
var writer = new StreamWriter(stream);
await writer.WriteLineAsync($"{port}@{protocol}");
await writer.FlushAsync();
var bytes = new byte[2];
await stream.ReadAsync(bytes, 0, 2);
var response = BitConverter.ToChar(bytes, 0);
switch (response)
{
case 'W':
Console.WriteLine(
"Warning: You are using a port which had been used before, if it was not you who previously used the port, please stop using it and refrain from doing so in the future.");
break;
case 'E':
throw new ArgumentException("Port is already in use.");
default:
{
Console.WriteLine("Connected.");
break;
}
}
if (protocol != null)
{
var client = protocol.CreateClient(protocol, ip);
client.Connection = connectionClient;
Console.WriteLine("Click CTRL+C to stop connection.");
await client.Listen();
Console.WriteLine("Connection ended, press any key to continue...");
Console.ReadKey();
}
}
catch
{
NetworkUtils.ProgramActive = false;
await Console.Error.WriteLineAsync("Caught an exception, connection ended.");
throw;
}
}
public override async Task <bool> Execute(DescribeInput input)
{
input.HostBuilder.ConfigureServices(x => x.AddTransient <IDescribedSystemPart, ConfigurationPreview>());
using (var host = input.BuildHost())
{
var config = host.Services.GetRequiredService <IConfiguration>();
var hosting = host.Services.GetService <IHostEnvironment>();
var about = new AboutThisAppPart(hosting, config);
var factories = host.Services.GetServices <IDescribedSystemPartFactory>();
var parts = host.Services.GetServices <IDescribedSystemPart>()
.Concat(factories.SelectMany(x => x.Parts()))
.Concat(new IDescribedSystemPart[] { about, new ReferencedAssemblies() }).ToArray();
foreach (var partWithServices in parts.OfType <IRequiresServices>())
{
partWithServices.Resolve(host.Services);
}
if (input.ListFlag)
{
Console.WriteLine("The registered system parts are");
foreach (var part in parts)
{
Console.WriteLine("* " + part.Title);
}
return(true);
}
if (input.TitleFlag.IsNotEmpty())
{
parts = parts.Where(x => x.Title == input.TitleFlag).ToArray();
}
else if (input.InteractiveFlag)
{
var prompt = new MultiSelectionPrompt <string>()
.Title("What part(s) of your application do you wish to view?")
.PageSize(10)
.AddChoices(parts.Select(x => x.Title));
var titles = AnsiConsole.Prompt(prompt);
parts = parts.Where(x => titles.Contains(x.Title)).ToArray();
}
if (!input.SilentFlag)
{
await WriteToConsole(parts);
}
if (input.FileFlag.IsNotEmpty())
{
using (var stream = new FileStream(input.FileFlag, FileMode.CreateNew, FileAccess.Write))
{
var writer = new StreamWriter(stream);
await WriteText(parts, writer);
await writer.FlushAsync();
}
Console.WriteLine("Wrote system description to file " + input.FileFlag);
}
return(true);
}
}
public async ValueTask WriteHeaderAsync <T>(T header, StreamWriter target)
{
await Task.Run(() => { serializer.Serialize(target, header); target.WriteLine(); target.WriteLine(BLOK_SEPARATOR); });
await target.FlushAsync();
}
private async Task SendLineAsync(string line)
{
await _writer.WriteLineAsync(line);
await _writer.FlushAsync();
}
public async Task TestInProcSessionContainer()
{
using (var server = CreateSocketServerBuilder <StringPackageInfo, MyPipelineFilter>()
.UseCommand <string, StringPackageInfo>(commandOptions =>
{
commandOptions.AddCommand <SESS>();
})
.UseInProcSessionContainer()
.BuildAsServer())
{
Assert.Equal("TestServer", server.Name);
var sessionContainer = server.ServiceProvider.GetSessionContainer();
Assert.NotNull(sessionContainer);
Assert.True(await server.StartAsync());
OutputHelper.WriteLine("Server started.");
var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await client.ConnectAsync(new IPEndPoint(IPAddress.Parse("127.0.0.1"), 4040));
OutputHelper.WriteLine("Connected.");
Thread.Sleep(1000);
Assert.Equal(1, sessionContainer.GetSessionCount());
var sessionID = string.Empty;
var closed = false;
using (var stream = new NetworkStream(client))
using (var streamReader = new StreamReader(stream, Utf8Encoding, true))
using (var streamWriter = new StreamWriter(stream, Utf8Encoding, 1024 * 1024 * 4))
{
await streamWriter.WriteAsync("SESS\r\n");
await streamWriter.FlushAsync();
sessionID = await streamReader.ReadLineAsync();
Assert.False(string.IsNullOrEmpty(sessionID));
var session = sessionContainer.GetSessionByID(sessionID);
Assert.NotNull(session);
Assert.Equal(sessionID, session.SessionID);
session.Closed += (s, e) =>
{
closed = true;
return(new ValueTask());
};
await session.Channel.CloseAsync();
}
await Task.Delay(1000);
Assert.Equal(0, sessionContainer.GetSessionCount());
Assert.Null(sessionContainer.GetSessionByID(sessionID));
Assert.True(closed);
await server.StopAsync();
}
}
public override async Task WriteToStreamAsync(Type type, object value, Stream stream, HttpContent content, TransportContext transportContext, CancellationToken cancellationToken)
{
// await Console.Out.WriteLineAsync("HB Started: " + Request.RequestUri.PathAndQuery);
var json = _formatter.GetContext(_request, value);
string r = _template.Render(_view, json);
StringBuilder html;
var donuts = new List <string>();
if (_request.Properties.ContainsKey("donut") &&
(bool)_request.Properties["donut"] == true)
{
// detect any master or sections and fill them
html = new StringBuilder(r);
}
else
{
html = FillSectionData(r, json);
// detect the donuts
int index = html.IndexOf("####donut:", 0, false);
int length = 4;
while (index != -1)
{
length = html.IndexOf("####", index + 10, false) - index - 10;
donuts.Add(html.ToString(index + 10, length));
if (index + length > html.Length)
{
break;
}
index = html.IndexOf("####donut:", index + length, false);
}
}
// execute any donuts
var sync = new object();
var donutContent = new Dictionary <string, string>();
if (donuts.Count > 0)
{
foreach (var donut in donuts)
{
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, "http://" + _request.RequestUri.DnsSafeHost + (_request.RequestUri.IsDefaultPort ? "" : ":" + _request.RequestUri.Port) + "/" + donut))
{
request.Properties.Add("donut", true);
// ensure any AB testing in donut actions uses the same grouping
if (_request.Properties.ContainsKey("experiment"))
{
request.Properties.Add("experiment", _request.Properties["experiment"]);
}
// so we can use the same identify and context information in higher up
// donut functions.
if (_request.Properties.ContainsKey("MS_OwinContext"))
{
request.Properties.Add("MS_OwinContext", _request.Properties["MS_OwinContext"]);
}
// temp: try catch to debug "bad headers"
foreach (var header in _request.Headers)
{
try
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
catch (Exception e)
{
Console.Out.WriteLineAsync("Handlebars - Add Header: " + e.Message);
}
}
// this was previously causing a deadlock, never use .Result!!!! it is the
// root of all things evil.
using (HttpResponseMessage response = await _client.SendAsync(request, CancellationToken.None))
{
if (response.IsSuccessStatusCode)
{
try
{
var donutHtml = await response.Content.ReadAsStringAsync();
lock (sync)
donutContent.Add(donut, donutHtml);
}
catch (Exception exp)
{
lock (sync)
donutContent.Add(donut, exp.Message);
}
}
}
}
}
// wait for the donut requests
// Console.Out.WriteLineAsync("HandlebarsMediaTypeFormatter: Donuts() " + sw.ElapsedMilliseconds + "ms");
}
// StreamWriter writer = new StreamWriter(writeStream);
foreach (var donut in donutContent)
{
html.Replace("####donut:" + donut.Key + "####", donut.Value);
}
// Console.Out.WriteLineAsync("HandlebarsMediaTypeFormatter: Replace() " + sw.ElapsedMilliseconds + "ms");
// await writer.WriteAsync(html.ToString());
var writer = new StreamWriter(stream);
string output, contentType;
if (_request.Properties.ContainsKey("hb-as-javascript"))
{
var lines = html.ToString()
.Split(new[] { "\n\r", "\n", "\r" }, StringSplitOptions.RemoveEmptyEntries);
var sb = new StringBuilder();
foreach (var line in lines)
{
sb.Append("document.write('" +
HandlebarsUtilities.ToJavaScriptString(line)
+ "');");
}
output = sb.ToString();
contentType = "application/javascript";
}
else
{
contentType = "text/html";
output = html.ToString();
}
content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
await writer.WriteAsync(output);
await writer.FlushAsync();
return;
}
public async Task TestMultipleServerHost()
{
var serverName1 = "TestServer1";
var serverName2 = "TestServer2";
var hostBuilder = MultipleServerHostBuilder.Create()
.ConfigureAppConfiguration((hostingContext, config) =>
{
config.Sources.Clear();
config.AddJsonFile("Config/multiple_server.json", optional: false, reloadOnChange: true);
})
.AddServer <MySocketService, StringPackageInfo, CommandLinePipelineFilter>(builder =>
{
builder
.ConfigureServerOptions((ctx, config) =>
{
return(config.GetSection(serverName1));
}).UseSessionHandler(async(s) =>
{
await s.SendAsync(Utf8Encoding.GetBytes($"{s.Server.Name}\r\n"));
})
.UseCommand(commandOptions =>
{
// register all commands in one assembly
commandOptions.AddCommandAssembly(typeof(MIN).GetTypeInfo().Assembly);
});
})
.AddWebSocketServer <MyWebSocketService>(builder =>
{
builder
.ConfigureServerOptions((ctx, config) =>
{
return(config.GetSection(serverName2));
})
.UseCommand <StringPackageInfo, StringPackageConverter>(commandOptions =>
{
commandOptions.AddCommand <ADD>();
commandOptions.AddCommand <MULT>();
commandOptions.AddCommand <SUB>();
});
})
.ConfigureLogging((hostCtx, loggingBuilder) =>
{
loggingBuilder.AddConsole();
loggingBuilder.AddDebug();
});
using (var host = hostBuilder.Build())
{
await host.StartAsync();
var client = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await client.ConnectAsync(new IPEndPoint(IPAddress.Loopback, DefaultServerPort));
using (var stream = new NetworkStream(client))
using (var streamReader = new StreamReader(stream, Utf8Encoding, true))
using (var streamWriter = new StreamWriter(stream, Utf8Encoding, 1024 * 1024 * 4))
{
var line = await streamReader.ReadLineAsync();
Assert.Equal(serverName1, line);
await streamWriter.WriteAsync("MIN 8 6 3\r\n");
await streamWriter.FlushAsync();
line = await streamReader.ReadLineAsync();
Assert.Equal("3", line);
await streamWriter.WriteAsync("SORT 8 6 3\r\n");
await streamWriter.FlushAsync();
line = await streamReader.ReadLineAsync();
Assert.Equal("SORT 3 6 8", line);
}
var websocket = new ClientWebSocket();
await websocket.ConnectAsync(new Uri($"ws://localhost:{AlternativeServerPort}"), CancellationToken.None);
Assert.Equal(WebSocketState.Open, websocket.State);
var receiveBuffer = new byte[256];
Assert.Equal("11", await GetWebSocketReply(websocket, receiveBuffer, "ADD 5 6"));
Assert.Equal("8", await GetWebSocketReply(websocket, receiveBuffer, "SUB 10 2"));
Assert.Equal("21", await GetWebSocketReply(websocket, receiveBuffer, "MULT 3 7"));
await websocket.CloseAsync(WebSocketCloseStatus.NormalClosure, string.Empty, CancellationToken.None);
Assert.Equal(WebSocketState.Closed, websocket.State);
await host.StopAsync();
}
}
public async Task Search(CommandContext ctx, [RemainingText, Description("Product ID, module, or function name. **Case sensitive**")] string search)
{
if (string.IsNullOrEmpty(search))
{
await ctx.ReactWithAsync(Config.Reactions.Failure, "No meaningful search query provided").ConfigureAwait(false);
return;
}
var productCodes = ProductCodeLookup.GetProductIds(search);
if (productCodes.Any())
{
await ReturnSyscallsByGameAsync(ctx, productCodes.First()).ConfigureAwait(false);
return;
}
if (ctx.User.Id == 216724245957312512UL && !search.StartsWith("sys_", StringComparison.InvariantCultureIgnoreCase))
{
await ctx.RespondAsync($"This is not a _syscall_, {ctx.User.Mention}").ConfigureAwait(false);
return;
}
using var db = new ThumbnailDb();
if (db.SyscallInfo.Any(sci => sci.Function == search))
{
var productInfoList = db.SyscallToProductMap.AsNoTracking()
.Where(m => m.SyscallInfo.Function == search)
.Select(m => new { m.Product.ProductCode, Name = m.Product.Name.StripMarks() ?? "???" })
.Distinct()
.ToList();
var groupedList = productInfoList
.GroupBy(m => m.Name, m => m.ProductCode, StringComparer.InvariantCultureIgnoreCase)
.OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase)
.ToList();
if (groupedList.Any())
{
var bigList = groupedList.Count >= Config.MaxSyscallResultLines;
var result = new StringBuilder();
var fullList = bigList ? new StringBuilder() : null;
result.AppendLine($"List of games using `{search}`:```");
var c = 0;
foreach (var gi in groupedList)
{
var productIds = string.Join(", ", gi.Distinct().OrderBy(pc => pc).AsEnumerable());
if (c < Config.MaxSyscallResultLines)
{
result.AppendLine($"{gi.Key.Trim(60)} [{productIds}]");
}
if (bigList)
{
fullList.AppendLine($"{gi.Key} [{productIds}]");
}
c++;
}
await ctx.SendAutosplitMessageAsync(result.Append("```")).ConfigureAwait(false);
if (bigList)
{
using var memoryStream = Config.MemoryStreamManager.GetStream();
using var streamWriter = new StreamWriter(memoryStream, Encoding.UTF8);
await streamWriter.WriteAsync(fullList).ConfigureAwait(false);
await streamWriter.FlushAsync().ConfigureAwait(false);
memoryStream.Seek(0, SeekOrigin.Begin);
await ctx.RespondWithFileAsync($"{search}.txt", memoryStream, $"See attached file for full list of {groupedList.Count} entries").ConfigureAwait(false);
}
}
else
{
await ctx.RespondAsync($"No games found that use `{search}`").ConfigureAwait(false);
}
}
else
{
var result = new StringBuilder("Unknown entity name");
var functions = await db.SyscallInfo.Select(sci => sci.Function).Distinct().ToListAsync().ConfigureAwait(false);
var substrFuncs = functions.Where(f => f.Contains(search, StringComparison.InvariantCultureIgnoreCase));
var fuzzyFuncs = functions
.Select(f => (name: f, score: search.GetFuzzyCoefficientCached(f)))
.Where(i => i.score > 0.6)
.OrderByDescending(i => i.score)
.Select(i => i.name)
.ToList();
functions = substrFuncs
.Concat(fuzzyFuncs)
.Distinct()
.OrderBy(f => f, StringComparer.OrdinalIgnoreCase)
.ToList();
var functionsFound = functions.Any();
if (functionsFound)
{
result.AppendLine(", possible functions:```");
foreach (var f in functions)
{
result.AppendLine(f);
}
result.AppendLine("```");
}
await ctx.SendAutosplitMessageAsync(result).ConfigureAwait(false);
}
}
//int frameCount = 0;
async void Discover()
{
Debug.Log("Discovering...");
//make handshake with TCP_client, and the port is set to be 4444
tcpSocket = new StreamSocket();
HostName networkHost = new HostName(CommandSet.IP.Trim());
try
{
await tcpSocket.ConnectAsync(networkHost, CommandSet.DISCOVERY_PORT.ToString());
Debug.Log("tcpSocket Connected!");
} catch (Exception e) {
Debug.Log(e.ToString());
Debug.Log(SocketError.GetStatus(e.HResult).ToString());
return;
}
//Write data to the echo server.
Stream streamOut = tcpSocket.OutputStream.AsStreamForWrite();
StreamWriter writer = new StreamWriter(streamOut);
//when the drone receive the message bellow, it will return the confirmation
string handshake_Message = "{\"controller_type\":\"computer\", \"controller_name\":\"halley\", \"d2c_port\":\"43210\", \"arstream2_client_stream_port\":\"55004\",\"arstream2_client_control_port\":\"55005\"}";
try
{
await writer.WriteLineAsync(handshake_Message);
await writer.FlushAsync();
Debug.Log("tcpSocket writer successful!");
} catch (Exception e) {
Debug.Log(e.ToString());
Debug.Log(SocketError.GetStatus(e.HResult).ToString());
return;
}
//Read data from the echo server.
Stream streamIn = tcpSocket.InputStream.AsStreamForRead();
StreamReader reader = new StreamReader(streamIn);
string receive_Message = await reader.ReadLineAsync();
if (receive_Message == null)
{
Debug.Log("Discover failed");
//return -1;
isConnected = false;
}
else
{
Debug.Log("The message from the drone shows: " + receive_Message);
//initialize
initPCMD();
initCMD();
//All State setting
generateAllStates();
generateAllSettings();
isConnected = true;
Debug.Log("isConnected = true");
pcmdThreadActive();
//return 1;
}
}
public async override Task WriteResponseBodyAsync(OutputFormatterWriteContext context)
{
var response = context.HttpContext.Response;
Type type = context.Object.GetType();
Type itemType;
if (type.GetGenericArguments().Length > 0)
{
itemType = type.GetGenericArguments()[0];
}
else
{
itemType = type.GetElementType();
}
var streamWriter = new StreamWriter(response.Body, Encoding.GetEncoding(_options.Encoding));
if (_options.UseSingleLineHeaderInCsv)
{
await streamWriter.WriteLineAsync(
string.Join(
_options.CsvDelimiter, itemType.GetProperties().Select(x => x.GetCustomAttribute <DisplayAttribute>(false)?.Name ?? x.Name)
)
);
}
foreach (var obj in (IEnumerable <object>)context.Object)
{
var vals = obj.GetType().GetProperties().Select(
pi => new
{
Value = pi.GetValue(obj, null)
}
);
string valueLine = string.Empty;
foreach (var val in vals)
{
if (val.Value != null)
{
var _val = val.Value.ToString();
//Escape quotas
_val = _val.Replace("\"", "\"\"");
//Check if the value contans a delimiter and place it in quotes if so
if (_val.Contains(_options.CsvDelimiter))
{
_val = string.Concat("\"", _val, "\"");
}
//Replace any \r or \n special characters from a new line with a space
if (_val.Contains("\r"))
{
_val = _val.Replace("\r", " ");
}
if (_val.Contains("\n"))
{
_val = _val.Replace("\n", " ");
}
valueLine = string.Concat(valueLine, _val, _options.CsvDelimiter);
}
else
{
valueLine = string.Concat(valueLine, string.Empty, _options.CsvDelimiter);
}
}
await streamWriter.WriteLineAsync(valueLine.TrimEnd(_options.CsvDelimiter.ToCharArray()));
}
await streamWriter.FlushAsync();
}
private async Task <T> GetKeyObjectFromFile <T>(string name, IKeyJsonOps <T> keyOp)
{
var secretStorageType = System.Environment.GetEnvironmentVariable(Constants.AzureWebJobsSecretStorageType);
// Assume the default version to not be one
var isVersionOne = !string.IsNullOrEmpty(FunctionSiteExtensionVersion) && (FunctionSiteExtensionVersion.StartsWith("1.0") || FunctionSiteExtensionVersion.Equals("~1"));
// If it is version one (default is Files) and the secret storage type not Files (not default or set manually), throw an error.
// Or if it's not version one (default is Blob), and the secret storage is not Files (manually set), throw an error.
var isStorageFiles = isVersionOne
? string.IsNullOrEmpty(secretStorageType) || !secretStorageType.Equals("Blob", StringComparison.OrdinalIgnoreCase)
: !string.IsNullOrEmpty(secretStorageType) && secretStorageType.Equals("Files", StringComparison.OrdinalIgnoreCase);
if (!isStorageFiles)
{
throw new InvalidOperationException($"Runtime keys are stored on blob storage. This API doesn't support this configuration. " +
$"Please change Environment variable {Constants.AzureWebJobsSecretStorageType} value to 'Files'. For more info, visit https://aka.ms/funcsecrets");
}
string keyPath = GetFunctionSecretsFilePath(name);
string key = null;
if (!FileSystemHelpers.FileExists(keyPath) || FileSystemHelpers.FileInfoFromFileName(keyPath).Length == 0)
{
FileSystemHelpers.EnsureDirectory(Path.GetDirectoryName(keyPath));
try
{
using (var fileStream = FileSystemHelpers.OpenFile(keyPath, FileMode.Create, FileAccess.Write, FileShare.None))
// getting the lock early (instead of acquire the lock at "new StreamWriter(fileStream)")
// so no redundant work is being done (generate secrets)
{
string jsonContent = keyOp.GenerateKeyJson(SecurityUtility.GenerateSecretStringsKeyPair(keyOp.NumberOfKeysInDefaultFormat), FunctionSiteExtensionVersion, out key);
using (var sw = new StringWriter())
using (var sr = new System.IO.StringReader(jsonContent))
{
// write json to memory
// since JsonConvert has no method to format a json string
new JsonTextWriter(sw)
{
Formatting = Formatting.Indented
}.WriteToken(new JsonTextReader(sr));
using (var streamWriter = new StreamWriter(fileStream))
{
await streamWriter.WriteAsync(sw.ToString());
await streamWriter.FlushAsync();
}
}
}
return(keyOp.GenerateKeyObject(key, name));
}
catch (IOException)
{
// failed to open file => function runtime has the handler
// fallback to read key files
}
}
string jsonStr = null;
int timeOut = 5;
while (true)
{
try
{
jsonStr = await FileSystemHelpers.ReadAllTextFromFileAsync(keyPath);
break;
}
catch (Exception)
{
if (timeOut == 0)
{
throw new TimeoutException($"Fail to read {keyPath}, the file is being held by another process");
}
timeOut--;
await Task.Delay(250);
}
}
bool isEncrypted;
key = keyOp.GetKeyValueFromJson(jsonStr, out isEncrypted);
if (isEncrypted)
{
key = SecurityUtility.DecryptSecretString(key);
}
return(keyOp.GenerateKeyObject(key, name));
}
private async Task SendCommandAction()
{
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
_logger.LogTrace("Подключение к терминалу Quik для отправки команд...");
_commandClient = await ReconnectIfMissingConnectionAsync(_commandClient, _commandClientSemaphoreSlim, _cancellationTokenSource.Token);
_logger.LogTrace("Подключение к терминалу Quik для отправки команд установлено.");
try
{
using (var stream = new NetworkStream(_commandClient.Client))
using (var writer = new StreamWriter(stream))
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
var command = _commandEnvelopeQueue.Take(_cancellationTokenSource.Token);
if (!_pendingResultContainer.TryGet(command.Id, out var pendingResult))
{
_logger.LogWarning($"Среди находящихся в ожидании результатов команд нет результата для команды с идентификатором: {command.Id}. Возможно истекло время ожидания результата команды.");
continue;
}
if (!TrySerializeCommand(command, out var serializedCommandEnvelope))
{
continue;
}
try
{
await writer.WriteLineAsync(serializedCommandEnvelope);
await writer.FlushAsync();
}
catch (IOException)
{
// При ошибке ввода/вывода (например при перезапуске терминала Quik или остановке скрипта)
// возвращаем команду обратно в очередь, чтобы отправить повторно после установки
// соединения с терминалом Quik.
_commandEnvelopeQueue.Add(command);
throw;
}
}
}
}
catch (IOException e)
{
_logger.LogError(e, "Исключение при отправке команды в терминал Quik.");
}
}
}
catch (OperationCanceledException e)
{
_logger.LogTrace(e, "Отправка команд в терминал Quik остановлена.");
}
catch (Exception e)
{
_logger.LogError(e, "Исключение при отправке команды.");
_cancellationTokenSource.Cancel();
throw new QuikSharpException("Исключение при отправке команд.", e);
}
finally
{
_commandClient = await CloseClientAsync(_commandClient, _commandClientSemaphoreSlim);
}
}
public async Task Serialize(Hl7.Fhir.Model.Bundle bundle, Stream outputStream, bool pretty = false)
{
await using Utf8JsonWriter writer = new Utf8JsonWriter(outputStream, pretty ? _indentedWriterOptions : _writerOptions);
await using StreamWriter streamWriter = new StreamWriter(outputStream, leaveOpen: true);
writer.WriteStartObject();
writer.WriteString("resourceType", bundle.ResourceType.GetLiteral());
writer.WriteString("id", bundle.Id);
SerializeMetadata();
writer.WriteString("type", bundle.Type?.GetLiteral());
SerializeLinks();
if (bundle.Total.HasValue)
{
writer.WriteNumber("total", bundle.Total.Value);
}
await SerializeEntries();
writer.WriteEndObject();
await writer.FlushAsync();
void SerializeMetadata()
{
if (bundle.Meta != null)
{
writer.WriteStartObject("meta");
writer.WriteString("lastUpdated", bundle.Meta?.LastUpdated?.ToInstantString());
writer.WriteEndObject();
}
}
void SerializeLinks()
{
if (bundle.Link?.Any() == true)
{
writer.WriteStartArray("link");
foreach (var link in bundle.Link)
{
writer.WriteStartObject();
writer.WritePropertyName("relation");
writer.WriteStringValue(link.Relation);
writer.WritePropertyName("url");
writer.WriteStringValue(link.Url);
writer.WriteEndObject();
}
writer.WriteEndArray();
}
}
async Task SerializeEntries()
{
if (bundle.Entry?.Any() == true)
{
writer.WriteStartArray("entry");
foreach (var entry in bundle.Entry)
{
if (!(entry is RawBundleEntryComponent rawBundleEntry))
{
throw new ArgumentException("BundleSerializer can only be used when all Entry elements are of type RawBundleEntryComponent.", nameof(bundle));
}
bool wroteFullUrl = false;
writer.WriteStartObject();
if (!string.IsNullOrEmpty(rawBundleEntry.FullUrl))
{
writer.WriteString("fullUrl", rawBundleEntry.FullUrl);
await writer.FlushAsync();
await streamWriter.WriteAsync(",");
wroteFullUrl = true;
}
await writer.FlushAsync();
await streamWriter.WriteAsync("\"resource\":");
await streamWriter.FlushAsync();
await rawBundleEntry.ResourceElement.SerializeToStreamAsUtf8Json(outputStream);
if (!wroteFullUrl && (rawBundleEntry?.Search?.Mode != null || rawBundleEntry.Request != null || rawBundleEntry.Response != null))
{
// If fullUrl was written, the Utf8JsonWriter knows it needs to write a comma before the next property since a comma is needed, and will do so.
// If fullUrl wasn't written, since we are writing resource in a separate writer, we need to add this comma manually.
await streamWriter.WriteAsync(",");
await streamWriter.FlushAsync();
}
if (rawBundleEntry?.Search?.Mode != null)
{
writer.WriteStartObject("search");
writer.WriteString("mode", rawBundleEntry.Search?.Mode?.GetLiteral());
writer.WriteEndObject();
}
if (rawBundleEntry.Request != null)
{
writer.WriteStartObject("request");
writer.WriteString("method", rawBundleEntry.Request.Method.GetLiteral());
writer.WriteString("url", rawBundleEntry.Request.Url);
writer.WriteEndObject();
}
if (rawBundleEntry.Response != null)
{
writer.WriteStartObject("response");
writer.WriteString("etag", rawBundleEntry.Response.Etag);
writer.WriteString("lastModified", rawBundleEntry.Response.LastModified?.ToInstantString());
writer.WriteEndObject();
}
writer.WriteEndObject();
}
writer.WriteEndArray();
}
}
}
public async Task FlashAsync()
{
await _writer.FlushAsync();
}
