public static void HandleSwagger(IContext context, VersionTable <SiteHost> versions)
{
var path = context.Request.Url.AbsolutePath;
if (path == "/swagger")
{
context.Response.Redirect("/swagger/");
return;
}
if (path == "/swagger/")
{
HttpConverters.SendFile(@"swagger-ui\index.html", context);
return;
}
if (path == "/swagger/wrapperLoadFunction.js")
{
// Query the loaded versions, build a JS document with the named versions to load
BuildStartupFunction(context, versions);
return;
}
if (path.StartsWith("/swagger/json")) // generator functions
{
// populate each version from the Huygens servers in the version table
var requestedVersion = path.Substring(14);
Trace.TraceInformation("Looking for version " + requestedVersion);
SendApiSpecForVersion(context, versions, requestedVersion);
return;
}
var guessPath = @"swagger-ui\" + (path.Substring(9).Replace("/", "\\"));
Trace.TraceInformation("Trying to send a swagger file at '" + guessPath + "' from request '" + path + "'");
HttpConverters.SendFile(guessPath, context);
}
public static async Task LoadTestForQuery(ServiceTable serviceTable, VersionTable versionTable)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var version = await versionTable.GetCurrent(ResourceType.Service);
if (version != null)
{
stopWatch.Stop();
Console.WriteLine($"verion: {version.RowKey}, takes {stopWatch.ElapsedMilliseconds} milli-seconds");
}
stopWatch.Start();
var services = await serviceTable.GetFromParitionKey(version.RowKey);
var models = new List <ServiceViewModel>();
foreach (var entity in services)
{
try
{
models.Add(ServiceViewModel.FromEntity((ServiceEntity)entity));
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
stopWatch.Stop();
Console.WriteLine($"query {models.Count} takes {stopWatch.ElapsedMilliseconds} milli-seconds");
}
public void _02_CanUpdateVersion()
{
var entity = new VersionTable {
Name = "test " + DateTime.Now
};
using (var session = factory.OpenSession()) {
session.Save(entity);
session.Flush();
Assert.Greater(entity.Id, 0);
Assert.Greater(entity.Version, 0);
var v1 = entity.Version;
entity.Name = "update " + DateTime.Now;
session.Update(entity);
session.Flush();
var v2 = entity.Version;
Assert.Greater(v2, v1);
session.Delete(entity);
session.Flush();
}
}
public void TestInit()
{
// RedisVersionDb.Instance.PipelineMode = true;
//this.versionDb = RedisVersionDb.Instance;
//this.clientManager = ((RedisVersionDb)this.versionDb).RedisManager;
//int partition = versionDb.PhysicalPartitionByKey(DEFAULT_KEY);
//using (RedisClient redisClient = (RedisClient)this.clientManager.GetClient(TEST_REDIS_DB, partition))
//{
// // 1. flush the test db
// redisClient.ChangeDb(TEST_REDIS_DB);
// redisClient.FlushDb();
// // 2. create version table, if table is null, it means the table with the same tableId has existed
// VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);
// // 3. load data
// Transaction tx = new Transaction(null, this.versionDb);
// tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
// tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
// tx.Commit();
//}
this.versionDb = CassandraVersionDb.Instance(4);
VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);
Transaction tx = new Transaction(null, this.versionDb);
tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
tx.Commit();
}
/// <summary>
/// Use number of data bits(header + eci header + data bits from EncoderBase) to search for proper version to use
/// between min and max boundary.
/// Boundary define by DynamicSearchIndicator method.
/// </summary>
private static int BinarySearch(int numDataBits, ErrorCorrectionLevel level, int lowerVersionNum, int higherVersionNum)
{
int middleVersionNumber;
while (lowerVersionNum <= higherVersionNum)
{
middleVersionNumber = (lowerVersionNum + higherVersionNum) / 2;
QRCodeVersion version = VersionTable.GetVersionByNum(middleVersionNumber);
int numECCodewords = version.GetECBlocksByLevel(level).NumErrorCorrectionCodewards;
int dataCodewords = version.TotalCodewords - numECCodewords;
if (dataCodewords << 3 == numDataBits)
{
return(middleVersionNumber);
}
if (dataCodewords << 3 > numDataBits)
{
higherVersionNum = middleVersionNumber - 1;
}
else
{
lowerVersionNum = middleVersionNumber + 1;
}
}
return(lowerVersionNum);
}
/// <summary>Gets version table from PE/PE+ resources</summary>
/// <param name="reader">Mapped bytes</param>
/// <param name="type">Type of version table</param>
/// <param name="padding">Base padding</param>
/// <exception cref="T:NotImplementedException">Unknown table row type</exception>
/// <returns>Readed version table</returns>
private VersionTable GetVersionTable(PinnedBufferReader reader, VersionTableType type, ref UInt32 padding)
{
VersionTable result = new VersionTable();
UInt32 length = padding;
result.Table = reader.BytesToStructure <WinNT.Resource.StringTable>(ref padding);
result.szKey = reader.BytesToStringUni(ref padding);
List <VersionItem> items = new List <VersionItem>();
while (padding - length < result.Table.wLength)
{
padding = NativeMethods.AlignToInt(padding);
switch (type)
{
case VersionTableType.StringFileInfo:
items.Add(this.GetStringVersionItem(reader, ref padding));
break;
case VersionTableType.VarFileInfo:
items.Add(this.GetBinaryVersionItem(reader, result, ref padding));
break;
default: throw new NotImplementedException();
}
padding = NativeMethods.AlignToInt(padding);
}
result.Items = items.ToArray();
return(result);
}
GetLocalObjectPool(VersionTable versionTable, int workerId)
{
var visitor = versionTable.GetWorkerLocalVisitor(workerId)
as SingletonVersionTableVisitor;
return(visitor.recordPool);
}
private static int DataBits(int version, ErrorCorrectionLevel level)
{
int totalCodewords = VersionTable.GetVersionByNum(version).TotalCodewords;
int totalECCodewords = VersionTable.GetVersionByNum(version).GetECBlocksByLevel(level).NumErrorCorrectionCodewards;
return((totalCodewords - totalECCodewords) * 8);
}
protected override async Task OnExecuteAsync(CommandLineApplication app)
{
ValidateParameters();
var storageAccount = AzureStorageAccount.CreateStorageAccountFromConnectionString(ConnectionString);
var serviceTable = new ServiceTable(AzureStorageAccount.GetStorageTable(ConnectionString));
var versionTable = new VersionTable(AzureStorageAccount.GetStorageTable(ConnectionString));
await Helper.LoadTestForQuery(serviceTable, versionTable);
}
public static VersionTableTestProperties GetVersionInfo(int versionNum, ErrorCorrectionLevel level)
{
Version version = VersionTable.GetVersionByNum(versionNum);
int totalNumCodewords = version.TotalCodewords;
ErrorCorrectionBlocks ecBlocks = version.GetECBlocksByLevel(level);
int numECCodewords = ecBlocks.NumErrorCorrectionCodewards;
string ecBlockString = ErrorCorrectionBlocksToString(ecBlocks);
return(new VersionTableTestProperties(versionNum, totalNumCodewords, level, numECCodewords, ecBlockString));
}
public MainRequestHandler(ISecurityCheck security)
{
_security = security;
ExternalEndpoint = AnySetting("PrimaryCallbackAddress");
var settingValid = bool.TryParse(AnySetting("UpgradeHttp"), out var httpUpgradeSetting);
UpgradeHttp = HttpsAvailable && httpUpgradeSetting && settingValid;
var timer = new Stopwatch();
timer.Start();
_versionTable = new VersionTable <SiteHost>();
_baseFolder = AppDomain.CurrentDomain.BaseDirectory;
if (Directory.Exists(AnySetting("HostedSitesRootDirectory")))
{
// use directly specified folder
_watchFolder = AnySetting("HostedSitesRootDirectory");
}
else
{
// use general working folder (best for Azure?)
_watchFolder = Path.Combine(_baseFolder, "uploads");
Directory.CreateDirectory(_watchFolder);
}
AvailableAppScanner = new PluginScanner(_watchFolder);
AvailableAppScanner.PluginsChanged += AvailableAppWatcher_PluginsChanged;
// Load initial versions out-of-band so we don't appear dead at start-up
new Thread(() =>
{
try
{
_firstScan = true;
Trace.TraceInformation("Initial scan started");
AvailableAppScanner.RefreshPlugins();
Trace.TraceInformation("Initial scan complete");
}
finally
{
_firstScan = false;
timer.Stop();
_warmUp = timer.Elapsed;
Console.WriteLine("Ready");
}
})
{
IsBackground = true
}.Start();
}
private void testSourceButton_Click(object sender, EventArgs e)
{
var scan = new PluginScanner(sourceFolderTxt.Text);
scan.RefreshPlugins();
var table = new VersionTable <string>();
sourceTestResult.Text = "Scanning";
sourceTestResult.Refresh();
string[] all = null;
var task1 = new Thread(() =>
{
all = scan.CurrentlyAvailable.ToArray();
});
task1.Start();
while (task1.IsAlive)
{
Application.DoEvents();
}
if (!all.Any())
{
sourceTestResult.Text = "No binaries found";
return;
}
List <string> versions = null;
var task2 = new Thread(() =>
{
foreach (var path in all)
{
table.SubmitVersion(path, (typePath, versionName, major) => versionName.Replace("_", "."));
}
versions = table.AllVersions().ToList();
});
task2.Start();
while (task2.IsAlive)
{
Application.DoEvents();
}
if (!versions.Any())
{
sourceFolderTxt.Text = "No versioned entry points found";
return;
}
sourceTestResult.Text = "Found: " + string.Join("; ", versions);
}
private async Task _LoadDataTables()
{
Task[] loaders = new Task[] {
Task.Run(() => _versions = new VersionTable()),
Task.Run(() => _colors = new ColorTable()),
Task.Run(() => _items = new ItemTable()),
Task.Run(() => _recipes = new RecipeTable()),
Task.Run(() => _researchs = new ResearchTable()),
Task.Run(() => _schematics = new SchematicTable()),
};
await Task.WhenAll(loaders);
}
protected override async Task OnExecuteAsync(CommandLineApplication app)
{
var keyvaultAddress = $"https://{KeyVaultName}.vault.azure.net/";
var azureHelper = new AzureHelper(TenantId);
var storageAccount = CloudStorageAccount.Parse(
azureHelper.GetSecretValue(
keyvaultAddress, "StorageAccountConnectionString").GetAwaiter().GetResult());
var tableClient = storageAccount.CreateCloudTableClient();
var serviceTable = new ServiceTable(tableClient);
var versionTable = new VersionTable(tableClient);
await Helper.LoadTestForQuery(serviceTable, versionTable);
}
/// <summary>Получить элемент версии в сохранённый в бинарном виде</summary>
/// <param name="reader">Allocated bytes in memory</param>
/// <param name="item">Version table</param>
/// <param name="padding">Отступ от начала массива</param>
/// <returns>Элемент версии</returns>
private VersionItem GetBinaryVersionItem(PinnedBufferReader reader, VersionTable item, ref UInt32 padding)
{
VersionItem result = new VersionItem();
if (item.szKey == "Translation")
{
result.Value = reader.BytesToStructure <Translation>(padding);
String text = result.Value.ToString();
}
else
{
result.Value = reader.GetBytes(padding, item.Table.wValueLength);
}
padding += item.Table.wValueLength;
return(result);
}
/// <summary>
/// Handles the one-time migration of data from the binary file format to SQLite.
/// </summary>
public static void Main(string[] args)
{
const string databaseLocation = @"C:\git\csharp\hn-reader\data";
const string dataLocation = @"C:\git\csharp\hn-reader";
var dbName = Path.Combine(databaseLocation, "hn-data.sqlite");
if (File.Exists(dbName))
{
throw new InvalidOperationException("Database already exists! " + dbName);
}
using (var connection = Connector.ConnectToFile(dbName))
{
var command = new SQLiteCommand(Schema.Create, connection);
command.ExecuteNonQuery();
VersionTable.Write(1, connection);
var backfilledFiles = Directory.GetFiles(dataLocation, "*-complete.bin");
var trackers = new DataTrackers();
using (var transaction = connection.BeginTransaction())
{
foreach (var backfilledFile in backfilledFiles)
{
WriteFileIntoDatabase(backfilledFile, connection, trackers);
Console.WriteLine("Completed file: " + Path.GetFileName(backfilledFile));
}
transaction.Commit();
}
using (var transaction = connection.BeginTransaction())
{
WriteFileIntoDatabase(Path.Combine(dataLocation, "hn.bin"), connection, trackers);
transaction.Commit();
}
LastWriteTable.Write(trackers.MaxId, connection);
DateRangeTable.Write(trackers.MinDate, trackers.MaxDate, connection);
}
}
private static void BuildStartupFunction(IContext context, VersionTable <SiteHost> versions)
{
var list = versions.VersionNames()?.ToList() ?? new List <string>();
var sb = new StringBuilder(1024);
sb.Append(@"window.onload = function () { var ui = SwaggerUIBundle({
urls: [");
if (list.Count < 1)
{
sb.Append("{ url: './json/0-0.0.0.0', name: 'No versions loaded!'},");
}
// add version links
foreach (var vers in list)
{
sb.Append("{ url: './json/");
sb.Append(vers);
sb.Append("', name: '");
sb.Append(vers);
sb.Append("'},");
}
sb.Append(@"],");
// Try to pass correct version header in 'try it out'
sb.Append("requestInterceptor: function (req) {\r\n" +
" if (! req.loadSpec) {\r\n" +
" req.headers.Version = document.getElementById('select').selectedOptions[0].innerText.split(\'-\')[0];\r\n" +
" }\r\n" +
" return req;\r\n" +
" },");
sb.Append(@"
dom_id: '#swagger-ui', deepLinking: true, presets: [ SwaggerUIBundle.presets.apis, SwaggerUIStandalonePreset ],
plugins: [ SwaggerUIBundle.plugins.DownloadUrl ],
layout: 'StandaloneLayout'
});
window.ui = ui;
}");
SendStringBuilder(context, sb);
}
/// <summary>
/// Decide which version group it belong to
/// </summary>
/// <param name="numBits">Number of bits for bitlist where it contain DataBits encode from input content and ECI header</param>
/// <param name="level">Error correction level</param>
/// <returns>Version group index for VERSION_GROUP</returns>
private static int DynamicSearchIndicator(int numBits, ErrorCorrectionLevel level)
{
int[] charCountIndicator = CharCountIndicatorTable.GetCharCountIndicatorSet();
int loopLength = VERSION_GROUP.Length;
for (int i = 0; i < loopLength; i++)
{
int totalBits = numBits + NumBitsModeIndicator + charCountIndicator[i];
QRCodeVersion version = VersionTable.GetVersionByNum(VERSION_GROUP[i]);
int numECCodewords = version.GetECBlocksByLevel(level).NumErrorCorrectionCodewards;
int dataCodewords = version.TotalCodewords - numECCodewords;
if (totalBits <= dataCodewords * 8)
{
return(i);
}
}
throw new InputOutOfBoundaryException($"QRCode do not have enough space for {(numBits + NumBitsModeIndicator + charCountIndicator[2])} bits");
}
private static VersionControlStruct FillVCStruct(int versionNum, ErrorCorrectionLevel level)
{
if (versionNum is < 1 or > 40)
{
throw new InvalidOperationException($"Unexpected version number: {versionNum}");
}
VersionControlStruct vcStruct = new();
int version = versionNum;
QRCodeVersion versionData = VersionTable.GetVersionByNum(versionNum);
int numTotalBytes = versionData.TotalCodewords;
ErrorCorrectionBlocks ecBlocks = versionData.GetECBlocksByLevel(level);
int numDataBytes = numTotalBytes - ecBlocks.NumErrorCorrectionCodewards;
int numECBlocks = ecBlocks.NumBlocks;
VersionDetail vcDetail = new(version, numTotalBytes, numDataBytes, numECBlocks);
vcStruct.VersionDetail = vcDetail;
return(vcStruct);
}
/// <summary>
/// 修改实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public void Update(VersionTable model)
{
this._repoVersionTable.Update(model);
}
/// <summary>
/// 添加实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public VersionTable Insert(VersionTable model)
{
return(this._repoVersionTable.Insert(model));
}
/// <summary>
/// 添加实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public VersionTable Insert(VersionTable model)
{
return(this._VersionTableBiz.Insert(model));
}
/// <summary>
/// 删除实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public void Delete(VersionTable model)
{
this._VersionTableBiz.Delete(model);
}
public static async Task <string> GetVersionAsync()
{
string version = await VersionTable.ReadVersion(CommandReadModel.ReadVersionTable());
return(version);
}
private bool MigrateNext(long dbVersion, MigratorDatabase database)
{
using (database.BeginTransaction())
{
dbVersion = VersionTable.GetCurrentVersionWithLock(database);
if (dbVersion >= LatestSchemaVersion)
{
return(false);
}
Interceptor?.PreMigration(database.Name, dbVersion, LatestSchemaVersion);
var migrations = Migrations
.Where(x => x.Version > dbVersion)
.ToArray();
var firstMigration = migrations.First();
if (firstMigration.DisableTransaction)
{
using (var noTransactionDatabase = database.Clone())
ExecuteMigration(migrations.First(), noTransactionDatabase);
if (firstMigration.Version > dbVersion)
{
VersionTable.SetVersion(database, firstMigration.Version);
}
database.Commit();
if (migrations.Length == 1)
{
Interceptor?.PostMigration(database.Name, dbVersion, LatestSchemaVersion);
return(false);
}
return(true);
}
foreach (var migration in migrations)
{
if (migration.DisableTransaction)
{
database.Commit();
return(true);
}
ExecuteMigration(migration, database);
if (migration.Version > dbVersion)
{
VersionTable.SetVersion(database, migration.Version);
}
}
database.Commit();
Interceptor?.PostMigration(database.Name, dbVersion, LatestSchemaVersion);
return(false);
}
}
public static string Generate(VersionTable <SiteHost> _versionTable, TimeSpan _warmUp, string _watchFolder, bool _isScanning, Exception _lastScanError)
{
var body = T.g("body");
var page = T.g("html")[
T.g("head")[
T.g("title")["Wrapper proxy test page"],
T.g("style")[".good {stroke: #0A0; } .bad {stroke: #A00; } path { stroke-width: 2.5px; fill: none; opacity: 0.5;}"]
],
body
];
body.Add(T.g("h1")["Status"]);
if (_warmUp.Ticks == 0)
{
body.Add(T.g("p")["The proxy is starting up. Versions will be listed below as they are ready."]);
}
else
{
body.Add(T.g("p")["Three flavours"]);
body.Add(T.g("p")["The proxy is active. Initial warm up took: " + _warmUp]);
}
body.Add(T.g("p")["Currently loaded: " + _versionTable.VersionsAvailable()]);
body.Add(T.g("p")["Watch folder: ", T.g("tt")[_watchFolder], " ", (_isScanning) ? ("Scan is in progress") : ("Scanner is idle")]);
// ReSharper disable once InconsistentlySynchronizedField
if (_lastScanError != null)
{
body.Add(T.g("p")["Last scan error: ", _lastScanError.ToString()]);
}
// run a health check against all versions and spit them out here...
body.Add(T.g("h1")["Health check"]);
var list = _versionTable.AllVersions().ToList();
foreach (var version in list)
{
var result = version.HostedSite.DirectCall(HealthRequest);
body.Add(T.g("h3")["Version " + version.VersionName]);
body.Add(T.g("p")[version.CallCount + " calls, " + (100 * version.SuccessRate).ToString("0.00") + "% successful"]);
body.Add(T.g("p")[result.StatusCode + " " + result.StatusMessage]);
if (result.Content != null)
{
body.Add(T.g("pre")[Encoding.UTF8.GetString(result.Content)]);
}
if (version.LastError != null)
{
body.Add(T.g("p")["Last proxy error: " + version.LastError]);
}
}
body.Add(T.g("h1")["Recent log entries"]);
// This double container lets us put the scroll-bar on the left. I just like it better that way :-)
body.Add(T.g("div", "style", "direction:rtl;overflow-y:scroll;height:40%;")[
T.g("pre", "style", "direction: ltr;")[LocalTrace.ReadAll()]
]);
// Some graphs
body.Add(T.g("h1")["Recent History"], T.g("p")["Logarithmic time scale. Left is last few seconds, right is last week."]);
foreach (var version in list)
{
body.Add(T.g("h3")["Version " + version.MajorVersion]);
body.Add(T.g("div")[RenderGraph(version.SuccessHistory, version.FailureHistory)]);
}
return(page.ToString());
}
/// <summary>
/// 删除实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public void Delete(VersionTable model)
{
this._repoVersionTable.Delete(model);
}
/// <summary>
/// 修改实体
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public void Update(VersionTable model)
{
this._VersionTableBiz.Update(model);
}
private long GetDbVersion(MigratorDatabase database)
{
VersionTable.CreateIfNotExisting(database);
return(VersionTable.GetCurrentVersion(database));
}
private static void SendApiSpecForVersion(IContext context, VersionTable <SiteHost> versions, string requestedVersion)
{
var result = versions.GetExactVersion(requestedVersion);
var sb = new StringBuilder(1024);
if (result.IsFailure)
{
// Craft a special response
AppendSwaggerErrorMessage(sb, result.FailureCause?.Message ?? "Unknown error");
SendStringBuilder(context, sb);
return;
}
var major = requestedVersion.Split('-').FirstOrDefault() ?? "0";
// get a host setting to inject in the swagger
var redirUri = new Uri(MainRequestHandler.ExternalEndpoint, UriKind.Absolute);
var selfAuthority = new Uri(MainRequestHandler.ExternalEndpoint, UriKind.Absolute).Authority;
if (MainRequestHandler.UpgradeHttp && redirUri.Scheme == "http")
{
selfAuthority = redirUri.Host; // configured address would get bounced, to try guess a better uri
}
var proxy = result.ResultData;
var routes = new[] { "/swagger/docs/v1", "/swagger/v1/swagger.json", "/swagger/docs/v" + major, "/swagger/v" + major + "/swagger.json" }; // we expect the raw swagger JSON to be in one of these paths
foreach (var route in routes)
{
var rq = new SerialisableRequest {
Method = "GET",
RequestUri = route,
Headers = new Dictionary <string, string>()
};
var tx = proxy.Request(rq);
if (tx.StatusCode < 299) // found it
{
var spec = Json.DefrostDynamic(Encoding.UTF8.GetString(tx.Content));
spec.host = selfAuthority;
spec.info.version = requestedVersion;
if (MainRequestHandler.UpgradeHttp)
{
spec.schemes = new[] { "https" };
}
else if (MainRequestHandler.HttpsAvailable)
{
spec.schemes = new[] { "http", "https" };
}
else
{
spec.schemes = new[] { "http" };
}
spec.securityDefinitions = new Dictionary <string, object> {
{ "Bearer Token", new { type = "apiKey", name = "Authorization", @in = "header" } }
};
spec.security = new[] {
new Dictionary <string, object> {
{ "Bearer Token", new object[0] }
}
};
sb.Append(Json.Freeze(spec));
SendStringBuilder(context, sb);
return;
}
}
AppendSwaggerErrorMessage(sb, "Failed to find a JSON endpoint in the hosted API. Make sure that hosted APIs respond to one of:<br/>" + string.Join("<br/>", routes));
SendStringBuilder(context, sb);
}
