// データベースから最新の情報を取得するタスク
async Task ReadMainLoopAsync(CancellationToken cancel)
{
int numCycle = 0;
int numError = 0;
$"ThinDatabase.ReadMainLoopAsync: Waiting for start."._Debug();
await TaskUtil.AwaitWithPollAsync(Timeout.Infinite, 1000, () => this.IsStarted, cancel);
$"ThinDatabase.ReadMainLoopAsync: Started."._Debug();
while (cancel.IsCancellationRequested == false)
{
numCycle++;
$"ThinDatabase.ReadMainLoopAsync numCycle={numCycle}, numError={numError} Start."._Debug();
long startTick = Time.HighResTick64;
bool ok = false;
try
{
await ReadCoreAsync(cancel);
ok = true;
}
catch (Exception ex)
{
ex._Error();
}
long endTick = Time.HighResTick64;
if (ok)
{
LastDbReadTookMsecs = (int)(endTick - startTick);
}
else
{
LastDbReadTookMsecs = 0;
}
$"ThinDatabase.ReadMainLoopAsync numCycle={numCycle}, numError={numError} End. Took time: {endTick - startTick}"._Debug();
await cancel._WaitUntilCanceledAsync(Util.GenRandInterval(Controller.CurrentValue_ControllerDbFullReloadIntervalMsecs));
}
}
// データベースを更新するタスク
async Task WriteMainLoopAsync(CancellationToken cancel)
{
int numCycle = 0;
int numError = 0;
$"ThinDatabase.WriteMainLoopAsync: Waiting for start."._Debug();
await TaskUtil.AwaitWithPollAsync(Timeout.Infinite, 1000, () => this.IsStarted, cancel);
$"ThinDatabase.WriteMainLoopAsync: Started."._Debug();
while (cancel.IsCancellationRequested == false)
{
if (this.LazyUpdateJobQueue.Count >= 1)
{
numCycle++;
$"ThinDatabase.WriteMainLoopAsync numCycle={numCycle}, numError={numError} Start."._Debug();
long startTick = Time.HighResTick64;
int num = 0;
try
{
num = await WriteCoreAsync(cancel);
}
catch (Exception ex)
{
ex._Error();
}
long endTick = Time.HighResTick64;
$"ThinDatabase.WriteMainLoopAsync numCycle={numCycle}, numError={numError} End. Written items: {num}, Took time: {endTick - startTick}"._Debug();
}
await cancel._WaitUntilCanceledAsync(Util.GenRandInterval(Controller.CurrentValue_ControllerDbWriteUpdateIntervalMsecs));
}
}
async Task ReadMainLoop(CancellationToken cancel)
{
try
{
var st = this.Reader.StreamReader;
int numFailed = 0;
while (true)
{
await TaskUtil.AwaitWithPollAsync(Timeout.Infinite, this.Emitter !.Options.Delay, () => this.Reader.StreamReader.IsReadyToRead(), cancel);
IReadOnlyList <ReadOnlyMemory <byte> > dataToWrite = DequeueAll(out long totalSize, this.Emitter.Options.DefragmentWriteBlockSize);
if (totalSize == 0)
{
if (cancel.IsCancellationRequested)
{
break;
}
else
{
continue;
}
}
L_RETRY:
try
{
await Emitter.EmitAsync(dataToWrite);
numFailed = 0;
if (this.Reader.StreamReader.IsReadyToRead() == false)
{
await Emitter.FlushAsync();
}
}
catch (Exception ex)
{
ex._Debug();
numFailed++;
if (cancel.IsCancellationRequested == false)
{
await cancel._WaitUntilCanceledAsync(Util.GenRandIntervalWithRetry(CoresConfig.LazyWriteBufferSettings.ErrorRetryIntervalStd, numFailed, CoresConfig.LazyWriteBufferSettings.ErrorRetryIntervalMax));
goto L_RETRY;
}
else
{
break;
}
}
}
}
finally
{
try
{
await Emitter !.CloseAsync();
}
catch { }
}
}
protected override async Task GetValueAsync(SortedDictionary <string, string> ret, RefInt nextPollingInterval, CancellationToken cancel = default)
{
SnmpWorkSettings settings = Host.Settings;
if (settings.PingTargets._IsSamei("none") || settings.PingTargets._IsSamei("null"))
{
return;
}
string hopsStr = "Hops";
if (settings.HopsToTTL)
{
hopsStr = "TTL";
}
string[] pingTargets = settings.PingTargets._Split(StringSplitOptions.RemoveEmptyEntries, ",");
numPerform++;
foreach (string pingTarget in pingTargets)
{
cancel.ThrowIfCancellationRequested();
ParseTargetString(pingTarget, out string hostname, out string alias);
bool ok = false;
try
{
IPAddress ipAddress = await LocalNet.GetIpAsync(hostname, cancel : cancel);
if (FirstPing.IsFirstCall())
{
// JIT 対策
try
{
await LocalNet.SendPingAsync(ipAddress, pingCancel : cancel);
}
catch { }
}
int numTry = 3;
if (numPerform >= 2)
{
// SpeedTest が動作中の場合は SpeedTest が完了するまで待機する
await TaskUtil.AwaitWithPollAsync(Timeout.Infinite, 10, () => !SpeedTestClient.IsInProgress, cancel);
// 試行回数を指定する
numTry = Math.Min(Math.Max(settings.PingNumTry, 1), 100);
}
SendPingReply reply = await LocalNet.SendPingAndGetBestResultAsync(ipAddress, pingCancel : cancel, numTry : numTry);
if (reply.Ok)
{
double rtt = reply.RttDouble;
rtt = Math.Min(rtt, 2.0);
int ttl = reply.Ttl;
bool ttl_ok = false;
if (ttl == 0)
{
// Use ping command to get TTL
try
{
var result = await EasyExec.ExecAsync(ipAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6?Consts.LinuxCommands.Ping6 : Consts.LinuxCommands.Ping,
$"-W 1 -c 1 {ipAddress.ToString()}",
cancel : cancel,
throwOnErrorExitCode : false);
string[] lines = result.OutputStr._GetLines(true);
foreach (string line in lines)
{
OneLineParams param = new OneLineParams(line, ' ', false);
string ttlStr = param._GetStrFirst("ttl");
if (ttlStr._IsFilled())
{
ttl = ttlStr._ToInt();
ttl_ok = true;
break;
}
}
}
catch (Exception ex)
{
ex._Debug();
}
}
else
{
ttl_ok = true;
}
ret.TryAdd($"Time - {alias}", (rtt * 1000.0).ToString("F3"));
if (ttl > 128)
{
ttl -= 128;
}
else if (ttl > 64)
{
ttl -= 64;
}
int hops = 64 - ttl;
if (ttl_ok == false)
{
hops = 0;
}
hops._SetMax(0);
hops._SetMin(64);
ret.TryAdd($"{hopsStr} - {alias}", hops.ToString());
ok = true;
}
}
catch (Exception ex)
{
ex._Debug();
}
if (ok == false)
{
ret.TryAdd($"Time - {alias}", "");
ret.TryAdd($"{hopsStr} - {alias}", "0");
}
}
}
protected override async Task GetValueAsync(SortedDictionary <string, string> ret, RefInt nextPollingInterval, CancellationToken cancel = default)
{
SnmpWorkSettings settings = Host.Settings;
if (settings.PingTargets._IsSamei("none") || settings.PingTargets._IsSamei("null"))
{
return;
}
string[] pingTargets = settings.PingTargets._Split(StringSplitOptions.RemoveEmptyEntries, ",");
KeyValueList <string, IPAddress> kvList = new KeyValueList <string, IPAddress>();
// 名前解決
foreach (string pingTarget in pingTargets)
{
cancel.ThrowIfCancellationRequested();
try
{
ParseTargetString(pingTarget, out string hostname, out string alias);
IPAddress ip = await LocalNet.GetIpAsync(hostname, cancel : cancel);
kvList.Add(alias, ip);
}
catch (Exception ex)
{
ex._Debug();
}
}
List <Task <double> > taskList = new List <Task <double> >();
int interval = 0;
int count = 3;
// SpeedTest が動作中の場合は SpeedTest が完了するまで待機する
numPerform++;
if (numPerform >= 2)
{
interval = settings.PktLossIntervalMsec;
count = settings.PktLossTryCount;
await TaskUtil.AwaitWithPollAsync(Timeout.Infinite, 10, () => !SpeedTestClient.IsInProgress, cancel);
}
// 並列実行の開始
foreach (var kv in kvList)
{
taskList.Add(PerformOneAsync(kv.Value, count, settings.PktLossTimeoutMsecs, interval, cancel));
}
// すべて終了するまで待機し、結果を整理
for (int i = 0; i < kvList.Count; i++)
{
var kv = kvList[i];
var task = taskList[i];
double lossRate = await task._TryAwait();
double quality = 1.0 - lossRate;
quality = Math.Max(quality, 0.0);
quality = Math.Min(quality, 1.0);
ret.TryAdd($"{kv.Key}", ((double)quality * 100.0).ToString("F3"));
}
}
// GitLab のユーザーメンテナンス
async Task Loop1_MainteUsersAsync(CancellationToken cancel = default)
{
long lastHookTick = -1;
List <GitLabMainteClient.User> lastPendingUsers = new List <GitLabMainteClient.User>();
while (cancel.IsCancellationRequested == false)
{
// 新規申請中のユーザーが増えたらメールで知らせる
try
{
// ユーザーの列挙
var users = await this.GitLabClient.EnumUsersAsync(cancel);
var pendingUsers = users.Where(x => x.IsSystemUser() == false && x.state == "blocked_pending_approval").OrderBy(x => x.id);
var newPendingUsers = pendingUsers.Where(u => lastPendingUsers.Where(a => a.id == u.id).Any() == false);
StringWriter w = new StringWriter();
string url = this.Settings.GitLabClientSettings.GitLabBaseUrl._CombineUrl("/admin/users?filter=blocked_pending_approval").ToString();
string subject = $"{url._ParseUrl().Host} にユーザー {newPendingUsers.Select(x => ("[" + x.commit_email._NonNullTrim() + " " + x.name + " " + x.username + "]"))._Combine(" ,")} の参加申請がありました";
w.WriteLine(subject + "。");
w.WriteLine();
w.WriteLine($"GitLab のアドレス: {url}");
w.WriteLine();
w.WriteLine($"現在時刻: {DtOffsetNow._ToDtStr()}");
w.WriteLine();
w.WriteLine($"新しい申請中のユーザー ({newPendingUsers.Count()}):");
int num = 0;
foreach (var user in newPendingUsers)
{
lastPendingUsers.Add(user._CloneDeep());
w.WriteLine("- " + user.name + " " + user.username + " " + user.commit_email);
num++;
}
w.WriteLine();
w.WriteLine($"現在申請中のユーザー一覧 ({pendingUsers.Count()})");
foreach (var user in pendingUsers)
{
w.WriteLine("- " + user.name + " " + user.username + " " + user.commit_email);
}
w.WriteLine();
w.WriteLine($"GitLab のアドレス: {url}");
w.WriteLine();
w.WriteLine();
//Dbg.Where();
if (num >= 1)
{
await this.SendMailAsync(subject, w.ToString(), cancel);
}
}
catch (Exception ex)
{
ex._Error();
}
// すべてのユーザーをデフォルトグループに自動追加する
try
{
await this.JoinAllUsersToSpecificGroupAsync(this.Settings.DefaultGroupsAllUsersWillJoin, cancel);
}
catch (Exception ex)
{
ex._Error();
}
await TaskUtil.AwaitWithPollAsync(this.Settings.UsersListMainteIntervalMsecs, 500, () =>
{
long currentHookTick = this.HookFiredTick;
if (lastHookTick != currentHookTick)
{
lastHookTick = currentHookTick;
return(true);
}
return(false);
},
cancel,
true);
}
}
// Git リポジトリの自動ダウンロード
async Task Loop2_MainteUsersAsync(CancellationToken cancel = default)
{
long lastHookTick = -1;
while (cancel.IsCancellationRequested == false)
{
try
{
await Lfs.CreateDirectoryAsync(this.Settings.GitMirrorDataRootDir);
await Lfs.CreateDirectoryAsync(this.Settings.GitWebDataRootDir);
}
catch
{
}
try
{
// Git リポジトリを列挙
var projects = await this.GitLabClient.EnumProjectsAsync(cancel);
ConcurrentHashSet <string> dirNames = new ConcurrentHashSet <string>(StrCmpi);
// メモ: last_activity_at の値を信用してはならない。これは GitLab がキャッシュしているので、なかなか更新されない。
var targetProjects = projects.Where(p => p.empty_repo == false && p.path_with_namespace._IsFilled() && p.default_branch._IsFilled() && p.visibility._IsDiffi("private")).OrderByDescending(x => x.last_activity_at).ThenBy(x => x.path_with_namespace, StrCmpi);
await TaskUtil.ForEachAsync(8, targetProjects, async (proj, index, cancel) =>
{
cancel.ThrowIfCancellationRequested();
try
{
string dirname = proj.GenerateDirName();
dirNames.Add(dirname);
string gitRoot = this.Settings.GitMirrorDataRootDir._CombinePath(dirname);
string webRoot = this.Settings.GitWebDataRootDir._CombinePath(dirname);
await this.GitLabClient.GitPullFromRepositoryAsync(proj.path_with_namespace !, gitRoot, proj.default_branch !, cancel);
await this.SyncGitLocalRepositoryDirToWebRootDirAsync(gitRoot, webRoot, cancel);
}
catch (Exception ex)
{
ex._Error();
}
}, cancel);
// GitLab 上に存在せず local に存在する gitRoot を列挙して削除する
var existingLocalGitDirs = await Lfs.EnumDirectoryAsync(this.Settings.GitMirrorDataRootDir, cancel : cancel);
foreach (var d in existingLocalGitDirs.Where(x => x.IsDirectory && dirNames.Contains(x.Name) == false))
{
cancel.ThrowIfCancellationRequested();
try
{
await Lfs.DeleteDirectoryAsync(d.FullPath, true, cancel, true);
}
catch { }
}
// GitLab 上に存在せず local に存在する webRoot を列挙して削除する
var existingLocalWebDirs = await Lfs.EnumDirectoryAsync(this.Settings.GitWebDataRootDir, cancel : cancel);
foreach (var d in existingLocalWebDirs.Where(x => x.IsDirectory && dirNames.Contains(x.Name) == false))
{
cancel.ThrowIfCancellationRequested();
try
{
await Lfs.DeleteFileAsync(d.FullPath._CombinePath(Consts.FileNames.LogBrowserSecureJson), cancel : cancel);
}
catch { }
}
}
catch (Exception ex)
{
ex._Error();
}
await TaskUtil.AwaitWithPollAsync(this.Settings.ForceRepositoryUpdateIntervalMsecs, 500, () =>
{
long currentHookTick = this.HookFiredTick;
if (lastHookTick != currentHookTick)
{
lastHookTick = currentHookTick;
return(true);
}
return(false);
},
cancel,
true);
}
}
