public override async Task EmitAsync(IReadOnlyList <ReadOnlyMemory <byte> > dataToWrite, CancellationToken cancel = default)
{
if (IsClosed)
{
throw new ObjectDisposedException("LazyBufferFileEmitter");
}
bool firstOnFile = false;
if (file == null)
{
if (this.Options.AppendMode)
{
file = await this.Options.FilePath.OpenOrCreateAppendAsync(additionalFlags : FileFlags.AutoCreateDirectory, cancel : cancel);
if (file.Position == 0)
{
firstOnFile = true;
}
}
else
{
file = await this.Options.FilePath.CreateAsync(additionalFlags : FileFlags.AutoCreateDirectory, cancel : cancel);
firstOnFile = true;
}
}
if (firstOnFile)
{
if (this.Options.FileHeaderData.IsEmpty == false)
{
await file.WriteAsync(this.Options.FileHeaderData);
}
}
foreach (ReadOnlyMemory <byte> data in dataToWrite)
{
try
{
await file !.WriteAsync(data, cancel);
}
catch (Exception ex)
{
Con.WriteDebug($"LazyBufferFileEmitter -> EmitAsync: WriteAsync('{this.Options.FilePath.ToString()}' failed. error: {ex.ToString()}");
await cancel._WaitUntilCanceledAsync(1000);
file = null;
}
}
}
public static void WriteToFile(string path, string bodyString, Encoding?encoding = null, bool writeBom = false, bool noDebug = false)
{
bodyString = bodyString._NonNull();
bodyString = Str.NormalizeCrlf(bodyString, CrlfStyle.LocalPlatform);
Lfs.WriteStringToFile(path, bodyString, FileFlags.AutoCreateDirectory | FileFlags.WriteOnlyIfChanged,
encoding: encoding, writeBom: writeBom);
if (noDebug == false)
{
Con.WriteDebug($"--- WriteToFile \"{path}\" ---");
Con.WriteDebug(bodyString);
Con.WriteDebug($"--- EOF ---");
Con.WriteDebug();
}
}
protected override async Task LogReceiveImplAsync(IReadOnlyList <LogServerReceivedData> dataList)
{
SingletonSlim <string, MemoryBuffer <byte> > writeBufferList =
new SingletonSlim <string, MemoryBuffer <byte> >((filename) => new MemoryBuffer <byte>(), this.Options.DestFileSystem.PathParser.PathStringComparer);
foreach (LogServerReceivedData data in dataList)
{
Options.SetDestinationsProc(data, this.Options);
foreach (string fileName in data.DestFileNames)
{
var buffer = writeBufferList[fileName];
buffer.Write(data.BinaryData);
buffer.Write(Str.NewLine_Bytes_Windows);
}
}
foreach (string fileName in writeBufferList.Keys)
{
try
{
var buffer = writeBufferList[fileName];
var handle = await Options.DestFileSystem.GetRandomAccessHandleAsync(fileName, true, this.Options.FileFlags | FileFlags.AutoCreateDirectory);
var concurrentHandle = handle.GetConcurrentRandomAccess();
await concurrentHandle.AppendAsync(buffer.Memory);
await concurrentHandle.FlushAsync();
}
catch (Exception ex)
{
Con.WriteDebug($"LogReceiveImplAsync: Filename '{fileName}' write error: {ex.ToString()}");
}
}
}
public CertVaultCertificate(CertVault vault, DirectoryPath dirName, CertVaultCertType certType)
{
this.Vault = vault;
if (certType.EqualsAny(CertVaultCertType.Acme, CertVaultCertType.Static, CertVaultCertType.AutoGenerated) == false)
{
throw new ArgumentOutOfRangeException("certType");
}
try
{
dirName.CreateDirectory();
}
catch { }
CertificateStore?store = null;
this.CertType = certType;
this.DirName = dirName;
if (certType == CertVaultCertType.Static || certType == CertVaultCertType.AutoGenerated)
{
// Static / auto generated cert
var files = DirName.EnumDirectory().Where(x => x.IsDirectory == false);
string?p12file = files.Where(x => x.Name._IsExtensionMatch(Consts.Extensions.Filter_Pkcs12s)).SingleOrDefault()?.FullPath;
string?certfile = files.Where(x => x.Name._IsExtensionMatch(Consts.Extensions.Filter_Certificates)).SingleOrDefault()?.FullPath;
string?keyfile = files.Where(x => x.Name._IsExtensionMatch(Consts.Extensions.Filter_Keys)).SingleOrDefault()?.FullPath;
string?passwordfile = files.Where(x => x.Name._IsSamei(Consts.FileNames.CertVault_Password)).SingleOrDefault()?.FullPath;
string?password = null;
if (passwordfile != null)
{
password = FileSystem !.ReadStringFromFile(passwordfile, oneLine: true);
if (password._IsEmpty())
{
password = null;
}
}
if (p12file != null)
{
store = new CertificateStore(FileSystem !.ReadDataFromFile(p12file).Span, password);
}
else if (certfile != null && keyfile != null)
{
store = new CertificateStore(FileSystem !.ReadDataFromFile(certfile).Span, FileSystem.ReadDataFromFile(keyfile).Span, password);
}
else
{
store = null;
}
}
else
{
// ACME cert
FilePath fileName = DirName.Combine(DirName.GetThisDirectoryName() + Consts.Extensions.Certificate_Acme);
if (fileName.IsFileExists())
{
store = new CertificateStore(fileName.ReadDataFromFile().Span, this.Vault.AcmeCertKey !);
}
else
{
store = null;
}
}
Certificate?test = store?.PrimaryContainer.CertificateList[0];
if (test != null && store != null)
{
if (test.PublicKey.Equals(store.PrimaryContainer.PrivateKey.PublicKey) == false)
{
Con.WriteDebug($"CertVault: The public key certificate in the directory '{dirName}' doesn't match to the private key.");
store = null;
}
}
this.Store = store;
}
public TelnetStreamWatcherBase(TelnetStreamWatcherOptions options)
{
this.Options = options;
Listener = TcpIp.CreateListener(new TcpListenParam(async(listener, sock) =>
{
try
{
Con.WriteDebug($"TelnetStreamWatcher({this.ToString()}: Connected: {sock.EndPointInfo._GetObjectDump()}");
using (var destStream = sock.GetStream())
{
if (sock.Info.Ip.RemoteIPAddress == null || Options.IPAccessFilter(sock.Info.Ip.RemoteIPAddress) == false)
{
Con.WriteDebug($"TelnetStreamWatcher({this.ToString()}: Access denied: {sock.EndPointInfo._GetObjectDump()}");
await destStream.WriteAsync("You are not allowed to access to this service.\r\n\r\n"._GetBytes_Ascii());
await Task.Delay(100);
return;
}
using (var pipePoint = await SubscribeImplAsync())
{
// ソケットから Enter キー入力を待機する
Task keyInputTask = TaskUtil.StartAsyncTaskAsync(async() =>
{
using StreamReader lineReader = new StreamReader(destStream);
while (true)
{
string?line = await lineReader.ReadLineAsync();
if (line == null)
{
break;
}
line = line.Trim();
if (line._IsSamei("s"))
{
// Socket リストの表示
var list = LocalNet.GetSockList().OrderBy(x => x.Connected);
StringWriter w = new StringWriter();
w.WriteLine();
foreach (var sock in list)
{
string tmp = sock._GetObjectDump();
w.WriteLine(tmp);
}
w.WriteLine();
w.WriteLine($"Total sockets: {list.Count()}");
w.WriteLine();
byte[] data = w.ToString()._GetBytes_Ascii();
var pipe = pipePoint;
if (pipe.CounterPart != null)
{
lock (pipe.CounterPart.StreamWriter.LockObj)
{
if (pipe.CounterPart.StreamWriter.NonStopWriteWithLock(data, false, FastStreamNonStopWriteMode.DiscardExistingData) != 0)
{
// To avoid deadlock, CompleteWrite() must be called from other thread.
// (CompleteWrite() ==> Disconnect ==> Socket Log will recorded ==> ReceiveLog() ==> this function will be called!)
TaskUtil.StartSyncTaskAsync(() => pipe.CounterPart.StreamWriter.CompleteWrite(false), false, false)._LaissezFaire(true);
}
}
}
}
else
{
// GC
Dbg.WriteLine($"Manual GC is called by the administrator.");
long start = FastTick64.Now;
Dbg.GcCollect();
long end = FastTick64.Now;
long spentTime = end - start;
Dbg.WriteLine($"Manual GC Took Time: {spentTime} msecs.");
}
}
});
try
{
// ソケットに対して、pipePoint のストリームをそのまま非同期で流し込む
using (var pipeStub = pipePoint.GetNetAppProtocolStub())
using (var srcStream = pipeStub.GetStream())
{
await srcStream.CopyToAsync(destStream, sock.GrandCancel);
}
}
finally
{
await UnsubscribeImplAsync(pipePoint);
await pipePoint.CleanupAsync(new DisconnectedException());
await keyInputTask._TryAwait(noDebugMessage: true);
}
}
}
}
finally
{
Con.WriteDebug($"TelnetStreamWatcher({this.ToString()}: Disconnected: {sock.EndPointInfo._GetObjectDump()}");
}
},
"TelnetStreamWatcher",
this.Options.EndPoints.ToArray()));
this.AddIndirectDisposeLink(this.Listener);
}
public static void StartRepository(string repoUrl)
{
Module.CheckInitalized();
lock (RepositoryUpdateLock)
{
if (repoUrl.IndexOf("@") != -1)
{
throw new ArgumentException($"The repository name \'{repoUrl}\' must not contain '@'.");
}
if (Data.IsReadOnly)
{
throw new ApplicationException("Data.IsReadOnly");
}
lock (Data.DataLock)
{
GitGlobalFsRepository?repoData = Data.ManagedData.RepositoryList.Where(x => x.Url._IsSamei(repoUrl)).SingleOrDefault();
L_RETRY:
if (repoData == null)
{
string dirName = GenerateNewRepositoryDirName(repoUrl);
string dirFullPath = Lfs.PathParser.Combine(RepoDir, dirName);
Con.WriteDebug($"Clone the repository \"{repoUrl}\" to \"{dirFullPath}\" ...");
try
{
GitUtil.Clone(dirFullPath, repoUrl);
}
catch (Exception ex)
{
Con.WriteError($"GitUtil.Clone error: {ex.ToString()}");
throw;
}
Con.WriteDebug("Done.");
GitRepository gitRepo = new GitRepository(dirFullPath);
repoData = new GitGlobalFsRepository()
{
Url = repoUrl,
LocalWorkDir = dirName,
Repository = gitRepo,
};
Data.ManagedData.RepositoryList.Add(repoData);
}
else
{
repoData.Url = repoUrl;
if (repoData.Repository == null)
{
string dirName = repoData.LocalWorkDir._NullCheck();
string dirFullPath = Lfs.PathParser.Combine(RepoDir, dirName);
try
{
GitRepository gitRepo = new GitRepository(dirFullPath);
repoData.Repository = gitRepo;
}
catch (Exception ex)
{
Con.WriteDebug($"Repository local dir \"{dirFullPath}\" load error: {ex.ToString()}");
Con.WriteDebug($"Trying to clone as a new local dir.");
Data.ManagedData.RepositoryList.Remove(repoData);
Data.SyncWithStorage(HiveSyncFlags.SaveToFile, false);
repoData = null;
goto L_RETRY;
}
}
}
}
Data.SyncWithStorage(HiveSyncFlags.SaveToFile, false);
StartUpdateLoop();
}
}
public static void WriteJsonDebug(object?obj, bool includeNull = false, bool escapeHtml = false, int?maxDepth = Json.DefaultMaxDepth, bool compact = false, bool referenceHandling = false, Type?type = null) => Con.WriteDebug(obj._ObjectToJson(includeNull, escapeHtml, maxDepth, compact, referenceHandling, type: type));
public static async Task CopyFileAsync(FileSystem srcFileSystem, string srcPath, FileSystem destFileSystem, string destPath,
CopyFileParams?param = null, object?state = null, CancellationToken cancel = default, RefBool?readErrorIgnored = null)
{
if (readErrorIgnored == null)
{
readErrorIgnored = new RefBool(false);
}
readErrorIgnored.Set(false);
if (param == null)
{
param = new CopyFileParams();
}
srcPath = await srcFileSystem.NormalizePathAsync(srcPath, cancel : cancel);
destPath = await destFileSystem.NormalizePathAsync(destPath, cancel : cancel);
if (srcFileSystem == destFileSystem)
{
if (srcFileSystem.PathParser.PathStringComparer.Equals(srcPath, destPath))
{
throw new FileException(destPath, "Both source and destination is the same file.");
}
}
using (ProgressReporterBase reporter = param.ProgressReporterFactory.CreateNewReporter($"CopyFile '{srcFileSystem.PathParser.GetFileName(srcPath)}'", state))
{
using (var srcFile = await srcFileSystem.OpenAsync(srcPath, flags: param.Flags, cancel: cancel))
{
try
{
FileMetadata srcFileMetadata = await srcFileSystem.GetFileMetadataAsync(srcPath, param.MetadataCopier.OptimizedMetadataGetFlags, cancel);
bool destFileExists = await destFileSystem.IsFileExistsAsync(destPath, cancel);
using (var destFile = await destFileSystem.CreateAsync(destPath, flags: param.Flags, doNotOverwrite: !param.Overwrite, cancel: cancel))
{
try
{
reporter.ReportProgress(new ProgressData(0, srcFileMetadata.Size));
Ref <uint> srcZipCrc = new Ref <uint>();
long copiedSize = await CopyBetweenFileBaseAsync(srcFile, destFile, param, reporter, srcFileMetadata.Size, cancel, readErrorIgnored, srcZipCrc);
if (param.Flags.Bit(FileFlags.CopyFile_Verify) && param.IgnoreReadError == false)
{
// Verify を実施する
// キャッシュを無効にするため、一度ファイルを閉じて再度開く
await destFile.CloseAsync();
using (var destFile2 = await destFileSystem.OpenAsync(destPath, flags: param.Flags, cancel: cancel))
{
uint destZipCrc = await CalcZipCrc32HandleAsync(destFile2, param, cancel);
if (srcZipCrc.Value != destZipCrc)
{
// なんと Verify に失敗したぞ
throw new CoresException($"CopyFile_Verify error. Src file: '{srcPath}', Dest file: '{destPath}', srcCrc: {srcZipCrc.Value}, destCrc = {destZipCrc}");
}
}
}
reporter.ReportProgress(new ProgressData(copiedSize, copiedSize, true));
await destFile.CloseAsync();
try
{
await destFileSystem.SetFileMetadataAsync(destPath, param.MetadataCopier.Copy(srcFileMetadata), cancel);
}
catch (Exception ex)
{
Con.WriteDebug($"CopyFileAsync: '{destPath}': SetFileMetadataAsync failed. Error: {ex.Message}");
}
}
catch
{
if (destFileExists == false)
{
try
{
await destFile.CloseAsync();
}
catch { }
try
{
await destFileSystem.DeleteFileAsync(destPath);
}
catch { }
}
throw;
}
finally
{
await destFile.CloseAsync();
}
}
}
finally
{
await srcFile.CloseAsync();
}
}
}
}
async Task RealtimeRecvLoopAsync(CancellationToken cancel)
{
while (true)
{
cancel.ThrowIfCancellationRequested();
try
{
using (WebSocket ws = await Api !.RealtimeConnectAsync(cancel))
{
using (PipeStream st = ws.GetStream())
{
while (true)
{
ReadOnlyMemory <byte> recvData = await st.ReceiveAsync(cancel : cancel);
if (recvData.IsEmpty)
{
throw new DisconnectedException();
}
try
{
dynamic json = recvData._GetString_UTF8()._JsonToDynamic() !;
string realtimeStr = Json.SerializeDynamic(json);
new { Workspace = this.AccountInfoStr, DataJson = realtimeStr }._PostData("slack_realtime_log");
string channel = json !.channel;
string type = json.type;
if (type._IsSamei("message") || type._IsSamei("channel_marked") || type._IsSamei("im_marked") || type._IsSamei("group_marked"))
{
if (channel._IsFilled())
{
lock (UpdateChannelsListLock)
{
UpdateChannelsList.Add(channel);
}
UpdateChannelsEvent.Set(true);
}
}
}
catch (Exception ex)
{
Con.WriteDebug($"RealtimeRecvLoopAsync: Ignoring JSON Parse Error ({ex.Message}): '{recvData._GetString_UTF8()}'");
}
}
}
}
}
catch (Exception ex)
{
ex._Debug();
}
cancel.ThrowIfCancellationRequested();
await cancel._WaitUntilCanceledAsync(15 * 1000);
}
}