public static async Task <long> CopyBetweenFileBaseAsync(FileBase src, FileBase dest, CopyFileParams?param = null, ProgressReporterBase?reporter = null,
long estimatedSize = -1, CancellationToken cancel = default, RefBool?readErrorIgnored = null, Ref <uint>?srcZipCrc = null, long truncateSize = -1)
{
if (param == null)
{
param = new CopyFileParams();
}
if (reporter == null)
{
reporter = new NullProgressReporter(null);
}
if (readErrorIgnored == null)
{
readErrorIgnored = new RefBool();
}
if (srcZipCrc == null)
{
srcZipCrc = new Ref <uint>();
}
if (estimatedSize < 0)
{
estimatedSize = src.Size;
}
if (truncateSize >= 0)
{
estimatedSize = Math.Min(estimatedSize, truncateSize);
}
ZipCrc32 srcCrc = new ZipCrc32();
readErrorIgnored.Set(false);
checked
{
long currentPosition = 0;
if (param.IgnoreReadError)
{
long fileSize = src.Size;
int errorCounter = 0;
if (truncateSize >= 0)
{
fileSize = truncateSize; // Truncate
}
// Ignore read error mode
using (MemoryHelper.FastAllocMemoryWithUsing(param.IgnoreReadErrorSectorSize, out Memory <byte> buffer))
{
for (long pos = 0; pos < fileSize; pos += param.IgnoreReadErrorSectorSize)
{
int size = (int)Math.Min(param.IgnoreReadErrorSectorSize, (fileSize - pos));
Memory <byte> buffer2 = buffer.Slice(0, size);
int readSize = 0;
try
{
//if (pos >= 10000000 && pos <= (10000000 + 100000)) throw new ApplicationException("*err*");
readSize = await src.ReadRandomAsync(pos, buffer2, cancel);
}
catch (Exception ex)
{
errorCounter++;
if (errorCounter >= 100)
{
// Skip error display
if (errorCounter == 100)
{
Con.WriteError($"The read error counter of the file \"{src.FileParams.Path}\" exceeds 100. No more errors will be reported.");
}
}
else
{
// Display the error
Con.WriteError($"Ignoring the read error at the offset {pos} of the file \"{src.FileParams.Path}\". Error: {ex.Message}");
}
readErrorIgnored.Set(true);
}
if (readSize >= 1)
{
await dest.WriteRandomAsync(pos, buffer2.Slice(0, readSize), cancel);
}
currentPosition = pos + readSize;
reporter.ReportProgress(new ProgressData(currentPosition, fileSize));
}
}
}
else if (param.AsyncCopy == false)
{
// Normal copy
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer))
{
while (true)
{
Memory <byte> thisTimeBuffer = buffer;
if (truncateSize >= 0)
{
// Truncate
long remainSize = Math.Max(truncateSize - currentPosition, 0);
if (thisTimeBuffer.Length > remainSize)
{
thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
}
if (remainSize == 0)
{
break;
}
}
int readSize = await src.ReadAsync(thisTimeBuffer, cancel);
Debug.Assert(readSize <= thisTimeBuffer.Length);
if (readSize <= 0)
{
break;
}
ReadOnlyMemory <byte> sliced = thisTimeBuffer.Slice(0, readSize);
if (param.Flags.Bit(FileFlags.CopyFile_Verify))
{
srcCrc.Append(sliced.Span);
}
await dest.WriteAsync(sliced, cancel);
currentPosition += readSize;
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
}
}
else
{
// Async copy
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer1))
{
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer2))
{
Task?lastWriteTask = null;
int number = 0;
int writeSize = 0;
long currentReadPosition = 0;
Memory <byte>[] buffers = new Memory <byte>[2] {
buffer1, buffer2
};
while (true)
{
Memory <byte> buffer = buffers[(number++) % 2];
Memory <byte> thisTimeBuffer = buffer;
if (truncateSize >= 0)
{
// Truncate
long remainSize = Math.Max(truncateSize - currentReadPosition, 0);
if (thisTimeBuffer.Length > remainSize)
{
thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
}
}
int readSize = await src.ReadAsync(thisTimeBuffer, cancel);
Debug.Assert(readSize <= buffer.Length);
if (lastWriteTask != null)
{
await lastWriteTask;
currentPosition += writeSize;
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
if (readSize <= 0)
{
break;
}
currentReadPosition += readSize;
writeSize = readSize;
ReadOnlyMemory <byte> sliced = buffer.Slice(0, writeSize);
if (param.Flags.Bit(FileFlags.CopyFile_Verify))
{
srcCrc.Append(sliced.Span);
}
lastWriteTask = dest.WriteAsync(sliced, cancel);
}
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
}
}
srcZipCrc.Set(srcCrc.Value);
return(currentPosition);
}
}
public static void WriteJsonDebug(object?obj, bool includeNull = false, bool escapeHtml = false, int?maxDepth = Json.DefaultMaxDepth, bool compact = false, bool referenceHandling = false, Type?type = null, JsonFlags jsonFlags = JsonFlags.None) => Con.WriteDebug(obj._ObjectToJson(includeNull, escapeHtml, maxDepth, compact, referenceHandling, type: type, jsonFlags: jsonFlags));
public static async Task CopyFileAsync(FileSystem srcFileSystem, string srcPath, FileSystem destFileSystem, string destPath,
CopyFileParams?param = null, object?state = null, CancellationToken cancel = default, RefBool?readErrorIgnored = null)
{
if (readErrorIgnored == null)
{
readErrorIgnored = new RefBool(false);
}
readErrorIgnored.Set(false);
if (param == null)
{
param = new CopyFileParams();
}
srcPath = await srcFileSystem.NormalizePathAsync(srcPath, cancel : cancel);
destPath = await destFileSystem.NormalizePathAsync(destPath, cancel : cancel);
if (srcFileSystem == destFileSystem)
{
if (srcFileSystem.PathParser.PathStringComparer.Equals(srcPath, destPath))
{
throw new FileException(destPath, "Both source and destination is the same file.");
}
}
using (ProgressReporterBase reporter = param.ProgressReporterFactory.CreateNewReporter($"CopyFile '{srcFileSystem.PathParser.GetFileName(srcPath)}'", state))
{
using (var srcFile = await srcFileSystem.OpenAsync(srcPath, flags: param.Flags, cancel: cancel))
{
try
{
FileMetadata srcFileMetadata = await srcFileSystem.GetFileMetadataAsync(srcPath, param.MetadataCopier.OptimizedMetadataGetFlags, cancel);
bool destFileExists = await destFileSystem.IsFileExistsAsync(destPath, cancel);
using (var destFile = await destFileSystem.CreateAsync(destPath, flags: param.Flags, doNotOverwrite: !param.Overwrite, cancel: cancel))
{
try
{
reporter.ReportProgress(new ProgressData(0, srcFileMetadata.Size));
Ref <uint> srcZipCrc = new Ref <uint>();
long copiedSize = await CopyBetweenFileBaseAsync(srcFile, destFile, param, reporter, srcFileMetadata.Size, cancel, readErrorIgnored, srcZipCrc);
if (param.Flags.Bit(FileFlags.CopyFile_Verify) && param.IgnoreReadError == false)
{
// Verify を実施する
// キャッシュを無効にするため、一度ファイルを閉じて再度開く
await destFile.CloseAsync();
using (var destFile2 = await destFileSystem.OpenAsync(destPath, flags: param.Flags, cancel: cancel))
{
uint destZipCrc = await CalcZipCrc32HandleAsync(destFile2, param, cancel);
if (srcZipCrc.Value != destZipCrc)
{
// なんと Verify に失敗したぞ
throw new CoresException($"CopyFile_Verify error. Src file: '{srcPath}', Dest file: '{destPath}', srcCrc: {srcZipCrc.Value}, destCrc = {destZipCrc}");
}
}
}
reporter.ReportProgress(new ProgressData(copiedSize, copiedSize, true));
await destFile.CloseAsync();
try
{
await destFileSystem.SetFileMetadataAsync(destPath, param.MetadataCopier.Copy(srcFileMetadata), cancel);
}
catch (Exception ex)
{
Con.WriteDebug($"CopyFileAsync: '{destPath}': SetFileMetadataAsync failed. Error: {ex.Message}");
}
}
catch
{
if (destFileExists == false)
{
try
{
await destFile.CloseAsync();
}
catch { }
try
{
await destFileSystem.DeleteFileAsync(destPath);
}
catch { }
}
throw;
}
finally
{
await destFile.CloseAsync();
}
}
}
finally
{
await srcFile.CloseAsync();
}
}
}
}
async Task RealtimeRecvLoopAsync(CancellationToken cancel)
{
while (true)
{
cancel.ThrowIfCancellationRequested();
try
{
using (WebSocket ws = await Api !.RealtimeConnectAsync(cancel))
{
using (PipeStream st = ws.GetStream())
{
while (true)
{
ReadOnlyMemory <byte> recvData = await st.ReceiveAsync(cancel : cancel);
if (recvData.IsEmpty)
{
throw new DisconnectedException();
}
try
{
dynamic json = recvData._GetString_UTF8()._JsonToDynamic() !;
string realtimeStr = Json.SerializeDynamic(json);
new { Workspace = this.AccountInfoStr, DataJson = realtimeStr }._PostData("slack_realtime_log");
string channel = json !.channel;
string type = json.type;
if (type._IsSamei("message") || type._IsSamei("channel_marked") || type._IsSamei("im_marked") || type._IsSamei("group_marked"))
{
if (channel._IsFilled())
{
lock (UpdateChannelsListLock)
{
UpdateChannelsList.Add(channel);
}
UpdateChannelsEvent.Set(true);
}
}
}
catch (Exception ex)
{
Con.WriteDebug($"RealtimeRecvLoopAsync: Ignoring JSON Parse Error ({ex.Message}): '{recvData._GetString_UTF8()}'");
}
}
}
}
}
catch (Exception ex)
{
ex._Debug();
}
cancel.ThrowIfCancellationRequested();
await cancel._WaitUntilCanceledAsync(15 * 1000);
}
}