示例#1
0
        public static async Task <long> CopyBetweenFileBaseAsync(FileBase src, FileBase dest, CopyFileParams?param = null, ProgressReporterBase?reporter = null,
                                                                 long estimatedSize = -1, CancellationToken cancel = default, RefBool?readErrorIgnored = null, Ref <uint>?srcZipCrc = null, long truncateSize = -1)
        {
            if (param == null)
            {
                param = new CopyFileParams();
            }
            if (reporter == null)
            {
                reporter = new NullProgressReporter(null);
            }
            if (readErrorIgnored == null)
            {
                readErrorIgnored = new RefBool();
            }
            if (srcZipCrc == null)
            {
                srcZipCrc = new Ref <uint>();
            }
            if (estimatedSize < 0)
            {
                estimatedSize = src.Size;
            }

            if (truncateSize >= 0)
            {
                estimatedSize = Math.Min(estimatedSize, truncateSize);
            }

            ZipCrc32 srcCrc = new ZipCrc32();

            readErrorIgnored.Set(false);

            checked
            {
                long currentPosition = 0;

                if (param.IgnoreReadError)
                {
                    long fileSize     = src.Size;
                    int  errorCounter = 0;

                    if (truncateSize >= 0)
                    {
                        fileSize = truncateSize; // Truncate
                    }
                    // Ignore read error mode
                    using (MemoryHelper.FastAllocMemoryWithUsing(param.IgnoreReadErrorSectorSize, out Memory <byte> buffer))
                    {
                        for (long pos = 0; pos < fileSize; pos += param.IgnoreReadErrorSectorSize)
                        {
                            int           size    = (int)Math.Min(param.IgnoreReadErrorSectorSize, (fileSize - pos));
                            Memory <byte> buffer2 = buffer.Slice(0, size);

                            int readSize = 0;

                            try
                            {
                                //if (pos >= 10000000 && pos <= (10000000 + 100000)) throw new ApplicationException("*err*");
                                readSize = await src.ReadRandomAsync(pos, buffer2, cancel);
                            }
                            catch (Exception ex)
                            {
                                errorCounter++;
                                if (errorCounter >= 100)
                                {
                                    // Skip error display
                                    if (errorCounter == 100)
                                    {
                                        Con.WriteError($"The read error counter of the file \"{src.FileParams.Path}\" exceeds 100. No more errors will be reported.");
                                    }
                                }
                                else
                                {
                                    // Display the error
                                    Con.WriteError($"Ignoring the read error at the offset {pos} of the file \"{src.FileParams.Path}\". Error: {ex.Message}");
                                }
                                readErrorIgnored.Set(true);
                            }

                            if (readSize >= 1)
                            {
                                await dest.WriteRandomAsync(pos, buffer2.Slice(0, readSize), cancel);
                            }

                            currentPosition = pos + readSize;
                            reporter.ReportProgress(new ProgressData(currentPosition, fileSize));
                        }
                    }
                }
                else if (param.AsyncCopy == false)
                {
                    // Normal copy
                    using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer))
                    {
                        while (true)
                        {
                            Memory <byte> thisTimeBuffer = buffer;

                            if (truncateSize >= 0)
                            {
                                // Truncate
                                long remainSize = Math.Max(truncateSize - currentPosition, 0);

                                if (thisTimeBuffer.Length > remainSize)
                                {
                                    thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
                                }

                                if (remainSize == 0)
                                {
                                    break;
                                }
                            }

                            int readSize = await src.ReadAsync(thisTimeBuffer, cancel);

                            Debug.Assert(readSize <= thisTimeBuffer.Length);

                            if (readSize <= 0)
                            {
                                break;
                            }

                            ReadOnlyMemory <byte> sliced = thisTimeBuffer.Slice(0, readSize);

                            if (param.Flags.Bit(FileFlags.CopyFile_Verify))
                            {
                                srcCrc.Append(sliced.Span);
                            }

                            await dest.WriteAsync(sliced, cancel);

                            currentPosition += readSize;
                            reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
                        }
                    }
                }
                else
                {
                    // Async copy
                    using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer1))
                    {
                        using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer2))
                        {
                            Task?lastWriteTask = null;
                            int  number        = 0;
                            int  writeSize     = 0;

                            long currentReadPosition = 0;

                            Memory <byte>[] buffers = new Memory <byte>[2] {
                                buffer1, buffer2
                            };

                            while (true)
                            {
                                Memory <byte> buffer = buffers[(number++) % 2];

                                Memory <byte> thisTimeBuffer = buffer;

                                if (truncateSize >= 0)
                                {
                                    // Truncate
                                    long remainSize = Math.Max(truncateSize - currentReadPosition, 0);

                                    if (thisTimeBuffer.Length > remainSize)
                                    {
                                        thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
                                    }
                                }

                                int readSize = await src.ReadAsync(thisTimeBuffer, cancel);

                                Debug.Assert(readSize <= buffer.Length);

                                if (lastWriteTask != null)
                                {
                                    await lastWriteTask;
                                    currentPosition += writeSize;
                                    reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
                                }

                                if (readSize <= 0)
                                {
                                    break;
                                }

                                currentReadPosition += readSize;

                                writeSize = readSize;

                                ReadOnlyMemory <byte> sliced = buffer.Slice(0, writeSize);

                                if (param.Flags.Bit(FileFlags.CopyFile_Verify))
                                {
                                    srcCrc.Append(sliced.Span);
                                }

                                lastWriteTask = dest.WriteAsync(sliced, cancel);
                            }

                            reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
                        }
                    }
                }

                srcZipCrc.Set(srcCrc.Value);

                return(currentPosition);
            }
        }
示例#2
0
 public static void WriteJsonDebug(object?obj, bool includeNull = false, bool escapeHtml = false, int?maxDepth = Json.DefaultMaxDepth, bool compact = false, bool referenceHandling = false, Type?type = null, JsonFlags jsonFlags = JsonFlags.None)
 => Con.WriteDebug(obj._ObjectToJson(includeNull, escapeHtml, maxDepth, compact, referenceHandling, type: type, jsonFlags: jsonFlags));
示例#3
0
        public static async Task CopyFileAsync(FileSystem srcFileSystem, string srcPath, FileSystem destFileSystem, string destPath,
                                               CopyFileParams?param = null, object?state = null, CancellationToken cancel = default, RefBool?readErrorIgnored = null)
        {
            if (readErrorIgnored == null)
            {
                readErrorIgnored = new RefBool(false);
            }

            readErrorIgnored.Set(false);

            if (param == null)
            {
                param = new CopyFileParams();
            }

            srcPath = await srcFileSystem.NormalizePathAsync(srcPath, cancel : cancel);

            destPath = await destFileSystem.NormalizePathAsync(destPath, cancel : cancel);

            if (srcFileSystem == destFileSystem)
            {
                if (srcFileSystem.PathParser.PathStringComparer.Equals(srcPath, destPath))
                {
                    throw new FileException(destPath, "Both source and destination is the same file.");
                }
            }

            using (ProgressReporterBase reporter = param.ProgressReporterFactory.CreateNewReporter($"CopyFile '{srcFileSystem.PathParser.GetFileName(srcPath)}'", state))
            {
                using (var srcFile = await srcFileSystem.OpenAsync(srcPath, flags: param.Flags, cancel: cancel))
                {
                    try
                    {
                        FileMetadata srcFileMetadata = await srcFileSystem.GetFileMetadataAsync(srcPath, param.MetadataCopier.OptimizedMetadataGetFlags, cancel);

                        bool destFileExists = await destFileSystem.IsFileExistsAsync(destPath, cancel);

                        using (var destFile = await destFileSystem.CreateAsync(destPath, flags: param.Flags, doNotOverwrite: !param.Overwrite, cancel: cancel))
                        {
                            try
                            {
                                reporter.ReportProgress(new ProgressData(0, srcFileMetadata.Size));

                                Ref <uint> srcZipCrc = new Ref <uint>();

                                long copiedSize = await CopyBetweenFileBaseAsync(srcFile, destFile, param, reporter, srcFileMetadata.Size, cancel, readErrorIgnored, srcZipCrc);

                                if (param.Flags.Bit(FileFlags.CopyFile_Verify) && param.IgnoreReadError == false)
                                {
                                    // Verify を実施する
                                    // キャッシュを無効にするため、一度ファイルを閉じて再度開く

                                    await destFile.CloseAsync();

                                    using (var destFile2 = await destFileSystem.OpenAsync(destPath, flags: param.Flags, cancel: cancel))
                                    {
                                        uint destZipCrc = await CalcZipCrc32HandleAsync(destFile2, param, cancel);

                                        if (srcZipCrc.Value != destZipCrc)
                                        {
                                            // なんと Verify に失敗したぞ
                                            throw new CoresException($"CopyFile_Verify error. Src file: '{srcPath}', Dest file: '{destPath}', srcCrc: {srcZipCrc.Value}, destCrc = {destZipCrc}");
                                        }
                                    }
                                }

                                reporter.ReportProgress(new ProgressData(copiedSize, copiedSize, true));

                                await destFile.CloseAsync();

                                try
                                {
                                    await destFileSystem.SetFileMetadataAsync(destPath, param.MetadataCopier.Copy(srcFileMetadata), cancel);
                                }
                                catch (Exception ex)
                                {
                                    Con.WriteDebug($"CopyFileAsync: '{destPath}': SetFileMetadataAsync failed. Error: {ex.Message}");
                                }
                            }
                            catch
                            {
                                if (destFileExists == false)
                                {
                                    try
                                    {
                                        await destFile.CloseAsync();
                                    }
                                    catch { }

                                    try
                                    {
                                        await destFileSystem.DeleteFileAsync(destPath);
                                    }
                                    catch { }
                                }

                                throw;
                            }
                            finally
                            {
                                await destFile.CloseAsync();
                            }
                        }
                    }
                    finally
                    {
                        await srcFile.CloseAsync();
                    }
                }
            }
        }
示例#4
0
        async Task RealtimeRecvLoopAsync(CancellationToken cancel)
        {
            while (true)
            {
                cancel.ThrowIfCancellationRequested();

                try
                {
                    using (WebSocket ws = await Api !.RealtimeConnectAsync(cancel))
                    {
                        using (PipeStream st = ws.GetStream())
                        {
                            while (true)
                            {
                                ReadOnlyMemory <byte> recvData = await st.ReceiveAsync(cancel : cancel);

                                if (recvData.IsEmpty)
                                {
                                    throw new DisconnectedException();
                                }

                                try
                                {
                                    dynamic json = recvData._GetString_UTF8()._JsonToDynamic() !;

                                    string realtimeStr = Json.SerializeDynamic(json);

                                    new { Workspace = this.AccountInfoStr, DataJson = realtimeStr }._PostData("slack_realtime_log");

                                    string channel = json !.channel;
                                    string type    = json.type;

                                    if (type._IsSamei("message") || type._IsSamei("channel_marked") || type._IsSamei("im_marked") || type._IsSamei("group_marked"))
                                    {
                                        if (channel._IsFilled())
                                        {
                                            lock (UpdateChannelsListLock)
                                            {
                                                UpdateChannelsList.Add(channel);
                                            }

                                            UpdateChannelsEvent.Set(true);
                                        }
                                    }
                                }
                                catch (Exception ex)
                                {
                                    Con.WriteDebug($"RealtimeRecvLoopAsync: Ignoring JSON Parse Error ({ex.Message}): '{recvData._GetString_UTF8()}'");
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    ex._Debug();
                }

                cancel.ThrowIfCancellationRequested();

                await cancel._WaitUntilCanceledAsync(15 * 1000);
            }
        }