/// <summary> /// Instantiates an instance of <c>ManualRequest</c>. /// </summary> /// <param name="paths">The list of paths to compare and preview.</param> /// <param name="filters">The list of filters to pass in.</param> /// <param name="syncConfig">The sync configuration to pass in.</param> /// <param name="tagName">The tag name to compare and preview.</param> protected ManualRequest(string[] paths, List<Filter> filters, SyncConfig syncConfig, string tagName) { _paths = paths; _filters = filters; _syncConfig = syncConfig; _tagName = tagName; }
/// <summary> /// Instantiates an instance of <c>AutoSyncRequest</c> for delete purposes. /// </summary> /// <param name="sourceName">The name of the source file or folder.</param> /// <param name="sourceParent">The parent of the source file or folder.</param> /// <param name="destinations">The list of destinations to propagate the source to.</param> /// <param name="requestType">The type of request, eg. create, update, delete or rename.</param> /// <param name="syncConfig">The sync configuration passed in.</param> /// <param name="tagList">The list of tags related to this request.</param> public AutoSyncRequest(string sourceName, string sourceParent, List<string> destinations, AutoSyncRequestType requestType, SyncConfig syncConfig, List<string> tagList) { _sourceName = sourceName; _sourceParent = sourceParent; _destinations = destinations; _requestType = requestType; _syncConfig = syncConfig; _tagList = tagList; }
public void Best_state_is_suggested_if_there_is_suggested_block_with_state() { IBlockTree blockTree = Substitute.For <IBlockTree>(); IReceiptStorage receiptStorage = Substitute.For <IReceiptStorage>(); IDb stateDb = Substitute.For <IDb>(); SyncConfig syncConfig = new SyncConfig(); syncConfig.PivotNumber = "1"; SyncProgressResolver syncProgressResolver = new SyncProgressResolver(blockTree, receiptStorage, stateDb, new MemDb(), syncConfig, LimboLogs.Instance); var head = Build.A.Block.WithHeader(Build.A.BlockHeader.WithNumber(5).WithStateRoot(TestItem.KeccakA).TestObject).TestObject; var suggested = Build.A.BlockHeader.WithNumber(6).WithStateRoot(TestItem.KeccakB).TestObject; blockTree.Head.Returns(head); blockTree.BestSuggestedHeader.Returns(suggested); blockTree.FindHeader(Arg.Any <Keccak>(), BlockTreeLookupOptions.TotalDifficultyNotNeeded).Returns(head.Header); stateDb.Get(head.StateRoot).Returns(new byte[] { 1 }); stateDb.Get(suggested.StateRoot).Returns(new byte[] { 1 }); Assert.AreEqual(suggested.Number, syncProgressResolver.FindBestFullState()); }
public void Best_state_is_suggested_if_there_is_suggested_block_with_state() { IBlockTree blockTree = Substitute.For <IBlockTree>(); IReceiptStorage receiptStorage = Substitute.For <IReceiptStorage>(); INodeDataDownloader nodeDataDownloader = Substitute.For <INodeDataDownloader>(); SyncConfig syncConfig = new SyncConfig(); syncConfig.PivotNumber = "1"; SyncProgressResolver syncProgressResolver = new SyncProgressResolver(blockTree, receiptStorage, nodeDataDownloader, syncConfig, LimboLogs.Instance); var head = Build.A.BlockHeader.WithNumber(5).TestObject; var suggested = Build.A.BlockHeader.WithNumber(6).TestObject; blockTree.Head.Returns(head); blockTree.BestSuggestedHeader.Returns(suggested); blockTree.FindHeader(Arg.Any <Keccak>(), BlockTreeLookupOptions.TotalDifficultyNotNeeded).Returns(head); nodeDataDownloader.IsFullySynced(head).Returns(true); nodeDataDownloader.IsFullySynced(suggested).Returns(true); Assert.AreEqual(suggested.Number, syncProgressResolver.FindBestFullState()); }
public void Is_fast_block_finished_returns_false_when_receipts_not_downloaded() { IBlockTree blockTree = Substitute.For <IBlockTree>(); IReceiptStorage receiptStorage = Substitute.For <IReceiptStorage>(); INodeDataDownloader nodeDataDownloader = Substitute.For <INodeDataDownloader>(); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastBlocks = true; syncConfig.DownloadBodiesInFastSync = true; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).TestObject); blockTree.LowestInsertedBody.Returns(Build.A.Block.WithNumber(1).TestObject); receiptStorage.LowestInsertedReceiptBlock.Returns(2); SyncProgressResolver syncProgressResolver = new SyncProgressResolver(blockTree, receiptStorage, nodeDataDownloader, syncConfig, LimboLogs.Instance); Assert.False(syncProgressResolver.IsFastBlocksFinished()); }
public void Is_fast_block_finished_returns_true_when_bodies_not_downloaded_and_we_do_not_want_to_download_bodies() { IBlockTree blockTree = Substitute.For <IBlockTree>(); IReceiptStorage receiptStorage = Substitute.For <IReceiptStorage>(); IDb stateDb = new MemDb(); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastBlocks = true; syncConfig.DownloadBodiesInFastSync = false; syncConfig.DownloadReceiptsInFastSync = true; syncConfig.PivotNumber = "1"; blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); blockTree.LowestInsertedBody.Returns(Build.A.Block.WithNumber(2).WithStateRoot(TestItem.KeccakB).TestObject); receiptStorage.LowestInsertedReceiptBlock.Returns(1); SyncProgressResolver syncProgressResolver = new SyncProgressResolver(blockTree, receiptStorage, stateDb, new MemDb(), syncConfig, LimboLogs.Instance); Assert.True(syncProgressResolver.IsFastBlocksFinished()); }
public async Task Setup() { _genesisBlock = Build.A.Block.WithNumber(0).TestObject; _blockTree = Build.A.BlockTree(_genesisBlock).OfChainLength(1).TestObject; IDbProvider dbProvider = await TestMemDbProvider.InitAsync(); _stateDb = dbProvider.StateDb; _codeDb = dbProvider.CodeDb; _receiptStorage = Substitute.For <IReceiptStorage>(); SyncConfig quickConfig = new SyncConfig(); quickConfig.FastSync = false; var stats = new NodeStatsManager(LimboLogs.Instance); _pool = new SyncPeerPool(_blockTree, stats, 25, LimboLogs.Instance); SyncConfig syncConfig = new SyncConfig(); SyncProgressResolver resolver = new SyncProgressResolver( _blockTree, _receiptStorage, _stateDb, dbProvider.BeamTempDb, new TrieStore(_stateDb, LimboLogs.Instance), syncConfig, LimboLogs.Instance); MultiSyncModeSelector syncModeSelector = new MultiSyncModeSelector(resolver, _pool, syncConfig, LimboLogs.Instance); _synchronizer = new Synchronizer(dbProvider, MainnetSpecProvider.Instance, _blockTree, _receiptStorage, Always.Valid, Always.Valid, _pool, stats, syncModeSelector, syncConfig, LimboLogs.Instance); _syncServer = new SyncServer( _stateDb, _codeDb, _blockTree, _receiptStorage, Always.Valid, Always.Valid, _pool, syncModeSelector, quickConfig, new WitnessCollector(new MemDb(), LimboLogs.Instance), LimboLogs.Instance); }
public async Task Finishes_when_all_downloaded() { IBlockTree blockTree = Substitute.For <IBlockTree>(); blockTree.LowestInsertedBeaconHeader.Returns(Build.A.BlockHeader.WithNumber(2000).TestObject); ISyncReport report = Substitute.For <ISyncReport>(); report.HeadersInQueue.Returns(new MeasuredProgress()); MeasuredProgress measuredProgress = new (); report.BeaconHeaders.Returns(measuredProgress); ISyncConfig syncConfig = new SyncConfig { FastSync = true, FastBlocks = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }; PoSSwitcher poSSwitcher = new(new MergeConfig(), syncConfig, new MemDb(), blockTree !, MainnetSpecProvider.Instance, LimboLogs.Instance); IBeaconPivot pivot = PreparePivot(2000, syncConfig, blockTree); BeaconHeadersSyncFeed feed = new (poSSwitcher, Substitute.For <ISyncModeSelector>(), blockTree, Substitute.For <ISyncPeerPool>(), syncConfig, report, pivot, new MergeConfig() { Enabled = true }, new NoopInvalidChainTracker(), LimboLogs.Instance); feed.InitializeFeed(); for (int i = 0; i < 6; i++) { await feed.PrepareRequest(); } blockTree.LowestInsertedBeaconHeader.Returns(Build.A.BlockHeader.WithNumber(1001).TestObject); HeadersSyncBatch?result = await feed.PrepareRequest(); result.Should().BeNull(); feed.CurrentState.Should().Be(SyncFeedState.Dormant); measuredProgress.CurrentValue.Should().Be(999); }
private SafeContext PrepareDownloader(DbContext dbContext, ISyncPeer syncPeer) { SafeContext ctx = new SafeContext(); ctx = new SafeContext(); BlockTree blockTree = Build.A.BlockTree().OfChainLength((int)BlockTree.BestSuggestedHeader.Number).TestObject; ctx.Pool = new SyncPeerPool(blockTree, new NodeStatsManager(new StatsConfig(), LimboLogs.Instance), 25, LimboLogs.Instance); ctx.Pool.Start(); ctx.Pool.AddPeer(syncPeer); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = true; ctx.SyncModeSelector = StaticSelector.StateNodesWithFastBlocks; ctx.Feed = new StateSyncFeed(dbContext.LocalCodeDb, dbContext.LocalStateDb, new MemDb(), ctx.SyncModeSelector, blockTree, _logManager); ctx.StateSyncDispatcher = new StateSyncDispatcher(ctx.Feed, ctx.Pool, new StateSyncAllocationStrategyFactory(), _logManager); ctx.StateSyncDispatcher.Start(CancellationToken.None); return(ctx); }
public void Setup() { _genesisBlock = Build.A.Block.WithNumber(0).TestObject; _blockTree = Build.A.BlockTree(_genesisBlock).OfChainLength(1).TestObject; _stateDb = new StateDb(); _codeDb = new StateDb(); _receiptsDb = new MemDb(); _receiptStorage = Substitute.For <IReceiptStorage>(); SyncConfig quickConfig = new SyncConfig(); quickConfig.FastSync = false; ISealValidator sealValidator = Build.A.SealValidator.ThatAlwaysReturnsTrue.TestObject; IBlockValidator blockValidator = Build.A.BlockValidator.ThatAlwaysReturnsTrue.TestObject; ITxValidator txValidator = Build.A.TransactionValidator.ThatAlwaysReturnsTrue.TestObject; var stats = new NodeStatsManager(new StatsConfig(), LimboLogs.Instance); _pool = new EthSyncPeerPool(_blockTree, stats, quickConfig, 25, LimboLogs.Instance); _synchronizer = new Synchronizer(MainNetSpecProvider.Instance, _blockTree, NullReceiptStorage.Instance, blockValidator, sealValidator, _pool, quickConfig, Substitute.For <INodeDataDownloader>(), LimboLogs.Instance); _syncServer = new SyncServer(_stateDb, _codeDb, _blockTree, _receiptStorage, sealValidator, _pool, _synchronizer, quickConfig, LimboLogs.Instance); }
public SyncingContext(SynchronizerType synchronizerType) { _logger = _logManager.GetClassLogger(); ISyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = synchronizerType == SynchronizerType.Fast; ISnapshotableDb stateDb = new StateDb(); ISnapshotableDb codeDb = new StateDb(); MemDb blockInfoDb = new MemDb(); BlockTree = new BlockTree(new MemDb(), new MemDb(), blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), new SingleReleaseSpecProvider(Constantinople.Instance, 1), NullTxPool.Instance, NullBloomStorage.Instance, _logManager); NodeStatsManager stats = new NodeStatsManager(new StatsConfig(), _logManager); SyncPeerPool = new EthSyncPeerPool(BlockTree, stats, 25, _logManager); NodeDataFeed feed = new NodeDataFeed(codeDb, stateDb, _logManager); NodeDataDownloader nodeDataDownloader = new NodeDataDownloader(SyncPeerPool, feed, NullDataConsumer.Instance, _logManager); Synchronizer = new Synchronizer( MainNetSpecProvider.Instance, BlockTree, NullReceiptStorage.Instance, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, SyncPeerPool, syncConfig, nodeDataDownloader, stats, _logManager); SyncServer = new SyncServer(stateDb, codeDb, BlockTree, NullReceiptStorage.Instance, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, SyncPeerPool, Synchronizer, syncConfig, _logManager); SyncPeerPool.Start(); Synchronizer.Start(); Synchronizer.SyncEvent += SynchronizerOnSyncEvent; AllInstances.Add(this); }
public void Can_keep_changing_in_fast_sync() { IEthSyncPeerPool syncPeerPool = Substitute.For <IEthSyncPeerPool>(); ISyncPeer syncPeer = Substitute.For <ISyncPeer>(); syncPeer.TotalDifficultyOnSessionStart.Returns((UInt256)(1024 * 1024)); PeerInfo peerInfo1 = new PeerInfo(syncPeer) { HeadNumber = 0, IsInitialized = true }; syncPeerPool.AllPeers.Returns(new[] { peerInfo1 }); syncPeerPool.UsefulPeers.Returns(new[] { peerInfo1 }); syncPeerPool.PeerCount.Returns(1); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = true; SyncModeSelector selector = new SyncModeSelector(syncPeerPool, syncConfig, LimboLogs.Instance); Assert.AreEqual(SyncMode.Headers, selector.Current); (long BestRemote, long BestLocalHeader, long BestLocalFullBlock, long BestLocalState, SyncMode ExpectedState, string Description)[] states =
private void Load() { XmlSerializer deserializer = new XmlSerializer(typeof(SyncConfig)); using (TextReader textReader = new StreamReader(FullPath)) { this.configXml = (SyncConfig)deserializer.Deserialize(textReader); } }
public T GetRequestProcessor <T>(SyncConfig config, string typeName) => GetProvivder <T>(config.Providers.Request.Processor, typeName);
/// <summary> /// Instantiates an instance of <c>AutoSyncRequest</c> for create and update purposes. /// </summary> /// <param name="sourceName">The name of the source file or folder.</param> /// <param name="sourceParent">The parent of the source file or folder.</param> /// <param name="destinations">The list of destinations to propagate the source to.</param> /// <param name="isFolder"></param> /// <param name="requestType">The type of request, eg. create, update, delete or rename.</param> /// <param name="syncConfig">The sync configuration passed in.</param> /// <param name="tagList">The list of tags related to this request.</param> public AutoSyncRequest(string sourceName, string sourceParent, List<string> destinations, bool? isFolder, AutoSyncRequestType requestType, SyncConfig syncConfig, List<string> tagList) : this(sourceName, sourceParent, destinations, requestType, syncConfig, tagList) { _isFolder = isFolder; _tagList = tagList; }
static void Main(string[] args) { //TestSpaceSizes(); Console.WriteLine("Starting the FileSync app... \n" + "If you want to stop its execution at any time, please, press CTRL+C"); Console.CancelKeyPress += (sender, e) => { Console.WriteLine("\n\n\n\n\nExiting..."); if (Console.ReadKey() != null) { Environment.Exit(0); } }; SyncConfig confInstance = null; try { Console.WriteLine("Initializing configuration..."); confInstance = new SyncConfig(); //string LogFolder = confInstance.Parameters["LogFolder"]; //MappingCsvFileName = confInstance.Parameters["FileMappingFile"]; Console.WriteLine("Folders for synchronization are following"); foreach (var entry in confInstance.FolderMappings) { Console.WriteLine(entry.Value.Item1 + " <=> " + entry.Value.Item2); } Console.WriteLine("Done"); } catch (Exception ex) { string mes = ex.Message; ExitApp("Could not initialize configuration. Check error log file for details \n" + mes); } var syncExec = new SyncExecution(confInstance); if (syncExec.SyncConfig.FolderMappings.Count == 0) { ExitApp("No folder mappings found in config file, exiting"); } // initialize source and destination files: try { Init.InitializeFiles(syncExec); } catch (Exception e) { ExitApp("Could not initialize files. \n" + e.Message); } // remove duplicates if this is configured: try { if (confInstance.Parameters["RemoveDuplicates"] == "yes") { DuplicatesHandling.RemoveDuplicates(syncExec); } } catch (Exception e) { ExitApp("Could not remove duplicates. \n" + e.Message); } try { // retrieve existing file mapping from CSV // and, if necessary, create additional mapping from paths Init.MapFiles(syncExec); } catch (Exception e) { ExitApp("Could not map files. \n" + e.Message); } try { syncExec.AppendActionListWithUpdateCreateMove(); } catch (Exception e) { ExitApp("Could not complete the stage of identifying updates, creations and moves. \n" + e.Message); } if (syncExec.AnyChangesNeeded) { SyncHelper.WriteActionsListToLog(syncExec); SyncHelper.PreviewChangesStats(syncExec); string proceedWithSync = ""; while (proceedWithSync != "yes" && proceedWithSync != "no") { Console.WriteLine("\nWould you like to perform synchronization (yes/no)?"); proceedWithSync = Console.ReadLine(); if (proceedWithSync == "yes") { syncExec.PerformActions(); SyncHelper.WriteFailedActionsToLog(syncExec); CSVHelper.SaveFileMappingToCsv(syncExec); } else if (proceedWithSync == "no") { Console.WriteLine("The synchronization has been cancelled, exiting the app"); Thread.Sleep(1500); Environment.Exit(0); } } if (syncExec.FailedActions.Count > 0) { Console.WriteLine("\nSome actions failed: for details, see the error log file -\n" + syncExec.SyncConfig.ErrorLogFile); } else { if (proceedWithSync == "yes") { Console.WriteLine("\n\nExecution completed successfully."); } } ExitApp(""); } else { CSVHelper.SaveFileMappingToCsv(syncExec); ExitApp("No changes needed"); } }
public SyncRunResult(SyncConfig syncConfig) { SyncConfig = syncConfig; }
public CleanupRunResult(SyncConfig syncConfig) { SyncConfig = syncConfig; }
public void Setup() { _localReceiptStorage = new InMemoryReceiptStorage(); _syncPeers = new List <LatencySyncPeerMock>(); _peerTrees = new Dictionary <LatencySyncPeerMock, IBlockTree>(); _peerMaxResponseSizes = new Dictionary <LatencySyncPeerMock, int>(); _pendingResponses = new Dictionary <LatencySyncPeerMock, FastBlocksBatch>(); _invalidBlocks = new Dictionary <LatencySyncPeerMock, HashSet <long> >(); _maliciousByRepetition = new HashSet <LatencySyncPeerMock>(); _maliciousByInvalidTxs = new HashSet <LatencySyncPeerMock>(); _maliciousByInvalidOmmers = new HashSet <LatencySyncPeerMock>(); _maliciousByShiftedOneForward = new HashSet <LatencySyncPeerMock>(); _maliciousByShiftedOneBack = new HashSet <LatencySyncPeerMock>(); _maliciousByShortAtStart = new HashSet <LatencySyncPeerMock>(); _maliciousByInvalidReceipts = new HashSet <LatencySyncPeerMock>(); _incorrectByTooShortMessages = new HashSet <LatencySyncPeerMock>(); _incorrectByTooLongMessages = new HashSet <LatencySyncPeerMock>(); _timingOut = new HashSet <LatencySyncPeerMock>(); _scheduledActions = new Dictionary <long, Action>(); LatencySyncPeerMock.RemoteIndex = 1; _time = 0; _syncPeerPool = Substitute.For <IEthSyncPeerPool>(); _syncPeerPool.WhenForAnyArgs(p => p.ReportNoSyncProgress(Arg.Any <SyncPeerAllocation>())) .Do(ci => { LatencySyncPeerMock mock = ((LatencySyncPeerMock)ci.Arg <SyncPeerAllocation>().Current.SyncPeer); mock.BusyUntil = _time + 5000; mock.IsReported = true; }); _syncPeerPool.WhenForAnyArgs(p => p.ReportNoSyncProgress(Arg.Any <PeerInfo>())) .Do(ci => { LatencySyncPeerMock mock = ((LatencySyncPeerMock)ci.Arg <PeerInfo>().SyncPeer); mock.BusyUntil = _time + 5000; mock.IsReported = true; }); _syncPeerPool.WhenForAnyArgs(p => p.ReportInvalid(Arg.Any <SyncPeerAllocation>())) .Do(ci => { LatencySyncPeerMock mock = ((LatencySyncPeerMock)ci.Arg <SyncPeerAllocation>().Current.SyncPeer); mock.BusyUntil = _time + 30000; mock.IsReported = true; }); _syncPeerPool.WhenForAnyArgs(p => p.ReportInvalid(Arg.Any <PeerInfo>())) .Do(ci => { LatencySyncPeerMock mock = ((LatencySyncPeerMock)ci.Arg <PeerInfo>().SyncPeer); mock.BusyUntil = _time + 30000; mock.IsReported = true; }); _syncPeerPool.AllPeers.Returns((ci) => _syncPeers.Select(sp => new PeerInfo(sp) { HeadNumber = sp.Tree.Head.Number })); _syncConfig = new SyncConfig(); _syncConfig.PivotHash = _validTree2048.Head.Hash.ToString(); _syncConfig.PivotNumber = _validTree2048.Head.Number.ToString(); _syncConfig.PivotTotalDifficulty = _validTree2048.Head.TotalDifficulty.ToString(); SetupLocalTree(); SetupFeed(); }
public T GetSyncPayloadLogProvider <T>(SyncConfig config, LogStrategy logStrategy) => GetProvivder <T>(config.Providers.Sync.PayloadLog, logStrategy.PayloadLogProvider);
// Handle build conflicts, that is, when a file and folder has the same name. private static void HandleBuildConflicts(List<string> typeConflicts, SyncConfig config) { foreach (string s in typeConflicts) { if (File.Exists(s)) { FileInfo info = new FileInfo(s); string conflictPath = Path.Combine(info.DirectoryName, config.ConflictDir); if (!Directory.Exists(conflictPath)) Directory.CreateDirectory(conflictPath); string currTime = String.Format("{0:MMddHHmmss}", DateTime.Now) + "_"; string dest = Path.Combine(conflictPath, currTime + info.Name); try { CommonMethods.CopyFile(s, dest); CommonMethods.DeleteFile(s); } catch (CopyFileException) { ServiceLocator.GetLogger(ServiceLocator.USER_LOG).Write(new LogData(LogEventType.FSCHANGE_ERROR, "Error copying file from " + s + " to " + dest)); } catch (DeleteFileException) { } } } }
/// <summary> /// Instantiates an instance of <c>AutoSyncRequest</c> for rename purposes. /// </summary> /// <param name="oldName">The old name of the object.</param> /// <param name="newName">The new name of the object.</param> /// <param name="sourceParent">The parent of the source file or folder.</param> /// <param name="destinations">The list of destinations to propagate the source to.</param> /// <param name="isFolder"></param> /// <param name="requestType">The type of request, eg. create, update, delete or rename.</param> /// <param name="syncConfig">The sync configuration passed in.</param> /// <param name="tagList">The list of tags related to this request.</param> public AutoSyncRequest(string oldName, string newName, string sourceParent, List<string> destinations, bool? isFolder, AutoSyncRequestType requestType, SyncConfig syncConfig, List<string> tagList) : this(newName, sourceParent, destinations, isFolder, requestType, syncConfig, tagList) { _oldName = oldName; }
private async Task IngestCoreAsync(SyncConfig syncConfig, string src, string target, SyncIngestResult result) { _cancellationToken.ThrowIfCancellationRequested(); Directory.CreateDirectory(target); foreach (var dir in Directory.EnumerateDirectories(src, "*", SearchOption.TopDirectoryOnly)) { _cancellationToken.ThrowIfCancellationRequested(); result.SourceDirCount++; await IngestCoreAsync(syncConfig, dir, Path.Combine(target, new DirectoryInfo(dir).Name), result); } foreach (var file in Directory.EnumerateFiles(src, "*", SearchOption.TopDirectoryOnly)) { _cancellationToken.ThrowIfCancellationRequested(); result.SourceFileCount++; var fileName = Path.GetFileName(file); if (_options.ExcludePatterns.Value != null && _options.ExcludePatterns.Value.Any()) { var matched = false; foreach (var excludePattern in _options.ExcludePatterns.Value) { if (excludePattern.IsMatch(fileName)) { matched = true; break; } } if (matched) { continue; } } if (_options.IncludePatterns.Value != null && _options.IncludePatterns.Value.Any()) { var matched = false; foreach (var includePattern in _options.IncludePatterns.Value) { if (includePattern.IsMatch(fileName)) { matched = true; break; } } if (!matched) { continue; } } var fileInfo = new FileInfo(file); if (fileInfo.LastWriteTime < _options.NewerModifyDateTime) { continue; } var targetFilePath = Path.Combine(target, fileName); var syncInfo = new SyncFileInfo(syncConfig, file, targetFilePath); if (!File.Exists(targetFilePath)) { _syncQueue.Enqueue(syncInfo); } else { if (_options.Force) { syncInfo.Overwrite = true; _syncQueue.Enqueue(syncInfo); } else if (_options.Strict) { var exactlySame = await FileCompareUtil.ExactlySameAsync(file, targetFilePath, _cancellationToken); if (!exactlySame) { syncInfo.Overwrite = true; _syncQueue.Enqueue(syncInfo); } } } } }
private SyncConfig Clone() { SyncConfig config = new SyncConfig(ArchiveName, ArchiveLimit, Recycle); return config; }
public CleanupIngestResult(SyncConfig syncConfig) { SyncConfig = syncConfig; }
public T GetSyncProcessor <T>(SyncConfig config, string typeName) => GetProvivder <T>(config.Providers.Sync.Processor, typeName);
/// <summary> /// Instantiates an instance of <c>ManualCompareRequest</c>. /// </summary> /// <param name="paths">The list of paths to compare and preview.</param> /// <param name="filters">The list of filters to pass in.</param> /// <param name="syncConfig">The sync configuration to pass in.</param> /// <param name="tagName">The tag name to compare and preview.</param> public ManualCompareRequest(string[] paths, List<Filter> filters, SyncConfig syncConfig, string tagName) : base(paths, filters, syncConfig, tagName) { }
public T GetSyncTransport <T>(SyncConfig config, TaskConfig taskConfig, string typeName) => GetProvivder <T>(config.Providers.Sync.Transport, typeName, taskConfig);
public SyncDataItem(SyncConfig SyncConfig, OracleDbOperation targetDb, OracleDbOperation localDb, string targetCode, string localCode, FtpHelper targetFtp, FtpHelper localFtp, StringBuilder sbLog) : base(targetDb, localDb, targetCode, localCode, targetFtp, localFtp, sbLog) { this.syncConfig = SyncConfig; }
/// <summary> /// Instantiates an instance of <c>SyncerVisitor</c> with the sync configuration and progress object passed in. /// </summary> /// <param name="syncConfig">The sync configuration to pass in.</param> /// <param name="progress">The progress object to pass in.</param> public SyncerVisitor(SyncConfig syncConfig, Progress progress) { _syncConfig = syncConfig; _syncProgress = progress; }
/// <summary> /// Instantiates an instance of <c>ManualSyncRequest</c>. /// </summary> /// <param name="paths">The list of paths to compare and sync.</param> /// <param name="filters">The list of filters to pass in.</param> /// <param name="syncConfig">The sync configuration to pass in.</param> /// <param name="tagName">The tag name to compare and sync.</param> /// <param name="notify">Determines whether or not to notify.</param> public ManualSyncRequest(string[] paths, List<Filter> filters, SyncConfig syncConfig, string tagName, bool notify) : base(paths, filters, syncConfig, tagName) { _notify = notify; }
public async Task Can_read_dependent_items_from_state_db_while_waiting_for_dependencies() { MemDb codeDb = new MemDb(); MemDb stateDB = new MemDb(); MemDb tempDb = new MemDb(); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = true; IBlockTree blockTree = Substitute.For <IBlockTree>(); ISyncPeerPool pool = Substitute.For <ISyncPeerPool>(); SyncProgressResolver syncProgressResolver = new SyncProgressResolver( blockTree, NullReceiptStorage.Instance, stateDB, new MemDb(), new TrieStore(stateDB.Innermost, LimboLogs.Instance), syncConfig, LimboLogs.Instance); ISyncModeSelector syncModeSelector = new MultiSyncModeSelector(syncProgressResolver, pool, syncConfig, LimboLogs.Instance); StateSyncFeed stateSyncFeed = new StateSyncFeed(codeDb, stateDB, tempDb, syncModeSelector, blockTree, LimboLogs.Instance); // so we want to setup a trie in a structure of -> branch into two leaves // so we can respond with the branch node and with leaves missing // and we can prove that we can read the branch from the temp DB while it is still missing from the State DB AccountDecoder accountDecoder = new AccountDecoder(); TrieNode leaf = TrieNodeFactory.CreateLeaf(new HexPrefix(true, new byte[] { 1, 2, 3 }), accountDecoder.Encode(Account.TotallyEmpty).Bytes); TrieNode branch = TrieNodeFactory.CreateBranch(); branch.SetChild(1, leaf); branch.ResolveKey(NullTrieNodeResolver.Instance, true); // PatriciaTree tree = new PatriciaTree(); // tree = new PatriciaTree(); // tree.Set(branch.Keccak.Bytes, branch.Value); stateSyncFeed.ResetStateRoot(0, branch.Keccak); var request = await stateSyncFeed.PrepareRequest(); BuildRequestAndHandleResponse(branch, request, stateSyncFeed); byte[] value = tempDb.Get(branch.Keccak); value.Should().BeEquivalentTo(branch.FullRlp); byte[] valueFromState = stateDB.Get(branch.Keccak); valueFromState.Should().BeNull(); request = await stateSyncFeed.PrepareRequest(); BuildRequestAndHandleResponse(leaf, request, stateSyncFeed); value = tempDb.Get(branch.Keccak); value.Should().BeNull(); valueFromState = stateDB.Get(branch.Keccak); valueFromState.Should().BeEquivalentTo(branch.FullRlp); }
/// <summary> /// Instantiates an instance of <c>ConflictVisitor</c> with a sync configuration. /// </summary> /// <param name="syncConfig">The sync configuration to pass in.</param> public ConflictVisitor(SyncConfig syncConfig) { _syncConfig = syncConfig; }
public SyncIngestResult(SyncConfig syncConfig) { SyncConfig = syncConfig; }
/// <summary> /// Create an initial XML configuration file with default settings and zero remote folders. /// </summary> private void CreateInitialConfig() { // Get the user name. string userName = "******"; if (Backend.Platform == PlatformID.Unix || Backend.Platform == PlatformID.MacOSX) { userName = Environment.UserName; if (string.IsNullOrEmpty(userName)) { userName = String.Empty; } else { userName = userName.TrimEnd(",".ToCharArray()); } } else { userName = Environment.UserName; } if (string.IsNullOrEmpty(userName)) { userName = "******"; } // Define the default XML configuration file. configXml = new SyncConfig() { Folders = new List<SyncConfig.Folder>(), User = new User() { EMail = "Unknown", Name = userName }, Notifications = true, SingleRepository = false, //Multiple repository for CmisSync Log4Net = createDefaultLog4NetElement() }; // Save it as an XML file. Save(); }
/// <summary> /// 写 /// </summary> /// <param name="sysConfig"></param> public void Write(SyncConfig sysConfig) { RegistryKey root = GetRoot(); if (regHelper.KeyExists(root, SysConfig.SyncServiceName)) { root.DeleteSubKeyTree(SysConfig.SyncServiceName); } RegistryKey serviceKey = root.CreateSubKey(SysConfig.SyncServiceName); serviceKey.SetValue(SysConfig.RegistryTimerInterval, sysConfig.TimerInterval); serviceKey.SetValue(SysConfig.RegistryWriteLog, sysConfig.WriteLog); serviceKey.SetValue(SysConfig.RegistrySyncType, sysConfig.SyncType); serviceKey.SetValue(SysConfig.RegistrySyncGisData, sysConfig.SyncGisData); serviceKey.SetValue(SysConfig.RegistryServiceUrl, sysConfig.ServiceUrl); serviceKey.SetValue(SysConfig.RegistryServiceMethod, sysConfig.ServiceMethod); RegistryKey localKey = serviceKey.CreateSubKey(SysConfig.RegistryLocals); foreach (var localSetting in sysConfig.LocalData) { RegistryKey localCodeKey = localKey.CreateSubKey(localSetting.Code); localCodeKey.SetValue(SysConfig.RegistryName, localSetting.Name); if (localSetting.DbConfig != null) { regHelper.SetValue(localCodeKey, SysConfig.RegistryDbIp, localSetting.DbConfig.Ip); regHelper.SetValue(localCodeKey, SysConfig.RegistryDbName, localSetting.DbConfig.Name); regHelper.SetValue(localCodeKey, SysConfig.RegistryDbUsername, localSetting.DbConfig.Username); regHelper.SetValue(localCodeKey, SysConfig.RegistryDbPassword, localSetting.DbConfig.Password); } if (localSetting.FtpConfig != null) { regHelper.SetValue(localCodeKey, SysConfig.RegistryFtpIp, localSetting.FtpConfig.Ip); regHelper.SetValue(localCodeKey, SysConfig.RegistryFtpPort, localSetting.FtpConfig.Port); regHelper.SetValue(localCodeKey, SysConfig.RegistryFtpUsername, localSetting.FtpConfig.Username); regHelper.SetValue(localCodeKey, SysConfig.RegistryFtpPassword, localSetting.FtpConfig.Password); } } RegistryKey targetKey = serviceKey.CreateSubKey(SysConfig.RegistryTargets); foreach (var targetSetting in sysConfig.TargetData) { RegistryKey targetCodeKey = targetKey.CreateSubKey(targetSetting.Code); targetCodeKey.SetValue(SysConfig.RegistryName, targetSetting.Name); if (targetSetting.DbConfig != null) { regHelper.SetValue(targetCodeKey, SysConfig.RegistryDbIp, targetSetting.DbConfig.Ip); regHelper.SetValue(targetCodeKey, SysConfig.RegistryDbName, targetSetting.DbConfig.Name); regHelper.SetValue(targetCodeKey, SysConfig.RegistryDbUsername, targetSetting.DbConfig.Username); regHelper.SetValue(targetCodeKey, SysConfig.RegistryDbPassword, targetSetting.DbConfig.Password); } if (targetSetting.DbConfig != null) { regHelper.SetValue(targetCodeKey, SysConfig.RegistryFtpIp, targetSetting.FtpConfig.Ip); regHelper.SetValue(targetCodeKey, SysConfig.RegistryFtpPort, targetSetting.FtpConfig.Port); regHelper.SetValue(targetCodeKey, SysConfig.RegistryFtpUsername, targetSetting.FtpConfig.Username); regHelper.SetValue(targetCodeKey, SysConfig.RegistryFtpPassword, targetSetting.FtpConfig.Password); } if (!string.IsNullOrEmpty(targetSetting.Jg)) { regHelper.SetValue(targetCodeKey, SysConfig.RegistryJg, targetSetting.Jg); } if (!string.IsNullOrEmpty(targetSetting.Time)) { regHelper.SetValue(targetCodeKey, SysConfig.RegistryTime, targetSetting.Time); } } }
public JsonService(IoService ioService) { this.ioService = ioService; this.config = new SyncConfig(); }
public CleanupInfo(SyncConfig syncConfig) { SyncConfig = syncConfig; }
/// <summary> /// 读 /// </summary> /// <returns></returns> public SyncConfig Read() { SyncConfig SyncSetting = new SyncConfig(); RegistryKey root = GetRoot(); if (regHelper.KeyExists(root, SysConfig.SyncServiceName)) { RegistryKey serviceKey = root.OpenSubKey(SysConfig.SyncServiceName); SyncSetting.TimerInterval = regHelper.GetValue <int>(serviceKey, SysConfig.RegistryTimerInterval); SyncSetting.WriteLog = regHelper.GetValue <bool>(serviceKey, SysConfig.RegistryWriteLog); SyncSetting.SyncType = regHelper.GetValue <int>(serviceKey, SysConfig.RegistrySyncType); SyncSetting.SyncGisData = regHelper.GetValue <bool>(serviceKey, SysConfig.RegistrySyncGisData); SyncSetting.ServiceUrl = regHelper.GetValue <string>(serviceKey, SysConfig.RegistryServiceUrl); SyncSetting.ServiceMethod = regHelper.GetValue <string>(serviceKey, SysConfig.RegistryServiceMethod); RegistryKey localKey = serviceKey.OpenSubKey(SysConfig.RegistryLocals); foreach (var keyName in localKey.GetSubKeyNames()) { RegistryKey localCodeKey = localKey.OpenSubKey(keyName); SyncConfigLocal localSetting = new SyncConfigLocal() { Code = keyName, Name = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryName), DbConfig = new DbConfig() { Ip = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryDbIp), Name = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryDbName), Username = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryDbUsername), Password = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryDbPassword) }, FtpConfig = new FtpConfig() { Ip = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryFtpIp), Port = regHelper.GetValue <int>(localCodeKey, SysConfig.RegistryFtpPort), Username = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryFtpUsername), Password = regHelper.GetValue <string>(localCodeKey, SysConfig.RegistryFtpPassword) } }; SyncSetting.LocalData.Add(localSetting); } RegistryKey targetKey = serviceKey.OpenSubKey(SysConfig.RegistryTargets); foreach (var keyName in targetKey.GetSubKeyNames()) { RegistryKey targetCodeKey = targetKey.OpenSubKey(keyName); SyncConfigTarget targetSetting = new SyncConfigTarget() { Code = keyName, Name = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryName), DbConfig = new DbConfig() { Ip = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryDbIp), Name = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryDbName), Username = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryDbUsername), Password = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryDbPassword) }, FtpConfig = new FtpConfig() { Ip = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryFtpIp), Port = regHelper.GetValue <int>(targetCodeKey, SysConfig.RegistryFtpPort), Username = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryFtpUsername), Password = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryFtpPassword) }, Jg = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryJg), Time = regHelper.GetValue <string>(targetCodeKey, SysConfig.RegistryTime) }; SyncSetting.TargetData.Add(targetSetting); } } return(SyncSetting); }
private SyncTestContext CreateSyncManager(int index) { Rlp.RegisterDecoders(typeof(ParityTraceDecoder).Assembly); var logManager = NoErrorLimboLogs.Instance; ConsoleAsyncLogger logger = new ConsoleAsyncLogger(LogLevel.Debug, "PEER " + index + " "); // var logManager = new OneLoggerLogManager(logger); var specProvider = new SingleReleaseSpecProvider(ConstantinopleFix.Instance, MainNetSpecProvider.Instance.ChainId); MemDb traceDb = new MemDb(); MemDb blockDb = new MemDb(); MemDb headerDb = new MemDb(); MemDb blockInfoDb = new MemDb(); StateDb codeDb = new StateDb(); StateDb stateDb = new StateDb(); var stateProvider = new StateProvider(stateDb, codeDb, logManager); stateProvider.CreateAccount(TestItem.AddressA, 10000.Ether()); stateProvider.Commit(specProvider.GenesisSpec); stateProvider.CommitTree(); stateDb.Commit(); var storageProvider = new StorageProvider(stateDb, stateProvider, logManager); var receiptStorage = new InMemoryReceiptStorage(); var ecdsa = new EthereumEcdsa(specProvider, logManager); var txPool = new TxPool(new InMemoryTransactionStorage(), new PendingTransactionThresholdValidator(), new Timestamp(), ecdsa, specProvider, logManager); var tree = new BlockTree(blockDb, headerDb, blockInfoDb, specProvider, txPool, logManager); var blockhashProvider = new BlockhashProvider(tree, LimboLogs.Instance); var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, logManager); var sealValidator = TestSealValidator.AlwaysValid; var headerValidator = new HeaderValidator(tree, sealValidator, specProvider, logManager); var txValidator = TestTxValidator.AlwaysValid; var ommersValidator = new OmmersValidator(tree, headerValidator, logManager); var blockValidator = new BlockValidator(txValidator, headerValidator, ommersValidator, specProvider, logManager); // var blockValidator = TestBlockValidator.AlwaysValid; SyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = _synchronizerType == SynchronizerType.Fast; var rewardCalculator = new RewardCalculator(specProvider); var txProcessor = new TransactionProcessor(specProvider, stateProvider, storageProvider, virtualMachine, logManager); var blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, txProcessor, stateDb, codeDb, traceDb, stateProvider, storageProvider, txPool, receiptStorage, syncConfig, logManager); var step = new TxSignaturesRecoveryStep(ecdsa, txPool, logManager); var processor = new BlockchainProcessor(tree, blockProcessor, step, logManager, true, true); var nodeStatsManager = new NodeStatsManager(new StatsConfig(), logManager); var syncPeerPool = new EthSyncPeerPool(tree, nodeStatsManager, syncConfig, logManager); StateProvider devState = new StateProvider(stateDb, codeDb, logManager); StorageProvider devStorage = new StorageProvider(stateDb, devState, logManager); var devEvm = new VirtualMachine(devState, devStorage, blockhashProvider, logManager); var devTxProcessor = new TransactionProcessor(specProvider, devState, devStorage, devEvm, logManager); var devBlockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, devTxProcessor, stateDb, codeDb, traceDb, devState, devStorage, txPool, receiptStorage, syncConfig, logManager); var devChainProcessor = new BlockchainProcessor(tree, devBlockProcessor, step, logManager, false, false); var producer = new DevBlockProducer(txPool, devChainProcessor, tree, new Timestamp(), logManager); NodeDataFeed feed = new NodeDataFeed(codeDb, stateDb, logManager); NodeDataDownloader downloader = new NodeDataDownloader(syncPeerPool, feed, logManager); Synchronizer synchronizer = new Synchronizer( tree, blockValidator, sealValidator, syncPeerPool, syncConfig, downloader, logManager); var syncServer = new SyncServer(stateDb, codeDb, tree, receiptStorage, TestSealValidator.AlwaysValid, syncPeerPool, synchronizer, logManager); ManualResetEventSlim waitEvent = new ManualResetEventSlim(); tree.NewHeadBlock += (s, e) => waitEvent.Set(); if (index == 0) { _genesis = Build.A.Block.Genesis.WithStateRoot(stateProvider.StateRoot).TestObject; producer.Start(); } syncPeerPool.Start(); synchronizer.Start(); processor.Start(); tree.SuggestBlock(_genesis); if (!waitEvent.Wait(1000)) { throw new Exception("No genesis"); } SyncTestContext context = new SyncTestContext(); context.Ecdsa = ecdsa; context.BlockchainProcessor = processor; context.PeerPool = syncPeerPool; context.StateProvider = stateProvider; context.Synchronizer = synchronizer; context.SyncServer = syncServer; context.Tree = tree; context.BlockProducer = producer; context.TxPool = txPool; context.Logger = logger; return(context); }
public SyncFileInfo(SyncConfig syncConfig, string sourceFile, string targetFile) { SyncConfig = syncConfig; SourceFile = sourceFile; TargetFile = targetFile; }
public void Db_size_are_computed_correctly( [Values(256 * MB, 512 * MB, 1 * GB, 4 * GB, 6 * GB, 16 * GB, 32 * GB, 64 * GB, 128 * GB)] long memoryHint, [Values(1u, 2u, 3u, 4u, 8u, 32u)] uint cpuCount, [Values(true, false)] bool fastSync, [Values(true, false)] bool fastBlocks) { // OK to throw here if (memoryHint == 256.MB()) { _txPoolConfig.Size = 128; _syncConfig.FastBlocks = false; _initConfig.DiagnosticMode = DiagnosticMode.MemDb; } _initConfig.MemoryHint = memoryHint; SetMemoryAllowances(cpuCount); SyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = fastSync; syncConfig.FastBlocks = fastBlocks; IDbConfig dbConfig = _dbConfig; ulong totalForHeaders = dbConfig.HeadersDbBlockCacheSize + dbConfig.HeadersDbWriteBufferNumber * dbConfig.HeadersDbWriteBufferSize; ulong totalForBlocks = dbConfig.BlocksDbBlockCacheSize + dbConfig.BlocksDbWriteBufferNumber * dbConfig.BlocksDbWriteBufferSize; ulong totalForInfos = dbConfig.BlockInfosDbBlockCacheSize + dbConfig.BlockInfosDbWriteBufferNumber * dbConfig.BlockInfosDbWriteBufferSize; ulong totalForReceipts = dbConfig.ReceiptsDbBlockCacheSize + dbConfig.ReceiptsDbWriteBufferNumber * dbConfig.ReceiptsDbWriteBufferSize; ulong totalForCode = dbConfig.CodeDbBlockCacheSize + dbConfig.CodeDbWriteBufferNumber * dbConfig.CodeDbWriteBufferSize; ulong totalForPending = dbConfig.PendingTxsDbBlockCacheSize + dbConfig.PendingTxsDbWriteBufferNumber * dbConfig.PendingTxsDbWriteBufferSize; ulong totalMem = (dbConfig.BlockCacheSize + dbConfig.WriteBufferNumber * dbConfig.WriteBufferSize) + totalForHeaders + totalForBlocks + totalForInfos + totalForReceipts + totalForCode + totalForPending; if (_initConfig.DiagnosticMode != DiagnosticMode.MemDb) { // some rounding differences are OK totalMem.Should().BeGreaterThan((ulong)((memoryHint - 200.MB()) * 0.6)); totalMem.Should().BeLessThan((ulong)((memoryHint - 200.MB()) * 0.9)); } else { _memoryHintMan.DbMemory.Should().BeGreaterThan((long)((memoryHint - 100.MB()) * 0.6)); _memoryHintMan.DbMemory.Should().BeLessThan((long)((memoryHint - 100.MB()) * 0.9)); } }
/// <summary> /// Create an initial XML configuration with default settings and zero remote folders. /// </summary> private void CreateConfigFromScratch() { // Get the user name. string userName = "******"; if (Backend.Platform == PlatformID.Unix || Backend.Platform == PlatformID.MacOSX) { userName = Environment.UserName; if (string.IsNullOrEmpty(userName)) { userName = String.Empty; } else { userName = userName.TrimEnd(",".ToCharArray()); } } else { userName = Environment.UserName; } if (string.IsNullOrEmpty(userName)) { userName = "******"; } // Define the default XML configuration file. configXml = new SyncConfig() { ConfigSchemaVersion = Config.SchemaVersion, Notifications = true, SingleRepository = false, // Multiple repository for CmisSync, but some CmisSync-derived products have different defaults. FrozenConfiguration = false, Log4Net = createDefaultLog4NetElement(), Folders = new List<SyncConfig.Folder>(), User = new User() { EMail = "Unknown", Name = userName } }; }