//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldSetLastPulledTransactionId() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldSetLastPulledTransactionId() { // given long lastFlushedTxId = 12; StoreId wantedStoreId = new StoreId(1, 2, 3, 4); AdvertisedSocketAddress localhost = new AdvertisedSocketAddress("127.0.0.1", 1234); CatchupAddressProvider catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(localhost); StoreCopyClient storeCopyClient = mock(typeof(StoreCopyClient)); when(storeCopyClient.CopyStoreFiles(eq(catchupAddressProvider), eq(wantedStoreId), any(typeof(StoreFileStreamProvider)), any(), any())).thenReturn(lastFlushedTxId); TxPullClient txPullClient = mock(typeof(TxPullClient)); when(txPullClient.PullTransactions(eq(localhost), eq(wantedStoreId), anyLong(), any())).thenReturn(new TxPullRequestResult(SUCCESS_END_OF_STREAM, 13)); TransactionLogCatchUpWriter writer = mock(typeof(TransactionLogCatchUpWriter)); RemoteStore remoteStore = new RemoteStore(NullLogProvider.Instance, mock(typeof(FileSystemAbstraction)), null, storeCopyClient, txPullClient, Factory(writer), Config.defaults(), new Monitors()); // when remoteStore.Copy(catchupAddressProvider, wantedStoreId, DatabaseLayout.of(new File("destination")), true); // then long previousTxId = lastFlushedTxId - 1; // the interface is defined as asking for the one preceding verify(txPullClient).pullTransactions(eq(localhost), eq(wantedStoreId), eq(previousTxId), any()); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldCloseDownTxLogWriterIfTxStreamingFails() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldCloseDownTxLogWriterIfTxStreamingFails() { // given StoreId storeId = new StoreId(1, 2, 3, 4); StoreCopyClient storeCopyClient = mock(typeof(StoreCopyClient)); TxPullClient txPullClient = mock(typeof(TxPullClient)); TransactionLogCatchUpWriter writer = mock(typeof(TransactionLogCatchUpWriter)); CatchupAddressProvider catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(null); RemoteStore remoteStore = new RemoteStore(NullLogProvider.Instance, mock(typeof(FileSystemAbstraction)), null, storeCopyClient, txPullClient, Factory(writer), Config.defaults(), new Monitors()); doThrow(typeof(CatchUpClientException)).when(txPullClient).pullTransactions(Null, eq(storeId), anyLong(), any()); // when try { remoteStore.Copy(catchupAddressProvider, storeId, DatabaseLayout.of(new File(".")), true); } catch (StoreCopyFailedException) { // expected } // then verify(writer).close(); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void failedFileCopyShouldRetry() throws StoreCopyFailedException, java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void FailedFileCopyShouldRetry() { // given a file will fail twice before succeeding _fileB.RemainingFailed = 2; // and remote node has a store // and local client has a store InMemoryStoreStreamProvider clientStoreFileStream = new InMemoryStoreStreamProvider(); // when catchup is performed for valid transactionId and StoreId CatchupAddressProvider catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(From(_catchupServer.address().Port)); _subject.copyStoreFiles(catchupAddressProvider, _serverHandler.StoreId, clientStoreFileStream, () => _defaultTerminationCondition, _targetLocation); // then the catchup is successful ISet <string> expectedFiles = new HashSet <string>(Arrays.asList(_fileA.Filename, _fileB.Filename, _indexFileA.Filename)); assertEquals(expectedFiles, clientStoreFileStream.FileStreams().Keys); // and assertEquals(FileContent(Relative(_fileA.Filename)), ClientFileContents(clientStoreFileStream, _fileA.Filename)); assertEquals(FileContent(Relative(_fileB.Filename)), ClientFileContents(clientStoreFileStream, _fileB.Filename)); // and verify server had exactly 2 failed calls before having a 3rd succeeding request assertEquals(3, _serverHandler.getRequestCount(_fileB.Filename)); // and verify server had exactly 1 call for all other files assertEquals(1, _serverHandler.getRequestCount(_fileA.Filename)); }
internal PersistentSnapshotDownloader(CatchupAddressProvider addressProvider, CommandApplicationProcess applicationProcess, CoreStateDownloader downloader, Log log, Org.Neo4j.causalclustering.helper.TimeoutStrategy_Timeout pauseStrategy, System.Func <DatabaseHealth> dbHealth, Monitors monitors) { this._applicationProcess = applicationProcess; this._addressProvider = addressProvider; this._downloader = downloader; this._log = log; this._timeout = pauseStrategy; this._dbHealth = dbHealth; this._monitor = monitors.NewMonitor(typeof(Monitor)); this._state = State.Initiated; this._keepRunning = true; }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: long copyStoreFiles(org.neo4j.causalclustering.catchup.CatchupAddressProvider catchupAddressProvider, org.neo4j.causalclustering.identity.StoreId expectedStoreId, StoreFileStreamProvider storeFileStreamProvider, System.Func<TerminationCondition> requestWiseTerminationCondition, java.io.File destDir) throws StoreCopyFailedException internal virtual long CopyStoreFiles(CatchupAddressProvider catchupAddressProvider, StoreId expectedStoreId, StoreFileStreamProvider storeFileStreamProvider, System.Func <TerminationCondition> requestWiseTerminationCondition, File destDir) { try { PrepareStoreCopyResponse prepareStoreCopyResponse = PrepareStoreCopy(catchupAddressProvider.Primary(), expectedStoreId, storeFileStreamProvider); CopyFilesIndividually(prepareStoreCopyResponse, expectedStoreId, catchupAddressProvider, storeFileStreamProvider, requestWiseTerminationCondition, destDir); CopyIndexSnapshotIndividually(prepareStoreCopyResponse, expectedStoreId, catchupAddressProvider, storeFileStreamProvider, requestWiseTerminationCondition); return(prepareStoreCopyResponse.LastTransactionId()); } catch (Exception e) when(e is CatchupAddressResolutionException || e is CatchUpClientException) { throw new StoreCopyFailedException(e); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldNotAppendToFileWhenRetryingWithNewFile() throws Throwable //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldNotAppendToFileWhenRetryingWithNewFile() { // given string fileName = "foo"; string copyFileName = "bar"; string unfinishedContent = "abcd"; string finishedContent = "abcdefgh"; IEnumerator <string> contents = Iterators.iterator(unfinishedContent, finishedContent); // and TestCatchupServerHandler halfWayFailingServerhandler = new TestCatchupServerHandlerAnonymousInnerClass(this, _logProvider, TestDirectory, _fsa, fileName, copyFileName, contents); Server halfWayFailingServer = null; try { // when ListenSocketAddress listenAddress = new ListenSocketAddress("localhost", PortAuthority.allocatePort()); halfWayFailingServer = (new CatchupServerBuilder(halfWayFailingServerhandler)).listenAddress(listenAddress).build(); halfWayFailingServer.Start(); CatchupAddressProvider addressProvider = CatchupAddressProvider.fromSingleAddress(new AdvertisedSocketAddress(listenAddress.Hostname, listenAddress.Port)); StoreId storeId = halfWayFailingServerhandler.StoreId; File databaseDir = TestDirectory.databaseDir(); StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider(databaseDir, _fsa, new Monitors()); // and _subject.copyStoreFiles(addressProvider, storeId, streamToDiskProvider, () => _defaultTerminationCondition, _targetLocation); // then assertEquals(FileContent(new File(databaseDir, fileName)), finishedContent); // and File fileCopy = new File(databaseDir, copyFileName); ByteBuffer buffer = ByteBuffer.wrap(new sbyte[finishedContent.Length]); using (StoreChannel storeChannel = _fsa.create(fileCopy)) { storeChannel.read(buffer); } assertEquals(finishedContent, new string( buffer.array(), Charsets.UTF_8 )); } finally { halfWayFailingServer.Stop(); halfWayFailingServer.Shutdown(); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void canPerformCatchup() throws StoreCopyFailedException, java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void CanPerformCatchup() { // given local client has a store InMemoryStoreStreamProvider storeFileStream = new InMemoryStoreStreamProvider(); // when catchup is performed for valid transactionId and StoreId CatchupAddressProvider catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(From(_catchupServer.address().Port)); _subject.copyStoreFiles(catchupAddressProvider, _serverHandler.StoreId, storeFileStream, () => _defaultTerminationCondition, _targetLocation); // then the catchup is successful ISet <string> expectedFiles = new HashSet <string>(Arrays.asList(_fileA.Filename, _fileB.Filename, _indexFileA.Filename)); assertEquals(expectedFiles, storeFileStream.FileStreams().Keys); assertEquals(FileContent(Relative(_fileA.Filename)), ClientFileContents(storeFileStream, _fileA.Filename)); assertEquals(FileContent(Relative(_fileB.Filename)), ClientFileContents(storeFileStream, _fileB.Filename)); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public void copy(org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider, org.neo4j.causalclustering.identity.StoreId expectedStoreId, org.neo4j.io.layout.DatabaseLayout destinationLayout, boolean rotateTransactionsManually) throws StoreCopyFailedException public virtual void Copy(CatchupAddressProvider addressProvider, StoreId expectedStoreId, DatabaseLayout destinationLayout, bool rotateTransactionsManually) { try { long lastFlushedTxId; StreamToDiskProvider streamToDiskProvider = new StreamToDiskProvider(destinationLayout.DatabaseDirectory(), _fs, _monitors); lastFlushedTxId = _storeCopyClient.copyStoreFiles(addressProvider, expectedStoreId, streamToDiskProvider, () => new MaximumTotalTime(_config.get(CausalClusteringSettings.store_copy_max_retry_time_per_request).Seconds, TimeUnit.SECONDS), destinationLayout.DatabaseDirectory()); _log.info("Store files need to be recovered starting from: %d", lastFlushedTxId); CatchupResult catchupResult = PullTransactions(addressProvider.Primary(), expectedStoreId, destinationLayout, lastFlushedTxId, true, true, rotateTransactionsManually); if (catchupResult != SUCCESS_END_OF_STREAM) { throw new StoreCopyFailedException("Failed to pull transactions: " + catchupResult); } } catch (Exception e) when(e is CatchupAddressResolutionException || e is IOException) { throw new StoreCopyFailedException(e); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldCopyStoreFilesAndPullTransactions() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldCopyStoreFilesAndPullTransactions() { // given StoreId storeId = new StoreId(1, 2, 3, 4); StoreCopyClient storeCopyClient = mock(typeof(StoreCopyClient)); TxPullClient txPullClient = mock(typeof(TxPullClient)); when(txPullClient.PullTransactions(any(), any(), anyLong(), any())).thenReturn(new TxPullRequestResult(SUCCESS_END_OF_STREAM, 13)); TransactionLogCatchUpWriter writer = mock(typeof(TransactionLogCatchUpWriter)); RemoteStore remoteStore = new RemoteStore(NullLogProvider.Instance, mock(typeof(FileSystemAbstraction)), null, storeCopyClient, txPullClient, Factory(writer), Config.defaults(), new Monitors()); // when AdvertisedSocketAddress localhost = new AdvertisedSocketAddress("127.0.0.1", 1234); CatchupAddressProvider catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(localhost); remoteStore.Copy(catchupAddressProvider, storeId, DatabaseLayout.of(new File("destination")), true); // then verify(storeCopyClient).copyStoreFiles(eq(catchupAddressProvider), eq(storeId), any(typeof(StoreFileStreamProvider)), any(), any()); verify(txPullClient).pullTransactions(eq(localhost), eq(storeId), anyLong(), any()); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public void replaceWithStoreFrom(org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider, org.neo4j.causalclustering.identity.StoreId expectedStoreId) throws java.io.IOException, StoreCopyFailedException, DatabaseShutdownException public virtual void ReplaceWithStoreFrom(CatchupAddressProvider addressProvider, StoreId expectedStoreId) { using (TemporaryStoreDirectory tempStore = new TemporaryStoreDirectory(_fs, _pageCache, _localDatabase.databaseLayout().databaseDirectory())) { _remoteStore.copy(addressProvider, expectedStoreId, tempStore.DatabaseLayout(), false); try { _copiedStoreRecovery.recoverCopiedStore(tempStore.DatabaseLayout()); } catch (Exception e) { /* * We keep the store until the next store copy attempt. If the exception * is fatal then the store will stay around for potential forensics. */ tempStore.KeepStore(); throw e; } _localDatabase.replaceWith(tempStore.DatabaseLayout().databaseDirectory()); } _log.info("Replaced store successfully"); }
internal override bool DownloadSnapshot(CatchupAddressProvider addressProvider) { return(After-- <= 0); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: private void copyIndexSnapshotIndividually(PrepareStoreCopyResponse prepareStoreCopyResponse, org.neo4j.causalclustering.identity.StoreId expectedStoreId, org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider, StoreFileStreamProvider storeFileStream, System.Func<TerminationCondition> terminationConditions) throws StoreCopyFailedException private void CopyIndexSnapshotIndividually(PrepareStoreCopyResponse prepareStoreCopyResponse, StoreId expectedStoreId, CatchupAddressProvider addressProvider, StoreFileStreamProvider storeFileStream, System.Func <TerminationCondition> terminationConditions) { StoreCopyClientMonitor storeCopyClientMonitor = _monitors.newMonitor(typeof(StoreCopyClientMonitor)); long lastTransactionId = prepareStoreCopyResponse.LastTransactionId(); LongIterator indexIds = prepareStoreCopyResponse.IndexIds.longIterator(); storeCopyClientMonitor.StartReceivingIndexSnapshots(); while (indexIds.hasNext()) { long indexId = indexIds.next(); storeCopyClientMonitor.StartReceivingIndexSnapshot(indexId); PersistentCallToSecondary(new GetIndexFilesRequest(expectedStoreId, indexId, lastTransactionId), filesCopyAdaptor(storeFileStream, _log), addressProvider, terminationConditions()); storeCopyClientMonitor.FinishReceivingIndexSnapshot(indexId); } storeCopyClientMonitor.FinishReceivingIndexSnapshots(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: private void copyFilesIndividually(PrepareStoreCopyResponse prepareStoreCopyResponse, org.neo4j.causalclustering.identity.StoreId expectedStoreId, org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider, StoreFileStreamProvider storeFileStream, System.Func<TerminationCondition> terminationConditions, java.io.File destDir) throws StoreCopyFailedException private void CopyFilesIndividually(PrepareStoreCopyResponse prepareStoreCopyResponse, StoreId expectedStoreId, CatchupAddressProvider addressProvider, StoreFileStreamProvider storeFileStream, System.Func <TerminationCondition> terminationConditions, File destDir) { StoreCopyClientMonitor storeCopyClientMonitor = _monitors.newMonitor(typeof(StoreCopyClientMonitor)); storeCopyClientMonitor.StartReceivingStoreFiles(); long lastTransactionId = prepareStoreCopyResponse.LastTransactionId(); foreach (File file in prepareStoreCopyResponse.Files) { storeCopyClientMonitor.StartReceivingStoreFile(Paths.get(destDir.ToString(), file.Name).ToString()); PersistentCallToSecondary(new GetStoreFileRequest(expectedStoreId, file, lastTransactionId), filesCopyAdaptor(storeFileStream, _log), addressProvider, terminationConditions()); storeCopyClientMonitor.FinishReceivingStoreFile(Paths.get(destDir.ToString(), file.Name).ToString()); } storeCopyClientMonitor.FinishReceivingStoreFiles(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: private void persistentCallToSecondary(org.neo4j.causalclustering.messaging.CatchUpRequest request, org.neo4j.causalclustering.catchup.CatchUpResponseAdaptor<StoreCopyFinishedResponse> copyHandler, org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider, TerminationCondition terminationCondition) throws StoreCopyFailedException private void PersistentCallToSecondary(CatchUpRequest request, CatchUpResponseAdaptor <StoreCopyFinishedResponse> copyHandler, CatchupAddressProvider addressProvider, TerminationCondition terminationCondition) { Org.Neo4j.causalclustering.helper.TimeoutStrategy_Timeout timeout = _backOffStrategy.newTimeout(); while (true) { try { AdvertisedSocketAddress address = addressProvider.Secondary(); _log.info(format("Sending request '%s' to '%s'", request, address)); StoreCopyFinishedResponse response = _catchUpClient.makeBlockingRequest(address, request, copyHandler); if (SuccessfulRequest(response, request)) { break; } } catch (CatchUpClientException e) { Exception cause = e.InnerException; if (cause is ConnectException) { _log.warn(cause.Message); } else { _log.warn(format("Request failed exceptionally '%s'.", request), e); } } catch (CatchupAddressResolutionException e) { _log.warn("Unable to resolve address for '%s'. %s", request, e.Message); } terminationCondition(); AwaitAndIncrementTimeout(timeout); } }
private void InitializeInstanceFields() { _catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(_expectedAdvertisedAddress); }
private void InitializeInstanceFields() { _catchupAddressProvider = CatchupAddressProvider.fromSingleAddress(_fromAddress); }
/// <summary> /// Tries to catchup this instance by downloading a snapshot. A snapshot consists of both the /// comparatively small state of the cluster state machines as well as the database store. The /// store is however caught up using two different approach. If it is possible to catchup by /// pulling transactions, then this will be sufficient, but if the store is lagging too far /// behind then a complete store copy will be attempted. /// </summary> /// <param name="addressProvider"> Provider of addresses to catchup from. </param> /// <returns> True if the operation succeeded, and false otherwise. </returns> /// <exception cref="LifecycleException"> A major database component failed to start or stop. </exception> /// <exception cref="IOException"> An issue with I/O. </exception> /// <exception cref="DatabaseShutdownException"> The database is shutting down. </exception> //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: boolean downloadSnapshot(org.neo4j.causalclustering.catchup.CatchupAddressProvider addressProvider) throws org.neo4j.kernel.lifecycle.LifecycleException, java.io.IOException, org.neo4j.causalclustering.catchup.storecopy.DatabaseShutdownException internal virtual bool DownloadSnapshot(CatchupAddressProvider addressProvider) { /* Extract some key properties before shutting it down. */ bool isEmptyStore = _localDatabase.Empty; /* * There is no reason to try to recover if there are no transaction logs and in fact it is * also problematic for the initial transaction pull during the snapshot download because the * kernel will create a transaction log with a header where previous index points to the same * index as that written down into the metadata store. This is problematic because we have no * guarantee that there are later transactions and we need at least one transaction in * the log to figure out the Raft log index (see {@link RecoverConsensusLogIndex}). */ if (_commitStateHelper.hasTxLogs(_localDatabase.databaseLayout())) { _log.info("Recovering local database"); Ensure(_localDatabase.start, "start local database"); Ensure(_localDatabase.stop, "stop local database"); } AdvertisedSocketAddress primary; StoreId remoteStoreId; try { primary = addressProvider.Primary(); remoteStoreId = _remoteStore.getStoreId(primary); } catch (Exception e) when(e is CatchupAddressResolutionException || e is StoreIdDownloadFailedException) { _log.warn("Store copy failed", e); return(false); } if (!isEmptyStore && !remoteStoreId.Equals(_localDatabase.storeId())) { _log.error("Store copy failed due to store ID mismatch"); return(false); } Ensure(_suspendOnStoreCopy.disable, "disable auxiliary services before store copy"); Ensure(_localDatabase.stopForStoreCopy, "stop local database for store copy"); _log.info("Downloading snapshot from core server at %s", primary); /* The core snapshot must be copied before the store, because the store has a dependency on * the state of the state machines. The store will thus be at or ahead of the state machines, * in consensus log index, and application of commands will bring them in sync. Any such commands * that carry transactions will thus be ignored by the transaction/token state machines, since they * are ahead, and the correct decisions for their applicability have already been taken as encapsulated * in the copied store. */ CoreSnapshot coreSnapshot; try { coreSnapshot = _catchUpClient.makeBlockingRequest(primary, new CoreSnapshotRequest(), new CatchUpResponseAdaptorAnonymousInnerClass(this)); } catch (CatchUpClientException e) { _log.warn("Store copy failed", e); return(false); } if (!isEmptyStore) { StoreId localStoreId = _localDatabase.storeId(); CatchupResult catchupResult; try { catchupResult = _remoteStore.tryCatchingUp(primary, localStoreId, _localDatabase.databaseLayout(), false, false); } catch (StoreCopyFailedException e) { _log.warn("Failed to catch up", e); return(false); } if (catchupResult == E_TRANSACTION_PRUNED) { _log.warn(format("Failed to pull transactions from (%s). They may have been pruned away", primary)); _localDatabase.delete(); isEmptyStore = true; } else if (catchupResult != SUCCESS_END_OF_STREAM) { _log.warn(format("Unexpected catchup operation result %s from %s", catchupResult, primary)); return(false); } } if (isEmptyStore) { try { _storeCopyProcess.replaceWithStoreFrom(addressProvider, remoteStoreId); } catch (StoreCopyFailedException e) { _log.warn("Failed to copy and replace store", e); return(false); } } /* We install the snapshot after the store has been downloaded, * so that we are not left with a state ahead of the store. */ _snapshotService.installSnapshot(coreSnapshot); _log.info("Core snapshot installed: " + coreSnapshot); /* Starting the database will invoke the commit process factory in * the EnterpriseCoreEditionModule, which has important side-effects. */ _log.info("Starting local database"); Ensure(_localDatabase.start, "start local database after store copy"); _coreStateMachines.installCommitProcess(_localDatabase.CommitProcess); Ensure(_suspendOnStoreCopy.enable, "enable auxiliary services after store copy"); return(true); }
internal override bool DownloadSnapshot(CatchupAddressProvider addressProvider) { Semaphore.acquireUninterruptibly(); return(true); }