private bool PullByLastToken(bool cleanAll) { //_indexer.BeginTransaction(); //indexTokenRepository.BeginTransaction(); try { var pullToken = indexTokenRepository.GetLastPullToken(_indexerModel.Id.ToString(), _indexerModel.EntityType); var pullResult = pullToken?.PullResult; if (pullResult == null) { _indexer.StartIndexing(cleanAll); } var lastTokenMessage = pullResult?.LastToken == null || !pullResult.IsValid() ? "Begin" : JsonConvert.SerializeObject(pullResult?.LastToken); pullResult = _puller.PullNext(pullResult?.LastToken); var nextTokenMessage = pullResult?.LastToken == null || !pullResult.IsValid() ? "Begin" : JsonConvert.SerializeObject(pullResult?.LastToken); Report($@"Pulled {pullResult?.Data?.Count() ?? 0} rows from LastToken: {lastTokenMessage}, got NextToken: {nextTokenMessage}"); _indexer.Persist(pullResult?.Data); indexTokenRepository.UpdateLastToken(_indexerModel.Id.ToString(), _indexerModel.EntityType, pullResult); var isValid = pullResult.IsValid(); if (!isValid) { Report("Reached last page..."); _indexer.EndIndexing(); indexTokenRepository.CleanUp(_indexerModel.Id.ToString(), _indexerModel.EntityType); } //_indexer.Commit(); //indexTokenRepository.Commit(); return(isValid); } catch { //_indexer.RollBack(); //indexTokenRepository.RollBack(); throw; // never let the while loop runs forever } }