private bool PullByLastToken(bool cleanAll)
{
//_indexer.BeginTransaction();
//indexTokenRepository.BeginTransaction();
try
{
var pullToken = indexTokenRepository.GetLastPullToken(_indexerModel.Id.ToString(), _indexerModel.EntityType);
var pullResult = pullToken?.PullResult;
if (pullResult == null)
{
_indexer.StartIndexing(cleanAll);
}
var lastTokenMessage = pullResult?.LastToken == null || !pullResult.IsValid()
? "Begin"
: JsonConvert.SerializeObject(pullResult?.LastToken);
pullResult = _puller.PullNext(pullResult?.LastToken);
var nextTokenMessage = pullResult?.LastToken == null || !pullResult.IsValid() ? "Begin" : JsonConvert.SerializeObject(pullResult?.LastToken);
Report($@"Pulled {pullResult?.Data?.Count() ?? 0} rows from LastToken: {lastTokenMessage}, got NextToken: {nextTokenMessage}");
_indexer.Persist(pullResult?.Data);
indexTokenRepository.UpdateLastToken(_indexerModel.Id.ToString(), _indexerModel.EntityType, pullResult);
var isValid = pullResult.IsValid();
if (!isValid)
{
Report("Reached last page...");
_indexer.EndIndexing();
indexTokenRepository.CleanUp(_indexerModel.Id.ToString(), _indexerModel.EntityType);
}
//_indexer.Commit();
//indexTokenRepository.Commit();
return(isValid);
}
catch
{
//_indexer.RollBack();
//indexTokenRepository.RollBack();
throw; // never let the while loop runs forever
}
}
