public override PopulationProgress Progress(PopulationProgress scanProgress) { // A general note on scanProgress.getTotal(). Before the scan is completed most progress parts will base their estimates on that value. // It is known that it may be slightly higher since it'll be based on store high-id, not the actual count. // This is fine, but it creates this small "jump" in the progress in the middle somewhere when it switches from scan to merge. // This also exists in the most basic population progress reports, but there it will be less visible since it will jump from // some close-to-100 percentage to 100% and ONLINE. // This progress report will consist of a couple of smaller parts, weighted differently based on empirically collected values. // The weights will not be absolutely correct in all environments, but they don't have to be either, it will just result in some // slices of the percentage progression range progressing at slightly different paces. However, progression of progress reporting // naturally fluctuates anyway due to data set and I/O etc. so this is not an actual problem. Org.Neo4j.Storageengine.Api.schema.PopulationProgress_MultiBuilder builder = PopulationProgress.multiple(); // Add scan progress (this one weights a bit heavier than the others) builder.Add(scanProgress, 4); // Add merge progress if (_allScanUpdates.Count > 0) { // The parts are merged in parallel so just take the first one and it will represent the whole merge progress. // It will be fairly accurate, but slightly off sometimes if other threads gets scheduling problems, i.e. if this part // finish far apart from others. long completed = 0; long total = 0; if (_scanCompleted) { // We know the actual entry count to write during merge since we have been monitoring those values ThreadLocalBlockStorage part = first(_allScanUpdates); completed = part.EntriesMergedConflict; total = part.TotalEntriesToMerge; } builder.Add(PopulationProgress.single(completed, total), 1); } // Add tree building incl. external updates PopulationProgress treeBuildProgress; if (_allScanUpdates.All(part => part.mergeStarted)) { long entryCount = _allScanUpdates.Select(part => part.count).Sum() + _externalUpdates.count(); treeBuildProgress = PopulationProgress.single(_numberOfAppliedScanUpdates + _numberOfAppliedExternalUpdates, entryCount); } else { treeBuildProgress = Org.Neo4j.Storageengine.Api.schema.PopulationProgress_Fields.None; } builder.Add(treeBuildProgress, 2); return(builder.Build()); }
private ThreadLocalBlockStorage NewThreadLocalBlockStorage() { lock (this) { Preconditions.checkState(!_cancellation.cancelled(), "Already closed"); Preconditions.checkState(!_scanCompleted, "Scan has already been completed"); try { int id = _allScanUpdates.Count; ThreadLocalBlockStorage blockStorage = new ThreadLocalBlockStorage(this, id); _allScanUpdates.Add(blockStorage); return(blockStorage); } catch (IOException e) { throw new UncheckedIOException(e); } } }