/// <summary>
/// Processes the instruction batch and checks for errors.
/// </summary>
/// <param name="processed">
/// Tracks which instructions have already been processed to avoid duplicates
/// </param>
/// Returns true if all instructions in the batch were processed, otherwise false if they could not be due to the app being shut down
/// </returns>
private bool ProcessDatabaseInstructions(
CacheRefresherCollection cacheRefreshers,
IReadOnlyCollection <RefreshInstruction> instructionBatch,
CacheInstruction instruction,
HashSet <RefreshInstruction> processed,
CancellationToken cancellationToken,
ref int lastId)
{
// Execute remote instructions & update lastId.
try
{
var result = NotifyRefreshers(cacheRefreshers, instructionBatch, processed, cancellationToken);
if (result)
{
// If all instructions were processed, set the last id.
lastId = instruction.Id;
}
return(result);
}
catch (Exception ex)
{
_logger.LogError(
ex,
"DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored",
instruction.Id,
instruction.Instructions);
// We cannot throw here because this invalid instruction will just keep getting processed over and over and errors
// will be thrown over and over. The only thing we can do is ignore and move on.
lastId = instruction.Id;
return(false);
}
}
private void RefreshByIds(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string jsonIds)
{
ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier);
foreach (var id in JsonConvert.DeserializeObject <int[]>(jsonIds))
{
refresher.Refresh(id);
}
}
public void Setup()
{
ServerRegistrar = new TestServerRegistrar();
ServerMessenger = new TestServerMessenger();
var cacheRefresherCollection = new CacheRefresherCollection(() => new[] { new TestCacheRefresher() });
_distributedCache = new Cms.Core.Cache.DistributedCache(ServerMessenger, cacheRefresherCollection);
}
private void RefreshByJson(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier,
string?jsonPayload)
{
IJsonCacheRefresher refresher = GetJsonRefresher(cacheRefreshers, uniqueIdentifier);
if (jsonPayload is not null)
{
refresher.Refresh(jsonPayload);
}
}
private ICacheRefresher GetRefresher(CacheRefresherCollection cacheRefreshers, Guid id)
{
ICacheRefresher refresher = cacheRefreshers[id];
if (refresher == null)
{
throw new InvalidOperationException("Cache refresher with ID \"" + id + "\" does not exist.");
}
return(refresher);
}
/// <summary>
/// Executes the instructions against the cache refresher instances.
/// </summary>
/// <returns>
/// Returns true if all instructions were processed, otherwise false if the processing was interrupted (i.e. by app shutdown).
/// </returns>
private bool NotifyRefreshers(
CacheRefresherCollection cacheRefreshers,
IEnumerable <RefreshInstruction> instructions,
HashSet <RefreshInstruction> processed,
CancellationToken cancellationToken)
{
foreach (RefreshInstruction instruction in instructions)
{
// Check if the app is shutting down, we need to exit if this happens.
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
// This has already been processed.
if (processed.Contains(instruction))
{
continue;
}
switch (instruction.RefreshType)
{
case RefreshMethodType.RefreshAll:
RefreshAll(cacheRefreshers, instruction.RefresherId);
break;
case RefreshMethodType.RefreshByGuid:
RefreshByGuid(cacheRefreshers, instruction.RefresherId, instruction.GuidId);
break;
case RefreshMethodType.RefreshById:
RefreshById(cacheRefreshers, instruction.RefresherId, instruction.IntId);
break;
case RefreshMethodType.RefreshByIds:
RefreshByIds(cacheRefreshers, instruction.RefresherId, instruction.JsonIds);
break;
case RefreshMethodType.RefreshByJson:
RefreshByJson(cacheRefreshers, instruction.RefresherId, instruction.JsonPayload);
break;
case RefreshMethodType.RemoveById:
RemoveById(cacheRefreshers, instruction.RefresherId, instruction.IntId);
break;
}
processed.Add(instruction);
}
return(true);
}
public static void ResumeDocumentCache(CacheRefresherCollection cacheRefresherCollection)
{
s_suspended = false;
StaticApplicationLogging.Logger.LogInformation("Resume document cache (reload:{Tried}).", s_tried);
if (s_tried == false)
{
return;
}
s_tried = false;
ICacheRefresher?pageRefresher = cacheRefresherCollection[ContentCacheRefresher.UniqueId];
pageRefresher?.RefreshAll();
}
/// <summary>
/// Initializes a new instance of the <see cref="BatchedDatabaseServerMessenger"/> class.
/// </summary>
public BatchedDatabaseServerMessenger(
IMainDom mainDom,
CacheRefresherCollection cacheRefreshers,
IServerRoleAccessor serverRoleAccessor,
ILogger <BatchedDatabaseServerMessenger> logger,
ISyncBootStateAccessor syncBootStateAccessor,
IHostingEnvironment hostingEnvironment,
ICacheInstructionService cacheInstructionService,
IJsonSerializer jsonSerializer,
IRequestCache requestCache,
IRequestAccessor requestAccessor,
LastSyncedFileManager lastSyncedFileManager,
IOptions <GlobalSettings> globalSettings)
: base(mainDom, cacheRefreshers, serverRoleAccessor, logger, true, syncBootStateAccessor, hostingEnvironment, cacheInstructionService, jsonSerializer, lastSyncedFileManager, globalSettings)
{
_requestCache = requestCache;
_requestAccessor = requestAccessor;
}
private void RefreshByIds(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, string?jsonIds)
{
ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier);
if (jsonIds is null)
{
return;
}
var ids = JsonConvert.DeserializeObject <int[]>(jsonIds);
if (ids is not null)
{
foreach (var id in ids)
{
refresher.Refresh(id);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseServerMessenger"/> class.
/// </summary>
protected DatabaseServerMessenger(
IMainDom mainDom,
CacheRefresherCollection cacheRefreshers,
IServerRoleAccessor serverRoleAccessor,
ILogger <DatabaseServerMessenger> logger,
bool distributedEnabled,
ISyncBootStateAccessor syncBootStateAccessor,
IHostingEnvironment hostingEnvironment,
ICacheInstructionService cacheInstructionService,
IJsonSerializer jsonSerializer,
LastSyncedFileManager lastSyncedFileManager,
IOptionsMonitor <GlobalSettings> globalSettings)
: base(distributedEnabled)
{
_cancellationToken = _cancellationTokenSource.Token;
_mainDom = mainDom;
_cacheRefreshers = cacheRefreshers;
_serverRoleAccessor = serverRoleAccessor;
_hostingEnvironment = hostingEnvironment;
Logger = logger;
_syncBootStateAccessor = syncBootStateAccessor;
CacheInstructionService = cacheInstructionService;
JsonSerializer = jsonSerializer;
_lastSyncedFileManager = lastSyncedFileManager;
GlobalSettings = globalSettings.CurrentValue;
_lastPruned = _lastSync = DateTime.UtcNow;
_syncIdle = new ManualResetEvent(true);
globalSettings.OnChange(x => GlobalSettings = x);
using (var process = Process.GetCurrentProcess())
{
// See notes on _localIdentity
LocalIdentity = Environment.MachineName // eg DOMAIN\SERVER
+ "/" + hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT
+ " [P" + process.Id // eg 1234
+ "/D" + AppDomain.CurrentDomain.Id // eg 22
+ "] " + Guid.NewGuid().ToString("N").ToUpper(); // make it truly unique
}
_initialized = new Lazy <SyncBootState?>(InitializeWithMainDom);
}
/// <inheritdoc/>
public ProcessInstructionsResult ProcessInstructions(
CacheRefresherCollection cacheRefreshers,
ServerRole serverRole,
CancellationToken cancellationToken,
string localIdentity,
DateTime lastPruned,
int lastId)
{
using (_profilingLogger.DebugDuration <CacheInstructionService>("Syncing from database..."))
using (ICoreScope scope = ScopeProvider.CreateCoreScope())
{
var numberOfInstructionsProcessed = ProcessDatabaseInstructions(cacheRefreshers, cancellationToken,
localIdentity, ref lastId);
// Check for pruning throttling.
if (cancellationToken.IsCancellationRequested || (DateTime.UtcNow - lastPruned) <=
_globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations)
{
scope.Complete();
return(ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId));
}
var instructionsWerePruned = false;
switch (serverRole)
{
case ServerRole.Single:
case ServerRole.SchedulingPublisher:
PruneOldInstructions();
instructionsWerePruned = true;
break;
}
scope.Complete();
return(instructionsWerePruned
? ProcessInstructionsResult.AsCompletedAndPruned(numberOfInstructionsProcessed, lastId)
: ProcessInstructionsResult.AsCompleted(numberOfInstructionsProcessed, lastId));
}
}
private IJsonCacheRefresher GetJsonRefresher(CacheRefresherCollection cacheRefreshers, Guid id) => GetJsonRefresher(GetRefresher(cacheRefreshers, id));
private void RemoveById(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, int id)
{
ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier);
refresher.Remove(id);
}
private void RefreshAll(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier)
{
ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier);
refresher.RefreshAll();
}
/// <summary>
/// Process instructions from the database.
/// </summary>
/// <remarks>
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// </remarks>
/// <returns>Number of instructions processed.</returns>
private int ProcessDatabaseInstructions(CacheRefresherCollection cacheRefreshers, CancellationToken cancellationToken, string localIdentity, ref int lastId)
{
// NOTE:
// We 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that
// would be a good idea since instructions could keep getting added and then all other threads will probably get stuck from serving requests
// (depending on what the cache refreshers are doing). I think it's best we do the one time check, process them and continue, if there are
// pending requests after being processed, they'll just be processed on the next poll.
//
// TODO: not true if we're running on a background thread, assuming we can?
// Only retrieve the top 100 (just in case there are tons).
// Even though MaxProcessingInstructionCount is by default 1000 we still don't want to process that many
// rows in one request thread since each row can contain a ton of instructions (until 7.5.5 in which case
// a row can only contain MaxProcessingInstructionCount).
const int MaxInstructionsToRetrieve = 100;
// Only process instructions coming from a remote server, and ignore instructions coming from
// the local server as they've already been processed. We should NOT assume that the sequence of
// instructions in the database makes any sense whatsoever, because it's all async.
// Tracks which ones have already been processed to avoid duplicates
var processed = new HashSet <RefreshInstruction>();
var numberOfInstructionsProcessed = 0;
// It would have been nice to do this in a Query instead of Fetch using a data reader to save
// some memory however we cannot do that because inside of this loop the cache refreshers are also
// performing some lookups which cannot be done with an active reader open.
IEnumerable <CacheInstruction> pendingInstructions = _cacheInstructionRepository.GetPendingInstructions(lastId, MaxInstructionsToRetrieve);
lastId = 0;
foreach (CacheInstruction instruction in pendingInstructions)
{
// If this flag gets set it means we're shutting down! In this case, we need to exit asap and cannot
// continue processing anything otherwise we'll hold up the app domain shutdown.
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (instruction.OriginIdentity == localIdentity)
{
// Just skip that local one but update lastId nevertheless.
lastId = instruction.Id;
continue;
}
// Deserialize remote instructions & skip if it fails.
if (!TryDeserializeInstructions(instruction, out JArray jsonInstructions))
{
lastId = instruction.Id; // skip
continue;
}
List <RefreshInstruction> instructionBatch = GetAllInstructions(jsonInstructions);
// Process as per-normal.
var success = ProcessDatabaseInstructions(cacheRefreshers, instructionBatch, instruction, processed, cancellationToken, ref lastId);
// If they couldn't be all processed (i.e. we're shutting down) then exit.
if (success == false)
{
_logger.LogInformation("The current batch of instructions was not processed, app is shutting down");
break;
}
numberOfInstructionsProcessed++;
}
return(numberOfInstructionsProcessed);
}
private void RefreshByGuid(CacheRefresherCollection cacheRefreshers, Guid uniqueIdentifier, Guid id)
{
ICacheRefresher refresher = GetRefresher(cacheRefreshers, uniqueIdentifier);
refresher.Refresh(id);
}
public DropDownPropertyEditorsMigration(IMigrationContext context, CacheRefresherCollection cacheRefreshers, IServerMessenger serverMessenger)
: base(context)
{
_cacheRefreshers = cacheRefreshers;
_serverMessenger = serverMessenger;
}
public DistributedCache(IServerMessenger serverMessenger, CacheRefresherCollection cacheRefreshers)
{
_serverMessenger = serverMessenger;
_cacheRefreshers = cacheRefreshers;
}
