public bool DeleteIndex(string indexName)
{
ILockFile lockFile = null;
var settingsFilename = GetSettingsFileName(indexName);
var lockFilename = settingsFilename + ".lock";
// acquire a lock file on the index
if (!_lockFileManager.TryAcquireLock(lockFilename, ref lockFile))
{
Logger.Information("Could not delete the index. Already in use.");
return(false);
}
using (lockFile) {
if (!_indexManager.HasIndexProvider())
{
return(false);
}
var searchProvider = _indexManager.GetSearchIndexProvider();
if (searchProvider.Exists(indexName))
{
searchProvider.DeleteIndex(indexName);
}
DeleteSettings(indexName);
}
return(true);
}
public void GenerateShouldNotRunIfLocked()
{
_appDataFolder.CreateFile(_warmupFilename, "");
ILockFile lockFile = null;
_lockFileManager.TryAcquireLock(_lockFilename, ref lockFile);
using (lockFile) {
_warmupUpdater.Generate();
Assert.That(_appDataFolder.ListFiles(WarmupFolder).Count(), Is.EqualTo(0));
}
_warmupUpdater.Generate();
Assert.That(_appDataFolder.ListFiles(WarmupFolder).Count(), Is.EqualTo(0));
}
public void LockShouldBeGrantedWhenDoesNotExist()
{
ILockFile lockFile = null;
var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.True(granted);
Assert.NotNull(lockFile);
Assert.True(_lockFileManager.IsLocked("foo.txt.lock"));
Assert.Equal(_appDataFolder.ListFiles("").Count(), 1);
}
public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
{
Guard.NotNull(ctx, nameof(ctx));
// TODO: (core) Check later if this is still necessary
//var dataTokens = _httpContextAccessor.HttpContext?.GetRouteData()?.DataTokens;
//if (dataTokens != null)
//{
// // Double seo code otherwise
// dataTokens["CultureCodeReplacement"] = string.Empty;
//}
var languageData = new Dictionary <int, LanguageData>();
foreach (var language in ctx.Languages)
{
var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);
if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
{
// Process only languages that are unlocked right now
// It is possible that an HTTP request triggered the generation
// of a language specific sitemap.
try
{
var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
var data = new LanguageData
{
Store = ctx.Store,
Language = language,
LockFile = lockFile,
LockFilePath = lockFilePath,
TempDir = sitemapDir + "~",
FinalDir = sitemapDir,
BaseUrl = await BuildBaseUrlAsync(ctx.Store, language)
};
_tenantRoot.TryDeleteDirectory(data.TempDir);
_tenantRoot.TryCreateDirectory(data.TempDir);
languageData[language.Id] = data;
}
catch
{
await lockFile.ReleaseAsync();
throw;
}
}
}
if (languageData.Count == 0)
{
Logger.Warn("XML sitemap rebuild already in process.");
return;
}
var languages = languageData.Values.Select(x => x.Language);
var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();
// All sitemaps grouped by language
var sitemaps = new Multimap <int, XmlSitemapNode>();
var compositeFileLock = new AsyncActionDisposable(async() =>
{
foreach (var data in languageData.Values)
{
await data.LockFile.ReleaseAsync();
}
});
await using (compositeFileLock)
{
// Impersonate
var prevCustomer = _services.WorkContext.CurrentCustomer;
// no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
// TODO: (core) Do not attempt to update CurrentCustomer entity if it is untracked (determine where applicable)
_services.WorkContext.CurrentCustomer = (await _customerService.GetCustomerBySystemNameAsync(SystemCustomerNames.SearchEngine, false)) ?? prevCustomer;
try
{
var nodes = new List <XmlSitemapNode>();
var providers = CreateProviders(ctx);
var total = (await providers.SelectAsync(x => x.GetTotalCountAsync()).ToListAsync(ctx.CancellationToken)).Sum();
var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
var hasIndex = totalSegments > 1;
var indexNodes = new Multimap <int, XmlSitemapNode>();
var segment = 0;
var numProcessed = 0;
CheckSitemapCount(totalSegments);
using (new DbContextScope(_db, autoDetectChanges: false, forceNoTracking: true, lazyLoading: false))
{
var entities = EnlistEntitiesAsync(providers);
await foreach (var batch in entities.SliceAsync(ctx.CancellationToken, MaximumSiteMapNodeCount))
{
if (ctx.CancellationToken.IsCancellationRequested)
{
break;
}
segment++;
numProcessed = segment * MaximumSiteMapNodeCount;
ctx.ProgressCallback?.Invoke(numProcessed, total, "{0} / {1}".FormatCurrent(numProcessed, total));
var slugs = await GetUrlRecordCollectionsForBatchAsync(batch.Select(x => x.Entry).ToList(), languageIds);
foreach (var data in languageData.Values)
{
var language = data.Language;
var baseUrl = data.BaseUrl;
// Create all node entries for this segment
var entries = batch
.Where(x => x.Entry.LanguageId.GetValueOrDefault() == 0 || x.Entry.LanguageId.Value == language.Id)
.Select(x => x.Provider.CreateNode(_linkGenerator, baseUrl, x.Entry, slugs[x.Entry.EntityName], language));
sitemaps[language.Id].AddRange(entries.Where(x => x != null));
// Create index node for this segment/language combination
if (hasIndex)
{
indexNodes[language.Id].Add(new XmlSitemapNode
{
LastMod = sitemaps[language.Id].Select(x => x.LastMod).Where(x => x.HasValue).DefaultIfEmpty().Max(),
Loc = GetSitemapIndexUrl(segment, baseUrl),
});
}
if (segment % 5 == 0 || segment == totalSegments)
{
// Commit every 5th segment (10.000 nodes) temporarily to disk to minimize RAM usage
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[language.Id]);
await SaveTempAsync(documents, data, segment - documents.Count + (hasIndex ? 1 : 0));
documents.Clear();
sitemaps.RemoveAll(language.Id);
}
}
slugs.Clear();
}
// Process custom nodes
if (!ctx.CancellationToken.IsCancellationRequested)
{
ctx.ProgressCallback?.Invoke(numProcessed, total, "Processing custom nodes".FormatCurrent(numProcessed, total));
await ProcessCustomNodesAsync(ctx, sitemaps);
foreach (var data in languageData.Values)
{
if (sitemaps.ContainsKey(data.Language.Id) && sitemaps[data.Language.Id].Count > 0)
{
var documents = GetSiteMapDocuments((IReadOnlyCollection <XmlSitemapNode>)sitemaps[data.Language.Id]);
await SaveTempAsync(documents, data, (segment + 1) - documents.Count + (hasIndex ? 1 : 0));
}
else if (segment == 0)
{
// Ensure that at least one entry exists. Otherwise,
// the system will try to rebuild again.
var homeNode = new XmlSitemapNode {
LastMod = DateTime.UtcNow, Loc = data.BaseUrl
};
var documents = GetSiteMapDocuments(new List <XmlSitemapNode> {
homeNode
});
await SaveTempAsync(documents, data, 0);
}
}
}
}
ctx.CancellationToken.ThrowIfCancellationRequested();
ctx.ProgressCallback?.Invoke(totalSegments, totalSegments, "Finalizing...'");
foreach (var data in languageData.Values)
{
// Create index documents (if any)
if (hasIndex && indexNodes.Any())
{
var indexDocument = CreateSitemapIndexDocument(indexNodes[data.Language.Id]);
await SaveTempAsync(new List <string> {
indexDocument
}, data, 0);
}
// Save finally (actually renames temp folder)
await SaveFinalAsync(data);
}
}
finally
{
// Undo impersonation
_services.WorkContext.CurrentCustomer = prevCustomer;
sitemaps.Clear();
foreach (var data in languageData.Values)
{
if (_tenantRoot.DirectoryExists(data.TempDir))
{
_tenantRoot.TryDeleteDirectory(data.TempDir);
}
}
}
}
}
/// <summary>
/// Tries to acquire a lock with the specified parameters
/// </summary>
/// <param name="name">Unique name of the lock</param>
/// <returns>The ILockFile instance on success or null if the lock couldn't be acquired.</returns>
public static ILockFile TryAcquireLock(this ILockFileManager lockFileManager, string name)
{
return(lockFileManager.TryAcquireLock(name, new TimeSpan(0, 0, 0, 4)));
}
public void LockShouldBeGrantedWhenDoesNotExist()
{
ILockFile lockFile = null;
var granted = _lockFileManager.TryAcquireLock("foo.txt.lock", ref lockFile);
Assert.That(granted, Is.True);
Assert.That(lockFile, Is.Not.Null);
Assert.That(_lockFileManager.IsLocked("foo.txt.lock"), Is.True);
Assert.That(_appDataFolder.ListFiles("").Count(), Is.EqualTo(1));
}
public void EnsureGenerate()
{
var baseUrl = _orchardServices.WorkContext.CurrentSite.BaseUrl;
var part = _orchardServices.WorkContext.CurrentSite.As <WarmupSettingsPart>();
// do nothing while the base url setting is not defined
if (String.IsNullOrWhiteSpace(baseUrl))
{
return;
}
// prevent multiple appdomains from rebuilding the static page concurrently (e.g., command line)
ILockFile lockFile = null;
if (!_lockFileManager.TryAcquireLock(_lockFilename, ref lockFile))
{
return;
}
using (lockFile) {
// check if we need to regenerate the pages by reading the last time it has been done
// 1- if the warmup file doesn't exists, generate the pages
// 2- otherwise, if the scheduled generation option is on, check if the delay is over
if (_appDataFolder.FileExists(_warmupPath))
{
try {
var warmupContent = _appDataFolder.ReadFile(_warmupPath);
var expired = XmlConvert.ToDateTimeOffset(warmupContent).AddMinutes(part.Delay);
if (expired > _clock.UtcNow)
{
return;
}
}
catch {
// invalid file, delete continue processing
_appDataFolder.DeleteFile(_warmupPath);
}
}
// delete peviously generated pages, by reading the Warmup Report file
try {
var encodedPrefix = WarmupUtility.EncodeUrl("http://www.");
foreach (var reportEntry in _reportManager.Read())
{
try {
// use FileName as the SiteBaseUrl could have changed in the meantime
var path = _appDataFolder.Combine(BaseFolder, reportEntry.Filename);
_appDataFolder.DeleteFile(path);
// delete the www-less version too if it's available
if (reportEntry.Filename.StartsWith(encodedPrefix, StringComparison.OrdinalIgnoreCase))
{
var filename = WarmupUtility.EncodeUrl("http://") + reportEntry.Filename.Substring(encodedPrefix.Length);
path = _appDataFolder.Combine(BaseFolder, filename);
_appDataFolder.DeleteFile(path);
}
}
catch (Exception e) {
Logger.Error(e, "Could not delete specific warmup file: ", reportEntry.Filename);
}
}
}
catch (Exception e) {
Logger.Error(e, "Could not read warmup report file");
}
var reportEntries = new List <ReportEntry>();
if (!String.IsNullOrEmpty(part.Urls))
{
// loop over every relative url to generate the contents
using (var urlReader = new StringReader(part.Urls)) {
string relativeUrl;
while (null != (relativeUrl = urlReader.ReadLine()))
{
if (String.IsNullOrWhiteSpace(relativeUrl))
{
continue;
}
string url = null;
relativeUrl = relativeUrl.Trim();
try {
url = VirtualPathUtility.RemoveTrailingSlash(baseUrl) + relativeUrl;
var filename = WarmupUtility.EncodeUrl(url.TrimEnd('/'));
var path = _appDataFolder.Combine(BaseFolder, filename);
var download = _webDownloader.Download(url);
if (download != null)
{
if (download.StatusCode == HttpStatusCode.OK)
{
// success
_appDataFolder.CreateFile(path, download.Content);
reportEntries.Add(new ReportEntry {
RelativeUrl = relativeUrl,
Filename = filename,
StatusCode = (int)download.StatusCode,
CreatedUtc = _clock.UtcNow
});
// if the base url contains http://www, then also render the www-less one);
if (url.StartsWith("http://www.", StringComparison.OrdinalIgnoreCase))
{
url = "http://" + url.Substring("http://www.".Length);
filename = WarmupUtility.EncodeUrl(url.TrimEnd('/'));
path = _appDataFolder.Combine(BaseFolder, filename);
_appDataFolder.CreateFile(path, download.Content);
}
}
else
{
reportEntries.Add(new ReportEntry {
RelativeUrl = relativeUrl,
Filename = filename,
StatusCode = (int)download.StatusCode,
CreatedUtc = _clock.UtcNow
});
}
}
else
{
// download failed
reportEntries.Add(new ReportEntry {
RelativeUrl = relativeUrl,
Filename = filename,
StatusCode = 0,
CreatedUtc = _clock.UtcNow
});
}
}
catch (Exception e) {
Logger.Error(e, "Could not extract warmup page content for: ", url);
}
}
}
}
_reportManager.Create(reportEntries);
// finally write the time the generation has been executed
_appDataFolder.CreateFile(_warmupPath, XmlConvert.ToString(_clock.UtcNow, XmlDateTimeSerializationMode.Utc));
}
}
