private Dictionary <string, string> GetLanguages()
{
return(_languageBranchRepository.ListEnabled()
.ToDictionary(x => x.LanguageID, x => x.Name));
}
public override string Execute()
{
var finalStatus = new StringBuilder();
var skippedReason = new StringBuilder();
var results = new BulkBatchResult();
var bulkCounter = 1;
var logMessage = $"Indexing starting. Content retrived and indexed in bulks of {_settings.BulkSize} items.";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
try
{
var languages = _languageBranchRepository.ListEnabled();
var contentReferences = GetContentReferences();
logMessage = $"Retrieved {contentReferences.Count} ContentReference items from the following languages: {String.Join(", ", languages.Select(l => l.LanguageID))}";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
while (contentReferences.Count > 0)
{
var contents = GetDescendentContents(contentReferences.Take(_settings.BulkSize).ToList(), languages, bulkCounter);
var batchResult = IndexContents(contents, bulkCounter);
results.Batches.AddRange(batchResult.Batches);
contentReferences.RemoveRange(0, contentReferences.Count >= _settings.BulkSize
? _settings.BulkSize
: contentReferences.Count);
bulkCounter++;
}
UpdateMappings(languages, CustomIndexName);
if (IsStopped)
{
return("Aborted by user");
}
}
catch (Exception ex)
{
_logger.Error(ex.Message, ex);
finalStatus.AppendLine();
finalStatus.AppendLine(ex.Message);
finalStatus.Append("<pre>").Append(ex.StackTrace).AppendLine("</pre>");
// If we re-throw here, stacktrace won't be displayed
}
var finished = $"Processed {results.Batches.Count} batches of {_settings.BulkSize} items, for a total of {results.Batches.Sum(b => b.Items.Length)} items to Elasticsearch index.";
for (var i = 1; i <= results.Batches.Count; i++)
{
if (results.Batches[i - 1].Errors)
{
var message = $" Batch {i} failed.";
foreach (var item in results.Batches[i - 1].Items.Where(item => item.Status >= 400))
{
message += item.ToString();
}
finished += message;
_logger.Warning(message);
}
}
finalStatus.Insert(0, finished);
OnStatusChanged(finalStatus.ToString());
_logger.Information(skippedReason.ToString());
_logger.Information(finished);
finalStatus.AppendLine(skippedReason.ToString().Replace("\n", "<br/>"));
return(finalStatus.ToString());
}
private static IEnumerable <CultureInfo> GetLanguages()
{
ILanguageBranchRepository languageRepo = ServiceLocator.Current.GetInstance <ILanguageBranchRepository>();
return(languageRepo.ListEnabled().Select(lb => lb.Culture));
}
public string[] GetGlobalLanguageOptions()
{
return(_languageBranchRepository.ListEnabled().Select(x => x.Culture.Name).ToArray());
}
public override string Execute()
{
if (BeforeIndexContent != null)
{
_logger.Debug("Firing subscribed event BeforeIndexContent");
BeforeIndexContent(EventArgs.Empty);
}
var finalStatus = new StringBuilder();
var skippedReason = new StringBuilder();
var results = new BulkBatchResult();
var logMessage = $"Indexing starting. Content retrived and indexed in bulks of {_settings.BulkSize} items.";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
try
{
var languages = _languageBranchRepository.ListEnabled();
if (!IndicesExists(languages))
{
throw new InvalidOperationException("One or more indices is missing, please create them.");
}
if (!Server.Plugins.Any(p => p.Component.Equals("ingest-attachment")))
{
throw new InvalidOperationException("Plugin 'ingest-attachment' is missing, please install it.");
}
var contentReferences = GetContentReferences();
logMessage = $"Retrieved {contentReferences.Count} ContentReference items from the following languages: {String.Join(", ", languages.Select(l => l.LanguageID))}";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
var contentList = new List <IContent>();
// Fetch all content to index
while (contentReferences.Count > 0)
{
if (IsStopped)
{
return("Aborted by user");
}
var contents = GetDescendentContents(contentReferences.Take(_settings.BulkSize).ToList(), languages);
contents.RemoveAll(_indexer.ShouldHideFromSearch);
contents.RemoveAll(_indexer.IsExludedType);
contentList.AddRange(contents);
var removeCount = contentReferences.Count >= _settings.BulkSize ? _settings.BulkSize : contentReferences.Count;
contentReferences.RemoveRange(0, removeCount);
}
// Is this the first run?
var isFirstRun = GetTotalDocumentCount(languages) == 0;
// Update mappings on first run after analyzing the actual content
if (isFirstRun)
{
var uniqueTypes = contentList.Select(content =>
{
var type = content.GetType();
return(type.Name.EndsWith("Proxy") ? type.BaseType : type);
})
.Distinct()
.ToArray();
UpdateMappings(languages, uniqueTypes);
}
var bulkCount = Math.Ceiling(contentList.Count / (double)_settings.BulkSize);
for (var i = 1; i <= bulkCount; i++)
{
OnStatusChanged($"Bulk {i} of {bulkCount} - Preparing bulk update of {_settings.BulkSize} items...");
var batch = contentList.Take(_settings.BulkSize);
var batchResult = IndexContents(batch);
results.Batches.AddRange(batchResult.Batches);
var removeCount = contentList.Count >= _settings.BulkSize ? _settings.BulkSize : contentList.Count;
contentList.RemoveRange(0, removeCount);
if (IsStopped)
{
return("Aborted by user");
}
}
}
catch (Exception ex)
{
_logger.Error(ex.Message, ex);
finalStatus.AppendLine();
finalStatus.AppendLine(ex.Message);
finalStatus.Append("<pre>").Append(ex.StackTrace).AppendLine("</pre>");
// If we re-throw here, stacktrace won't be displayed
}
var finishedBuilder = new StringBuilder($"Processed {results.Batches.Count} batches of {_settings.BulkSize} items, for a total of {results.Batches.Sum(b => b.Items.Length)} items to Elasticsearch index.");
for (var i = 1; i <= results.Batches.Count; i++)
{
if (results.Batches[i - 1].Errors)
{
var messageBuilder = new StringBuilder($" Batch {i} failed. Details: \n");
foreach (var item in results.Batches[i - 1].Items.Where(item => item.Status >= 400))
{
messageBuilder.AppendLine(item.ToString());
}
var message = messageBuilder.ToString();
finishedBuilder.AppendLine(message);
_logger.Warning(message);
}
}
finalStatus.Insert(0, finishedBuilder.ToString().Replace("\n", "<br/>"));
OnStatusChanged(finalStatus.ToString());
_logger.Information(skippedReason.ToString());
_logger.Information(finishedBuilder.ToString());
finalStatus.AppendLine(skippedReason.ToString().Replace("\n", "<br/>"));
return(finalStatus.ToString());
}
/// <summary>
/// Initializes a new instance of the <see cref="CmsContext" /> class.
/// </summary>
public CmsContext()
{
ContentReference.RootPage = new PageReference(1).CreateWritableClone();
////ContentReference.StartPage.CreateWritableClone().ID = 4;
SiteDefinition.Current = A.Fake <SiteDefinition>();
SiteDefinition.Current.StartPage = new PageReference(4);
this.StartPageUrl = "/";
Context.Current = A.Fake <IContext>();
ServiceLocator.SetLocator(A.Fake <IServiceLocator>());
this.ContentLoader = A.Fake <IContentLoader>();
this.UrlResolver = A.Fake <UrlResolver>();
this.TemplateResolver = A.Fake <TemplateResolver>();
this.ContentRepository = A.Fake <IContentRepository>();
this.ContentTypeRepository = A.Fake <IContentTypeRepository>();
this.LanguageBranchRepository = A.Fake <ILanguageBranchRepository>();
this.LoaderOptions = A.Fake <LoaderOptions>();
this.LocalizationService = A.Fake <LocalizationService>();
this.ProviderBasedLocalizationService = new ProviderBasedLocalizationService();
A.CallTo(() => Context.Current.RequestTime).Returns(DateTime.Now);
A.CallTo(() => ServiceLocator.Current.GetInstance <IContentRepository>()).Returns(this.ContentRepository);
A.CallTo(() => ServiceLocator.Current.GetInstance <IContentTypeRepository>())
.Returns(this.ContentTypeRepository);
A.CallTo(() => ServiceLocator.Current.GetInstance <ILanguageBranchRepository>())
.Returns(this.LanguageBranchRepository);
A.CallTo(() => ServiceLocator.Current.GetInstance <TemplateResolver>()).Returns(this.TemplateResolver);
A.CallTo(() => this.UrlResolver.GetUrl(ContentReference.StartPage)).Returns(this.StartPageUrl);
A.CallTo(() => ServiceLocator.Current.GetInstance <ProviderBasedLocalizationService>())
.Returns(this.ProviderBasedLocalizationService);
A.CallTo(() => ServiceLocator.Current.GetInstance <LocalizationService>())
.Returns(this.ProviderBasedLocalizationService);
// Also fake the TryGetExistingInstance for the LocalizationService
LocalizationService ignored;
A.CallTo(() => ServiceLocator.Current.TryGetExistingInstance(out ignored))
.Returns(true)
.AssignsOutAndRefParameters(this.ProviderBasedLocalizationService);
ILanguageBranchRepository languageBranchRepository = this.LanguageBranchRepository;
if (languageBranchRepository != null)
{
languageBranchRepository.Save(new LanguageBranch(MasterLanguage));
languageBranchRepository.Save(new LanguageBranch(SecondLanguage));
A.CallTo(() => languageBranchRepository.ListEnabled())
.Returns(new[] { new LanguageBranch(MasterLanguage), new LanguageBranch(SecondLanguage) });
}
this.RegisterMocks();
HttpContextBase = this.CreateFakeHttpContext();
}
private IList <LanguageBranch> GetEnabledSiteLanguages()
{
ILanguageBranchRepository languageBranchRepository = ServiceLocator.Current.GetInstance <ILanguageBranchRepository>();
return(languageBranchRepository.ListEnabled());
}
public override string Execute()
{
if (BeforeIndexContent != null)
{
_logger.Debug("Firing subscribed event BeforeIndexContent");
BeforeIndexContent(EventArgs.Empty);
}
var failed = false;
var finalStatus = new StringBuilder();
var skippedReason = new StringBuilder();
var results = new BulkBatchResult();
var logMessage = $"Indexing starting. Content retrieved and indexed in bulks of {_settings.BulkSize} items.";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
try
{
var languages = _languageBranchRepository.ListEnabled();
if (!IndicesExists(languages))
{
throw new InvalidOperationException("One or more indices is missing, please create them.");
}
if (!_serverInfoService.ListPlugins().Any(p => p.Component.Equals("ingest-attachment")))
{
throw new InvalidOperationException("Plugin 'ingest-attachment' is missing, please install it.");
}
List <ContentReference> contentReferences = GetContentReferences();
logMessage = $"Retrieved {contentReferences.Count} ContentReference items from the following languages: {String.Join(", ", languages.Select(l => l.LanguageID))}";
_logger.Information(logMessage);
OnStatusChanged(logMessage);
var contentList = new List <IContent>();
// Fetch all content to index
while (contentReferences.Count > 0)
{
if (IsStopped)
{
return("Aborted by user");
}
List <IContent> contents = GetDescendentContents(contentReferences.Take(_settings.BulkSize).ToList(), languages);
contents.RemoveAll(_indexer.SkipIndexing);
contents.RemoveAll(_indexer.IsExcludedType);
contentList.AddRange(contents);
var removeCount = contentReferences.Count >= _settings.BulkSize ? _settings.BulkSize : contentReferences.Count;
contentReferences.RemoveRange(0, removeCount);
}
var mediaData = contentList.OfType <MediaData>().ToList();
contentList.RemoveAll(c => c is MediaData);
var bulkCount = Math.Ceiling(contentList.Count / (double)_settings.BulkSize);
for (var i = 1; i <= bulkCount; i++)
{
OnStatusChanged($"Indexing bulk {i} of {bulkCount} (Bulk size: {_settings.BulkSize})");
var batch = contentList.Take(_settings.BulkSize);
var batchResult = IndexContents(batch);
results.Batches.AddRange(batchResult.Batches);
var removeCount = contentList.Count >= _settings.BulkSize ? _settings.BulkSize : contentList.Count;
contentList.RemoveRange(0, removeCount);
if (IsStopped)
{
return("Aborted by user");
}
}
// Index media files one by one regardless of bulk size.
var mediaBatchResults = IndexMediaData(mediaData);
results.Batches.AddRange(mediaBatchResults.Batches);
// Put Best Bets back into the index
RestoreBestBets(languages);
}
catch (Exception ex)
{
_logger.Error(ex.Message, ex);
finalStatus
.AppendLine()
.AppendLine(ex.Message)
.AppendLine()
.AppendLine("Stacktrace:")
.AppendLine(ex.StackTrace);
failed = true;
// If we re-throw here, stacktrace won't be displayed
}
var finishedBuilder = new StringBuilder($"Processed {results.Batches.Count} batches, for a total of {results.Batches.Sum(b => b?.Items?.Length ?? 0)} items to Elasticsearch index.\n");
for (var i = 1; i <= results.Batches.Count; i++)
{
if (results.Batches[i - 1]?.Errors ?? false)
{
var messageBuilder = new StringBuilder($" Batch {i} failed. Details: \n");
foreach (var item in results.Batches[i - 1].Items.Where(item => item.Status >= 400))
{
messageBuilder.AppendLine(item.ToString());
}
var message = messageBuilder.ToString();
finishedBuilder.AppendLine(message);
_logger.Warning(message);
}
}
finalStatus.Insert(0, finishedBuilder.ToString());
OnStatusChanged(finalStatus.ToString());
_logger.Information(skippedReason.ToString());
_logger.Information(finishedBuilder.ToString());
finalStatus.AppendLine(skippedReason.ToString());
var status = finalStatus.ToString().Replace("\n", "<br/>");
if (failed)
{
throw new Exception(status);
}
return(status);
}
