/// <summary>
/// Creates an accessor.
/// </summary>
/// <param name="indexer"> indexer Indexer used to index symbols in the symbol array.</param>
/// <param name="length"> length Length of a section of the symbol array.</param>
/// <param name="symbols"> symbols Symbol array.</param>
public AccessorBase
(IIndexer indexer, int length, ISymbolArray symbols)
{
Indexer = indexer;
Length = length;
this.symbols = symbols;
}
public ContentChangeTracker(IIndexer indexer, IPersister persister, IWorker worker, ConnectionMonitor connection, IErrorNotifier errors, DatabaseSection config)
{
this.indexer = indexer;
this.persister = persister;
this.worker = worker;
this.errors = errors;
this.async = config.Search.AsyncIndexing;
this.handleErrors = config.Search.HandleErrors;
RetryInterval = TimeSpan.FromMinutes(2);
if(config.Search.Enabled)
{
connection.Online += delegate
{
persister.ItemSaved += persister_ItemSaved;
persister.ItemMoving += persister_ItemMoving;
persister.ItemMoved += persister_ItemMoved;
persister.ItemCopied += persister_ItemCopied;
persister.ItemDeleted += persister_ItemDeleted;
};
connection.Offline += delegate
{
persister.ItemSaved -= persister_ItemSaved;
persister.ItemMoving -= persister_ItemMoving;
persister.ItemMoved -= persister_ItemMoved;
persister.ItemCopied -= persister_ItemCopied;
persister.ItemDeleted -= persister_ItemDeleted;
};
}
}
public void TestSetup()
{
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
public void Initialize()
{
_cwsDir = new RAMDirectory();
_pdfDir = new RAMDirectory();
_simpleDir = new RAMDirectory();
_conventionDir = new RAMDirectory();
//get all of the indexers and rebuild them all first
var indexers = new IIndexer[]
{
IndexInitializer.GetUmbracoIndexer(_cwsDir),
IndexInitializer.GetSimpleIndexer(_simpleDir),
IndexInitializer.GetUmbracoIndexer(_conventionDir)
};
foreach (var i in indexers)
{
try
{
i.RebuildIndex();
}
finally
{
var d = i as IDisposable;
if (d != null) d.Dispose();
}
}
//now get the multi index searcher for all indexes
_searcher = IndexInitializer.GetMultiSearcher(_pdfDir, _simpleDir, _conventionDir, _cwsDir);
}
public void Test(IIndexer indexer)
{
var releases = _feedFetcher.FetchRss(indexer);
if (releases.Any()) return;
try
{
var url = indexer.RecentFeed.First();
var xml = _httpProvider.DownloadString(url);
NewznabPreProcessor.Process(xml, url);
}
catch (ApiKeyException apiKeyException)
{
_logger.Warn("Indexer returned result for Newznab RSS URL, API Key appears to be invalid");
var apiKeyFailure = new ValidationFailure("ApiKey", "Invalid API Key");
throw new ValidationException(new List<ValidationFailure> { apiKeyFailure }.ToArray());
}
catch (Exception ex)
{
_logger.Warn("Indexer doesn't appear to be Newznab based");
var failure = new ValidationFailure("Url", "Invalid Newznab URL entered");
throw new ValidationException(new List<ValidationFailure> { failure }.ToArray());
}
}
public IndexOperations(IClient client, ILogger logger = null, IScroller scroller = null, IIndexer indexer = null, ISearcher searcher = null)
{
_logger = logger;
_searcher = searcher ?? new Searcher(client, logger);
_scroller = scroller ?? new Scroller(client, logger);
_indexer = indexer ?? new Indexer(client, logger);
}
public void CacheRssResults(IIndexer indexer, IEnumerable<ReleaseInfo> releases)
{
lock (cache)
{
var trackerCache = cache.Where(c => c.TrackerId == indexer.ID).FirstOrDefault();
if (trackerCache == null)
{
trackerCache = new TrackerCache();
trackerCache.TrackerId = indexer.ID;
trackerCache.TrackerName = indexer.DisplayName;
cache.Add(trackerCache);
}
foreach(var release in releases.OrderByDescending(i=>i.PublishDate))
{
var existingItem = trackerCache.Results.Where(i => i.Result.Guid == release.Guid).FirstOrDefault();
if (existingItem == null)
{
existingItem = new CachedResult();
existingItem.Created = DateTime.Now;
trackerCache.Results.Add(existingItem);
}
existingItem.Result = release;
}
// Prune cache
foreach(var tracker in cache)
{
tracker.Results = tracker.Results.OrderByDescending(i => i.Created).Take(MAX_RESULTS_PER_TRACKER).ToList();
}
}
}
EngineDisk()
{
this.engineConfiguration = EngineConfiguration.Instance;
this.logRep = FactoryRepositoryLog.GetRepositoryLog();
maxSentence = engineConfiguration.MaxSentence;
maxResultList = engineConfiguration.MaxResultList;
this.indexer = FactoryIndexer.GetIndexer();
}
public ReindexTextMigration(IRepository<int, ContentItem> repository, IIndexer indexer, LuceneAccesor accessor)
{
this.repository = repository;
this.indexer = indexer;
this.accessor = accessor;
Title = "Reindex all content using the lucene based search index";
Description = "Will re-index all items using lucene search database format.";
}
public IList<ReleaseInfo> Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria)
{
_logger.Debug("Searching for {0}", searchCriteria);
var result = Fetch(indexer, searchCriteria, 0).DistinctBy(c => c.DownloadUrl).ToList();
_logger.Info("Finished searching {0} for {1}. Found {2}", indexer, searchCriteria, result.Count);
return result;
}
public IList<ReleaseInfo> Fetch(IIndexer indexer, SingleEpisodeSearchCriteria searchCriteria)
{
_logger.Debug("Searching for {0}", searchCriteria);
var searchUrls = indexer.GetEpisodeSearchUrls(searchCriteria.QueryTitle, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber);
var result = Fetch(indexer, searchUrls);
_logger.Info("Finished searching {0} for {1}. Found {2}", indexer, searchCriteria, result.Count);
return result;
}
public virtual IList<ReleaseInfo> FetchRss(IIndexer indexer)
{
_logger.Debug("Fetching feeds from " + indexer);
var result = Fetch(indexer, indexer.RecentFeed);
_logger.Debug("Finished processing feeds from {0} found {1} releases", indexer, result.Count);
return result;
}
public IMapValue GetMapper(IIndexer item, ISyncIn attribute)
{
Func<IIndexer, ISyncIn, IMapValue> mapper;
if (_mappers.TryGetValue(attribute.GetType(), out mapper))
{
return mapper(item, attribute);
}
return null;
}
public AsyncIndexer(IIndexer indexer, IPersister persister, IWorker worker, IErrorNotifier errors, DatabaseSection config)
{
RetryInterval = TimeSpan.FromMinutes(2);
this.async = config.Search.AsyncIndexing;
this.handleErrors = config.Search.HandleErrors;
this.indexer = indexer;
this.persister = persister;
this.worker = worker;
this.errors = errors;
}
protected void Page_Load(object sender, EventArgs e)
{
index = FactoryIndexer.GetIndexer();
Process currentProc = Process.GetCurrentProcess();
long memoryUsed = currentProc.PrivateMemorySize64;
this.lblMemory.Text = "Memory: " + Useful.GetFormatedSizeString(memoryUsed);
this.lblFiles.Text = "Indexed Files: " + index.TotalDocumentQuantity;
this.lblWords.Text = "Total Word Quantity: " + index.TotalWordQuantity;
}
public static PersistentIndexer Resolve(IIndexer source, ITypeResolver typeResolver)
{
PersistentIndexer ind = new PersistentIndexer();
ind.FullyQualifiedName = source.FullyQualifiedName;
ind.Documentation = source.Documentation;
ind.modifiers = source.Modifiers;
ind.returnType = PersistentReturnType.Resolve (source.ReturnType, typeResolver);
foreach (IParameter p in source.Parameters)
ind.parameters.Add (PersistentParameter.Resolve (p, typeResolver));
ind.region = source.Region;
return ind;
}
public static void WriteTo(IIndexer ind, BinaryWriter writer, INameEncoder nameTable)
{
PersistentHelper.WriteString (ind.FullyQualifiedName, writer, nameTable);
PersistentHelper.WriteString (ind.Documentation, writer, nameTable);
writer.Write((uint)ind.Modifiers);
PersistentReturnType.WriteTo (ind.ReturnType, writer, nameTable);
writer.Write ((uint)ind.Parameters.Count);
foreach (IParameter p in ind.Parameters) {
PersistentParameter.WriteTo (p, writer, nameTable);
}
PersistentRegion.WriteTo (ind.Region, writer, nameTable);
}
protected void LoadIndexerConfig(IIndexer idx)
{
var configFilePath = GetIndexerConfigFilePath(idx);
if (File.Exists(configFilePath))
{
try
{
var fileStr = File.ReadAllText(configFilePath);
var jsonString = JToken.Parse(fileStr);
idx.LoadFromSavedConfiguration(jsonString);
}
catch (Exception ex)
{
logger.Error(ex, "Failed loading configuration for {0}, you must reconfigure this indexer", idx.DisplayName);
}
}
}
private IList<ReleaseInfo> Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria, int offset)
{
_logger.Debug("Searching for {0} offset: {1}", searchCriteria, offset);
var searchUrls = indexer.GetSeasonSearchUrls(searchCriteria.QueryTitle, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, offset);
var result = Fetch(indexer, searchUrls);
_logger.Info("{0} offset {1}. Found {2}", indexer, searchCriteria, result.Count);
if (result.Count > 90)
{
result.AddRange(Fetch(indexer, searchCriteria, offset + 90));
}
return result;
}
public virtual int CompareTo(IIndexer value)
{
int cmp;
cmp = base.CompareTo((IDecoration)value);
if (cmp != 0) {
return cmp;
}
if (FullyQualifiedName != null) {
cmp = FullyQualifiedName.CompareTo(value.FullyQualifiedName);
if (cmp != 0) {
return cmp;
}
}
if (ReturnType != null) {
cmp = ReturnType.CompareTo(value.ReturnType);
if (cmp != 0) {
return cmp;
}
}
if (Region != null) {
cmp = Region.CompareTo(value.Region);
if (cmp != 0) {
return cmp;
}
}
if (GetterRegion != null) {
cmp = GetterRegion.CompareTo(value.GetterRegion);
if (cmp != 0) {
return cmp;
}
}
if (SetterRegion != null) {
cmp = SetterRegion.CompareTo(value.SetterRegion);
if (cmp != 0) {
return cmp;
}
}
return DiffUtility.Compare(Parameters, value.Parameters);
}
protected void Application_Start()
{
var kernel = NinjectWebCommon.Kernel;
DashboardAccountContext context = kernel.Get<DashboardAccountContext>();
AreaRegistration.RegisterAllAreas();
GlobalConfiguration.Configure(WebApiConfig.Register);
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
if (!context.HasSetupError)
{
RouteConfig.RegisterRoutes(RouteTable.Routes);
}
else
{
RouteConfig.RegisterNoAccountRoutes(RouteTable.Routes);
}
BundleConfig.RegisterBundles(BundleTable.Bundles);
if (!context.HasSetupError)
{
ModelBinderConfig.Register();
HostVersionConfig.RegisterWarnings(kernel.Get<IHostVersionReader>());
}
_indexer = kernel.TryGet<IIndexer>();
if (_indexer != null)
{
// Using private threads for now. If indexing switches to async storage calls we need to
// either use the CLR threadpool or figure out how to schedule the async callbacks on the
// private threads.
for (int i = 0; i < Environment.ProcessorCount; i++)
{
new Thread(IndexerWorkerLoop).Start();
}
}
}
public int GetNewItemCount(IIndexer indexer, IEnumerable<ReleaseInfo> releases)
{
lock (cache)
{
int newItemCount = 0;
var trackerCache = cache.Where(c => c.TrackerId == indexer.ID).FirstOrDefault();
if (trackerCache != null)
{
foreach (var release in releases)
{
if (trackerCache.Results.Where(i => i.Result.Guid == release.Guid).Count() == 0)
{
newItemCount++;
}
}
}
else {
newItemCount++;
}
return newItemCount;
}
}
public void SaveConfig(IIndexer indexer, JToken obj)
{
var uID = Guid.NewGuid().ToString("N");
var configFilePath = GetIndexerConfigFilePath(indexer);
var configFilePathBak = configFilePath + ".bak";
var configFilePathTmp = configFilePath + "." + uID + ".tmp";
var content = obj.ToString();
logger.Debug(string.Format("Saving new config file: {0}", configFilePathTmp));
if (string.IsNullOrWhiteSpace(content))
{
throw new Exception(string.Format("New config content for {0} is empty, please report this bug.", indexer.ID));
}
if (content.Contains("\x00"))
{
throw new Exception(string.Format("New config content for {0} contains 0x00, please report this bug. Content: {1}", indexer.ID, content));
}
// make sure the config directory exists
if (!Directory.Exists(configService.GetIndexerConfigDir()))
Directory.CreateDirectory(configService.GetIndexerConfigDir());
// create new temporary config file
File.WriteAllText(configFilePathTmp, content);
var fileInfo = new FileInfo(configFilePathTmp);
if (fileInfo.Length == 0)
{
throw new Exception(string.Format("New config file {0} is empty, please report this bug.", configFilePathTmp));
}
// create backup file
File.Delete(configFilePathBak);
if (File.Exists(configFilePath))
{
try
{
File.Move(configFilePath, configFilePathBak);
}
catch (IOException ex)
{
logger.Error(string.Format("Error while moving {0} to {1}: {2}", configFilePath, configFilePathBak, ex.ToString()));
}
}
// replace the actual config file
File.Delete(configFilePath);
try
{
File.Move(configFilePathTmp, configFilePath);
}
catch (IOException ex)
{
logger.Error(string.Format("Error while moving {0} to {1}: {2}", configFilePathTmp, configFilePath, ex.ToString()));
}
}
public void OnLuceneDriverChanged (IIndexer source,
ICollection list_of_added_uris,
ICollection list_of_removed_uris,
ICollection list_of_renamed_uris)
{
// Since we are proxying events from the LuceneDriver, there
// will never been any renamed uris. Thus it is safe to
// substitute our own internal list of renamed uris.
if (ChangedEvent != null) {
ChangedEvent (this,
list_of_added_uris,
list_of_removed_uris,
renamed_uris);
}
renamed_uris.Clear ();
}
public ProvisionService(ILuceneService luceneService, IIndexer indexer)
{
this.luceneService = luceneService;
this.indexer = indexer;
}
public FhirIndex(Definitions definitions, IIndexer indexer, ISearcher searcher)
{
this.definitions = definitions;
this.indexer = indexer;
this.searcher = searcher;
}
/// <summary>
/// Initializes a new instance of the <see cref="Index"/> class.
/// </summary>
/// <param name="pipeline">The pipeline.</param>
/// <param name="indexer">The indexer.</param>
public Index(Pipeline pipeline, IIndexer indexer)
{
Pipeline = pipeline;
Indexer = indexer;
}
public void InitializeTest( )
{
schema = SchemaTests.CreateSchema(ID);
indexer = GetIndexer(schema);
}
public static ProcessorHandler <FilterLog> CreateProcessorHandler <TEventDTO>(
this IIndexer <EventLog <TEventDTO> > indexer)
where TEventDTO : class, IEventDTO, new()
{
return(new EventLogSearchIndexProcessorHandler <TEventDTO>(indexer));
}
public SearchEngine(MusicDatabase database, ReceiveResponse callback)
{
_localIndexer = new LocalIndexer(database);
Callback = callback;
Database = database;
}
public override async Task Invoke(IStepExecutionContext context = null)
{
try
{
Logger.Information($@"Updating index changes of {IndexModel.Name}/{IndexModel.Id}...");
var sourceConnection = connectionRepository.GetById(IndexModel.SourceConnectionId.ToString());
//var destConnection = connectionRepository.GetById(IndexModel.DestinationConnectionId.ToString());
IPuller puller = null;
IIndexer indexer = null;
if (IndexModel.EntityType == EntityType.Entity)
{
puller = pullers.Where(p => typeof(IEntityPuller).IsAssignableFrom(p.GetType()))
.Select(p => (IEntityPuller)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel?.SourceProcessorId,
sourceConnection?.ProviderId));
indexer = indexers.Where(p => typeof(IEntityIndexer).IsAssignableFrom(p.GetType()))
.Select(p => (IEntityIndexer)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel?.SourceProcessorId,
sourceConnection?.ProviderId));
}
else
{
var entity = entityRepository.GetById((IndexModel as AttributeModel).EntityId.ToString());
puller = pullers.Where(p => typeof(IAttributePuller).IsAssignableFrom(p.GetType()))
.Select(p => (IAttributePuller)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel.SourceProcessorId,
entity?.SourceProcessorId,
sourceConnection?.ProviderId));
indexer = indexers.Where(p => typeof(IAttributeIndexer).IsAssignableFrom(p.GetType()))
.Select(p => (IAttributeIndexer)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel.SourceProcessorId,
entity?.SourceProcessorId,
sourceConnection?.ProviderId));
}
var options = entityRepository.LoadOptions(IndexModel.Id.ToString(), IndexModel.EntityType)
.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
puller.SetIndex(IndexModel);
puller.SetOptions(options);
indexer.SetIndex(IndexModel);
indexer.SetOptions(options);
indexerManager.SetIndex(IndexModel);
indexerManager.SetPuller(puller);
indexerManager.SetIndexer(indexer);
indexerManager.OnReport(s =>
{
Logger.Information(s);
});
await indexerManager.PullNext();
Logger.Information($@"Updated index changes of {IndexModel.Name}/{IndexModel.Id}");
}
catch (Exception ex)
{
ErrorLogger.Error(ex, ex.Message);
throw;
}
}
public IndexerException(IIndexer Indexer, string message, Exception innerException)
: base(message, innerException)
{
this.Indexer = Indexer;
}
#pragma warning restore SA1401 // Fields should be private
protected Database(ITypeStore typeInfo, IIndexer indexer)
{
_typeInfo = typeInfo;
_indexer = indexer;
}
public ContentIndexService(IContentLoader contentLoader, IIndexer indexer)
{
_contentLoader = contentLoader;
_indexer = indexer;
}
public ElasticIndexerController(IContentLoader contentLoader, IIndexer indexer, IElasticSearchSettings settings)
{
_contentLoader = contentLoader;
_indexer = indexer;
_settings = settings;
}
public void SaveConfig(IIndexer indexer, JToken obj)
{
LastSavedConfig = obj;
}
public IndexerWrapper(IIndexer <TAttributeGroup, TDeclaringType, TTypeReference, TParameter, TAccessor> indexer)
{
WrappedObject = indexer;
}
public static ProcessorHandler <FilterLog> CreateProcessorHandler(
this IIndexer <FilterLog> indexer, Func <FilterLog, Task <bool> > asyncCriteria)
{
return(new FilterLogSearchIndexProcessorHandler(indexer, asyncCriteria));
}
public static ProcessorHandler <TSource> CreateProcessorHandler <TSource>(
this IIndexer <TSource> indexer)
where TSource : class, new()
{
return(new ProcessorHandler <TSource>((x) => indexer.IndexAsync(x)));
}
public TeamController(ITeamRepository teamRepository, IIndexer indexer)
{
_teamRepository = teamRepository;
_indexer = indexer;
}
public HomeController(ILogger <HomeController> logger, IIndexer indexer, ISimilarityDetectionAlgorithm similarityDetectionAlgorithm)
{
this.logger = logger;
this.indexer = indexer;
this.similarityDetectionAlgorithm = similarityDetectionAlgorithm;
}
private List<ReleaseInfo> Fetch(IIndexer indexer, IEnumerable<string> urls)
{
var result = new List<ReleaseInfo>();
foreach (var url in urls)
{
try
{
_logger.Debug("Downloading Feed " + url);
var xml = _httpProvider.DownloadString(url);
if (!string.IsNullOrWhiteSpace(xml))
{
result.AddRange(indexer.Parser.Process(xml, url));
}
else
{
_logger.Warn("{0} returned empty response.", url);
}
}
catch (WebException webException)
{
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
_logger.Warn("{0} server is currently unavailable. {1} {2}", indexer, url, webException.Message);
}
else
{
_logger.Warn("{0} {1} {2}", indexer, url, webException.Message);
}
}
catch (ApiKeyException)
{
_logger.Warn("Invalid API Key for {0} {1}", indexer, url);
}
catch (Exception feedEx)
{
feedEx.Data.Add("FeedUrl", url);
_logger.ErrorException("An error occurred while processing feed. " + url, feedEx);
}
}
result.ForEach(c => c.Indexer = indexer.Definition.Name);
return result;
}
public async Task <IActionResult> Results([FromQuery] ApiSearch requestt)
{
//TODO: Better way to parse querystring
ApiSearch request = new ApiSearch();
foreach (var t in Request.Query)
{
if (t.Key == "Tracker[]")
{
request.Tracker = t.Value.ToString().Split(',');
}
if (t.Key == "Category[]")
{
request.Category = t.Value.ToString().Split(',').Select(Int32.Parse).ToArray();
CurrentQuery.Categories = request.Category;
}
if (t.Key == "query")
{
request.Query = t.Value.ToString();
}
}
var manualResult = new ManualSearchResult();
var trackers = IndexerService.GetAllIndexers().ToList().Where(t => t.IsConfigured);
if (request.Tracker != null)
{
trackers = trackers.Where(t => request.Tracker.Contains(t.ID));
}
trackers = trackers.Where(t => t.CanHandleQuery(CurrentQuery));
var tasks = trackers.ToList().Select(t => t.ResultsForQuery(CurrentQuery)).ToList();
try
{
var aggregateTask = Task.WhenAll(tasks);
await aggregateTask;
}
catch (AggregateException aex)
{
foreach (var ex in aex.InnerExceptions)
{
logger.Error(ex);
}
}
catch (Exception ex)
{
logger.Error(ex);
}
manualResult.Indexers = tasks.Select(t =>
{
var resultIndexer = new ManualSearchResultIndexer();
IIndexer indexer = null;
if (t.Status == TaskStatus.RanToCompletion)
{
resultIndexer.Status = ManualSearchResultIndexerStatus.OK;
resultIndexer.Results = t.Result.Releases.Count();
resultIndexer.Error = null;
indexer = t.Result.Indexer;
}
else if (t.Exception.InnerException is IndexerException)
{
resultIndexer.Status = ManualSearchResultIndexerStatus.Error;
resultIndexer.Results = 0;
resultIndexer.Error = ((IndexerException)t.Exception.InnerException).ToString();
indexer = ((IndexerException)t.Exception.InnerException).Indexer;
}
else
{
resultIndexer.Status = ManualSearchResultIndexerStatus.Unknown;
resultIndexer.Results = 0;
resultIndexer.Error = null;
}
if (indexer != null)
{
resultIndexer.ID = indexer.ID;
resultIndexer.Name = indexer.DisplayName;
}
return(resultIndexer);
}).ToList();
manualResult.Results = tasks.Where(t => t.Status == TaskStatus.RanToCompletion).Where(t => t.Result.Releases.Any()).SelectMany(t =>
{
var searchResults = t.Result.Releases;
var indexer = t.Result.Indexer;
cacheService.CacheRssResults(indexer, searchResults);
return(searchResults.Select(result =>
{
var item = AutoMapper.Mapper.Map <TrackerCacheResult>(result);
item.Tracker = indexer.DisplayName;
item.TrackerId = indexer.ID;
item.Peers = item.Peers - item.Seeders; // Use peers as leechers
return item;
}));
}).OrderByDescending(d => d.PublishDate).ToList();
ConfigureCacheResults(manualResult.Results);
logger.Info(string.Format("Manual search for \"{0}\" on {1} with {2} results.", CurrentQuery.SanitizedSearchTerm, string.Join(", ", manualResult.Indexers.Select(i => i.ID)), manualResult.Results.Count()));
return(Json(manualResult));
}
public static ProcessorHandler <FilterLog> CreateProcessorHandler(
this IIndexer <FilterLog> indexer)
{
return(new FilterLogSearchIndexProcessorHandler(indexer));
}
public IndexerResult(IIndexer indexer, IEnumerable <ReleaseInfo> releases, bool isFromCache)
{
Indexer = indexer;
Releases = releases;
IsFromCache = isFromCache;
}
public IList<ReleaseInfo> Fetch(IIndexer indexer, SpecialEpisodeSearchCriteria searchCriteria)
{
var queryUrls = new List<String>();
foreach (var episodeQueryTitle in searchCriteria.EpisodeQueryTitles)
{
_logger.Debug("Performing query of {0} for {1}", indexer, episodeQueryTitle);
queryUrls.AddRange(indexer.GetSearchUrls(episodeQueryTitle));
}
var result = Fetch(indexer, queryUrls);
_logger.Info("Finished searching {0} for {1}. Found {2}", indexer, searchCriteria, result.Count);
return result;
}
private string GetIndexerConfigFilePath(IIndexer indexer)
{
return(Path.Combine(configService.GetIndexerConfigDir(), indexer.ID + ".json"));
}
public void SaveConfig(IIndexer indexer, JToken obj)
{
UpdateAggregateIndexer();
lock (configWriteLock)
{
var uID = Guid.NewGuid().ToString("N");
var configFilePath = GetIndexerConfigFilePath(indexer);
var configFilePathBak = configFilePath + ".bak";
var configFilePathTmp = configFilePath + "." + uID + ".tmp";
var content = obj.ToString();
logger.Debug(string.Format("Saving new config file: {0}", configFilePathTmp));
if (string.IsNullOrWhiteSpace(content))
{
throw new Exception(string.Format("New config content for {0} is empty, please report this bug.", indexer.ID));
}
if (content.Contains("\x00"))
{
throw new Exception(string.Format("New config content for {0} contains 0x00, please report this bug. Content: {1}", indexer.ID, content));
}
// make sure the config directory exists
if (!Directory.Exists(configService.GetIndexerConfigDir()))
{
Directory.CreateDirectory(configService.GetIndexerConfigDir());
}
// create new temporary config file
File.WriteAllText(configFilePathTmp, content);
var fileInfo = new FileInfo(configFilePathTmp);
if (fileInfo.Length == 0)
{
throw new Exception(string.Format("New config file {0} is empty, please report this bug.", configFilePathTmp));
}
// create backup file
File.Delete(configFilePathBak);
if (File.Exists(configFilePath))
{
try
{
File.Move(configFilePath, configFilePathBak);
}
catch (IOException ex)
{
logger.Error(string.Format("Error while moving {0} to {1}: {2}", configFilePath, configFilePathBak, ex.ToString()));
}
}
// replace the actual config file
File.Delete(configFilePath);
try
{
File.Move(configFilePathTmp, configFilePath);
}
catch (IOException ex)
{
logger.Error(string.Format("Error while moving {0} to {1}: {2}", configFilePathTmp, configFilePath, ex.ToString()));
}
}
}
public MemUtils(IIndexer memory)
{
Memory = memory;
}
// This is mostly a copy of LuceneQueryable.Flush + FSQ.PostAddHooks/PostRemoveHook
static bool FlushIndexer (IIndexer indexer)
{
IndexerRequest flushed_request;
if (pending_request.IsEmpty)
return false;
flushed_request = pending_request;
pending_request = new IndexerRequest ();
IndexerReceipt [] receipts;
receipts = indexer.Flush (flushed_request);
// Flush will return null if it encounters a shutdown during flushing
if (receipts == null)
return false;
fa_store.BeginTransaction ();
bool indexer_indexable_receipt = false;
foreach (IndexerReceipt raw_r in receipts) {
if (raw_r is IndexerAddedReceipt) {
// Update the file attributes
IndexerAddedReceipt r = (IndexerAddedReceipt) raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) {
Logger.Log.Debug ("Should not happen! Previously requested indexable with id #{0} has eloped!", r.Id);
continue;
}
// We don't need to write out any file attributes for
// children.
if (indexable.ParentUri != null)
continue;
string path = indexable.Uri.LocalPath;
FileAttributes attr;
attr = fa_store.ReadOrCreate (path);
attr.LastWriteTime = indexable.Timestamp;
attr.FilterName = r.FilterName;
attr.FilterVersion = r.FilterVersion;
fa_store.Write (attr);
} else if (raw_r is IndexerRemovedReceipt) {
// Update the file attributes
IndexerRemovedReceipt r = (IndexerRemovedReceipt) raw_r;
Indexable indexable = flushed_request.RetrieveRequestIndexable (r);
if (indexable == null) { // Should never happen
Log.Warn ("Unable to match indexable-remove #{0} to any request!", r.Id);
continue;
}
string path = indexable.Uri.LocalPath;
Logger.Log.Debug ("Removing: '{0}'", path);
fa_store.Drop (path);
} else if (raw_r is IndexerIndexablesReceipt) {
indexer_indexable_receipt = true;
}
}
pending_request.DeferredIndexables = flushed_request.DeferredIndexables;
// Reschedule if some indexable generated more indexables
if (indexer_indexable_receipt) {
pending_request.ContinueIndexing = true;
return true;
}
fa_store.CommitTransaction ();
return false;
}
public static ProcessorHandler <TransactionReceiptVO> CreateProcessorHandler <TFunctionMessage>(
this IIndexer <TransactionForFunctionVO <TFunctionMessage> > indexer)
where TFunctionMessage : FunctionMessage, new()
{
return(new FunctionMessageSearchIndexProcessorHandler <TFunctionMessage>(indexer));
}
public void Delete(IIndexer indexer)
{
var configFilePath = GetIndexerConfigFilePath(indexer);
File.Delete(configFilePath);
}
public static ProcessorHandler <TransactionReceiptVO> CreateProcessorHandler(
this IIndexer <TransactionReceiptVO> indexer,
Func <TransactionReceiptVO, Task <bool> > asyncCriteria)
{
return(new TransactionReceiptSearchIndexProcessorHandler(indexer, asyncCriteria));
}
private string GetIndexerConfigFilePath(IIndexer indexer)
{
return Path.Combine(configService.GetIndexerConfigDir(), indexer.ID + ".json");
}
public static ProcessorHandler <TransactionReceiptVO> CreateProcessorHandler(
this IIndexer <TransactionReceiptVO> indexer)
{
return(new TransactionReceiptSearchIndexProcessorHandler(indexer));
}
public ContentIndexer(IIndexer indexer, TextExtractor extractor)
{
this.extractor = extractor;
this.indexer = indexer;
}
public EventLogSearchIndexProcessorHandler(
IIndexer <EventLog <TEventDTO> > eventIndexer) :
base((eventLog) => eventIndexer.IndexAsync(eventLog))
{
EventIndexer = eventIndexer;
}
