/// <summary>
/// Shows how to perform accent insensitive indexing
/// Feature is supported in version 17.8.0 of the API
/// </summary>
public static void AccentInsensitiveIndexing()
{
//ExStart:AccentInsensitiveIndexing
// Enabling replacements during indexing
var settings = new IndexingSettings();
settings.UseCharacterReplacements = true;
// Creating index
Index index = new Index(Utilities.indexPath, settings);
// Clearing dictionary of replacements
index.Dictionaries.CharacterReplacements.Clear();
// Adding replacements
KeyValuePair <char, char>[] replacements = new KeyValuePair <char, char>[]
{
new KeyValuePair <char, char>('Ṝ', 'R'),
new KeyValuePair <char, char>('ṝ', 'r'),
};
index.Dictionaries.CharacterReplacements.AddRange(replacements);
// Import replacements from file. Existing replacements are staying.
index.Dictionaries.CharacterReplacements.Import(Utilities.replacementsFileName);
// Export replacements to file
index.Dictionaries.CharacterReplacements.Export(Utilities.exportedReplacementsFileName);
// Indexing
index.AddToIndex(Utilities.documentsPath);
//ExEnd:AccentInsensitiveIndexing
}
/// <summary>
/// Shows how to filter files during indexing
/// Feature is supported in version 17.9.0 or greater of the API
/// </summary>
public static void FilterFilesDuringIndexing()
{
//ExStart:FilterFilesDuringIndexing
// Creating indexing settings object
IndexingSettings settings = new IndexingSettings();
// Creating filter that only passes files from 600 KB to 1 MB in length
DocumentFilter byLength = DocumentFilter.CreateFileLengthRange(614400, 1048576);
// Creating filter that only passes text files
DocumentFilter byExtension = DocumentFilter.CreateFileExtension(".txt");
// Creating composite filter that only passes text files from 600 KB to 1 MB in length
DocumentFilter compositeFilter = DocumentFilter.CreateConjunction(byLength, byExtension);
// Setting filter
settings.DocumentFilter = compositeFilter;
// Creating index
Index index = new Index(Utilities.indexPath, settings);
// Indexing
index.AddToIndex(Utilities.documentsPath);
//ExEnd:FilterFilesDuringIndexing
}
public ElasticClientService(
ILogger <IElasticClientService> logger,
IElasticClientProvider elasticClientProvider,
IndexingSettings indexingSettings)
{
this.logger = logger;
this.maxDocumentsPerBatch = indexingSettings.MaxDocumentsPerBatch;
this.elasticClient = elasticClientProvider.Create(true);
}
public void Start(ulong currentBlockCount)
{
var indexerInstanceSettings = _container.Resolve <IIndexerInstanceSettings>();
var settings = new List <IIndexingSettings>();
// Blocks indexers
if (indexerInstanceSettings.IndexBlocks)
{
var lastRpcBlock = currentBlockCount;
var from = indexerInstanceSettings.StartBlock;
var to = indexerInstanceSettings.StopBlock ?? lastRpcBlock;
var partSize = (to - from) / (ulong)indexerInstanceSettings.ThreadAmount;
ulong?toBlock = from;
for (var i = 0; i < indexerInstanceSettings.ThreadAmount; i++)
{
var fromBlock = (ulong)toBlock + 1;
toBlock = fromBlock + partSize;
toBlock = toBlock < to ? toBlock : indexerInstanceSettings.StopBlock;
var indexerId = $"{indexerInstanceSettings.IndexerId}_thread_{i}";
var setting = new IndexingSettings
{
IndexerId = indexerId,
From = fromBlock,
To = toBlock
};
settings.Add(setting);
}
var blockIndexingActorDispatcherProps = Props.Create(() => new BlockIndexingActorDispatcher(settings,
_container.Resolve <ILog>(), _container.Resolve <IBlockIndexingActorFactory>()));
_blockIndexingActorDispatcher = _actorSystem.ActorOf(blockIndexingActorDispatcherProps, "block-indexing-actor-dispatcher");
}
// Balances indexer
if (indexerInstanceSettings.IndexBalances)
{
//_indexerInstanceSettings.BalancesStartBlock;
var erc20BalanceIndexingActorDispatcherProps = _actorSystem.DI().Props <Erc20BalanceIndexingActorDispatcher>();
_erc20BalanceIndexingActorDispatcher = _actorSystem.ActorOf(erc20BalanceIndexingActorDispatcherProps, "erc20-balance-indexing-actor-dispatcher");
}
// Contracts indexer
if (indexerInstanceSettings.IndexContracts)
{
//indexerInstanceSettings.ContractsIndexerThreadAmount
var erc20ContractIndexingActorDispatcherProps = _actorSystem.DI().Props <Erc20ContractIndexingActorDispatcher>();
_erc20ContractIndexingActorDispatcherProps = _actorSystem.ActorOf(erc20ContractIndexingActorDispatcherProps, "erc20-contract-indexing-actor-dispatcher");
}
}
public IndexingService(
ILogger <IndexingService> logger,
IIndexService indexService,
ILockWork lockWork,
IndexingSettings indexingSettings)
{
this.logger = logger;
this.indexService = indexService;
this.lockWork = lockWork;
this.IndexingFrequencySeconds = indexingSettings.IndexingFrequencySeconds;
}
/// <summary>
/// Create index in memory with index settings
/// </summary>
public static void CreateIndexInMemoryWithIndexSettings()
{
//ExStart:CreateIndexInMemoryWithIndexSettings
bool quickIndexing = true;
IndexingSettings settings = new IndexingSettings(quickIndexing);
// Create index on disk
Index index1 = new Index(settings);
// Create index on disk using Index Repository
IndexRepository repository = new IndexRepository();
Index index2 = repository.Create(settings);
//ExEnd:CreateIndexInMemoryWithIndexSettings
}
internal static void IndexMetaData()
{
//ExStart: IndexMetaData
// Creating indexing settings object
IndexingSettings settings = new IndexingSettings();
settings.IndexType = IndexType.MetadataIndex;
// Creating index
Index index = new Index(Utilities.indexPath, settings);
// Indexing
index.AddToIndex(Utilities.documentsPath);
//ExEnd: IndexMetaData
}
/// <summary>
/// Shows how to sutomatically detect encoding in text files
/// Feature is supported in version 17.9.0 or greater
/// </summary>
public static void AutomaticDetectEncoding()
{
//ExStart:AutomaticDetectEncoding
// Creating indexing settings object
IndexingSettings settings = new IndexingSettings();
// Enabling automatic encoding detection
settings.AutoDetectEncoding = true;
// Creating index
Index index = new Index(Utilities.indexPath, settings);
// Indexing
index.AddToIndex(Utilities.documentsPath);
//ExEnd:AutomaticDetectEncoding
}
public void InitDependencies()
{
TestConfig.ReconfigureServices();
var rpcReader = TestConfig.ServiceProvider.GetService <IRpcBlockReader>();
_indexingService = TestConfig.ServiceProvider.GetService <IIndexingService>();
_blockService = TestConfig.ServiceProvider.GetService <IBlockService>();
var indexingSettings = new IndexingSettings()
{
IndexerId = "TestIndexer",
From = 1,
To = 1
};
var logger = new Mock <ILog>();
_blockIndexingJob = new BlockIndexingJob(_blockService, _indexingService, indexingSettings, logger.Object, rpcReader);
}
/// <summary>
/// Shows how to cache text of Indexed Documents in index
/// Feature is supported in version 17.9.0 or greater of the API
/// </summary>
public static void CacheTextOfIndexedDocsInIndex()
{
//ExStart:CacheTextOfIndexedDocsInIndex
// Creating indexing settings object
IndexingSettings settings = new IndexingSettings();
// Enabling source document text caching with normal compression level
// From version 17.10 onwards, value "High" has been added to GroupDocs.Search.Compression enumeration.
// So in order to cache document's text with high compression level use "Compression.High"
settings.TextStorageSettings = new TextStorageSettings(Compression.Normal);
// Creating index
Index index = new Index(Utilities.indexPath, settings);
// Indexing
index.AddToIndex(Utilities.documentsPath);
//ExEnd:CacheTextOfIndexedDocsInIndex
}
/// <summary>
/// Creates index in memory with index settings
/// </summary>
public static void CreateIndexInMemoryWithIndexSettings()
{
//ExStart:CreateIndexInMemoryWithIndexSettings
//From version 17.9.0 quickIndexing has been marked obsolter and removed from the API
//These 2 lines of code are no more valid from version 17.9.0 onward
//bool quickIndexing = true;
//IndexingSettings settings = new IndexingSettings(quickIndexing);
IndexingSettings settings = new IndexingSettings();
// Create index on disk
Index index1 = new Index(settings);
// Create index on disk using Index Repository
IndexRepository repository = new IndexRepository();
Index index2 = repository.Create(settings);
//ExEnd:CreateIndexInMemoryWithIndexSettings
}
public static void CompactIndexing()
{
try
{
// Creating indexing settings object
IndexingSettings indexingSettings = new IndexingSettings();
// Setting compact index type
indexingSettings.IndexType = IndexType.CompactIndex;
// Creating index
Index index = new Index(Utilities.indexPath, indexingSettings);
// Indexing
index.AddToIndex(Utilities.documentsPath);
// Searching
SearchResults result = index.Search("Einstein");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
public IEnumerable <IIndexingSettings> GetBlockIndexingSettings()
{
var settings = new List <IIndexingSettings>();
// Blocks indexers
if (_indexerInstanceSettings.IndexBlocks)
{
var lastRpcBlock = (ulong)_rpcBlockReader.GetBlockCount().Result;
var from = _indexerInstanceSettings.StartBlock;
var to = _indexerInstanceSettings.StopBlock ?? lastRpcBlock;
var partSize = (to - from) / (ulong)_indexerInstanceSettings.ThreadAmount;
ulong?toBlock = from;
for (var i = 0; i < _indexerInstanceSettings.ThreadAmount; i++)
{
var fromBlock = (ulong)toBlock + 1;
toBlock = fromBlock + partSize;
toBlock = toBlock < to ? toBlock : _indexerInstanceSettings.StopBlock;
var indexerId = $"{_indexerInstanceSettings.IndexerId}_thread_{i}";
var setting = new IndexingSettings
{
IndexerId = indexerId,
From = fromBlock,
To = toBlock
};
settings.Add(setting);
}
}
return(settings);
}
