private async Task <CloudTable> CreateTableAync(string tableName, IndexingMode indexingMode, int?throughput, int?defaultTimeToLive, CancellationToken cancellationToken)
{
CloudTable cloudTable = _cloudTableClient.GetTableReference(tableName);
await cloudTable.CreateIfNotExistsAsync(indexingMode, throughput, defaultTimeToLive, cancellationToken);
return(cloudTable);
}
/// <summary>
/// Deserialize the object
/// </summary>
public virtual void DeserializeJson(JToken inputObject)
{
if (inputObject == null || inputObject.Type == JTokenType.Null)
{
return;
}
var automaticValue = inputObject["automatic"];
if (automaticValue != null && automaticValue.Type != JTokenType.Null)
{
Automatic = (bool)automaticValue;
}
var excludePathsValue = inputObject["ExcludePaths"];
if (excludePathsValue != null && excludePathsValue.Type != JTokenType.Null)
{
var excludePaths = new ExcludePaths();
excludePaths.DeserializeJson(excludePathsValue);
ExcludePaths = excludePaths;
}
var includePathsValue = inputObject["IncludePaths"];
if (includePathsValue != null && includePathsValue.Type != JTokenType.Null)
{
IncludePaths = (string)includePathsValue;
}
var indexingModeSequence = inputObject["indexingMode"];
if (indexingModeSequence != null && indexingModeSequence.Type != JTokenType.Null)
{
foreach (var indexingModeValue in (JArray)indexingModeSequence)
{
IndexingMode.Add((string)indexingModeValue);
}
}
var indexTypeValue = inputObject["indexType"];
if (indexTypeValue != null && indexTypeValue.Type != JTokenType.Null)
{
IndexType = (string)indexTypeValue;
}
var numericPrecisionValue = inputObject["NumericPrecision"];
if (numericPrecisionValue != null && numericPrecisionValue.Type != JTokenType.Null)
{
NumericPrecision = (int)numericPrecisionValue;
}
var pathValue = inputObject["Path"];
if (pathValue != null && pathValue.Type != JTokenType.Null)
{
Path = (string)pathValue;
}
var stringPrecisionValue = inputObject["StringPrecision"];
if (stringPrecisionValue != null && stringPrecisionValue.Type != JTokenType.Null)
{
StringPrecision = (int)stringPrecisionValue;
}
}
public virtual string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
{
// Does not seem to reach here
List<string> fixedFolderList = new List<string>();
List<string> dneFolderList = new List<string>();
foreach (string dirPath in mSettings.GetConfig().FolderList)
{
if (Directory.Exists(dirPath))
{
fixedFolderList.Add(dirPath);
}
else
{
dneFolderList.Add(dirPath);
}
}
if (dneFolderList.Count > 0)
{
StringBuilder sb = new System.Text.StringBuilder();
foreach (string dp in dneFolderList)
{
sb.AppendLine(dp);
}
MessageBox.Show("Following Index folders do not exist:" + Environment.NewLine + Environment.NewLine + sb.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
// Overrides by Sub Classes
mSettings.GetConfig().FolderList = fixedFolderList;
return new StringBuilder().ToString();
}
private static DocumentCollection GetDocumentCollection(
string collectionId,
string partitionKey = null,
IndexingMode indexingMode = IndexingMode.Consistent)
{
var documentCollection = new DocumentCollection();
documentCollection.Id = collectionId;
var rangeIndex = new RangeIndex(DataType.String);
rangeIndex.Precision = -1;
var indexingPolicy = new IndexingPolicy(rangeIndex);
indexingPolicy.IndexingMode = indexingMode;
documentCollection.IndexingPolicy = indexingPolicy;
if (partitionKey != null)
{
var partitionKeyDefinition = new PartitionKeyDefinition();
partitionKeyDefinition.Paths = new Collection <string> {
partitionKey
};
documentCollection.PartitionKey = partitionKeyDefinition;
}
return(documentCollection);
}
/// <summary>
/// The create document collection if not exists asynchronous.
/// </summary>
/// <param name="partitionId">
/// The partition id.
/// </param>
/// <param name="indexingMode">
/// The indexing mode.
/// </param>
/// <param name="defaultTtl">The default Time to live.</param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
public async Task CreateDocumentCollectionIfNotExistsAsync(
string partitionId,
IndexingMode indexingMode,
int?defaultTtl)
{
await this.CreateDocumentPartitionedCollectionIfNotExistsAsync(partitionId, indexingMode, defaultTtl).ConfigureAwait(false);
}
public void Add(List <I> intervals, IndexingMode mode, uint collectionID, int threads)
{
int start = 0, stop = 0, count = 0, range = (int)Math.Ceiling(intervals.Count / (double)threads);
var addedBookmarks = new ConcurrentDictionary <int, int>();
using (WorkQueue work = new WorkQueue(threads))
{
for (int i = 0; i < threads; i++)
{
start = i * range;
stop = (i + 1) * range;
if (stop > intervals.Count)
{
stop = intervals.Count;
}
work.Enqueue(new BatchIndex <C, I, M>(_di4_incIdx, collectionID, intervals, start, stop, mode, addedBookmarks).Run);
}
work.Complete(true, -1);
}
count = 0;
foreach (var item in addedBookmarks)
{
count += item.Value;
}
_indexesCardinality.AddOrUpdate(_keyCardinalityIncIndx, count);
}
public string fGetIndexFilePath(int folderID, IndexingMode mode)
{
//Dim strDirPath As String = GetConfig().FolderList.Item(folderID)
string lPath = string.Empty;
// strDirPath + "\" + GetConfig().GetIndexFileName
switch (mode)
{
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
string strDirPath = GetConfig().FolderList[folderID];
string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
string sDirName = Path.GetFileName(strDirPath);
string sep = GetOptions().IndividualIndexFileWordSeperator;
if (Directory.Exists(GetConfig().OutputDir))
{
lPath = GetConfig().OutputDir + Path.DirectorySeparatorChar + sDirName + sep + sDrive + sep + GetConfig().GetIndexFileName();
}
break;
case IndexingMode.IN_EACH_DIRECTORY:
// this is also what it is first initialized to
strDirPath = GetConfig().FolderList[folderID];
lPath = strDirPath + "\\" + GetConfig().GetIndexFileName();
break;
default:
lPath = GetConfig().GetIndexFilePath();
break;
}
return(lPath);
}
public virtual string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
{
// Does not seem to reach here
List <string> fixedFolderList = new List <string>();
List <string> dneFolderList = new List <string>();
foreach (string dirPath in mSettings.GetConfig().FolderList)
{
if (Directory.Exists(dirPath))
{
fixedFolderList.Add(dirPath);
}
else
{
dneFolderList.Add(dirPath);
}
}
if (dneFolderList.Count > 0)
{
StringBuilder sb = new System.Text.StringBuilder();
foreach (string dp in dneFolderList)
{
sb.AppendLine(dp);
}
MessageBox.Show("Following Index folders do not exist:" + Environment.NewLine + Environment.NewLine + sb.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
// Overrides by Sub Classes
mSettings.GetConfig().FolderList = fixedFolderList;
return(new StringBuilder().ToString());
}
public MainViewModel(ConfigValues configValues, WorkspaceDirectoryModel workspaceDirectoryModel,
TreeViewModel treeViewModel, FilesPatternProvider filesPatternProvider, CommandExecutor commandExecutor, PowershellTreeRestructurer powershellTreeRestructurer)
{
this.configValues = configValues;
this.WorkspaceDirectoryModel = workspaceDirectoryModel;
this.TreeViewModel = treeViewModel;
this.filesPatternProvider = filesPatternProvider;
this.commandExecutor = commandExecutor;
this.powershellTreeRestructurer = powershellTreeRestructurer;
this.TreeViewModel.PropertyChanged += (s, e) => { if (e.PropertyName == "NumberOfFiles")
{
this.OnPropertyChanged("TreeItemsResultString");
}
};
this.searchRegex = configValues.SearchRegex;
this.indexFilesMode = configValues.IndexFilesMode;
this.showAllFiles = configValues.ShowAllFiles;
this.filesPatternProvider.IncludeAllFiles = this.showAllFiles;
this.filesPatternProvider.ExcludePaths = configValues.ExcludePaths;
this.filesPatternProvider.IndexFilesMode = configValues.IndexFilesMode;
this.powershellTreeRestructurer.ShowRegions = !configValues.SortFunctions;
NodeComparerProvider.SortFunctions = configValues.SortFunctions;
this.syncWithActiveDocument = configValues.SyncWithActiveDocument;
var searchField = (indexFilesMode != IndexingMode.NO_FILES ? FullTextFieldType.CatchAll : FullTextFieldType.Name);
this.SearchOptions = new SearchOptions(searchField, string.Empty, this.searchRegex);
}
public async Task SetIndexMode(IndexingMode mode)
{
var collection = new DocumentCollection {
Id = _configurationService.DocumentDb.Collection
};
collection.IndexingPolicy.IndexingMode = mode;
await DocumentDbClient.ReplaceDocumentCollectionAsync(collection);
}
public IndexingPolicyBuilder()
{
this.includedPaths = new HashSet <IncludedPath>(IndexingPolicy.IncludedPathEqualityComparer.Singleton);
this.excludedPaths = new HashSet <ExcludedPath>(IndexingPolicy.ExcludedPathEqualityComparer.Singleton);
this.compositeIndexes = new HashSet <HashSet <CompositePath> >();
this.spatialIndexes = new HashSet <SpatialIndex>(SpatialIndex.SpatialIndexEqualityComparer.Singleton);
this.Automatic = true;
this.IndexingMode = IndexingMode.Consistent;
}
/// <summary>
/// Creates a new Simple Lucene Indexer
/// </summary>
/// <param name="indexDir">Directory</param>
/// <param name="analyzer">Analyzer</param>
/// <param name="schema">Index Schema</param>
/// <param name="mode">Indexing Mode</param>
public BaseSimpleLuceneIndexer(Directory indexDir, Analyzer analyzer, IFullTextIndexSchema schema, IndexingMode mode)
{
if (this._mode == IndexingMode.Custom) throw new ArgumentException("Cannot use IndexingMode.Custom with the BaseSimpleLuceneIndexer");
this._mode = mode;
this._indexDir = indexDir;
this._analyzer = analyzer;
this._schema = schema;
this._writer = new IndexWriter(indexDir, analyzer);
this._searcher = new LucSearch.IndexSearcher(this._indexDir, true);
}
public CorpusIndexManager(IndexingMode indexingMode, string rootDirectory, string indexDirectory, string tbxIateLocation = null, string logLocation = null, int?parallelIndexingLimit = null, bool readOnly = true)
{
m_readOnly = readOnly;
m_indexingMode = indexingMode;
if (!Directory.Exists(rootDirectory))
{
throw new ArgumentException("The source directory must exist!");
}
/*if (Directory.Exists(indexDirectory) && !Directory.EnumerateFileSystemEntries(indexDirectory).GetEnumerator().MoveNext())
* {
* throw new ArgumentException("The indexing directory must be empty!");
* }*/
m_rootSourceDirectory = rootDirectory;
if (logLocation != null)
{
m_logLocation = logLocation;
if (!m_logLocation.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
m_logLocation += Path.DirectorySeparatorChar;
}
if (!Directory.Exists(m_logLocation))
{
Directory.CreateDirectory(m_logLocation);
}
}
m_rootIndexDirectory = indexDirectory;
if (!m_rootIndexDirectory.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
m_rootIndexDirectory += Path.DirectorySeparatorChar;
}
if (!Directory.Exists(m_rootIndexDirectory))
{
Directory.CreateDirectory(m_rootIndexDirectory);
}
if (tbxIateLocation != null)
{
Tbx sourceTbx = null;
var serializerTbx = new System.Xml.Serialization.XmlSerializer(typeof(Tbx));
using (var xmlReader = XmlReader.Create(tbxIateLocation))
{
sourceTbx = (Tbx)serializerTbx.Deserialize(xmlReader);
}
m_parserFactory = new LegalTextParserFactory(sourceTbx);
}
SetupWriters();
}
/// <summary>
/// Creates a new Simple Lucene Indexer.
/// </summary>
/// <param name="indexDir">Directory.</param>
/// <param name="analyzer">Analyzer.</param>
/// <param name="schema">Index Schema.</param>
/// <param name="mode">Indexing Mode.</param>
public BaseSimpleLuceneIndexer(Directory indexDir, Analyzer analyzer, IFullTextIndexSchema schema, IndexingMode mode)
{
if (this._mode == IndexingMode.Custom)
{
throw new ArgumentException("Cannot use IndexingMode.Custom with the BaseSimpleLuceneIndexer");
}
this._mode = mode;
this._indexDir = indexDir;
this._analyzer = analyzer;
this._schema = schema;
this._writer = new IndexWriter(indexDir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
}
internal BatchIndex(BPlusTree <C, B> di4, uint collectionID, List <I> intervals, int start, int stop, IndexingMode mode, ConcurrentDictionary <int, int> addedBookmarks)
{
_di4 = di4;
_collectionID = collectionID;
_intervals = intervals;
_start = start;
_stop = stop;
_mode = mode;
_addedBookmarks = addedBookmarks;
_bCounter = new BookmarkCounter();
update.bookmarkCounter = _bCounter;
}
public Task <CloudTable> CreateIfNotExistsAsync(string tableName,
IndexingMode indexingMode,
CancellationToken cancellationToken,
int?throughput = null,
int?defaultTimeToLive = null)
{
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(nameof(tableName), "tableName can not be null or empty");
}
return(CreateTableAync(tableName, indexingMode, throughput, defaultTimeToLive, cancellationToken));
}
public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
{
IndexingMode indexingMode = (IndexingMode)value;
switch (indexingMode)
{
case IndexingMode.ALL_FILES: return(true);
case IndexingMode.NO_FILES: return(false);
default: return(null);
}
}
/// <summary>
/// Gets the hash code
/// </summary>
/// <returns>Hash code</returns>
public override int GetHashCode()
{
// credit: http://stackoverflow.com/a/263416/677735
unchecked // Overflow is fine, just wrap
{
var hash = 41;
// Suitable nullity checks etc, of course :)
if (Automatic != null)
{
hash = hash * 57 + Automatic.GetHashCode();
}
if (IndexingMode != null)
{
hash = hash * 57 + IndexingMode.GetHashCode();
}
if (IncludePaths != null)
{
hash = hash * 57 + IncludePaths.GetHashCode();
}
if (IndexType != null)
{
hash = hash * 57 + IndexType.GetHashCode();
}
if (NumericPrecision != null)
{
hash = hash * 57 + NumericPrecision.GetHashCode();
}
if (StringPrecision != null)
{
hash = hash * 57 + StringPrecision.GetHashCode();
}
if (Path != null)
{
hash = hash * 57 + Path.GetHashCode();
}
if (ExcludePaths != null)
{
hash = hash * 57 + ExcludePaths.GetHashCode();
}
return(hash);
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Create a new instance of the index manager. There should always be only one instance of the
/// index manager.
/// </summary>
///
/// <remarks> Semantika d.o.o.,. </remarks>
///
/// <param name="indexLocation"> The configuration of the index. </param>
////////////////////////////////////////////////////////////////////////////////////////////////////
public IndexManager(IndexingMode indexingMode, string indexLocation, bool isReadOnly = false)
{
//Go through all index types and create the management structure for individual indexes
var indexObjectTypes = Enum.GetValues(typeof(IndexObjectType)).Cast <IndexObjectType>().ToArray();
m_indexingMode = indexingMode;
if (indexingMode == IndexingMode.IndexPerLanguage)
{
m_indexTypes = new IndexType[indexObjectTypes.Length * m_supportedLanguages.Length];
for (int i = 0; i < m_supportedLanguages.Length; i++)
{
for (int j = 0; j < indexObjectTypes.Length; j++)
{
m_indexTypes[i * indexObjectTypes.Length + j] = new IndexType
{
Language = m_supportedLanguages[i],
ObjectType = indexObjectTypes[j]
};
}
}
}
else
{
m_indexTypes = new IndexType[indexObjectTypes.Length];
for (int j = 0; j < indexObjectTypes.Length; j++)
{
m_indexTypes[j] = new IndexType
{
Language = m_singleIndexLangConstant,
ObjectType = indexObjectTypes[j]
};
}
}
foreach (IndexType curType in m_indexTypes)
{
var directoryPath = indexLocation + "\\FTS_" + curType.ToString();
IndexManagementObjects indexManagementObjects = new IndexManagementObjects
{
CurrentReaderUsers = 0,
Directory = FSDirectory.Open(directoryPath),
OldReaders = new ConcurrentDictionary <IndexReader, int>()
};
m_managementObjects[curType] = indexManagementObjects;
}
IsReadOnly = isReadOnly;
m_taskScheduler = TaskScheduler.Default;
}
/// <summary>
/// Returns true if IndexingPolicy instances are equal
/// </summary>
/// <param name="other">Instance of IndexingPolicy to be compared</param>
/// <returns>Boolean</returns>
public bool Equals(IndexingPolicy other)
{
// credit: http://stackoverflow.com/a/10454552/677735
if (other == null)
{
return(false);
}
return
((
Automatic == other.Automatic ||
Automatic != null &&
Automatic.Equals(other.Automatic)
) &&
(
IndexingMode == other.IndexingMode ||
IndexingMode != null &&
IndexingMode.SequenceEqual(other.IndexingMode)
) &&
(
IncludePaths == other.IncludePaths ||
IncludePaths != null &&
IncludePaths.Equals(other.IncludePaths)
) &&
(
IndexType == other.IndexType ||
IndexType != null &&
IndexType.Equals(other.IndexType)
) &&
(
NumericPrecision == other.NumericPrecision ||
NumericPrecision != null &&
NumericPrecision.Equals(other.NumericPrecision)
) &&
(
StringPrecision == other.StringPrecision ||
StringPrecision != null &&
StringPrecision.Equals(other.StringPrecision)
) &&
(
Path == other.Path ||
Path != null &&
Path.Equals(other.Path)
) &&
(
ExcludePaths == other.ExcludePaths ||
ExcludePaths != null &&
ExcludePaths.Equals(other.ExcludePaths)
));
}
public MarcellCorpus(IndexingMode indexingMode, string indexPath)
{
lock (indexInitLock)
{
if (indexManager == null)
{
//Configure the index manager
indexManager = new IndexManager(indexingMode, indexPath, true);
//Prepare the default implementation of
m_searcherCache[defaultSearchParameters] = GetSearcher(defaultSearchParameters);
WarmUp();
}
}
}
public CloudTable CreateIfNotExists(string tableName,
IndexingMode indexingMode,
int?throughput = null,
int?defaultTimeToLive = null)
{
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(nameof(tableName), "tableName can not be null or empty");
}
CloudTable cloudTable = _cloudTableClient.GetTableReference(tableName);
cloudTable.CreateIfNotExists(indexingMode, throughput, defaultTimeToLive);
return(cloudTable);
}
private string getOutputSwitch(string folderPath, IndexingMode mode)
{
switch (mode)
{
case IndexingMode.IN_EACH_DIRECTORY:
this.setCurrentIndexFilePath(folderPath + Path.DirectorySeparatorChar + mSettings.GetConfig().GetIndexFileName());
return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
string strOutputFileName = null;
string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
if (Path.GetFileName(folderPath).Length > 0)
{
strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + Path.GetFileName(folderPath) + sep + mSettings.GetConfig().GetIndexFileName();
}
else
{
strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + mSettings.GetConfig().GetIndexFileName();
}
this.setCurrentIndexFilePath(mSettings.GetConfig().OutputDir + Path.DirectorySeparatorChar + strOutputFileName);
return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
case IndexingMode.IN_ONE_FOLDER_MERGED:
if (mSettings.GetConfig().MergeFiles)
{
this.setCurrentIndexFilePath(this.mSettings.GetConfig().GetIndexFilePath());
if (isFirstEntryToSingleFile == true)
{
isFirstEntryToSingleFile = false;
return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
}
else
{
return(">>" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
}
}
break;
}
return(null);
}
private string getOutputSwitch(string folderPath, IndexingMode mode)
{
switch (mode)
{
case IndexingMode.IN_EACH_DIRECTORY:
this.setCurrentIndexFilePath(folderPath + Path.DirectorySeparatorChar + mSettings.GetConfig().GetIndexFileName());
return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
string strOutputFileName = null;
string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
if (Path.GetFileName(folderPath).Length > 0)
{
strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + Path.GetFileName(folderPath) + sep + mSettings.GetConfig().GetIndexFileName();
}
else
{
strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + mSettings.GetConfig().GetIndexFileName();
}
this.setCurrentIndexFilePath(mSettings.GetConfig().OutputDir + Path.DirectorySeparatorChar + strOutputFileName);
return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
case IndexingMode.IN_ONE_FOLDER_MERGED:
if (mSettings.GetConfig().MergeFiles)
{
this.setCurrentIndexFilePath(this.mSettings.GetConfig().GetIndexFilePath());
if (isFirstEntryToSingleFile == true)
{
isFirstEntryToSingleFile = false;
return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
}
else
{
return ">>" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
}
}
break;
}
return null;
}
/// <summary>
/// The create document partitioned collection if not exists async.
/// </summary>
/// <param name="partitionId">
/// The partition id.
/// </param>
/// <param name="indexingMode">
/// The indexing mode.
/// </param>
/// <param name="defaultTtl">The default TTL</param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
/// <exception cref="ArgumentNullException">
/// null value in parameters
/// </exception>
private async Task CreateDocumentPartitionedCollectionIfNotExistsAsync(
string partitionId,
IndexingMode indexingMode,
int?defaultTtl)
{
if (string.IsNullOrWhiteSpace(partitionId))
{
throw new ArgumentNullException(nameof(partitionId));
}
try
{
await
this.documentDbClient.ReadDocumentCollectionAsync(
UriFactory.CreateDocumentCollectionUri(this.databaseName, this.collectionName))
.ConfigureAwait(false);
}
catch (DocumentClientException de)
{
// If the document collection does not exist, create a new partitioned collection
if (de.StatusCode == HttpStatusCode.NotFound)
{
var partitionKey = new PartitionKeyDefinition();
partitionKey.Paths.Add(partitionId);
var collectionInfo = new DocumentCollection
{
Id = this.collectionName,
PartitionKey = partitionKey,
IndexingPolicy = new IndexingPolicy {
Automatic = true, IndexingMode = indexingMode
},
DefaultTimeToLive = defaultTtl
};
await this.documentDbClient.CreateDocumentCollectionAsync(UriFactory.CreateDatabaseUri(this.databaseName), collectionInfo).ConfigureAwait(false);
}
else
{
throw;
}
}
}
public static LucField.Index ToLuceneIndexingMode(IndexingMode mode)
{
if (mode == IndexingMode.Default)
{
mode = IndexingInfo.DefaultIndexingMode;
}
switch (mode)
{
case IndexingMode.Analyzed: return(LucField.Index.ANALYZED);
case IndexingMode.AnalyzedNoNorms: return(LucField.Index.ANALYZED_NO_NORMS);
case IndexingMode.No: return(LucField.Index.NO);
case IndexingMode.NotAnalyzed: return(LucField.Index.NOT_ANALYZED);
case IndexingMode.NotAnalyzedNoNorms: return(LucField.Index.NOT_ANALYZED_NO_NORMS);
default: throw new ArgumentOutOfRangeException("Not supported IndexingMode: " + mode);
}
}
public void Add(List <I> intervals, IndexingMode mode, uint collectionID)
{
Add(intervals, mode, collectionID, Environment.ProcessorCount);
}
public override string IndexNow(IndexingMode IndexMode, bool bWriteToFile = true)
{
string fp = null;
StringBuilder sb = new StringBuilder();
List<string> folderList = new List<string>();
folderList = mSettings.GetConfig().FolderList;
TreeNetIndexer treeNetLib = new TreeNetIndexer(mSettings);
string ext = mSettings.GetConfig().IndexFileExt;
switch (IndexMode)
{
case IndexingMode.IN_EACH_DIRECTORY:
IndexInEachDir(mSettings);
break;
case IndexingMode.IN_ONE_FOLDER_MERGED:
if (mSettings.GetConfig().MergeFiles)
{
fp = mSettings.fGetIndexFilePath(-1, IndexingMode.IN_ONE_FOLDER_MERGED);
if (mSettings.GetConfig().FolderList.Count > 1)
{
for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 2; i++)
{
string strDirPath = mSettings.GetConfig().FolderList[i];
TreeDir dir = new TreeDir(strDirPath);
this.CurrentDirMessage = "Indexing " + strDirPath;
if (ext.Contains(".html"))
{
treeNetLib.mBooMoreFilesToCome = true;
treeNetLib.IndexRootFolderToHtml(strDirPath, sb, false);
}
else
{
treeNetLib.IndexFolderToTxt(strDirPath, sb, false);
}
this.Progress += 1;
}
}
TreeDir lastDir = new TreeDir(mSettings.GetConfig().FolderList[mSettings.GetConfig().FolderList.Count - 1]);
this.CurrentDirMessage = "Indexing " + lastDir.DirectoryPath();
if (ext.Contains(".html"))
{
treeNetLib.mBooFirstIndexFile = false || mSettings.GetConfig().FolderList.Count == 1;
treeNetLib.IndexRootFolderToHtml(lastDir.DirectoryPath(), sb, true);
}
else
{
treeNetLib.IndexFolderToTxt(lastDir.DirectoryPath(), sb, mSettings.GetConfig().AddFooter);
}
if (mSettings.GetConfig().ZipMergedFile)
{
mSettings.ZipAdminFile(fp, null);
}
this.Progress += 1;
}
break;
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
// DO NOT MERGE INDEX FILES
if (!Directory.Exists(mSettings.GetConfig().OutputDir))
{
MessageBox.Show(string.Format("{0} does not exist." + Environment.NewLine + Environment.NewLine + "Please change the Output folder in Configuration." + Environment.NewLine + "The index file will be created in the same folder you chose to index.", mSettings.GetConfig().OutputDir), Application.ProductName, MessageBoxButtons.OK);
}
for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
{
string strDirPath = mSettings.GetConfig().FolderList[i];
string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
string sDirName = Path.GetFileName(strDirPath);
string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
//where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName
// New Behavior for getting where location
fp = mSettings.fGetIndexFilePath(i, IndexingMode.IN_ONE_FOLDER_SEPERATE);
mSettings.GetConfig().SetIndexPath(fp);
//MsgBox(where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName)
this.CurrentDirMessage = "Indexing " + strDirPath;
if (ext.Contains(".html"))
{
treeNetLib.mBooFirstIndexFile = true;
treeNetLib.IndexRootFolderToHtml(strDirPath, sb, mSettings.GetConfig().AddFooter);
}
else
{
treeNetLib.IndexFolderToTxt(strDirPath, sb, mSettings.GetConfig().AddFooter);
}
if (mSettings.GetConfig().ZipFilesInOutputDir)
{
mSettings.ZipAdminFile(fp, null);
}
this.Progress += 1;
}
break;
}
if (bWriteToFile)
{
using (StreamWriter sw = new StreamWriter(fp, false))
{
sw.Write(sb.ToString());
sw.Close();
}
}
return sb.ToString();
}
/// <summary>
/// Defines the <see cref="Container"/>'s <see cref="Cosmos.IndexingMode"/>.
/// </summary>
/// <param name="indexingMode">An <see cref="Cosmos.IndexingMode"/></param>
/// <returns>An instance of <see cref="IndexingPolicyDefinition{T}"/>.</returns>
/// <remarks>
/// If multiple calls are made to this method within the same <see cref="IndexingPolicyDefinition{T}"/>, the last one will apply.
/// </remarks>
public IndexingPolicyDefinition <T> WithIndexingMode(IndexingMode indexingMode)
{
this.indexingPolicy.IndexingMode = indexingMode;
return(this);
}
internal ExecutionReport Add(
uint collectionID,
Dictionary <string, Dictionary <char, List <I> > > intervals,
char strand,
IndexingMode indexinMode,
MaxDegreeOfParallelism maxDegreeOfParallelism)
{
int totalIntervals = 0;
switch (_memory)
{
case Memory.HDD:
if (_chrSection == null)
{
_chrSection = new ChrSection();
}
ConfigurationManager.RefreshSection(_sectionTitle);
switch (_hddPerformance)
{
// TODO:
// This case is not complete, because other operations
// are not supporting this method.
case HDDPerformance.LeastMemory:
_stpWtch.Restart();
foreach (var chr in intervals)
{
foreach (var strandEntry in chr.Value)
{
using (var di4 = new Di4 <C, I, M>(GetDi4Options(GetDi4File(chr.Key, strand)))) // this might be wrong
{
di4.Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
totalIntervals += strandEntry.Value.Count;
}
}
}
_stpWtch.Stop();
break;
case HDDPerformance.Fastest:
/// Initialize by a sequential foreach loop.
foreach (var chr in intervals)
{
foreach (var strandEntry in chr.Value)
{
if (!chrs.ContainsKey(chr.Key))
{
chrs.Add(chr.Key, new Dictionary <char, Di4 <C, I, M> >());
}
if (!chrs[chr.Key].ContainsKey(strand))
{
chrs[chr.Key].Add(strand, new Di4 <C, I, M>(GetDi4Options(GetDi4File(chr.Key, strand))));
}
}
}
_stpWtch.Restart();
/// Populate inside a parallel foreach loop.
Parallel.ForEach(intervals,
new ParallelOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism.chrDegree
},
chr =>
{
foreach (var strandEntry in chr.Value)
{
chrs[chr.Key][strand].Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
//chrs[chr.Key][strand].Commit();
totalIntervals += strandEntry.Value.Count;
}
});
_stpWtch.Stop();
break;
}
if (_config.Sections[_sectionTitle] == null)
{
_config.Sections.Add(_sectionTitle, _chrSection);
}
_config.Save(ConfigurationSaveMode.Modified);
ConfigurationManager.RefreshSection(_sectionTitle);
break;
case Memory.RAM:
_stpWtch.Restart();
foreach (var chr in intervals)
{
foreach (var strandEntry in chr.Value)
{
if (!chrs.ContainsKey(chr.Key))
{
chrs.Add(chr.Key, new Dictionary <char, Di4 <C, I, M> >());
}
if (!chrs[chr.Key].ContainsKey(strand))
{
chrs[chr.Key].Add(strand, new Di4 <C, I, M>(GetDi4Options()));
}
chrs[chr.Key][strand].Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
totalIntervals += strandEntry.Value.Count;
}
}
_stpWtch.Stop();
break;
}
return(new ExecutionReport(totalIntervals, _stpWtch.Elapsed));
}
public ExecutionReport Add(uint collectionID, Dictionary <string, Dictionary <char, List <I> > > peaks, IndexingMode indexingMode, MaxDegreeOfParallelism maxDegreeOfParallelism)
{
return(genome.Add(collectionID, peaks, '*', indexingMode, maxDegreeOfParallelism));
}
public override string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
{
TreeWalkIndexer tree = new TreeWalkIndexer(mSettings);
bool isMergeFile = mSettings.GetConfig().MergeFiles;
bool isRemoveBranches = mSettings.GetConfig().RemoveTreeBranches;
for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
{
string batFilePath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\temp" + i.ToString() + ".bat";
string CURRENT_DIR = mSettings.GetConfig().FolderList[i];
string TREE_COMMAND = "%windir%\\system32\\tree.com " + tree.getSourceSwitch(CURRENT_DIR) + tree.getAsciiSwitch() + tree.getAddFilesSwitch() + tree.getOutputSwitch(CURRENT_DIR, mIndexMode);
using (StreamWriter sw = new StreamWriter(batFilePath))
{
sw.WriteLine(TREE_COMMAND);
//1.5.3.4 Didn't tag index files created in the same folder witout appending
if (mSettings.GetConfig().AddFooter)
{
if (mIndexMode == IndexingMode.IN_EACH_DIRECTORY | i == mSettings.GetConfig().FolderList.Count - 1 | (isMergeFile == false & mIndexMode == IndexingMode.IN_ONE_FOLDER_MERGED))
{
sw.WriteLine(mSettings.getBlankLine(tree.getCurrentIndexFilePath()));
sw.WriteLine(mSettings.GetFooterText(tree.getCurrentIndexFilePath(), IndexingEngine.TreeLib, false));
}
}
sw.WriteLine("DEL " + (char)34 + batFilePath + (char)34);
}
Process proc = new Process();
proc = mSettings.StartHiddenProcess(batFilePath, true);
if (isRemoveBranches)
{
tree.removeTreeBranches(tree.getCurrentIndexFilePath());
}
if (mIndexMode == IndexingMode.IN_EACH_DIRECTORY | i == mSettings.GetConfig().FolderList.Count - 1 | (isMergeFile == false & mIndexMode == IndexingMode.IN_ONE_FOLDER_MERGED))
{
if (mSettings.GetConfig().ZipFilesInEachDir)
{
mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
}
if (mSettings.GetConfig().ZipMergedFile)
{
mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
}
}
if (mIndexMode == IndexingMode.IN_ONE_FOLDER_SEPERATE)
{
if (mSettings.GetConfig().ZipFilesInOutputDir)
{
//MsgBox(tree.getCurrentIndexFilePath())
mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
}
}
if (proc.HasExited)
{
this.Progress += 1;
this.CurrentDirMessage = "Indexed " + mSettings.GetConfig().FolderList[i];
}
}
return File.ReadAllText(tree.getCurrentIndexFilePath());
}
public IndexingPolicyImpl <ParentImplT, IParentT, DefinitionParentT, UpdateParentT> WithIndexingMode(IndexingMode indexingMode)
{
Inner.IndexingMode = indexingMode;
return(this);
}
/// <summary>
/// Initializes an instance of the IndexField with a named System.DateTime value and indexing metadata.
/// </summary>
/// <param name="name">The name of the term.</param>
/// <param name="value">System.DateTime value</param>
/// <param name="mode">Indexing mode.</param>
/// <param name="store">Index storing mode.</param>
/// <param name="termVector">Term vector handling.</param>
public IndexField(string name, DateTime value, IndexingMode mode, IndexStoringMode store, IndexTermVector termVector) : base(name, value)
{
Mode = mode; Store = store; TermVector = termVector;
}
public override string IndexNow(IndexingMode IndexMode, bool bWriteToFile = true)
{
string fp = null;
StringBuilder sb = new StringBuilder();
List <string> folderList = new List <string>();
folderList = mSettings.GetConfig().FolderList;
TreeNetIndexer treeNetLib = new TreeNetIndexer(mSettings);
string ext = mSettings.GetConfig().IndexFileExt;
switch (IndexMode)
{
case IndexingMode.IN_EACH_DIRECTORY:
IndexInEachDir(mSettings);
break;
case IndexingMode.IN_ONE_FOLDER_MERGED:
if (mSettings.GetConfig().MergeFiles)
{
fp = mSettings.fGetIndexFilePath(-1, IndexingMode.IN_ONE_FOLDER_MERGED);
if (mSettings.GetConfig().FolderList.Count > 1)
{
for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 2; i++)
{
string strDirPath = mSettings.GetConfig().FolderList[i];
TreeDir dir = new TreeDir(strDirPath);
this.CurrentDirMessage = "Indexing " + strDirPath;
if (ext.Contains(".html"))
{
treeNetLib.mBooMoreFilesToCome = true;
treeNetLib.IndexRootFolderToHtml(strDirPath, sb, false);
}
else
{
treeNetLib.IndexFolderToTxt(strDirPath, sb, false);
}
this.Progress += 1;
}
}
TreeDir lastDir = new TreeDir(mSettings.GetConfig().FolderList[mSettings.GetConfig().FolderList.Count - 1]);
this.CurrentDirMessage = "Indexing " + lastDir.DirectoryPath();
if (ext.Contains(".html"))
{
treeNetLib.mBooFirstIndexFile = false || mSettings.GetConfig().FolderList.Count == 1;
treeNetLib.IndexRootFolderToHtml(lastDir.DirectoryPath(), sb, true);
}
else
{
treeNetLib.IndexFolderToTxt(lastDir.DirectoryPath(), sb, true);
}
if (mSettings.GetConfig().ZipMergedFile)
{
mSettings.ZipAdminFile(fp, null);
}
this.Progress += 1;
}
break;
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
// DO NOT MERGE INDEX FILES
if (!Directory.Exists(mSettings.GetConfig().OutputDir))
{
MessageBox.Show(string.Format("{0} does not exist." + Environment.NewLine + Environment.NewLine + "Please change the Output folder in Configuration." + Environment.NewLine + "The index file will be created in the same folder you chose to index.", mSettings.GetConfig().OutputDir), Application.ProductName, MessageBoxButtons.OK);
}
for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
{
string strDirPath = mSettings.GetConfig().FolderList[i];
string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
string sDirName = Path.GetFileName(strDirPath);
string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
//where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName
// New Behavior for getting where location
fp = mSettings.fGetIndexFilePath(i, IndexingMode.IN_ONE_FOLDER_SEPERATE);
mSettings.GetConfig().SetIndexPath(fp);
//MsgBox(where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName)
this.CurrentDirMessage = "Indexing " + strDirPath;
if (ext.Contains(".html"))
{
treeNetLib.mBooFirstIndexFile = true;
treeNetLib.IndexRootFolderToHtml(strDirPath, sb, true);
}
else
{
treeNetLib.IndexFolderToTxt(strDirPath, sb, true);
}
if (mSettings.GetConfig().ZipFilesInOutputDir)
{
mSettings.ZipAdminFile(fp, null);
}
this.Progress += 1;
}
break;
}
if (bWriteToFile)
{
using (StreamWriter sw = new StreamWriter(fp, false))
{
sw.Write(sb.ToString());
sw.Close();
}
}
return(sb.ToString());
}
public string fGetIndexFilePath(int folderID, IndexingMode mode)
{
//Dim strDirPath As String = GetConfig().FolderList.Item(folderID)
string lPath = string.Empty;
// strDirPath + "\" + GetConfig().GetIndexFileName
switch (mode)
{
case IndexingMode.IN_ONE_FOLDER_SEPERATE:
string strDirPath = GetConfig().FolderList[folderID];
string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
string sDirName = Path.GetFileName(strDirPath);
string sep = GetOptions().IndividualIndexFileWordSeperator;
if (Directory.Exists(GetConfig().OutputDir))
{
lPath = GetConfig().OutputDir + Path.DirectorySeparatorChar + sDirName + sep + sDrive + sep + GetConfig().GetIndexFileName();
}
break;
case IndexingMode.IN_EACH_DIRECTORY:
// this is also what it is first initialized to
strDirPath = GetConfig().FolderList[folderID];
lPath = strDirPath + "\\" + GetConfig().GetIndexFileName();
break;
default:
lPath = GetConfig().GetIndexFilePath();
break;
}
return lPath;
}