private async Task <CloudTable> CreateTableAync(string tableName, IndexingMode indexingMode, int?throughput, int?defaultTimeToLive, CancellationToken cancellationToken)
        {
            CloudTable cloudTable = _cloudTableClient.GetTableReference(tableName);
            await cloudTable.CreateIfNotExistsAsync(indexingMode, throughput, defaultTimeToLive, cancellationToken);

            return(cloudTable);
        }
Пример #2
0
        /// <summary>
        ///     Deserialize the object
        /// </summary>
        public virtual void DeserializeJson(JToken inputObject)
        {
            if (inputObject == null || inputObject.Type == JTokenType.Null)
            {
                return;
            }
            var automaticValue = inputObject["automatic"];

            if (automaticValue != null && automaticValue.Type != JTokenType.Null)
            {
                Automatic = (bool)automaticValue;
            }
            var excludePathsValue = inputObject["ExcludePaths"];

            if (excludePathsValue != null && excludePathsValue.Type != JTokenType.Null)
            {
                var excludePaths = new ExcludePaths();
                excludePaths.DeserializeJson(excludePathsValue);
                ExcludePaths = excludePaths;
            }
            var includePathsValue = inputObject["IncludePaths"];

            if (includePathsValue != null && includePathsValue.Type != JTokenType.Null)
            {
                IncludePaths = (string)includePathsValue;
            }
            var indexingModeSequence = inputObject["indexingMode"];

            if (indexingModeSequence != null && indexingModeSequence.Type != JTokenType.Null)
            {
                foreach (var indexingModeValue in (JArray)indexingModeSequence)
                {
                    IndexingMode.Add((string)indexingModeValue);
                }
            }
            var indexTypeValue = inputObject["indexType"];

            if (indexTypeValue != null && indexTypeValue.Type != JTokenType.Null)
            {
                IndexType = (string)indexTypeValue;
            }
            var numericPrecisionValue = inputObject["NumericPrecision"];

            if (numericPrecisionValue != null && numericPrecisionValue.Type != JTokenType.Null)
            {
                NumericPrecision = (int)numericPrecisionValue;
            }
            var pathValue = inputObject["Path"];

            if (pathValue != null && pathValue.Type != JTokenType.Null)
            {
                Path = (string)pathValue;
            }
            var stringPrecisionValue = inputObject["StringPrecision"];

            if (stringPrecisionValue != null && stringPrecisionValue.Type != JTokenType.Null)
            {
                StringPrecision = (int)stringPrecisionValue;
            }
        }
Пример #3
0
        public virtual string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
        {
            // Does not seem to reach here

            List<string> fixedFolderList = new List<string>();
            List<string> dneFolderList = new List<string>();

            foreach (string dirPath in mSettings.GetConfig().FolderList)
            {
                if (Directory.Exists(dirPath))
                {
                    fixedFolderList.Add(dirPath);
                }
                else
                {
                    dneFolderList.Add(dirPath);
                }
            }

            if (dneFolderList.Count > 0)
            {
                StringBuilder sb = new System.Text.StringBuilder();
                foreach (string dp in dneFolderList)
                {
                    sb.AppendLine(dp);
                }
                MessageBox.Show("Following Index folders do not exist:" + Environment.NewLine + Environment.NewLine + sb.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
            }

            // Overrides by Sub Classes

            mSettings.GetConfig().FolderList = fixedFolderList;
            return new StringBuilder().ToString();
        }
        private static DocumentCollection GetDocumentCollection(
            string collectionId,
            string partitionKey       = null,
            IndexingMode indexingMode = IndexingMode.Consistent)
        {
            var documentCollection = new DocumentCollection();

            documentCollection.Id = collectionId;

            var rangeIndex = new RangeIndex(DataType.String);

            rangeIndex.Precision = -1;
            var indexingPolicy = new IndexingPolicy(rangeIndex);

            indexingPolicy.IndexingMode       = indexingMode;
            documentCollection.IndexingPolicy = indexingPolicy;

            if (partitionKey != null)
            {
                var partitionKeyDefinition = new PartitionKeyDefinition();
                partitionKeyDefinition.Paths = new Collection <string> {
                    partitionKey
                };
                documentCollection.PartitionKey = partitionKeyDefinition;
            }

            return(documentCollection);
        }
 /// <summary>
 /// The create document collection if not exists asynchronous.
 /// </summary>
 /// <param name="partitionId">
 /// The partition id.
 /// </param>
 /// <param name="indexingMode">
 /// The indexing mode.
 /// </param>
 /// <param name="defaultTtl">The default Time to live.</param>
 /// <returns>
 /// The <see cref="Task"/>.
 /// </returns>
 public async Task CreateDocumentCollectionIfNotExistsAsync(
     string partitionId,
     IndexingMode indexingMode,
     int?defaultTtl)
 {
     await this.CreateDocumentPartitionedCollectionIfNotExistsAsync(partitionId, indexingMode, defaultTtl).ConfigureAwait(false);
 }
Пример #6
0
Файл: Di4.cs Проект: olgatei/Di4
        public void Add(List <I> intervals, IndexingMode mode, uint collectionID, int threads)
        {
            int start = 0, stop = 0, count = 0, range = (int)Math.Ceiling(intervals.Count / (double)threads);
            var addedBookmarks = new ConcurrentDictionary <int, int>();

            using (WorkQueue work = new WorkQueue(threads))
            {
                for (int i = 0; i < threads; i++)
                {
                    start = i * range;
                    stop  = (i + 1) * range;
                    if (stop > intervals.Count)
                    {
                        stop = intervals.Count;
                    }
                    work.Enqueue(new BatchIndex <C, I, M>(_di4_incIdx, collectionID, intervals, start, stop, mode, addedBookmarks).Run);
                }

                work.Complete(true, -1);
            }

            count = 0;
            foreach (var item in addedBookmarks)
            {
                count += item.Value;
            }
            _indexesCardinality.AddOrUpdate(_keyCardinalityIncIndx, count);
        }
        public string fGetIndexFilePath(int folderID, IndexingMode mode)
        {
            //Dim strDirPath As String = GetConfig().FolderList.Item(folderID)
            string lPath = string.Empty;

            // strDirPath + "\" + GetConfig().GetIndexFileName

            switch (mode)
            {
            case IndexingMode.IN_ONE_FOLDER_SEPERATE:
                string strDirPath = GetConfig().FolderList[folderID];
                string sDrive     = Path.GetPathRoot(strDirPath).Substring(0, 1);
                string sDirName   = Path.GetFileName(strDirPath);
                string sep        = GetOptions().IndividualIndexFileWordSeperator;
                if (Directory.Exists(GetConfig().OutputDir))
                {
                    lPath = GetConfig().OutputDir + Path.DirectorySeparatorChar + sDirName + sep + sDrive + sep + GetConfig().GetIndexFileName();
                }
                break;

            case IndexingMode.IN_EACH_DIRECTORY:
                // this is also what it is first initialized to
                strDirPath = GetConfig().FolderList[folderID];
                lPath      = strDirPath + "\\" + GetConfig().GetIndexFileName();
                break;

            default:
                lPath = GetConfig().GetIndexFilePath();
                break;
            }

            return(lPath);
        }
Пример #8
0
        public virtual string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
        {
            // Does not seem to reach here

            List <string> fixedFolderList = new List <string>();
            List <string> dneFolderList   = new List <string>();

            foreach (string dirPath in mSettings.GetConfig().FolderList)
            {
                if (Directory.Exists(dirPath))
                {
                    fixedFolderList.Add(dirPath);
                }
                else
                {
                    dneFolderList.Add(dirPath);
                }
            }

            if (dneFolderList.Count > 0)
            {
                StringBuilder sb = new System.Text.StringBuilder();
                foreach (string dp in dneFolderList)
                {
                    sb.AppendLine(dp);
                }
                MessageBox.Show("Following Index folders do not exist:" + Environment.NewLine + Environment.NewLine + sb.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
            }

            // Overrides by Sub Classes

            mSettings.GetConfig().FolderList = fixedFolderList;
            return(new StringBuilder().ToString());
        }
Пример #9
0
        public MainViewModel(ConfigValues configValues, WorkspaceDirectoryModel workspaceDirectoryModel,
                             TreeViewModel treeViewModel, FilesPatternProvider filesPatternProvider, CommandExecutor commandExecutor, PowershellTreeRestructurer powershellTreeRestructurer)
        {
            this.configValues                   = configValues;
            this.WorkspaceDirectoryModel        = workspaceDirectoryModel;
            this.TreeViewModel                  = treeViewModel;
            this.filesPatternProvider           = filesPatternProvider;
            this.commandExecutor                = commandExecutor;
            this.powershellTreeRestructurer     = powershellTreeRestructurer;
            this.TreeViewModel.PropertyChanged += (s, e) => { if (e.PropertyName == "NumberOfFiles")
                                                              {
                                                                  this.OnPropertyChanged("TreeItemsResultString");
                                                              }
            };

            this.searchRegex    = configValues.SearchRegex;
            this.indexFilesMode = configValues.IndexFilesMode;
            this.showAllFiles   = configValues.ShowAllFiles;
            this.filesPatternProvider.IncludeAllFiles   = this.showAllFiles;
            this.filesPatternProvider.ExcludePaths      = configValues.ExcludePaths;
            this.filesPatternProvider.IndexFilesMode    = configValues.IndexFilesMode;
            this.powershellTreeRestructurer.ShowRegions = !configValues.SortFunctions;
            NodeComparerProvider.SortFunctions          = configValues.SortFunctions;

            this.syncWithActiveDocument = configValues.SyncWithActiveDocument;
            var searchField = (indexFilesMode != IndexingMode.NO_FILES ? FullTextFieldType.CatchAll : FullTextFieldType.Name);

            this.SearchOptions = new SearchOptions(searchField, string.Empty, this.searchRegex);
        }
Пример #10
0
        public async Task SetIndexMode(IndexingMode mode)
        {
            var collection = new DocumentCollection {
                Id = _configurationService.DocumentDb.Collection
            };

            collection.IndexingPolicy.IndexingMode = mode;
            await DocumentDbClient.ReplaceDocumentCollectionAsync(collection);
        }
Пример #11
0
 public IndexingPolicyBuilder()
 {
     this.includedPaths    = new HashSet <IncludedPath>(IndexingPolicy.IncludedPathEqualityComparer.Singleton);
     this.excludedPaths    = new HashSet <ExcludedPath>(IndexingPolicy.ExcludedPathEqualityComparer.Singleton);
     this.compositeIndexes = new HashSet <HashSet <CompositePath> >();
     this.spatialIndexes   = new HashSet <SpatialIndex>(SpatialIndex.SpatialIndexEqualityComparer.Singleton);
     this.Automatic        = true;
     this.IndexingMode     = IndexingMode.Consistent;
 }
Пример #12
0
 /// <summary>
 /// Creates a new Simple Lucene Indexer
 /// </summary>
 /// <param name="indexDir">Directory</param>
 /// <param name="analyzer">Analyzer</param>
 /// <param name="schema">Index Schema</param>
 /// <param name="mode">Indexing Mode</param>
 public BaseSimpleLuceneIndexer(Directory indexDir, Analyzer analyzer, IFullTextIndexSchema schema, IndexingMode mode)
 {
     if (this._mode == IndexingMode.Custom) throw new ArgumentException("Cannot use IndexingMode.Custom with the BaseSimpleLuceneIndexer");
     this._mode = mode;
     this._indexDir = indexDir;
     this._analyzer = analyzer;
     this._schema = schema;
     this._writer = new IndexWriter(indexDir, analyzer);
     this._searcher = new LucSearch.IndexSearcher(this._indexDir, true);
 }
Пример #13
0
        public CorpusIndexManager(IndexingMode indexingMode, string rootDirectory, string indexDirectory, string tbxIateLocation = null, string logLocation = null, int?parallelIndexingLimit = null, bool readOnly = true)
        {
            m_readOnly     = readOnly;
            m_indexingMode = indexingMode;

            if (!Directory.Exists(rootDirectory))
            {
                throw new ArgumentException("The source directory must exist!");
            }

            /*if (Directory.Exists(indexDirectory) && !Directory.EnumerateFileSystemEntries(indexDirectory).GetEnumerator().MoveNext())
             * {
             *  throw new ArgumentException("The indexing directory must be empty!");
             * }*/

            m_rootSourceDirectory = rootDirectory;

            if (logLocation != null)
            {
                m_logLocation = logLocation;
                if (!m_logLocation.EndsWith(Path.DirectorySeparatorChar.ToString()))
                {
                    m_logLocation += Path.DirectorySeparatorChar;
                }
                if (!Directory.Exists(m_logLocation))
                {
                    Directory.CreateDirectory(m_logLocation);
                }
            }

            m_rootIndexDirectory = indexDirectory;
            if (!m_rootIndexDirectory.EndsWith(Path.DirectorySeparatorChar.ToString()))
            {
                m_rootIndexDirectory += Path.DirectorySeparatorChar;
            }
            if (!Directory.Exists(m_rootIndexDirectory))
            {
                Directory.CreateDirectory(m_rootIndexDirectory);
            }

            if (tbxIateLocation != null)
            {
                Tbx sourceTbx     = null;
                var serializerTbx = new System.Xml.Serialization.XmlSerializer(typeof(Tbx));
                using (var xmlReader = XmlReader.Create(tbxIateLocation))
                {
                    sourceTbx = (Tbx)serializerTbx.Deserialize(xmlReader);
                }

                m_parserFactory = new LegalTextParserFactory(sourceTbx);
            }

            SetupWriters();
        }
 /// <summary>
 /// Creates a new Simple Lucene Indexer.
 /// </summary>
 /// <param name="indexDir">Directory.</param>
 /// <param name="analyzer">Analyzer.</param>
 /// <param name="schema">Index Schema.</param>
 /// <param name="mode">Indexing Mode.</param>
 public BaseSimpleLuceneIndexer(Directory indexDir, Analyzer analyzer, IFullTextIndexSchema schema, IndexingMode mode)
 {
     if (this._mode == IndexingMode.Custom)
     {
         throw new ArgumentException("Cannot use IndexingMode.Custom with the BaseSimpleLuceneIndexer");
     }
     this._mode     = mode;
     this._indexDir = indexDir;
     this._analyzer = analyzer;
     this._schema   = schema;
     this._writer   = new IndexWriter(indexDir, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
 }
Пример #15
0
 internal BatchIndex(BPlusTree <C, B> di4, uint collectionID, List <I> intervals, int start, int stop, IndexingMode mode, ConcurrentDictionary <int, int> addedBookmarks)
 {
     _di4                   = di4;
     _collectionID          = collectionID;
     _intervals             = intervals;
     _start                 = start;
     _stop                  = stop;
     _mode                  = mode;
     _addedBookmarks        = addedBookmarks;
     _bCounter              = new BookmarkCounter();
     update.bookmarkCounter = _bCounter;
 }
        public Task <CloudTable> CreateIfNotExistsAsync(string tableName,
                                                        IndexingMode indexingMode,
                                                        CancellationToken cancellationToken,
                                                        int?throughput        = null,
                                                        int?defaultTimeToLive = null)
        {
            if (string.IsNullOrEmpty(tableName))
            {
                throw new ArgumentNullException(nameof(tableName), "tableName can not be null  or empty");
            }

            return(CreateTableAync(tableName, indexingMode, throughput, defaultTimeToLive, cancellationToken));
        }
        public object Convert(object value, Type targetType, object parameter, CultureInfo culture)
        {
            IndexingMode indexingMode = (IndexingMode)value;

            switch (indexingMode)
            {
            case IndexingMode.ALL_FILES: return(true);

            case IndexingMode.NO_FILES: return(false);

            default: return(null);
            }
        }
Пример #18
0
        /// <summary>
        ///     Gets the hash code
        /// </summary>
        /// <returns>Hash code</returns>
        public override int GetHashCode()
        {
            // credit: http://stackoverflow.com/a/263416/677735
            unchecked // Overflow is fine, just wrap
            {
                var hash = 41;
                // Suitable nullity checks etc, of course :)

                if (Automatic != null)
                {
                    hash = hash * 57 + Automatic.GetHashCode();
                }

                if (IndexingMode != null)
                {
                    hash = hash * 57 + IndexingMode.GetHashCode();
                }

                if (IncludePaths != null)
                {
                    hash = hash * 57 + IncludePaths.GetHashCode();
                }

                if (IndexType != null)
                {
                    hash = hash * 57 + IndexType.GetHashCode();
                }

                if (NumericPrecision != null)
                {
                    hash = hash * 57 + NumericPrecision.GetHashCode();
                }

                if (StringPrecision != null)
                {
                    hash = hash * 57 + StringPrecision.GetHashCode();
                }

                if (Path != null)
                {
                    hash = hash * 57 + Path.GetHashCode();
                }

                if (ExcludePaths != null)
                {
                    hash = hash * 57 + ExcludePaths.GetHashCode();
                }

                return(hash);
            }
        }
        ////////////////////////////////////////////////////////////////////////////////////////////////////
        /// <summary>
        /// Create a new instance of the index manager. There should always be only one instance of the
        /// index manager.
        /// </summary>
        ///
        /// <remarks>   Semantika d.o.o.,. </remarks>
        ///
        /// <param name="indexLocation">  The configuration of the index. </param>
        ////////////////////////////////////////////////////////////////////////////////////////////////////
        public IndexManager(IndexingMode indexingMode, string indexLocation, bool isReadOnly = false)
        {
            //Go through all index types and create the management structure for individual indexes
            var indexObjectTypes = Enum.GetValues(typeof(IndexObjectType)).Cast <IndexObjectType>().ToArray();

            m_indexingMode = indexingMode;

            if (indexingMode == IndexingMode.IndexPerLanguage)
            {
                m_indexTypes = new IndexType[indexObjectTypes.Length * m_supportedLanguages.Length];
                for (int i = 0; i < m_supportedLanguages.Length; i++)
                {
                    for (int j = 0; j < indexObjectTypes.Length; j++)
                    {
                        m_indexTypes[i * indexObjectTypes.Length + j] = new IndexType
                        {
                            Language   = m_supportedLanguages[i],
                            ObjectType = indexObjectTypes[j]
                        };
                    }
                }
            }
            else
            {
                m_indexTypes = new IndexType[indexObjectTypes.Length];
                for (int j = 0; j < indexObjectTypes.Length; j++)
                {
                    m_indexTypes[j] = new IndexType
                    {
                        Language   = m_singleIndexLangConstant,
                        ObjectType = indexObjectTypes[j]
                    };
                }
            }

            foreach (IndexType curType in m_indexTypes)
            {
                var directoryPath = indexLocation + "\\FTS_" + curType.ToString();
                IndexManagementObjects indexManagementObjects = new IndexManagementObjects
                {
                    CurrentReaderUsers = 0,
                    Directory          = FSDirectory.Open(directoryPath),
                    OldReaders         = new ConcurrentDictionary <IndexReader, int>()
                };

                m_managementObjects[curType] = indexManagementObjects;
            }
            IsReadOnly      = isReadOnly;
            m_taskScheduler = TaskScheduler.Default;
        }
Пример #20
0
        /// <summary>
        ///     Returns true if IndexingPolicy instances are equal
        /// </summary>
        /// <param name="other">Instance of IndexingPolicy to be compared</param>
        /// <returns>Boolean</returns>
        public bool Equals(IndexingPolicy other)
        {
            // credit: http://stackoverflow.com/a/10454552/677735
            if (other == null)
            {
                return(false);
            }

            return
                ((
                     Automatic == other.Automatic ||
                     Automatic != null &&
                     Automatic.Equals(other.Automatic)
                     ) &&
                 (
                     IndexingMode == other.IndexingMode ||
                     IndexingMode != null &&
                     IndexingMode.SequenceEqual(other.IndexingMode)
                 ) &&
                 (
                     IncludePaths == other.IncludePaths ||
                     IncludePaths != null &&
                     IncludePaths.Equals(other.IncludePaths)
                 ) &&
                 (
                     IndexType == other.IndexType ||
                     IndexType != null &&
                     IndexType.Equals(other.IndexType)
                 ) &&
                 (
                     NumericPrecision == other.NumericPrecision ||
                     NumericPrecision != null &&
                     NumericPrecision.Equals(other.NumericPrecision)
                 ) &&
                 (
                     StringPrecision == other.StringPrecision ||
                     StringPrecision != null &&
                     StringPrecision.Equals(other.StringPrecision)
                 ) &&
                 (
                     Path == other.Path ||
                     Path != null &&
                     Path.Equals(other.Path)
                 ) &&
                 (
                     ExcludePaths == other.ExcludePaths ||
                     ExcludePaths != null &&
                     ExcludePaths.Equals(other.ExcludePaths)
                 ));
        }
Пример #21
0
        public MarcellCorpus(IndexingMode indexingMode, string indexPath)
        {
            lock (indexInitLock)
            {
                if (indexManager == null)
                {
                    //Configure the index manager
                    indexManager = new IndexManager(indexingMode, indexPath, true);

                    //Prepare the default implementation of
                    m_searcherCache[defaultSearchParameters] = GetSearcher(defaultSearchParameters);
                    WarmUp();
                }
            }
        }
        public CloudTable CreateIfNotExists(string tableName,
                                            IndexingMode indexingMode,
                                            int?throughput        = null,
                                            int?defaultTimeToLive = null)
        {
            if (string.IsNullOrEmpty(tableName))
            {
                throw new ArgumentNullException(nameof(tableName), "tableName can not be null  or empty");
            }

            CloudTable cloudTable = _cloudTableClient.GetTableReference(tableName);

            cloudTable.CreateIfNotExists(indexingMode, throughput, defaultTimeToLive);
            return(cloudTable);
        }
Пример #23
0
        private string getOutputSwitch(string folderPath, IndexingMode mode)
        {
            switch (mode)
            {
            case IndexingMode.IN_EACH_DIRECTORY:
                this.setCurrentIndexFilePath(folderPath + Path.DirectorySeparatorChar + mSettings.GetConfig().GetIndexFileName());

                return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);

            case IndexingMode.IN_ONE_FOLDER_SEPERATE:

                string strOutputFileName = null;
                string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
                if (Path.GetFileName(folderPath).Length > 0)
                {
                    strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + Path.GetFileName(folderPath) + sep + mSettings.GetConfig().GetIndexFileName();
                }
                else
                {
                    strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + mSettings.GetConfig().GetIndexFileName();
                }

                this.setCurrentIndexFilePath(mSettings.GetConfig().OutputDir + Path.DirectorySeparatorChar + strOutputFileName);

                return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);

            case IndexingMode.IN_ONE_FOLDER_MERGED:
                if (mSettings.GetConfig().MergeFiles)
                {
                    this.setCurrentIndexFilePath(this.mSettings.GetConfig().GetIndexFilePath());
                    if (isFirstEntryToSingleFile == true)
                    {
                        isFirstEntryToSingleFile = false;
                        return(">" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
                    }
                    else
                    {
                        return(">>" + (char)34 + this.getCurrentIndexFilePath() + (char)34);
                    }
                }
                break;
            }

            return(null);
        }
Пример #24
0
        private string getOutputSwitch(string folderPath, IndexingMode mode)
        {
            switch (mode)
            {
                case IndexingMode.IN_EACH_DIRECTORY:
                    this.setCurrentIndexFilePath(folderPath + Path.DirectorySeparatorChar + mSettings.GetConfig().GetIndexFileName());

                    return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
                case IndexingMode.IN_ONE_FOLDER_SEPERATE:

                    string strOutputFileName = null;
                    string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;
                    if (Path.GetFileName(folderPath).Length > 0)
                    {
                        strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + Path.GetFileName(folderPath) + sep + mSettings.GetConfig().GetIndexFileName();
                    }
                    else
                    {
                        strOutputFileName = Path.GetPathRoot(folderPath).Substring(0, 1) + sep + mSettings.GetConfig().GetIndexFileName();
                    }

                    this.setCurrentIndexFilePath(mSettings.GetConfig().OutputDir + Path.DirectorySeparatorChar + strOutputFileName);

                    return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
                case IndexingMode.IN_ONE_FOLDER_MERGED:
                    if (mSettings.GetConfig().MergeFiles)
                    {
                        this.setCurrentIndexFilePath(this.mSettings.GetConfig().GetIndexFilePath());
                        if (isFirstEntryToSingleFile == true)
                        {
                            isFirstEntryToSingleFile = false;
                            return ">" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
                        }
                        else
                        {
                            return ">>" + (char)34 + this.getCurrentIndexFilePath() + (char)34;
                        }
                    }
                    break;
            }

            return null;
        }
        /// <summary>
        /// The create document partitioned collection if not exists async.
        /// </summary>
        /// <param name="partitionId">
        /// The partition id.
        /// </param>
        /// <param name="indexingMode">
        /// The indexing mode.
        /// </param>
        /// <param name="defaultTtl">The default TTL</param>
        /// <returns>
        /// The <see cref="Task"/>.
        /// </returns>
        /// <exception cref="ArgumentNullException">
        /// null value in parameters
        /// </exception>
        private async Task CreateDocumentPartitionedCollectionIfNotExistsAsync(
            string partitionId,
            IndexingMode indexingMode,
            int?defaultTtl)
        {
            if (string.IsNullOrWhiteSpace(partitionId))
            {
                throw new ArgumentNullException(nameof(partitionId));
            }

            try
            {
                await
                this.documentDbClient.ReadDocumentCollectionAsync(
                    UriFactory.CreateDocumentCollectionUri(this.databaseName, this.collectionName))
                .ConfigureAwait(false);
            }
            catch (DocumentClientException de)
            {
                // If the document collection does not exist, create a new partitioned collection
                if (de.StatusCode == HttpStatusCode.NotFound)
                {
                    var partitionKey = new PartitionKeyDefinition();
                    partitionKey.Paths.Add(partitionId);
                    var collectionInfo = new DocumentCollection
                    {
                        Id             = this.collectionName,
                        PartitionKey   = partitionKey,
                        IndexingPolicy = new IndexingPolicy {
                            Automatic = true, IndexingMode = indexingMode
                        },
                        DefaultTimeToLive = defaultTtl
                    };
                    await this.documentDbClient.CreateDocumentCollectionAsync(UriFactory.CreateDatabaseUri(this.databaseName), collectionInfo).ConfigureAwait(false);
                }
                else
                {
                    throw;
                }
            }
        }
Пример #26
0
        public static LucField.Index ToLuceneIndexingMode(IndexingMode mode)
        {
            if (mode == IndexingMode.Default)
            {
                mode = IndexingInfo.DefaultIndexingMode;
            }

            switch (mode)
            {
            case IndexingMode.Analyzed: return(LucField.Index.ANALYZED);

            case IndexingMode.AnalyzedNoNorms: return(LucField.Index.ANALYZED_NO_NORMS);

            case IndexingMode.No: return(LucField.Index.NO);

            case IndexingMode.NotAnalyzed: return(LucField.Index.NOT_ANALYZED);

            case IndexingMode.NotAnalyzedNoNorms: return(LucField.Index.NOT_ANALYZED_NO_NORMS);

            default: throw new ArgumentOutOfRangeException("Not supported IndexingMode: " + mode);
            }
        }
Пример #27
0
Файл: Di4.cs Проект: olgatei/Di4
 public void Add(List <I> intervals, IndexingMode mode, uint collectionID)
 {
     Add(intervals, mode, collectionID, Environment.ProcessorCount);
 }
Пример #28
0
        public override string IndexNow(IndexingMode IndexMode, bool bWriteToFile = true)
        {
            string fp = null;
            StringBuilder sb = new StringBuilder();
            List<string> folderList = new List<string>();
            folderList = mSettings.GetConfig().FolderList;
            TreeNetIndexer treeNetLib = new TreeNetIndexer(mSettings);

            string ext = mSettings.GetConfig().IndexFileExt;

            switch (IndexMode)
            {
                case IndexingMode.IN_EACH_DIRECTORY:
                    IndexInEachDir(mSettings);
                    break;
                case IndexingMode.IN_ONE_FOLDER_MERGED:
                    if (mSettings.GetConfig().MergeFiles)
                    {
                        fp = mSettings.fGetIndexFilePath(-1, IndexingMode.IN_ONE_FOLDER_MERGED);

                        if (mSettings.GetConfig().FolderList.Count > 1)
                        {
                            for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 2; i++)
                            {
                                string strDirPath = mSettings.GetConfig().FolderList[i];
                                TreeDir dir = new TreeDir(strDirPath);

                                this.CurrentDirMessage = "Indexing " + strDirPath;

                                if (ext.Contains(".html"))
                                {
                                    treeNetLib.mBooMoreFilesToCome = true;
                                    treeNetLib.IndexRootFolderToHtml(strDirPath, sb, false);
                                }
                                else
                                {
                                    treeNetLib.IndexFolderToTxt(strDirPath, sb, false);
                                }

                                this.Progress += 1;
                            }
                        }

                        TreeDir lastDir = new TreeDir(mSettings.GetConfig().FolderList[mSettings.GetConfig().FolderList.Count - 1]);
                        this.CurrentDirMessage = "Indexing " + lastDir.DirectoryPath();

                        if (ext.Contains(".html"))
                        {
                            treeNetLib.mBooFirstIndexFile = false || mSettings.GetConfig().FolderList.Count == 1;
                            treeNetLib.IndexRootFolderToHtml(lastDir.DirectoryPath(), sb, true);
                        }
                        else
                        {
                            treeNetLib.IndexFolderToTxt(lastDir.DirectoryPath(), sb, mSettings.GetConfig().AddFooter);
                        }

                        if (mSettings.GetConfig().ZipMergedFile)
                        {
                            mSettings.ZipAdminFile(fp, null);
                        }

                        this.Progress += 1;
                    }
                    break;
                case IndexingMode.IN_ONE_FOLDER_SEPERATE:
                    // DO NOT MERGE INDEX FILES
                    if (!Directory.Exists(mSettings.GetConfig().OutputDir))
                    {
                        MessageBox.Show(string.Format("{0} does not exist." + Environment.NewLine + Environment.NewLine + "Please change the Output folder in Configuration." + Environment.NewLine + "The index file will be created in the same folder you chose to index.", mSettings.GetConfig().OutputDir), Application.ProductName, MessageBoxButtons.OK);
                    }

                    for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
                    {
                        string strDirPath = mSettings.GetConfig().FolderList[i];

                        string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
                        string sDirName = Path.GetFileName(strDirPath);
                        string sep = mSettings.GetOptions().IndividualIndexFileWordSeperator;

                        //where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName
                        // New Behavior for getting where location
                        fp = mSettings.fGetIndexFilePath(i, IndexingMode.IN_ONE_FOLDER_SEPERATE);
                        mSettings.GetConfig().SetIndexPath(fp);
                        //MsgBox(where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName)

                        this.CurrentDirMessage = "Indexing " + strDirPath;

                        if (ext.Contains(".html"))
                        {
                            treeNetLib.mBooFirstIndexFile = true;
                            treeNetLib.IndexRootFolderToHtml(strDirPath, sb, mSettings.GetConfig().AddFooter);
                        }
                        else
                        {
                            treeNetLib.IndexFolderToTxt(strDirPath, sb, mSettings.GetConfig().AddFooter);
                        }

                        if (mSettings.GetConfig().ZipFilesInOutputDir)
                        {
                            mSettings.ZipAdminFile(fp, null);
                        }

                        this.Progress += 1;
                    }
                    break;
            }

            if (bWriteToFile)
            {
                using (StreamWriter sw = new StreamWriter(fp, false))
                {
                    sw.Write(sb.ToString());
                    sw.Close();
                }
            }
            return sb.ToString();
        }
 /// <summary>
 /// Defines the <see cref="Container"/>'s <see cref="Cosmos.IndexingMode"/>.
 /// </summary>
 /// <param name="indexingMode">An <see cref="Cosmos.IndexingMode"/></param>
 /// <returns>An instance of <see cref="IndexingPolicyDefinition{T}"/>.</returns>
 /// <remarks>
 /// If multiple calls are made to this method within the same <see cref="IndexingPolicyDefinition{T}"/>, the last one will apply.
 /// </remarks>
 public IndexingPolicyDefinition <T> WithIndexingMode(IndexingMode indexingMode)
 {
     this.indexingPolicy.IndexingMode = indexingMode;
     return(this);
 }
Пример #30
0
        internal ExecutionReport Add(
            uint collectionID,
            Dictionary <string, Dictionary <char, List <I> > > intervals,
            char strand,
            IndexingMode indexinMode,
            MaxDegreeOfParallelism maxDegreeOfParallelism)
        {
            int totalIntervals = 0;

            switch (_memory)
            {
            case Memory.HDD:
                if (_chrSection == null)
                {
                    _chrSection = new ChrSection();
                }
                ConfigurationManager.RefreshSection(_sectionTitle);

                switch (_hddPerformance)
                {
                // TODO:
                // This case is not complete, because other operations
                // are not supporting this method.
                case HDDPerformance.LeastMemory:
                    _stpWtch.Restart();
                    foreach (var chr in intervals)
                    {
                        foreach (var strandEntry in chr.Value)
                        {
                            using (var di4 = new Di4 <C, I, M>(GetDi4Options(GetDi4File(chr.Key, strand))))        // this might be wrong
                            {
                                di4.Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
                                totalIntervals += strandEntry.Value.Count;
                            }
                        }
                    }
                    _stpWtch.Stop();
                    break;

                case HDDPerformance.Fastest:
                    /// Initialize by a sequential foreach loop.
                    foreach (var chr in intervals)
                    {
                        foreach (var strandEntry in chr.Value)
                        {
                            if (!chrs.ContainsKey(chr.Key))
                            {
                                chrs.Add(chr.Key, new Dictionary <char, Di4 <C, I, M> >());
                            }
                            if (!chrs[chr.Key].ContainsKey(strand))
                            {
                                chrs[chr.Key].Add(strand, new Di4 <C, I, M>(GetDi4Options(GetDi4File(chr.Key, strand))));
                            }
                        }
                    }

                    _stpWtch.Restart();
                    /// Populate inside a parallel foreach loop.
                    Parallel.ForEach(intervals,
                                     new ParallelOptions {
                        MaxDegreeOfParallelism = maxDegreeOfParallelism.chrDegree
                    },
                                     chr =>
                    {
                        foreach (var strandEntry in chr.Value)
                        {
                            chrs[chr.Key][strand].Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
                            //chrs[chr.Key][strand].Commit();
                            totalIntervals += strandEntry.Value.Count;
                        }
                    });
                    _stpWtch.Stop();
                    break;
                }


                if (_config.Sections[_sectionTitle] == null)
                {
                    _config.Sections.Add(_sectionTitle, _chrSection);
                }

                _config.Save(ConfigurationSaveMode.Modified);
                ConfigurationManager.RefreshSection(_sectionTitle);
                break;

            case Memory.RAM:
                _stpWtch.Restart();
                foreach (var chr in intervals)
                {
                    foreach (var strandEntry in chr.Value)
                    {
                        if (!chrs.ContainsKey(chr.Key))
                        {
                            chrs.Add(chr.Key, new Dictionary <char, Di4 <C, I, M> >());
                        }
                        if (!chrs[chr.Key].ContainsKey(strand))
                        {
                            chrs[chr.Key].Add(strand, new Di4 <C, I, M>(GetDi4Options()));
                        }

                        chrs[chr.Key][strand].Add(strandEntry.Value, indexinMode, collectionID, maxDegreeOfParallelism.di4Degree);
                        totalIntervals += strandEntry.Value.Count;
                    }
                }
                _stpWtch.Stop();
                break;
            }

            return(new ExecutionReport(totalIntervals, _stpWtch.Elapsed));
        }
Пример #31
0
 public ExecutionReport Add(uint collectionID, Dictionary <string, Dictionary <char, List <I> > > peaks, IndexingMode indexingMode, MaxDegreeOfParallelism maxDegreeOfParallelism)
 {
     return(genome.Add(collectionID, peaks, '*', indexingMode, maxDegreeOfParallelism));
 }
Пример #32
0
        public override string IndexNow(IndexingMode mIndexMode, bool bWriteToFile = true)
        {
            TreeWalkIndexer tree = new TreeWalkIndexer(mSettings);
            bool isMergeFile = mSettings.GetConfig().MergeFiles;
            bool isRemoveBranches = mSettings.GetConfig().RemoveTreeBranches;

            for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
            {
                string batFilePath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\temp" + i.ToString() + ".bat";
                string CURRENT_DIR = mSettings.GetConfig().FolderList[i];
                string TREE_COMMAND = "%windir%\\system32\\tree.com " + tree.getSourceSwitch(CURRENT_DIR) + tree.getAsciiSwitch() + tree.getAddFilesSwitch() + tree.getOutputSwitch(CURRENT_DIR, mIndexMode);

                using (StreamWriter sw = new StreamWriter(batFilePath))
                {
                    sw.WriteLine(TREE_COMMAND);
                    //1.5.3.4 Didn't tag index files created in the same folder witout appending
                    if (mSettings.GetConfig().AddFooter)
                    {
                        if (mIndexMode == IndexingMode.IN_EACH_DIRECTORY | i == mSettings.GetConfig().FolderList.Count - 1 | (isMergeFile == false & mIndexMode == IndexingMode.IN_ONE_FOLDER_MERGED))
                        {
                            sw.WriteLine(mSettings.getBlankLine(tree.getCurrentIndexFilePath()));
                            sw.WriteLine(mSettings.GetFooterText(tree.getCurrentIndexFilePath(), IndexingEngine.TreeLib, false));
                        }
                    }
                    sw.WriteLine("DEL " + (char)34 + batFilePath + (char)34);
                }

                Process proc = new Process();
                proc = mSettings.StartHiddenProcess(batFilePath, true);

                if (isRemoveBranches)
                {
                    tree.removeTreeBranches(tree.getCurrentIndexFilePath());
                }

                if (mIndexMode == IndexingMode.IN_EACH_DIRECTORY | i == mSettings.GetConfig().FolderList.Count - 1 | (isMergeFile == false & mIndexMode == IndexingMode.IN_ONE_FOLDER_MERGED))
                {
                    if (mSettings.GetConfig().ZipFilesInEachDir)
                    {
                        mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
                    }
                    if (mSettings.GetConfig().ZipMergedFile)
                    {
                        mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
                    }
                }

                if (mIndexMode == IndexingMode.IN_ONE_FOLDER_SEPERATE)
                {
                    if (mSettings.GetConfig().ZipFilesInOutputDir)
                    {
                        //MsgBox(tree.getCurrentIndexFilePath())
                        mSettings.ZipAdminFile(tree.getCurrentIndexFilePath(), null);
                    }
                }

                if (proc.HasExited)
                {
                    this.Progress += 1;
                    this.CurrentDirMessage = "Indexed " + mSettings.GetConfig().FolderList[i];
                }
            }

            return File.ReadAllText(tree.getCurrentIndexFilePath());
        }
 public IndexingPolicyImpl <ParentImplT, IParentT, DefinitionParentT, UpdateParentT> WithIndexingMode(IndexingMode indexingMode)
 {
     Inner.IndexingMode = indexingMode;
     return(this);
 }
Пример #34
0
 /// <summary>
 /// Initializes an instance of the IndexField with a named System.DateTime value and indexing metadata.
 /// </summary>
 /// <param name="name">The name of the term.</param>
 /// <param name="value">System.DateTime value</param>
 /// <param name="mode">Indexing mode.</param>
 /// <param name="store">Index storing mode.</param>
 /// <param name="termVector">Term vector handling.</param>
 public IndexField(string name, DateTime value, IndexingMode mode, IndexStoringMode store, IndexTermVector termVector) : base(name, value)
 {
     Mode = mode; Store = store; TermVector = termVector;
 }
        public override string IndexNow(IndexingMode IndexMode, bool bWriteToFile = true)
        {
            string        fp         = null;
            StringBuilder sb         = new StringBuilder();
            List <string> folderList = new List <string>();

            folderList = mSettings.GetConfig().FolderList;
            TreeNetIndexer treeNetLib = new TreeNetIndexer(mSettings);

            string ext = mSettings.GetConfig().IndexFileExt;

            switch (IndexMode)
            {
            case IndexingMode.IN_EACH_DIRECTORY:
                IndexInEachDir(mSettings);
                break;

            case IndexingMode.IN_ONE_FOLDER_MERGED:
                if (mSettings.GetConfig().MergeFiles)
                {
                    fp = mSettings.fGetIndexFilePath(-1, IndexingMode.IN_ONE_FOLDER_MERGED);

                    if (mSettings.GetConfig().FolderList.Count > 1)
                    {
                        for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 2; i++)
                        {
                            string  strDirPath = mSettings.GetConfig().FolderList[i];
                            TreeDir dir        = new TreeDir(strDirPath);

                            this.CurrentDirMessage = "Indexing " + strDirPath;

                            if (ext.Contains(".html"))
                            {
                                treeNetLib.mBooMoreFilesToCome = true;
                                treeNetLib.IndexRootFolderToHtml(strDirPath, sb, false);
                            }
                            else
                            {
                                treeNetLib.IndexFolderToTxt(strDirPath, sb, false);
                            }

                            this.Progress += 1;
                        }
                    }

                    TreeDir lastDir = new TreeDir(mSettings.GetConfig().FolderList[mSettings.GetConfig().FolderList.Count - 1]);
                    this.CurrentDirMessage = "Indexing " + lastDir.DirectoryPath();

                    if (ext.Contains(".html"))
                    {
                        treeNetLib.mBooFirstIndexFile = false || mSettings.GetConfig().FolderList.Count == 1;
                        treeNetLib.IndexRootFolderToHtml(lastDir.DirectoryPath(), sb, true);
                    }
                    else
                    {
                        treeNetLib.IndexFolderToTxt(lastDir.DirectoryPath(), sb, true);
                    }

                    if (mSettings.GetConfig().ZipMergedFile)
                    {
                        mSettings.ZipAdminFile(fp, null);
                    }

                    this.Progress += 1;
                }
                break;

            case IndexingMode.IN_ONE_FOLDER_SEPERATE:
                // DO NOT MERGE INDEX FILES
                if (!Directory.Exists(mSettings.GetConfig().OutputDir))
                {
                    MessageBox.Show(string.Format("{0} does not exist." + Environment.NewLine + Environment.NewLine + "Please change the Output folder in Configuration." + Environment.NewLine + "The index file will be created in the same folder you chose to index.", mSettings.GetConfig().OutputDir), Application.ProductName, MessageBoxButtons.OK);
                }

                for (int i = 0; i <= mSettings.GetConfig().FolderList.Count - 1; i++)
                {
                    string strDirPath = mSettings.GetConfig().FolderList[i];

                    string sDrive   = Path.GetPathRoot(strDirPath).Substring(0, 1);
                    string sDirName = Path.GetFileName(strDirPath);
                    string sep      = mSettings.GetOptions().IndividualIndexFileWordSeperator;

                    //where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName
                    // New Behavior for getting where location
                    fp = mSettings.fGetIndexFilePath(i, IndexingMode.IN_ONE_FOLDER_SEPERATE);
                    mSettings.GetConfig().SetIndexPath(fp);
                    //MsgBox(where = mSettings.GetConfig().OutputDir + "\" + sDrive + sep + sDirName + sep + mSettings.GetConfig().GetIndexFileName)

                    this.CurrentDirMessage = "Indexing " + strDirPath;

                    if (ext.Contains(".html"))
                    {
                        treeNetLib.mBooFirstIndexFile = true;
                        treeNetLib.IndexRootFolderToHtml(strDirPath, sb, true);
                    }
                    else
                    {
                        treeNetLib.IndexFolderToTxt(strDirPath, sb, true);
                    }

                    if (mSettings.GetConfig().ZipFilesInOutputDir)
                    {
                        mSettings.ZipAdminFile(fp, null);
                    }

                    this.Progress += 1;
                }
                break;
            }

            if (bWriteToFile)
            {
                using (StreamWriter sw = new StreamWriter(fp, false))
                {
                    sw.Write(sb.ToString());
                    sw.Close();
                }
            }
            return(sb.ToString());
        }
        public string fGetIndexFilePath(int folderID, IndexingMode mode)
        {
            //Dim strDirPath As String = GetConfig().FolderList.Item(folderID)
            string lPath = string.Empty;
            // strDirPath + "\" + GetConfig().GetIndexFileName

            switch (mode)
            {
                case IndexingMode.IN_ONE_FOLDER_SEPERATE:
                    string strDirPath = GetConfig().FolderList[folderID];
                    string sDrive = Path.GetPathRoot(strDirPath).Substring(0, 1);
                    string sDirName = Path.GetFileName(strDirPath);
                    string sep = GetOptions().IndividualIndexFileWordSeperator;
                    if (Directory.Exists(GetConfig().OutputDir))
                    {
                        lPath = GetConfig().OutputDir + Path.DirectorySeparatorChar + sDirName + sep + sDrive + sep + GetConfig().GetIndexFileName();
                    }
                    break;
                case IndexingMode.IN_EACH_DIRECTORY:
                    // this is also what it is first initialized to
                    strDirPath = GetConfig().FolderList[folderID];
                    lPath = strDirPath + "\\" + GetConfig().GetIndexFileName();
                    break;
                default:
                    lPath = GetConfig().GetIndexFilePath();
                    break;
            }

            return lPath;
        }