예제 #1
0
        /// <param name="targetTable"> target table or view</param>
        /// <returns></returns>
        private SearchJob BuildSearchJob(string targetTable)
        {
            const int IndexCacheCount = 3;

            SearchJob search = new SearchJob();

            BuildIndexRoot();

            string indexPath = _indexDir;

            if (string.IsNullOrEmpty(indexPath))
            {
                indexPath = BuildIndexPath(targetTable);
            }

            if (!IndexExists(indexPath))
            {
                throw new Exception("Invalid index path: " + indexPath);
            }

            search.IndexesToSearch.Add(indexPath);
            search.SearchFlags        = search.SearchFlags | SearchFlags.dtsSearchDelayDocInfo;
            search.MaxFilesToRetrieve = _maxHitCount;

            if (_enableDtIndexCache)
            {
                if (_cache == null)
                {
                    _cache = new IndexCache(IndexCacheCount);
                }
                search.SetIndexCache(_cache);
            }

            return(search);
        }
예제 #2
0
        /// <summary>
        /// Compare our current children (presumably just read from the repo) with the current state of the file system and adjust for any changes
        /// ***Currently does not contain logic to maintain items that are unavailable in the file system***
        /// </summary>
        /// <param name="progress">The progress.</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <param name="recursive">if set to <c>true</c> [recursive].</param>
        /// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
        /// <returns>Task.</returns>
        protected override Task ValidateChildrenInternal(IProgress <double> progress, CancellationToken cancellationToken, bool?recursive = null, bool forceRefreshMetadata = false)
        {
            //we don't directly validate our children
            //but we do need to clear out the index cache...
            if (IndexCache != null)
            {
                IndexCache.Clear();
            }

            ResetDynamicChildren();

            return(NullTaskResult);
        }
예제 #3
0
        public bool DropIndex(string memberPath)
        {
            IndexWrapper index;

            if (IndexCache.TryGetValue(memberPath, out index))
            {
                IndexCache.Remove(memberPath);
                Remove(memberPath, _ => true);
                index.Drop();
                return(true);
            }
            return(false);
        }
예제 #4
0
        private static void SetState()
        {
            var state = StorageState.Instance;

            state.TopOfChain = null;

            var cacheL1             = new IndexCache();
            var cacheL2             = new LinkedListCache();
            var longTermUserStorage = new LongTermUserStorage();

            state.SetNextInChain(cacheL1);
            state.SetNextInChain(cacheL2);
            state.SetNextInChain(longTermUserStorage);
        }
예제 #5
0
        /// <summary>
        /// Returns the indexed children for this user from the cache. Caches them if not already there.
        /// </summary>
        /// <param name="user">The user.</param>
        /// <param name="indexBy">The index by.</param>
        /// <returns>IEnumerable{BaseItem}.</returns>
        private IEnumerable <BaseItem> GetIndexedChildren(User user, string indexBy)
        {
            List <BaseItem> result;
            var             cacheKey = user.Name + indexBy;

            IndexCache.TryGetValue(cacheKey, out result);

            if (result == null)
            {
                //not cached - cache it
                Func <User, IEnumerable <BaseItem> > indexing;
                IndexByOptions.TryGetValue(indexBy, out indexing);
                result = BuildIndex(indexBy, indexing, user);
            }
            return(result);
        }
예제 #6
0
        public bool EnsureIndex(IndexDefinition indexDefinition)
        {
            // Try to not rebuild index, so we check if there exists an equal defintion already
            IndexWrapper existing;

            if (IndexCache.TryGetValue(indexDefinition.Path, out existing))
            {
                if (indexDefinition.Options.Equals(existing.IndexDefinition.Options))
                {
                    // phew, existing index was already matching. bail out
                    return(false);
                }
                DropIndex(indexDefinition.Path);
            }

            // An index can not both exclude certain values and at the same time include certain values
            var hasIncludeValues = (indexDefinition.Options.IncludeValues != null) &&
                                   (indexDefinition.Options.IncludeValues.Length > 0);
            var hasExcludeValues = (indexDefinition.Options.ExcludeValues != null) &&
                                   (indexDefinition.Options.ExcludeValues.Length > 0);

            if (hasIncludeValues && hasExcludeValues)
            {
                throw new KiwiDbException("An index can not have an exclusion list and an inclusion list at the same time.");
            }

            var index = new IndexWrapper(this, indexDefinition)
            {
                IsChanged = true
            };

            IndexCache.Add(indexDefinition.Path, index);
            Insert(indexDefinition.Path, JSON.FromObject(indexDefinition));

            // rebuild index
            foreach (var record in MasterTable.Scan())
            {
                UpdateIndex(record.Key, null, record.Value);
            }
            return(true);
        }
예제 #7
0
            /// <summary>
            /// Standardowy konstruktor
            /// </summary>
            /// <param name="diskFileReader">Strumień binarny odczytu pliku danych (ustawiony na początek pliku)</param>
            /// <param name="indexCache">Cache indeksu słownika</param>
            /// <param name="memoryCache">Cache podręczny tokenów jeszcze nie zapisanych na dysku</param>
            public DiskAndCacheFusion(BinaryReader diskFileReader, IndexCache indexCache, Dictionary<string, uint> memoryCache)
            {
                this.diskFileReader = diskFileReader;
                this.indexCache = indexCache;

                cacheEnumerator = memoryCache.OrderBy(x => x.Key, StringComparer.Ordinal).GetEnumerator();

                getNextFromDisk();
                getNextFromMemory();
            }
예제 #8
0
        /// <summary>
        /// Dodaje wpis indeksu do pliku oraz pamięci podręcznej
        /// </summary>
        /// <param name="entry">Wpis indeksu do dodania</param>
        /// <param name="cache">Pamięć podręczna do dodania wpisu</param>
        private void writeIndex(IndexEntry entry, IndexCache cache)
        {
            if (entry.firstToken == null)
                entry.firstToken = "";

            if (entry.lastToken == null)
                entry.lastToken = "";

            indexTmpFile.Write((long)entry.offset);
            indexTmpFile.Write(entry.firstToken);

            int rest = IndexEntry.stringMaxLength - Misc.bytesLenInFile(entry.firstToken);
            while (rest-- != 0)
                indexTmpFile.Write((byte)0);

            indexTmpFile.Write(entry.firstTokenId);

            indexTmpFile.Write(entry.lastToken);

            rest = IndexEntry.stringMaxLength - Misc.bytesLenInFile(entry.lastToken);
            while (rest-- != 0)
                indexTmpFile.Write((byte)0);

            indexTmpFile.Write(entry.lastTokenId);

            cache.add(entry);
        }
예제 #9
0
        /// <summary>
        /// Złącza pamięć podręczną słownika ze słownikiem na dystku, tworząc nowy słownik na dysku, zaweirający także wpisy z cache'u.
        /// Czyści pamięć podręczną. Otwiera na nowo strumienie pliku słownika i indeksu.
        /// </summary>
        private void mergeToFile()
        {
            /* Z pliku słownika będziemy przepisywać - wróć na początek */
            dictionaryFile.Seek(0, SeekOrigin.Begin);

            indexFile.Seek(0, SeekOrigin.Begin);

            /* Utwórz BinaryWriter dla tymczasowego pliku słownika (trafi tu suma cache'u i słownika dyskowego */
            dictionaryTmpFile = new BinaryWriter(new FileStream(dictionaryTmpFilePath, FileMode.Create), System.Text.Encoding.UTF8);

            /* Utwórz BinaryWriter dla pliku indeksu */
            indexTmpFile = new BinaryWriter(new FileStream(indexTmpFilePath, FileMode.Create), System.Text.Encoding.UTF8);

            IndexCache newIndexCache = new IndexCache();

            long offset = 0, startOffset = 0;

            DiskAndCacheFusion fusion = new DiskAndCacheFusion(dictionaryFileBR, indexCache, cacheDictionary);
            KeyValuePair<string, uint>[] tokensChunk = fusion.getNextN(indexInterval);

            while (tokensChunk.Length != 0)
            {
                KeyValuePair<string, uint> first = tokensChunk[0];
                KeyValuePair<string, uint> last = tokensChunk[tokensChunk.Length - 1];

                int firstLen = Misc.bytesLenInFile(first.Key);
                int lastLen = Misc.bytesLenInFile(last.Key);

                IndexEntry entry = new IndexEntry();

                if (firstLen > IndexEntry.stringMaxLength)
                {
                    entry.firstToken = null;
                    entry.firstTokenId = 0;

                    writeTokenToTmpFile(first.Key, first.Value, ref offset);
                }
                else
                {
                    entry.firstToken = first.Key;
                    entry.firstTokenId = first.Value;
                }

                for (int i = 1, max = tokensChunk.Length - 2; i <= max; i++)
                    writeTokenToTmpFile(tokensChunk[i].Key, tokensChunk[i].Value, ref offset);

                if (lastLen > IndexEntry.stringMaxLength)
                {
                    entry.lastToken = null;
                    entry.lastTokenId = 0;

                    writeTokenToTmpFile(last.Key, last.Value, ref offset);
                }
                else
                {
                    entry.lastToken = last.Key;
                    entry.lastTokenId = last.Value;
                }

                entry.offset = startOffset;

                writeIndex(entry, newIndexCache);

                startOffset = offset;

                tokensChunk = fusion.getNextN(indexInterval);
            }

            /* Pamięć podręczna została przepisana (dopisana) do pliku - wyczyść */
            cacheDictionary.Clear();

            /* Zamknij writer pliku tymczasowego słownika */
            dictionaryTmpFile.Flush();
            dictionaryTmpFile.Close();

            /* Zamknij writera pliku indeksu */
            indexTmpFile.Flush();
            indexTmpFile.Close();

            /* Zamknij reader pliku słownika (będzie otwarty na nowo po podmianie) */
            dictionaryFileBR.Close();
            indexFileBR.Close();

            /* Podmień plik słownika na plik tymczasowy */
            File.Delete(dictionaryFilePath);
            File.Move(dictionaryTmpFilePath, dictionaryFilePath);

            File.Delete(indexFilePath);
            File.Move(indexTmpFilePath, indexFilePath);

            /* Otwórz na nowo słownik i indeks */
            openDictionaryAndIndex();

            indexCache = newIndexCache;
        }
예제 #10
0
        public IIndex GetIndex(string memberPath)
        {
            IndexWrapper indexWrapper;

            return(IndexCache.TryGetValue(memberPath, out indexWrapper) ? indexWrapper : null);
        }