/// <param name="targetTable"> target table or view</param>
/// <returns></returns>
private SearchJob BuildSearchJob(string targetTable)
{
const int IndexCacheCount = 3;
SearchJob search = new SearchJob();
BuildIndexRoot();
string indexPath = _indexDir;
if (string.IsNullOrEmpty(indexPath))
{
indexPath = BuildIndexPath(targetTable);
}
if (!IndexExists(indexPath))
{
throw new Exception("Invalid index path: " + indexPath);
}
search.IndexesToSearch.Add(indexPath);
search.SearchFlags = search.SearchFlags | SearchFlags.dtsSearchDelayDocInfo;
search.MaxFilesToRetrieve = _maxHitCount;
if (_enableDtIndexCache)
{
if (_cache == null)
{
_cache = new IndexCache(IndexCacheCount);
}
search.SetIndexCache(_cache);
}
return(search);
}
/// <summary>
/// Compare our current children (presumably just read from the repo) with the current state of the file system and adjust for any changes
/// ***Currently does not contain logic to maintain items that are unavailable in the file system***
/// </summary>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <param name="recursive">if set to <c>true</c> [recursive].</param>
/// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
/// <returns>Task.</returns>
protected override Task ValidateChildrenInternal(IProgress <double> progress, CancellationToken cancellationToken, bool?recursive = null, bool forceRefreshMetadata = false)
{
//we don't directly validate our children
//but we do need to clear out the index cache...
if (IndexCache != null)
{
IndexCache.Clear();
}
ResetDynamicChildren();
return(NullTaskResult);
}
public bool DropIndex(string memberPath)
{
IndexWrapper index;
if (IndexCache.TryGetValue(memberPath, out index))
{
IndexCache.Remove(memberPath);
Remove(memberPath, _ => true);
index.Drop();
return(true);
}
return(false);
}
private static void SetState()
{
var state = StorageState.Instance;
state.TopOfChain = null;
var cacheL1 = new IndexCache();
var cacheL2 = new LinkedListCache();
var longTermUserStorage = new LongTermUserStorage();
state.SetNextInChain(cacheL1);
state.SetNextInChain(cacheL2);
state.SetNextInChain(longTermUserStorage);
}
/// <summary>
/// Returns the indexed children for this user from the cache. Caches them if not already there.
/// </summary>
/// <param name="user">The user.</param>
/// <param name="indexBy">The index by.</param>
/// <returns>IEnumerable{BaseItem}.</returns>
private IEnumerable <BaseItem> GetIndexedChildren(User user, string indexBy)
{
List <BaseItem> result;
var cacheKey = user.Name + indexBy;
IndexCache.TryGetValue(cacheKey, out result);
if (result == null)
{
//not cached - cache it
Func <User, IEnumerable <BaseItem> > indexing;
IndexByOptions.TryGetValue(indexBy, out indexing);
result = BuildIndex(indexBy, indexing, user);
}
return(result);
}
public bool EnsureIndex(IndexDefinition indexDefinition)
{
// Try to not rebuild index, so we check if there exists an equal defintion already
IndexWrapper existing;
if (IndexCache.TryGetValue(indexDefinition.Path, out existing))
{
if (indexDefinition.Options.Equals(existing.IndexDefinition.Options))
{
// phew, existing index was already matching. bail out
return(false);
}
DropIndex(indexDefinition.Path);
}
// An index can not both exclude certain values and at the same time include certain values
var hasIncludeValues = (indexDefinition.Options.IncludeValues != null) &&
(indexDefinition.Options.IncludeValues.Length > 0);
var hasExcludeValues = (indexDefinition.Options.ExcludeValues != null) &&
(indexDefinition.Options.ExcludeValues.Length > 0);
if (hasIncludeValues && hasExcludeValues)
{
throw new KiwiDbException("An index can not have an exclusion list and an inclusion list at the same time.");
}
var index = new IndexWrapper(this, indexDefinition)
{
IsChanged = true
};
IndexCache.Add(indexDefinition.Path, index);
Insert(indexDefinition.Path, JSON.FromObject(indexDefinition));
// rebuild index
foreach (var record in MasterTable.Scan())
{
UpdateIndex(record.Key, null, record.Value);
}
return(true);
}
/// <summary>
/// Standardowy konstruktor
/// </summary>
/// <param name="diskFileReader">Strumień binarny odczytu pliku danych (ustawiony na początek pliku)</param>
/// <param name="indexCache">Cache indeksu słownika</param>
/// <param name="memoryCache">Cache podręczny tokenów jeszcze nie zapisanych na dysku</param>
public DiskAndCacheFusion(BinaryReader diskFileReader, IndexCache indexCache, Dictionary<string, uint> memoryCache)
{
this.diskFileReader = diskFileReader;
this.indexCache = indexCache;
cacheEnumerator = memoryCache.OrderBy(x => x.Key, StringComparer.Ordinal).GetEnumerator();
getNextFromDisk();
getNextFromMemory();
}
/// <summary>
/// Dodaje wpis indeksu do pliku oraz pamięci podręcznej
/// </summary>
/// <param name="entry">Wpis indeksu do dodania</param>
/// <param name="cache">Pamięć podręczna do dodania wpisu</param>
private void writeIndex(IndexEntry entry, IndexCache cache)
{
if (entry.firstToken == null)
entry.firstToken = "";
if (entry.lastToken == null)
entry.lastToken = "";
indexTmpFile.Write((long)entry.offset);
indexTmpFile.Write(entry.firstToken);
int rest = IndexEntry.stringMaxLength - Misc.bytesLenInFile(entry.firstToken);
while (rest-- != 0)
indexTmpFile.Write((byte)0);
indexTmpFile.Write(entry.firstTokenId);
indexTmpFile.Write(entry.lastToken);
rest = IndexEntry.stringMaxLength - Misc.bytesLenInFile(entry.lastToken);
while (rest-- != 0)
indexTmpFile.Write((byte)0);
indexTmpFile.Write(entry.lastTokenId);
cache.add(entry);
}
/// <summary>
/// Złącza pamięć podręczną słownika ze słownikiem na dystku, tworząc nowy słownik na dysku, zaweirający także wpisy z cache'u.
/// Czyści pamięć podręczną. Otwiera na nowo strumienie pliku słownika i indeksu.
/// </summary>
private void mergeToFile()
{
/* Z pliku słownika będziemy przepisywać - wróć na początek */
dictionaryFile.Seek(0, SeekOrigin.Begin);
indexFile.Seek(0, SeekOrigin.Begin);
/* Utwórz BinaryWriter dla tymczasowego pliku słownika (trafi tu suma cache'u i słownika dyskowego */
dictionaryTmpFile = new BinaryWriter(new FileStream(dictionaryTmpFilePath, FileMode.Create), System.Text.Encoding.UTF8);
/* Utwórz BinaryWriter dla pliku indeksu */
indexTmpFile = new BinaryWriter(new FileStream(indexTmpFilePath, FileMode.Create), System.Text.Encoding.UTF8);
IndexCache newIndexCache = new IndexCache();
long offset = 0, startOffset = 0;
DiskAndCacheFusion fusion = new DiskAndCacheFusion(dictionaryFileBR, indexCache, cacheDictionary);
KeyValuePair<string, uint>[] tokensChunk = fusion.getNextN(indexInterval);
while (tokensChunk.Length != 0)
{
KeyValuePair<string, uint> first = tokensChunk[0];
KeyValuePair<string, uint> last = tokensChunk[tokensChunk.Length - 1];
int firstLen = Misc.bytesLenInFile(first.Key);
int lastLen = Misc.bytesLenInFile(last.Key);
IndexEntry entry = new IndexEntry();
if (firstLen > IndexEntry.stringMaxLength)
{
entry.firstToken = null;
entry.firstTokenId = 0;
writeTokenToTmpFile(first.Key, first.Value, ref offset);
}
else
{
entry.firstToken = first.Key;
entry.firstTokenId = first.Value;
}
for (int i = 1, max = tokensChunk.Length - 2; i <= max; i++)
writeTokenToTmpFile(tokensChunk[i].Key, tokensChunk[i].Value, ref offset);
if (lastLen > IndexEntry.stringMaxLength)
{
entry.lastToken = null;
entry.lastTokenId = 0;
writeTokenToTmpFile(last.Key, last.Value, ref offset);
}
else
{
entry.lastToken = last.Key;
entry.lastTokenId = last.Value;
}
entry.offset = startOffset;
writeIndex(entry, newIndexCache);
startOffset = offset;
tokensChunk = fusion.getNextN(indexInterval);
}
/* Pamięć podręczna została przepisana (dopisana) do pliku - wyczyść */
cacheDictionary.Clear();
/* Zamknij writer pliku tymczasowego słownika */
dictionaryTmpFile.Flush();
dictionaryTmpFile.Close();
/* Zamknij writera pliku indeksu */
indexTmpFile.Flush();
indexTmpFile.Close();
/* Zamknij reader pliku słownika (będzie otwarty na nowo po podmianie) */
dictionaryFileBR.Close();
indexFileBR.Close();
/* Podmień plik słownika na plik tymczasowy */
File.Delete(dictionaryFilePath);
File.Move(dictionaryTmpFilePath, dictionaryFilePath);
File.Delete(indexFilePath);
File.Move(indexTmpFilePath, indexFilePath);
/* Otwórz na nowo słownik i indeks */
openDictionaryAndIndex();
indexCache = newIndexCache;
}
public IIndex GetIndex(string memberPath)
{
IndexWrapper indexWrapper;
return(IndexCache.TryGetValue(memberPath, out indexWrapper) ? indexWrapper : null);
}