/// <summary> /// Insert a word in the database and then insert all the parts /// </summary> /// <param name="word"></param> /// <param name="token"></param> /// <returns></returns> private async Task <long> InsertWordAsync(IWord word, CancellationToken token) { // sanity check. Contract.Assert(_wordsHelper != null); var wordId = await _wordsHelper.GetIdAsync(word.Value, token).ConfigureAwait(false); if (wordId != -1) { // already exists return(wordId); } try { // then insert it. wordId = await _wordsHelper.InsertAndGetIdAsync(word.Value, token).ConfigureAwait(false); if (wordId == -1) { _logger.Error($"There was an issue getting the word id: {word.Value} from the persister"); return(-1); } // we now can add/find the parts for that word. var partIds = await GetOrInsertParts(word, token).ConfigureAwait(false); // marry the word id, (that we just added). // with the partIds, (that we just added). await _wordsParts.AddOrUpdateWordParts(wordId, new HashSet <long>(partIds), token).ConfigureAwait(false); // all done return the id of the word. return(wordId); } catch (OperationCanceledException) { _logger.Warning("Received cancellation request - Insert single word"); throw; } catch (Exception ex) { _logger.Exception(ex); throw; } }
/// <inheritdoc /> public async Task<IList<FileInfo>> ParseDirectoryAsync(DirectoryInfo directory, CancellationToken token) { try { // sanity check if (!helper.File.CanReadDirectory(directory)) { _logger.Warning($"Cannot Parse Directory: {directory.FullName}"); return null; } IEnumerable<FileInfo> files; try { files = directory.EnumerateFiles().ToArray(); } catch (SecurityException) { // we cannot access/enumerate this file // but we might as well continue _logger.Verbose($"Security error while parsing directory: {directory.FullName}."); return null; } catch (UnauthorizedAccessException) { // we cannot access/enumerate this file // but we might as well continue _logger.Verbose($"Unauthorized Access while parsing directory: {directory.FullName}."); return null; } catch (Exception e) { _logger.Error($"Exception while parsing directory: {directory.FullName}. {e.Message}"); return null; } // all the posible files. var posibleFiles = new ConcurrentBag<FileInfo>(); // check if we need to use threads or not. var processorCount = Environment.ProcessorCount; if (files.Count() < processorCount) { foreach (var file in files) { token.ThrowIfCancellationRequested(); if (!helper.File.CanReadFile(file)) { continue; } posibleFiles.Add(file); } } else { // partition the files into managable groups // so we don't create thoushands of tasks. var partitions = Partitioner.Create(files).GetPartitions(processorCount * 2); var tasks = partitions.Select(partition => Task.Run(() => { using(partition) { while (partition.MoveNext()) { token.ThrowIfCancellationRequested(); var file = partition.Current; if (helper.File.CanReadFile(file)) { posibleFiles.Add(file); } } } }, token)).ToArray(); // we then wait for them all await Wait.WhenAll(tasks, _logger, token).ConfigureAwait(false); } // if we found nothing we return null. return posibleFiles.Any() ? posibleFiles.ToList() : null; } catch (OperationCanceledException) { _logger.Warning("Received cancellation request - directory parsing"); throw; } }