/// <summary>
/// Insert a word in the database and then insert all the parts
/// </summary>
/// <param name="word"></param>
/// <param name="token"></param>
/// <returns></returns>
private async Task <long> InsertWordAsync(IWord word, CancellationToken token)
{
// sanity check.
Contract.Assert(_wordsHelper != null);
var wordId = await _wordsHelper.GetIdAsync(word.Value, token).ConfigureAwait(false);
if (wordId != -1)
{
// already exists
return(wordId);
}
try
{
// then insert it.
wordId = await _wordsHelper.InsertAndGetIdAsync(word.Value, token).ConfigureAwait(false);
if (wordId == -1)
{
_logger.Error($"There was an issue getting the word id: {word.Value} from the persister");
return(-1);
}
// we now can add/find the parts for that word.
var partIds = await GetOrInsertParts(word, token).ConfigureAwait(false);
// marry the word id, (that we just added).
// with the partIds, (that we just added).
await _wordsParts.AddOrUpdateWordParts(wordId, new HashSet <long>(partIds), token).ConfigureAwait(false);
// all done return the id of the word.
return(wordId);
}
catch (OperationCanceledException)
{
_logger.Warning("Received cancellation request - Insert single word");
throw;
}
catch (Exception ex)
{
_logger.Exception(ex);
throw;
}
}
/// <inheritdoc />
public async Task<IList<FileInfo>> ParseDirectoryAsync(DirectoryInfo directory, CancellationToken token)
{
try
{
// sanity check
if (!helper.File.CanReadDirectory(directory))
{
_logger.Warning($"Cannot Parse Directory: {directory.FullName}");
return null;
}
IEnumerable<FileInfo> files;
try
{
files = directory.EnumerateFiles().ToArray();
}
catch (SecurityException)
{
// we cannot access/enumerate this file
// but we might as well continue
_logger.Verbose($"Security error while parsing directory: {directory.FullName}.");
return null;
}
catch (UnauthorizedAccessException)
{
// we cannot access/enumerate this file
// but we might as well continue
_logger.Verbose($"Unauthorized Access while parsing directory: {directory.FullName}.");
return null;
}
catch (Exception e)
{
_logger.Error($"Exception while parsing directory: {directory.FullName}. {e.Message}");
return null;
}
// all the posible files.
var posibleFiles = new ConcurrentBag<FileInfo>();
// check if we need to use threads or not.
var processorCount = Environment.ProcessorCount;
if (files.Count() < processorCount)
{
foreach (var file in files)
{
token.ThrowIfCancellationRequested();
if (!helper.File.CanReadFile(file))
{
continue;
}
posibleFiles.Add(file);
}
}
else
{
// partition the files into managable groups
// so we don't create thoushands of tasks.
var partitions = Partitioner.Create(files).GetPartitions(processorCount * 2);
var tasks = partitions.Select(partition => Task.Run(() =>
{
using(partition)
{
while (partition.MoveNext())
{
token.ThrowIfCancellationRequested();
var file = partition.Current;
if (helper.File.CanReadFile(file))
{
posibleFiles.Add(file);
}
}
}
}, token)).ToArray();
// we then wait for them all
await Wait.WhenAll(tasks, _logger, token).ConfigureAwait(false);
}
// if we found nothing we return null.
return posibleFiles.Any() ? posibleFiles.ToList() : null;
}
catch (OperationCanceledException)
{
_logger.Warning("Received cancellation request - directory parsing");
throw;
}
}