public void Run()
{
var scopedTimer = new ScopedTimer();
using (scopedTimer)
{
IsScanningFolder = true;
checkIndexForFile = new CheckIndexForFileQueue(() => IsScanningFolder, numTasksToUse, serverUrl, basePath, alias);
prepareFile = new PrepareFileQueue(() => checkIndexForFile.IsActive, numTasksToUse, serverUrl);
getExif = new GetExifQueue(() => prepareFile.IsActive, numTasksToUse, basePath);
createMedia = new CreateMediaQueue(() => prepareFile.IsActive || getExif.IsActive, numTasksToUse, basePath);
indexMedia = new IndexMediaQueue(() => createMedia.IsActive, numTasksToUse, serverUrl, count => IndexedFileCount += count);
checkIndexForFile.Start(prepareFile.Enqueue);
prepareFile.Start(createMedia.Enqueue, getExif.Enqueue);
getExif.Start(createMedia.Enqueue);
createMedia.Start(indexMedia.Enqueue);
indexMedia.Start();
scanner.ScanPath(basePath, checkIndexForFile.Enqueue);
IsScanningFolder = false;
logger.Error("Done scanning files");
checkIndexForFile.WaitForCompletion();
prepareFile.WaitForCompletion();
getExif.WaitForCompletion();
createMedia.WaitForCompletion();
indexMedia.WaitForCompletion();
}
ElapsedMsec = scopedTimer.TotalMilliseconds;
}
static public bool ShouldAdd(ElasticsearchClient client, ref CandidateFile candidate)
{
var searchQuery = new { query = new { term = new { _id = candidate.AliasedPath } } };
ElasticsearchResponse<string> response;
var timer = new ScopedTimer();
using (timer)
{
response = client.Search<string>(Media.IndexName, JsonConvert.SerializeObject(searchQuery));
}
if (!FirstSearchMsec.HasValue)
{
FirstSearchMsec = timer.TotalMilliseconds;
}
if (!response.Success)
{
response.LogFailure(logger);
return false;
}
candidate.Signature = CalculateSignature(candidate.FullFilename);
candidate.LengthInBytes = new FileInfo(candidate.FullFilename).Length;
var searchResponse = JsonConvert.DeserializeObject<SearchResponse>(response.Response);
// if ((timer.TotalMilliseconds - searchResponse.Took) > 200)
// {
// logger.Warn("Search took {1} milliseconds for: {0}", candidate.AliasedPath, timer.TotalMilliseconds);
// }
// If there's more than one matching path or the signature is different, go through the add process
// If it's not already in the index, add it.
if (searchResponse.Hits.Total != 1)
return true;
var media = searchResponse.Hits.Hits.First().Media;
if (media.Signature != candidate.Signature || media.LengthInBytes != candidate.LengthInBytes)
{
logger.Error("Add {0} due to mis-match ({0}, {1} -- {2}, {3})", candidate.AliasedPath,
media.Signature, media.LengthInBytes,
candidate.Signature, candidate.LengthInBytes);
return true;
}
return false;
}