private async Task AddDocuments(List <string> lexical, List <Element> elements, List <IDocumentationDocument> documentationDocuments, List <string> paths = null)
{
if (paths == null)
{
paths = new List <string>();
}
foreach (var document in documentationDocuments.Where(w => w is ISearchableDocument || w.DocumentType == DocumentationDocumentType.Folder))
{
if (document.DocumentType == DocumentationDocumentType.Folder && document.SubDocumentsCount > 0)
{
List <string> tempFolderPaths = new List <string>(paths)
{
document.Key
};
await AddDocuments(lexical, elements, document.SubDocuments.ToList(), tempFolderPaths);
continue;
}
ISearchableDocument searchableDocument = (ISearchableDocument)document;
string content = "";
try
{
content = await searchableDocument.GetSearchableContent();
} catch (Exception exception)
{
_Logger?.LogWarning($"Unable to get searchable content for document \"{document.Path}\". Skipping. Reason: {exception.Message}");
}
var tokens = await _SearchPipeline.Analyse(new string[] { document.Name, content });
List <string> tempPaths = new List <string>(paths)
{
document.Key
};
var documentFullPath = string.Join("/", tempPaths);
foreach (var token in tokens)
{
if (!lexical.Contains(token))
{
lexical.Add(token);
}
}
elements.Add(new Element
{
Path = documentFullPath,
LexicalIndexes = tokens.Select(s => lexical.IndexOf(s)).Distinct().ToArray()
});;
}
}
private async Task AddDocuments(IndexWriter writer, List <IDocumentationDocument> documentationDocuments, List <string> paths = null)
{
if (paths == null)
{
paths = new List <string>();
}
foreach (var document in documentationDocuments.Where(w => w is ISearchableDocument || w.DocumentType == DocumentationDocumentType.Folder))
{
if (document.DocumentType == DocumentationDocumentType.Folder && document.SubDocumentsCount > 0)
{
List <string> tempFolderPaths = new List <string>(paths);
tempFolderPaths.Add(document.Key);
await AddDocuments(writer, document.SubDocuments.ToList(), tempFolderPaths);
continue;
}
ISearchableDocument searchableDocument = (ISearchableDocument)document;
string content = await searchableDocument.GetContent();
List <string> tempPaths = new List <string>(paths);
tempPaths.Add(document.Key);
do
{
string contentSlice = "";
int sliceLength = content.Length > 32000 ? 32000 : content.Length;
contentSlice = content.Substring(0, sliceLength);
content = content.Remove(0, sliceLength);
Document doc = new Document
{
new TextField("name", searchableDocument.Name, Field.Store.YES),
new TextField("content", contentSlice, Field.Store.YES),
new StringField("keypath", string.Join("/", tempPaths), Field.Store.YES)
};
writer.AddDocument(doc);
} while (content.Length > 0);
}
}
