private IEnumerable <SearchableDocument> GetSearchableDocument(IPublishedContent content)
{
var fileName = Path.GetFileName(content.Url);
var extension = Path.GetExtension(fileName)?.Trim('.');
bool isFileExtensionAllowedForIndex = _settings.IndexingDocumentTypesKey.Contains(extension, StringComparison.OrdinalIgnoreCase);
if (!content.Url.IsNullOrEmpty())
{
var physicalPath = HostingEnvironment.MapPath(content.Url);
if (!File.Exists(physicalPath))
{
_logger.Error <DocumentIndexer>(new FileNotFoundException($"Could not find file \"{physicalPath}\""));
return(Enumerable.Empty <SearchableDocument>());
}
var base64File = isFileExtensionAllowedForIndex ? Convert.ToBase64String(File.ReadAllBytes(physicalPath)) : string.Empty;
var result = new SearchableDocument
{
Id = content.Id,
Title = fileName,
Url = content.Url.ToLinkModel(),
Data = base64File,
Type = SearchableTypeEnum.Document.ToInt()
};
return(result.ToEnumerable());
}
return(Enumerable.Empty <SearchableDocument>());
}
void AddDocuments(IndexWriter writer, Node node)
{
string url = node.PublicUrl;
Stream file_stream = GetHelpStream(url.Substring(9));
if (file_stream == null) //Error
{
return;
}
XmlDocument xdoc = new XmlDocument();
xdoc.Load(new XmlTextReader(file_stream));
//Obtain the title
XmlNode nelem = xdoc.DocumentElement;
string title = nelem.Attributes["number"].Value + ": " + nelem.Attributes["title"].Value;
//Obtain the text
StringBuilder s = new StringBuilder();
GetTextNode(nelem, s);
string text = s.ToString();
//Obtain the examples
StringBuilder s2 = new StringBuilder();
GetExamples(nelem, s2);
string examples = s2.ToString();
//Write to the Lucene Index all the parts
SearchableDocument doc = new SearchableDocument();
doc.Title = title;
doc.HotText = title.Substring(title.IndexOf(':'));
doc.Url = url;
doc.Text = text;
doc.Examples = examples;
writer.AddDocument(doc.LuceneDoc);
if (node.IsLeaf)
{
return;
}
foreach (Node n in node.ChildNodes)
{
AddDocuments(writer, n);
}
}
public override void PopulateSearchableIndex(IndexWriter writer)
{
foreach (Node n in Tree.RootNode.ChildNodes)
{
XmlSerializer reader = new XmlSerializer(typeof(ErrorDocumentation));
ErrorDocumentation d = (ErrorDocumentation)reader.Deserialize(GetHelpStream(n.Element.Substring(6)));
SearchableDocument doc = new SearchableDocument();
doc.Title = d.ErrorName;
doc.Url = n.Element;
doc.Text = d.Details != null?d.Details.ToString() : string.Empty;
doc.Examples = d.Examples.Cast <string> ().Aggregate((e1, e2) => e1 + Environment.NewLine + e2);
doc.HotText = d.ErrorName;
writer.AddDocument(doc.LuceneDoc);
}
}
void AddDocuments (IndexWriter writer, Node node)
{
string url = node.PublicUrl;
Stream file_stream = GetHelpStream (url.Substring (9));
if (file_stream == null) //Error
return;
XmlDocument xdoc = new XmlDocument ();
xdoc.Load (new XmlTextReader (file_stream));
//Obtain the title
XmlNode nelem = xdoc.DocumentElement;
string title = nelem.Attributes["number"].Value + ": " + nelem.Attributes["title"].Value;
//Obtain the text
StringBuilder s = new StringBuilder ();
GetTextNode (nelem, s);
string text = s.ToString ();
//Obtain the examples
StringBuilder s2 = new StringBuilder ();
GetExamples (nelem, s2);
string examples = s2.ToString ();
//Write to the Lucene Index all the parts
SearchableDocument doc = new SearchableDocument ();
doc.Title = title;
doc.HotText = title.Substring (title.IndexOf (':'));
doc.Url = url;
doc.Text = text;
doc.Examples = examples;
writer.AddDocument (doc.LuceneDoc);
if (node.IsLeaf)
return;
foreach (Node n in node.ChildNodes)
AddDocuments (writer, n);
}
public override void PopulateSearchableIndex (IndexWriter writer)
{
foreach (Node n in Tree.RootNode.Nodes) {
XmlSerializer reader = new XmlSerializer (typeof (ErrorDocumentation));
ErrorDocumentation d = (ErrorDocumentation)reader.Deserialize (GetHelpStream (n.Element.Substring (6)));
SearchableDocument doc = new SearchableDocument ();
doc.Title = d.ErrorName;
doc.Url = n.Element;
doc.Text = d.Details != null ? d.Details.ToString () : string.Empty;
doc.Examples = d.Examples.Cast<string> ().Aggregate ((e1, e2) => e1 + Environment.NewLine + e2);
doc.HotText = d.ErrorName;
writer.AddDocument (doc.LuceneDoc);
}
}
public void Index(SearchableDocument content)
{
_elasticSearchRepository.EnsureMappingExist();
_elasticSearchRepository.Save(content);
}
public override void PopulateSearchableIndex (IndexWriter writer)
{
StringBuilder text = new StringBuilder ();
SearchableDocument searchDoc = new SearchableDocument ();
foreach (Node ns_node in Tree.RootNode.Nodes) {
foreach (Node type_node in ns_node.Nodes) {
string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' '));
string full = ns_node.Caption + "." + typename;
string url = type_node.PublicUrl;
string doc_tag = GetKindFromCaption (type_node.Caption);
string rest, hash;
var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash);
var xdoc = XDocument.Load (GetHelpStream (id));
if (xdoc == null)
continue;
if (string.IsNullOrEmpty (doc_tag))
continue;
// For classes, structures or interfaces add a doc for the overview and
// add a doc for every constructor, method, event, ...
// doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"
if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I') {
// Adds a doc for every overview of every type
SearchableDocument doc = searchDoc.Reset ();
doc.Title = type_node.Caption;
doc.HotText = typename;
doc.Url = url;
doc.FullTitle = full;
var node_sel = xdoc.Root.Element ("Docs");
text.Clear ();
GetTextFromNode (node_sel, text);
doc.Text = text.ToString ();
text.Clear ();
GetExamples (node_sel, text);
doc.Examples = text.ToString ();
writer.AddDocument (doc.LuceneDoc);
var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac"));
//Add docs for contructors, methods, etc.
foreach (Node c in type_node.Nodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators
if (c.Element == "*")
continue;
const float innerTypeBoost = 0.2f;
IEnumerable<Node> ncnodes = c.Nodes;
// The rationale is that we need to properly handle method overloads
// so for those method node which have children, flatten them
if (c.Caption == "Methods") {
ncnodes = ncnodes
.Where (n => n.Nodes == null || n.Nodes.Count == 0)
.Concat (ncnodes.Where (n => n.Nodes.Count > 0).SelectMany (n => n.Nodes));
} else if (c.Caption == "Operators") {
ncnodes = ncnodes
.Where (n => !n.Caption.EndsWith ("Conversion"))
.Concat (ncnodes.Where (n => n.Caption.EndsWith ("Conversion")).SelectMany (n => n.Nodes));
}
var prematchedMembers = xdoc.Root.Element ("Members").Elements ("Member").ToLookup (n => (string)n.Attribute ("MemberName"), n => n);
foreach (Node nc in ncnodes) {
XElement docsNode = null;
try {
docsNode = GetDocsFromCaption (xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O', prematchedMembers);
} catch {}
if (docsNode == null) {
Console.Error.WriteLine ("Problem: {0}", nc.PublicUrl);
continue;
}
SearchableDocument doc_nod = searchDoc.Reset ();
doc_nod.Title = LargeName (nc) + " " + EcmaDoc.EtcKindToCaption (c.Caption[0]);
doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption;
doc_nod.HotText = string.Empty;
/* Disable constructors hottext indexing as it's often "polluting" search queries
because it has the same hottext than standard types */
if (c.Caption != "Constructors") {
//dont add the parameters to the hottext
int ppos = nc.Caption.IndexOf ('(');
doc_nod.HotText = ppos != -1 ? nc.Caption.Substring (0, ppos) : nc.Caption;
}
var urlnc = nc.PublicUrl;
doc_nod.Url = urlnc;
text.Clear ();
GetTextFromNode (docsNode, text);
doc_nod.Text = text.ToString ();
text.Clear ();
GetExamples (docsNode, text);
doc_nod.Examples = text.ToString ();
Document lucene_doc = doc_nod.LuceneDoc;
lucene_doc.Boost = innerTypeBoost;
writer.AddDocument (lucene_doc);
// Objective-C binding specific parsing of [Export] attributes
if (exportParsable) {
try {
var exports = docsNode.Parent.Elements ("Attributes").Elements ("Attribute").Elements ("AttributeName")
.Select (a => (string)a).Where (txt => txt.Contains ("Foundation.Export"));
foreach (var exportNode in exports) {
var parts = exportNode.Split ('"');
if (parts.Length != 3) {
Console.WriteLine ("Export attribute not found or not usable in {0}", exportNode);
continue;
}
var export = parts[1];
var export_node = searchDoc.Reset ();
export_node.Title = export + " Export";
export_node.FullTitle = ns_node.Caption + '.' + typename + "::" + export;
export_node.Url = urlnc;
export_node.HotText = export;
export_node.Text = string.Empty;
export_node.Examples = string.Empty;
lucene_doc = export_node.LuceneDoc;
lucene_doc.Boost = innerTypeBoost;
writer.AddDocument (lucene_doc);
}
} catch (Exception e){
Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e);
}
}
}
}
// doc_tag == "Enumeration"
} else if (doc_tag[0] == 'E'){
var members = xdoc.Root.Element ("Members").Elements ("Member");
if (members == null)
continue;
text.Clear ();
foreach (var member_node in members) {
string enum_value = (string)member_node.Attribute ("MemberName");
text.Append (enum_value);
text.Append (" ");
GetTextFromNode (member_node.Element ("Docs"), text);
text.AppendLine ();
}
SearchableDocument doc = searchDoc.Reset ();
text.Clear ();
GetExamples (xdoc.Root.Element ("Docs"), text);
doc.Examples = text.ToString ();
doc.Title = type_node.Caption;
doc.HotText = (string)xdoc.Root.Attribute ("Name");
doc.FullTitle = full;
doc.Url = url;
doc.Text = text.ToString();
writer.AddDocument (doc.LuceneDoc);
// doc_tag == "Delegate"
} else if (doc_tag[0] == 'D'){
SearchableDocument doc = searchDoc.Reset ();
doc.Title = type_node.Caption;
doc.HotText = (string)xdoc.Root.Attribute ("Name");
doc.FullTitle = full;
doc.Url = url;
var node_sel = xdoc.Root.Element ("Docs");
text.Clear ();
GetTextFromNode (node_sel, text);
doc.Text = text.ToString();
text.Clear ();
GetExamples (node_sel, text);
doc.Examples = text.ToString();
writer.AddDocument (doc.LuceneDoc);
}
}
}
}
public override void PopulateSearchableIndex(IndexWriter writer)
{
StringBuilder text = new StringBuilder();
SearchableDocument searchDoc = new SearchableDocument();
foreach (Node ns_node in Tree.RootNode.ChildNodes)
{
foreach (Node type_node in ns_node.ChildNodes)
{
string typename = type_node.Caption.Substring(0, type_node.Caption.IndexOf(' '));
string full = ns_node.Caption + "." + typename;
string url = type_node.PublicUrl;
string doc_tag = GetKindFromCaption(type_node.Caption);
string rest, hash;
var id = GetInternalIdForInternalUrl(type_node.GetInternalUrl(), out hash);
var xdoc = XDocument.Load(GetHelpStream(id));
if (xdoc == null)
{
continue;
}
if (string.IsNullOrEmpty(doc_tag))
{
continue;
}
// For classes, structures or interfaces add a doc for the overview and
// add a doc for every constructor, method, event, ...
// doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface"
if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I')
{
// Adds a doc for every overview of every type
SearchableDocument doc = searchDoc.Reset();
doc.Title = type_node.Caption;
doc.HotText = typename;
doc.Url = url;
doc.FullTitle = full;
var node_sel = xdoc.Root.Element("Docs");
text.Clear();
GetTextFromNode(node_sel, text);
doc.Text = text.ToString();
text.Clear();
GetExamples(node_sel, text);
doc.Examples = text.ToString();
writer.AddDocument(doc.LuceneDoc);
var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith("MonoTouch") || ns_node.Caption.StartsWith("MonoMac"));
//Add docs for contructors, methods, etc.
foreach (Node c in type_node.ChildNodes) // c = Constructors || Fields || Events || Properties || Methods || Operators
{
if (c.Element == "*")
{
continue;
}
const float innerTypeBoost = 0.2f;
IEnumerable <Node> ncnodes = c.ChildNodes;
// The rationale is that we need to properly handle method overloads
// so for those method node which have children, flatten them
if (c.Caption == "Methods")
{
ncnodes = ncnodes
.Where(n => n.ChildNodes == null || n.ChildNodes.Count == 0)
.Concat(ncnodes.Where(n => n.ChildNodes.Count > 0).SelectMany(n => n.ChildNodes));
}
else if (c.Caption == "Operators")
{
ncnodes = ncnodes
.Where(n => !n.Caption.EndsWith("Conversion"))
.Concat(ncnodes.Where(n => n.Caption.EndsWith("Conversion")).SelectMany(n => n.ChildNodes));
}
var prematchedMembers = xdoc.Root.Element("Members").Elements("Member").ToLookup(n => (string)n.Attribute("MemberName"), n => n);
foreach (Node nc in ncnodes)
{
XElement docsNode = null;
try {
docsNode = GetDocsFromCaption(xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O', prematchedMembers);
} catch {}
if (docsNode == null)
{
Console.Error.WriteLine("Problem: {0}", nc.PublicUrl);
continue;
}
SearchableDocument doc_nod = searchDoc.Reset();
doc_nod.Title = LargeName(nc) + " " + EcmaDoc.EtcKindToCaption(c.Caption[0]);
doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption;
doc_nod.HotText = string.Empty;
/* Disable constructors hottext indexing as it's often "polluting" search queries
* because it has the same hottext than standard types */
if (c.Caption != "Constructors")
{
//dont add the parameters to the hottext
int ppos = nc.Caption.IndexOf('(');
doc_nod.HotText = ppos != -1 ? nc.Caption.Substring(0, ppos) : nc.Caption;
}
var urlnc = nc.PublicUrl;
doc_nod.Url = urlnc;
text.Clear();
GetTextFromNode(docsNode, text);
doc_nod.Text = text.ToString();
text.Clear();
GetExamples(docsNode, text);
doc_nod.Examples = text.ToString();
Document lucene_doc = doc_nod.LuceneDoc;
lucene_doc.Boost = innerTypeBoost;
writer.AddDocument(lucene_doc);
// Objective-C binding specific parsing of [Export] attributes
if (exportParsable)
{
try {
var exports = docsNode.Parent.Elements("Attributes").Elements("Attribute").Elements("AttributeName")
.Select(a => (string)a).Where(txt => txt.Contains("Foundation.Export"));
foreach (var exportNode in exports)
{
var parts = exportNode.Split('"');
if (parts.Length != 3)
{
Console.WriteLine("Export attribute not found or not usable in {0}", exportNode);
continue;
}
var export = parts[1];
var export_node = searchDoc.Reset();
export_node.Title = export + " Export";
export_node.FullTitle = ns_node.Caption + '.' + typename + "::" + export;
export_node.Url = urlnc;
export_node.HotText = export;
export_node.Text = string.Empty;
export_node.Examples = string.Empty;
lucene_doc = export_node.LuceneDoc;
lucene_doc.Boost = innerTypeBoost;
writer.AddDocument(lucene_doc);
}
} catch (Exception e) {
Console.WriteLine("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e);
}
}
}
}
// doc_tag == "Enumeration"
}
else if (doc_tag[0] == 'E')
{
var members = xdoc.Root.Element("Members").Elements("Member");
if (members == null)
{
continue;
}
text.Clear();
foreach (var member_node in members)
{
string enum_value = (string)member_node.Attribute("MemberName");
text.Append(enum_value);
text.Append(" ");
GetTextFromNode(member_node.Element("Docs"), text);
text.AppendLine();
}
SearchableDocument doc = searchDoc.Reset();
text.Clear();
GetExamples(xdoc.Root.Element("Docs"), text);
doc.Examples = text.ToString();
doc.Title = type_node.Caption;
doc.HotText = (string)xdoc.Root.Attribute("Name");
doc.FullTitle = full;
doc.Url = url;
doc.Text = text.ToString();
writer.AddDocument(doc.LuceneDoc);
// doc_tag == "Delegate"
}
else if (doc_tag[0] == 'D')
{
SearchableDocument doc = searchDoc.Reset();
doc.Title = type_node.Caption;
doc.HotText = (string)xdoc.Root.Attribute("Name");
doc.FullTitle = full;
doc.Url = url;
var node_sel = xdoc.Root.Element("Docs");
text.Clear();
GetTextFromNode(node_sel, text);
doc.Text = text.ToString();
text.Clear();
GetExamples(node_sel, text);
doc.Examples = text.ToString();
writer.AddDocument(doc.LuceneDoc);
}
}
}
}
