private IEnumerable <SearchableDocument> GetSearchableDocument(IPublishedContent content) { var fileName = Path.GetFileName(content.Url); var extension = Path.GetExtension(fileName)?.Trim('.'); bool isFileExtensionAllowedForIndex = _settings.IndexingDocumentTypesKey.Contains(extension, StringComparison.OrdinalIgnoreCase); if (!content.Url.IsNullOrEmpty()) { var physicalPath = HostingEnvironment.MapPath(content.Url); if (!File.Exists(physicalPath)) { _logger.Error <DocumentIndexer>(new FileNotFoundException($"Could not find file \"{physicalPath}\"")); return(Enumerable.Empty <SearchableDocument>()); } var base64File = isFileExtensionAllowedForIndex ? Convert.ToBase64String(File.ReadAllBytes(physicalPath)) : string.Empty; var result = new SearchableDocument { Id = content.Id, Title = fileName, Url = content.Url.ToLinkModel(), Data = base64File, Type = SearchableTypeEnum.Document.ToInt() }; return(result.ToEnumerable()); } return(Enumerable.Empty <SearchableDocument>()); }
void AddDocuments(IndexWriter writer, Node node) { string url = node.PublicUrl; Stream file_stream = GetHelpStream(url.Substring(9)); if (file_stream == null) //Error { return; } XmlDocument xdoc = new XmlDocument(); xdoc.Load(new XmlTextReader(file_stream)); //Obtain the title XmlNode nelem = xdoc.DocumentElement; string title = nelem.Attributes["number"].Value + ": " + nelem.Attributes["title"].Value; //Obtain the text StringBuilder s = new StringBuilder(); GetTextNode(nelem, s); string text = s.ToString(); //Obtain the examples StringBuilder s2 = new StringBuilder(); GetExamples(nelem, s2); string examples = s2.ToString(); //Write to the Lucene Index all the parts SearchableDocument doc = new SearchableDocument(); doc.Title = title; doc.HotText = title.Substring(title.IndexOf(':')); doc.Url = url; doc.Text = text; doc.Examples = examples; writer.AddDocument(doc.LuceneDoc); if (node.IsLeaf) { return; } foreach (Node n in node.ChildNodes) { AddDocuments(writer, n); } }
public override void PopulateSearchableIndex(IndexWriter writer) { foreach (Node n in Tree.RootNode.ChildNodes) { XmlSerializer reader = new XmlSerializer(typeof(ErrorDocumentation)); ErrorDocumentation d = (ErrorDocumentation)reader.Deserialize(GetHelpStream(n.Element.Substring(6))); SearchableDocument doc = new SearchableDocument(); doc.Title = d.ErrorName; doc.Url = n.Element; doc.Text = d.Details != null?d.Details.ToString() : string.Empty; doc.Examples = d.Examples.Cast <string> ().Aggregate((e1, e2) => e1 + Environment.NewLine + e2); doc.HotText = d.ErrorName; writer.AddDocument(doc.LuceneDoc); } }
void AddDocuments (IndexWriter writer, Node node) { string url = node.PublicUrl; Stream file_stream = GetHelpStream (url.Substring (9)); if (file_stream == null) //Error return; XmlDocument xdoc = new XmlDocument (); xdoc.Load (new XmlTextReader (file_stream)); //Obtain the title XmlNode nelem = xdoc.DocumentElement; string title = nelem.Attributes["number"].Value + ": " + nelem.Attributes["title"].Value; //Obtain the text StringBuilder s = new StringBuilder (); GetTextNode (nelem, s); string text = s.ToString (); //Obtain the examples StringBuilder s2 = new StringBuilder (); GetExamples (nelem, s2); string examples = s2.ToString (); //Write to the Lucene Index all the parts SearchableDocument doc = new SearchableDocument (); doc.Title = title; doc.HotText = title.Substring (title.IndexOf (':')); doc.Url = url; doc.Text = text; doc.Examples = examples; writer.AddDocument (doc.LuceneDoc); if (node.IsLeaf) return; foreach (Node n in node.ChildNodes) AddDocuments (writer, n); }
public override void PopulateSearchableIndex (IndexWriter writer) { foreach (Node n in Tree.RootNode.Nodes) { XmlSerializer reader = new XmlSerializer (typeof (ErrorDocumentation)); ErrorDocumentation d = (ErrorDocumentation)reader.Deserialize (GetHelpStream (n.Element.Substring (6))); SearchableDocument doc = new SearchableDocument (); doc.Title = d.ErrorName; doc.Url = n.Element; doc.Text = d.Details != null ? d.Details.ToString () : string.Empty; doc.Examples = d.Examples.Cast<string> ().Aggregate ((e1, e2) => e1 + Environment.NewLine + e2); doc.HotText = d.ErrorName; writer.AddDocument (doc.LuceneDoc); } }
public void Index(SearchableDocument content) { _elasticSearchRepository.EnsureMappingExist(); _elasticSearchRepository.Save(content); }
public override void PopulateSearchableIndex (IndexWriter writer) { StringBuilder text = new StringBuilder (); SearchableDocument searchDoc = new SearchableDocument (); foreach (Node ns_node in Tree.RootNode.Nodes) { foreach (Node type_node in ns_node.Nodes) { string typename = type_node.Caption.Substring (0, type_node.Caption.IndexOf (' ')); string full = ns_node.Caption + "." + typename; string url = type_node.PublicUrl; string doc_tag = GetKindFromCaption (type_node.Caption); string rest, hash; var id = GetInternalIdForInternalUrl (type_node.GetInternalUrl (), out hash); var xdoc = XDocument.Load (GetHelpStream (id)); if (xdoc == null) continue; if (string.IsNullOrEmpty (doc_tag)) continue; // For classes, structures or interfaces add a doc for the overview and // add a doc for every constructor, method, event, ... // doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface" if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I') { // Adds a doc for every overview of every type SearchableDocument doc = searchDoc.Reset (); doc.Title = type_node.Caption; doc.HotText = typename; doc.Url = url; doc.FullTitle = full; var node_sel = xdoc.Root.Element ("Docs"); text.Clear (); GetTextFromNode (node_sel, text); doc.Text = text.ToString (); text.Clear (); GetExamples (node_sel, text); doc.Examples = text.ToString (); writer.AddDocument (doc.LuceneDoc); var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith ("MonoTouch") || ns_node.Caption.StartsWith ("MonoMac")); //Add docs for contructors, methods, etc. foreach (Node c in type_node.Nodes) { // c = Constructors || Fields || Events || Properties || Methods || Operators if (c.Element == "*") continue; const float innerTypeBoost = 0.2f; IEnumerable<Node> ncnodes = c.Nodes; // The rationale is that we need to properly handle method overloads // so for those method node which have children, flatten them if (c.Caption == "Methods") { ncnodes = ncnodes .Where (n => n.Nodes == null || n.Nodes.Count == 0) .Concat (ncnodes.Where (n => n.Nodes.Count > 0).SelectMany (n => n.Nodes)); } else if (c.Caption == "Operators") { ncnodes = ncnodes .Where (n => !n.Caption.EndsWith ("Conversion")) .Concat (ncnodes.Where (n => n.Caption.EndsWith ("Conversion")).SelectMany (n => n.Nodes)); } var prematchedMembers = xdoc.Root.Element ("Members").Elements ("Member").ToLookup (n => (string)n.Attribute ("MemberName"), n => n); foreach (Node nc in ncnodes) { XElement docsNode = null; try { docsNode = GetDocsFromCaption (xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O', prematchedMembers); } catch {} if (docsNode == null) { Console.Error.WriteLine ("Problem: {0}", nc.PublicUrl); continue; } SearchableDocument doc_nod = searchDoc.Reset (); doc_nod.Title = LargeName (nc) + " " + EcmaDoc.EtcKindToCaption (c.Caption[0]); doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption; doc_nod.HotText = string.Empty; /* Disable constructors hottext indexing as it's often "polluting" search queries because it has the same hottext than standard types */ if (c.Caption != "Constructors") { //dont add the parameters to the hottext int ppos = nc.Caption.IndexOf ('('); doc_nod.HotText = ppos != -1 ? nc.Caption.Substring (0, ppos) : nc.Caption; } var urlnc = nc.PublicUrl; doc_nod.Url = urlnc; text.Clear (); GetTextFromNode (docsNode, text); doc_nod.Text = text.ToString (); text.Clear (); GetExamples (docsNode, text); doc_nod.Examples = text.ToString (); Document lucene_doc = doc_nod.LuceneDoc; lucene_doc.Boost = innerTypeBoost; writer.AddDocument (lucene_doc); // Objective-C binding specific parsing of [Export] attributes if (exportParsable) { try { var exports = docsNode.Parent.Elements ("Attributes").Elements ("Attribute").Elements ("AttributeName") .Select (a => (string)a).Where (txt => txt.Contains ("Foundation.Export")); foreach (var exportNode in exports) { var parts = exportNode.Split ('"'); if (parts.Length != 3) { Console.WriteLine ("Export attribute not found or not usable in {0}", exportNode); continue; } var export = parts[1]; var export_node = searchDoc.Reset (); export_node.Title = export + " Export"; export_node.FullTitle = ns_node.Caption + '.' + typename + "::" + export; export_node.Url = urlnc; export_node.HotText = export; export_node.Text = string.Empty; export_node.Examples = string.Empty; lucene_doc = export_node.LuceneDoc; lucene_doc.Boost = innerTypeBoost; writer.AddDocument (lucene_doc); } } catch (Exception e){ Console.WriteLine ("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e); } } } } // doc_tag == "Enumeration" } else if (doc_tag[0] == 'E'){ var members = xdoc.Root.Element ("Members").Elements ("Member"); if (members == null) continue; text.Clear (); foreach (var member_node in members) { string enum_value = (string)member_node.Attribute ("MemberName"); text.Append (enum_value); text.Append (" "); GetTextFromNode (member_node.Element ("Docs"), text); text.AppendLine (); } SearchableDocument doc = searchDoc.Reset (); text.Clear (); GetExamples (xdoc.Root.Element ("Docs"), text); doc.Examples = text.ToString (); doc.Title = type_node.Caption; doc.HotText = (string)xdoc.Root.Attribute ("Name"); doc.FullTitle = full; doc.Url = url; doc.Text = text.ToString(); writer.AddDocument (doc.LuceneDoc); // doc_tag == "Delegate" } else if (doc_tag[0] == 'D'){ SearchableDocument doc = searchDoc.Reset (); doc.Title = type_node.Caption; doc.HotText = (string)xdoc.Root.Attribute ("Name"); doc.FullTitle = full; doc.Url = url; var node_sel = xdoc.Root.Element ("Docs"); text.Clear (); GetTextFromNode (node_sel, text); doc.Text = text.ToString(); text.Clear (); GetExamples (node_sel, text); doc.Examples = text.ToString(); writer.AddDocument (doc.LuceneDoc); } } } }
public override void PopulateSearchableIndex(IndexWriter writer) { StringBuilder text = new StringBuilder(); SearchableDocument searchDoc = new SearchableDocument(); foreach (Node ns_node in Tree.RootNode.ChildNodes) { foreach (Node type_node in ns_node.ChildNodes) { string typename = type_node.Caption.Substring(0, type_node.Caption.IndexOf(' ')); string full = ns_node.Caption + "." + typename; string url = type_node.PublicUrl; string doc_tag = GetKindFromCaption(type_node.Caption); string rest, hash; var id = GetInternalIdForInternalUrl(type_node.GetInternalUrl(), out hash); var xdoc = XDocument.Load(GetHelpStream(id)); if (xdoc == null) { continue; } if (string.IsNullOrEmpty(doc_tag)) { continue; } // For classes, structures or interfaces add a doc for the overview and // add a doc for every constructor, method, event, ... // doc_tag == "Class" || doc_tag == "Structure" || doc_tag == "Interface" if (doc_tag[0] == 'C' || doc_tag[0] == 'S' || doc_tag[0] == 'I') { // Adds a doc for every overview of every type SearchableDocument doc = searchDoc.Reset(); doc.Title = type_node.Caption; doc.HotText = typename; doc.Url = url; doc.FullTitle = full; var node_sel = xdoc.Root.Element("Docs"); text.Clear(); GetTextFromNode(node_sel, text); doc.Text = text.ToString(); text.Clear(); GetExamples(node_sel, text); doc.Examples = text.ToString(); writer.AddDocument(doc.LuceneDoc); var exportParsable = doc_tag[0] == 'C' && (ns_node.Caption.StartsWith("MonoTouch") || ns_node.Caption.StartsWith("MonoMac")); //Add docs for contructors, methods, etc. foreach (Node c in type_node.ChildNodes) // c = Constructors || Fields || Events || Properties || Methods || Operators { if (c.Element == "*") { continue; } const float innerTypeBoost = 0.2f; IEnumerable <Node> ncnodes = c.ChildNodes; // The rationale is that we need to properly handle method overloads // so for those method node which have children, flatten them if (c.Caption == "Methods") { ncnodes = ncnodes .Where(n => n.ChildNodes == null || n.ChildNodes.Count == 0) .Concat(ncnodes.Where(n => n.ChildNodes.Count > 0).SelectMany(n => n.ChildNodes)); } else if (c.Caption == "Operators") { ncnodes = ncnodes .Where(n => !n.Caption.EndsWith("Conversion")) .Concat(ncnodes.Where(n => n.Caption.EndsWith("Conversion")).SelectMany(n => n.ChildNodes)); } var prematchedMembers = xdoc.Root.Element("Members").Elements("Member").ToLookup(n => (string)n.Attribute("MemberName"), n => n); foreach (Node nc in ncnodes) { XElement docsNode = null; try { docsNode = GetDocsFromCaption(xdoc, c.Caption[0] == 'C' ? ".ctor" : nc.Caption, c.Caption[0] == 'O', prematchedMembers); } catch {} if (docsNode == null) { Console.Error.WriteLine("Problem: {0}", nc.PublicUrl); continue; } SearchableDocument doc_nod = searchDoc.Reset(); doc_nod.Title = LargeName(nc) + " " + EcmaDoc.EtcKindToCaption(c.Caption[0]); doc_nod.FullTitle = ns_node.Caption + '.' + typename + "::" + nc.Caption; doc_nod.HotText = string.Empty; /* Disable constructors hottext indexing as it's often "polluting" search queries * because it has the same hottext than standard types */ if (c.Caption != "Constructors") { //dont add the parameters to the hottext int ppos = nc.Caption.IndexOf('('); doc_nod.HotText = ppos != -1 ? nc.Caption.Substring(0, ppos) : nc.Caption; } var urlnc = nc.PublicUrl; doc_nod.Url = urlnc; text.Clear(); GetTextFromNode(docsNode, text); doc_nod.Text = text.ToString(); text.Clear(); GetExamples(docsNode, text); doc_nod.Examples = text.ToString(); Document lucene_doc = doc_nod.LuceneDoc; lucene_doc.Boost = innerTypeBoost; writer.AddDocument(lucene_doc); // Objective-C binding specific parsing of [Export] attributes if (exportParsable) { try { var exports = docsNode.Parent.Elements("Attributes").Elements("Attribute").Elements("AttributeName") .Select(a => (string)a).Where(txt => txt.Contains("Foundation.Export")); foreach (var exportNode in exports) { var parts = exportNode.Split('"'); if (parts.Length != 3) { Console.WriteLine("Export attribute not found or not usable in {0}", exportNode); continue; } var export = parts[1]; var export_node = searchDoc.Reset(); export_node.Title = export + " Export"; export_node.FullTitle = ns_node.Caption + '.' + typename + "::" + export; export_node.Url = urlnc; export_node.HotText = export; export_node.Text = string.Empty; export_node.Examples = string.Empty; lucene_doc = export_node.LuceneDoc; lucene_doc.Boost = innerTypeBoost; writer.AddDocument(lucene_doc); } } catch (Exception e) { Console.WriteLine("Problem processing {0} for MonoTouch/MonoMac exports\n\n{0}", e); } } } } // doc_tag == "Enumeration" } else if (doc_tag[0] == 'E') { var members = xdoc.Root.Element("Members").Elements("Member"); if (members == null) { continue; } text.Clear(); foreach (var member_node in members) { string enum_value = (string)member_node.Attribute("MemberName"); text.Append(enum_value); text.Append(" "); GetTextFromNode(member_node.Element("Docs"), text); text.AppendLine(); } SearchableDocument doc = searchDoc.Reset(); text.Clear(); GetExamples(xdoc.Root.Element("Docs"), text); doc.Examples = text.ToString(); doc.Title = type_node.Caption; doc.HotText = (string)xdoc.Root.Attribute("Name"); doc.FullTitle = full; doc.Url = url; doc.Text = text.ToString(); writer.AddDocument(doc.LuceneDoc); // doc_tag == "Delegate" } else if (doc_tag[0] == 'D') { SearchableDocument doc = searchDoc.Reset(); doc.Title = type_node.Caption; doc.HotText = (string)xdoc.Root.Attribute("Name"); doc.FullTitle = full; doc.Url = url; var node_sel = xdoc.Root.Element("Docs"); text.Clear(); GetTextFromNode(node_sel, text); doc.Text = text.ToString(); text.Clear(); GetExamples(node_sel, text); doc.Examples = text.ToString(); writer.AddDocument(doc.LuceneDoc); } } } }