public List<ProgramElement> Parse(string filename)
        {
            var programElements = new List<ProgramElement>();

            XmlTextReader reader = new XmlTextReader(filename);

            while (reader.Read())
            {
                string text = String.Empty;

                if (reader.NodeType == XmlNodeType.Text)
                {
                    text = reader.Value;
                }
                else if (reader.NodeType == XmlNodeType.Element)
                {
                    while (reader.MoveToNextAttribute())
                    {
                        text += reader.Value + " ";
                    }
                }

                if (!String.IsNullOrWhiteSpace(text))
                {
                    var cleanedText = text.TrimStart(' ', '\n', '\r', '\t');
                    cleanedText = cleanedText.TrimEnd(' ', '\n', '\r', '\t');
                    var linenum = reader.LineNumber;
                    var snippet = SrcMLParsingUtils.RetrieveSource(cleanedText);
                    var pe = new TextFileElement(filename, snippet, cleanedText);
                    programElements.Add(pe);
                }
            }

            return programElements;
        }
        public List<ProgramElement> Parse(string filename)
        {
            var list = new List<ProgramElement>();
            var termSeparators = new char[] {' ', '\n', '\t', '\r'};
            try
            {
                int charactersInCurrentChunk = 0;
                int currentLineNumber = 1;
                int startingLineNumber = 1;
                using (var sr = new StreamReader(filename))
                {
                    StringBuilder fileText = new StringBuilder();
                    //fileText.Append(Environment.NewLine); //in order to start line numbers at 1 instead of 0
                    string line = String.Empty;
                    while ((line = sr.ReadLine()) != null)
                    {
                        fileText.Append(line + Environment.NewLine);
                        charactersInCurrentChunk += line.Length;
                        currentLineNumber++;

                        if (charactersInCurrentChunk >= MaxLengthOfTermInLucene)
                        {
                            if (!String.IsNullOrWhiteSpace(fileText.ToString()))
                            {
                                var element = new TextFileElement(startingLineNumber, 0, filename, fileText.ToString(), fileText.ToString());
                                list.Add(element);
                                startingLineNumber = currentLineNumber;
                                charactersInCurrentChunk = 0;
                                fileText = new StringBuilder();
                            }
                        }
                    }

                    var fileString = fileText.ToString();
                    if (!String.IsNullOrWhiteSpace(fileString))
                    {
                        var element = new TextFileElement(startingLineNumber, 0, filename, fileText.ToString(), fileText.ToString());
                        list.Add(element);
                    }
                }
            }
            catch (Exception e)
            {
                LogEvents.ParserGenericFileError(this, filename);
            }

            return list;
        }
 private static IEnumerable<string> ExtractTextLineElement(TextFileElement element)
 {
     return GetDefaultLetterWords(element.Body);
 }