public async Task <IActionResult> Article(string fileName)
{
using (var fileStream = new FileStream(HeaderParser.FindAddress(fileName),
FileMode.Open, FileAccess.Read))
{
using (StreamReader reader = new StreamReader(fileStream))
{
ArticleParser.CreateNew();
for (var i = 0; i < 9; i++)
{
var line = await reader.ReadLineAsync();
ArticleParser.ParsLine(line);
}
var markdown = await reader.ReadToEndAsync();
var pipeline = new MarkdownPipelineBuilder().UseAdvancedExtensions().Build();
var model = ArticleParser.GetPostDetaile();
model.Html = Markdown.ToHtml(markdown, pipeline);
ViewBag.Summary = model.Summary;
ViewBag.Keywords = model.Keywords;
ViewBag.Author = model.Author;
return(View(model: model));
}
}
}
public virtual ActionResult Generate(string key)
{
var parser =
new ArticleParser(
new ParserOptions
{
PathToDocumentationDirectory = GetArticleDirectory(),
RootUrl = string.Format("{0}://{1}{2}", Request.Url.Scheme, Request.Url.Authority, Url.Content("~")),
ImagesUrl = DocsController.GetImagesUrl(DocumentStore)
}, new NoOpGitFileInformationProvider());
DocumentStore
.DatabaseCommands
.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery { Query = "Tag:ArticlePages" })
.WaitForCompletion();
foreach (var page in parser.Parse())
{
DocumentSession.Store(page);
Parallel.ForEach(
page.Images,
image =>
{
if (System.IO.File.Exists(image.ImagePath) == false)
return;
using (var file = System.IO.File.OpenRead(image.ImagePath))
{
DocumentStore.DatabaseCommands.PutAttachment(image.ImageKey, null, file, new RavenJObject());
}
});
}
foreach (var toc in parser.GenerateTableOfContents())
{
DocumentSession.Store(toc);
}
DocumentSession.SaveChanges();
while (true)
{
var stats = DocumentStore.DatabaseCommands.GetStatistics();
if (stats.StaleIndexes.Any() == false)
break;
Thread.Sleep(500);
}
if (string.IsNullOrEmpty(key))
return RedirectToAction(MVC.Articles.ActionNames.Articles);
return RedirectToAction(MVC.Articles.ActionNames.Articles, MVC.Articles.Name, new { key = key });
}
private void AppendWord(FlowDocument document, WordInfo word)
{
ArticleParser parser = new ArticleParser();
WordArticle article = parser.Parse(word.Word.Name, word.Word.Description);
Paragraph paragraph = FlowDocumentStyles.CreateParagraph();
document.Blocks.Add(paragraph);
IEnumerable <WordFormGroup> formGroups = article.FormGroups.Where(fg => fg.TranslationGroups.SelectMany(tg => tg.Translations).Contains(Translation));
foreach (WordFormGroup formGroup in formGroups)
{
bool firstForm = true;
foreach (string form in formGroup.Forms)
{
Hyperlink link = new Hyperlink();
link.Command = Commands.NavigateTo;
link.CommandParameter = NavigationTarget.ToWord(word.Word.Name);
paragraph.Inlines.Add(link);
Run run = new Run(form);
link.Inlines.Add(run);
if (firstForm)
{
FlowDocumentStyles.FormatWord(run);
firstForm = false;
}
else
{
FlowDocumentStyles.FormatWordForm(run);
}
paragraph.Inlines.Add(new LineBreak());
}
foreach (WordTranslationGroup translationGroup in formGroup.TranslationGroups.Where(tg => tg.Translations.Contains(Translation)))
{
foreach (string translation in translationGroup.Translations)
{
Run run = new Run("\u2022 " + translation);
paragraph.Inlines.Add(run);
FlowDocumentStyles.FormatTranslation(run);
paragraph.Inlines.Add(new LineBreak());
}
foreach (string example in translationGroup.Examples)
{
Run run = new Run("Example: " + example);
paragraph.Inlines.Add(run);
FlowDocumentStyles.FormatExample(run);
paragraph.Inlines.Add(new LineBreak());
}
}
}
}
public void Test()
{
ArticleParser parser = new ArticleParser();
WordArticle article = parser.Parse("a", "b");
Assert.That(article.FormGroups.Count, Is.EqualTo(1));
Assert.That(article.FormGroups[0].Forms, Is.EquivalentTo(new string[] { "a" }));
Assert.That(article.FormGroups[0].TranslationGroups.Count, Is.EqualTo(1));
Assert.That(article.FormGroups[0].TranslationGroups[0].Translations, Is.EquivalentTo(new string[] { "b" }));
Assert.That(article.Format(), Is.EqualTo("#a\nb\n"));
}
private List <ArticlePage> GetParsedPages(Options options)
{
var parser =
new ArticleParser(
new ParserOptions
{
PathToDocumentationDirectory = options.ArticleDirectory,
RootUrl = DummyRootUrl,
ImageUrlGenerator = ImageUrlGenerator
}, new NoOpGitFileInformationProvider());
var parserOutput = parser.Parse();
return(parserOutput.Pages.ToList());
}
public virtual ActionResult Generate(string key)
{
var parser =
new ArticleParser(
new ParserOptions
{
PathToDocumentationDirectory = GetArticleDirectory(),
RootUrl = string.Format("{0}://{1}{2}", Request.Url.Scheme, Request.Url.Authority, Url.Content("~")),
ImagesUrl = DocsController.GetImagesUrl(DocumentStore)
}, new NoOpGitFileInformationProvider());
DocumentStore
.DatabaseCommands
.DeleteByIndex("Raven/DocumentsByEntityName", new IndexQuery {
Query = "Tag:ArticlePages"
})
.WaitForCompletion();
foreach (var page in parser.Parse())
{
DocumentSession.Store(page);
Parallel.ForEach(
page.Images,
image =>
{
if (System.IO.File.Exists(image.ImagePath) == false)
{
return;
}
using (var file = System.IO.File.OpenRead(image.ImagePath))
{
DocumentStore.DatabaseCommands.PutAttachment(image.ImageKey, null, file, new RavenJObject());
}
});
}
foreach (var toc in parser.GenerateTableOfContents())
{
DocumentSession.Store(toc);
}
DocumentSession.SaveChanges();
while (true)
{
var stats = DocumentStore.DatabaseCommands.GetStatistics();
if (stats.StaleIndexes.Any() == false)
{
break;
}
Thread.Sleep(500);
}
if (string.IsNullOrEmpty(key))
{
return(RedirectToAction(MVC.Articles.ActionNames.Articles));
}
return(RedirectToAction(MVC.Articles.ActionNames.Articles, MVC.Articles.Name, new { key = key }));
}
public virtual ActionResult Generate(string key)
{
var parser =
new ArticleParser(
new ParserOptions
{
PathToDocumentationDirectory = GetArticleDirectory(),
RootUrl = string.Format("{0}://{1}{2}", Request.Url.Scheme, Request.Url.Authority, Url.Content("~")),
ImageUrlGenerator = DocsController.GetImageUrlGenerator(HttpContext, DocumentStore, ArticleType.Articles)
}, new NoOpGitFileInformationProvider());
var query = DocumentSession
.Advanced
.DocumentQuery <ArticlePage>()
.GetIndexQuery();
DocumentStore
.Operations
.Send(new DeleteByQueryOperation(query))
.WaitForCompletion();
DocumentSession.SaveChanges();
var toDispose = new List <IDisposable>();
var parserOutput = parser.Parse();
try
{
foreach (var page in parserOutput.Pages)
{
DocumentSession.Store(page);
foreach (var image in page.Images)
{
var fileInfo = new FileInfo(image.ImagePath);
if (fileInfo.Exists == false)
{
continue;
}
var file = fileInfo.OpenRead();
toDispose.Add(file);
var documentId = page.Id;
var fileName = Path.GetFileName(image.ImagePath);
DocumentSession.Advanced.Attachments.Store(documentId, fileName, file, AttachmentsController.FileExtensionToContentTypeMapping[fileInfo.Extension]);
}
}
foreach (var toc in parserOutput.TableOfContents)
{
DocumentSession.Store(toc);
}
DocumentSession.SaveChanges();
}
finally
{
foreach (var disposable in toDispose)
{
disposable?.Dispose();
}
}
while (true)
{
var stats = DocumentStore.Maintenance.Send(new GetStatisticsOperation());
if (stats.StaleIndexes.Any() == false)
{
break;
}
Thread.Sleep(500);
}
if (string.IsNullOrEmpty(key))
{
return(RedirectToAction(MVC.Articles.ActionNames.Articles));
}
return(RedirectToAction(MVC.Articles.ActionNames.Articles, MVC.Articles.Name, new { key = key }));
}
private void ParseArticles()
{
var articleParser = new ArticleParser(_coreConfigSection.InputPath, _repository);
_repository.Articles = articleParser.ParseArticles();
}
public void ParseSgg()
{
var path = @"C:\Users\Mikhail\source\repos\SolomonSrc\SolomonTests\one_appeals_stavropol_with_comments.json";
ArticleParser.ParseArticles(path);
}
public ArticleParserTest()
{
_articleParser = new ArticleParser(new ParserFactory());
}
private Dictionary <string, WordStatsInfo> parseArticles(IEnumerable <ArticleHeader> headers)
{
// Check for stop
if (_stopRequested)
{
_isDone = true;
return(null);
}
doReport(MessageKind.Info, "Downloading articles...");
// Push the articles into the queue
Queue <ArticleHeader> tasks = new Queue <ArticleHeader>(headers);
// Create the working threads and pass them all
// the same collection of article headers to work on
Task[] workers = new Task[Constants.MAX_THREADS];
// The logger
Action <MessageKind, string> reportAction = (k, m) => doReport(k, m);
// The resulting global statistics
var globalWordStats = new Dictionary <string, WordStatsInfo>();
// Initiate the required number of workers and
// make them toll
for (int s = 0; s < Constants.MAX_THREADS; s++)
{
// Create the context for this worker
ArticleParserContext context = new ArticleParserContext()
{
id = s + 1, // The worker's own index
shouldStop = stopEvent.Token, // So that a worker knows when to stop
articles = tasks, // Bunch of tasks they work on
reportAction = reportAction, // Let him report some messages
globalStats = globalWordStats // Where the workers puts statistics
};
// Start the worker and pass it its context
workers[s] = Task.Factory.StartNew((state) =>
{
// Start the work
ArticleParser.processArticles(state);
},
state: context);
}
// Wait for the threads to complete
Task.WaitAll(workers);
doReport(MessageKind.Result, "All workers finished");
doReport(MessageKind.Info, "Processing the stats...");
// Sort by frequencies
var wordFrequencies = globalWordStats.OrderBy(c => c.Value.count)
.ToList();
//var wordFrequencies = globalWordStats.OrderByDescending(c => c.Value.count)
// .Take(Constants.TOP_WORDS_LIMIT) // Sorry was too many of words otherwise
// .ToList();
// Print the stats
doReport(MessageKind.Result, "Count | Word");
doReport(MessageKind.Result, "-------|-----------------------------------------");
foreach (var word in wordFrequencies)
{
doReport(MessageKind.Result, String.Format("{0} | {1}",
word.Value.count.ToString().PadLeft(6),
System.Threading.Thread.CurrentThread.CurrentCulture.TextInfo.ToTitleCase(word.Key)
));
}
return(globalWordStats);
}
