public ActionResult Index()
{
var context = new WikiContext();
var pages = context.Pages.OrderByDescending(o => o.Id).ToList();
return(View(pages));
}
public AccountsController(UserManager <ApplicationUser> userManager, IMapper mapper, WikiContext appDbContext, SignInManager <ApplicationUser> signInManager)
{
_userManager = userManager;
_mapper = mapper;
_appDbContext = appDbContext;
this.signInManager = signInManager;
}
public ActionResult Index()
{
using (var context = new WikiContext())
{
return(View("Index", context.WikiReferences.ToList()));
}
}
//creating this by hand w/o dbservice on purpose
public static void CreateFourPagesWithTags(WikiContext context)
{
CreatePage("title", "eins zwei drei vier fuenf sechs sieben acht neun zehn elf zwoelf dreizehn test vierzehn fuenfzehn sechzehn siebzehn achtzehn neunzehn zwanzig", context);
CreatePage("title2", "eins zwei drei vier fuenf sechs sieben acht neun zehn elf zwoelf dreizehn test vierzehn fuenfzehn sechzehn siebzehn achtzehn neunzehn zwanzig", context);
CreatePage("title3", "eins zwei drei vier fuenf sechs sieben acht neun zehn elf zwoelf dreizehn test vierzehn fuenfzehn sechzehn siebzehn achtzehn neunzehn zwanzig", context);
CreatePage("title4", "eins zwei drei vier fuenf sechs sieben acht neun zehn elf zwoelf dreizehn NOOO vierzehn fuenfzehn sechzehn siebzehn achtzehn neunzehn zwanzig", context);
CreateTag("eins", context);
CreateTag("zwei", context);
CreateTag("drei", context);
CreateTag("vier", context);
CreatePageTagRef(1, 1, context);
CreatePageTagRef(1, 2, context);
CreatePageTagRef(1, 3, context);
CreatePageTagRef(1, 4, context);
CreatePageTagRef(2, 1, context);
CreatePageTagRef(2, 2, context);
CreatePageTagRef(2, 3, context);
CreatePageTagRef(2, 4, context);
CreatePageTagRef(3, 1, context);
CreatePageTagRef(3, 2, context);
CreatePageTagRef(3, 3, context);
CreatePageTagRef(3, 4, context);
CreatePageTagRef(4, 1, context);
CreatePageTagRef(4, 2, context);
CreatePageTagRef(4, 3, context);
CreatePageTagRef(4, 4, context);
context.SaveChanges();
}
public ActionResult Show(string id)
{
var cat = new WikiContext().Pages.Where(o => o.Category == id).ToList();
ViewBag.Title = id;
return(View(cat));
}
private static void WebParseCompanyInfoboxes()
{
var stopwatch = Stopwatch.StartNew();
var webParser = new InfoboxWebParser("https://en.wikipedia.org/w/index.php?title=Special:WhatLinksHere/Template:Infobox_company&limit=500");
var urls = webParser.GetArticlesUrl();
Console.WriteLine("Found {0} articles ({1})", urls.Count, stopwatch.Elapsed.ToString());
var groups = urls.Select((u, i) => new { Url = u, Index = i }).GroupBy(a => a.Index / 1000).ToList();
Console.WriteLine("Start parsing infoboxes for {0} groups of 1000 articles", groups.Count);
foreach (var group in groups.Skip(69))
{
var boxes = new List <ParsedInfobox>();
foreach (var url in group)
{
boxes.AddRange(webParser.GetArticleInfobox(url.Url));
}
using (var db = new WikiContext())
{
db.ParsedInfoboxes.AddRange(boxes);
db.SaveChanges();
}
Console.WriteLine("Group #{0} done", group.Key);
}
Console.Write("Parsed and persisted company infoboxes in {0}", stopwatch.Elapsed.ToString());
}
/// <summary>
/// Extracted from wikipedia dumps (step 3/3).
/// Write the infobox properties to a CSV file for easy consumption.
/// TODO: this step could be removed by moving the execution of the dump parsing directly in the ETL project.
/// </summary>
/// <param name="delimiter">The CSV delimiter to use when writing the CSV file</param>
private static void WriteInfoboxPropertiesToCsv(string delimiter = "\t")
{
var applicationDirectory = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "/../../";
var fileName = string.Format("wiki-dumps-infoboxes-{0}.csv", "1"); // TODO: give a specific identifier to identify the parsing version
var filePath = applicationDirectory + "Results/" + fileName;
using (var db = new WikiContext())
{
using (var mem = new FileStream(filePath, FileMode.Create))
using (var writer = new StreamWriter(mem))
using (var csvWriter = new CsvWriter(writer))
{
csvWriter.Configuration.Delimiter = delimiter;
csvWriter.WriteField("PageTitle");
csvWriter.WriteField("InfoboxId");
csvWriter.WriteField("Key");
csvWriter.WriteField("Value");
csvWriter.NextRecord();
foreach (var prop in db.RawInfoboxProperties)
{
csvWriter.WriteField(prop.PageTitle);
csvWriter.WriteField(prop.InfoboxId);
csvWriter.WriteField(prop.PropKey);
csvWriter.WriteField(prop.PropValue.Replace("\n", "/n")); // FIXME: hack to "escape" line returns in order not to screw up the CSV file
csvWriter.NextRecord();
}
/*writer.Flush();
* var result = Encoding.UTF8.GetString(mem.ToArray());
* Console.WriteLine(result);*/
}
}
}
public ArticleController(UserManager <ApplicationUser> userManager, IMapper mapper, WikiContext appDbContext, IWikiRepository repository)
{
_userManager = userManager;
_mapper = mapper;
_appDbContext = appDbContext;
this.repository = repository;
}
private static void CreateTag(string v, WikiContext context)
{
Tag t = new Tag();
t.Name = v;
context.Tags.Add(t);
context.SaveChanges();
}
public ActionResult Show(int id)
{
var page = new WikiContext().ArticleViewModels.Where(o => o.Id == id).FirstOrDefault();
if (page != null)
ViewBag.Title = page.Name;
else ViewBag.Title = "Error";
return View(page);
}
// GET: ArticleViewModel
public ActionResult Index(string name)
{
var context = new WikiContext(connectionString);
List<ArticleViewModel> pages = null;
if (name == null)
pages = context.ArticleViewModels.ToList();
else pages = context.ArticleViewModels.Where(o => o.Title == name).ToList();
return View(pages);
}
private static void CreatePage(string title, string text, WikiContext context)
{
Page p = new Page();
p.Title = title;
p.Content = text;
context.Pages.Add(p);
context.SaveChanges();
}
private static void CreatePageTagRef(int pId, int tId, WikiContext context)
{
PageTag pt = new PageTag();
pt.PageId = pId;
pt.TagId = tId;
context.PageTags.Add(pt);
context.SaveChanges();
}
public SearchHelperTest()
{
var conn = new SqliteConnection("DataSource=:memory:");
conn.Open();
var options = new DbContextOptionsBuilder <WikiContext>().UseSqlite(conn).Options;
var context = new WikiContext(options);
context.Database.EnsureCreated();
this.context = context;
TestDataCreator.CreateFourPagesWithTags(context);
}
public ActionResult Submit()
{
var title = Request["title"];
var category = Request["category"];
var content = Request["content"];
var id = 0;
int.TryParse(Request["pId"], out id);
Page page = null;
if (content != null && title != null && category != null)
{
page = new Page()
{
Name = title,
Category = category,
Text = content,
Edits = new List <EditData>()
{
new EditData()
{
Owner = page, Description = "Page Created", Time = DateTime.UtcNow
}
}
};
try
{
var wc = new WikiContext();
if (id == 0)
{
wc.Pages.Add(page);
}
else
{
var p = wc.Pages.First(o => o.Id == id);
p.Name = title;
p.Text = content;
p.Category = category;
p.Edits.Add(new EditData()
{
Owner = p, Time = DateTime.UtcNow, Description = Request["edit"]
});
page = p;
}
wc.SaveChanges();
} catch { page = null; }
}
return(View(page));
}
public static EntityFrameworkEntityManager Create(string sDataSource, string sCatalog, string sUser, string sPassword)
{
var sConnection = new SqlConnectionStringBuilder()
{
DataSource = sDataSource,
InitialCatalog = sCatalog,
UserID = sUser,
Password = sPassword,
}.ConnectionString;
var oContext = new WikiContext(sConnection);
return EntityFrameworkEntityManager.Create(oContext);
}
public ActionResult Show(int id)
{
var page = new WikiContext().Pages.Where(o => o.Id == id).FirstOrDefault();
if (page != null)
{
ViewBag.Title = page.Name;
}
else
{
ViewBag.Title = "Error";
}
return(View(page));
}
// GET: Page
public ActionResult Index(string name)
{
var context = new WikiContext();
List <Page> pages = null;
if (name == null)
{
pages = context.Pages.ToList();
}
else
{
pages = context.Pages.Where(o => o.Name == name).ToList();
}
return(View(pages));
}
/// <summary>
/// Extracted from wikipedia dumps (step 1/3).
/// Browse the wiki dumps and extract the markdown for each company infobox.
/// </summary>
private static void ParseCompanyInfoboxesInDumps()
{
var generalStopwatch = Stopwatch.StartNew();
var dumpDir = Utilities.PathToDownloadDirectory;
foreach (var filePath in Directory.EnumerateFiles(dumpDir))
{
Console.WriteLine("Start parsing infoboxes in file {0}", Path.GetFileName(filePath));
var stopwatch = Stopwatch.StartNew();
var infoboxes = new List <RawDumpParsedInfobox>();
var wikiReader = new XmlDumpFileReader(filePath);
Predicate <string> pageFilterer = s => s.Contains(":"); // Real Wikipedia pages contains ":" (others are conversations etc.)
var page = wikiReader.ReadNext(pageFilterer);
while (page != null)
{
var boxes = page.GetInfoboxTexts("company").Select(s => new RawDumpParsedInfobox()
{
Markdown = HttpUtility.HtmlDecode(s),
PageTitle = page.Title
});
infoboxes.AddRange(boxes);
page = wikiReader.ReadNext(pageFilterer);
}
stopwatch.Stop();
Console.WriteLine("Parsed {0} infoboxes in {1}", infoboxes.Count, stopwatch.Elapsed.ToString());
stopwatch.Restart();
// Persist infoboxes
using (var db = new WikiContext())
{
db.RawDumpParsedInfoboxes.AddRange(infoboxes);
db.SaveChanges();
}
stopwatch.Stop();
Console.WriteLine("Persisted {0} infoboxes in {1}", infoboxes.Count, stopwatch.Elapsed.ToString());
Console.WriteLine("--");
}
generalStopwatch.Stop();
Console.WriteLine("Total infobox parsing time: {0}", generalStopwatch.Elapsed.ToString());
}
public static void InitializeDb(string connectionString)
{
var options = new DbContextOptionsBuilder <WikiContext>().UseSqlite(connectionString).Options;
using (WikiContext db = new WikiContext(options))
{
db.Database.EnsureCreated();
//Only Initialize if DB is Empty
if (db.Pages.Any())
{
return;
}
CreateOverviewPage(db);
}
}
/// <summary>
/// Extracted from wikipedia dumps (step 2/3).
/// Process the markdown for each company infobox and extract the properties (key/value pairs).
/// </summary>
private static void ProcessCompanyInfoboxes()
{
var batchSize = 1000;
var index = 0;
using (var db = new WikiContext())
{
var infoboxes = db.RawDumpParsedInfoboxes.OrderBy(box => box.Id).Skip(index * batchSize).Take(batchSize).ToList();
while (infoboxes.Any())
{
if (index % 10 == 0)
{
Console.WriteLine("Start parsing batch #{0}", index);
}
foreach (var infobox in infoboxes)
{
var markdownRegex = new Regex(@"(\n\s*\||\|\s*\n)(?=[\sa-zA-Z_]+=)", RegexOptions.Compiled);
var parts = markdownRegex.Split(infobox.Markdown).ToList();
var properties = parts.Where(line => line.Contains("="))
.Select(line => line.Split(new char[] { '=' }, 2))
.Select(tup => new RawInfoboxProperty
{
PropKey = tup.First().Trim(),
PropValue = tup.Last().Trim(),
PageTitle = infobox.PageTitle,
InfoboxId = infobox.Id
})
.ToList();
db.RawInfoboxProperties.AddRange(properties);
}
db.SaveChanges();
index++;
infoboxes = db.RawDumpParsedInfoboxes.OrderBy(box => box.Id).Skip(index * batchSize).Take(batchSize).ToList();
}
}
// One infobox doesn't have any property -> ATMNet (normal)
}
private static void CreateOverviewPage(WikiContext db)
{
Tag t1 = new Tag();
t1.Name = "WikiCore";
t1.Color = 1;
db.Tags.Add(t1);
Page p = new Page();
p.Title = "Start";
p.Content = "Welcome to __WikiCore__.\r\n\r\nWikiCore is a modest, small and fast Wiki featuring [MarkDown](https://daringfireball.net/projects/markdown/) editing.\r\n\r\nUnlike regular Wikis pages are organized with tags.\r\n\r\nPlease report Bugs in a [GitHub-Issue](https://github.com/philphilphil/WikiCore/issues).";
db.Pages.Add(p);
var pt = new PageTag {
Tag = t1, Page = p
};
db.PageTags.Add(pt);
db.SaveChanges();
}
public ActionResult Post()
{
var query = Request.Form["SearchQuery"];
if (string.IsNullOrWhiteSpace(query))
{
using (var context = new WikiContext())
{
return(View("Index", context.WikiReferences.ToList()));
}
}
query = Regex.Replace(query, @"\s+", " ").Trim();
query = query.Replace(" ", " OR ");
var s = FtsInterceptor.Fts(query);
using (var context = new WikiContext())
{
return(View("Index", context.WikiReferences
.Where(r => r.Content.Contains(s)).ToList()));
}
}
private static void ParseInfoboxProperties()
{
// We chose the strategy to do as much as possible directly in SQL, performance-wise.
// However, we execute queries directly in SQL to limit the overhead of EntityFramework
using (var db = new WikiContext())
{
// Queries can take a long time to run; set the timeout to 6 hours
db.Database.CommandTimeout = 6 * 60 * 60;
var stopwatch = Stopwatch.StartNew();
// Update the Template Property of the parsed Infoboxes
int nbInfoboxTemplatesUpdated = db.Database.ExecuteSqlCommand(@"
UPDATE Infoboxes
SET Template = TRIM(SUBSTRING(RawText, (CHARINDEX('{{Infobox ', RawText) + LEN('{{Infobox')), (CHARINDEX('|', RawText) - LEN('{{Infobox') - 1)))
WHERE RawText like '%{{Infobox%' and RawText like '%|%'");
Console.WriteLine("{0} templates were extracted for infoboxes", nbInfoboxTemplatesUpdated);
// Extract the infoboxes' properties in markup text and create the infoboxes' properties with them
int nbOfInfoboxPropertiesCreated = db.Database.ExecuteSqlCommand(@"
INSERT INTO InfoboxProperties (Infobox_Id, RawText)
SELECT Id, value
FROM Infoboxes
CROSS APPLY string_split(SUBSTRING(RawText, CHARINDEX('|', RawText) + 1, LEN(RawText)), '|')
WHERE RawText like '%{{Infobox%' and RawText like '%|%'
ORDER BY Infoboxes.Id");
Console.WriteLine("{0} infobox properties were created", nbOfInfoboxPropertiesCreated);
// Parse the infoboxes' properties' raw text
int nbOfPropertiesUpdated = db.Database.ExecuteSqlCommand(@"
UPDATE InfoboxProperties
SET Key = TRIM(SUBSTRING(RawText, 0, CHARINDEX('='))),
Value = TRIM(SUBSTRING(RawText, CHARINDEX('='), LEN(RawText))
WHERE RawText like '%=%'");
Console.WriteLine("{0} infobox properties have been parsed", nbOfPropertiesUpdated);
}
}
public EditRepository(WikiContext context)
{
this.db = context;
}
public NationsController(WikiContext context)
{
_context = context;
}
public ActionResult Edit(int id)
{
var page = new WikiContext().ArticleViewModels.FirstOrDefault(o => o.Id == id);
return View(page);
}
public BranchRepository(WikiContext context) : base(context)
{
}
public AssetsController(WikiContext context, IAssetRepository assetRepository)
{
_context = context;
_assetRepository = assetRepository;
}
public ResultRepository(WikiContext context) : base(context)
{
}