private async Task _FindCategoryPoemsRhythmsInternal(int catId, bool retag, string rhythm)
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"FindCategoryPoemsRhythms Cat {catId}", "Query data")).Result;
try
{
var metres = await context.GanjoorMetres.OrderBy(m => m.Rhythm).AsNoTracking().ToArrayAsync();
var rhythms = metres.Select(m => m.Rhythm).ToArray();
GanjoorMetre preDeterminedMetre = string.IsNullOrEmpty(rhythm) ? null : metres.Where(m => m.Rhythm == rhythm).Single();
var poems = await context.GanjoorPoems.Where(p => p.CatId == catId).ToListAsync();
int i = 0;
using (HttpClient httpClient = new HttpClient())
{
foreach (var poem in poems)
{
if (retag || poem.GanjoorMetreId == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, i ++);
if (preDeterminedMetre == null)
{
var res = await _FindPoemRhythm(poem.Id, context, httpClient, rhythms);
if (!string.IsNullOrEmpty(res.Result))
{
poem.GanjoorMetreId = metres.Where(m => m.Rhythm == res.Result).Single().Id;
context.GanjoorPoems.Update(poem);
await context.SaveChangesAsync();
}
}
else
{
poem.GanjoorMetreId = preDeterminedMetre.Id;
context.GanjoorPoems.Update(poem);
await context.SaveChangesAsync();
}
if (poem.GanjoorMetreId != null && !string.IsNullOrEmpty(poem.RhymeLetters))
{
await _UpdateRelatedPoems(context, (int)poem.GanjoorMetreId, poem.RhymeLetters);
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 99);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
/// <summary>
/// Start finding missing rhthms
/// </summary>
/// <param name="onlyPoemsWithRhymes"></param>
/// <param name="poemsNum"></param>
/// <returns></returns>
public RServiceResult <bool> StartFindingMissingRhythms(bool onlyPoemsWithRhymes, int poemsNum = 1000)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"StartFindingMissingRhythms", "Query data")).Result;
try
{
var poemIds = await context.GanjoorPoems.AsNoTracking()
.Where(p =>
p.GanjoorMetreId == null && (onlyPoemsWithRhymes == false || !string.IsNullOrEmpty(p.RhymeLetters))
&&
false == (context.GanjoorVerses.Where(v => v.PoemId == p.Id && (v.VersePosition == VersePosition.Paragraph || v.VersePosition == VersePosition.Single)).Any())
&&
false == (context.GanjoorPoemProbableMetres.Where(r => r.PoemId == p.Id).Any())
)
.Take(poemsNum)
.Select(p => p.Id)
.ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Total: {poemIds.Length}");
var metres = await context.GanjoorMetres.OrderBy(m => m.Rhythm).AsNoTracking().Select(m => m.Rhythm).ToArrayAsync();
using (HttpClient httpClient = new HttpClient())
{
for (int i = 0; i < poemIds.Length; i++)
{
var id = poemIds[i];
var res = await _FindPoemRhythm(id, context, httpClient, metres, true);
if (res.Result == null)
{
res.Result = "";
}
GanjoorPoemProbableMetre prometre = new GanjoorPoemProbableMetre()
{
PoemId = id,
Metre = res.Result
};
context.GanjoorPoemProbableMetres.Add(prometre);
await jobProgressServiceEF.UpdateJob(job.Id, i);
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start filling GanjoorLink table OriginalSource values
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartFillingGanjoorLinkOriginalSources()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FillingGanjoorLinkOriginalSources", "Updating")).Result;
try
{
var links = await context.GanjoorLinks.ToListAsync();
for (int i = 0; i < links.Count; i++)
{
var link = links[i];
var itemInfo = await context.Items
.Include(i => i.Tags)
.ThenInclude(t => t.RTag)
.Where(i => i.Id == link.ItemId).SingleAsync();
var sourceTag = itemInfo.Tags.Where(t => t.RTag.FriendlyUrl == "source").FirstOrDefault();
if (sourceTag != null)
{
if (!string.IsNullOrEmpty(sourceTag.ValueSupplement) && (sourceTag.ValueSupplement.IndexOf("http") == 0))
{
link.OriginalSourceUrl = sourceTag.ValueSupplement;
link.LinkToOriginalSource = true;
context.GanjoorLinks.Update(link);
}
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start generating related poems info
/// </summary>
/// <param name="regenerate"></param>
/// <returns></returns>
public RServiceResult <bool> StartGeneratingRelatedPoemsInfo(bool regenerate)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("GeneratingRelatedPoemsInfo", "Query")).Result;
try
{
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Query");
var poemIds = await context.GanjoorPoems.AsNoTracking().Where(p => !string.IsNullOrEmpty(p.RhymeLetters) && p.GanjoorMetreId != null).Select(p => p.Id).ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Updating Related Poems");
int percent = 0;
for (int i = 0; i < poemIds.Count; i++)
{
if (!regenerate)
{
if (await context.GanjoorCachedRelatedPoems.AnyAsync(r => r.PoemId == poemIds[i]))
{
continue;
}
}
if (i * 100 / poemIds.Count > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
await _UpdatePoemRelatedPoemsInfoNoSaveChanges(context, poemIds[i]);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// examine comments for long links
/// </summary>
/// <returns></returns>
public RServiceResult <bool> FindAndFixLongUrlsInComments()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FindAndFixLongUrlsInComments", "Query data")).Result;
try
{
var comments = await context.GanjoorComments.Where(c => c.HtmlComment.Contains("href=")).ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Examining {comments.Length} Comments");
int percent = 0;
for (int i = 0; i < comments.Length; i++)
{
if (i * 100 / comments.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var comment = comments[i];
string commentText = await _ProcessCommentHtml(comment.HtmlComment, context);
if (commentText != comment.HtmlComment)
{
comment.HtmlComment = commentText;
context.Update(comment);
await context.SaveChangesAsync();
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// generate missing default numberings and start counting
/// </summary>
/// <returns></returns>
public RServiceResult <bool> GenerateMissingDefaultNumberings()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("GenerateMissingDefaultNumberings", "Query")).Result;
try
{
var cats = await context.GanjoorCategories.Where(c => c.ParentId != null).ToListAsync();
foreach (var cat in cats)
{
var numbering = await context.GanjoorNumberings.Where(n => n.StartCatId == cat.Id && n.Name == cat.Title).FirstOrDefaultAsync();
if (numbering == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, 0, cat.FullUrl);
numbering = new GanjoorNumbering()
{
Name = cat.Title,
StartCatId = cat.Id,
EndCatId = cat.Id
};
context.GanjoorNumberings.Add(numbering);
await context.SaveChangesAsync();
Recount(numbering.Id); //start counting
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start filling poems couplet indices
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartFillingPoemsCoupletIndices()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FillingPoemsCoupletIndices", "Query data")).Result;
try
{
var poemIds = await context.GanjoorPoems.AsNoTracking().Select(p => p.Id).ToListAsync();
int percent = 0;
for (int i = 0; i < poemIds.Count; i++)
{
if (i * 100 / poemIds.Count > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
await _FillPoemCoupletIndices(context, poemIds[i]);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start updating stats page
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartUpdatingMundexPage(Guid editingUserId)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("UpdateMundexPage", "House Keeping")).Result;
try
{
await _PerformMundexHouseKeepingAndPreparation(context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 10, "Mundex Page");
await _UpdateMundexPage(editingUserId, context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 60, "Mundex by poet page");
await _UpdateMundexByPoetPage(editingUserId, context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start generating sub cats TOC
/// </summary>
/// <param name="userId"></param>
/// <param name="catId"></param>
/// <returns></returns>
public RServiceResult <bool> StartGeneratingSubCatsTOC(Guid userId, int catId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"GeneratingSubCatsTOC {catId}", "Query data")).Result;
try
{
await _GeneratingSubCatsTOC(userId, context, jobProgressServiceEF, job, catId);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return(new RServiceResult <bool>(true));
}
/// <summary>
/// import GanjoorPage entity data from MySql
/// </summary>
/// <returns></returns>
public RServiceResult <bool> ImportFromMySql()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC
using (RMuseumDbContext contextReport = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(contextReport);
var job = (await jobProgressServiceEF.NewJob("GanjoorService:ImportFromMySql", "pre open connection")).Result;
if (string.IsNullOrEmpty(Configuration.GetSection("AudioMySqlServer")["ReportedCommentsDatabase"]))
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, "ReportedCommentsDatabase is not set");
return;
}
MusicCatalogueService catalogueService = new MusicCatalogueService(Configuration, context);
RServiceResult <bool> musicCatalogueRes = await catalogueService.ImportFromMySql("MusicCatalogueImportFromMySql", jobProgressServiceEF, job);
if (!musicCatalogueRes.Result)
{
return;
}
try
{
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT ID, post_author, post_date, post_date_gmt, post_content, post_title, post_category, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count, " +
"COALESCE((SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='_wp_page_template'), '') AS template," +
"(SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='otherpoetid') AS other_poet_id " +
"FROM ganja_posts",
connection))
{
using (DataTable srcData = new DataTable())
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - mysql 1")).Result;
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - processing mysql data")).Result;
foreach (DataRow row in srcData.Rows)
{
GanjoorPageType pageType =
row["post_type"].ToString() == "post" && row["comment_status"].ToString() != "closed" ?
GanjoorPageType.PoemPage
:
row["template"].ToString() == "comspage.php" ?
GanjoorPageType.AllComments
:
row["template"].ToString() == "relations.php" ?
GanjoorPageType.ProsodySimilars
:
row["template"].ToString() == "vazn.php" ?
GanjoorPageType.ProsodyAndStats
:
GanjoorPageType.None;
int?poetId = row["post_author"].ToString() == "1" ? (int?)null : int.Parse(row["post_author"].ToString());
if (poetId == 36)//رشحه
{
continue;
}
if (poetId != null)
{
if (!await context.GanjoorPoets.Where(poet => poet.Id == poetId).AnyAsync())
{
continue;
}
}
GanjoorPage page = new GanjoorPage()
{
Id = int.Parse(row["ID"].ToString()),
GanjoorPageType = pageType,
Published = true,
PageOrder = -1,
Title = row["post_title"].ToString(),
UrlSlug = row["post_name"].ToString(),
HtmlText = row["post_content"].ToString(),
ParentId = row["post_parent"].ToString() == "0" ? (int?)null : int.Parse(row["post_parent"].ToString()),
PoetId = poetId,
SecondPoetId = row["other_poet_id"] == DBNull.Value ? (int?)null : int.Parse(row["other_poet_id"].ToString()),
PostDate = (DateTime)row["post_date"]
};
if (pageType == GanjoorPageType.PoemPage)
{
var poem = await context.GanjoorPoems.Where(p => p.Id == page.Id).FirstOrDefaultAsync();
if (poem == null)
{
continue;
}
page.PoemId = poem.Id;
}
if (poetId != null && pageType == GanjoorPageType.None)
{
GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId == null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
page.GanjoorPageType = GanjoorPageType.PoetPage;
page.CatId = cat.Id;
}
else
{
cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId != null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
page.GanjoorPageType = GanjoorPageType.CatPage;
page.CatId = cat.Id;
}
}
}
context.GanjoorPages.Add(page);
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - finalizing")).Result;
await context.SaveChangesAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - pre fetch data")).Result;
var orphanPages = await context.GanjoorPages.Include(p => p.Poem).Where(p => p.FullUrl == null).ToListAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - post fetch data")).Result;
double count = orphanPages.Count;
int i = 0;
foreach (var page in orphanPages)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, i++, "phase 2")).Result;
string fullUrl = page.UrlSlug;
string fullTitle = page.Title;
if (page.GanjoorPageType == GanjoorPageType.PoemPage)
{
fullTitle = page.Poem.FullTitle;
fullUrl = page.Poem.FullUrl;
}
else
{
if (page.ParentId != null)
{
GanjoorPage parent = await context.GanjoorPages.Where(p => p.Id == page.ParentId).SingleAsync();
while (parent != null)
{
fullUrl = parent.UrlSlug + "/" + fullUrl;
fullTitle = parent.Title + " » " + fullTitle;
parent = parent.ParentId == null ? null : await context.GanjoorPages.Where(p => p.Id == parent.ParentId).SingleAsync();
}
}
else
{
GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
fullUrl = cat.FullUrl;
while (cat.ParentId != null)
{
cat = await context.GanjoorCategories.Where(c => c.Id == cat.ParentId).SingleOrDefaultAsync();
if (cat != null)
{
fullTitle = cat.Title + " » " + fullTitle;
}
}
}
else
{
cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.ParentId == null).SingleOrDefaultAsync();
if (cat != null)
{
fullUrl = $"{cat.UrlSlug}/{page.UrlSlug}";
}
}
}
}
if (!string.IsNullOrEmpty(fullUrl) && fullUrl.IndexOf('/') != 0)
{
fullUrl = $"/{fullUrl}";
}
page.FullUrl = fullUrl;
page.FullTitle = fullTitle;
context.Update(page);
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "phase 2 - finalizing")).Result;
await context.SaveChangesAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - pre mysql data fetch")).Result;
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT meta_key, post_id, meta_value FROM ganja_postmeta WHERE meta_key IN ( 'vazn', 'ravi', 'src', 'srcslug', 'oldtag' )",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - mysql 2")).Result;
using (DataTable srcData = new DataTable())
{
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - processing meta data")).Result;
int r = 0;
foreach (DataRow row in srcData.Rows)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, r++, "phase 3 - processing meta data")).Result;
int poemId = int.Parse(row["post_id"].ToString());
var poem = await context.GanjoorPoems.Where(p => p.Id == poemId).FirstOrDefaultAsync();
if (poem == null)
{
continue;
}
string metaKey = row["meta_key"].ToString();
string metaValue = row["meta_value"].ToString();
switch (metaKey)
{
case "vazn":
{
GanjoorMetre metre = await context.GanjoorMetres.Where(m => m.Rhythm == metaValue).SingleOrDefaultAsync();
if (metre == null)
{
metre = new GanjoorMetre()
{
Rhythm = metaValue,
VerseCount = 0
};
context.GanjoorMetres.Add(metre);
await context.SaveChangesAsync();
}
poem.GanjoorMetreId = metre.Id;
}
break;
case "ravi":
poem.RhymeLetters = metaValue;
break;
case "src":
poem.SourceName = metaValue;
break;
case "srcslug":
poem.SourceUrlSlug = metaValue;
break;
case "oldtag":
poem.OldTag = metaValue;
switch (poem.OldTag)
{
case "بدایع":
poem.OldTagPageUrl = "/saadi/badaye";
break;
case "خواتیم":
poem.OldTagPageUrl = "/saadi/khavatim";
break;
case "طیبات":
poem.OldTagPageUrl = "/saadi/tayyebat";
break;
case "غزلیات قدیم":
poem.OldTagPageUrl = "/saadi/ghazaliyat-e-ghadim";
break;
case "ملمعات":
poem.OldTagPageUrl = "/saadi/molammaat";
break;
}
break;
}
context.GanjoorPoems.Update(poem);
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - finalizing meta data")).Result;
await context.SaveChangesAsync();
var resApprovedPoemSongs = await _ImportPoemSongsDataFromMySql("_ImportPoemSongsDataFromMySql", context, jobProgressServiceEF, job, true);
if (!resApprovedPoemSongs.Result)
{
return;
}
var resPendingPoemSongs = await _ImportPoemSongsDataFromMySql("_ImportPoemSongsDataFromMySql", context, jobProgressServiceEF, job, false);
if (!resPendingPoemSongs.Result)
{
return;
}
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT poem_id, mimage_id FROM ganja_mimages",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - mysql N")).Result;
using (DataTable srcData = new DataTable())
{
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - processing meta data")).Result;
int r = 0;
foreach (DataRow row in srcData.Rows)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, r++, "phase N - processing meta data")).Result;
int poemId = int.Parse(row["poem_id"].ToString());
Guid imageId = Guid.Parse(row["mimage_id"].ToString());
var link = await context.GanjoorLinks.Include(l => l.Item).ThenInclude(i => i.Images).
Where(l => l.GanjoorPostId == poemId && l.Item.Images.First().Id == imageId)
.FirstOrDefaultAsync();
if (link != null)
{
link.DisplayOnPage = true;
context.GanjoorLinks.Update(link);
}
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - finalizing meta data")).Result;
await context.SaveChangesAsync();
}
catch (Exception jobExp)
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, jobExp.ToString());
}
var resComments = await _ImportCommentsDataFromMySql("_ImportCommentsDataFromMySql", context, jobProgressServiceEF, job);
if (!resComments.Result)
{
return;
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "Finished", true);
}
});
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start counting
/// </summary>
/// <returns></returns>
public RServiceResult <bool> Recount(int numberingId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("NumberingService::Count", "Query Cats")).Result;
try
{
var numbering = await context.GanjoorNumberings.Include(n => n.StartCat).Where(n => n.Id == numberingId).SingleOrDefaultAsync();
var cats = await _FindTargetCategories(context, numbering.StartCat, numbering.EndCatId);
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Deleting Old Data");
var oldNumbers = await context.GanjoorVerseNumbers.Where(n => n.NumberingId == numberingId).ToListAsync();
context.GanjoorVerseNumbers.RemoveRange(oldNumbers);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Counting");
int number = 0;
int coupletnumber = 0;
int paragraphnumber = 0;
int totalVerseCount = 0;
foreach (var cat in cats)
{
var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == cat.Id).OrderBy(p => p.Id).ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Counting:: {cat.Title}");
foreach (var poem in poems)
{
totalVerseCount += await context.GanjoorVerses.Where(v => v.PoemId == poem.Id).CountAsync();
var verses = await context.GanjoorVerses.AsNoTracking()
.Where(v => v.PoemId == poem.Id && v.VersePosition != VersePosition.Left && v.VersePosition != VersePosition.CenteredVerse2 && v.VersePosition != VersePosition.Comment)
.OrderBy(v => v.VOrder)
.ToListAsync();
for (int coupletIndex = 0; coupletIndex < verses.Count; coupletIndex++)
{
number++;
bool isPoemVerse = verses[coupletIndex].VersePosition == VersePosition.Right || verses[coupletIndex].VersePosition == VersePosition.CenteredVerse1;
if (isPoemVerse)
{
coupletnumber++;
}
else
{
paragraphnumber++;
}
GanjoorVerseNumber verseNumber = new GanjoorVerseNumber()
{
NumberingId = numberingId,
PoemId = poem.Id,
CoupletIndex = coupletIndex,
Number = number,
IsPoemVerse = isPoemVerse,
SameTypeNumber = isPoemVerse ? coupletnumber : paragraphnumber
};
context.GanjoorVerseNumbers.Add(verseNumber);
}
}
}
numbering.TotalLines = number;
numbering.TotalVerses = totalVerseCount;
numbering.TotalCouplets = coupletnumber;
numbering.TotalParagraphs = paragraphnumber;
numbering.LastCountingDate = DateTime.Now;
context.GanjoorNumberings.Update(numbering);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start updating stats page
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartUpdatingStatsPage(Guid editingUserId)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("UpdateStatsPage", "Total Poets Stats")).Result;
try
{
var poetsCoupletCounts =
await context.GanjoorVerses.Include(v => v.Poem).ThenInclude(p => p.Cat).ThenInclude(c => c.Poet).AsNoTracking()
.Where(v =>
v.Poem.Cat.Poet.Published
&&
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1))
.GroupBy(v => new { v.Poem.Cat.PoetId })
.Select(g => new { PoetId = g.Key.PoetId, Count = g.Count() })
.ToListAsync();
poetsCoupletCounts.Sort((a, b) => b.Count - a.Count);
var sumPoetsCouplets = poetsCoupletCounts.Sum(c => c.Count);
var rhythmsCoupletCounts =
await context.GanjoorVerses.Include(v => v.Poem).ThenInclude(p => p.Cat).ThenInclude(c => c.Poet).AsNoTracking()
.Where(v =>
v.Poem.Cat.Poet.Published
&&
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1))
.GroupBy(v => new { v.Poem.GanjoorMetreId })
.Select(g => new { GanjoorMetreId = g.Key.GanjoorMetreId, Count = g.Count() })
.ToListAsync();
rhythmsCoupletCounts.Sort((a, b) => b.Count - a.Count);
var sumRhythmsCouplets = rhythmsCoupletCounts.Sum(c => c.Count);
var dbPage = await context.GanjoorPages.Where(p => p.FullUrl == "/vazn").SingleAsync();
var poets = await context.GanjoorPoets.ToListAsync();
string htmlText = $"<p>تا تاریخ {LanguageUtils.FormatDate(DateTime.Now)} مجموعاً {LanguageUtils.FormatMoney(sumPoetsCouplets)} بیت شعر از طریق سایت گنجور در دسترس قرار گرفته است. در جدول زیر که شاعران در آنها بر اساس تعداد ابیات اشعارشان به صورت نزولی مرتب شدهاند با کلیک بر روی نام هر شاعر میتوانید آمار اوزان اشعار او را مشاهده کنید.</p>{Environment.NewLine}";
htmlText += $"<p>توجه فرمایید که این آمار به دلایلی از قبیل وجود چند نسخه از آثار شعرا در سایت (مثل آثار خیام یا وجود متن خلاصه و کامل هفت اورنگ جامی) و همینطور یک بیت محسوب شدن مصرعهای بند قالبهای ترکیبی مثل مخمسها تقریبی و حدودی است و افزونگی دارد.</p>{Environment.NewLine}";
htmlText += $"<table>{Environment.NewLine}" +
$"<tr class=\"h\">{Environment.NewLine}" +
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
$"<td class=\"c2\">شاعر</td>{Environment.NewLine}" +
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
$"</tr>{Environment.NewLine}";
for (int i = 0; i < poetsCoupletCounts.Count; i++)
{
if (i % 2 == 0)
{
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
}
else
{
htmlText += $"<tr>{Environment.NewLine}";
}
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c2\"><a href=\"{(await context.GanjoorCategories.Where(c => c.ParentId == null && c.PoetId == poetsCoupletCounts[i].PoetId).SingleAsync()).FullUrl}/vazn\">{poets.Where(p => p.Id == poetsCoupletCounts[i].PoetId).Single().Nickname}</a></td>{Environment.NewLine}";
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(poetsCoupletCounts[i].Count)}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c4\">{(poetsCoupletCounts[i].Count * 100.0 / sumPoetsCouplets).ToString("N2", new CultureInfo("fa-IR")).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"</tr>{Environment.NewLine}";
}
htmlText += $"</table>{Environment.NewLine}";
var rhythms = await context.GanjoorMetres.ToListAsync();
htmlText += $"<p>فهرست زیر نیز آمار اشعار گنجور را از لحاظ اوزان عروضی نشان میدهد:</p>{Environment.NewLine}";
htmlText += $"<table>{Environment.NewLine}" +
$"<tr class=\"h\">{Environment.NewLine}" +
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
$"<td class=\"c2\">وزن</td>{Environment.NewLine}" +
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
$"</tr>{Environment.NewLine}";
for (int i = 0; i < rhythmsCoupletCounts.Count; i++)
{
if (i % 2 == 0)
{
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
}
else
{
htmlText += $"<tr>{Environment.NewLine}";
}
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
var rhythm = rhythms.Where(r => r.Id == rhythmsCoupletCounts[i].GanjoorMetreId).SingleOrDefault();
if (rhythm != null)
{
rhythm.VerseCount = rhythmsCoupletCounts[i].Count;
context.Update(rhythm);
}
string rhythmName = rhythmsCoupletCounts[i].GanjoorMetreId == null ? "وزنیابی نشده" :
$"<a href=\"/vazn/?v={Uri.EscapeDataString(rhythm.Rhythm)}\">{rhythms.Where(r => r.Id == rhythmsCoupletCounts[i].GanjoorMetreId).Single().Rhythm}</a>";
htmlText += $"<td class=\"c2\">{rhythmName}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(rhythmsCoupletCounts[i].Count)}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c4\">{(rhythmsCoupletCounts[i].Count * 100.0 / sumRhythmsCouplets).ToString("N2", new CultureInfo("fa-IR")).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"</tr>{Environment.NewLine}";
}
htmlText += $"</table>{Environment.NewLine}";
await context.SaveChangesAsync(); //store rhythm[s].VerseCount
await _UpdatePageHtmlText(context, editingUserId, dbPage, "به روزرسانی خودکار صفحهٔ آمار وزنها", htmlText);
foreach (var poetInfo in poetsCoupletCounts)
{
var poet = poets.Where(p => p.Id == poetInfo.PoetId).Single();
await jobProgressServiceEF.UpdateJob(job.Id, poetInfo.PoetId, poet.Nickname);
await _UpdatePoetStatsPage(editingUserId, poet, rhythms, context);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// import from sqlite
/// </summary>
/// <param name="poetId"></param>
/// <param name="file"></param>
/// <returns></returns>
public async Task <RServiceResult <bool> > ImportFromSqlite(int poetId, IFormFile file)
{
try
{
string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string filePath = Path.Combine(dir, file.FileName);
if (File.Exists(filePath))
{
File.Delete(filePath);
}
using (FileStream fsMain = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(fsMain);
}
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ImportFromSqlite", "Query data")).Result;
try
{
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
IDbConnection sqlite = sqliteConnection;
var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList();
if (poets.Count != 1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1");
}
var poet = await context.GanjoorPoets.AsNoTracking().Where(p => p.Id == poetId).SingleAsync();
var cat = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poetId && c.ParentId == null).SingleAsync();
var catPage = await context.GanjoorPages.AsNoTracking().Where(p => p.FullUrl == cat.FullUrl).SingleAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Importing");
var resImport = await _ImportSQLiteCatChildren(context, sqlite, poetId, await sqlite.QuerySingleAsync <int>($"SELECT id FROM cat WHERE parent_id = 0"), cat, poet.Nickname, jobProgressServiceEF, job, catPage.Id);
if (string.IsNullOrEmpty(resImport))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
else
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resImport);
}
}
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
File.Delete(filePath);
}
);
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start generating gdb files
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartBatchGenerateGDBFiles()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("BatchGenerateGDBFiles", "Query Data")).Result;
try
{
string outDir = Configuration.GetSection("Ganjoor")["GDBStorage"];
string imgDir = Configuration.GetSection("Ganjoor")["GDBStorageImageSource"];
string xmlFile = Configuration.GetSection("Ganjoor")["GDBListXMLFile"];
string preExisitingListXMLFile = Configuration.GetSection("Ganjoor")["GDBPreExisitingListXMLFile"];
List <GDBInfo> programList = new List <GDBInfo>();
if (!string.IsNullOrEmpty(preExisitingListXMLFile))
{
if (File.Exists(preExisitingListXMLFile))
{
programList = GDBListProcessor.RetrieveListFromFile(preExisitingListXMLFile, out string exception);
if (programList == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "GDBListProcessor.RetrieveListFromFile", false, exception);
return;
}
}
}
var poets = await context.GanjoorPoets.AsNoTracking().ToListAsync();
List <GDBInfo> lstFiles = new List <GDBInfo>();
foreach (var poet in poets)
{
if (!await context.GanjoorPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id).AnyAsync())
{
continue;
}
await jobProgressServiceEF.UpdateJob(job.Id, poet.Id);
var gdbGeneration = await _ExportToSqlite(context, poet.Id, outDir, null, false);
if (!string.IsNullOrEmpty(gdbGeneration.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, gdbGeneration.ExceptionString);
return;
}
string gdbFile = gdbGeneration.Result;
string pngFile = Path.Combine(imgDir, $"{poet.Id}.png");
bool hasImage = File.Exists(pngFile);
using (var archiveStream = new MemoryStream())
{
using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, true))
{
var zipGDBFileEntry = archive.CreateEntry(Path.GetFileName(gdbFile), CompressionLevel.Optimal);
using (var zipStream = zipGDBFileEntry.Open())
{
var gdbBytes = File.ReadAllBytes(gdbFile);
zipStream.Write(gdbBytes, 0, gdbBytes.Length);
}
if (hasImage)
{
var zipImgFileEntry = archive.CreateEntry(Path.GetFileName(pngFile), CompressionLevel.Optimal);
using (var zipStream = zipImgFileEntry.Open())
{
var pngBytes = File.ReadAllBytes(pngFile);
zipStream.Write(pngBytes, 0, pngBytes.Length);
}
}
}
string zipFile = Path.Combine(outDir, Path.GetFileNameWithoutExtension(gdbFile) + ".zip");
if (File.Exists(zipFile))
{
File.Delete(zipFile);
}
byte[] zipArray = archiveStream.ToArray();
File.WriteAllBytes(zipFile, zipArray);
var catPoet = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poet.Id && c.ParentId == null).SingleAsync();
var lowestPoemID = await context.GanjoorPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id)
.MinAsync(p => p.Id);
lstFiles.Add
(
new GDBInfo()
{
CatName = poet.Nickname,
CatID = catPoet.Id,
PoetID = poet.Id,
DownloadUrl = $"http://i.ganjoor.net/android/gdb/{Path.GetFileName(zipFile)}",
BlogUrl = "",
FileExt = ".zip",
ImageUrl = (hasImage ? $"http://i.ganjoor.net/android/img/{poet.Id}.png" : ""),
FileSizeInByte = zipArray.Length,
LowestPoemID = lowestPoemID,
PubDate = DateTime.Now
}
);
}
File.Delete(gdbFile);
}
if (File.Exists(xmlFile))
{
File.Delete(xmlFile);
}
if (programList.Count > 0)
{
lstFiles.AddRange(programList);
}
lstFiles.Sort((a, b) => a.CatName.CompareTo(b.CatName));
GDBListProcessor.Save(xmlFile, "مجموعههای قابل دریافت از گنجور", "", "", lstFiles);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// Apply corrections from sqlite
/// </summary>
/// <param name="poetId"></param>
/// <param name="file"></param>
/// <param name="note"></param>
/// <returns></returns>
public async Task <RServiceResult <bool> > ApplyCorrectionsFromSqlite(int poetId, IFormFile file, string note)
{
try
{
string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string filePath = Path.Combine(dir, file.FileName);
if (File.Exists(filePath))
{
File.Delete(filePath);
}
using (FileStream fsMain = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(fsMain);
}
string email = $"{Configuration.GetSection("Ganjoor")["SystemEmail"]}";
var userId = (await _appUserService.FindUserByEmail(email)).Result.Id;
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ApplyCorrectionsFromSqlite", "Query data")).Result;
try
{
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
IDbConnection sqlite = sqliteConnection;
var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList();
if (poets.Count != 1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1");
}
int poemNumber = 0;
foreach (var poem in await sqlite.QueryAsync($"SELECT * FROM poem ORDER BY id"))
{
poemNumber++;
await jobProgressServiceEF.UpdateJob(job.Id, poemNumber, "", false);
int poemId = (int)poem.id;
GanjoorPoem dbPoem = await context.GanjoorPoems.Include(p => p.Cat).Where(p => p.Id == poemId).SingleOrDefaultAsync();
if (dbPoem == null)
{
continue;
}
if (dbPoem.Cat.PoetId != poetId)
{
continue;
}
string comment = $"<p>تغییرات حاصل از پردازش {note}</p>{Environment.NewLine}";
bool anyChanges = false;
var dbPage = await context.GanjoorPages.Where(p => p.Id == poemId).SingleOrDefaultAsync();
GanjoorPageSnapshot snapshot = new GanjoorPageSnapshot()
{
GanjoorPageId = poemId,
MadeObsoleteByUserId = (Guid)userId,
RecordDate = DateTime.Now,
Note = note,
Title = dbPage.Title,
UrlSlug = dbPage.UrlSlug,
HtmlText = dbPage.HtmlText,
};
string poemTitle = poem.title;
if (poemTitle != dbPoem.Title)
{
anyChanges = true;
comment += $"<p>تغییر عنوان از «{dbPoem.Title}» به «{poemTitle}»</p>{Environment.NewLine}";
dbPoem.Title = poemTitle;
dbPoem.FullTitle = $"{dbPoem.Cat.FullUrl} » {dbPoem.Title}";
context.GanjoorPoems.Update(dbPoem);
}
var sqliteVerses = new List <dynamic>(await sqlite.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder"));
var dbVerses = await context.GanjoorVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
int vIndex = 0;
while (vIndex < sqliteVerses.Count && vIndex < dbVerses.Count)
{
if (sqliteVerses[vIndex].vorder != dbVerses[vIndex].VOrder)
{
vIndex = -1;
break;
}
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
if (text == dbVerses[vIndex].Text)
{
vIndex++;
continue;
}
comment += $"<p>تغییر مصرع {vIndex + 1} از «{dbVerses[vIndex].Text}» به «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
dbVerses[vIndex].Text = text;
context.GanjoorVerses.Update(dbVerses[vIndex]);
anyChanges = true;
vIndex++;
}
if (vIndex != -1)
{
while (vIndex < dbVerses.Count)
{
comment += $"<p>حذف مصرع {vIndex + 1} با متن «{dbVerses[vIndex].Text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.GanjoorVerses.Remove(dbVerses[vIndex]);
vIndex++;
anyChanges = true;
}
while (vIndex < sqliteVerses.Count)
{
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
int vOrder = int.Parse(sqliteVerses[vIndex].vorder.ToString());
int position = int.Parse(sqliteVerses[vIndex].position.ToString());
comment += $"<p>اضافه شدن مصرع {vIndex + 1} با متن «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.GanjoorVerses.Add
(
new GanjoorVerse()
{
PoemId = poemId,
VOrder = vOrder,
VersePosition = (VersePosition)position,
Text = text
}
);
vIndex++;
anyChanges = true;
}
if (anyChanges)
{
await _FillPoemCoupletIndices(context, poemId);
GanjoorComment sysComment = new GanjoorComment()
{
UserId = userId,
AuthorIpAddress = "127.0.0.1",
CommentDate = DateTime.Now,
HtmlComment = comment,
PoemId = poemId,
Status = PublishStatus.Published,
};
context.GanjoorComments.Add(sysComment);
context.GanjoorPageSnapshots.Add(snapshot);
await context.SaveChangesAsync();
var poemVerses = await context.GanjoorVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
bool needsNewVOrder = false;
for (int i = 0; i < poemVerses.Count; i++)
{
if (poemVerses[i].VOrder != (i + 1))
{
poemVerses[i].VOrder = i + 1;
needsNewVOrder = true;
}
}
if (needsNewVOrder)
{
context.GanjoorVerses.UpdateRange(poemVerses);
}
dbPoem.PlainText = PreparePlainText(poemVerses);
dbPoem.HtmlText = PrepareHtmlText(poemVerses);
dbPage.HtmlText = dbPoem.HtmlText;
dbPage.Title = dbPoem.Title;
dbPage.FullTitle = dbPoem.FullTitle;
try
{
var poemRhymeLettersRes = LanguageUtils.FindRhyme(poemVerses);
if (!string.IsNullOrEmpty(poemRhymeLettersRes.Rhyme))
{
dbPoem.RhymeLetters = poemRhymeLettersRes.Rhyme;
}
}
catch
{
}
context.GanjoorPoems.Update(dbPoem);
context.GanjoorPages.Update(dbPage);
await context.SaveChangesAsync();
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
File.Delete(filePath);
}
);
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
return(new RServiceResult <bool>(true));
}
/// <summary>
/// separate verses in poem.PlainText with Environment.NewLine instead of SPACE
/// </summary>
/// <param name="catId"></param>
/// <returns></returns>
public RServiceResult <bool> RegerneratePoemsPlainText(int catId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"RegerneratePoemsPlainText {catId}", "Query data")).Result;
try
{
var poems = catId == 0 ? await context.GanjoorPoems.ToArrayAsync() : await context.GanjoorPoems.Where(p => p.CatId == catId).ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Updating PlainText/Poem Html {catId}");
int percent = 0;
for (int i = 0; i < poems.Length; i++)
{
if (i * 100 / poems.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var poem = poems[i];
var verses = await context.GanjoorVerses.AsNoTracking().Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync();
poem.PlainText = PreparePlainText(verses);
poem.HtmlText = PrepareHtmlText(verses);
}
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Finalizing PlainText/Poem Html {catId}");
context.GanjoorPoems.UpdateRange(poems);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 50, $"Updating pages HTML {catId}");
//the following line always gets timeout, so it is being replaced by a loop
//await context.Database.ExecuteSqlRawAsync(
// "UPDATE p SET p.HtmlText = (SELECT poem.HtmlText FROM GanjoorPoems poem WHERE poem.Id = p.Id) FROM GanjoorPages p WHERE p.GanjoorPageType = 3 ");
foreach (var poem in poems)
{
var page = await context.GanjoorPages.Where(p => p.Id == poem.Id).SingleAsync();
page.HtmlText = poem.HtmlText;
context.GanjoorPages.Update(page);
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// build sitemap
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartBuildingSitemap()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("BuildSitemap", "Query data")).Result;
try
{
string xmlSitemap = Configuration.GetSection("Ganjoor")["SitemapLocation"];
if (File.Exists(xmlSitemap))
{
File.Delete(xmlSitemap);
}
string dir = Path.GetDirectoryName(xmlSitemap);
await jobProgressServiceEF.UpdateJob(job.Id, 0, "", false);
List <string> sitemaps = new List <string>();
string firstSitemap = Path.Combine(dir, $"1.xml");
sitemaps.Add(firstSitemap);
var urls = await context.GanjoorPages.Where(p => p.PoetId == null).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync();
urls.Add("/map");
urls.Add("/photos");
urls.Add("/faq");
urls.Remove("/audioclip");
urls.Remove("/simi");
urls.Remove("/tags");
urls.Remove("/amar");
urls.Remove("/tools");
urls.Remove("/embed");
WriteSitemap(firstSitemap,
urls
);
foreach (var poet in await context.GanjoorPoets.Where(p => p.Published).ToListAsync())
{
await jobProgressServiceEF.UpdateJob(job.Id, poet.Id, "", false);
string poetSitemap = Path.Combine(dir, $"{poet.Id}.xml");
WriteSitemap(poetSitemap,
await context.GanjoorPages.Where(p => p.PoetId == poet.Id).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync()
);
sitemaps.Add(poetSitemap);
}
XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9";
XNamespace xsiNs = "http://www.w3.org/2001/XMLSchema-instance";
XDocument xDoc = new XDocument(
new XDeclaration("1.0", "UTF-8", "no"),
new XElement(ns + "sitemapindex",
new XAttribute(XNamespace.Xmlns + "xsi", xsiNs),
new XAttribute(xsiNs + "schemaLocation",
"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"),
from sitemap in sitemaps
select new XElement(ns + "sitemap",
new XElement(ns + "loc", $"https://ganjoor.net/{Path.GetFileName(sitemap)}"))
)
);
xDoc.Save(xmlSitemap);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
private async Task DoInitializeRecords(RMuseumDbContext context)
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("DonationService::DoInitializeRecords", "Processing")).Result;
try
{
string htmlText = await context.GanjoorPages.Where(p => p.UrlSlug == "donate").Select(p => p.HtmlText).AsNoTracking().SingleAsync();
List <DonationPageRow> rows = new List <DonationPageRow>();
int nStartIndex = htmlText.IndexOf("<td class=\"ddate\">");
int rowNumber = 0;
while (nStartIndex != -1)
{
rowNumber++;
DonationPageRow row = new DonationPageRow();
nStartIndex += "<td class=\"ddate\">".Length;
row.Date = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"damount\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : damount");
return;
}
nStartIndex += "<td class=\"damount\">".Length;
row.Amount = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"ddonator\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : ddonator");
return;
}
nStartIndex += "<td class=\"ddonator\">".Length;
row.Donor = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"dusage\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : dusage");
return;
}
nStartIndex += "<td class=\"dusage\">".Length;
row.Usage = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"drem\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : drem");
return;
}
nStartIndex += "<td class=\"drem\">".Length;
row.Remaining = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
rows.Add(row);
nStartIndex = htmlText.IndexOf("<td class=\"ddate\">", nStartIndex);
}
DateTime recordDate = DateTime.Now.AddDays(-2);
for (int i = rows.Count - 1; i >= 0; i--)
{
GanjoorDonation donation = new GanjoorDonation()
{
ImportedRecord = true,
DateString = rows[i].Date,
RecordDate = recordDate,
AmountString = rows[i].Amount,
DonorName = rows[i].Donor, //needs to be cleaned from href values
ExpenditureDesc = rows[i].Usage,
Remaining = 0
};
if (
rows[i].Remaining != "۲۰ دلار" //one record
&&
rows[i].Remaining.ToEnglishNumbers() != "0"
)
{
donation.Remaining = decimal.Parse(rows[i].Remaining.Replace("٬", "").Replace("تومان", "").Trim().ToEnglishNumbers());
}
if (donation.AmountString.Contains("تومان"))
{
donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("تومان", "").Trim().ToEnglishNumbers());
donation.Unit = "تومان";
}
if (donation.AmountString.Contains("دلار"))
{
donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("دلار", "").Trim().ToEnglishNumbers());
donation.Unit = "دلار";
}
if (donation.ExpenditureDesc == "هنوز هزینه نشده.")
{
donation.ExpenditureDesc = "";
}
context.GanjoorDonations.Add(donation);
await context.SaveChangesAsync(); //in order to make Id columns filled in desired order
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
/// <summary>
/// start removing original images
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartRemovingOriginalImages()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("RemovingOriginalImages", "Removing")).Result;
try
{
var srcPath = Configuration.GetSection("PictureFileService")["StoragePath"];
var trashPath = Configuration.GetSection("PictureFileService")["TrashStoragePath"];
var images = await context.PictureFiles.Where(p => p.StoredFileName != null && p.SrcUrl != null && p.NormalSizeImageStoredFileName.IndexOf("orig") != 0).ToListAsync();
int progress = 0;
for (int i = 0; i < images.Count; i++)
{
var image = images[i];
string targetDir = Path.Combine(trashPath, image.FolderName);
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
Directory.CreateDirectory(Path.Combine(targetDir, "orig"));
}
string srcFileName = Path.Combine(Path.Combine(srcPath, image.FolderName), image.StoredFileName);
if (File.Exists(srcFileName))
{
string targetFileName = Path.Combine(Path.Combine(trashPath, image.FolderName), image.StoredFileName);
File.Move(srcFileName, targetFileName, true);
image.StoredFileName = null;
context.Update(image);
if ((i * 100 / images.Count) > progress)
{
progress = i * 100 / images.Count;
await jobProgressServiceEF.UpdateJob(job.Id, progress);
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// examine site pages for broken links
/// </summary>
/// <returns></returns>
public RServiceResult <bool> HealthCheckContents()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("HealthCheckContents", "Query data")).Result;
try
{
var pages = await context.GanjoorPages.ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Examining Pages");
var previousErrors = await context.GanjoorHealthCheckErrors.ToArrayAsync();
context.RemoveRange(previousErrors);
await context.SaveChangesAsync();
int percent = 0;
for (int i = 0; i < pages.Length; i++)
{
if (i * 100 / pages.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var hrefs = pages[i].HtmlText.Split(new[] { "href=\"" }, StringSplitOptions.RemoveEmptyEntries).Where(o => o.StartsWith("http")).Select(o => o.Substring(0, o.IndexOf("\"")));
foreach (string url in hrefs)
{
if (url == "https://ganjoor.net" || url == "https://ganjoor.net/" || url.IndexOf("https://ganjoor.net/vazn/?") == 0 || url.IndexOf("https://ganjoor.net/simi/?v") == 0)
{
continue;
}
if (url.IndexOf("http://ganjoor.net") == 0)
{
context.GanjoorHealthCheckErrors.Add
(
new GanjoorHealthCheckError()
{
ReferrerPageUrl = pages[i].FullUrl,
TargetUrl = url,
BrokenLink = false,
MulipleTargets = false
}
);
await context.SaveChangesAsync();
}
else
if (url.IndexOf("https://ganjoor.net") == 0)
{
var testUrl = url.Substring("https://ganjoor.net".Length);
if (testUrl[testUrl.Length - 1] == '/')
{
testUrl = testUrl.Substring(0, testUrl.Length - 1);
}
var pageCount = await context.GanjoorPages.Where(p => p.FullUrl == testUrl).CountAsync();
if (pageCount != 1)
{
context.GanjoorHealthCheckErrors.Add
(
new GanjoorHealthCheckError()
{
ReferrerPageUrl = pages[i].FullUrl,
TargetUrl = url,
BrokenLink = pageCount == 0,
MulipleTargets = pageCount != 0
}
);
await context.SaveChangesAsync();
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}