private async Task _FindCategoryPoemsRhythmsInternal(int catId, bool retag, string rhythm)
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"FindCategoryPoemsRhythms Cat {catId}", "Query data")).Result;
try
{
var metres = await context.GanjoorMetres.OrderBy(m => m.Rhythm).AsNoTracking().ToArrayAsync();
var rhythms = metres.Select(m => m.Rhythm).ToArray();
GanjoorMetre preDeterminedMetre = string.IsNullOrEmpty(rhythm) ? null : metres.Where(m => m.Rhythm == rhythm).Single();
var poems = await context.GanjoorPoems.Where(p => p.CatId == catId).ToListAsync();
int i = 0;
using (HttpClient httpClient = new HttpClient())
{
foreach (var poem in poems)
{
if (retag || poem.GanjoorMetreId == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, i ++);
if (preDeterminedMetre == null)
{
var res = await _FindPoemRhythm(poem.Id, context, httpClient, rhythms);
if (!string.IsNullOrEmpty(res.Result))
{
poem.GanjoorMetreId = metres.Where(m => m.Rhythm == res.Result).Single().Id;
context.GanjoorPoems.Update(poem);
await context.SaveChangesAsync();
}
}
else
{
poem.GanjoorMetreId = preDeterminedMetre.Id;
context.GanjoorPoems.Update(poem);
await context.SaveChangesAsync();
}
if (poem.GanjoorMetreId != null && !string.IsNullOrEmpty(poem.RhymeLetters))
{
await _UpdateRelatedPoems(context, (int)poem.GanjoorMetreId, poem.RhymeLetters);
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 99);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
/// <summary>
/// start filling GanjoorLink table OriginalSource values
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartFillingGanjoorLinkOriginalSources()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FillingGanjoorLinkOriginalSources", "Updating")).Result;
try
{
var links = await context.GanjoorLinks.ToListAsync();
for (int i = 0; i < links.Count; i++)
{
var link = links[i];
var itemInfo = await context.Items
.Include(i => i.Tags)
.ThenInclude(t => t.RTag)
.Where(i => i.Id == link.ItemId).SingleAsync();
var sourceTag = itemInfo.Tags.Where(t => t.RTag.FriendlyUrl == "source").FirstOrDefault();
if (sourceTag != null)
{
if (!string.IsNullOrEmpty(sourceTag.ValueSupplement) && (sourceTag.ValueSupplement.IndexOf("http") == 0))
{
link.OriginalSourceUrl = sourceTag.ValueSupplement;
link.LinkToOriginalSource = true;
context.GanjoorLinks.Update(link);
}
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start generating related poems info
/// </summary>
/// <param name="regenerate"></param>
/// <returns></returns>
public RServiceResult <bool> StartGeneratingRelatedPoemsInfo(bool regenerate)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("GeneratingRelatedPoemsInfo", "Query")).Result;
try
{
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Query");
var poemIds = await context.GanjoorPoems.AsNoTracking().Where(p => !string.IsNullOrEmpty(p.RhymeLetters) && p.GanjoorMetreId != null).Select(p => p.Id).ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Updating Related Poems");
int percent = 0;
for (int i = 0; i < poemIds.Count; i++)
{
if (!regenerate)
{
if (await context.GanjoorCachedRelatedPoems.AnyAsync(r => r.PoemId == poemIds[i]))
{
continue;
}
}
if (i * 100 / poemIds.Count > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
await _UpdatePoemRelatedPoemsInfoNoSaveChanges(context, poemIds[i]);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// examine comments for long links
/// </summary>
/// <returns></returns>
public RServiceResult <bool> FindAndFixLongUrlsInComments()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FindAndFixLongUrlsInComments", "Query data")).Result;
try
{
var comments = await context.GanjoorComments.Where(c => c.HtmlComment.Contains("href=")).ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Examining {comments.Length} Comments");
int percent = 0;
for (int i = 0; i < comments.Length; i++)
{
if (i * 100 / comments.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var comment = comments[i];
string commentText = await _ProcessCommentHtml(comment.HtmlComment, context);
if (commentText != comment.HtmlComment)
{
comment.HtmlComment = commentText;
context.Update(comment);
await context.SaveChangesAsync();
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
private async Task _GeneratingSubCatsTOC(Guid userId, RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job, int catId)
{
foreach (var cat in await context.GanjoorCategories.AsNoTracking().Where(c => c.ParentId == catId).ToListAsync())
{
await jobProgressServiceEF.UpdateJob(job.Id, cat.Id);
var page = await context.GanjoorPages.Where(p => p.FullUrl == cat.FullUrl).SingleAsync();
context.GanjoorPageSnapshots.Add
(
new GanjoorPageSnapshot()
{
GanjoorPageId = page.Id,
MadeObsoleteByUserId = userId,
HtmlText = page.HtmlText,
Note = "تولید گروهی فهرستهای زیربخشها",
RecordDate = DateTime.Now
}
);
page.HtmlText = (await _GenerateTableOfContents(context, cat.Id, GanjoorTOC.TitlesAndFirstVerse)).Result;
context.GanjoorPages.Update(page);
await context.SaveChangesAsync();
await _GeneratingSubCatsTOC(userId, context, jobProgressServiceEF, job, cat.Id);
}
}
/// <summary>
/// generate missing default numberings and start counting
/// </summary>
/// <returns></returns>
public RServiceResult <bool> GenerateMissingDefaultNumberings()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("GenerateMissingDefaultNumberings", "Query")).Result;
try
{
var cats = await context.GanjoorCategories.Where(c => c.ParentId != null).ToListAsync();
foreach (var cat in cats)
{
var numbering = await context.GanjoorNumberings.Where(n => n.StartCatId == cat.Id && n.Name == cat.Title).FirstOrDefaultAsync();
if (numbering == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, 0, cat.FullUrl);
numbering = new GanjoorNumbering()
{
Name = cat.Title,
StartCatId = cat.Id,
EndCatId = cat.Id
};
context.GanjoorNumberings.Add(numbering);
await context.SaveChangesAsync();
Recount(numbering.Id); //start counting
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start filling poems couplet indices
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartFillingPoemsCoupletIndices()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("FillingPoemsCoupletIndices", "Query data")).Result;
try
{
var poemIds = await context.GanjoorPoems.AsNoTracking().Select(p => p.Id).ToListAsync();
int percent = 0;
for (int i = 0; i < poemIds.Count; i++)
{
if (i * 100 / poemIds.Count > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
await _FillPoemCoupletIndices(context, poemIds[i]);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start updating stats page
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartUpdatingMundexPage(Guid editingUserId)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("UpdateMundexPage", "House Keeping")).Result;
try
{
await _PerformMundexHouseKeepingAndPreparation(context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 10, "Mundex Page");
await _UpdateMundexPage(editingUserId, context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 60, "Mundex by poet page");
await _UpdateMundexByPoetPage(editingUserId, context, jobProgressServiceEF, job);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start generating sub cats TOC
/// </summary>
/// <param name="userId"></param>
/// <param name="catId"></param>
/// <returns></returns>
public RServiceResult <bool> StartGeneratingSubCatsTOC(Guid userId, int catId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"GeneratingSubCatsTOC {catId}", "Query data")).Result;
try
{
await _GeneratingSubCatsTOC(userId, context, jobProgressServiceEF, job, catId);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return(new RServiceResult <bool>(true));
}
/// <summary>
/// import from sqlite
/// </summary>
/// <param name="poetId"></param>
/// <param name="file"></param>
/// <returns></returns>
public async Task <RServiceResult <bool> > ImportFromSqlite(int poetId, IFormFile file)
{
try
{
string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string filePath = Path.Combine(dir, file.FileName);
if (File.Exists(filePath))
{
File.Delete(filePath);
}
using (FileStream fsMain = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(fsMain);
}
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ImportFromSqlite", "Query data")).Result;
try
{
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
IDbConnection sqlite = sqliteConnection;
var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList();
if (poets.Count != 1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1");
}
var poet = await context.GanjoorPoets.AsNoTracking().Where(p => p.Id == poetId).SingleAsync();
var cat = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poetId && c.ParentId == null).SingleAsync();
var catPage = await context.GanjoorPages.AsNoTracking().Where(p => p.FullUrl == cat.FullUrl).SingleAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Importing");
var resImport = await _ImportSQLiteCatChildren(context, sqlite, poetId, await sqlite.QuerySingleAsync <int>($"SELECT id FROM cat WHERE parent_id = 0"), cat, poet.Nickname, jobProgressServiceEF, job, catPage.Id);
if (string.IsNullOrEmpty(resImport))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
else
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resImport);
}
}
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
File.Delete(filePath);
}
);
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
return(new RServiceResult <bool>(true));
}
/// <summary>
/// start generating gdb files
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartBatchGenerateGDBFiles()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("BatchGenerateGDBFiles", "Query Data")).Result;
try
{
string outDir = Configuration.GetSection("Ganjoor")["GDBStorage"];
string imgDir = Configuration.GetSection("Ganjoor")["GDBStorageImageSource"];
string xmlFile = Configuration.GetSection("Ganjoor")["GDBListXMLFile"];
string preExisitingListXMLFile = Configuration.GetSection("Ganjoor")["GDBPreExisitingListXMLFile"];
List <GDBInfo> programList = new List <GDBInfo>();
if (!string.IsNullOrEmpty(preExisitingListXMLFile))
{
if (File.Exists(preExisitingListXMLFile))
{
programList = GDBListProcessor.RetrieveListFromFile(preExisitingListXMLFile, out string exception);
if (programList == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "GDBListProcessor.RetrieveListFromFile", false, exception);
return;
}
}
}
var poets = await context.GanjoorPoets.AsNoTracking().ToListAsync();
List <GDBInfo> lstFiles = new List <GDBInfo>();
foreach (var poet in poets)
{
if (!await context.GanjoorPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id).AnyAsync())
{
continue;
}
await jobProgressServiceEF.UpdateJob(job.Id, poet.Id);
var gdbGeneration = await _ExportToSqlite(context, poet.Id, outDir, null, false);
if (!string.IsNullOrEmpty(gdbGeneration.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, gdbGeneration.ExceptionString);
return;
}
string gdbFile = gdbGeneration.Result;
string pngFile = Path.Combine(imgDir, $"{poet.Id}.png");
bool hasImage = File.Exists(pngFile);
using (var archiveStream = new MemoryStream())
{
using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, true))
{
var zipGDBFileEntry = archive.CreateEntry(Path.GetFileName(gdbFile), CompressionLevel.Optimal);
using (var zipStream = zipGDBFileEntry.Open())
{
var gdbBytes = File.ReadAllBytes(gdbFile);
zipStream.Write(gdbBytes, 0, gdbBytes.Length);
}
if (hasImage)
{
var zipImgFileEntry = archive.CreateEntry(Path.GetFileName(pngFile), CompressionLevel.Optimal);
using (var zipStream = zipImgFileEntry.Open())
{
var pngBytes = File.ReadAllBytes(pngFile);
zipStream.Write(pngBytes, 0, pngBytes.Length);
}
}
}
string zipFile = Path.Combine(outDir, Path.GetFileNameWithoutExtension(gdbFile) + ".zip");
if (File.Exists(zipFile))
{
File.Delete(zipFile);
}
byte[] zipArray = archiveStream.ToArray();
File.WriteAllBytes(zipFile, zipArray);
var catPoet = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poet.Id && c.ParentId == null).SingleAsync();
var lowestPoemID = await context.GanjoorPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id)
.MinAsync(p => p.Id);
lstFiles.Add
(
new GDBInfo()
{
CatName = poet.Nickname,
CatID = catPoet.Id,
PoetID = poet.Id,
DownloadUrl = $"http://i.ganjoor.net/android/gdb/{Path.GetFileName(zipFile)}",
BlogUrl = "",
FileExt = ".zip",
ImageUrl = (hasImage ? $"http://i.ganjoor.net/android/img/{poet.Id}.png" : ""),
FileSizeInByte = zipArray.Length,
LowestPoemID = lowestPoemID,
PubDate = DateTime.Now
}
);
}
File.Delete(gdbFile);
}
if (File.Exists(xmlFile))
{
File.Delete(xmlFile);
}
if (programList.Count > 0)
{
lstFiles.AddRange(programList);
}
lstFiles.Sort((a, b) => a.CatName.CompareTo(b.CatName));
GDBListProcessor.Save(xmlFile, "مجموعههای قابل دریافت از گنجور", "", "", lstFiles);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// Apply corrections from sqlite
/// </summary>
/// <param name="poetId"></param>
/// <param name="file"></param>
/// <param name="note"></param>
/// <returns></returns>
public async Task <RServiceResult <bool> > ApplyCorrectionsFromSqlite(int poetId, IFormFile file, string note)
{
try
{
string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string filePath = Path.Combine(dir, file.FileName);
if (File.Exists(filePath))
{
File.Delete(filePath);
}
using (FileStream fsMain = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(fsMain);
}
string email = $"{Configuration.GetSection("Ganjoor")["SystemEmail"]}";
var userId = (await _appUserService.FindUserByEmail(email)).Result.Id;
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ApplyCorrectionsFromSqlite", "Query data")).Result;
try
{
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
IDbConnection sqlite = sqliteConnection;
var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList();
if (poets.Count != 1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1");
}
int poemNumber = 0;
foreach (var poem in await sqlite.QueryAsync($"SELECT * FROM poem ORDER BY id"))
{
poemNumber++;
await jobProgressServiceEF.UpdateJob(job.Id, poemNumber, "", false);
int poemId = (int)poem.id;
GanjoorPoem dbPoem = await context.GanjoorPoems.Include(p => p.Cat).Where(p => p.Id == poemId).SingleOrDefaultAsync();
if (dbPoem == null)
{
continue;
}
if (dbPoem.Cat.PoetId != poetId)
{
continue;
}
string comment = $"<p>تغییرات حاصل از پردازش {note}</p>{Environment.NewLine}";
bool anyChanges = false;
var dbPage = await context.GanjoorPages.Where(p => p.Id == poemId).SingleOrDefaultAsync();
GanjoorPageSnapshot snapshot = new GanjoorPageSnapshot()
{
GanjoorPageId = poemId,
MadeObsoleteByUserId = (Guid)userId,
RecordDate = DateTime.Now,
Note = note,
Title = dbPage.Title,
UrlSlug = dbPage.UrlSlug,
HtmlText = dbPage.HtmlText,
};
string poemTitle = poem.title;
if (poemTitle != dbPoem.Title)
{
anyChanges = true;
comment += $"<p>تغییر عنوان از «{dbPoem.Title}» به «{poemTitle}»</p>{Environment.NewLine}";
dbPoem.Title = poemTitle;
dbPoem.FullTitle = $"{dbPoem.Cat.FullUrl} » {dbPoem.Title}";
context.GanjoorPoems.Update(dbPoem);
}
var sqliteVerses = new List <dynamic>(await sqlite.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder"));
var dbVerses = await context.GanjoorVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
int vIndex = 0;
while (vIndex < sqliteVerses.Count && vIndex < dbVerses.Count)
{
if (sqliteVerses[vIndex].vorder != dbVerses[vIndex].VOrder)
{
vIndex = -1;
break;
}
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
if (text == dbVerses[vIndex].Text)
{
vIndex++;
continue;
}
comment += $"<p>تغییر مصرع {vIndex + 1} از «{dbVerses[vIndex].Text}» به «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
dbVerses[vIndex].Text = text;
context.GanjoorVerses.Update(dbVerses[vIndex]);
anyChanges = true;
vIndex++;
}
if (vIndex != -1)
{
while (vIndex < dbVerses.Count)
{
comment += $"<p>حذف مصرع {vIndex + 1} با متن «{dbVerses[vIndex].Text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.GanjoorVerses.Remove(dbVerses[vIndex]);
vIndex++;
anyChanges = true;
}
while (vIndex < sqliteVerses.Count)
{
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
int vOrder = int.Parse(sqliteVerses[vIndex].vorder.ToString());
int position = int.Parse(sqliteVerses[vIndex].position.ToString());
comment += $"<p>اضافه شدن مصرع {vIndex + 1} با متن «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.GanjoorVerses.Add
(
new GanjoorVerse()
{
PoemId = poemId,
VOrder = vOrder,
VersePosition = (VersePosition)position,
Text = text
}
);
vIndex++;
anyChanges = true;
}
if (anyChanges)
{
await _FillPoemCoupletIndices(context, poemId);
GanjoorComment sysComment = new GanjoorComment()
{
UserId = userId,
AuthorIpAddress = "127.0.0.1",
CommentDate = DateTime.Now,
HtmlComment = comment,
PoemId = poemId,
Status = PublishStatus.Published,
};
context.GanjoorComments.Add(sysComment);
context.GanjoorPageSnapshots.Add(snapshot);
await context.SaveChangesAsync();
var poemVerses = await context.GanjoorVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
bool needsNewVOrder = false;
for (int i = 0; i < poemVerses.Count; i++)
{
if (poemVerses[i].VOrder != (i + 1))
{
poemVerses[i].VOrder = i + 1;
needsNewVOrder = true;
}
}
if (needsNewVOrder)
{
context.GanjoorVerses.UpdateRange(poemVerses);
}
dbPoem.PlainText = PreparePlainText(poemVerses);
dbPoem.HtmlText = PrepareHtmlText(poemVerses);
dbPage.HtmlText = dbPoem.HtmlText;
dbPage.Title = dbPoem.Title;
dbPage.FullTitle = dbPoem.FullTitle;
try
{
var poemRhymeLettersRes = LanguageUtils.FindRhyme(poemVerses);
if (!string.IsNullOrEmpty(poemRhymeLettersRes.Rhyme))
{
dbPoem.RhymeLetters = poemRhymeLettersRes.Rhyme;
}
}
catch
{
}
context.GanjoorPoems.Update(dbPoem);
context.GanjoorPages.Update(dbPage);
await context.SaveChangesAsync();
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
File.Delete(filePath);
}
);
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
return(new RServiceResult <bool>(true));
}
private async Task DoInitializeRecords(RMuseumDbContext context)
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("DonationService::DoInitializeRecords", "Processing")).Result;
try
{
string htmlText = await context.GanjoorPages.Where(p => p.UrlSlug == "donate").Select(p => p.HtmlText).AsNoTracking().SingleAsync();
List <DonationPageRow> rows = new List <DonationPageRow>();
int nStartIndex = htmlText.IndexOf("<td class=\"ddate\">");
int rowNumber = 0;
while (nStartIndex != -1)
{
rowNumber++;
DonationPageRow row = new DonationPageRow();
nStartIndex += "<td class=\"ddate\">".Length;
row.Date = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"damount\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : damount");
return;
}
nStartIndex += "<td class=\"damount\">".Length;
row.Amount = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"ddonator\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : ddonator");
return;
}
nStartIndex += "<td class=\"ddonator\">".Length;
row.Donor = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"dusage\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : dusage");
return;
}
nStartIndex += "<td class=\"dusage\">".Length;
row.Usage = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
nStartIndex = htmlText.IndexOf("<td class=\"drem\">", nStartIndex);
if (nStartIndex == -1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, $"{rowNumber} : drem");
return;
}
nStartIndex += "<td class=\"drem\">".Length;
row.Remaining = htmlText.Substring(nStartIndex, htmlText.IndexOf("</td>", nStartIndex) - nStartIndex);
rows.Add(row);
nStartIndex = htmlText.IndexOf("<td class=\"ddate\">", nStartIndex);
}
DateTime recordDate = DateTime.Now.AddDays(-2);
for (int i = rows.Count - 1; i >= 0; i--)
{
GanjoorDonation donation = new GanjoorDonation()
{
ImportedRecord = true,
DateString = rows[i].Date,
RecordDate = recordDate,
AmountString = rows[i].Amount,
DonorName = rows[i].Donor, //needs to be cleaned from href values
ExpenditureDesc = rows[i].Usage,
Remaining = 0
};
if (
rows[i].Remaining != "۲۰ دلار" //one record
&&
rows[i].Remaining.ToEnglishNumbers() != "0"
)
{
donation.Remaining = decimal.Parse(rows[i].Remaining.Replace("٬", "").Replace("تومان", "").Trim().ToEnglishNumbers());
}
if (donation.AmountString.Contains("تومان"))
{
donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("تومان", "").Trim().ToEnglishNumbers());
donation.Unit = "تومان";
}
if (donation.AmountString.Contains("دلار"))
{
donation.Amount = decimal.Parse(donation.AmountString.Replace("٬", "").Replace("دلار", "").Trim().ToEnglishNumbers());
donation.Unit = "دلار";
}
if (donation.ExpenditureDesc == "هنوز هزینه نشده.")
{
donation.ExpenditureDesc = "";
}
context.GanjoorDonations.Add(donation);
await context.SaveChangesAsync(); //in order to make Id columns filled in desired order
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
/// <summary>
/// start updating stats page
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartUpdatingStatsPage(Guid editingUserId)
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("UpdateStatsPage", "Total Poets Stats")).Result;
try
{
var poetsCoupletCounts =
await context.GanjoorVerses.Include(v => v.Poem).ThenInclude(p => p.Cat).ThenInclude(c => c.Poet).AsNoTracking()
.Where(v =>
v.Poem.Cat.Poet.Published
&&
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1))
.GroupBy(v => new { v.Poem.Cat.PoetId })
.Select(g => new { PoetId = g.Key.PoetId, Count = g.Count() })
.ToListAsync();
poetsCoupletCounts.Sort((a, b) => b.Count - a.Count);
var sumPoetsCouplets = poetsCoupletCounts.Sum(c => c.Count);
var rhythmsCoupletCounts =
await context.GanjoorVerses.Include(v => v.Poem).ThenInclude(p => p.Cat).ThenInclude(c => c.Poet).AsNoTracking()
.Where(v =>
v.Poem.Cat.Poet.Published
&&
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1))
.GroupBy(v => new { v.Poem.GanjoorMetreId })
.Select(g => new { GanjoorMetreId = g.Key.GanjoorMetreId, Count = g.Count() })
.ToListAsync();
rhythmsCoupletCounts.Sort((a, b) => b.Count - a.Count);
var sumRhythmsCouplets = rhythmsCoupletCounts.Sum(c => c.Count);
var dbPage = await context.GanjoorPages.Where(p => p.FullUrl == "/vazn").SingleAsync();
var poets = await context.GanjoorPoets.ToListAsync();
string htmlText = $"<p>تا تاریخ {LanguageUtils.FormatDate(DateTime.Now)} مجموعاً {LanguageUtils.FormatMoney(sumPoetsCouplets)} بیت شعر از طریق سایت گنجور در دسترس قرار گرفته است. در جدول زیر که شاعران در آنها بر اساس تعداد ابیات اشعارشان به صورت نزولی مرتب شدهاند با کلیک بر روی نام هر شاعر میتوانید آمار اوزان اشعار او را مشاهده کنید.</p>{Environment.NewLine}";
htmlText += $"<p>توجه فرمایید که این آمار به دلایلی از قبیل وجود چند نسخه از آثار شعرا در سایت (مثل آثار خیام یا وجود متن خلاصه و کامل هفت اورنگ جامی) و همینطور یک بیت محسوب شدن مصرعهای بند قالبهای ترکیبی مثل مخمسها تقریبی و حدودی است و افزونگی دارد.</p>{Environment.NewLine}";
htmlText += $"<table>{Environment.NewLine}" +
$"<tr class=\"h\">{Environment.NewLine}" +
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
$"<td class=\"c2\">شاعر</td>{Environment.NewLine}" +
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
$"</tr>{Environment.NewLine}";
for (int i = 0; i < poetsCoupletCounts.Count; i++)
{
if (i % 2 == 0)
{
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
}
else
{
htmlText += $"<tr>{Environment.NewLine}";
}
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c2\"><a href=\"{(await context.GanjoorCategories.Where(c => c.ParentId == null && c.PoetId == poetsCoupletCounts[i].PoetId).SingleAsync()).FullUrl}/vazn\">{poets.Where(p => p.Id == poetsCoupletCounts[i].PoetId).Single().Nickname}</a></td>{Environment.NewLine}";
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(poetsCoupletCounts[i].Count)}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c4\">{(poetsCoupletCounts[i].Count * 100.0 / sumPoetsCouplets).ToString("N2", new CultureInfo("fa-IR")).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"</tr>{Environment.NewLine}";
}
htmlText += $"</table>{Environment.NewLine}";
var rhythms = await context.GanjoorMetres.ToListAsync();
htmlText += $"<p>فهرست زیر نیز آمار اشعار گنجور را از لحاظ اوزان عروضی نشان میدهد:</p>{Environment.NewLine}";
htmlText += $"<table>{Environment.NewLine}" +
$"<tr class=\"h\">{Environment.NewLine}" +
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
$"<td class=\"c2\">وزن</td>{Environment.NewLine}" +
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
$"</tr>{Environment.NewLine}";
for (int i = 0; i < rhythmsCoupletCounts.Count; i++)
{
if (i % 2 == 0)
{
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
}
else
{
htmlText += $"<tr>{Environment.NewLine}";
}
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
var rhythm = rhythms.Where(r => r.Id == rhythmsCoupletCounts[i].GanjoorMetreId).SingleOrDefault();
if (rhythm != null)
{
rhythm.VerseCount = rhythmsCoupletCounts[i].Count;
context.Update(rhythm);
}
string rhythmName = rhythmsCoupletCounts[i].GanjoorMetreId == null ? "وزنیابی نشده" :
$"<a href=\"/vazn/?v={Uri.EscapeDataString(rhythm.Rhythm)}\">{rhythms.Where(r => r.Id == rhythmsCoupletCounts[i].GanjoorMetreId).Single().Rhythm}</a>";
htmlText += $"<td class=\"c2\">{rhythmName}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(rhythmsCoupletCounts[i].Count)}</td>{Environment.NewLine}";
htmlText += $"<td class=\"c4\">{(rhythmsCoupletCounts[i].Count * 100.0 / sumRhythmsCouplets).ToString("N2", new CultureInfo("fa-IR")).ToPersianNumbers()}</td>{Environment.NewLine}";
htmlText += $"</tr>{Environment.NewLine}";
}
htmlText += $"</table>{Environment.NewLine}";
await context.SaveChangesAsync(); //store rhythm[s].VerseCount
await _UpdatePageHtmlText(context, editingUserId, dbPage, "به روزرسانی خودکار صفحهٔ آمار وزنها", htmlText);
foreach (var poetInfo in poetsCoupletCounts)
{
var poet = poets.Where(p => p.Id == poetInfo.PoetId).Single();
await jobProgressServiceEF.UpdateJob(job.Id, poetInfo.PoetId, poet.Nickname);
await _UpdatePoetStatsPage(editingUserId, poet, rhythms, context);
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// build sitemap
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartBuildingSitemap()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("BuildSitemap", "Query data")).Result;
try
{
string xmlSitemap = Configuration.GetSection("Ganjoor")["SitemapLocation"];
if (File.Exists(xmlSitemap))
{
File.Delete(xmlSitemap);
}
string dir = Path.GetDirectoryName(xmlSitemap);
await jobProgressServiceEF.UpdateJob(job.Id, 0, "", false);
List <string> sitemaps = new List <string>();
string firstSitemap = Path.Combine(dir, $"1.xml");
sitemaps.Add(firstSitemap);
var urls = await context.GanjoorPages.Where(p => p.PoetId == null).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync();
urls.Add("/map");
urls.Add("/photos");
urls.Add("/faq");
urls.Remove("/audioclip");
urls.Remove("/simi");
urls.Remove("/tags");
urls.Remove("/amar");
urls.Remove("/tools");
urls.Remove("/embed");
WriteSitemap(firstSitemap,
urls
);
foreach (var poet in await context.GanjoorPoets.Where(p => p.Published).ToListAsync())
{
await jobProgressServiceEF.UpdateJob(job.Id, poet.Id, "", false);
string poetSitemap = Path.Combine(dir, $"{poet.Id}.xml");
WriteSitemap(poetSitemap,
await context.GanjoorPages.Where(p => p.PoetId == poet.Id).OrderBy(p => p.Id).Select(p => p.FullUrl).ToListAsync()
);
sitemaps.Add(poetSitemap);
}
XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9";
XNamespace xsiNs = "http://www.w3.org/2001/XMLSchema-instance";
XDocument xDoc = new XDocument(
new XDeclaration("1.0", "UTF-8", "no"),
new XElement(ns + "sitemapindex",
new XAttribute(XNamespace.Xmlns + "xsi", xsiNs),
new XAttribute(xsiNs + "schemaLocation",
"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"),
from sitemap in sitemaps
select new XElement(ns + "sitemap",
new XElement(ns + "loc", $"https://ganjoor.net/{Path.GetFileName(sitemap)}"))
)
);
xDoc.Save(xmlSitemap);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
private async Task <RServiceResult <bool> > _ImportCommentsDataFromMySql(string jobName, RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job)
{
try
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - processing reported comments")).Result;
List <GanjoorCommentAbuseReport> reportedComments = await _MySqlImportReportedComments();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - pre mysql data fetch")).Result;
string connectionString =
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True";
using (MySqlConnection connection = new MySqlConnection
(
connectionString
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT comment_ID, comment_post_ID, comment_author, comment_author_email, comment_author_url, comment_author_IP, comment_date, comment_content, comment_approved FROM ganja_comments WHERE comment_type <> 'pingback' ORDER BY comment_ID",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - mysql")).Result;
using (DataTable data = new DataTable())
{
await src.FillAsync(data);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - processing approved poem songs")).Result;
int count = data.Rows.Count;
int i = 0;
int percent = -1;
foreach (DataRow row in data.Rows)
{
GanjoorComment comment = new GanjoorComment()
{
PoemId = int.Parse(row["comment_post_ID"].ToString()),
AuthorName = row["comment_author"].ToString(),
AuthorEmail = row["comment_author_email"].ToString(),
AuthorUrl = row["comment_author_url"].ToString(),
AuthorIpAddress = row["comment_author_IP"].ToString(),
CommentDate = (DateTime)row["comment_date"],
HtmlComment = _PrepareCommentHtml(row["comment_content"].ToString()),
Status = row["comment_approved"].ToString() == "1" ? PublishStatus.Published : PublishStatus.Awaiting
};
var poem = await context.GanjoorPoems.Where(p => p.Id == comment.PoemId).SingleOrDefaultAsync();
if (poem == null)
{
continue;
}
context.GanjoorComments.Add(comment);
int originalCommentId = int.Parse(row["comment_post_ID"].ToString());
var complaints = reportedComments.Where(c => c.GanjoorCommentId == originalCommentId).ToList();
if (complaints.Count > 0)
{
await context.SaveChangesAsync(); //save this comment to make its ID valid
foreach (var complaint in complaints)
{
context.GanjoorReportedComments.Add
(
new GanjoorCommentAbuseReport()
{
GanjoorCommentId = comment.Id,
ReasonCode = complaint.ReasonCode,
ReasonText = complaint.ReasonText,
}
);
}
}
i++;
if (i * 100 / count > percent)
{
percent = i * 100 / count;
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - {i} of {count}")).Result;
}
}
await context.SaveChangesAsync();
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - assigning comments to users")).Result;
foreach (var user in await context.Users.ToListAsync())
{
foreach (var comment in await context.GanjoorComments.Where(u => u.AuthorEmail == user.Email).ToListAsync())
{
comment.UserId = user.Id;
context.GanjoorComments.Update(comment);
}
}
await context.SaveChangesAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - finished")).Result;
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, exp.ToString());
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
private async Task <RServiceResult <bool> > _ImportPoemSongsDataFromMySql(string jobName, RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job, bool approved)
{
try
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - pre mysql data fetch")).Result;
string connectionString =
approved ?
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
:
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["SongsUsername"]};pwd={Configuration.GetSection("AudioMySqlServer")["SongsPassword"]};database={Configuration.GetSection("AudioMySqlServer")["SongsDatabase"]};charset=utf8;convert zero datetime=True";
using (MySqlConnection connection = new MySqlConnection
(
connectionString
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT poem_id, artist_name, artist_beeptunesurl, album_name, album_beeptunesurl, track_name, track_beeptunesurl, ptrack_typeid FROM ganja_ptracks ORDER BY id",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - mysql")).Result;
using (DataTable data = new DataTable())
{
await src.FillAsync(data);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - processing approved poem songs")).Result;
foreach (DataRow row in data.Rows)
{
PoemMusicTrack track = new PoemMusicTrack()
{
TrackType = (PoemMusicTrackType)int.Parse(row["ptrack_typeid"].ToString()),
PoemId = int.Parse(row["poem_id"].ToString()),
ArtistName = row["artist_name"].ToString(),
ArtistUrl = row["artist_beeptunesurl"].ToString(),
AlbumName = row["album_name"].ToString(),
AlbumUrl = row["album_beeptunesurl"].ToString(),
TrackName = row["track_name"].ToString(),
TrackUrl = row["track_beeptunesurl"].ToString(),
ApprovalDate = DateTime.Now,
Description = "",
Approved = approved
};
var poem = await context.GanjoorPoems.Where(p => p.Id == track.PoemId).SingleOrDefaultAsync();
if (poem == null)
{
continue;
}
switch (track.TrackType)
{
case PoemMusicTrackType.BeepTunesOrKhosousi:
case PoemMusicTrackType.iTunes:
{
GanjoorTrack catalogueTrack = await context.GanjoorMusicCatalogueTracks.Where(m => m.Url == track.TrackUrl).FirstOrDefaultAsync();
if (catalogueTrack != null)
{
track.GanjoorTrackId = catalogueTrack.Id;
}
GanjoorSinger singer = await context.GanjoorSingers.Where(s => s.Url == track.ArtistUrl).FirstOrDefaultAsync();
if (singer != null)
{
track.SingerId = singer.Id;
}
}
break;
case PoemMusicTrackType.Golha:
{
track.AlbumName = $"{track.ArtistName} » {track.AlbumName}";
track.ArtistName = "";
track.GolhaTrackId = int.Parse(track.ArtistUrl);
track.ArtistUrl = "";
}
break;
}
context.GanjoorPoemMusicTracks.Add(track);
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - finalizing approved poem songs data")).Result;
await context.SaveChangesAsync();
}
}
}
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, exp.ToString());
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start counting
/// </summary>
/// <returns></returns>
public RServiceResult <bool> Recount(int numberingId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("NumberingService::Count", "Query Cats")).Result;
try
{
var numbering = await context.GanjoorNumberings.Include(n => n.StartCat).Where(n => n.Id == numberingId).SingleOrDefaultAsync();
var cats = await _FindTargetCategories(context, numbering.StartCat, numbering.EndCatId);
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Deleting Old Data");
var oldNumbers = await context.GanjoorVerseNumbers.Where(n => n.NumberingId == numberingId).ToListAsync();
context.GanjoorVerseNumbers.RemoveRange(oldNumbers);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, "Counting");
int number = 0;
int coupletnumber = 0;
int paragraphnumber = 0;
int totalVerseCount = 0;
foreach (var cat in cats)
{
var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == cat.Id).OrderBy(p => p.Id).ToListAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Counting:: {cat.Title}");
foreach (var poem in poems)
{
totalVerseCount += await context.GanjoorVerses.Where(v => v.PoemId == poem.Id).CountAsync();
var verses = await context.GanjoorVerses.AsNoTracking()
.Where(v => v.PoemId == poem.Id && v.VersePosition != VersePosition.Left && v.VersePosition != VersePosition.CenteredVerse2 && v.VersePosition != VersePosition.Comment)
.OrderBy(v => v.VOrder)
.ToListAsync();
for (int coupletIndex = 0; coupletIndex < verses.Count; coupletIndex++)
{
number++;
bool isPoemVerse = verses[coupletIndex].VersePosition == VersePosition.Right || verses[coupletIndex].VersePosition == VersePosition.CenteredVerse1;
if (isPoemVerse)
{
coupletnumber++;
}
else
{
paragraphnumber++;
}
GanjoorVerseNumber verseNumber = new GanjoorVerseNumber()
{
NumberingId = numberingId,
PoemId = poem.Id,
CoupletIndex = coupletIndex,
Number = number,
IsPoemVerse = isPoemVerse,
SameTypeNumber = isPoemVerse ? coupletnumber : paragraphnumber
};
context.GanjoorVerseNumbers.Add(verseNumber);
}
}
}
numbering.TotalLines = number;
numbering.TotalVerses = totalVerseCount;
numbering.TotalCouplets = coupletnumber;
numbering.TotalParagraphs = paragraphnumber;
numbering.LastCountingDate = DateTime.Now;
context.GanjoorNumberings.Update(numbering);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
private async Task <string> _ImportSQLiteCatChildren(RMuseumDbContext context, IDbConnection sqlite, int poetId, int sqliteParentCatId, GanjoorCat parentCat, string parentFullTitle, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job, int parentPagId)
{
try
{
string catHtmlText = "";
foreach (var cat in await sqlite.QueryAsync($"SELECT * FROM cat WHERE parent_id = {sqliteParentCatId} ORDER BY id"))
{
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Importing - {cat.text}");
var poetCatId = 1 + await context.GanjoorCategories.MaxAsync(c => c.Id);
string catTitle = cat.text;
string url = GPersianTextSync.Farglisize(catTitle);
if (catTitle.IndexOf('|') != -1)
{
string[] catParts = catTitle.Split('|', StringSplitOptions.RemoveEmptyEntries);
if (catParts.Length == 2)
{
catTitle = catParts[0].Trim();
url = catParts[1].Trim();
}
}
else
{
switch (catTitle)
{
case "دیوان اشعار":
url = "divan";
break;
case "قصاید":
case "قصائد":
case "قصیده":
case "قصیده ها":
url = "ghaside";
break;
case "غزلیات":
case "غزل":
case "غزل ها":
url = "ghazal";
break;
case "قطعات":
case "مقطعات":
case "قطعه":
url = "ghete";
break;
case "مثنویات":
case "مثنوی":
case "مثنوی ها":
url = "masnavi";
break;
case "ترکیبات":
case "ترکیب بند":
url = "tarkib";
break;
case "ترجیعات":
case "ترجیع بند":
url = "tarjee";
break;
case "مسمطات":
case "مسمط":
url = "mosammat";
break;
case "مخمسات":
case "مخمس":
url = "mokhammas";
break;
case "رباعیات":
case "رباعی":
case "رباعی ها":
url = "robaee";
break;
case "ملمعات":
case "ملمع":
url = "molamma";
break;
case "هجویات":
case "هجو":
url = "hajv";
break;
case "هزلیات":
case "هزل":
url = "hazl";
break;
case "مراثی":
case "مرثیه":
case "رثا":
case "مرثیه ها":
url = "marsie";
break;
case "مفردات":
url = "mofradat";
break;
case "ملحقات":
url = "molhaghat";
break;
case "اشعار عربی":
url = "arabi";
break;
case "ماده تاریخها":
case "ماده تاریخها":
case "ماده تاریخ":
url = "tarikh";
break;
case "معمیات":
url = "moammiyat";
break;
case "چیستان":
url = "chistan";
break;
case "لغز":
case "لغزها":
url = "loghaz";
break;
}
}
GanjoorCat dbCat = new GanjoorCat()
{
Id = poetCatId,
PoetId = poetId,
Title = catTitle,
UrlSlug = url,
FullUrl = $"{parentCat.FullUrl}/{url}",
ParentId = parentCat.Id,
TableOfContentsStyle = GanjoorTOC.Analyse,
Published = true,
};
context.GanjoorCategories.Add(dbCat);
var maxPageId = await context.GanjoorPages.MaxAsync(p => p.Id);
if (await context.GanjoorPoems.MaxAsync(p => p.Id) > maxPageId)
{
maxPageId = await context.GanjoorPoems.MaxAsync(p => p.Id);
}
var catPageId = 1 + maxPageId;
while (await context.GanjoorPoems.Where(p => p.Id == catPageId).AnyAsync())
{
catPageId++;
}
GanjoorPage dbPageCat = new GanjoorPage()
{
Id = catPageId,
GanjoorPageType = GanjoorPageType.CatPage,
Published = false,
PageOrder = -1,
Title = dbCat.Title,
FullTitle = $"{parentFullTitle} » {dbCat.Title}",
UrlSlug = dbCat.UrlSlug,
FullUrl = dbCat.FullUrl,
HtmlText = "",
PoetId = poetId,
CatId = poetCatId,
PostDate = DateTime.Now,
ParentId = parentPagId
};
context.GanjoorPages.Add(dbPageCat);
await context.SaveChangesAsync();
catHtmlText += $"<p><a href=\"{dbCat.FullUrl}\">{dbCat.Title}</a></p>{Environment.NewLine}";
var resChild = await _ImportSQLiteCatChildren(context, sqlite, poetId, (int)cat.id, dbCat, $"{parentFullTitle} » {dbCat.Title}", jobProgressServiceEF, job, dbPageCat.Id);
if (!string.IsNullOrEmpty(resChild))
{
return(resChild);
}
}
var maxPoemId = await context.GanjoorPoems.MaxAsync(p => p.Id);
if (await context.GanjoorPages.MaxAsync(p => p.Id) > maxPoemId)
{
maxPoemId = await context.GanjoorPages.MaxAsync(p => p.Id);
}
var poemId = 1 + maxPoemId;
int poemNumber = 0;
foreach (var poem in await sqlite.QueryAsync($"SELECT * FROM poem WHERE cat_id = {sqliteParentCatId} ORDER BY id"))
{
poemNumber++;
await jobProgressServiceEF.UpdateJob(job.Id, poemNumber, "", false);
string title = poem.title;
string urlSlug = $"sh{poemNumber}";
if (title.IndexOf('|') != -1)
{
string[] titleParts = title.Split('|', StringSplitOptions.RemoveEmptyEntries);
if (titleParts.Length == 2)
{
title = titleParts[0].Trim();
urlSlug = titleParts[1].Trim();
}
}
GanjoorPoem dbPoem = new GanjoorPoem()
{
Id = poemId,
CatId = parentCat.Id,
Title = title,
UrlSlug = urlSlug,
FullTitle = $"{parentFullTitle} » {title}",
FullUrl = $"{parentCat.FullUrl}/{urlSlug}",
Published = true,
};
List <GanjoorVerse> poemVerses = new List <GanjoorVerse>();
foreach (var verse in await sqlite.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder"))
{
int vOrder = int.Parse(verse.vorder.ToString());
int position = int.Parse(verse.position.ToString());
string text = verse.text;
GanjoorVerse dbVerse = new GanjoorVerse()
{
PoemId = poemId,
VOrder = vOrder,
VersePosition = (VersePosition)position,
Text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim()
};
poemVerses.Add(dbVerse);
}
if (poemVerses.Count == 0)
{
poemNumber--;
continue;
}
dbPoem.PlainText = PreparePlainText(poemVerses);
dbPoem.HtmlText = PrepareHtmlText(poemVerses);
context.GanjoorPoems.Add(dbPoem);
await context.SaveChangesAsync();
foreach (var dbVerse in poemVerses)
{
context.GanjoorVerses.Add(dbVerse);
await context.SaveChangesAsync();//id set should be in order
}
await _FillPoemCoupletIndices(context, poemId);
try
{
var poemRhymeLettersRes = LanguageUtils.FindRhyme(poemVerses);
if (!string.IsNullOrEmpty(poemRhymeLettersRes.Rhyme))
{
dbPoem.RhymeLetters = poemRhymeLettersRes.Rhyme;
context.GanjoorPoems.Update(dbPoem);
}
}
catch
{
}
GanjoorPage dbPoemPage = new GanjoorPage()
{
Id = poemId,
GanjoorPageType = GanjoorPageType.PoemPage,
Published = false,
PageOrder = -1,
Title = dbPoem.Title,
FullTitle = dbPoem.FullTitle,
UrlSlug = dbPoem.UrlSlug,
FullUrl = dbPoem.FullUrl,
HtmlText = dbPoem.HtmlText,
PoetId = poetId,
CatId = parentCat.Id,
PoemId = poemId,
PostDate = DateTime.Now,
ParentId = parentPagId
};
context.GanjoorPages.Add(dbPoemPage);
await context.SaveChangesAsync();
catHtmlText += $"<p><a href=\"{dbPoemPage.FullUrl}\">{dbPoemPage.Title}</a></p>{Environment.NewLine}";
poemId++;
}
if (!string.IsNullOrEmpty(catHtmlText))
{
var parentCatPage = await context.GanjoorPages.Where(p => p.FullUrl == parentCat.FullUrl).SingleAsync();
parentCatPage.HtmlText += catHtmlText;
context.GanjoorPages.Update(parentCatPage);
}
await context.SaveChangesAsync();
}
catch (Exception exp)
{
return(exp.ToString());
}
return("");
}
/// <summary>
/// examine site pages for broken links
/// </summary>
/// <returns></returns>
public RServiceResult <bool> HealthCheckContents()
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("HealthCheckContents", "Query data")).Result;
try
{
var pages = await context.GanjoorPages.ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Examining Pages");
var previousErrors = await context.GanjoorHealthCheckErrors.ToArrayAsync();
context.RemoveRange(previousErrors);
await context.SaveChangesAsync();
int percent = 0;
for (int i = 0; i < pages.Length; i++)
{
if (i * 100 / pages.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var hrefs = pages[i].HtmlText.Split(new[] { "href=\"" }, StringSplitOptions.RemoveEmptyEntries).Where(o => o.StartsWith("http")).Select(o => o.Substring(0, o.IndexOf("\"")));
foreach (string url in hrefs)
{
if (url == "https://ganjoor.net" || url == "https://ganjoor.net/" || url.IndexOf("https://ganjoor.net/vazn/?") == 0 || url.IndexOf("https://ganjoor.net/simi/?v") == 0)
{
continue;
}
if (url.IndexOf("http://ganjoor.net") == 0)
{
context.GanjoorHealthCheckErrors.Add
(
new GanjoorHealthCheckError()
{
ReferrerPageUrl = pages[i].FullUrl,
TargetUrl = url,
BrokenLink = false,
MulipleTargets = false
}
);
await context.SaveChangesAsync();
}
else
if (url.IndexOf("https://ganjoor.net") == 0)
{
var testUrl = url.Substring("https://ganjoor.net".Length);
if (testUrl[testUrl.Length - 1] == '/')
{
testUrl = testUrl.Substring(0, testUrl.Length - 1);
}
var pageCount = await context.GanjoorPages.Where(p => p.FullUrl == testUrl).CountAsync();
if (pageCount != 1)
{
context.GanjoorHealthCheckErrors.Add
(
new GanjoorHealthCheckError()
{
ReferrerPageUrl = pages[i].FullUrl,
TargetUrl = url,
BrokenLink = pageCount == 0,
MulipleTargets = pageCount != 0
}
);
await context.SaveChangesAsync();
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// separate verses in poem.PlainText with Environment.NewLine instead of SPACE
/// </summary>
/// <param name="catId"></param>
/// <returns></returns>
public RServiceResult <bool> RegerneratePoemsPlainText(int catId)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob($"RegerneratePoemsPlainText {catId}", "Query data")).Result;
try
{
var poems = catId == 0 ? await context.GanjoorPoems.ToArrayAsync() : await context.GanjoorPoems.Where(p => p.CatId == catId).ToArrayAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Updating PlainText/Poem Html {catId}");
int percent = 0;
for (int i = 0; i < poems.Length; i++)
{
if (i * 100 / poems.Length > percent)
{
percent++;
await jobProgressServiceEF.UpdateJob(job.Id, percent);
}
var poem = poems[i];
var verses = await context.GanjoorVerses.AsNoTracking().Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync();
poem.PlainText = PreparePlainText(verses);
poem.HtmlText = PrepareHtmlText(verses);
}
await jobProgressServiceEF.UpdateJob(job.Id, 0, $"Finalizing PlainText/Poem Html {catId}");
context.GanjoorPoems.UpdateRange(poems);
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 50, $"Updating pages HTML {catId}");
//the following line always gets timeout, so it is being replaced by a loop
//await context.Database.ExecuteSqlRawAsync(
// "UPDATE p SET p.HtmlText = (SELECT poem.HtmlText FROM GanjoorPoems poem WHERE poem.Id = p.Id) FROM GanjoorPages p WHERE p.GanjoorPageType = 3 ");
foreach (var poem in poems)
{
var page = await context.GanjoorPages.Where(p => p.Id == poem.Id).SingleAsync();
page.HtmlText = poem.HtmlText;
context.GanjoorPages.Update(page);
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
/// <summary>
/// import catalogue from ganjoor.net MySql db
/// </summary>
/// <param name="jobName"></param>
/// <param name="jobProgressServiceEF"></param>
/// <param name="job"></param>
/// <returns></returns>
public async Task <RServiceResult <bool> > ImportFromMySql(string jobName, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job)
{
try
{
using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC
{
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["SongsUsername"]};pwd={Configuration.GetSection("AudioMySqlServer")["SongsPassword"]};database={Configuration.GetSection("AudioMySqlServer")["SongsDatabase"]};charset=utf8;convert zero datetime=True"
))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - import golha data - pre open connection")).Result;
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT col_id, name FROM golha_collections ORDER BY col_id",
connection))
{
using (DataTable data = new DataTable())
{
await src.FillAsync(data);
foreach (DataRow row in data.Rows)
{
GolhaCollection collection = new GolhaCollection()
{
Id = int.Parse(row["col_id"].ToString()),
Name = row["name"].ToString(),
Programs = new List <GolhaProgram>()
};
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - import golha data - golha_collections: {collection.Id}")).Result;
using (MySqlDataAdapter srcPrograms = new MySqlDataAdapter(
$"SELECT program_id, title, progarm_order, url, mp3 FROM golha_programs WHERE col_id = {collection.Id} ORDER BY program_id",
connection))
{
using (DataTable programData = new DataTable())
{
await srcPrograms.FillAsync(programData);
foreach (DataRow golhaProgram in programData.Rows)
{
GolhaProgram program = new GolhaProgram()
{
Id = int.Parse(golhaProgram["program_id"].ToString()),
Title = golhaProgram["title"].ToString(),
ProgramOrder = int.Parse(golhaProgram["progarm_order"].ToString()),
Url = golhaProgram["url"].ToString(),
Mp3 = golhaProgram["mp3"].ToString(),
Tracks = new List <GolhaTrack>()
};
using (MySqlDataAdapter srcTracks = new MySqlDataAdapter(
$"SELECT track_id, track_no, timing, title FROM golha_tracks WHERE program_id = {program.Id} ORDER BY track_no",
connection))
{
using (DataTable trackData = new DataTable())
{
await srcTracks.FillAsync(trackData);
foreach (DataRow golhaTrack in trackData.Rows)
{
program.Tracks.Add
(
new GolhaTrack()
{
Id = int.Parse(golhaTrack["track_id"].ToString()),
TrackNo = int.Parse(golhaTrack["track_no"].ToString()),
Timing = golhaTrack["timing"].ToString(),
Title = golhaTrack["title"].ToString(),
Blocked = false,
}
);
}
}
}
collection.Programs.Add(program);
}
}
}
context.GolhaCollections.Add(collection);
await context.SaveChangesAsync();
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - import singers data")).Result;
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT artist_id, artist_name, artist_beeptunesurl FROM ganja_artists ORDER BY artist_id",
connection))
{
using (DataTable data = new DataTable())
{
await src.FillAsync(data);
foreach (DataRow row in data.Rows)
{
int artistId = int.Parse(row["artist_id"].ToString());
GanjoorSinger singer = new GanjoorSinger()
{
Name = row["artist_name"].ToString(),
Url = row["artist_beeptunesurl"].ToString(),
Albums = new List <GanjoorAlbum>()
};
using (MySqlDataAdapter srcAlbums = new MySqlDataAdapter(
$"SELECT album_id, album_name, album_beeptunesurl FROM ganja_albums WHERE album_artistid = {artistId} ORDER BY album_id",
connection))
{
using (DataTable dataAlbums = new DataTable())
{
await srcAlbums.FillAsync(dataAlbums);
foreach (DataRow rowAlbum in dataAlbums.Rows)
{
int albumId = int.Parse(rowAlbum["album_id"].ToString());
GanjoorAlbum album = new GanjoorAlbum()
{
Name = rowAlbum["album_name"].ToString(),
Url = rowAlbum["album_beeptunesurl"].ToString(),
Tracks = new List <GanjoorTrack>()
};
using (MySqlDataAdapter srcTracks = new MySqlDataAdapter(
$"SELECT track_name, track_beeptunesurl FROM ganja_tracks WHERE album_id = {albumId} ORDER BY track_id",
connection))
{
using (DataTable dataTracks = new DataTable())
{
await srcTracks.FillAsync(dataTracks);
foreach (DataRow rowTrack in dataTracks.Rows)
{
album.Tracks.Add
(
new GanjoorTrack()
{
Name = rowTrack["track_name"].ToString(),
Url = rowTrack["track_beeptunesurl"].ToString(),
Blocked = false
}
);
}
}
}
singer.Albums.Add(album);
}
}
}
context.GanjoorSingers.Add(singer);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - import singers data - {singer.Name}")).Result;
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, $"{jobName} - finalizing singers data")).Result;
await context.SaveChangesAsync();
}
}
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, exp.ToString());
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// start removing original images
/// </summary>
/// <returns></returns>
public RServiceResult <bool> StartRemovingOriginalImages()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions <RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("RemovingOriginalImages", "Removing")).Result;
try
{
var srcPath = Configuration.GetSection("PictureFileService")["StoragePath"];
var trashPath = Configuration.GetSection("PictureFileService")["TrashStoragePath"];
var images = await context.PictureFiles.Where(p => p.StoredFileName != null && p.SrcUrl != null && p.NormalSizeImageStoredFileName.IndexOf("orig") != 0).ToListAsync();
int progress = 0;
for (int i = 0; i < images.Count; i++)
{
var image = images[i];
string targetDir = Path.Combine(trashPath, image.FolderName);
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
Directory.CreateDirectory(Path.Combine(targetDir, "orig"));
}
string srcFileName = Path.Combine(Path.Combine(srcPath, image.FolderName), image.StoredFileName);
if (File.Exists(srcFileName))
{
string targetFileName = Path.Combine(Path.Combine(trashPath, image.FolderName), image.StoredFileName);
File.Move(srcFileName, targetFileName, true);
image.StoredFileName = null;
context.Update(image);
if ((i * 100 / images.Count) > progress)
{
progress = i * 100 / images.Count;
await jobProgressServiceEF.UpdateJob(job.Id, progress);
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
/// <summary>
/// import GanjoorPage entity data from MySql
/// </summary>
/// <returns></returns>
public RServiceResult <bool> ImportFromMySql()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC
using (RMuseumDbContext contextReport = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(contextReport);
var job = (await jobProgressServiceEF.NewJob("GanjoorService:ImportFromMySql", "pre open connection")).Result;
if (string.IsNullOrEmpty(Configuration.GetSection("AudioMySqlServer")["ReportedCommentsDatabase"]))
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, "ReportedCommentsDatabase is not set");
return;
}
MusicCatalogueService catalogueService = new MusicCatalogueService(Configuration, context);
RServiceResult <bool> musicCatalogueRes = await catalogueService.ImportFromMySql("MusicCatalogueImportFromMySql", jobProgressServiceEF, job);
if (!musicCatalogueRes.Result)
{
return;
}
try
{
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT ID, post_author, post_date, post_date_gmt, post_content, post_title, post_category, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count, " +
"COALESCE((SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='_wp_page_template'), '') AS template," +
"(SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='otherpoetid') AS other_poet_id " +
"FROM ganja_posts",
connection))
{
using (DataTable srcData = new DataTable())
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - mysql 1")).Result;
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - processing mysql data")).Result;
foreach (DataRow row in srcData.Rows)
{
GanjoorPageType pageType =
row["post_type"].ToString() == "post" && row["comment_status"].ToString() != "closed" ?
GanjoorPageType.PoemPage
:
row["template"].ToString() == "comspage.php" ?
GanjoorPageType.AllComments
:
row["template"].ToString() == "relations.php" ?
GanjoorPageType.ProsodySimilars
:
row["template"].ToString() == "vazn.php" ?
GanjoorPageType.ProsodyAndStats
:
GanjoorPageType.None;
int?poetId = row["post_author"].ToString() == "1" ? (int?)null : int.Parse(row["post_author"].ToString());
if (poetId == 36)//رشحه
{
continue;
}
if (poetId != null)
{
if (!await context.GanjoorPoets.Where(poet => poet.Id == poetId).AnyAsync())
{
continue;
}
}
GanjoorPage page = new GanjoorPage()
{
Id = int.Parse(row["ID"].ToString()),
GanjoorPageType = pageType,
Published = true,
PageOrder = -1,
Title = row["post_title"].ToString(),
UrlSlug = row["post_name"].ToString(),
HtmlText = row["post_content"].ToString(),
ParentId = row["post_parent"].ToString() == "0" ? (int?)null : int.Parse(row["post_parent"].ToString()),
PoetId = poetId,
SecondPoetId = row["other_poet_id"] == DBNull.Value ? (int?)null : int.Parse(row["other_poet_id"].ToString()),
PostDate = (DateTime)row["post_date"]
};
if (pageType == GanjoorPageType.PoemPage)
{
var poem = await context.GanjoorPoems.Where(p => p.Id == page.Id).FirstOrDefaultAsync();
if (poem == null)
{
continue;
}
page.PoemId = poem.Id;
}
if (poetId != null && pageType == GanjoorPageType.None)
{
GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId == null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
page.GanjoorPageType = GanjoorPageType.PoetPage;
page.CatId = cat.Id;
}
else
{
cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId != null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
page.GanjoorPageType = GanjoorPageType.CatPage;
page.CatId = cat.Id;
}
}
}
context.GanjoorPages.Add(page);
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - finalizing")).Result;
await context.SaveChangesAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - pre fetch data")).Result;
var orphanPages = await context.GanjoorPages.Include(p => p.Poem).Where(p => p.FullUrl == null).ToListAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - post fetch data")).Result;
double count = orphanPages.Count;
int i = 0;
foreach (var page in orphanPages)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, i++, "phase 2")).Result;
string fullUrl = page.UrlSlug;
string fullTitle = page.Title;
if (page.GanjoorPageType == GanjoorPageType.PoemPage)
{
fullTitle = page.Poem.FullTitle;
fullUrl = page.Poem.FullUrl;
}
else
{
if (page.ParentId != null)
{
GanjoorPage parent = await context.GanjoorPages.Where(p => p.Id == page.ParentId).SingleAsync();
while (parent != null)
{
fullUrl = parent.UrlSlug + "/" + fullUrl;
fullTitle = parent.Title + " » " + fullTitle;
parent = parent.ParentId == null ? null : await context.GanjoorPages.Where(p => p.Id == parent.ParentId).SingleAsync();
}
}
else
{
GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync();
if (cat != null)
{
fullUrl = cat.FullUrl;
while (cat.ParentId != null)
{
cat = await context.GanjoorCategories.Where(c => c.Id == cat.ParentId).SingleOrDefaultAsync();
if (cat != null)
{
fullTitle = cat.Title + " » " + fullTitle;
}
}
}
else
{
cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.ParentId == null).SingleOrDefaultAsync();
if (cat != null)
{
fullUrl = $"{cat.UrlSlug}/{page.UrlSlug}";
}
}
}
}
if (!string.IsNullOrEmpty(fullUrl) && fullUrl.IndexOf('/') != 0)
{
fullUrl = $"/{fullUrl}";
}
page.FullUrl = fullUrl;
page.FullTitle = fullTitle;
context.Update(page);
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "phase 2 - finalizing")).Result;
await context.SaveChangesAsync();
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - pre mysql data fetch")).Result;
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT meta_key, post_id, meta_value FROM ganja_postmeta WHERE meta_key IN ( 'vazn', 'ravi', 'src', 'srcslug', 'oldtag' )",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - mysql 2")).Result;
using (DataTable srcData = new DataTable())
{
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - processing meta data")).Result;
int r = 0;
foreach (DataRow row in srcData.Rows)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, r++, "phase 3 - processing meta data")).Result;
int poemId = int.Parse(row["post_id"].ToString());
var poem = await context.GanjoorPoems.Where(p => p.Id == poemId).FirstOrDefaultAsync();
if (poem == null)
{
continue;
}
string metaKey = row["meta_key"].ToString();
string metaValue = row["meta_value"].ToString();
switch (metaKey)
{
case "vazn":
{
GanjoorMetre metre = await context.GanjoorMetres.Where(m => m.Rhythm == metaValue).SingleOrDefaultAsync();
if (metre == null)
{
metre = new GanjoorMetre()
{
Rhythm = metaValue,
VerseCount = 0
};
context.GanjoorMetres.Add(metre);
await context.SaveChangesAsync();
}
poem.GanjoorMetreId = metre.Id;
}
break;
case "ravi":
poem.RhymeLetters = metaValue;
break;
case "src":
poem.SourceName = metaValue;
break;
case "srcslug":
poem.SourceUrlSlug = metaValue;
break;
case "oldtag":
poem.OldTag = metaValue;
switch (poem.OldTag)
{
case "بدایع":
poem.OldTagPageUrl = "/saadi/badaye";
break;
case "خواتیم":
poem.OldTagPageUrl = "/saadi/khavatim";
break;
case "طیبات":
poem.OldTagPageUrl = "/saadi/tayyebat";
break;
case "غزلیات قدیم":
poem.OldTagPageUrl = "/saadi/ghazaliyat-e-ghadim";
break;
case "ملمعات":
poem.OldTagPageUrl = "/saadi/molammaat";
break;
}
break;
}
context.GanjoorPoems.Update(poem);
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - finalizing meta data")).Result;
await context.SaveChangesAsync();
var resApprovedPoemSongs = await _ImportPoemSongsDataFromMySql("_ImportPoemSongsDataFromMySql", context, jobProgressServiceEF, job, true);
if (!resApprovedPoemSongs.Result)
{
return;
}
var resPendingPoemSongs = await _ImportPoemSongsDataFromMySql("_ImportPoemSongsDataFromMySql", context, jobProgressServiceEF, job, false);
if (!resPendingPoemSongs.Result)
{
return;
}
using (MySqlConnection connection = new MySqlConnection
(
$"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True"
))
{
connection.Open();
using (MySqlDataAdapter src = new MySqlDataAdapter(
"SELECT poem_id, mimage_id FROM ganja_mimages",
connection))
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - mysql N")).Result;
using (DataTable srcData = new DataTable())
{
await src.FillAsync(srcData);
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - processing meta data")).Result;
int r = 0;
foreach (DataRow row in srcData.Rows)
{
job = (await jobProgressServiceEF.UpdateJob(job.Id, r++, "phase N - processing meta data")).Result;
int poemId = int.Parse(row["poem_id"].ToString());
Guid imageId = Guid.Parse(row["mimage_id"].ToString());
var link = await context.GanjoorLinks.Include(l => l.Item).ThenInclude(i => i.Images).
Where(l => l.GanjoorPostId == poemId && l.Item.Images.First().Id == imageId)
.FirstOrDefaultAsync();
if (link != null)
{
link.DisplayOnPage = true;
context.GanjoorLinks.Update(link);
}
}
}
}
}
job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase N - finalizing meta data")).Result;
await context.SaveChangesAsync();
}
catch (Exception jobExp)
{
await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, jobExp.ToString());
}
var resComments = await _ImportCommentsDataFromMySql("_ImportCommentsDataFromMySql", context, jobProgressServiceEF, job);
if (!resComments.Result)
{
return;
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "Finished", true);
}
});
return(new RServiceResult <bool>(true));
}
catch (Exception exp)
{
return(new RServiceResult <bool>(false, exp.ToString()));
}
}
private async Task _PerformMundexHouseKeepingAndPreparation(RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job)
{
var singers = await context.GanjoorSingers.ToListAsync();
foreach (var singer in singers)
{
if (!string.IsNullOrEmpty(singer.Url) && singer.Url.Length > 0 && singer.Url[singer.Url.Length - 1] == '/')
{
singer.Url = singer.Url.Substring(0, singer.Url.Length - 1);
context.GanjoorSingers.Update(singer);
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 2);
var poemMusicTracks =
await context.GanjoorPoemMusicTracks
.Where(m =>
m.TrackType == PoemMusicTrackType.BeepTunesOrKhosousi
&&
m.Approved)
.ToListAsync();
IConfigurationRoot configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("appsettings.json")
.Build();
ImageFileServiceEF imageFileService = new ImageFileServiceEF(context, configuration);
using (HttpClient httpClient = new HttpClient())
foreach (var poemMusicTrack in poemMusicTracks)
{
if (!string.IsNullOrEmpty(poemMusicTrack.ArtistUrl) && poemMusicTrack.ArtistUrl.Length > 0 && poemMusicTrack.ArtistUrl[poemMusicTrack.ArtistUrl.Length - 1] == '/')
{
poemMusicTrack.ArtistUrl = poemMusicTrack.ArtistUrl.Substring(0, poemMusicTrack.ArtistUrl.Length - 1);
context.GanjoorPoemMusicTracks.Update(poemMusicTrack);
}
var singer =
poemMusicTrack.SingerId != null?
context.GanjoorSingers.Where(s => s.Id == poemMusicTrack.SingerId).FirstOrDefault()
:
context.GanjoorSingers.Where(s => s.Url == poemMusicTrack.ArtistUrl).FirstOrDefault();
if (singer == null)
{
singer = new GanjoorSinger()
{
Name = poemMusicTrack.ArtistName,
Url = poemMusicTrack.ArtistUrl
};
context.GanjoorSingers.Add(singer);
await context.SaveChangesAsync();
}
if (poemMusicTrack.SingerId == null)
{
poemMusicTrack.SingerId = singer.Id;
context.GanjoorPoemMusicTracks.Update(poemMusicTrack);
}
//singer image:
if (singer.Url.Contains("beeptunes.com/artist/"))
{
if (singer.RImageId == null)
{
var bUrl = singer.Url;
var beepId = bUrl.Substring(bUrl.LastIndexOf("/") + 1);
var response = await httpClient.GetAsync($"https://newapi.beeptunes.com/public/artist/info/?artistId={beepId}");
if (response.IsSuccessStatusCode)
{
dynamic bpArtist = JsonConvert.DeserializeObject(await response.Content.ReadAsStringAsync());
if (bpArtist.artistImage != null)
{
try
{
var imageResult = await httpClient.GetAsync(bpArtist.artistImage.ToString());
if (imageResult.IsSuccessStatusCode)
{
using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync())
{
RServiceResult <RImage> image = await imageFileService.Add(null, imageStream, $"{beepId}.jpg", Path.Combine(configuration.GetSection("PictureFileService")["StoragePath"], "SingerImages"));
if (string.IsNullOrEmpty(image.ExceptionString))
{
image = await imageFileService.Store(image.Result);
if (string.IsNullOrEmpty(image.ExceptionString))
{
singer.RImageId = image.Result.Id;
context.GanjoorSingers.Update(singer);
await context.SaveChangesAsync();
}
}
}
}
}
catch
{
}
}
}
}
}
}
await context.SaveChangesAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 3);
}