private static async System.Threading.Tasks.Task GetDiffsContent(string old, string @new, int howManyWeeksToSave)
{
if (old != null && @new != null)
{
var oldTree = JsonConvert.DeserializeObject <Rootobject>(old);
var newTree = JsonConvert.DeserializeObject <Rootobject>(@new);
var result = oldTree.GetDiffs(newTree);
if (result.Any())
{
var file = new DirectoryInfo(Pathes.pathToDataDirectory).GetFiles().Where(fi => fi.Name == ConfigJson.diffsFileName).SingleOrDefault();
if (file == null)
{
var json = JsonConvert.SerializeObject(result, Formatting.Indented);
var path = Path.Combine(Pathes.pathToDataDirectory, ConfigJson.diffsFileName);
var file_manager = new ClassLibrary.File_Manager();
file_manager.OpenFile(path, "Append", json);
}
else
{
var fileManager = new ClassLibrary.File_Manager();
var fileDataObj = fileManager.OpenFile(file.FullName, "Read", null);
string readedDiffsFile = fileDataObj.fileData;
var parsedFile = JsonConvert.DeserializeObject <IEnumerable <(Item old, Item @new)> >(readedDiffsFile);
var concatedObj = parsedFile.Concat(result);
var dateTimeList = new List <DateTime>();
foreach (var homework in concatedObj)
{
string dateTime;
DateTime dateTimeAsDateTime;
var notEmptyItem = (homework.old ?? homework.@new);
if ((notEmptyItem) != null)
{
dateTime = notEmptyItem.datetime_from;
dateTimeAsDateTime = DateTime.ParseExact(dateTime, DateTimesFormats.FullDateTime, null);
dateTimeList.Add(dateTimeAsDateTime);
}
}
var maxDateTimeSaved = dateTimeList.Max().AddDays(-7 * howManyWeeksToSave);
var recentItemsOnly = concatedObj.Where(changedHomework =>
{
var timeAsDateTime = DateTime.ParseExact((changedHomework.old ?? changedHomework.@new).datetime_from, DateTimesFormats.FullDateTime, null);
return(timeAsDateTime >= maxDateTimeSaved);
});
var newData = JsonConvert.SerializeObject(recentItemsOnly, Formatting.Indented);
if (newData.Any())
{
var file_manager = new ClassLibrary.File_Manager();
file_manager.OpenFile(file.FullName, "Write", newData);
}
}
}
}
}
public static async Task <SortedList <(DateTime, string), List <JToken> > > ParseDiffsFile(FileInfo diffsFile)
{
var sortedList = new SortedList <(DateTime, string), List <JToken> >();
if (diffsFile != null)
{
var fileManager = new ClassLibrary.File_Manager();
var result = fileManager.OpenFile(diffsFile.FullName, "Read", null);
var readedDiffsFile = result.fileData;
var diffsJson = JArray.Parse(readedDiffsFile);
foreach (var obj in diffsJson)
{
var item = GetNotNullItem(obj);
var datetime = DateTime.ParseExact(item["datetime_from"].ToString(), DateTimesFormats.FullDateTime, null);
var subject = item["subject_name"].ToString();
bool keyContains = sortedList.ContainsKey((datetime, subject));
if (!keyContains)
{
var list = new List <JToken>();
list.Add(obj);
sortedList.Add((datetime, subject), list);
}
else
{
List <JToken> @object = sortedList[(datetime, subject)];
private static async Task WriteProcessIdToFile(string id)
{
var fm = new ClassLibrary.File_Manager();
var path = Path.Combine(Pathes.pathToWorkDirectory, "WebSiteProcessId.txt");
fm.OpenFile(path, "Write", id);
}
private static async Task WriteProcessIdToFile(string id)
{
// Do not pay attention at File_Manager. It is my own API.
var fm = new ClassLibrary.File_Manager();
var path = Path.Combine(Pathes.pathToWorkDirectory, "WebSiteProcessId.txt");
fm.OpenFile(path, "Write", id);
}
private static void ErrorWriter(string bodyStr)
{
var currentTime = DateTime.Now.ToString(DateTimesFormats.FullDateTime);
string fileName = $"{ConfigJson.JavaScriptErrorsFileName}";
var path = Path.Combine(Pathes.pathToReports, fileName);
var content = $"{currentTime}{Environment.NewLine}{bodyStr}{Environment.NewLine + Environment.NewLine}";
var fileManager = new ClassLibrary.File_Manager();
fileManager.OpenFile(path, "Append", content);
}
private static async Task WriteProcessDataToFile(string id, DateTime systemRestartTime)
{
var json = new JObject();
json.Add(ProcessID, id);
json.Add(SystemRestartTime, systemRestartTime);
string jsonStr = JsonConvert.SerializeObject(json);
var fm = new ClassLibrary.File_Manager();
var path = pathToWebSiteProcessDataFile;
fm.OpenFile(path, "Write", jsonStr);
}
private static JObject GetWebSiteProcessData()
{
if (!File.Exists(pathToWebSiteProcessDataFile))
{
return(null);
}
var fm = new ClassLibrary.File_Manager();
var path = pathToWebSiteProcessDataFile;
string jsonStr = fm.OpenFile(path, "Read", null).fileData;
var jobj = JObject.Parse(jsonStr);
return(jobj);
}
private static int GetWebSiteProcessId()
{
var directoryFiles = new DirectoryInfo(Pathes.pathToWorkDirectory).GetFiles();
var file = directoryFiles
.Where(file => file.Name == "WebSiteProcessId.txt")
.SingleOrDefault();
if (file == null)
{
return(-1);
}
var fm = new ClassLibrary.File_Manager();
var path = Path.Combine(Pathes.pathToWorkDirectory, "WebSiteProcessId.txt");
string idStr = fm.OpenFile(path, "Read", null).fileData;
int id = Convert.ToInt32(idStr);
return(id);
}
internal static async Task <Cookie> GetCookieByAuthorizationAsync(string pathToCookieFile)
{
/* ОЧЕНЬ ВАЖНО!
* В DefaultViewPort стоит параметр null, при котором в развернутом виде КОМПЬЮТЕРНАЯ ВЕРСИЯ,
* а в окне достаточно маленького размера у некоторых сайтов МОБИЛЬНАЯ ВЕРСИЯ!
* Следовательно, испольнование CSS-селекторов может быть удачным, а может и нет,
* в зависимости от размера браузера.
* В headless-режиме окно маленького размера. */
using (var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
DefaultViewport = null,
//Args = PuppeteerSharpLaunchArgs.args
}))
{
Stopwatch timer = new Stopwatch();
var p = await GetPage(browser, "about:blank");
/* 10 попыток подключения нужны, если сервер плохой, или интернета вдруг нету.*/
bool success = false;
int connectCount = 0;
timer.Start();
while (connectCount < 10 && !success)
{
try
{
await p.GoToAsync("https://dnevnik2.petersburgedu.ru");
success = true;
Log.Information("Подключение к сайту {Site}: успешно!", "https://dnevnik2.petersburgedu.ru");
Log.Information($"{timer.ElapsedMilliseconds}");
timer.Restart();
}
catch (PuppeteerException e)
{
Log.Error(e, "GoToAsync({Site} failed", "https://dnevnik2.petersburgedu.ru");
Log.Information("Подключение к сайту {Site}: Ошибка, повторяю...", "https://dnevnik2.petersburgedu.ru");
Log.Information("Попытка № {attempt}", connectCount + 1);
await System.Threading.Tasks.Task.Delay(3000);
connectCount++;
}
}
WaitForSelectorOptions WaitForSelectorTimeout = new WaitForSelectorOptions {
Timeout = DefaultTimeout
};
Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");
const string button = "body > app-root > n3-grid > app-login > div > div.notice > div > app-login-form > div > button";
await p.WaitForSelectorAsync(button, WaitForSelectorTimeout);
await System.Threading.Tasks.Task.Delay(10000);
await p.ClickAsync(button);
Log.Information("Первый клик {Button}: успешно!", "Войти с ЕСИА");
Log.Information($"{timer.ElapsedMilliseconds}");
timer.Restart();
p = await GetPage(browser, "https://esia.gosuslugi.ru");
Log.Information($"DefaultTimeout: {p.DefaultTimeout}");
Log.Information($"DefaultNavigationTimeout: {p.DefaultNavigationTimeout}");
// Авторизация
await p.WaitForSelectorAsync("#mobileOrEmail", WaitForSelectorTimeout);
await p.FocusAsync("#mobileOrEmail");
await p.Keyboard.TypeAsync(ConfigJson.Login);
Log.Information($"Login: {timer.ElapsedMilliseconds}");
timer.Restart();
await p.WaitForSelectorAsync("#password", WaitForSelectorTimeout);
await p.FocusAsync("#password");
await p.Keyboard.TypeAsync(ConfigJson.Password);
Log.Information($"Password: {timer.ElapsedMilliseconds}");
timer.Restart();
await p.WaitForSelectorAsync("#loginByPwdButton > span", WaitForSelectorTimeout);
await p.ClickAsync("#loginByPwdButton > span");
Log.Information($"ClickAuthorizationButton: {timer.ElapsedMilliseconds}");
Log.Information("Авторизация: успешно!");
timer.Stop();
/* Куки нужны для того, чтобы сайт меня опознал
* при отправке http-запроса на сервер эл. дневника */
// 10 попыток получения cookie.
Cookie cookie;
int count = 0;
int attempts = (DefaultTimeout / 1000);
do
{
if (count > attempts)
{
throw new Exception("Cookie X-JMT-Token is not present.");
}
await System.Threading.Tasks.Task.Delay(1000);
var cookies = await p.GetCookiesAsync();
cookie = cookies.Where(c => c.Name == "X-JWT-Token").Select(c => new Cookie(c.Name, c.Value)).SingleOrDefault();
count++;
}while (cookie == null || cookie.Value == "");
//Здесь и далее безголовый браузер уже не нужен
await browser.CloseAsync();
var cookieAsJson = JsonConvert.SerializeObject(cookie);
//await File.WriteAllTextAsync(pathToCookieFile, cookieAsJson);
var fm = new ClassLibrary.File_Manager();
fm.OpenFile(pathToCookieFile, "Write", cookieAsJson);
return(cookie);
}
}
private static async System.Threading.Tasks.Task Main()
{
string pathToCookieFile = Path.Combine(Pathes.pathToAuthorizationDataDirectory, ConfigJson.AuthorizationCookieFileName);
ConfigureLogger();
await InstallBrowserAsync();
string cookieFile;
if (File.Exists(pathToCookieFile))
{
var fileManager = new ClassLibrary.File_Manager();
var result = fileManager.OpenFile(pathToCookieFile, "Read", null);
cookieFile = result.fileData;
var json = JObject.Parse(cookieFile);
var cookieName = json["Name"].ToString();
var cookieValue = json["Value"].ToString();
cookie = new Cookie(cookieName, cookieValue);
}
else
{
Log.Information("Файл cookie отсутствует. Авторизация");
Authorization.DefaultTimeout = ConfigJson.DefaultTimeout;
cookie = await Authorization.GetCookieByAuthorizationAsync(pathToCookieFile);
}
// Соединяемся с электронным дневником и получаем JSON
var monday_sunday = GetMondaySunday();
var mondayDate = monday_sunday.monday.Date;
var sundayDate = monday_sunday.sunday.Date;
int howManyWeeksToDownload = ConfigJson.HowManyWeeksToDownload;
var dataFromServerList = new SortedList <DateTime, string>();
var dataFromFileSystemList = new SortedList <DateTime, string>();
var serverDataObj = new JObject();
EnsureDirectoryExists(Pathes.pathToDataDirectory);
var lastFile = new DirectoryInfo(Pathes.pathToDataDirectory)
.GetFiles()
.OrderByDescending(fi => fi.CreationTime)
.Where(file => file.Name == ConfigJson.serverFileName)
.FirstOrDefault();
if (lastFile != null)
{
var fileManager = new ClassLibrary.File_Manager();
var path2 = Path.Combine(Pathes.pathToDataDirectory, ConfigJson.serverFileName);
var result = fileManager.OpenFile(path2, "Read", null);
var json = result.fileData;
var p = JObject.Parse(json);
for (int i = 0; i < p.Count; i++)
{
var date = DateTime.ParseExact(p[i.ToString()]["data"]["Monday"].ToString(), ConfigJson.DateTimesFormats.FullDateTime, null).Date;
var jsonWeek = JsonConvert.SerializeObject(p[i.ToString()]);
dataFromFileSystemList.Add(date, jsonWeek);
}
}
for (int i = 0; i < howManyWeeksToDownload; i++)
{
Log.Information(i.ToString());
if (i != 0)
{
mondayDate = mondayDate.AddDays(-7);
sundayDate = sundayDate.AddDays(-7);
}
string jsonContentAsString = GetDataFromServer(mondayDate, sundayDate, pathToCookieFile).Result;
var strAsJson = JsonConvert.DeserializeObject <JObject>(jsonContentAsString);
var date = DateTime.ParseExact(strAsJson["data"]["Monday"].ToString(), "dd.MM.yyyy H:mm:ss", null).Date;
dataFromServerList.Add(date, jsonContentAsString);
serverDataObj.Add(new JProperty(i.ToString(), strAsJson));
}
var path = Path.Combine(Pathes.pathToDataDirectory, ConfigJson.serverFileName);
var content = JsonConvert.SerializeObject(serverDataObj, Formatting.Indented);
var file_manager = new ClassLibrary.File_Manager();
file_manager.OpenFile(path, "Write", content);
var orderedDates = dataFromFileSystemList.Keys.Concat(dataFromServerList.Keys).Distinct().OrderByDescending(d => d);
var howManyWeeksToSaveInDiffsJson = ConfigJson.HowManyWeeksToSave;
foreach (var date in orderedDates)
{
bool fileExists = dataFromFileSystemList.TryGetValue(date, out var fileJson);
bool serverContains = dataFromServerList.TryGetValue(date, out var serverJson);
if (fileExists && serverContains)
{
string newDz;
newDz = await GetDiffsContent(fileJson, serverJson, howManyWeeksToSaveInDiffsJson);
if (newDz != null)
{
string dzMessage = "";
var json = JArray.Parse(newDz);
IEnumerable <JToken> items = json.AsEnumerable();
items = items.OrderBy(item => DateTime.ParseExact(
(item["Item1"].HasValues ?
item["Item1"] :
item["Item2"])["datetime_from"].ToString(),
DateTimesFormats.FullDateTime, null));
foreach (var item in items)
{
var item1 = item["Item1"];
var item2 = item["Item2"];
/*if (item1.Any())
* {
* var updateTime = DateTime.ParseExact(item1["updateTime"].ToString(), DateTimesFormats.FullDateTime, null).ToString(DateTimesFormats.No_seconds);
* var subjectStatus = item1["SubjectStatus"].ToString();
* var homeworkStatus = item1["HomeworkStatus"].ToString();
*
* if (subjectStatus == "deleted" && item1["tasks"].Any())
* {
* //< p class="dzDeleted">У(пбу) @updateTime</p>
* }
* }*/
if (item2.Any())
{
//var updateTime = DateTime.ParseExact(item2["updateTime"].ToString(), DateTimesFormats.FullDateTime, null).ToString(DateTimesFormats.No_seconds);
var subjectStatus = item2["SubjectStatus"].ToString();
var homeworkStatus = item2["HomeworkStatus"].ToString();
if ((subjectStatus == "new" && item2["tasks"].Any()) ||
(homeworkStatus == "changed" && item1.Any() && !item1["tasks"].Any() && item2["tasks"].Any()))
{
var subject = item2["subject_name"].ToString();
var subject_date = DateTime.ParseExact(item2["datetime_from"].ToString(), ConfigJson.DateTimesFormats.FullDateTime, null).Date;
string dayOfWeek = "";
string subjectDateStr = "";
if (subject_date != DateTime.Now.Date)
{
dayOfWeek = DayOfWeekExtention.ToRussianString(subject_date.DayOfWeek);
}
var monday_sunday2 = GetMondaySunday();
var mondayDate2 = monday_sunday.monday.Date;
if ((subject_date - mondayDate2).TotalDays < 0)
{
subjectDateStr = subject_date.ToString(ConfigJson.DateTimesFormats.No_year);
}
string itog = "";
if (dayOfWeek != "")
{
itog += ($"{dayOfWeek},\n");
if (subjectDateStr != "")
{
itog += ($"{subjectDateStr}:\n");
}
}
itog += ($"{subject}:\n");
dzMessage += itog;
foreach (var task in item2["tasks"])
{
dzMessage += ($"Появилось: {task["task_name"].ToString()}\n");
}
//<p class="dzAdded">П @updateTime</p>
}
else if (item1.Any() && item1["tasks"].Any() && item2["tasks"].Any())
{
var subject = item2["subject_name"].ToString();
var subject_date = DateTime.ParseExact(item2["datetime_from"].ToString(), ConfigJson.DateTimesFormats.FullDateTime, null).Date;
string dayOfWeek = "";
string subjectDateStr = "";
if (subject_date != DateTime.Now.Date)
{
dayOfWeek = DayOfWeekExtention.ToRussianString(subject_date.DayOfWeek);
}
var monday_sunday2 = GetMondaySunday();
var mondayDate2 = monday_sunday.monday.Date;
if ((subject_date - mondayDate2).TotalDays < 0)
{
subjectDateStr = subject_date.ToString(ConfigJson.DateTimesFormats.No_year);
}
string itog = "";
if (dayOfWeek != "")
{
itog += ($"{dayOfWeek},\n");
if (subjectDateStr != "")
{
itog += ($"{subjectDateStr}:\n");
}
}
itog += ($"{subject}:\n");
dzMessage += itog;
foreach (var task in item2["tasks"])
{
dzMessage += ($"Изменилось: {task["task_name"].ToString()}\n");
}
}
/*else if (homeworkStatus == "changed")
* {
*
* if (item1.Any() && item1["tasks"].Any() && item2["tasks"].Any())
* {
* //<p class="dzChanged">И @updateTime</p>
* }
* else if (item1.Any() && !item1["tasks"].Any() && item2["tasks"].Any())
* {
* //<p class="dzAdded">П(прб) @updateTime</p>
* }
* else if (item1.Any() && !item2["tasks"].Any() && item1["tasks"].Any())
* {
* //<p class="dzDeleted">У @updateTime</p>
* }
* }*/
}
await SendNewDzToServer(dzMessage);
dzMessage = "";
}
}
}
}
Log.Information("Скрипт выполнен успешно!");
Log.CloseAndFlush(); /*отправляем логи на сервер логов*/
}
private static async System.Threading.Tasks.Task Main()
{
string pathToCookieFile = Path.Combine(Pathes.pathToAuthorizationDataDirectory, ConfigJson.AuthorizationCookieFileName);
ConfigureLogger();
await InstallBrowserAsync();
string cookieFile;
if (File.Exists(pathToCookieFile))
{
var fileManager = new ClassLibrary.File_Manager();
var result = fileManager.OpenFile(pathToCookieFile, "Read", null);
cookieFile = result.fileData;
var json = JObject.Parse(cookieFile);
var cookieName = json["Name"].ToString();
var cookieValue = json["Value"].ToString();
cookie = new Cookie(cookieName, cookieValue);
}
else
{
Log.Information("Файл cookie отсутствует. Авторизация");
Authorization.DefaultTimeout = ConfigJson.DefaultTimeout;
cookie = await Authorization.GetCookieByAuthorizationAsync(pathToCookieFile);
}
// Соединяемся с электронным дневником и получаем JSON
var monday_sunday = GetMondaySunday();
var mondayDate = monday_sunday.monday.Date;
var sundayDate = monday_sunday.sunday.Date;
int howManyWeeksToDownload = ConfigJson.HowManyWeeksToDownload;
var dataFromServerList = new SortedList <DateTime, string>();
var dataFromFileSystemList = new SortedList <DateTime, string>();
var serverDataObj = new JObject();
EnsureDirectoryExists(Pathes.pathToDataDirectory);
var lastFile = new DirectoryInfo(Pathes.pathToDataDirectory)
.GetFiles()
.OrderByDescending(fi => fi.CreationTime)
.Where(file => file.Name == ConfigJson.serverFileName)
.FirstOrDefault();
if (lastFile != null)
{
var fileManager = new ClassLibrary.File_Manager();
var path2 = Path.Combine(Pathes.pathToDataDirectory, ConfigJson.serverFileName);
var result = fileManager.OpenFile(path2, "Read", null);
var json = result.fileData;
var p = JObject.Parse(json);
for (int i = 0; i < p.Count; i++)
{
var date = DateTime.ParseExact(p[i.ToString()]["data"]["Monday"].ToString(), ConfigJson.DateTimesFormats.FullDateTime, null).Date;
var jsonWeek = JsonConvert.SerializeObject(p[i.ToString()]);
dataFromFileSystemList.Add(date, jsonWeek);
}
}
for (int i = 0; i < howManyWeeksToDownload; i++)
{
Log.Information(i.ToString());
if (i != 0)
{
mondayDate = mondayDate.AddDays(-7);
sundayDate = sundayDate.AddDays(-7);
}
string jsonContentAsString = GetDataFromServer(mondayDate, sundayDate, pathToCookieFile).Result;
var strAsJson = JsonConvert.DeserializeObject <JObject>(jsonContentAsString);
var date = DateTime.ParseExact(strAsJson["data"]["Monday"].ToString(), "dd.MM.yyyy H:mm:ss", null).Date;
dataFromServerList.Add(date, jsonContentAsString);
serverDataObj.Add(new JProperty(i.ToString(), strAsJson));
}
var path = Path.Combine(Pathes.pathToDataDirectory, ConfigJson.serverFileName);
var content = JsonConvert.SerializeObject(serverDataObj, Formatting.Indented);
var file_manager = new ClassLibrary.File_Manager();
file_manager.OpenFile(path, "Write", content);
var orderedDates = dataFromFileSystemList.Keys.Concat(dataFromServerList.Keys).Distinct().OrderByDescending(d => d);
var howManyWeeksToSaveInDiffsJson = ConfigJson.HowManyWeeksToSave;
foreach (var date in orderedDates)
{
bool fileExists = dataFromFileSystemList.TryGetValue(date, out var fileJson);
bool serverContains = dataFromServerList.TryGetValue(date, out var serverJson);
if (fileExists && serverContains)
{
await GetDiffsContent(fileJson, serverJson, howManyWeeksToSaveInDiffsJson);
}
}
Log.Information("Скрипт выполнен успешно!");
Log.CloseAndFlush(); /*отправляем логи на сервер логов*/
}