public void UpdateDictionary()
{
VmCurrentWord[] words = GetWordsCollectionFromJson();
using (var db = new WordContext())
{
var existedWords = db.Words.ToArray();
foreach (VmCurrentWord word in words)
{
if (!Array.Exists(existedWords, element => element.Name_en == word.Name_en))
{
db.Words.Update(new VmWord
{
Name_ru = word.Name_ru,
Name_en = word.Name_en,
FourDaysLearnPhase = true,
LearnDay = 0,
RepeatIterationNum = 0,
NextRepeatDate = DateTime.Today,
DailyReapeatCountForEng = 0,
DailyReapeatCountForRus = 0
});
Console.WriteLine("Updating word \"{0}\"", word.Name_en);
}
else
{
Console.WriteLine("Skipped word \"{0}\"", word.Name_en);
}
}
var count = db.SaveChanges();
Console.WriteLine("{0} records saved to database", count);
}
}
//public string Update([FromBody] VmWord[] words)
// TODO: use better name for VmWordAndCollocationUpdating
public string Update([FromBody] VmWordAndCollocationUpdating wordAndCollocationUpdating)
{
using (var db = new WordContext())
{
foreach (VmWord word in wordAndCollocationUpdating.words)
{
if (word == null)
{
Console.WriteLine("Word is null");
throw new ArgumentNullException("Word is null");
}
else
{
db.Words.Update(word);
Console.WriteLine("Updating word \"{0}\" id {1}", word.Name_en, word.Id);
}
}
DateTime dateToday = DateTime.Now;
int collocationDelayPeriod = 4;
foreach (VmCollocation collocation in wordAndCollocationUpdating.collocations)
{
if (collocation == null)
{
Console.WriteLine("collocation is null");
throw new ArgumentNullException("collocation is null");
}
else
{
if (collocation.NotUsedToday == false)
{
collocation.NextRepeatDate = dateToday.AddDays(collocationDelayPeriod);
collocation.NotUsedToday = true;
}
db.Collocations.Update(collocation);
Console.WriteLine("Updating collocation \"{0}\"", collocation.AudioUrl);
}
}
db.SaveChanges();
}
return("succes");
}
public void UpdateSchedule()
{
DateTime dateToday = DateTime.Now;
VmWord[] renewingIteration;
using (var db = new WordContext())
{
// TODO: move it to separated methods
// Auto-removing duplicates
var allWords = db.Words.ToArray();
var duplicates = db.Words.Where(x => allWords
.Count(n => ((n.Name_ru == x.Name_ru) &&
(n.Name_en == x.Name_en))) > 1)
.GroupBy(p => p.Name_ru)
.Select(p => p.LastOrDefault());
// TODO: dell all duplicates in one time,
// now p.Skip(1) doesn't make it works
foreach (VmWord word in duplicates)
{
db.Words.Remove(word);
Console.WriteLine("Removing duplicate \"{0}\" id {1}", word.Name_en, word.Id);
}
// Manually-removing duplicates
var duplicatesToResolve = db.Words.Where(x => allWords
.Count(n => ((n.Name_ru != x.Name_ru) && (n.Name_en == x.Name_en)) ||
((n.Name_ru == x.Name_ru) && (n.Name_en != x.Name_en))) > 1)
.GroupBy(p => p.Name_ru)
.Select(p => p.LastOrDefault());
if (duplicatesToResolve.Count() > 0)
{
throw new Exception("There are duplicates thet should be resolved");
}
// Renewing Schedule
renewingIteration = db.Words.Where(p => (p.NextRepeatDate <= dateToday) &&
(p.DailyReapeatCountForEng >= minReapeatCountPerDayIteration) &&
(p.DailyReapeatCountForRus >= minReapeatCountPerDayIteration) &&
(p.FourDaysLearnPhase == false)).ToArray();
var iterationIncrement = 7;
foreach (VmWord word in renewingIteration)
{
var iteration = iterationIncrement * getIterationLenght(word.RepeatIterationNum);
word.NextRepeatDate = dateToday.AddDays(iteration);
word.DailyReapeatCountForEng = 0;
word.DailyReapeatCountForRus = 0;
word.RepeatIterationNum++;
db.Words.Update(word);
Console.WriteLine("Set new day for repeating word \"{0}\" iterations {1} id {2}",
word.Name_en, word.RepeatIterationNum, word.Id);
}
db.SaveChanges();
}
}
public void Download()
{
VmWord[] words;
List <VmCollocation> collocations;
using (var db = new WordContext())
{
words = db.Words.Where(p => p.Name_ru.IndexOf(' ') < 0 &&
(p.Name_en.IndexOf(' ') < 0)).ToArray();
}
string htmlCode;
string tempURL;
foreach (VmWord word in words)
{
using (WebClient client = new WebClient())
{
client.Encoding = System.Text.Encoding.UTF8;
htmlCode = client.DownloadString("https://www.ldoceonline.com/dictionary/" + word.Name_en);
}
var sourcePath = "https://www.ldoceonline.com/";
var urlFirstPart = "data-src-mp3=\"/";
var searchStartPattern = "\"EXAMPLE\"";
var searchStartIndex = htmlCode.IndexOf(searchStartPattern);
var indexesOfUrl = new List <int>();
var urls = new List <string>();
string lang = "en";
int tagCounter = 0;
string[] nameParts = new string[2];
if (searchStartIndex >= 0)
{
int urlIndex = searchStartIndex;
int nameTempIndex;
int urlSubstrStart, urlSubstrEnd;
string url;
string nameSearchStartTag = "/span>";
int nameStartIndex;
string nextCharPair, fileName;
int nameEnd;
string excludeTagClass = "\"GLOSS\"";
while (urlIndex < htmlCode.LastIndexOf(urlFirstPart))
{
urlSubstrStart = htmlCode.IndexOf(urlFirstPart, urlIndex) + urlFirstPart.Length;
if (urlSubstrStart < 0)
{
break;
}
urlSubstrEnd = htmlCode.IndexOf(".mp3", urlSubstrStart) + 4;
url = htmlCode.Substring(urlSubstrStart, urlSubstrEnd - urlSubstrStart);
urls.Add(url);
Console.WriteLine(url);
nameTempIndex = urlSubstrEnd;
nameStartIndex = htmlCode.IndexOf(nameSearchStartTag, nameTempIndex) + 6;
nextCharPair = htmlCode.Substring(nameStartIndex, 2);
tagCounter = 0;
if (nextCharPair == "<s")
{
tagCounter++;
nameStartIndex = htmlCode.IndexOf(">", nameStartIndex) + 1;
// TODO: remove repetitation
nextCharPair = htmlCode.Substring(nameStartIndex, 2);
if (nextCharPair == "<s")
{
throw new ArgumentNullException("Unknow tags arangment");
}
}
nameParts[0] = "";
nameParts[1] = "";
if (tagCounter > 0)
{
if (tagCounter > 1)
{
throw new ArgumentNullException("Unknow tags arangment");
}
nameEnd = htmlCode.IndexOf("</span>", nameStartIndex);
fileName = htmlCode.Substring(nameStartIndex, nameEnd - nameStartIndex);
nameParts[0] = fileName;
nameStartIndex = nameEnd + 7;
}
nameEnd = htmlCode.IndexOf("</span>", nameStartIndex);
fileName = htmlCode.Substring(nameStartIndex, nameEnd - nameStartIndex);
// TODO: solve hotfixes
// COLLOINEXA found there:
// I promise not to <span class="COLLOINEXA">disturb anything
if (fileName.IndexOf(excludeTagClass) < 0 &&
fileName.IndexOf("COLLOINEXA") < 0)
{
nameParts[1] = fileName;
}
fileName = nameParts[0] + nameParts[1];
urlIndex = urlSubstrEnd;
Console.WriteLine(fileName);
using (var db = new WordContext())
{
tempURL = Path.Combine("/" + "audio", collocationPath, lang, fileName + ".mp3");
if (fileName != "")
{
GetAndSave(sourcePath + url, fileName, lang);
var isCollocationExist = db.Collocations.Any(p => p.AudioUrl == tempURL);
if (!isCollocationExist)
{
db.Collocations.Add(new VmCollocation
{
Lang = "en",
AudioUrl = tempURL,
NotUsedToday = true
});
}
db.SaveChanges();
}
}
}
}
}
using (var db = new WordContext())
{
// TODO: check for dublicates
var allCollocations = db.Collocations.ToList();
var duplicates = allCollocations.Where(p => allCollocations
.Count(z => z.AudioUrl == p.AudioUrl) > 1)
.GroupBy(j => j.AudioUrl)
.Select(p => p.LastOrDefault()).ToList();
foreach (VmCollocation collocation in duplicates)
{
db.Collocations.Remove(collocation);
Console.WriteLine("Removing duplicate \"{0}\" id {1}", collocation.AudioUrl, collocation.Id);
}
collocations = db.Collocations.ToList();
var collocationsTemp = Directory.GetFiles(Path.Combine(audioPath, collocationPath)).ToList();
var collocationsForExclude = collocationsTemp
.Where(p => collocations.All(z => z.AudioUrl == p)).ToList();
var collocationsNew = collocationsTemp.Except(collocationsForExclude);
foreach (string collocation in collocationsNew)
{
collocations.Add(new VmCollocation {
Lang = "en",
AudioUrl = collocation,
NotUsedToday = true,
});
}
db.SaveChanges();
}
}
