private void Test(string lang, string[] texts, string[][] pairs = null)
{
LanguageDetector detector;
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddAllLanguages();
foreach (string text in texts)
{
Assert.AreEqual(lang, detector.Detect(text));
}
if (pairs != null)
{
foreach (string[] pair in pairs)
{
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddLanguages(pair);
detector.AddLanguages(lang);
foreach (string text in texts)
{
Assert.AreEqual(lang, detector.Detect(text));
}
}
}
}
public AnswerModel <string> Execute(string text)
{
AnswerModel <string> answer = new AnswerModel <string>();
LanguageDetector detector = new LanguageDetector();
detector.AddAllLanguages();
Assert.AreEqual("lv", detector.Detect("Привет"));
answer.Property = "Определен язык: ";
answer.Value = "Определен язык: ";
return(answer);
}
public TwitterRetriever(string accessToken, string accessTokenSecret, string consumerKey, string consumerSecret)
{
tokens = new OAuthTokens()
{
AccessToken = accessToken,
AccessTokenSecret = accessTokenSecret,
ConsumerKey = consumerKey,
ConsumerSecret = consumerSecret
};
so = new SearchOptions();
so.ResultType = SearchOptionsResultType.Recent;
so.IncludeEntities = false;
so.Count = 100;
detector = new LanguageDetector();
detector.AddAllLanguages(AppDomain.CurrentDomain.BaseDirectory + "Profiles");
}
public void Issue_2()
{
string text = "Výsledky kola švýcarské hokejové ligy";
LanguageDetector detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddAllLanguages();
Assert.AreEqual("slk", detector.Detect(text));
Assert.AreEqual(1, detector.DetectAll(text).Count());
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.ConvergenceThreshold = 0.9;
detector.MaxIterations = 50;
detector.AddAllLanguages();
Assert.AreEqual("slk", detector.Detect(text));
Assert.AreEqual(2, detector.DetectAll(text).Count());
}
private void Test(string lang, string[] texts, string[][] pairs = null)
{
LanguageDetector detector;
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddAllLanguages();
foreach (string text in texts)
Assert.AreEqual(lang, detector.Detect(text));
if (pairs != null)
{
foreach (string[] pair in pairs)
{
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddLanguages(pair);
detector.AddLanguages(lang);
foreach (string text in texts)
Assert.AreEqual(lang, detector.Detect(text));
}
}
}
public static async Task <List <Words> > Search(string searchTerm)
{
LanguageDetector detector = new LanguageDetector();
detector.AddAllLanguages();
HttpClient client = new HttpClient();
var loginPage = await client.GetStringAsync("https://www.altmetric.com/explorer/login");
Match matchObject = Regex.Match(loginPage, @"name=""authenticity_token"" value=""(?<key>.+)""");
string token = string.Empty;
if (matchObject.Success)
{
token = matchObject.Groups["key"].Value;
}
Dictionary <string, string> formFields = new Dictionary <string, string>()
{
{ "email", "*****@*****.**" },
{ "password", "bigdatachallenge" },
{ "authenticity_token", token },
{ "commit", "Sign in" }
};
FormUrlEncodedContent content = new FormUrlEncodedContent(formFields);
var response = await client.PostAsync("https://www.altmetric.com/explorer/login", content);
var searchResults =
await client.GetStringAsync("https://www.altmetric.com/explorer/json_data/research_outputs?q=" + searchTerm +
"&scope=all");
Console.WriteLine("A");
var serializer = new System.Web.Script.Serialization.JavaScriptSerializer();
serializer.MaxJsonLength = Int32.MaxValue;
dynamic papersDict = serializer.DeserializeObject(searchResults);
List <Words> newsList = new List <Words>();
List <Task <string> > taskList = new List <Task <string> >();
List <int> scoreList = new List <int>();
if (papersDict["outputs"].Length == 0)
{
return(newsList);
}
Console.WriteLine("B");
for (int i = 0; i < Math.Min(10, papersDict["outputs"].Length); i++)
{
string altId = papersDict["outputs"][i]["id"].ToString();
int score = papersDict["outputs"][i]["score"];
scoreList.Add(score);
taskList.Add(client.GetStringAsync("https://api.altmetric.com/v1/fetch/id/" + altId + "?key=ef2e9b9961415ba4b6510ec82c3e9cba"));
}
int counter = 0;
while (taskList.Count > 0)
{
Console.WriteLine(counter);
Task <string> firstFinishedTask = await Task.WhenAny(taskList);
taskList.Remove(firstFinishedTask);
string detailsText = await firstFinishedTask;
dynamic details = serializer.DeserializeObject(detailsText);
if (details["posts"].ContainsKey("news") && details["posts"]["news"].Length > 0)
{
for (int j = 0; j < Math.Min(3, details["posts"]["news"].Length); j++)
{
if (details["posts"]["news"][j].ContainsKey("title") &&
details["posts"]["news"][j].ContainsKey("url"))
{
string title = details["posts"]["news"][j]["title"];
if (detector.Detect(title) == "en" && details["posts"]["news"][j]["url"] != null)
{
var request = new HttpRequestMessage(HttpMethod.Head, details["posts"]["news"][j]["url"]);
try
{
var validityResponse = await client.SendAsync(request);
if (validityResponse.IsSuccessStatusCode)
{
Words newsArticle = new Words(title, details["posts"]["news"][j]["url"], scoreList[counter], WordType.Article);
newsList.Add(newsArticle);
}
}
catch (HttpRequestException e)
{
}
}
}
}
}
if (details["posts"].ContainsKey("blogs") && details["posts"]["blogs"].Length > 0)
{
string title = details["posts"]["blogs"][0]["title"];
if (detector.Detect(title) == "en" && details["posts"]["blogs"][0]["url"] != null)
{
var request = new HttpRequestMessage(HttpMethod.Head, details["posts"]["blogs"][0]["url"]);
try
{
var validityResponse = await client.SendAsync(request);
if (validityResponse.IsSuccessStatusCode)
{
Words blogPost = new Words(title, details["posts"]["blogs"][0]["url"], scoreList[counter], WordType.Blog);
newsList.Add(blogPost);
}
}
catch (HttpRequestException e)
{
}
}
}
if (details["posts"].ContainsKey("video") && details["posts"]["video"].Length > 0)
{
string title = details["posts"]["video"][0]["title"];
if (detector.Detect(title) == "en" && details["posts"]["video"][0]["url"] != null)
{
var request = new HttpRequestMessage(HttpMethod.Head, details["posts"]["video"][0]["url"]);
try
{
var validityResponse = await client.SendAsync(request);
if (validityResponse.IsSuccessStatusCode)
{
Words video = new Words(title, details["posts"]["video"][0]["url"], scoreList[counter], WordType.Video);
newsList.Add(video);
}
}
catch (HttpRequestException e)
{
}
}
}
counter++;
}
client.Dispose();
return(newsList);
}
public TranslationService()
{
_detector = new LanguageDetector();
_detector.AddAllLanguages();
}
public DetectLanguageService()
{
detector = new LanguageDetector();
detector.AddAllLanguages();
}
private static void mergeMovies()
{
string statusText = "All Done";
try
{
using (FolderBrowserDialog Folder_Browser_Dialog = main.get_folderbrowserdialog())
{
if (Folder_Browser_Dialog == null)
{
return;
}
List <string> Merging_Movies = new List <string>();
List <string> commands = new List <string>();
List <string> commands_report = new List <string>();
if (new choice_box().ShowDialog() == DialogResult.OK)
{
main.change_button_text("Stop Merging", main.get_control_by_name("Merge_Button"), msgType.error);
if (choice.series)
{
//needs more tinkering
foreach (FileInfo fInfo in new DirectoryInfo(Folder_Browser_Dialog.SelectedPath).GetFiles("*.*", SearchOption.AllDirectories).Where(f => extension.videoEXT.Contains(f.Extension.ToLower())).ToArray())
{
commands.Add(@"""" + Application.StartupPath + @"\MKVToolNix\/k mkvmerge.exe --ui-language en --output ^""" + Properties.Settings.Default.merge_destination + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + @".mkv^"" --language 0:und --language 1:und ^""^(^"" ^""" + fInfo.DirectoryName + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + "" + fInfo.Extension + @"^"" ^""^)^"" --language 0:en --track-name ^""0:English Subtitle ^"" --default-track 0:yes --forced-track 0:yes ^""^(^"" ^""" + fInfo.DirectoryName + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + @".srt ^"" ^""^)^"" --track-order 0:0,0:1,1:0");
if (new DirectoryInfo(Folder_Browser_Dialog.SelectedPath).GetFiles(Path.GetFileNameWithoutExtension(fInfo.Name) + ".*", SearchOption.AllDirectories).Where(f => extension.subtitleEXT.Contains(f.Extension.ToLower())).ToArray().Count() > 0)
{
commands_report.Add("Done.");
}
else
{
commands_report.Add("Subtitle File Not Found.");
}
Merging_Movies.Add(Path.GetFileNameWithoutExtension(fInfo.Name));
main.log(@"""" + Application.StartupPath + @"\MKVToolNix\/k mkvmerge.exe --ui-language en --output ^""" + Properties.Settings.Default.merge_destination + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + @".mkv^"" --language 0:und --language 1:und ^""^(^"" ^""" + fInfo.DirectoryName + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + "" + fInfo.Extension + @"^"" ^""^)^"" --language 0:en --track-name ^""0:English Subtitle ^"" --default-track 0:yes --forced-track 0:yes ^""^(^"" ^""" + fInfo.DirectoryName + @"\" + Path.GetFileNameWithoutExtension(fInfo.Name) + @".srt ^"" ^""^)^"" --track-order 0:0,0:1,1:0", msgType.message);
}
}
else
{
List <DirectoryInfo> allDirectory = new List <DirectoryInfo>()
{
new DirectoryInfo(Folder_Browser_Dialog.SelectedPath)
};
if (choice.container)
{
allDirectory = new DirectoryInfo(Folder_Browser_Dialog.SelectedPath).GetDirectories().ToList();
}
foreach (DirectoryInfo dInfo in allDirectory)
{
FileInfo[] allFileInfo = new DirectoryInfo(dInfo.FullName).GetFiles("*.*", SearchOption.AllDirectories);
FileInfo[] vidInfo = allFileInfo.Where(f => extension.videoEXT.Contains(f.Extension.ToLower())).ToArray();
FileInfo[] srtInfo = allFileInfo.Where(f => extension.subtitleEXT.Contains(f.Extension.ToLower())).ToArray();
string vidName = Path.GetFileNameWithoutExtension(vidInfo[0].Name);
string commandStr = @"""" + Application.StartupPath + @"\MKVToolNix\/c mkvmerge.exe --ui-language en --output ^""" + Properties.Settings.Default.merge_destination + @"\" + Path.GetFileNameWithoutExtension(dInfo.Name) + @".mkv^"" --language 0:und --language 1:und ^""^(^"" ^""" + dInfo.FullName + @"\" + vidName + vidInfo[0].Extension + @"^"" ^""^)^"" ";
string trackOrder = "--track-order 0:0,0:1,";
foreach (FileInfo srt in srtInfo)
{
string dLanguage = string.Empty;
using (StreamReader sReader = new StreamReader(srt.FullName))
{
LanguageDetector detector = new LanguageDetector();
detector.AddAllLanguages();
dLanguage = detector.Detect(sReader.ReadToEnd());
}
int objIndex = Array.IndexOf(srtInfo, srt);
commandStr += @"--language " + objIndex + @":" + dLanguage + @" ";
if (dLanguage == "en")
{
commandStr += @"--default-track " + objIndex + @":yes --forced-track " + objIndex + @":yes ";
}
commandStr += @"^""^(^"" ^""" + dInfo.FullName + @"\" + Path.GetFileNameWithoutExtension(srt.Name) + srt.Extension + @" ^"" ^""^)^"" ";
trackOrder += (objIndex + 1) + ":0,";
}
commandStr += trackOrder.Substring(0, trackOrder.LastIndexOf(","));
if (srtInfo.Length > 0)
{
commands_report.Add("Done.");
}
else
{
commands_report.Add("Muxed without subtitle.");
}
commands.Add(commandStr);
Merging_Movies.Add(Path.GetFileNameWithoutExtension(dInfo.Name) + ".mkv");
}
}
main.max_progressbar(commands.Count * 100, main.ProgressBar);
main.max_progressbar(100, main.Mini_ProgressBar);
mergeInProgress = true;
foreach (string merge_command in commands)
{
main.log("Multiplexing Movie --> " + Merging_Movies[commands.IndexOf(merge_command)] + "... ", msgType.message);
main.change_status("Merging " + (currentIndex + 1) + " of " + commands.Count, msgType.message);
Process p = Process.Start(new ProcessStartInfo()
{
FileName = "cmd.exe", Arguments = merge_command, RedirectStandardInput = true, RedirectStandardOutput = true, UseShellExecute = false, CreateNoWindow = true, WindowStyle = ProcessWindowStyle.Hidden
});
p.OutputDataReceived += new DataReceivedEventHandler(process_OutputDataReceived);
p.BeginOutputReadLine();
do
{
try
{
if (cancellationPending)
{
p.CancelOutputRead();
foreach (Process mkvmerge in Process.GetProcessesByName("mkvmerge"))
{
mkvmerge.Kill();
}
p.Kill();
p.Close();
statusText = "Canceled by user.";
main.log("Canceled", msgType.error, true);
goto Finish;
}
}
catch (Exception) { }
} while (!p.HasExited);
p.WaitForExit();
currentIndex += 1;
if (commands_report[commands.IndexOf(merge_command)] == "Done.")
{
main.log(commands_report[commands.IndexOf(merge_command)], msgType.success, true);
}
else
{
main.log(commands_report[commands.IndexOf(merge_command)], msgType.error, true);
}
}
}
}
Finish :;
}
catch (Exception ex)
{
main.log(ex.ToString(), msgType.error);
statusText = "Error";
}
finally
{
main.update_progressbar(0, main.ProgressBar);
main.update_progressbar(0, main.Mini_ProgressBar);
if (statusText == "Error")
{
main.change_status(statusText, msgType.error);
}
else
{
main.change_status(statusText, msgType.success);
}
mergeInProgress = false;
main.change_button_text("Merge Movies", main.get_control_by_name("Merge_Button"), msgType.baseColor);
}
}
public void Issue_2()
{
string text = "Výsledky kola švýcarské hokejové ligy";
LanguageDetector detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.AddAllLanguages();
Assert.AreEqual("sk", detector.Detect(text));
Assert.AreEqual(1, detector.DetectAll(text).Count());
detector = new LanguageDetector();
detector.RandomSeed = 1;
detector.ConvergenceThreshold = 0.9;
detector.MaxIterations = 50;
detector.AddAllLanguages();
Assert.AreEqual("sk", detector.Detect(text));
Assert.AreEqual(2, detector.DetectAll(text).Count());
}
