private void Form1_Load(object sender, EventArgs e)
{
//fastBasePath = fastMiniBasePath; // use part of datas
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
Directory.CreateDirectory(fastBasePath);
CheckCache("apps.txt");
CheckCache("developers.txt");
CheckCache("video.txt");
apps = AppData.load_from_file(fastBasePath + "apps.txt");
developers = DeveloperData.Deserialize(fastBasePath + "developers.txt");
videos = YoutubeData.load_from_file(fastBasePath + "video.txt");
videos = videos.DistinctBy(v => v.videoId).ToList();
genreComboBox.Items.AddRange(Genres.All.ToArray());
Connect();
//SavePart();
//SymbolsInName();
//NameWordsCount(false);
//PopularNameWords(); <---
//AppsByGenres(true);
//SimpleDevelopersAnalisis(false);
//SimpleDevelopersAnalisis(true);
//DevelopersAnalisis("Аркады и экшн");
//YoutubeAnalisis();
GenerateReport();
}
protected override void OnFinished()
{
AppData.save_to_file(apps, _appsFileName);
DeveloperData.Serialize(developers, _developersFileName);
SaveExceptions(_exceptionsPath, _exceptions);
YoutubeData.save_to_file(_videos, _videoFileName);
}
public DeveloperVote(DeveloperData developer, string genre, DateTime date, int addition)
{
Developer = developer;
Genre = genre;
Date = date;
Addition = addition;
}
protected override IEnumerator LoadSceneData(AsyncOperation op)
{
yield return(base.LoadSceneData(op));
if (_dataGO != null)
{
_back = _dataGO.GetComponent <DeveloperData>();
if (_back != null)
{
_back.BackBtn.onClick.AddListener(OnBackClicked);
}
else
{
Debug.Log("_Back is null");
}
}
else
{
Debug.Log("_dataGO is null");
}
}
void SavePart(int step = 10)
{
List <AppData> appsPart = new List <AppData>();
List <DeveloperData> developersPart = new List <DeveloperData>();
List <YoutubeData> videosPart = new List <YoutubeData>();
foreach (DeveloperData developer in developers.TakeEvery(step))
{
developersPart.Add(developer);
foreach (AppData app in developer.apps)
{
appsPart.Add(app);
if (app.video != null)
{
videosPart.Add(app.video);
}
}
}
AppData.save_to_file(appsPart, fastMiniBasePath + "apps.txt");
YoutubeData.save_to_file(videosPart, fastMiniBasePath + "video.txt");
DeveloperData.Serialize(developersPart, fastMiniBasePath + "developers.txt");
}
protected override List <string> ProcessUrlAndGetLinks(WebClient webClient, string current_url, int processId)
{
progress.Increment();
progress.LogEvery(100);
//Console.WriteLine("\n({0}, {1}) ID={3} download {2}", visited_urls.Count, urls_to_visit.Count, current_url, processId);
string stage = null;
string html = null;
List <string> filtered_links = null;
try
{
stage = "downloading";
html = webClient.DownloadString(current_url);
HtmlDocument document = new HtmlDocument();
stage = "loading_html_document";
document.LoadHtml(html);
HtmlNode root = document.DocumentNode;
stage = "get_links";
List <string> links =
root.SelectNodes("//a[@href]")
.Select(el => normalize_url(current_url, el.GetAttributeValue("href", null)))
.ToList();
filtered_links = links.Where(good_url).ToList();
if (AppData.IsAppUrl(current_url))
{
stage = "parse_app";
AppData app = new AppData(document, current_url);
lock (apps)
{
apps.Add(app);
}
SavePage(_appsPath, html, app.Id);
stage = "parse_youtube_page";
ProcessVideoId(webClient, app.videoID);
}
else if (DeveloperData.IsDeveloperUrl(current_url))
{
stage = "parse_developer";
DeveloperData developer = new DeveloperData(document, current_url);
lock (developers)
{
developers.Add(developer);
}
SavePage(_developersPath, html, developer.name);
}
else
{
SavePage(_otherPagesPath, html);
}
}
catch (Exception e)
{
AddException(e, current_url, stage);
SaveFaultyPage(html, stage, e);
}
return(filtered_links ?? new List <string>());
}
