private void Form1_Load(object sender, EventArgs e) { //fastBasePath = fastMiniBasePath; // use part of datas Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; Directory.CreateDirectory(fastBasePath); CheckCache("apps.txt"); CheckCache("developers.txt"); CheckCache("video.txt"); apps = AppData.load_from_file(fastBasePath + "apps.txt"); developers = DeveloperData.Deserialize(fastBasePath + "developers.txt"); videos = YoutubeData.load_from_file(fastBasePath + "video.txt"); videos = videos.DistinctBy(v => v.videoId).ToList(); genreComboBox.Items.AddRange(Genres.All.ToArray()); Connect(); //SavePart(); //SymbolsInName(); //NameWordsCount(false); //PopularNameWords(); <--- //AppsByGenres(true); //SimpleDevelopersAnalisis(false); //SimpleDevelopersAnalisis(true); //DevelopersAnalisis("Аркады и экшн"); //YoutubeAnalisis(); GenerateReport(); }
protected override void OnFinished() { AppData.save_to_file(apps, _appsFileName); DeveloperData.Serialize(developers, _developersFileName); SaveExceptions(_exceptionsPath, _exceptions); YoutubeData.save_to_file(_videos, _videoFileName); }
public DeveloperVote(DeveloperData developer, string genre, DateTime date, int addition) { Developer = developer; Genre = genre; Date = date; Addition = addition; }
protected override IEnumerator LoadSceneData(AsyncOperation op) { yield return(base.LoadSceneData(op)); if (_dataGO != null) { _back = _dataGO.GetComponent <DeveloperData>(); if (_back != null) { _back.BackBtn.onClick.AddListener(OnBackClicked); } else { Debug.Log("_Back is null"); } } else { Debug.Log("_dataGO is null"); } }
void SavePart(int step = 10) { List <AppData> appsPart = new List <AppData>(); List <DeveloperData> developersPart = new List <DeveloperData>(); List <YoutubeData> videosPart = new List <YoutubeData>(); foreach (DeveloperData developer in developers.TakeEvery(step)) { developersPart.Add(developer); foreach (AppData app in developer.apps) { appsPart.Add(app); if (app.video != null) { videosPart.Add(app.video); } } } AppData.save_to_file(appsPart, fastMiniBasePath + "apps.txt"); YoutubeData.save_to_file(videosPart, fastMiniBasePath + "video.txt"); DeveloperData.Serialize(developersPart, fastMiniBasePath + "developers.txt"); }
protected override List <string> ProcessUrlAndGetLinks(WebClient webClient, string current_url, int processId) { progress.Increment(); progress.LogEvery(100); //Console.WriteLine("\n({0}, {1}) ID={3} download {2}", visited_urls.Count, urls_to_visit.Count, current_url, processId); string stage = null; string html = null; List <string> filtered_links = null; try { stage = "downloading"; html = webClient.DownloadString(current_url); HtmlDocument document = new HtmlDocument(); stage = "loading_html_document"; document.LoadHtml(html); HtmlNode root = document.DocumentNode; stage = "get_links"; List <string> links = root.SelectNodes("//a[@href]") .Select(el => normalize_url(current_url, el.GetAttributeValue("href", null))) .ToList(); filtered_links = links.Where(good_url).ToList(); if (AppData.IsAppUrl(current_url)) { stage = "parse_app"; AppData app = new AppData(document, current_url); lock (apps) { apps.Add(app); } SavePage(_appsPath, html, app.Id); stage = "parse_youtube_page"; ProcessVideoId(webClient, app.videoID); } else if (DeveloperData.IsDeveloperUrl(current_url)) { stage = "parse_developer"; DeveloperData developer = new DeveloperData(document, current_url); lock (developers) { developers.Add(developer); } SavePage(_developersPath, html, developer.name); } else { SavePage(_otherPagesPath, html); } } catch (Exception e) { AddException(e, current_url, stage); SaveFaultyPage(html, stage, e); } return(filtered_links ?? new List <string>()); }