private void CheckForSources() { if (userId == 339) { return; //pageadmin } string srcId = FindFBSource(activity.RawActivity.BodyStr); if (!string.IsNullOrEmpty(srcId)) { Logger.Instance.Debug($"found fb source {srcId} in post.looking for page"); PageSource record = PageSources.GetBySourceId(srcId, "fb_page"); if (record == null) { //create a page and assign this action to it Task.Run(() => CreatePageAndAssignActivity(srcId, activity.ActivityId)); } else { //like or increase like to this page LikeSourceIfPossible(record.page_id, "sitepage_page"); } } }
public void DoFetching(int checkingIntervalInMinutes = 1) { //load sourcelist //PageSources.Add(390, "1154233056", "Tal Gilad", "fb_user"); while (IsActive) { //go through sources foreach (var source in PageSources.GetByType("fb_page").Select(s => s.source_id).ToList()) { FetchPosts(source, "fb_page"); } foreach (var source in PageSources.GetByType("fb_user").Select(s => s.source_id).ToList()) { FetchPosts(source, "fb_user"); } //update last fetch time _LastFetchTime = DateTime.Now; if (!IsActive) { return; } //go to sleep int sleepSeconds = checkingIntervalInMinutes * 60; while (sleepSeconds-- > 0 && IsActive) { Thread.Sleep(1000); } } Logger.Instance.Info("Fetching process ended !"); }
private void CreatePageAndAssignActivity(string fbSrcId, ulong actionId) { try { Logger.Instance.Debug($"Creating page for {fbSrcId}"); var fbPage = FacebookService.Default.GetPageAsync(fbSrcId).Result; //get photo stream var photo = FacebookService.Default.Client.GetStreamContentAsync($"{fbSrcId}/picture", "type=large").Result; // Create koobeca page var pageId = KoobecaService.Default.CreatePageAsync(fbPage.name, fbPage.link, fbPage.description, fbPage.categoty, new System.IO.Stream[1] { photo }); if (pageId > 0) { PageSources.Add((ulong)pageId, fbSrcId, fbPage.name, "fb_page"); LikeSourceIfPossible((uint)pageId, "sitepage_page"); } } catch (Exception e) { Logger.Instance.Error($"{e.ToString()}"); } }
protected abstract string GetRecordSourcePageString(IAudioBook audioBook, PageSources pageSource, CancellationToken cancellationToken);
protected abstract void DownloadRecord(IDbContainer db, IAudioBook record, PageSources pageSource, CancellationToken cancellationToken);
public async Task DownloadRecords(IReadOnlyCollection <string> recordsKeys, PageSources pageSource, CancellationToken cancellationToken) { var mainConfig = Config.Load <MainConfig>(); await Task.Factory.StartNew(() => { var recordActualityPeriod = mainConfig.RecordActualityPeriod; using (var autoSave = Context.I.DbContainerAutoSave) { var dbContainer = autoSave.DBContainer; IAudioBook[] records; if (recordsKeys == null) { if (pageSource == PageSources.CacheOnly) { records = dbContainer.AudioBookSet.GetRecordsByWebSite(WebSiteId).ToArray(); } else { records = dbContainer.AudioBookSet.GetRecordsUpdatedBefore(WebSiteId, DateTime.Now.Subtract(TimeSpan.FromDays(recordActualityPeriod))).ToArray(); } } else { records = dbContainer.AudioBookSet .GetRecordsByKeys(recordsKeys as HashSet <string> ?? recordsKeys.ToHashSet()).ToArray(); } var sw = new Stopwatch(); sw.Start(); var normalizerPlugin = Context.I.ComponentFactory.GetCreators <INormalizationLogicPlugin>() .First() .GetInstance <INormalizationLogicPlugin>(); var waitingForSave = new List <IAudioBook>(); for (var z = 0; z < records.Length; z++) { try { var record = records[z]; ProgressMessage.ReportComplex(z, records.Length, $"{DisplayName}: {z} / {records.Length}"); DownloadRecord(dbContainer, record, pageSource, cancellationToken); record.LastUpdate = DateTime.Now; waitingForSave.Add(record); if (record.Created == default(DateTime)) { record.Created = DateTime.Now; } if (cancellationToken.IsCancellationRequested) { break; } if (sw.Elapsed > TimeSpan.FromSeconds(30) || z == records.Length - 1) { sw.Restart(); normalizerPlugin.Normalize(waitingForSave, dbContainer); if (cancellationToken.IsCancellationRequested) { break; } dbContainer.AudioBookSet.AddChangedRecords(waitingForSave.ToArray()); dbContainer.SaveChanges(); if (cancellationToken.IsCancellationRequested) { break; } } } catch (Exception ex) { Context.I.MainLog.Error(ex); } } ProgressMessage.ReportComplex(records.Length, records.Length, $"{DisplayName}: {records.Length} / {records.Length}"); if (!cancellationToken.IsCancellationRequested) { dbContainer.SaveChanges(); } else { dbContainer.AutoSaveChanges = false; } } }, cancellationToken); }
protected override string GetRecordSourcePageString([NotNull] IAudioBook audioBook, PageSources pageSource, CancellationToken cancellationToken) { string result = null; using (var autoSave = Context.I.DbContainerAutoSave) { var dbContainer = autoSave.DBContainer; var recordPageKey = audioBook.GetPageKey(); var pageData = pageSource == PageSources.WebOnly ? null : dbContainer.BinaryDataSet.GetByKey(recordPageKey); if (pageData != null) { var data = pageData.GetData(); using (var ms = new MemoryStream(data)) { var document = new HtmlDocument(); document.Load(ms); if (cancellationToken.IsCancellationRequested) { return(null); } CleanupRecordPage(document); result = document.DocumentNode.InnerHtml; if (cancellationToken.IsCancellationRequested) { return(null); } pageData = dbContainer.BinaryDataSet.CreateBinaryData(); pageData.Key = recordPageKey; pageData.SetString(result, true); dbContainer.BinaryDataSet.AddChangedBinaryData(pageData); } } else if (pageSource != PageSources.CacheOnly) { using (var webClient = WebClientPool.GetPoolItem()) { var pageUrl = GetRecordPageUrl(audioBook); result = webClient.Target.DownloadString(pageUrl); var document = new HtmlDocument(); document.LoadHtml(result); if (cancellationToken.IsCancellationRequested) { return(null); } CleanupRecordPage(document); result = document.DocumentNode.InnerHtml; if (cancellationToken.IsCancellationRequested) { return(null); } pageData = dbContainer.BinaryDataSet.CreateBinaryData(); pageData.Key = recordPageKey; pageData.SetString(result, true); dbContainer.BinaryDataSet.AddChangedBinaryData(pageData); } } } return(result); }
protected override void DownloadRecord(IDbContainer dbContainer, IAudioBook record, PageSources pageSource, CancellationToken cancellationToken) { string pageHtml = null; if (pageSource != PageSources.WebOnly) { //var pageMetaId = record.GetPageMetaKey(); //var metaPage = dbContainer.BinaryDataSet.GetByKey(pageMetaId); //if (metaPage != null) //{ // pageHtml = metaPage.GetString(); //} var page = dbContainer.BinaryDataSet.GetByKey(record.GetPageKey()); if (page != null) { using (var ms = new MemoryStream(page.GetData())) { var doc = new HtmlDocument(); doc.Load(ms); var savedHtml = doc.DocumentNode.OuterHtml; CleanupRecordPage(doc); pageHtml = doc.DocumentNode.InnerHtml; if (savedHtml != pageHtml) { page.SetString(doc.DocumentNode.InnerHtml, true); dbContainer.BinaryDataSet.AddChangedBinaryData(page); } } } } if (pageHtml.IsNullOrEmpty() && pageSource != PageSources.CacheOnly) { pageHtml = DownloadRecordMetaPageFromWeb(record, dbContainer, cancellationToken); } if (pageHtml.IsNullOrEmpty() || cancellationToken.IsCancellationRequested) { return; } ParseRecord(dbContainer, record, pageHtml); }
private void AddPublicFigures() { PageSources.GetByType("fb_user").Select(s => AddSource(SourcesMap.Instance.GetSource(s.page_id, "user"))).ToArray(); }
public void DoPosting(string user, string password) { //login var koobecaClient = new KoobecaClient(); var koobecaService = new KoobecaService(koobecaClient); var getAccountTask = koobecaService.GetAccountAsync(user, password); Task.WaitAll(getAccountTask); var account = getAccountTask.Result; if (string.IsNullOrEmpty(account.oauth_token)) { Logger.Instance.Error("Failed to login to koobeca"); return; } //load the cache PageSources.GetByType("fb_page").Select(p => _PageSourceCache[p.source_id] = p).ToArray(); Logger.Instance.Debug($"page source cache loaded with {_PageSourceCache.Count} items."); //take from the queue while (IsActive) { if (_PostQueue.Count > 0) { var item = _PostQueue.Dequeue(); if (item.Type == FBPostType.Picture) { try { //make the picture include the bigger text StringBuilder bigMsg = new StringBuilder(); bigMsg.AppendLine(item.message); bigMsg.AppendLine("------------------------"); bigMsg.AppendLine(item.name); bigMsg.AppendLine(item.description); item.message = bigMsg.ToString(); //get all puctures List <string> picUrls = new List <string>(); picUrls.AddRange(item.attachments.data.Where(a => a.type == "photo").Select(a => a.media.image.src).ToList()); picUrls.AddRange(item.attachments.data.Where(a => a.type == "album").SelectMany(a => a.subattachments.data.Where(sa => sa.type == "photo").Select(sa => sa.media.image.src)).ToList()); var webClient = new WebClient(); item.Photos = new Stream[picUrls.Count]; for (int i = 0; i < picUrls.Count; i++) { var url = picUrls[i]; byte[] imageBytes = webClient.DownloadData(url); item.Photos[i] = new MemoryStream(imageBytes); } } catch (Exception e) { Logger.Instance.Error($"failed to get photo stream from {item.full_picture} , {e.Message}"); } } Logger.Instance.Info($"Got post from FB source:{item.source} of type {item.Type} , looking for target in Koobeca"); double timeSpent; PublishPost(_KoobecaService, item, out timeSpent); Published?.Invoke(item.source, timeSpent / 1000); // throw event to listeners } else { Thread.Sleep(1000); } } Logger.Instance.Info($"Publishing ended."); }
private bool PublishPost(KoobecaService service, FBPost item, out double timeSpanMiliSec) { DateTime start = DateTime.Now; CurrentlyPublished = item; try { //if (string.IsNullOrEmpty(item.Link)) // item.Link = item.URL; Logger.Instance.Info($"Publishing {item.source_type} {item.source} : {item.message} : link {item.link}"); if (item.Photos == null) { var effectiveLink = UrlUnshortener.GetRealUrl(item.link); if (effectiveLink != item.link) { Logger.Instance.Info($"effective link is: {effectiveLink}"); item.link = effectiveLink; } } else { //no link in case of photo item.link = string.Empty; } if (!_PageSourceCache.TryGetValue(item.source, out PageSource source)) { source = PageSources.GetBySourceId(item.source, item.source_type); if (source == null) { throw new Exception($"{item.source}:{item.link} not found in page ids"); } else { //add it to the cache _PageSourceCache[item.source] = source; } } ulong pageId = source.page_id; //target id (legacy is page) if (ActualPublishing) { KoobecaService.PostTarget targetType = source.source_type == "fb_page" ? KoobecaService.PostTarget.Page : KoobecaService.PostTarget.User; int videoId = 0; if (item.link.Contains(".youtube.com")) { //post video var vidReult = service.PostVideoAsync("1", "everyone", item.name, item.description, item.link).Result; if (!vidReult.error) { videoId = vidReult.body.response.video_id; } } var result = service.PostOnWallAsync(item.Privacy, videoId, item.message, item.link, targetType, (int)pageId, item.Photos).Result; if (result.error) { Logger.Instance.Error(result.ToString()); } } else { Thread.Sleep(1000); // just sleep second } timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds; Logger.Instance.Info($"Time Spent in {timeSpanMiliSec} ms"); //File.AppendAllText(_PublishedPath,item.ID + "\n"); PublishCount++; return(true); } catch (Exception e) { Logger.Instance.Error(e.ToString()); PublishErrorCount++; timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds; return(false); } finally { CurrentlyPublished = null; } }