Ejemplo n.º 1
0
        private void CheckForSources()
        {
            if (userId == 339)
            {
                return;               //pageadmin
            }
            string srcId = FindFBSource(activity.RawActivity.BodyStr);

            if (!string.IsNullOrEmpty(srcId))
            {
                Logger.Instance.Debug($"found fb source {srcId} in post.looking for page");

                PageSource record = PageSources.GetBySourceId(srcId, "fb_page");
                if (record == null)
                {
                    //create a page and assign this action to it
                    Task.Run(() => CreatePageAndAssignActivity(srcId, activity.ActivityId));
                }
                else
                {
                    //like or increase like to this page
                    LikeSourceIfPossible(record.page_id, "sitepage_page");
                }
            }
        }
Ejemplo n.º 2
0
        public void DoFetching(int checkingIntervalInMinutes = 1)
        {
            //load sourcelist
            //PageSources.Add(390, "1154233056", "Tal Gilad", "fb_user");

            while (IsActive)
            {
                //go through sources
                foreach (var source in PageSources.GetByType("fb_page").Select(s => s.source_id).ToList())
                {
                    FetchPosts(source, "fb_page");
                }
                foreach (var source in PageSources.GetByType("fb_user").Select(s => s.source_id).ToList())
                {
                    FetchPosts(source, "fb_user");
                }

                //update last fetch time
                _LastFetchTime = DateTime.Now;
                if (!IsActive)
                {
                    return;
                }
                //go to sleep
                int sleepSeconds = checkingIntervalInMinutes * 60;
                while (sleepSeconds-- > 0 && IsActive)
                {
                    Thread.Sleep(1000);
                }
            }

            Logger.Instance.Info("Fetching process ended !");
        }
Ejemplo n.º 3
0
        private void CreatePageAndAssignActivity(string fbSrcId, ulong actionId)
        {
            try
            {
                Logger.Instance.Debug($"Creating page for {fbSrcId}");

                var fbPage = FacebookService.Default.GetPageAsync(fbSrcId).Result;
                //get photo stream
                var photo = FacebookService.Default.Client.GetStreamContentAsync($"{fbSrcId}/picture", "type=large").Result;
                // Create koobeca page
                var pageId = KoobecaService.Default.CreatePageAsync(fbPage.name, fbPage.link, fbPage.description, fbPage.categoty, new System.IO.Stream[1] {
                    photo
                });
                if (pageId > 0)
                {
                    PageSources.Add((ulong)pageId, fbSrcId, fbPage.name, "fb_page");
                    LikeSourceIfPossible((uint)pageId, "sitepage_page");
                }
            }
            catch (Exception e)
            {
                Logger.Instance.Error($"{e.ToString()}");
            }
        }
Ejemplo n.º 4
0
 protected abstract string GetRecordSourcePageString(IAudioBook audioBook, PageSources pageSource,
                                                     CancellationToken cancellationToken);
Ejemplo n.º 5
0
 protected abstract void DownloadRecord(IDbContainer db, IAudioBook record, PageSources pageSource,
                                        CancellationToken cancellationToken);
Ejemplo n.º 6
0
        public async Task DownloadRecords(IReadOnlyCollection <string> recordsKeys, PageSources pageSource,
                                          CancellationToken cancellationToken)
        {
            var mainConfig = Config.Load <MainConfig>();

            await Task.Factory.StartNew(() =>
            {
                var recordActualityPeriod = mainConfig.RecordActualityPeriod;

                using (var autoSave = Context.I.DbContainerAutoSave)
                {
                    var dbContainer = autoSave.DBContainer;

                    IAudioBook[] records;

                    if (recordsKeys == null)
                    {
                        if (pageSource == PageSources.CacheOnly)
                        {
                            records = dbContainer.AudioBookSet.GetRecordsByWebSite(WebSiteId).ToArray();
                        }
                        else
                        {
                            records =
                                dbContainer.AudioBookSet.GetRecordsUpdatedBefore(WebSiteId,
                                                                                 DateTime.Now.Subtract(TimeSpan.FromDays(recordActualityPeriod))).ToArray();
                        }
                    }
                    else
                    {
                        records = dbContainer.AudioBookSet
                                  .GetRecordsByKeys(recordsKeys as HashSet <string> ?? recordsKeys.ToHashSet()).ToArray();
                    }

                    var sw = new Stopwatch();
                    sw.Start();

                    var normalizerPlugin =
                        Context.I.ComponentFactory.GetCreators <INormalizationLogicPlugin>()
                        .First()
                        .GetInstance <INormalizationLogicPlugin>();

                    var waitingForSave = new List <IAudioBook>();

                    for (var z = 0; z < records.Length; z++)
                    {
                        try
                        {
                            var record = records[z];
                            ProgressMessage.ReportComplex(z, records.Length, $"{DisplayName}: {z} / {records.Length}");
                            DownloadRecord(dbContainer, record, pageSource, cancellationToken);
                            record.LastUpdate = DateTime.Now;
                            waitingForSave.Add(record);

                            if (record.Created == default(DateTime))
                            {
                                record.Created = DateTime.Now;
                            }

                            if (cancellationToken.IsCancellationRequested)
                            {
                                break;
                            }
                            if (sw.Elapsed > TimeSpan.FromSeconds(30) || z == records.Length - 1)
                            {
                                sw.Restart();
                                normalizerPlugin.Normalize(waitingForSave, dbContainer);

                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }
                                dbContainer.AudioBookSet.AddChangedRecords(waitingForSave.ToArray());
                                dbContainer.SaveChanges();
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Context.I.MainLog.Error(ex);
                        }
                    }

                    ProgressMessage.ReportComplex(records.Length, records.Length,
                                                  $"{DisplayName}: {records.Length} / {records.Length}");

                    if (!cancellationToken.IsCancellationRequested)
                    {
                        dbContainer.SaveChanges();
                    }
                    else
                    {
                        dbContainer.AutoSaveChanges = false;
                    }
                }
            }, cancellationToken);
        }
        protected override string GetRecordSourcePageString([NotNull] IAudioBook audioBook, PageSources pageSource,
                                                            CancellationToken cancellationToken)
        {
            string result = null;

            using (var autoSave = Context.I.DbContainerAutoSave)
            {
                var dbContainer   = autoSave.DBContainer;
                var recordPageKey = audioBook.GetPageKey();
                var pageData      = pageSource == PageSources.WebOnly
                    ? null
                    : dbContainer.BinaryDataSet.GetByKey(recordPageKey);

                if (pageData != null)
                {
                    var data = pageData.GetData();
                    using (var ms = new MemoryStream(data))
                    {
                        var document = new HtmlDocument();
                        document.Load(ms);
                        if (cancellationToken.IsCancellationRequested)
                        {
                            return(null);
                        }

                        CleanupRecordPage(document);
                        result = document.DocumentNode.InnerHtml;

                        if (cancellationToken.IsCancellationRequested)
                        {
                            return(null);
                        }
                        pageData     = dbContainer.BinaryDataSet.CreateBinaryData();
                        pageData.Key = recordPageKey;
                        pageData.SetString(result, true);
                        dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
                    }
                }
                else if (pageSource != PageSources.CacheOnly)
                {
                    using (var webClient = WebClientPool.GetPoolItem())
                    {
                        var pageUrl = GetRecordPageUrl(audioBook);
                        result = webClient.Target.DownloadString(pageUrl);

                        var document = new HtmlDocument();
                        document.LoadHtml(result);
                        if (cancellationToken.IsCancellationRequested)
                        {
                            return(null);
                        }

                        CleanupRecordPage(document);
                        result = document.DocumentNode.InnerHtml;

                        if (cancellationToken.IsCancellationRequested)
                        {
                            return(null);
                        }
                        pageData     = dbContainer.BinaryDataSet.CreateBinaryData();
                        pageData.Key = recordPageKey;
                        pageData.SetString(result, true);
                        dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
                    }
                }
            }

            return(result);
        }
        protected override void DownloadRecord(IDbContainer dbContainer, IAudioBook record, PageSources pageSource,
                                               CancellationToken cancellationToken)
        {
            string pageHtml = null;

            if (pageSource != PageSources.WebOnly)
            {
                //var pageMetaId = record.GetPageMetaKey();
                //var metaPage = dbContainer.BinaryDataSet.GetByKey(pageMetaId);
                //if (metaPage != null)
                //{
                //    pageHtml = metaPage.GetString();
                //}

                var page = dbContainer.BinaryDataSet.GetByKey(record.GetPageKey());

                if (page != null)
                {
                    using (var ms = new MemoryStream(page.GetData()))
                    {
                        var doc = new HtmlDocument();
                        doc.Load(ms);
                        var savedHtml = doc.DocumentNode.OuterHtml;
                        CleanupRecordPage(doc);
                        pageHtml = doc.DocumentNode.InnerHtml;

                        if (savedHtml != pageHtml)
                        {
                            page.SetString(doc.DocumentNode.InnerHtml, true);
                            dbContainer.BinaryDataSet.AddChangedBinaryData(page);
                        }
                    }
                }
            }

            if (pageHtml.IsNullOrEmpty() && pageSource != PageSources.CacheOnly)
            {
                pageHtml = DownloadRecordMetaPageFromWeb(record, dbContainer, cancellationToken);
            }

            if (pageHtml.IsNullOrEmpty() || cancellationToken.IsCancellationRequested)
            {
                return;
            }

            ParseRecord(dbContainer, record, pageHtml);
        }
Ejemplo n.º 9
0
 private void AddPublicFigures()
 {
     PageSources.GetByType("fb_user").Select(s => AddSource(SourcesMap.Instance.GetSource(s.page_id, "user"))).ToArray();
 }
Ejemplo n.º 10
0
        public void DoPosting(string user, string password)
        {
            //login
            var koobecaClient  = new KoobecaClient();
            var koobecaService = new KoobecaService(koobecaClient);
            var getAccountTask = koobecaService.GetAccountAsync(user, password);

            Task.WaitAll(getAccountTask);
            var account = getAccountTask.Result;

            if (string.IsNullOrEmpty(account.oauth_token))
            {
                Logger.Instance.Error("Failed to login to koobeca");
                return;
            }

            //load the cache
            PageSources.GetByType("fb_page").Select(p => _PageSourceCache[p.source_id] = p).ToArray();
            Logger.Instance.Debug($"page source cache loaded with {_PageSourceCache.Count} items.");

            //take from the queue
            while (IsActive)
            {
                if (_PostQueue.Count > 0)
                {
                    var item = _PostQueue.Dequeue();

                    if (item.Type == FBPostType.Picture)
                    {
                        try
                        {
                            //make the picture include the bigger text
                            StringBuilder bigMsg = new StringBuilder();
                            bigMsg.AppendLine(item.message);
                            bigMsg.AppendLine("------------------------");
                            bigMsg.AppendLine(item.name);
                            bigMsg.AppendLine(item.description);
                            item.message = bigMsg.ToString();

                            //get all puctures
                            List <string> picUrls = new List <string>();
                            picUrls.AddRange(item.attachments.data.Where(a => a.type == "photo").Select(a => a.media.image.src).ToList());
                            picUrls.AddRange(item.attachments.data.Where(a => a.type == "album").SelectMany(a => a.subattachments.data.Where(sa => sa.type == "photo").Select(sa => sa.media.image.src)).ToList());
                            var webClient = new WebClient();

                            item.Photos = new Stream[picUrls.Count];
                            for (int i = 0; i < picUrls.Count; i++)
                            {
                                var    url        = picUrls[i];
                                byte[] imageBytes = webClient.DownloadData(url);
                                item.Photos[i] = new MemoryStream(imageBytes);
                            }
                        }
                        catch (Exception e)
                        {
                            Logger.Instance.Error($"failed to get photo stream from {item.full_picture} , {e.Message}");
                        }
                    }
                    Logger.Instance.Info($"Got post from FB source:{item.source} of type {item.Type} , looking for target in Koobeca");
                    double timeSpent;
                    PublishPost(_KoobecaService, item, out timeSpent);
                    Published?.Invoke(item.source, timeSpent / 1000); // throw event to listeners
                }
                else
                {
                    Thread.Sleep(1000);
                }
            }

            Logger.Instance.Info($"Publishing ended.");
        }
Ejemplo n.º 11
0
        private bool PublishPost(KoobecaService service, FBPost item, out double timeSpanMiliSec)
        {
            DateTime start = DateTime.Now;

            CurrentlyPublished = item;
            try
            {
                //if (string.IsNullOrEmpty(item.Link))
                //    item.Link = item.URL;
                Logger.Instance.Info($"Publishing {item.source_type} {item.source} : {item.message} : link {item.link}");

                if (item.Photos == null)
                {
                    var effectiveLink = UrlUnshortener.GetRealUrl(item.link);
                    if (effectiveLink != item.link)
                    {
                        Logger.Instance.Info($"effective link is: {effectiveLink}");
                        item.link = effectiveLink;
                    }
                }
                else
                {
                    //no link in case of photo
                    item.link = string.Empty;
                }

                if (!_PageSourceCache.TryGetValue(item.source, out PageSource source))
                {
                    source = PageSources.GetBySourceId(item.source, item.source_type);
                    if (source == null)
                    {
                        throw new Exception($"{item.source}:{item.link} not found in page ids");
                    }
                    else
                    {
                        //add it to the cache
                        _PageSourceCache[item.source] = source;
                    }
                }


                ulong pageId = source.page_id; //target id (legacy is page)
                if (ActualPublishing)
                {
                    KoobecaService.PostTarget targetType = source.source_type == "fb_page" ? KoobecaService.PostTarget.Page : KoobecaService.PostTarget.User;
                    int videoId = 0;
                    if (item.link.Contains(".youtube.com"))
                    {
                        //post video
                        var vidReult = service.PostVideoAsync("1", "everyone", item.name, item.description, item.link).Result;
                        if (!vidReult.error)
                        {
                            videoId = vidReult.body.response.video_id;
                        }
                    }

                    var result = service.PostOnWallAsync(item.Privacy, videoId, item.message, item.link, targetType, (int)pageId, item.Photos).Result;
                    if (result.error)
                    {
                        Logger.Instance.Error(result.ToString());
                    }
                }
                else
                {
                    Thread.Sleep(1000); // just sleep second
                }

                timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds;
                Logger.Instance.Info($"Time Spent in {timeSpanMiliSec} ms");

                //File.AppendAllText(_PublishedPath,item.ID + "\n");
                PublishCount++;
                return(true);
            }
            catch (Exception e)
            {
                Logger.Instance.Error(e.ToString());
                PublishErrorCount++;
                timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds;
                return(false);
            }
            finally
            {
                CurrentlyPublished = null;
            }
        }