private void CheckForSources()
{
if (userId == 339)
{
return; //pageadmin
}
string srcId = FindFBSource(activity.RawActivity.BodyStr);
if (!string.IsNullOrEmpty(srcId))
{
Logger.Instance.Debug($"found fb source {srcId} in post.looking for page");
PageSource record = PageSources.GetBySourceId(srcId, "fb_page");
if (record == null)
{
//create a page and assign this action to it
Task.Run(() => CreatePageAndAssignActivity(srcId, activity.ActivityId));
}
else
{
//like or increase like to this page
LikeSourceIfPossible(record.page_id, "sitepage_page");
}
}
}
public void DoFetching(int checkingIntervalInMinutes = 1)
{
//load sourcelist
//PageSources.Add(390, "1154233056", "Tal Gilad", "fb_user");
while (IsActive)
{
//go through sources
foreach (var source in PageSources.GetByType("fb_page").Select(s => s.source_id).ToList())
{
FetchPosts(source, "fb_page");
}
foreach (var source in PageSources.GetByType("fb_user").Select(s => s.source_id).ToList())
{
FetchPosts(source, "fb_user");
}
//update last fetch time
_LastFetchTime = DateTime.Now;
if (!IsActive)
{
return;
}
//go to sleep
int sleepSeconds = checkingIntervalInMinutes * 60;
while (sleepSeconds-- > 0 && IsActive)
{
Thread.Sleep(1000);
}
}
Logger.Instance.Info("Fetching process ended !");
}
private void CreatePageAndAssignActivity(string fbSrcId, ulong actionId)
{
try
{
Logger.Instance.Debug($"Creating page for {fbSrcId}");
var fbPage = FacebookService.Default.GetPageAsync(fbSrcId).Result;
//get photo stream
var photo = FacebookService.Default.Client.GetStreamContentAsync($"{fbSrcId}/picture", "type=large").Result;
// Create koobeca page
var pageId = KoobecaService.Default.CreatePageAsync(fbPage.name, fbPage.link, fbPage.description, fbPage.categoty, new System.IO.Stream[1] {
photo
});
if (pageId > 0)
{
PageSources.Add((ulong)pageId, fbSrcId, fbPage.name, "fb_page");
LikeSourceIfPossible((uint)pageId, "sitepage_page");
}
}
catch (Exception e)
{
Logger.Instance.Error($"{e.ToString()}");
}
}
protected abstract string GetRecordSourcePageString(IAudioBook audioBook, PageSources pageSource,
CancellationToken cancellationToken);
protected abstract void DownloadRecord(IDbContainer db, IAudioBook record, PageSources pageSource,
CancellationToken cancellationToken);
public async Task DownloadRecords(IReadOnlyCollection <string> recordsKeys, PageSources pageSource,
CancellationToken cancellationToken)
{
var mainConfig = Config.Load <MainConfig>();
await Task.Factory.StartNew(() =>
{
var recordActualityPeriod = mainConfig.RecordActualityPeriod;
using (var autoSave = Context.I.DbContainerAutoSave)
{
var dbContainer = autoSave.DBContainer;
IAudioBook[] records;
if (recordsKeys == null)
{
if (pageSource == PageSources.CacheOnly)
{
records = dbContainer.AudioBookSet.GetRecordsByWebSite(WebSiteId).ToArray();
}
else
{
records =
dbContainer.AudioBookSet.GetRecordsUpdatedBefore(WebSiteId,
DateTime.Now.Subtract(TimeSpan.FromDays(recordActualityPeriod))).ToArray();
}
}
else
{
records = dbContainer.AudioBookSet
.GetRecordsByKeys(recordsKeys as HashSet <string> ?? recordsKeys.ToHashSet()).ToArray();
}
var sw = new Stopwatch();
sw.Start();
var normalizerPlugin =
Context.I.ComponentFactory.GetCreators <INormalizationLogicPlugin>()
.First()
.GetInstance <INormalizationLogicPlugin>();
var waitingForSave = new List <IAudioBook>();
for (var z = 0; z < records.Length; z++)
{
try
{
var record = records[z];
ProgressMessage.ReportComplex(z, records.Length, $"{DisplayName}: {z} / {records.Length}");
DownloadRecord(dbContainer, record, pageSource, cancellationToken);
record.LastUpdate = DateTime.Now;
waitingForSave.Add(record);
if (record.Created == default(DateTime))
{
record.Created = DateTime.Now;
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (sw.Elapsed > TimeSpan.FromSeconds(30) || z == records.Length - 1)
{
sw.Restart();
normalizerPlugin.Normalize(waitingForSave, dbContainer);
if (cancellationToken.IsCancellationRequested)
{
break;
}
dbContainer.AudioBookSet.AddChangedRecords(waitingForSave.ToArray());
dbContainer.SaveChanges();
if (cancellationToken.IsCancellationRequested)
{
break;
}
}
}
catch (Exception ex)
{
Context.I.MainLog.Error(ex);
}
}
ProgressMessage.ReportComplex(records.Length, records.Length,
$"{DisplayName}: {records.Length} / {records.Length}");
if (!cancellationToken.IsCancellationRequested)
{
dbContainer.SaveChanges();
}
else
{
dbContainer.AutoSaveChanges = false;
}
}
}, cancellationToken);
}
protected override string GetRecordSourcePageString([NotNull] IAudioBook audioBook, PageSources pageSource,
CancellationToken cancellationToken)
{
string result = null;
using (var autoSave = Context.I.DbContainerAutoSave)
{
var dbContainer = autoSave.DBContainer;
var recordPageKey = audioBook.GetPageKey();
var pageData = pageSource == PageSources.WebOnly
? null
: dbContainer.BinaryDataSet.GetByKey(recordPageKey);
if (pageData != null)
{
var data = pageData.GetData();
using (var ms = new MemoryStream(data))
{
var document = new HtmlDocument();
document.Load(ms);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
CleanupRecordPage(document);
result = document.DocumentNode.InnerHtml;
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
pageData = dbContainer.BinaryDataSet.CreateBinaryData();
pageData.Key = recordPageKey;
pageData.SetString(result, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
}
}
else if (pageSource != PageSources.CacheOnly)
{
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = GetRecordPageUrl(audioBook);
result = webClient.Target.DownloadString(pageUrl);
var document = new HtmlDocument();
document.LoadHtml(result);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
CleanupRecordPage(document);
result = document.DocumentNode.InnerHtml;
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
pageData = dbContainer.BinaryDataSet.CreateBinaryData();
pageData.Key = recordPageKey;
pageData.SetString(result, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
}
}
}
return(result);
}
protected override void DownloadRecord(IDbContainer dbContainer, IAudioBook record, PageSources pageSource,
CancellationToken cancellationToken)
{
string pageHtml = null;
if (pageSource != PageSources.WebOnly)
{
//var pageMetaId = record.GetPageMetaKey();
//var metaPage = dbContainer.BinaryDataSet.GetByKey(pageMetaId);
//if (metaPage != null)
//{
// pageHtml = metaPage.GetString();
//}
var page = dbContainer.BinaryDataSet.GetByKey(record.GetPageKey());
if (page != null)
{
using (var ms = new MemoryStream(page.GetData()))
{
var doc = new HtmlDocument();
doc.Load(ms);
var savedHtml = doc.DocumentNode.OuterHtml;
CleanupRecordPage(doc);
pageHtml = doc.DocumentNode.InnerHtml;
if (savedHtml != pageHtml)
{
page.SetString(doc.DocumentNode.InnerHtml, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(page);
}
}
}
}
if (pageHtml.IsNullOrEmpty() && pageSource != PageSources.CacheOnly)
{
pageHtml = DownloadRecordMetaPageFromWeb(record, dbContainer, cancellationToken);
}
if (pageHtml.IsNullOrEmpty() || cancellationToken.IsCancellationRequested)
{
return;
}
ParseRecord(dbContainer, record, pageHtml);
}
private void AddPublicFigures()
{
PageSources.GetByType("fb_user").Select(s => AddSource(SourcesMap.Instance.GetSource(s.page_id, "user"))).ToArray();
}
public void DoPosting(string user, string password)
{
//login
var koobecaClient = new KoobecaClient();
var koobecaService = new KoobecaService(koobecaClient);
var getAccountTask = koobecaService.GetAccountAsync(user, password);
Task.WaitAll(getAccountTask);
var account = getAccountTask.Result;
if (string.IsNullOrEmpty(account.oauth_token))
{
Logger.Instance.Error("Failed to login to koobeca");
return;
}
//load the cache
PageSources.GetByType("fb_page").Select(p => _PageSourceCache[p.source_id] = p).ToArray();
Logger.Instance.Debug($"page source cache loaded with {_PageSourceCache.Count} items.");
//take from the queue
while (IsActive)
{
if (_PostQueue.Count > 0)
{
var item = _PostQueue.Dequeue();
if (item.Type == FBPostType.Picture)
{
try
{
//make the picture include the bigger text
StringBuilder bigMsg = new StringBuilder();
bigMsg.AppendLine(item.message);
bigMsg.AppendLine("------------------------");
bigMsg.AppendLine(item.name);
bigMsg.AppendLine(item.description);
item.message = bigMsg.ToString();
//get all puctures
List <string> picUrls = new List <string>();
picUrls.AddRange(item.attachments.data.Where(a => a.type == "photo").Select(a => a.media.image.src).ToList());
picUrls.AddRange(item.attachments.data.Where(a => a.type == "album").SelectMany(a => a.subattachments.data.Where(sa => sa.type == "photo").Select(sa => sa.media.image.src)).ToList());
var webClient = new WebClient();
item.Photos = new Stream[picUrls.Count];
for (int i = 0; i < picUrls.Count; i++)
{
var url = picUrls[i];
byte[] imageBytes = webClient.DownloadData(url);
item.Photos[i] = new MemoryStream(imageBytes);
}
}
catch (Exception e)
{
Logger.Instance.Error($"failed to get photo stream from {item.full_picture} , {e.Message}");
}
}
Logger.Instance.Info($"Got post from FB source:{item.source} of type {item.Type} , looking for target in Koobeca");
double timeSpent;
PublishPost(_KoobecaService, item, out timeSpent);
Published?.Invoke(item.source, timeSpent / 1000); // throw event to listeners
}
else
{
Thread.Sleep(1000);
}
}
Logger.Instance.Info($"Publishing ended.");
}
private bool PublishPost(KoobecaService service, FBPost item, out double timeSpanMiliSec)
{
DateTime start = DateTime.Now;
CurrentlyPublished = item;
try
{
//if (string.IsNullOrEmpty(item.Link))
// item.Link = item.URL;
Logger.Instance.Info($"Publishing {item.source_type} {item.source} : {item.message} : link {item.link}");
if (item.Photos == null)
{
var effectiveLink = UrlUnshortener.GetRealUrl(item.link);
if (effectiveLink != item.link)
{
Logger.Instance.Info($"effective link is: {effectiveLink}");
item.link = effectiveLink;
}
}
else
{
//no link in case of photo
item.link = string.Empty;
}
if (!_PageSourceCache.TryGetValue(item.source, out PageSource source))
{
source = PageSources.GetBySourceId(item.source, item.source_type);
if (source == null)
{
throw new Exception($"{item.source}:{item.link} not found in page ids");
}
else
{
//add it to the cache
_PageSourceCache[item.source] = source;
}
}
ulong pageId = source.page_id; //target id (legacy is page)
if (ActualPublishing)
{
KoobecaService.PostTarget targetType = source.source_type == "fb_page" ? KoobecaService.PostTarget.Page : KoobecaService.PostTarget.User;
int videoId = 0;
if (item.link.Contains(".youtube.com"))
{
//post video
var vidReult = service.PostVideoAsync("1", "everyone", item.name, item.description, item.link).Result;
if (!vidReult.error)
{
videoId = vidReult.body.response.video_id;
}
}
var result = service.PostOnWallAsync(item.Privacy, videoId, item.message, item.link, targetType, (int)pageId, item.Photos).Result;
if (result.error)
{
Logger.Instance.Error(result.ToString());
}
}
else
{
Thread.Sleep(1000); // just sleep second
}
timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds;
Logger.Instance.Info($"Time Spent in {timeSpanMiliSec} ms");
//File.AppendAllText(_PublishedPath,item.ID + "\n");
PublishCount++;
return(true);
}
catch (Exception e)
{
Logger.Instance.Error(e.ToString());
PublishErrorCount++;
timeSpanMiliSec = (DateTime.Now - start).TotalMilliseconds;
return(false);
}
finally
{
CurrentlyPublished = null;
}
}