private async Task GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrLikedByCrawler:GetUrlsAsync: {0}", "User not logged in");
ShellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, Blog.Name);
PostQueue.CompleteAdding();
return;
}
long pagination = CreateStartPagination();
// TODO: find way to parallelize without losing content.
foreach (int crawlerNumber in Enumerable.Range(0, 1))
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(pagination, crawlerNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
UpdateBlogStats(true);
}
private async Task GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrLikedByCrawler:GetUrlsAsync: {0}", "User not logged in");
ShellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, Blog.Name);
PostQueue.CompleteAdding();
return;
}
long pagination = CreateStartPagination();
nextPage.Add(Blog.Url + (TumblrLikedByBlog.IsLikesUrl(Blog.Url) ? "?before=" : "/page/1/") + pagination);
foreach (int crawlerNumber in Enumerable.Range(0, ShellService.Settings.ConcurrentScans))
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(crawlerNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
UpdateBlogStats(true);
}
private async Task <bool> GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
GenerateTags();
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrHiddenCrawler:GetUrlsAsync: {0}", "User not logged in");
ShellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, Blog.Name);
PostQueue.CompleteAdding();
incompleteCrawl = true;
return(incompleteCrawl);
}
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
jsonQueue.CompleteAdding();
PostQueue.CompleteAdding();
UpdateBlogStats();
return(incompleteCrawl);
}
private void AddToPostQueue(string listname, string emailAddress, string key, string url, string postData)
{
PostQueue pq = new PostQueue();
if (listname?.Length > 50)
{
pq.Name = listname?.Substring(0, 50);
}
else
{
pq.Name = listname;
}
pq.Method = "GET";
pq.EmailAddress = emailAddress;
pq.PostUrl = url;
pq.PostData = key + postData;
pq.OkValue = "Thank You";
pq.IgnoreValues = "";
GloshareContext db = new GloshareContext();
try
{
db.PostQueues.Add(pq);
db.SaveChanges();
}
catch (Exception e)
{
//EmailUtility.SendErrorEmail(e, "CBR", "ListtrakUtility", "UpdateListrak()");
}
}
private async Task GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrTagSearchCrawler:GetUrlsAsync: {0}", "User not logged in");
ShellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, Blog.Name);
PostQueue.CompleteAdding();
return;
}
GenerateTags();
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync());
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(true);
}
// Esempio con errore: file globale associato a un destinatario
private static async Task Scenario_3()
{
try
{
string account = Utils.GetAccount();
string uploadSessionId = await GetUploadSessionId(account);
File personale1 = await PostFile(account, uploadSessionId, "personale1", "application/pdf", "private", null);
File personale2 = await PostFile(account, uploadSessionId, "personale2", "application/pdf", "private", null);
File globale1 = await PostFile(account, uploadSessionId, "globale1", "application/pdf", "global", AttachmentOptions.CreateAttachmentOptions(true, "bw", "A4", 80, true, false));
Attachments attachments = Attachments.CreateAttachments(uploadSessionId, new List <File> {
personale1, personale2, globale1
});
Recipient recipient1 = Recipient.CreateRecipient("Via Emilia Ovest 129/2", "43126", "Parma", "PR", "it",
"pt", "RACCOMANDATA1",
"person", "Winton", "Marsalis", "Multidialogo Srl",
"*****@*****.**", null,
new List <string> {
personale1.Id
},
"sendposta", null,
null);
Recipient recipient2 = Recipient.CreateRecipient("Via Zarotto 63", "43123", "Collecchio", "PR", "it",
"pt", "RACCOMANDATA1AR",
"person", "Clara", "Schumann", "ASA Srl",
"*****@*****.**", null,
new List <string> {
globale1.Id, personale2.Id
},
"sendposta", null,
null);
PostQueueDto postQueueDto = PostQueue.CreatePostQueue(Sender.CreateSender(), attachments,
new List <Recipient> {
recipient1, recipient2
},
"Convocazione assemblea", "Caro sei convocato per l'assemblea. Visualizza l'allegato. Grazie.",
false, false, false, false, "Test scenario 1", null);
await SendPostQueueRequest(account, postQueueDto);
}
catch (ApiDialogException e)
{
Console.WriteLine("Errore: " + e.Message);
}
}
public override async Task IsBlogOnlineAsync()
{
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrHiddenCrawler:GetUrlsAsync: {0}", "User not logged in");
ShellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, Blog.Name);
PostQueue.CompleteAdding();
}
try
{
tumblrKey = await UpdateTumblrKeyAsync("https://www.tumblr.com/dashboard/blog/" + Blog.Name);
string document = await GetSvcPageAsync("1", "0");
Blog.Online = true;
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.RequestCanceled)
{
return;
}
if (HandleServiceUnavailableWebException(webException))
{
Blog.Online = true;
}
if (HandleNotFoundWebException(webException))
{
Blog.Online = false;
}
if (HandleLimitExceededWebException(webException))
{
Blog.Online = true;
}
}
catch (TimeoutException timeoutException)
{
HandleTimeoutException(timeoutException, Resources.OnlineChecking);
Blog.Online = false;
}
catch (Exception ex) when(ex.Message == "Acceptance of privacy consent needed!")
{
Blog.Online = false;
}
}
void Start()
{
instance = PostQueue.GetInstance;
port = new SerialPort(IniFileManager.instance.ReadValue("COM", "PORT"), 38400, Parity.None, 8, StopBits.One);
port.ReadTimeout = 100;
Connect();
serialThread = new Thread(new ThreadStart(OnSerialThreadStart));
isSerialWorking = true;
serialThread.Start();
while (!serialThread.IsAlive)
{
; // wait for thread to start
}
UnityEngine.Debug.Log("Thread started : " + Time.realtimeSinceStartup);
}
private async Task GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
GenerateTags();
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync());
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(true);
}
private async Task GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
tumblrKey = await UpdateTumblrKeyAsync("https://www.tumblr.com/search/" + Blog.Name);
GenerateTags();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
UpdateBlogStats();
}
private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
// page already loaded in GetHighestPostIdCoreAsync(), so retrieve new number of posts already there
await Task.Run(() => Task.CompletedTask);
//await UpdateTotalPostCountAsync();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
private async Task <bool> GetUrlsAsync()
{
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
trackedTasks = new List <Task>();
GenerateTags();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
jsonQueue.CompleteAdding();
PostQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
await UpdateTotalPostCountAsync();
foreach (int pageNumber in GetPageNumbers())
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
trackedTasks.Add(CrawlPageAsync(pageNumber));
}
await Task.WhenAll(trackedTasks);
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats();
return(incompleteCrawl);
}
protected void AddToDownloadList(TumblrPost addToList)
{
PostQueue.Add(addToList);
StatisticsBag.Add(addToList);
}
private void OnDisable()
{
queue = null;
}
// Use this for initialization
void OnEnable()
{
queue = PostQueue.GetInstance;
}
private async Task <bool> GetUrlsAsync()
{
trackedTasks = new List <Task>();
semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans);
GenerateTags();
await IsBlogOnlineAsync();
if (!Blog.Online)
{
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
return(true);
}
Blog.Posts = twUser.Data.User.Legacy.StatusesCount;
if (Blog.PageSize == 0)
{
Blog.PageSize = 50;
}
int currentPage = (Blog.Posts > 3200) ? (Blog.Posts - 3200) / 20 + 3200 / Blog.PageSize + 1 : Blog.Posts / Blog.PageSize + 1;
if (Blog.Posts > 3200)
{
currentPage += 50;
}
int pageNo = 1;
while (true)
{
await semaphoreSlim.WaitAsync();
if (!completeGrab)
{
break;
}
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
await CrawlPageAsync(pageNo);
if (currentPage > 0)
{
currentPage--;
pageNo++;
}
else
{
break;
}
}
PostQueue.CompleteAdding();
jsonQueue.CompleteAdding();
UpdateBlogStats(GetLastPostId() != 0);
return(incompleteCrawl);
}
