static void Main(string[] args)
{
//setup ES connections, change as you see fit
var settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("emailfrominstagram");
ElasticSearch = new ElasticClient(settings);
settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("emailfrominstagramtargets");
ElasticSearchTarget = new ElasticClient(settings);
//lets start with #bnw shall we
TagQueue.Enqueue("bnw");
TagQueue.Enqueue("bnw_society");
TagQueue.Enqueue("photooftheday");
TagQueue.Enqueue("instamood");
TagQueue.Enqueue("instagood");
TagQueue.Enqueue("mood");
TagQueue.Enqueue("style");
TagQueue.Enqueue("loveit");
TagQueue.Enqueue("dayshots");
TagQueue.Enqueue("artwork");
TagQueue.Enqueue("naturelover");
TagQueue.Enqueue("detail");
TagQueue.Enqueue("igmasters");
TagQueue.Enqueue("igmood");
TagQueue.Enqueue("instamood");
TagQueue.Enqueue("instatravel");
TagQueue.Enqueue("moodstagram");
TagQueue.Enqueue("picoftheday");
Console.WriteLine("STARTING...");
try
{
//start up threads
List <Thread> threads = new List <Thread>();
for (int i = 0; i < 4; i++)
{
Thread t = new Thread(() => Work(i));
t.Start();
threads.Add(t);
}
// Await threads
foreach (Thread thread in threads)
{
thread.Join();
}
}
catch (Exception ex)
{
Console.WriteLine("ERROR, " + ex);
}
}
//find tags to look at
static void ExtractTagTargets(string input)
{
//parse for our finders
input = input.Replace("\\n", " ");
input = input.Replace("\n", " ");
Regex tagRegex = new Regex(@"(?<=#)\w+", RegexOptions.IgnoreCase);
MatchCollection tagMatches = tagRegex.Matches(input);
if (TagQueue != null && TagQueue.Count < 1000)
{
foreach (Match foundTag in tagMatches)
{
lock (TagQueue)
{
if (!TagQueue.Contains(foundTag.ToString()))
{
TagQueue.Enqueue(foundTag.ToString());
}
}
}
}
}
static async Task ReadPage(int t)
{
//check if we have any users to scan
//MODES
// t - read user accounts
// f - read hashtags
var readUserMode = UserQueue.Any();
//t1 preempts running out of targets
if (t == 1 && UserQueue.Any() && UserQueue.Count < 100)
{
readUserMode = false;
}
string page = "";
string target = "";
//make the target url and dequeue in a locked state
if (readUserMode)
{
lock (UserQueue)
{
UserQueue.TryDequeue(out target);
}
page = UserUrl + target;
}
else
{
lock (TagQueue)
{
TagQueue.TryDequeue(out target);
}
page = TagUrl + target;
}
try
{
//check for locking error
if (!string.IsNullOrWhiteSpace(target))
{
using (HttpClient client = new HttpClient())
{
using (HttpResponseMessage response = await client.GetAsync(page))
{
using (HttpContent content = response.Content)
{
// Read the string.
string result = await content.ReadAsStringAsync();
if (result != null)
{
await ProcessPage(target, result, readUserMode);
//display update
Console.WriteLine($"[t{t}] UserQueue {(UserQueue.Any() ? UserQueue.Count : 0)} | TagQueue {(TagQueue.Any() ? TagQueue.Count : 0)} || {(readUserMode ? "/" : "#")}{target}");
Thread.Sleep(1500);
}
}
}
}
}
else
{
Console.WriteLine("THREAD LOCKING ERROR");
Thread.Sleep(5000);
}
}
catch (Exception e)
{
Console.WriteLine($"ERROR WITH READING PAGE ON THREAD {t}");
}
}