private void ConfigureIoC()
{
Container = new WindsorContainer();
var documentStore = new EmbeddableDocumentStore {
DataDirectory = "../data"
};
documentStore.Initialize();
var connectionLookup = new ConnectionLookup();
var taskNotifier = new TaskNotifier(connectionLookup);
var queueFactory = new DefaultQueueFactory(taskNotifier, documentStore);
var userAccountService = new DummyUserAccountService();
var taskDistributor = new TaskDistributor(queueFactory, userAccountService);
sessionTimeoutEventSink = taskDistributor;
Container.Register(Component.For <IDocumentStore>().Instance(documentStore).LifestyleSingleton());
Container.Register(Component.For <INotifyUsersOfTasks>().Instance(taskNotifier).LifestyleSingleton());
Container.Register(Component.For <TaskDistributor, IReactToUserLoggedIn, IReactToUserLoggedOff>().Instance(taskDistributor).LifestyleSingleton());
Container.Register(Component.For <IControllerFactory>()
.Instance(new MyControllerFactory(Container))
.LifeStyle.Is(Castle.Core.LifestyleType.Singleton));
Container.Register(AllTypes.FromThisAssembly().BasedOn <IController>().LifestyleTransient());
GlobalHost.DependencyResolver.Register(typeof(TaskDistributor), () => taskDistributor);
GlobalHost.DependencyResolver.Register(typeof(ConnectionLookup), () => connectionLookup);
GlobalHost.DependencyResolver.Register(typeof(IBus), () => Bus);
GlobalHost.DependencyResolver.Register(typeof(UserTaskEndpoint), () => new UserTaskEndpoint(taskDistributor, connectionLookup, Bus));
}
protected override async Task HandleInternalUnstatable(WeiboPhotoListTaskData taskData,
CancellationToken cancellationChangeToken)
{
var url = string.Format(Options.Value.UrlTemplate, taskData.SinceId, taskData.Page,
(DateTime.Now.ToUniversalTime() - new DateTime(1970, 01, 01, 0, 0, 0, DateTimeKind.Utc)).Milliseconds);
var client = await GetHttpClient();
var rsp = await client.GetAsync(url, cancellationChangeToken);
var responseString = await rsp.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <Dictionary <string, string> >(responseString);
var data = json["data"];
if (!string.IsNullOrEmpty(data))
{
var html = new CQ(data);
var imageData = html[".photo_cont>a.ph_ar_box>img.photo_pict"].Select(t => t.GetAttribute("src"))
.Where(t => !string.IsNullOrEmpty(t)).Select(t =>
{
var r = Regex.Replace(t, "/thumb\\d+?/", "/large/");
if (r.StartsWith("//"))
{
r = $"https:{r}";
}
var index = r.IndexOf('?');
if (index > -1)
{
r = r.Substring(0, index);
}
return(r);
});
var newTaskData = imageData.Select(a => (TaskData) new DownloadTaskData
{
Url = a,
RelativeFilename = Path.GetFileName(a)
}).ToList();
var nextPageData = html["div[node-type='loading']"].Attr("action-data");
if (!string.IsNullOrEmpty(nextPageData))
{
var sinceIdMatch = Regex.Match(nextPageData, "since_id=(?<sinceId>[^&]+)");
if (sinceIdMatch.Success)
{
var sinceId = sinceIdMatch.Groups["sinceId"].Value;
newTaskData.Insert(0,
new WeiboPhotoListTaskData {
Page = taskData.Page + 1, SinceId = sinceId
});
}
}
await TaskDistributor.Distribute(newTaskData);
}
}
/// <summary>
/// Starts the given Task when this Task ended successfully.
/// </summary>
/// <param name="that">The that.</param>
/// <param name="followingTask">The task to start.</param>
/// <returns>
/// This task.
/// </returns>
public static CustomTask Then(this CustomTask that, CustomTask followingTask)
{
TaskDistributor target = null;
if (CustomThread.CurrentThread is TaskWorker)
{
target = ((TaskWorker)CustomThread.CurrentThread).TaskDistributor;
}
return(that.Then(followingTask, target));
}
protected override async Task HandleInternalUnstatable(TaobaoGetItemListTaskData taskData,
CancellationToken ct)
{
var client = await GetHttpClient();
var url = string.Format(Options.Value.ListUrlTpl, taskData.Page);
var rsp = await client.GetAsync(url, ct);
var html = rsp.StatusCode == HttpStatusCode.Redirect
? await client.GetStringAsync(rsp.Headers.Location)
: await rsp.Content.ReadAsStringAsync();
var cq = new CQ(html.Replace("\\\"", "\""));
var searchResultSpan = cq[".search-result span"];
var count = int.Parse(searchResultSpan.Text().Trim());
if (count > 0)
{
var itemIds = cq[".shop-filter"].NextAll().Children(".item").Select(t => t.GetAttribute("data-id"))
.ToList();
if (itemIds.Any())
{
var newTaskData = new List <TaskData> {
new TaobaoGetItemListTaskData {
Page = taskData.Page + 1
}
};
if (DbContextProvider != null)
{
var db = await DbContextProvider.Get();
var soonestCheckDt = DateTime.Now.AddDays(-7);
var skippedItems = await db.TaobaoItems
.Where(t => itemIds.Contains(t.ItemId) && t.LastCheckDt > soonestCheckDt)
.Select(a => a.ItemId).ToListAsync(ct);
itemIds.RemoveAll(t => skippedItems.Contains(t));
}
if (itemIds.Any())
{
newTaskData.AddRange(itemIds
.Select(t => new TaobaoGetItemTaskData {
ItemId = t
}).ToList());
}
await TaskDistributor.Distribute(newTaskData);
}
}
}
/// <summary>
/// Update the min distributor
/// </summary>
public void Compute()
{
int nbIterations = 0;
for (int t = 0; t < _mDistributors.Length; t++)
{
nbIterations += _mDistributors[t].CurrentIteration;
if (_mDistributors[t].Fitness < _minDistributorFitness)
{
_minDistributorFitness = _mDistributors[t].Fitness;
MinDistributor = _mDistributors[t];
}
}
_mCurIteration = nbIterations;
}
static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("usage: dotnet {0} <filename> <number of tasks>", AppDomain.CurrentDomain.FriendlyName);
return;
}
try
{
foreach (KeyValuePair <string, List <int> > personTasks in TaskDistributor.Distribute(File.ReadAllLines(args[0]), int.Parse(args[1])))
{
Console.WriteLine("{0}: {1}", personTasks.Key, string.Join(", ", personTasks.Value.Select((task) => task.ToString())));
}
}
catch (Exception e)
{
Console.WriteLine("Error: " + e.Message);
}
}
protected override async Task <TaskDataExecutionResult> HandleInternal(YandeListTaskData data,
CancellationToken ct)
{
var client = await GetHttpClient();
var listUrl = string.Format(Options.Value.ListUrlTpl, data.Page);
var html = await client.GetStringAsync(listUrl);
var cQuery = new CQ(html);
var posts = cQuery["#post-list-posts li"];
var imageUrls = new List <string>();
var newTaskData = new List <TaskData>();
var stop = false;
foreach (var post in posts)
{
var id = int.Parse(post.Attributes["id"].Substring(1));
if (data.LastImageIds?.Contains(id) == true)
{
stop = true;
break;
}
var url = post.Cq().Children("a").Attr("href");
if (url.StartsWith("//"))
{
url = "https:" + url;
}
imageUrls.Add(url);
data.CrawledImageIds.Add(id);
}
if (!stop)
{
newTaskData.Add(new YandeListTaskData
{
Page = data.Page + 1, LastImageIds = data.LastImageIds
});
}
if (imageUrls.Any())
{
newTaskData.AddRange(imageUrls.Select(t =>
{
var newData = new DownloadTaskData {
Url = t
};
var filename = WebUtility.UrlDecode(t.Substring(t.LastIndexOf('/') + 1));
var regexSearch = new string(Path.GetInvalidFileNameChars()) +
new string(Path.GetInvalidPathChars());
var r = new Regex($"[{Regex.Escape(regexSearch)}]");
newData.RelativeFilename = r.Replace(filename, "");
return(newData);
}));
}
await TaskDistributor.Distribute(newTaskData);
return(TaskDataExecutionResult.Complete);
}
/// <summary>
/// Performs the given Func parallel for each element in the enumerable.
/// </summary>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <typeparam name="T"></typeparam>
/// <param name="that">The that.</param>
/// <param name="action">The Func to perform for each element.</param>
/// <param name="target">The TaskDistributor instance on which the operation should perform.</param>
/// <returns>
/// IEnumerable of created tasks.
/// </returns>
public static IEnumerable <CustomTask <TResult> > ParallelForEach <TResult, T>(this IEnumerable <T> that,
Func <T, TResult> action,
TaskDistributor target)
{
var res = that.Select(tmp => CustomTask.Create(() => action(tmp)).Run(target)).CreateList();
return(res);
}
public UserTaskEndpoint(TaskDistributor taskDistributor, ConnectionLookup connectionLookup, IBus messageBus)
{
this.taskDistributor = taskDistributor;
this.connectionLookup = connectionLookup;
this.messageBus = messageBus;
}
public static Task RunAsync(this object that, string methodName, TaskDistributor target, params object[] args)
{
return(that.RunAsync <object>(methodName, target, args));
}
public static Task <T> RunAsync <T>(this object that, string methodName, TaskDistributor target, params object[] args)
{
return(Task.Create <T>(that, methodName, args).Run(target));
}
//TODO figure out why this is not used
//public Task FindNearestAssignedTask(Task fromTask)
//{
// int minDistance = Int32.MaxValue;
// Task minTask = null;
// foreach (Task task in _tasks)
// {
// // assigned?
// if (_taskDistributedWorker[task.Id] != -1 && fromTask.Id != task.Id)
// {
// //int distance = task.DistanceTo(fromTask);
// int cost = task.CostTo(fromTask);
// if (cost < minDistance)
// {
// minDistance = cost;
// minTask = task;
// }
// }
// }
// return minTask;
//}
private void Configure(Configuration config)
{
_rand = new Random(config.RandomSeed);
ConfigureTasks(config);
_workers = config.Workers;
_distributor = config.Distributor;
_taskDistributedWorker = new int[_tasks.Count];
_lastTaskDistributedWorker = new int[_tasks.Count];
_sequencers = new TaskSequencer[_workers.Count];
_distributedTasks = new List<Task>[_workers.Count];
Sequences = new List<TaskSequence>(_workers.Count);
for (int t = 0; t < _workers.Count; t++)
{
_distributedTasks[t] = new List<Task>(_tasks.Count);
_sequencers[t] = new TaskSequencer(_tasks.Count);
Sequences.Add(null);
}
for (int t = 0; t < _tasks.Count; t++)
{
_lastTaskDistributedWorker[t] = -1;
}
}
private static bool StartInstances(XmlDocument xmlDoc, out TaskCreator taskCreator, out TaskDistributor taskDistributor,
out ReportAggregator reportAggregator, out IncentiveEngine incentiveEngine, out Repository repository,
out TransactionNode transactionNode)
{
taskCreator = null;
taskDistributor = null;
reportAggregator = null;
incentiveEngine = null;
repository = null;
transactionNode = null;
int port;
string serverMgrURL;
string repositoryURL = null;
string transactionNodeURL = null;
XmlNode xmlNode;
try
{
xmlNode = xmlDoc.GetElementsByTagName(Global.SERVER_MANAGER_NAME)[0];
if (xmlNode != null)
{
serverMgrURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.SERVER_MANAGER_NAME}";
}
else
{
Console.WriteLine("Server Manager Parameters are not defined in the XML!");
return(false);
}
Console.WriteLine("[SERVER MANAGER] - Node Started");
Console.WriteLine("----------------------------" + Environment.NewLine);
/*********REPOSITORY**************/
xmlNode = xmlDoc.GetElementsByTagName(Global.REPOSITORY_NAME)[0];
if (xmlNode != null)
{
repositoryURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.REPOSITORY_NAME}";
int.TryParse(xmlNode.Attributes["port"].Value, out port);
repository = new Repository(xmlNode.Attributes["hostName"].Value, port, Global.REPOSITORY_NAME);
}
/**************TRANSACTION NODE********/
xmlNode = xmlDoc.GetElementsByTagName(Global.TRANSACTION_NODE_NAME)[0];
if (xmlNode != null)
{
int.TryParse(xmlNode.Attributes["port"].Value, out port);
transactionNode = new TransactionNode(xmlNode.Attributes["hostName"].Value, port, Global.TRANSACTION_NODE_NAME);
transactionNodeURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.TRANSACTION_NODE_NAME}";
}
/**************TASK CREATOR********/
xmlNode = xmlDoc.GetElementsByTagName(Global.TASK_CREATOR_NAME)[0];
if (xmlNode != null)
{
int.TryParse(xmlNode.Attributes["port"].Value, out port);
taskCreator = new TaskCreator(xmlNode.Attributes["hostName"].Value, port, Global.TASK_CREATOR_NAME);
}
/***********TASK DISTRIBUTOR********/
xmlNode = xmlDoc.GetElementsByTagName(Global.TASK_DISTRIBUTOR_NAME)[0];
if (xmlNode != null)
{
int.TryParse(xmlNode.Attributes["port"].Value, out port);
taskDistributor = new TaskDistributor(xmlNode.Attributes["hostName"].Value, port, Global.TASK_DISTRIBUTOR_NAME);
}
/*********REPORT AGGREGATOR**********/
xmlNode = xmlDoc.GetElementsByTagName(Global.REPORT_AGGREGATOR_NAME)[0];
if (xmlNode != null)
{
int.TryParse(xmlNode.Attributes["port"].Value, out port);
reportAggregator = new ReportAggregator(xmlNode.Attributes["hostName"].Value, port, Global.REPORT_AGGREGATOR_NAME);
}
/***********INCENTIVE ENGINE********/
xmlNode = xmlDoc.GetElementsByTagName(Global.INCENTIVE_ENGINE_NAME)[0];
if (xmlNode != null)
{
int.TryParse(xmlNode.Attributes["port"].Value, out port);
incentiveEngine = new IncentiveEngine(xmlNode.Attributes["hostName"].Value, port, Global.INCENTIVE_ENGINE_NAME,
repositoryURL, transactionNodeURL);
}
return(true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
}
private void Create(Configuration config)
{
_rand = new Random(config.RandomSeed);
_mDistributors = new TaskDistributor[config.NumberDistributors];
_threads = new Thread[config.NumberDistributors];
_recomputeFitnesses = new bool[config.NumberDistributors];
var taskDistributorConfiguration = new TaskDistributor.Configuration {Workers = config.Workers, Tasks = config.Tasks, Optimizer = this};
for (int t = 0; t < config.NumberDistributors; t++)
{
Console.WriteLine(t);
taskDistributorConfiguration.RandomSeed = _rand.Next();
taskDistributorConfiguration.StartProgressPercent = t*100/config.NumberDistributors;
taskDistributorConfiguration.EndProgressPercent = taskDistributorConfiguration.StartProgressPercent + 100/config.NumberDistributors;
_mDistributors[t] = new TaskDistributor(taskDistributorConfiguration);
_recomputeFitnesses[t] = false;
_threads[t] = new Thread(OptimizeThread);
}
Start();
}
/// <summary>
/// Performs the given Func sequential for each element in the enumerable.
/// </summary>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <typeparam name="T"></typeparam>
/// <param name="that">The that.</param>
/// <param name="action">The Func to perform for each element.</param>
/// <param name="target">The TaskDistributor instance on which the operation should perform.</param>
/// <returns>
/// IEnumerable of created tasks.
/// </returns>
public static IEnumerable <CustomTask <TResult> > SequentialForEach <TResult, T>(this IEnumerable <T> that,
Func <T, TResult> action, TaskDistributor target)
{
var result = new List <CustomTask <TResult> >();
CustomTask lastTask = null;
foreach (var element in that)
{
var tmp = element;
var task = CustomTask.Create(() => action(tmp));
if (lastTask == null)
{
task.Run(target);
}
else
{
lastTask.WhenEnded(() => task.Run(target));
}
lastTask = task;
result.Add(task);
}
return(result);
}
public Configuration(List<Worker> workers, List<Task> tasks = null, TaskDistributor distributor = null, int randomSeed = 0)
{
Tasks = tasks;
Workers = workers;
Distributor = distributor;
RandomSeed = randomSeed;
}
/// <summary>
/// Starts the Func as async Task on the given TaskDistributor.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="that">The that.</param>
/// <param name="target">The TaskDistributor instance on which the operation should perform.</param>
/// <returns>
/// The task.
/// </returns>
public static CustomTask <T> RunAsync <T>(this Func <T> that, TaskDistributor target)
{
return(target.Dispatch(that));
}
/// <summary>
/// Starts the Action as async Task on the given TaskDistributor.
/// </summary>
/// <param name="that">The that.</param>
/// <param name="target">The TaskDistributor instance on which the operation should perform.</param>
/// <returns>
/// The task.
/// </returns>
public static CustomTask RunAsync(this Action that, TaskDistributor target)
{
return(target.Dispatch(that));
}
public UserTaskEndpoint(TaskDistributor taskDistributor, ConnectionLookup connectionLookup, IBus messageBus)
{
this.taskDistributor = taskDistributor;
this.connectionLookup = connectionLookup;
this.messageBus = messageBus;
}
private void ConfigureIoC()
{
Container = new WindsorContainer();
var documentStore = new EmbeddableDocumentStore { DataDirectory = "../data" };
documentStore.Initialize();
var connectionLookup = new ConnectionLookup();
var taskNotifier = new TaskNotifier(connectionLookup);
var queueFactory = new DefaultQueueFactory(taskNotifier, documentStore);
var userAccountService = new DummyUserAccountService();
var taskDistributor = new TaskDistributor(queueFactory, userAccountService);
sessionTimeoutEventSink = taskDistributor;
Container.Register(Component.For<IDocumentStore>().Instance(documentStore).LifestyleSingleton());
Container.Register(Component.For<INotifyUsersOfTasks>().Instance(taskNotifier).LifestyleSingleton());
Container.Register(Component.For<TaskDistributor, IReactToUserLoggedIn, IReactToUserLoggedOff>().Instance(taskDistributor).LifestyleSingleton());
Container.Register(Component.For<IControllerFactory>()
.Instance(new MyControllerFactory(Container))
.LifeStyle.Is(Castle.Core.LifestyleType.Singleton));
Container.Register(AllTypes.FromThisAssembly().BasedOn<IController>().LifestyleTransient());
GlobalHost.DependencyResolver.Register(typeof(TaskDistributor), () => taskDistributor);
GlobalHost.DependencyResolver.Register(typeof(ConnectionLookup), () => connectionLookup);
GlobalHost.DependencyResolver.Register(typeof(IBus), () => Bus);
GlobalHost.DependencyResolver.Register(typeof(UserTaskEndpoint), () => new UserTaskEndpoint(taskDistributor, connectionLookup, Bus));
}
protected override async Task HandleInternalUnstatable(TaobaoGetItemTaskData taskData,
CancellationToken ct)
{
var client = await GetHttpClient();
var url = string.Format(Options.Value.UrlTemplate, taskData.ItemId);
var html = await client.GetStringAsync(url);
var cq = new CQ(html);
var title = cq["#J_Title h3"].Attr("data-title").Trim();
//china url
var match = Regex.Match(html, @"descUrl\s*\:\slocation.*(?<url>\/\/.*?)',").Groups["url"];
//world url
if (!match.Success)
{
match = Regex.Match(html, "descUrlSSL\\s*\\:\\s*\"(?<url>\\/\\/.*?)\".*").Groups["url"];
}
var descUrl = match.Value;
if (descUrl.StartsWith("//"))
{
descUrl = $"https:{descUrl}";
}
var descJsonp = await client.GetStringAsync(descUrl);
var descHtml = Regex.Match(descJsonp, @"var\s*desc\s*=\s*'\s*(?<html>[\s\S]*)\s*'\s*;")
.Groups["html"].Value;
var descCq = new CQ(descHtml);
var urlList = await ImageUrlListExtractor.ExtractImageUrlList(descCq);
var index = 0;
var newTaskData = urlList.Select(t =>
{
var filename = $"{index++}_{t.Substring(t.LastIndexOf('/') + 1)}";
if (filename.Contains("?"))
{
filename = filename.Substring(0, filename.IndexOf('?'));
}
var data = new DownloadTaskData
{
RelativeFilename = $"{title}/{filename}",
Url = t
};
return(data);
}).ToList();
if (DbContextProvider != null)
{
var db = await DbContextProvider.Get();
var record = await db.TaobaoItems.FirstOrDefaultAsync(t => t.ItemId == taskData.ItemId, ct);
if (record == null)
{
record = new TaobaoItem
{
ItemId = taskData.ItemId
};
db.Add(record);
}
record.LastCheckDt = DateTime.Now;
await db.SaveChangesAsync(ct);
}
await TaskDistributor.Distribute(newTaskData);
}
/// <summary>
/// Starts the Enumerator as async Task on the given TaskDistributor.
/// </summary>
/// <param name="that">The that.</param>
/// <param name="target">The TaskDistributor instance on which the operation should perform.</param>
/// <returns>
/// The task.
/// </returns>
public static CustomTask RunAsync(this IEnumerator that, TaskDistributor target)
{
return(target.Dispatch(CustomTask.Create(that)));
}
