private void ConfigureIoC() { Container = new WindsorContainer(); var documentStore = new EmbeddableDocumentStore { DataDirectory = "../data" }; documentStore.Initialize(); var connectionLookup = new ConnectionLookup(); var taskNotifier = new TaskNotifier(connectionLookup); var queueFactory = new DefaultQueueFactory(taskNotifier, documentStore); var userAccountService = new DummyUserAccountService(); var taskDistributor = new TaskDistributor(queueFactory, userAccountService); sessionTimeoutEventSink = taskDistributor; Container.Register(Component.For <IDocumentStore>().Instance(documentStore).LifestyleSingleton()); Container.Register(Component.For <INotifyUsersOfTasks>().Instance(taskNotifier).LifestyleSingleton()); Container.Register(Component.For <TaskDistributor, IReactToUserLoggedIn, IReactToUserLoggedOff>().Instance(taskDistributor).LifestyleSingleton()); Container.Register(Component.For <IControllerFactory>() .Instance(new MyControllerFactory(Container)) .LifeStyle.Is(Castle.Core.LifestyleType.Singleton)); Container.Register(AllTypes.FromThisAssembly().BasedOn <IController>().LifestyleTransient()); GlobalHost.DependencyResolver.Register(typeof(TaskDistributor), () => taskDistributor); GlobalHost.DependencyResolver.Register(typeof(ConnectionLookup), () => connectionLookup); GlobalHost.DependencyResolver.Register(typeof(IBus), () => Bus); GlobalHost.DependencyResolver.Register(typeof(UserTaskEndpoint), () => new UserTaskEndpoint(taskDistributor, connectionLookup, Bus)); }
protected override async Task HandleInternalUnstatable(WeiboPhotoListTaskData taskData, CancellationToken cancellationChangeToken) { var url = string.Format(Options.Value.UrlTemplate, taskData.SinceId, taskData.Page, (DateTime.Now.ToUniversalTime() - new DateTime(1970, 01, 01, 0, 0, 0, DateTimeKind.Utc)).Milliseconds); var client = await GetHttpClient(); var rsp = await client.GetAsync(url, cancellationChangeToken); var responseString = await rsp.Content.ReadAsStringAsync(); var json = JsonConvert.DeserializeObject <Dictionary <string, string> >(responseString); var data = json["data"]; if (!string.IsNullOrEmpty(data)) { var html = new CQ(data); var imageData = html[".photo_cont>a.ph_ar_box>img.photo_pict"].Select(t => t.GetAttribute("src")) .Where(t => !string.IsNullOrEmpty(t)).Select(t => { var r = Regex.Replace(t, "/thumb\\d+?/", "/large/"); if (r.StartsWith("//")) { r = $"https:{r}"; } var index = r.IndexOf('?'); if (index > -1) { r = r.Substring(0, index); } return(r); }); var newTaskData = imageData.Select(a => (TaskData) new DownloadTaskData { Url = a, RelativeFilename = Path.GetFileName(a) }).ToList(); var nextPageData = html["div[node-type='loading']"].Attr("action-data"); if (!string.IsNullOrEmpty(nextPageData)) { var sinceIdMatch = Regex.Match(nextPageData, "since_id=(?<sinceId>[^&]+)"); if (sinceIdMatch.Success) { var sinceId = sinceIdMatch.Groups["sinceId"].Value; newTaskData.Insert(0, new WeiboPhotoListTaskData { Page = taskData.Page + 1, SinceId = sinceId }); } } await TaskDistributor.Distribute(newTaskData); } }
/// <summary> /// Starts the given Task when this Task ended successfully. /// </summary> /// <param name="that">The that.</param> /// <param name="followingTask">The task to start.</param> /// <returns> /// This task. /// </returns> public static CustomTask Then(this CustomTask that, CustomTask followingTask) { TaskDistributor target = null; if (CustomThread.CurrentThread is TaskWorker) { target = ((TaskWorker)CustomThread.CurrentThread).TaskDistributor; } return(that.Then(followingTask, target)); }
protected override async Task HandleInternalUnstatable(TaobaoGetItemListTaskData taskData, CancellationToken ct) { var client = await GetHttpClient(); var url = string.Format(Options.Value.ListUrlTpl, taskData.Page); var rsp = await client.GetAsync(url, ct); var html = rsp.StatusCode == HttpStatusCode.Redirect ? await client.GetStringAsync(rsp.Headers.Location) : await rsp.Content.ReadAsStringAsync(); var cq = new CQ(html.Replace("\\\"", "\"")); var searchResultSpan = cq[".search-result span"]; var count = int.Parse(searchResultSpan.Text().Trim()); if (count > 0) { var itemIds = cq[".shop-filter"].NextAll().Children(".item").Select(t => t.GetAttribute("data-id")) .ToList(); if (itemIds.Any()) { var newTaskData = new List <TaskData> { new TaobaoGetItemListTaskData { Page = taskData.Page + 1 } }; if (DbContextProvider != null) { var db = await DbContextProvider.Get(); var soonestCheckDt = DateTime.Now.AddDays(-7); var skippedItems = await db.TaobaoItems .Where(t => itemIds.Contains(t.ItemId) && t.LastCheckDt > soonestCheckDt) .Select(a => a.ItemId).ToListAsync(ct); itemIds.RemoveAll(t => skippedItems.Contains(t)); } if (itemIds.Any()) { newTaskData.AddRange(itemIds .Select(t => new TaobaoGetItemTaskData { ItemId = t }).ToList()); } await TaskDistributor.Distribute(newTaskData); } } }
/// <summary> /// Update the min distributor /// </summary> public void Compute() { int nbIterations = 0; for (int t = 0; t < _mDistributors.Length; t++) { nbIterations += _mDistributors[t].CurrentIteration; if (_mDistributors[t].Fitness < _minDistributorFitness) { _minDistributorFitness = _mDistributors[t].Fitness; MinDistributor = _mDistributors[t]; } } _mCurIteration = nbIterations; }
static void Main(string[] args) { if (args.Length < 2) { Console.WriteLine("usage: dotnet {0} <filename> <number of tasks>", AppDomain.CurrentDomain.FriendlyName); return; } try { foreach (KeyValuePair <string, List <int> > personTasks in TaskDistributor.Distribute(File.ReadAllLines(args[0]), int.Parse(args[1]))) { Console.WriteLine("{0}: {1}", personTasks.Key, string.Join(", ", personTasks.Value.Select((task) => task.ToString()))); } } catch (Exception e) { Console.WriteLine("Error: " + e.Message); } }
protected override async Task <TaskDataExecutionResult> HandleInternal(YandeListTaskData data, CancellationToken ct) { var client = await GetHttpClient(); var listUrl = string.Format(Options.Value.ListUrlTpl, data.Page); var html = await client.GetStringAsync(listUrl); var cQuery = new CQ(html); var posts = cQuery["#post-list-posts li"]; var imageUrls = new List <string>(); var newTaskData = new List <TaskData>(); var stop = false; foreach (var post in posts) { var id = int.Parse(post.Attributes["id"].Substring(1)); if (data.LastImageIds?.Contains(id) == true) { stop = true; break; } var url = post.Cq().Children("a").Attr("href"); if (url.StartsWith("//")) { url = "https:" + url; } imageUrls.Add(url); data.CrawledImageIds.Add(id); } if (!stop) { newTaskData.Add(new YandeListTaskData { Page = data.Page + 1, LastImageIds = data.LastImageIds }); } if (imageUrls.Any()) { newTaskData.AddRange(imageUrls.Select(t => { var newData = new DownloadTaskData { Url = t }; var filename = WebUtility.UrlDecode(t.Substring(t.LastIndexOf('/') + 1)); var regexSearch = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()); var r = new Regex($"[{Regex.Escape(regexSearch)}]"); newData.RelativeFilename = r.Replace(filename, ""); return(newData); })); } await TaskDistributor.Distribute(newTaskData); return(TaskDataExecutionResult.Complete); }
/// <summary> /// Performs the given Func parallel for each element in the enumerable. /// </summary> /// <typeparam name="TResult">The type of the result.</typeparam> /// <typeparam name="T"></typeparam> /// <param name="that">The that.</param> /// <param name="action">The Func to perform for each element.</param> /// <param name="target">The TaskDistributor instance on which the operation should perform.</param> /// <returns> /// IEnumerable of created tasks. /// </returns> public static IEnumerable <CustomTask <TResult> > ParallelForEach <TResult, T>(this IEnumerable <T> that, Func <T, TResult> action, TaskDistributor target) { var res = that.Select(tmp => CustomTask.Create(() => action(tmp)).Run(target)).CreateList(); return(res); }
public UserTaskEndpoint(TaskDistributor taskDistributor, ConnectionLookup connectionLookup, IBus messageBus) { this.taskDistributor = taskDistributor; this.connectionLookup = connectionLookup; this.messageBus = messageBus; }
public static Task RunAsync(this object that, string methodName, TaskDistributor target, params object[] args) { return(that.RunAsync <object>(methodName, target, args)); }
public static Task <T> RunAsync <T>(this object that, string methodName, TaskDistributor target, params object[] args) { return(Task.Create <T>(that, methodName, args).Run(target)); }
//TODO figure out why this is not used //public Task FindNearestAssignedTask(Task fromTask) //{ // int minDistance = Int32.MaxValue; // Task minTask = null; // foreach (Task task in _tasks) // { // // assigned? // if (_taskDistributedWorker[task.Id] != -1 && fromTask.Id != task.Id) // { // //int distance = task.DistanceTo(fromTask); // int cost = task.CostTo(fromTask); // if (cost < minDistance) // { // minDistance = cost; // minTask = task; // } // } // } // return minTask; //} private void Configure(Configuration config) { _rand = new Random(config.RandomSeed); ConfigureTasks(config); _workers = config.Workers; _distributor = config.Distributor; _taskDistributedWorker = new int[_tasks.Count]; _lastTaskDistributedWorker = new int[_tasks.Count]; _sequencers = new TaskSequencer[_workers.Count]; _distributedTasks = new List<Task>[_workers.Count]; Sequences = new List<TaskSequence>(_workers.Count); for (int t = 0; t < _workers.Count; t++) { _distributedTasks[t] = new List<Task>(_tasks.Count); _sequencers[t] = new TaskSequencer(_tasks.Count); Sequences.Add(null); } for (int t = 0; t < _tasks.Count; t++) { _lastTaskDistributedWorker[t] = -1; } }
private static bool StartInstances(XmlDocument xmlDoc, out TaskCreator taskCreator, out TaskDistributor taskDistributor, out ReportAggregator reportAggregator, out IncentiveEngine incentiveEngine, out Repository repository, out TransactionNode transactionNode) { taskCreator = null; taskDistributor = null; reportAggregator = null; incentiveEngine = null; repository = null; transactionNode = null; int port; string serverMgrURL; string repositoryURL = null; string transactionNodeURL = null; XmlNode xmlNode; try { xmlNode = xmlDoc.GetElementsByTagName(Global.SERVER_MANAGER_NAME)[0]; if (xmlNode != null) { serverMgrURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.SERVER_MANAGER_NAME}"; } else { Console.WriteLine("Server Manager Parameters are not defined in the XML!"); return(false); } Console.WriteLine("[SERVER MANAGER] - Node Started"); Console.WriteLine("----------------------------" + Environment.NewLine); /*********REPOSITORY**************/ xmlNode = xmlDoc.GetElementsByTagName(Global.REPOSITORY_NAME)[0]; if (xmlNode != null) { repositoryURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.REPOSITORY_NAME}"; int.TryParse(xmlNode.Attributes["port"].Value, out port); repository = new Repository(xmlNode.Attributes["hostName"].Value, port, Global.REPOSITORY_NAME); } /**************TRANSACTION NODE********/ xmlNode = xmlDoc.GetElementsByTagName(Global.TRANSACTION_NODE_NAME)[0]; if (xmlNode != null) { int.TryParse(xmlNode.Attributes["port"].Value, out port); transactionNode = new TransactionNode(xmlNode.Attributes["hostName"].Value, port, Global.TRANSACTION_NODE_NAME); transactionNodeURL = $"tcp://{xmlNode.Attributes["hostName"].Value}:{xmlNode.Attributes["port"].Value}/{Global.TRANSACTION_NODE_NAME}"; } /**************TASK CREATOR********/ xmlNode = xmlDoc.GetElementsByTagName(Global.TASK_CREATOR_NAME)[0]; if (xmlNode != null) { int.TryParse(xmlNode.Attributes["port"].Value, out port); taskCreator = new TaskCreator(xmlNode.Attributes["hostName"].Value, port, Global.TASK_CREATOR_NAME); } /***********TASK DISTRIBUTOR********/ xmlNode = xmlDoc.GetElementsByTagName(Global.TASK_DISTRIBUTOR_NAME)[0]; if (xmlNode != null) { int.TryParse(xmlNode.Attributes["port"].Value, out port); taskDistributor = new TaskDistributor(xmlNode.Attributes["hostName"].Value, port, Global.TASK_DISTRIBUTOR_NAME); } /*********REPORT AGGREGATOR**********/ xmlNode = xmlDoc.GetElementsByTagName(Global.REPORT_AGGREGATOR_NAME)[0]; if (xmlNode != null) { int.TryParse(xmlNode.Attributes["port"].Value, out port); reportAggregator = new ReportAggregator(xmlNode.Attributes["hostName"].Value, port, Global.REPORT_AGGREGATOR_NAME); } /***********INCENTIVE ENGINE********/ xmlNode = xmlDoc.GetElementsByTagName(Global.INCENTIVE_ENGINE_NAME)[0]; if (xmlNode != null) { int.TryParse(xmlNode.Attributes["port"].Value, out port); incentiveEngine = new IncentiveEngine(xmlNode.Attributes["hostName"].Value, port, Global.INCENTIVE_ENGINE_NAME, repositoryURL, transactionNodeURL); } return(true); } catch (Exception ex) { Console.WriteLine(ex.Message); return(false); } }
private void Create(Configuration config) { _rand = new Random(config.RandomSeed); _mDistributors = new TaskDistributor[config.NumberDistributors]; _threads = new Thread[config.NumberDistributors]; _recomputeFitnesses = new bool[config.NumberDistributors]; var taskDistributorConfiguration = new TaskDistributor.Configuration {Workers = config.Workers, Tasks = config.Tasks, Optimizer = this}; for (int t = 0; t < config.NumberDistributors; t++) { Console.WriteLine(t); taskDistributorConfiguration.RandomSeed = _rand.Next(); taskDistributorConfiguration.StartProgressPercent = t*100/config.NumberDistributors; taskDistributorConfiguration.EndProgressPercent = taskDistributorConfiguration.StartProgressPercent + 100/config.NumberDistributors; _mDistributors[t] = new TaskDistributor(taskDistributorConfiguration); _recomputeFitnesses[t] = false; _threads[t] = new Thread(OptimizeThread); } Start(); }
/// <summary> /// Performs the given Func sequential for each element in the enumerable. /// </summary> /// <typeparam name="TResult">The type of the result.</typeparam> /// <typeparam name="T"></typeparam> /// <param name="that">The that.</param> /// <param name="action">The Func to perform for each element.</param> /// <param name="target">The TaskDistributor instance on which the operation should perform.</param> /// <returns> /// IEnumerable of created tasks. /// </returns> public static IEnumerable <CustomTask <TResult> > SequentialForEach <TResult, T>(this IEnumerable <T> that, Func <T, TResult> action, TaskDistributor target) { var result = new List <CustomTask <TResult> >(); CustomTask lastTask = null; foreach (var element in that) { var tmp = element; var task = CustomTask.Create(() => action(tmp)); if (lastTask == null) { task.Run(target); } else { lastTask.WhenEnded(() => task.Run(target)); } lastTask = task; result.Add(task); } return(result); }
public Configuration(List<Worker> workers, List<Task> tasks = null, TaskDistributor distributor = null, int randomSeed = 0) { Tasks = tasks; Workers = workers; Distributor = distributor; RandomSeed = randomSeed; }
/// <summary> /// Starts the Func as async Task on the given TaskDistributor. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="that">The that.</param> /// <param name="target">The TaskDistributor instance on which the operation should perform.</param> /// <returns> /// The task. /// </returns> public static CustomTask <T> RunAsync <T>(this Func <T> that, TaskDistributor target) { return(target.Dispatch(that)); }
/// <summary> /// Starts the Action as async Task on the given TaskDistributor. /// </summary> /// <param name="that">The that.</param> /// <param name="target">The TaskDistributor instance on which the operation should perform.</param> /// <returns> /// The task. /// </returns> public static CustomTask RunAsync(this Action that, TaskDistributor target) { return(target.Dispatch(that)); }
public UserTaskEndpoint(TaskDistributor taskDistributor, ConnectionLookup connectionLookup, IBus messageBus) { this.taskDistributor = taskDistributor; this.connectionLookup = connectionLookup; this.messageBus = messageBus; }
private void ConfigureIoC() { Container = new WindsorContainer(); var documentStore = new EmbeddableDocumentStore { DataDirectory = "../data" }; documentStore.Initialize(); var connectionLookup = new ConnectionLookup(); var taskNotifier = new TaskNotifier(connectionLookup); var queueFactory = new DefaultQueueFactory(taskNotifier, documentStore); var userAccountService = new DummyUserAccountService(); var taskDistributor = new TaskDistributor(queueFactory, userAccountService); sessionTimeoutEventSink = taskDistributor; Container.Register(Component.For<IDocumentStore>().Instance(documentStore).LifestyleSingleton()); Container.Register(Component.For<INotifyUsersOfTasks>().Instance(taskNotifier).LifestyleSingleton()); Container.Register(Component.For<TaskDistributor, IReactToUserLoggedIn, IReactToUserLoggedOff>().Instance(taskDistributor).LifestyleSingleton()); Container.Register(Component.For<IControllerFactory>() .Instance(new MyControllerFactory(Container)) .LifeStyle.Is(Castle.Core.LifestyleType.Singleton)); Container.Register(AllTypes.FromThisAssembly().BasedOn<IController>().LifestyleTransient()); GlobalHost.DependencyResolver.Register(typeof(TaskDistributor), () => taskDistributor); GlobalHost.DependencyResolver.Register(typeof(ConnectionLookup), () => connectionLookup); GlobalHost.DependencyResolver.Register(typeof(IBus), () => Bus); GlobalHost.DependencyResolver.Register(typeof(UserTaskEndpoint), () => new UserTaskEndpoint(taskDistributor, connectionLookup, Bus)); }
protected override async Task HandleInternalUnstatable(TaobaoGetItemTaskData taskData, CancellationToken ct) { var client = await GetHttpClient(); var url = string.Format(Options.Value.UrlTemplate, taskData.ItemId); var html = await client.GetStringAsync(url); var cq = new CQ(html); var title = cq["#J_Title h3"].Attr("data-title").Trim(); //china url var match = Regex.Match(html, @"descUrl\s*\:\slocation.*(?<url>\/\/.*?)',").Groups["url"]; //world url if (!match.Success) { match = Regex.Match(html, "descUrlSSL\\s*\\:\\s*\"(?<url>\\/\\/.*?)\".*").Groups["url"]; } var descUrl = match.Value; if (descUrl.StartsWith("//")) { descUrl = $"https:{descUrl}"; } var descJsonp = await client.GetStringAsync(descUrl); var descHtml = Regex.Match(descJsonp, @"var\s*desc\s*=\s*'\s*(?<html>[\s\S]*)\s*'\s*;") .Groups["html"].Value; var descCq = new CQ(descHtml); var urlList = await ImageUrlListExtractor.ExtractImageUrlList(descCq); var index = 0; var newTaskData = urlList.Select(t => { var filename = $"{index++}_{t.Substring(t.LastIndexOf('/') + 1)}"; if (filename.Contains("?")) { filename = filename.Substring(0, filename.IndexOf('?')); } var data = new DownloadTaskData { RelativeFilename = $"{title}/{filename}", Url = t }; return(data); }).ToList(); if (DbContextProvider != null) { var db = await DbContextProvider.Get(); var record = await db.TaobaoItems.FirstOrDefaultAsync(t => t.ItemId == taskData.ItemId, ct); if (record == null) { record = new TaobaoItem { ItemId = taskData.ItemId }; db.Add(record); } record.LastCheckDt = DateTime.Now; await db.SaveChangesAsync(ct); } await TaskDistributor.Distribute(newTaskData); }
/// <summary> /// Starts the Enumerator as async Task on the given TaskDistributor. /// </summary> /// <param name="that">The that.</param> /// <param name="target">The TaskDistributor instance on which the operation should perform.</param> /// <returns> /// The task. /// </returns> public static CustomTask RunAsync(this IEnumerator that, TaskDistributor target) { return(target.Dispatch(CustomTask.Create(that))); }