private async Task ScrapProducts(List <string> links, ParamsDTO queryParams) { if (links != null) { foreach (var url in links) { var item = GetProduct(url, queryParams); if (item.Result != null) { if (!item.Result.Name.StartsWith("---")) { await _hubContext.Clients.All.SendAsync("Send", $"{++productCountSuccess} {item.Result.Name} {item.Result.Price} {item.Result.Description} {item.Result.ImgHref}"); } if (url == lastLink) { await _hubContext.Clients.All.SendAsync("Send", $"Количество спарсенных {productCountSuccess}, ошибок {productCountError}"); } System.IO.File.AppendAllText(filename, $"{productCountSuccess};{item.Result.ItemLink};{item.Result.Name};{item.Result.Price};{item.Result.Description};{item.Result.ImgHref};" + $"{item.Result.Brand};{item.Result.Category};{item.Result.DescriptionText};{item.Result.OldPrice}" + Environment.NewLine, Encoding.GetEncoding("utf-8")); } else { await _hubContext.Clients.All.SendAsync("Send", $"Error due parsing product"); ++productCountError; } } } else { await _hubContext.Clients.All.SendAsync("Send", $"Error due parsing product list"); } }
public async Task StartScraping(ParamsDTO queryParams) { try { HtmlWeb web = new HtmlWeb(); var htmlDoc = web.Load(queryParams.HomeUrl); var products = htmlDoc.DocumentNode.SelectNodes(queryParams.ProductList); level++; var links = products.Descendants("a").Select(a => a.Attributes["href"].Value).ToList(); if (level == 1) { lastLink = links.Last(); sitename = UrlConverter.GetSiteHostWithProtocol(queryParams.HomeUrl); SetFilename(); } await ScrapProducts(links, queryParams); } catch (Exception ex) { await _hubContext.Clients.All.SendAsync("Send", $"Error due parsing website or product list"); throw (ex); } }
public bool UpdateAppParams(ParamsDTO dto) { using (var client = new HttpClient()) { // New code: client.BaseAddress = BaseAddress; client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = client.PostAsJsonAsync("api/ParamsDTOs", dto).Result; return(response.IsSuccessStatusCode); } }
private async Task <ProductModel> GetProduct(string url, ParamsDTO queryParams) { await Task.Delay(TimeSpan.FromSeconds(1)); HtmlWeb web = new HtmlWeb(); var htmlDoc = web.Load(sitename + url); var product = htmlDoc.DocumentNode.SelectSingleNode(queryParams.Name) != null || htmlDoc.DocumentNode.SelectSingleNode(queryParams.Description) != null; if (product) { var img = htmlDoc.DocumentNode.SelectSingleNode(queryParams.ImgHref); return(new ProductModel() { ItemLink = url, Name = htmlDoc.DocumentNode.SelectSingleNode(queryParams.Name) != null?htmlDoc.DocumentNode.SelectSingleNode(queryParams.Name).InnerText : null, Description = htmlDoc.DocumentNode.SelectSingleNode(queryParams.Description) != null?Regex.Replace(htmlDoc.DocumentNode.SelectSingleNode(queryParams.Description).InnerHtml, @"\t|\n|\r", "").Replace(" ", " ") : null, Price = htmlDoc.DocumentNode.SelectSingleNode(queryParams.Price) != null?HttpUtility.HtmlDecode(htmlDoc.DocumentNode.SelectSingleNode(queryParams.Price).InnerHtml).Replace(" ", " ") : null, ImgHref = img != null ? sitename + string.Join("," + sitename, img.Descendants("img").Select(z => z.Attributes["src"].Value).ToList()) : string.Empty, Brand = (queryParams.Brand != null && htmlDoc.DocumentNode.SelectSingleNode(queryParams.Brand) != null) ? htmlDoc.DocumentNode.SelectSingleNode(queryParams.Brand).InnerText : null, Category = (queryParams.Category != null && htmlDoc.DocumentNode.SelectSingleNode(queryParams.Category) != null) ? htmlDoc.DocumentNode.SelectSingleNode(queryParams.Category).InnerText : null, DescriptionText = (queryParams.DescriptionText != null && htmlDoc.DocumentNode.SelectSingleNode(queryParams.DescriptionText) != null) ? Regex.Replace(htmlDoc.DocumentNode.SelectSingleNode(queryParams.DescriptionText).InnerText, @"\t|\n|\r", "").Replace(" ", " ") : null, OldPrice = (queryParams.OldPrice != null && htmlDoc.DocumentNode.SelectSingleNode(queryParams.OldPrice) != null) ? HttpUtility.HtmlDecode(htmlDoc.DocumentNode.SelectSingleNode(queryParams.OldPrice).InnerHtml).Replace(" ", " ") : null }); } else { var isProductListPage = htmlDoc.DocumentNode.SelectNodes(queryParams.ProductList); // проверка если мы на странице продуктов, то снова парсим if (isProductListPage != null) { var model = queryParams; model.HomeUrl = queryParams.HomeUrl + url; await StartScraping(model); // возвращаем модель без данных, чтобы не возвращался null return(new ProductModel() { Name = "---" }); } } return(null); }
public IActionResult GetUsuarioDisplay([FromServices] UsuariosDAO _UsuariosDAO, [FromQuery] ParamsDTO Params) { ResultadoDTO _Resultado = new ResultadoDTO(); int _IDUsuario = User.Identity.GetIDUsuario(); try { List <UsuarioDisplayDTO> _UsuarioDisplay = _UsuariosDAO.SelectUsuarioDisplay(_IDUsuario, Params.DataAtualizacao); _Resultado = ResultadoHelper.PreparaResultado(_UsuarioDisplay); } catch (Exception ex) { return(BadRequest(ex)); } return(Ok(_Resultado)); }
public IActionResult GetSemanaDias([FromQuery] ParamsDTO Params, [FromServices] TreinosDAO TreinosDAO) { if (!ModelState.IsValid) { return(BadRequest(ModelState)); } List <SemanaDiasDTO> _SemanaDias = new List <SemanaDiasDTO>(); ResultadoDTO _Resultado = new ResultadoDTO(); try { _SemanaDias = TreinosDAO.SelectSemanaDias(); _Resultado = ResultadoHelper.PreparaResultado(_SemanaDias); } catch (Exception ex) { return(BadRequest(ex.InnerException)); } return(Ok(_Resultado)); }
public IActionResult GetTreino([FromQuery] ParamsDTO Params, [FromServices] TreinosDAO TreinosDAO) { if (!ModelState.IsValid) { return(BadRequest(ModelState)); } Object _Result = new Object(); ResultadoDTO _Resultado = new ResultadoDTO(); try { int _IDUsuario = User.Identity.GetIDUsuario(); _Result = TreinosDAO.SelectTreino(_IDUsuario); _Resultado = ResultadoHelper.PreparaResultado(_Result); } catch (Exception ex) { return(BadRequest(ex.InnerException)); } return(Ok(_Resultado)); }
private void HandleCreateApp(object param) { List <object> addedList = new List <object>(); try { // int childrenCnt = (from i in DbContext.ProjectParts where i.ParentPart == Part.Id select i).Count(); if (childrenCnt > 0) { throw new Exception("只能在没有子结点的部位创建测点"); } int appCnt = (from s in DbContext.Apps where s.AppName == NewApp.AppName || s.CalculateName == NewApp.CalculateName select s).Count(); if (appCnt > 0) { throw new Exception("新测点的名称或计算名称已存在,无法创建"); } App needAddApp = new App(); needAddApp.Id = Guid.NewGuid(); needAddApp.ProjectPartID = Part.Id; needAddApp.AppName = NewApp.AppName; needAddApp.CalculateName = NewApp.CalculateName; needAddApp.BuriedTime = NewApp.BuriedTime; needAddApp.X = NewApp.X; needAddApp.Y = NewApp.Y; needAddApp.Z = NewApp.Z; needAddApp.OtherInfo = NewApp.OtherInfo; DbContext.AddToApps(needAddApp); addedList.Add(needAddApp); DbContext.SaveChanges(); _currentApps.Add(needAddApp); if (AllowClone) { //clone app params if (CloneAppName.Trim().Length == 0) { throw new Exception("当选择克隆时,模板测点不能为空"); } if (cloneApp == null) { cloneApp = (from s in DbContext.Apps where s.AppName == CloneAppName select s).FirstOrDefault(); if (cloneApp == null) { throw new Exception(string.Format("找不到测点编号为{0}的测点", CloneAppName)); } } DbContext.LoadProperty(cloneApp, "AppParams"); var paramsList = new List <AppParam>(); var formulaList = new List <Formula>(); foreach (var item in cloneApp.AppParams) { AppParam newParam = null; if (item is ConstantParam) { newParam = new ConstantParam(); (newParam as ConstantParam).Val = (item as ConstantParam).Val; } else if (item is MessureParam) { newParam = new MessureParam(); } else if (item is CalculateParam) { newParam = new CalculateParam(); } //set values newParam.Id = Guid.NewGuid(); newParam.AppId = needAddApp.Id; newParam.ParamName = item.ParamName; newParam.ParamSymbol = item.ParamSymbol; newParam.PrecisionNum = item.PrecisionNum; newParam.UnitSymbol = item.UnitSymbol; newParam.Order = item.Order; newParam.Description = item.Description; paramsList.Add(newParam); //DbContext.AddToAppParams(newParam); //addedList.Add(newParam); //clone formules if (item is CalculateParam) { CalculateParam cp = item as CalculateParam; var formulae = (from i in DbContext.Formulae where i.ParamId == cp.Id select i).ToList(); foreach (var fl in formulae) { Formula newfl = new Formula(); newfl.Id = Guid.NewGuid(); newfl.ParamId = newParam.Id; newfl.FormulaExpression = fl.FormulaExpression; newfl.StartDate = fl.StartDate; newfl.EndDate = fl.EndDate; newfl.CalculateOrder = fl.CalculateOrder; formulaList.Add(newfl); //DbContext.AddToFormulae(newfl); //addedList.Add(newfl); } } } ParamsDTO dto = new ParamsDTO(); dto.AddedParams = paramsList; dto.AddedFormulae = formulaList; DbContext.UpdateAppParams(dto); } var msg = new DialogMessage("测点创建成功并已添加到相关的工程部位中。", result => { }); msg.Caption = "创建成功"; msg.Button = MessageBoxButton.OK; msg.Icon = MessageBoxImage.Information; Messenger.Default.Send <DialogMessage>(msg); } catch (Exception ex) { Messenger.Default.Send <Exception>(ex); //reject the changes foreach (object item in addedList) { DbContext.Detach(item); } } finally { // DbContext.Detach(NewApp); } }
/// <summary> /// 只更新对AppParams和Formula的修改,一次提交由服务端进行事物处理 /// </summary> /// <returns></returns> public void UpdateAppParams() { using (var client = new HttpClient()) { ParamsDTO dto = new ParamsDTO(); dto.AddedParams = new List <AppParam>(); dto.UpdatedParams = new List <AppParam>(); dto.DeletedParams = new List <AppParam>(); dto.AddedFormulae = new List <Formula>(); dto.UpdatedFormulae = new List <Formula>(); dto.DeletedFormulae = new List <Formula>(); foreach (var item in this.Entities.ToList()) { if (item.State == EntityStates.Added || item.State == EntityStates.Deleted || item.State == EntityStates.Modified) { if (item.Entity is AppParam) { var entity = item.Entity as AppParam; if (item.State == EntityStates.Added) { dto.AddedParams.Add(entity); } else if (item.State == EntityStates.Modified) { dto.UpdatedParams.Add(entity); } else if (item.State == EntityStates.Deleted) { dto.DeletedParams.Add(entity); } this.Detach(item.Entity); } else if (item.Entity is Formula) { var entity = item.Entity as Formula; if (item.State == EntityStates.Added) { dto.AddedFormulae.Add(entity); } else if (item.State == EntityStates.Modified) { dto.UpdatedFormulae.Add(entity); } else if (item.State == EntityStates.Deleted) { dto.DeletedFormulae.Add(entity); } this.Detach(item.Entity); } } } // New code: client.BaseAddress = BaseAddress; client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = client.PostAsJsonAsync("api/ParamsDTOs", dto).Result; if (response.IsSuccessStatusCode == false) { throw new Exception("update app params and formulae error!"); } } }
// POST odata/Apps public IHttpActionResult Post(JObject rdto) { if (!ModelState.IsValid) { return(BadRequest(ModelState)); } // Newtonsoft.Json.Serialization.ITraceWriter traceWriter = new Newtonsoft.Json.Serialization.MemoryTraceWriter(); var converter = new Helper.AppParamConverter(); ParamsDTO dto = null; try { //TraceWriter = traceWriter, dto = JsonConvert.DeserializeObject <ParamsDTO>(rdto.ToString(), new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, Converters = { converter } }); } catch (Exception) { throw; } //delay load //check logic in entity context db = new DamWCFContext(true); if (dto.AddedParams != null) { db.AppParams.AddRange(dto.AddedParams); } if (dto.UpdatedParams != null) { foreach (var item in dto.UpdatedParams) { db.AppParams.Attach(item); db.Entry <AppParam>(item).State = EntityState.Modified; } } if (dto.DeletedParams != null) { foreach (var item in dto.DeletedParams) { db.AppParams.Attach(item); db.Entry <AppParam>(item).State = EntityState.Deleted; } } ; if (dto.AddedFormulae != null) { db.Formulae.AddRange(dto.AddedFormulae); } if (dto.UpdatedFormulae != null) { foreach (var item in dto.UpdatedFormulae) { db.Formulae.Attach(item); db.Entry <Formula>(item).State = EntityState.Modified; } } if (dto.DeletedFormulae != null) { foreach (var item in dto.DeletedFormulae) { db.Formulae.Attach(item); db.Entry <Formula>(item).State = EntityState.Deleted; } } try { db.SaveChanges(); } catch (DbUpdateException) { throw; } return(Created <string>(this.Url.Request.RequestUri, "ok")); //return Created(dto); }
public async Task T_params_Composite() { using (var client = new HttpClient()) { //get app Uri uri = new Uri(TestConfig.serviceUrl); var context = new DamServiceRef.Container(uri); context.Format.UseJson(); var appItem = context.Apps.Where(s => s.AppName == "第二支仪器").SingleOrDefault(); var conParam1 = new ConstantParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "sc1", ParamSymbol = "sc1", PrecisionNum = 2, UnitSymbol = "no", Val = 1, Order = 1, Description = "no description", }; var mesParam1 = new MessureParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "sm1", ParamSymbol = "sm1", PrecisionNum = 2, UnitSymbol = "no", Order = 1, Description = "no description", }; var calParam1 = new CalculateParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "scal1", ParamSymbol = "scal1", PrecisionNum = 2, UnitSymbol = "no", Order = 1, Description = "no description", }; var formula1 = new Formula() { Id = Guid.NewGuid(), ParamId = calParam1.Id, StartDate = DateTimeOffset.MinValue, EndDate = DateTimeOffset.MaxValue, CalculateOrder = 1, FormulaExpression = "sc1+sm1" }; ParamsDTO dto = new ParamsDTO() { Id = appItem.Id, }; dto.AddedParams = new List <AppParam>() { conParam1, mesParam1, calParam1 }; dto.AddedFormulae = new List <Formula>() { formula1 }; // New code: client.BaseAddress = new Uri(TestConfig.baseAddress); client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, " insert param fail"); //add updated ,delete var mesParam2 = new MessureParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "sm2", ParamSymbol = "sm2", PrecisionNum = 2, UnitSymbol = "no", Order = 1, Description = "no description", }; var calParam2 = new CalculateParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "scal2", ParamSymbol = "scal2", PrecisionNum = 2, UnitSymbol = "no", Order = 1, Description = "no description", }; var formula2 = new Formula() { Id = Guid.NewGuid(), ParamId = calParam2.Id, StartDate = DateTimeOffset.MinValue, EndDate = DateTimeOffset.MaxValue, CalculateOrder = 1, FormulaExpression = "sm2+sc1" }; dto = new ParamsDTO() { Id = appItem.Id, }; dto.AddedParams = new List <AppParam>() { calParam2 }; dto.AddedFormulae = new List <Formula>() { formula2 }; // fail because no corresponding mesparam response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsFalse(response.IsSuccessStatusCode, "constraint fail"); //fail because delete mes1 which cal param1 use dto = new ParamsDTO() { Id = appItem.Id, }; dto.AddedParams = new List <AppParam>() { mesParam2, calParam2 }; dto.AddedFormulae = new List <Formula>() { formula2 }; dto.DeletedParams = new List <AppParam>() { mesParam1 }; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsFalse(response.IsSuccessStatusCode, "constraint fail"); //delete mes1 and calc1 dto = new ParamsDTO() { Id = appItem.Id, }; conParam1.Val = 2; dto.AddedParams = new List <AppParam>() { mesParam2, calParam2 }; dto.AddedFormulae = new List <Formula>() { formula2 }; dto.UpdatedParams = new List <AppParam>() { conParam1 }; dto.DeletedParams = new List <AppParam>() { mesParam1, calParam1 }; dto.DeletedFormulae = new List <Formula>() { formula1 }; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, "constraint fail"); //delete all params dto = new ParamsDTO() { Id = appItem.Id, }; dto.DeletedParams = new List <AppParam>() { conParam1, mesParam2, calParam2 }; dto.DeletedFormulae = new List <Formula>() { formula2 }; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, "delete formulae fail"); //reload mesparam context = new DamServiceRef.Container(uri); var cnt = context.AppParams.Where(s => s.Id == mesParam1.Id).Count(); Assert.AreEqual(0, cnt, "delete test fail"); } }
public async Task T_paramsUpdate() { using (var client = new HttpClient()) { //get app Uri uri = new Uri(TestConfig.serviceUrl); var context = new DamServiceRef.Container(uri); context.Format.UseJson(); var appItem = context.Apps.Expand("AppParams").Where(s => s.AppName == "第一支仪器").SingleOrDefault(); // New code: client.BaseAddress = new Uri(TestConfig.baseAddress); client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); ParamsDTO dto = new ParamsDTO() { Id = appItem.Id, }; var conParam1 = appItem.AppParams.OfType <ConstantParam>().SingleOrDefault(); var mesParam1 = appItem.AppParams.OfType <MessureParam>().SingleOrDefault(); var calParam1 = appItem.AppParams.OfType <CalculateParam>().SingleOrDefault(); conParam1.Order += 1; mesParam1.Order += 1; calParam1.Order += 1; dto.UpdatedParams = new List <AppParam>() { conParam1, mesParam1, calParam1 }; HttpResponseMessage response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, " update param fail"); //all update success //test acid mesParam1.ParamSymbol = "modify"; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); context.Detach(mesParam1); Assert.IsFalse(response.IsSuccessStatusCode, "test formulae fail"); //reload mesparam var itemInDb = context.AppParams.Where(s => s.Id == mesParam1.Id).SingleOrDefault(); Assert.AreNotEqual(mesParam1.ParamSymbol, itemInDb.ParamSymbol, "acid test fail"); } }
public async Task T_paramsMes() { using (var client = new HttpClient()) { //get app Uri uri = new Uri(TestConfig.serviceUrl); var context = new DamServiceRef.Container(uri); context.Format.UseJson(); var appItem = context.Apps.Where(s => s.AppName == "第一支仪器").SingleOrDefault(); // New code: client.BaseAddress = new Uri(TestConfig.baseAddress); client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); ParamsDTO dto = new ParamsDTO() { Id = appItem.Id, }; var param = new MessureParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "mtest2", ParamSymbol = "mtest2", PrecisionNum = 2, UnitSymbol = "no", Order = 1, Description = "no description", }; dto.AddedParams = new List <AppParam>() { param }; HttpResponseMessage response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, "add param fail"); //modify dto = new ParamsDTO() { Id = appItem.Id, }; param.Order += 1; dto.UpdatedParams = new List <AppParam>() { param }; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, "delete param fail"); //now deleted added param dto = new ParamsDTO() { Id = appItem.Id, }; dto.DeletedParams = new List <AppParam>() { param }; response = await client.PostAsJsonAsync("api/ParamsDTOs", dto); Assert.IsTrue(response.IsSuccessStatusCode, "delete param fail"); } }
public async Task T_paramsConst2() { //get app Uri uri = new Uri(TestConfig.serviceUrl); var context = new DamServiceRef.Container(uri); context.Format.UseJson(); var appItem = context.Apps.Where(s => s.AppName == "第一支仪器").SingleOrDefault(); // New code: ParamsDTO dto = new ParamsDTO() { Id = appItem.Id, }; var conParam1 = new ConstantParam() { Id = Guid.NewGuid(), AppId = appItem.Id, ParamName = "c2", ParamSymbol = "c2", PrecisionNum = 2, UnitSymbol = "no", Val = 1, Order = 1, Description = "no description", }; dto.AddedParams = new List <AppParam>() { conParam1 }; Assert.IsTrue(context.UpdateAppParams(dto), "add param fail"); //modify dto = new ParamsDTO() { Id = appItem.Id, }; conParam1.Order += 1; dto.UpdatedParams = new List <AppParam>() { conParam1 }; Assert.IsTrue(context.UpdateAppParams(dto), "delete param fail"); //now deleted added param dto = new ParamsDTO() { Id = appItem.Id, }; dto.DeletedParams = new List <AppParam>() { conParam1 }; Assert.IsTrue(context.UpdateAppParams(dto), "delete param fail"); }
public void StartScraping(ParamsDTO queryParams) { _scraper.StartScraping(queryParams); }