public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { if (receiveData == null) { receiveData = new DataToSentTable(); } var file = new ReadFileToString { FileEnconding = Encoding.UTF8, FileToRead = fileName }; var data = await file.LoadData(); var dt = new DataTable { TableName = "FileContents" }; dt.Columns.Add(new DataColumn($"FilePath", typeof(string))); dt.Columns.Add(new DataColumn($"FileContents", typeof(string))); dt.Rows.Add(fileName, data); FastAddTable(receiveData, dt); return(receiveData); }
public override async Task <string> GetData() { var file = new ReadFileToString { FileEnconding = Encoding.UTF8, FileToRead = this.adress }; var data = await file.LoadData(); return(data); }
public async Task <string[]> StreamData() { var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.NameFile }; var data = await file.LoadData(); var splitLines = data.Split(Separator); return(await Task.FromResult(splitLines)); }
public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.File }; var data = await file.LoadData(); var doc = new HtmlDocument(); doc.LoadHtml(data); var metas = doc.DocumentNode.SelectNodes("//head/meta"); if ((metas?.Count ?? 0) == 0) { throw new ArgumentException("not found head meta"); } var ret = new DataToSentTable(); var dt = new DataTable { TableName = $"TableMeta" }; dt.Columns.Add("meta_name"); dt.Columns.Add("meta_content"); if (metas != null) { foreach (var meta in metas) { var attr = meta.Attributes; if (!attr.Contains("name")) { continue; } var arr = new string[] { attr["name"].Value, attr["content"].Value }; dt.Rows.Add(arr); } } var id = ret.AddNewTable(dt); ret.Metadata.AddTable(dt, id); return(ret); }
protected async Task <HtmlNodeCollection> Find(string find) { var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.File }; var data = await file.LoadData(); var doc = new HtmlDocument(); doc.LoadHtml(data); return(doc.DocumentNode.SelectNodes(find)); }
public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { var ret = new DataToSentTable(); var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.File }; var data = await file.LoadData(); bool web = file.FileType == FileType.Web; var startWeb = ""; if (web) { var uri = new Uri(File); startWeb = uri.GetLeftPart(UriPartial.Authority); } var tr = new AHrefToTable(); var dt = tr.TransformToTable(data); if (web) { foreach (DataRow item in dt.Rows) { string val = item["href"]?.ToString(); if (val == null) { continue; } if (val.StartsWith("/")) { val = (startWeb + val).Replace("//", "/"); } item["href"] = val; } } var id = ret.AddNewTable(dt); ret.Metadata.AddTable(dt, id); return(ret); }
public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.File }; var data = await file.LoadData(); var doc = new HtmlDocument(); doc.LoadHtml(data); var nodes = doc.DocumentNode.SelectNodes(XPathExpression); //maybe return empty table? if ((nodes?.Count ?? 0) == 0) { throw new ArgumentException($"not found {XPathExpression}"); } var ret = new DataToSentTable(); int nrTable = 0; var dt = new DataTable { TableName = $"Table{nrTable}" }; dt.Columns.Add("item"); dt.Columns.Add("item_html"); var id = ret.AddNewTable(dt); ret.Metadata.AddTable(dt, id); if (nodes != null) { foreach (var node in nodes) { dt.Rows.Add(new[] { node.InnerText, node.OuterHtml }); } } return(ret); }
async Task <List <KeyValuePair <long, string> > > components() { var ret = new List <KeyValuePair <long, string> >(); //TODO: replace with json stankins //TODO: replace with another component that does better cachet var f = new ReadFileToString(); f.FileToRead = urlCachet + "/api/v1/components?per_page=1000"; var data = await f.LoadData(); var token = JToken.Parse(data); var comps = token["data"] as JArray; foreach (var item in comps) { int id = int.Parse(item["id"].ToString()); ret.Add(new KeyValuePair <long, string>(id, item["name"].ToString())); } return(ret); }
public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { var file = new ReadFileToString { FileEnconding = this.Encoding, FileToRead = this.File }; var data = await file.LoadData(); bool web = file.FileType == FileType.Web; var ret = new DataToSentTable(); var dt = new DataTable { TableName = $"TableLinks" }; var regex = new Regex(Expression); var names = regex. GetGroupNames(). Where(it => !int.TryParse(it, out var _)). ToArray(); foreach (var n in names) { dt.Columns.Add(new DataColumn(n, typeof(string))); } RegexOptions options = RegexOptions.Multiline; MatchCollection matches = Regex.Matches(data, Expression, options); foreach (Match g in matches) { if (!g.Success) { continue; } var item = new string[names.Length]; var groups = g.Groups; int i = -1; foreach (var n in names) { i++; Group gr; try { gr = groups[n]; } catch { //TODO: log exception or put in exception table continue; } if (gr.Success) { item[i] = gr.Value; } } dt.Rows.Add(item); } var id = ret.AddNewTable(dt); ret.Metadata.AddTable(dt, id); return(ret); }
public override async Task <IDataToSent> TransformData(IDataToSent receiveData) { if (receiveData == null) { receiveData = new DataToSentTable(); } var file = new ReadFileToString { FileEnconding = this.encoding, FileToRead = this.fileNameYaml }; var data = await file.LoadData(); var yaml = new YamlDevOpsVisitor(); yaml.LoadFromString(data); var jobs = yaml.jobs; var dtJobs = new DataTable("jobs"); dtJobs.Columns.Add(new DataColumn("name", typeof(string))); dtJobs.Columns.Add(new DataColumn("condition", typeof(string))); dtJobs.Columns.Add(new DataColumn("pool", typeof(string))); var dtSteps = new DataTable("steps"); dtSteps.Columns.Add("jobName", typeof(string)); dtSteps.Columns.Add(new DataColumn("displayName", typeof(string))); dtSteps.Columns.Add(new DataColumn("name", typeof(string))); dtSteps.Columns.Add(new DataColumn("value", typeof(string))); var dtDepends = new DataTable("jobDepends"); dtDepends.Columns.Add("jobName", typeof(string)); dtDepends.Columns.Add("jobNameDepends", typeof(string)); int idJob = 0; foreach (var job in jobs) { idJob++; if (string.IsNullOrEmpty(job.Name)) { job.Name = $"NewJob{idJob}"; } dtJobs.Rows.Add(job.Name, job.condition, job.pool.ToString()); foreach (var step in job.Steps) { dtSteps.Rows.Add(job.Name, step.DisplayName, step.Name, step.Value); } foreach (var name in job.DependsOn) { dtDepends.Rows.Add(job.Name, name); } } var id = receiveData.AddNewTable(dtJobs); receiveData.Metadata.AddTable(dtJobs, id); id = receiveData.AddNewTable(dtSteps); receiveData.Metadata.AddTable(dtSteps, id); id = receiveData.AddNewTable(dtDepends); receiveData.Metadata.AddTable(dtDepends, id); return(await Task.FromResult(receiveData)); }