public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
if (receiveData == null)
{
receiveData = new DataToSentTable();
}
var file = new ReadFileToString
{
FileEnconding = Encoding.UTF8,
FileToRead = fileName
};
var data = await file.LoadData();
var dt = new DataTable
{
TableName = "FileContents"
};
dt.Columns.Add(new DataColumn($"FilePath", typeof(string)));
dt.Columns.Add(new DataColumn($"FileContents", typeof(string)));
dt.Rows.Add(fileName, data);
FastAddTable(receiveData, dt);
return(receiveData);
}
public override async Task <string> GetData()
{
var file = new ReadFileToString
{
FileEnconding = Encoding.UTF8,
FileToRead = this.adress
};
var data = await file.LoadData();
return(data);
}
public async Task <string[]> StreamData()
{
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.NameFile
};
var data = await file.LoadData();
var splitLines = data.Split(Separator);
return(await Task.FromResult(splitLines));
}
public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.File
};
var data = await file.LoadData();
var doc = new HtmlDocument();
doc.LoadHtml(data);
var metas = doc.DocumentNode.SelectNodes("//head/meta");
if ((metas?.Count ?? 0) == 0)
{
throw new ArgumentException("not found head meta");
}
var ret = new DataToSentTable();
var dt = new DataTable
{
TableName = $"TableMeta"
};
dt.Columns.Add("meta_name");
dt.Columns.Add("meta_content");
if (metas != null)
{
foreach (var meta in metas)
{
var attr = meta.Attributes;
if (!attr.Contains("name"))
{
continue;
}
var arr = new string[] { attr["name"].Value, attr["content"].Value };
dt.Rows.Add(arr);
}
}
var id = ret.AddNewTable(dt);
ret.Metadata.AddTable(dt, id);
return(ret);
}
protected async Task <HtmlNodeCollection> Find(string find)
{
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.File
};
var data = await file.LoadData();
var doc = new HtmlDocument();
doc.LoadHtml(data);
return(doc.DocumentNode.SelectNodes(find));
}
public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
var ret = new DataToSentTable();
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.File
};
var data = await file.LoadData();
bool web = file.FileType == FileType.Web;
var startWeb = "";
if (web)
{
var uri = new Uri(File);
startWeb = uri.GetLeftPart(UriPartial.Authority);
}
var tr = new AHrefToTable();
var dt = tr.TransformToTable(data);
if (web)
{
foreach (DataRow item in dt.Rows)
{
string val = item["href"]?.ToString();
if (val == null)
{
continue;
}
if (val.StartsWith("/"))
{
val = (startWeb + val).Replace("//", "/");
}
item["href"] = val;
}
}
var id = ret.AddNewTable(dt);
ret.Metadata.AddTable(dt, id);
return(ret);
}
public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.File
};
var data = await file.LoadData();
var doc = new HtmlDocument();
doc.LoadHtml(data);
var nodes = doc.DocumentNode.SelectNodes(XPathExpression);
//maybe return empty table?
if ((nodes?.Count ?? 0) == 0)
{
throw new ArgumentException($"not found {XPathExpression}");
}
var ret = new DataToSentTable();
int nrTable = 0;
var dt = new DataTable
{
TableName = $"Table{nrTable}"
};
dt.Columns.Add("item");
dt.Columns.Add("item_html");
var id = ret.AddNewTable(dt);
ret.Metadata.AddTable(dt, id);
if (nodes != null)
{
foreach (var node in nodes)
{
dt.Rows.Add(new[] { node.InnerText, node.OuterHtml });
}
}
return(ret);
}
async Task <List <KeyValuePair <long, string> > > components()
{
var ret = new List <KeyValuePair <long, string> >();
//TODO: replace with json stankins
//TODO: replace with another component that does better cachet
var f = new ReadFileToString();
f.FileToRead = urlCachet + "/api/v1/components?per_page=1000";
var data = await f.LoadData();
var token = JToken.Parse(data);
var comps = token["data"] as JArray;
foreach (var item in comps)
{
int id = int.Parse(item["id"].ToString());
ret.Add(new KeyValuePair <long, string>(id, item["name"].ToString()));
}
return(ret);
}
public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
var file = new ReadFileToString
{
FileEnconding = this.Encoding,
FileToRead = this.File
};
var data = await file.LoadData();
bool web = file.FileType == FileType.Web;
var ret = new DataToSentTable();
var dt = new DataTable
{
TableName = $"TableLinks"
};
var regex = new Regex(Expression);
var names = regex.
GetGroupNames().
Where(it => !int.TryParse(it, out var _)).
ToArray();
foreach (var n in names)
{
dt.Columns.Add(new DataColumn(n, typeof(string)));
}
RegexOptions options = RegexOptions.Multiline;
MatchCollection matches = Regex.Matches(data, Expression, options);
foreach (Match g in matches)
{
if (!g.Success)
{
continue;
}
var item = new string[names.Length];
var groups = g.Groups;
int i = -1;
foreach (var n in names)
{
i++;
Group gr;
try
{
gr = groups[n];
}
catch
{
//TODO: log exception or put in exception table
continue;
}
if (gr.Success)
{
item[i] = gr.Value;
}
}
dt.Rows.Add(item);
}
var id = ret.AddNewTable(dt);
ret.Metadata.AddTable(dt, id);
return(ret);
}
public override async Task <IDataToSent> TransformData(IDataToSent receiveData)
{
if (receiveData == null)
{
receiveData = new DataToSentTable();
}
var file = new ReadFileToString
{
FileEnconding = this.encoding,
FileToRead = this.fileNameYaml
};
var data = await file.LoadData();
var yaml = new YamlDevOpsVisitor();
yaml.LoadFromString(data);
var jobs = yaml.jobs;
var dtJobs = new DataTable("jobs");
dtJobs.Columns.Add(new DataColumn("name", typeof(string)));
dtJobs.Columns.Add(new DataColumn("condition", typeof(string)));
dtJobs.Columns.Add(new DataColumn("pool", typeof(string)));
var dtSteps = new DataTable("steps");
dtSteps.Columns.Add("jobName", typeof(string));
dtSteps.Columns.Add(new DataColumn("displayName", typeof(string)));
dtSteps.Columns.Add(new DataColumn("name", typeof(string)));
dtSteps.Columns.Add(new DataColumn("value", typeof(string)));
var dtDepends = new DataTable("jobDepends");
dtDepends.Columns.Add("jobName", typeof(string));
dtDepends.Columns.Add("jobNameDepends", typeof(string));
int idJob = 0;
foreach (var job in jobs)
{
idJob++;
if (string.IsNullOrEmpty(job.Name))
{
job.Name = $"NewJob{idJob}";
}
dtJobs.Rows.Add(job.Name, job.condition, job.pool.ToString());
foreach (var step in job.Steps)
{
dtSteps.Rows.Add(job.Name, step.DisplayName, step.Name, step.Value);
}
foreach (var name in job.DependsOn)
{
dtDepends.Rows.Add(job.Name, name);
}
}
var id = receiveData.AddNewTable(dtJobs);
receiveData.Metadata.AddTable(dtJobs, id);
id = receiveData.AddNewTable(dtSteps);
receiveData.Metadata.AddTable(dtSteps, id);
id = receiveData.AddNewTable(dtDepends);
receiveData.Metadata.AddTable(dtDepends, id);
return(await Task.FromResult(receiveData));
}