private void StoreLinks(List <string> filtredLinks, DataTable dt)
{
for (int i = 0; i < filtredLinks.Count; i++)
{
if (dt.Select($"Url = '{filtredLinks[i]}'").Length == 0)
{
try
{
var existingUrl = this.webDbContext.UrlAdresses.Where(url => url.urlAddress == filtredLinks[i]).FirstOrDefault();
if (existingUrl == null)
{
existingUrl = new UrlAddress(filtredLinks[i]);
this.webDbContext.UrlAdresses.Add(existingUrl).State = EntityState.Added;
this.webDbContext.SaveChanges();
}
Response reponseTimeModel = new Response(existingUrl.Id, this.GetResponseTime(filtredLinks[i]));
this.webDbContext.Responses.Add(reponseTimeModel);
DataRow row = dt.NewRow();
row["Url"] = filtredLinks[i];
row["Response Time"] = reponseTimeModel.ResponseTime + " s";
row["UrlLength"] = filtredLinks[i].Length;
row["id"] = existingUrl.Id;
dt.Rows.Add(row);
}
catch (WebException)
{
}
}
}
this.webDbContext.SaveChanges();
}
public void DownloadImagesFromUrl()
{
using (WebClient webClient = new WebClient())
{
//https://msdn.microsoft.com/en-us/library/system.windows.forms.savefiledialog(v=vs.110).aspx use instead hardcoding the file paths.
foreach (var image in ListOfImagePaths)
{
//Use instead of string for the url address, instance of Url class (.Net) so you can avoid hardcoded indexes like that.
var primaryUrlEndIndex = UrlAddress.IndexOf(@"/", 9);
var primaryUrl = UrlAddress.Substring(0, primaryUrlEndIndex + 1);
var index = image.LastIndexOf(@"/");
var filePath = image.Substring(index + 1);
if (!File.Exists(DOWNLOADFILEPATH + filePath))
{
if (image.StartsWith("/"))
{
webClient.DownloadFile(new Uri(primaryUrl + image.Substring(1)), DOWNLOADFILEPATH + filePath);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Downloading image {image}");
}
else if (image.StartsWith(@"http"))
{
webClient.DownloadFile(new Uri(image), DOWNLOADFILEPATH + filePath);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Downloading image {image}");
}
}
}
}
}
public bool TryParse(string urlstring, out UrlAddress url)
{
url = new UrlAddress();
var match = Regex.Match(urlstring,
@"^(http|https|ftp)://(([\w\-]+)(\.[\w\-]+)+)(/([\w\-]+(/[\w\-\.]+)*))?(/?|(\?([\w]+=[\w]*(&([\w]+)=([\w]*))*)))$");
if (!match.Success)
{
url = null;
return(false);
}
url.Scheme = match.Groups[1].Value;
url.Host = match.Groups[2].Value;
url.UrlPath = match.Groups[6].Value;
url.Parameters = new List <Parameter>();
foreach (var param in match.Groups[10].ToString().Split('&'))
{
var paramMatch = Regex.Match(param, @"([\w]+)=([\w]*)");
if (!paramMatch.Success)
{
url.Parameters = null;
return(true);
}
url.Parameters.Add(new Parameter(paramMatch.Groups[1].Value, paramMatch.Groups[2].Value));
}
return(true);
}
/// <summary>
/// Retrieves and converts URLs from file.
/// </summary>
/// <param name="fileStream">FileStream for file with URLs.</param>
/// <returns>List of UrlAddress objects.</returns>
private List <UrlAddress> GetAdressesFromFile(FileStream fileStream)
{
List <UrlAddress> adresses = new List <UrlAddress>();
using (StreamReader reader = new StreamReader(fileStream))
{
string urlString = reader.ReadLine();
while (urlString != null)
{
UrlAddress address = UrlParser.Parse(urlString);
if (address != null)
{
adresses.Add(UrlParser.Parse(urlString));
this.logger.Info(urlString + " was succesfully written.");
}
else
{
this.logger.Error(urlString + " was not written.");
}
urlString = reader.ReadLine();
}
}
return(adresses);
}
/// <summary>
/// Method ParseFile convert initial file's lines to XML
/// </summary>
/// <param name="filePath"> line of initial file</param>
/// <returns>List of UrlAddresses that was taking from initial file</returns>
public List <UrlAddress> ParseFile(string filePath)
{
var file = new StreamReader(filePath);
var urlAddresses = new List <UrlAddress>();
try
{
string line;
while ((line = file.ReadLine()) != null)
{
var url = new Uri(line);
var urlAddress = new UrlAddress
{
HostName = url.Host
};
if (url.Segments.Length != 0)
{
var uri = new UriTag();
List <Segment> listSeg = new List <Segment>();
var seg = new Segment();
for (var i = 1; i < url.Segments.Length - 1; i++)
{
seg.Segments = url.Segments[i].Remove(url.Segments[i].Length - 1);
listSeg.Add(seg);
}
seg.Segments = url.Segments[url.Segments.Length - 1];
listSeg.Add(seg);
uri.Segments = listSeg;
urlAddress.UriTags = uri;
}
if (!string.IsNullOrEmpty(url.Query))
{
var param = url.Query.Remove(0, 1).Split('&');
foreach (var s in param)
{
var par = s.Split('=');
var urlPar = new Parameter()
{
Key = par[0],
Value = par[1]
};
urlAddress.Parametrs.Add(urlPar);
}
}
urlAddresses.Add(urlAddress);
}
}
catch (Exception)
{
Logger?.Info("Data is not valid.");
}
return(urlAddresses);
}
private UrlAddress Mapper(IDataReader reader)
{
UrlAddress singleItem = new UrlAddress();
int startingIndex = 0; //startingOrdinal
singleItem.Id = reader.GetSafeInt32(startingIndex++);
singleItem.Url = reader.GetSafeString(startingIndex++);
singleItem.CreatedDate = reader.GetSafeDateTime(startingIndex++);
singleItem.ModifiedDate = reader.GetSafeDateTime(startingIndex++);
singleItem.ModifiedBy = reader.GetSafeString(startingIndex++);
return(singleItem);
}
public List <UrlAddress> GetAll()
{
List <UrlAddress> list = new List <UrlAddress>();
DataProvider.ExecuteCmd("dbo.UrlAddress_SelectAll"
, inputParamMapper : null
, singleRecordMapper : delegate(IDataReader reader, short set)
{
UrlAddress singleItem = Mapper(reader);
list.Add(singleItem);
});
return(list);
}
public UrlAddress Get(int id)
{
UrlAddress singleItem = null;
DataProvider.ExecuteCmd("dbo.UrlAddress_SelectById"
, inputParamMapper : delegate(SqlParameterCollection paramCollection)
{
paramCollection.AddWithValue("@Id", id);
}
, singleRecordMapper : delegate(IDataReader reader, short set)
{
singleItem = Mapper(reader);
});
return(singleItem);
}
public static XElement ToXml(this UrlAddress url)
{
XElement xml = new XElement("URL");
xml.Add(new XElement("Schema", url.Scheme));
xml.Add(new XElement("Host", url.Host));
if (ReferenceEquals(null, url.UrlPath))
{
return(xml);
}
xml.Add(new XElement("UrlPath", url.UrlPath));
xml.Add(new XElement("Parameters"));
if (!ReferenceEquals(null, url.Parameters))
{
foreach (var param in url.Parameters)
{
xml.Element("Parameters")?.Add(
new XElement("Parameter",
new XElement("Name", param.Name),
new XElement("Value", param.Value)));
}
}
return(xml);
}
