/// <summary>
/// Called when the spider is ready to process an HTML
/// URL. Download the contents of the URL to a local file.
/// </summary>
/// <param name="url">The URL that the spider is about to process.</param>
/// <param name="parse">An object that will allow you you to parse the HTML on this page.</param>
public void SpiderProcessURL(Uri url, SpiderParseHTML parse)
{
String filename = URLUtility.convertFilename(this.path, url, true);
Stream os = new FileStream(filename, FileMode.Create);
parse.Stream.OutputStream = os;
parse.ReadAll();
os.Close();
}
public void FindBestIcon()
{
if (this.SiteUri == null)
{
this.myLogger.LogWarn("No URL. Stop Find Best Icon");
return;
}
FindBestIconForURI(this.SiteUri);
if (this.BestImage == null) // try with www
{
Uri testURI = this.SiteUri;
if (!testURI.Host.StartsWith("www."))
{
FindBestIconForURI(new Uri(
this.SiteUri.Scheme + "://www." + this.SiteUri.DnsSafeHost));
}
}
string rootDomain = URLUtility.RootDomain(this.SiteUri.AbsoluteUri);
if (!string.IsNullOrEmpty(rootDomain))
{
if (this.BestImage == null) // try with root domain prefix
{
FindBestIconForURI(new Uri(this.SiteUri.Scheme + "://" + rootDomain));
}
if (this.BestImage == null) // try with host prefix and www
{
FindBestIconForURI(new Uri(this.SiteUri.Scheme + "://www." + rootDomain));
}
}
if (this.RootUriForIco != null)
{
if (this.BestImage == null) // try with host prefix
{
FindBestIconForURI(new Uri(
this.RootUriForIco.Scheme + "://" + this.RootUriForIco.Host));
}
if (this.BestImage == null) // try with host prefix and www
{
FindBestIconForURI(new Uri(
this.RootUriForIco.Scheme + "://www." + this.RootUriForIco.Host));
}
}
if (this.BestImage == null)
{
FindBestIconForURI(this.OriginalSiteUri);
}
}
public void TestIsValidURL()
{
Assert.That(URLUtility.isValidURLString("http://github.com/vitdeck/VitDeck"), Is.True);
Assert.That(URLUtility.isValidURLString("https://github.com/vitdeck/VitDeck/blob/56b11a7dbbca3b8cf6c4104ca443f78f2beae9c1/LICENSE?query=test#1"), Is.True);
Assert.That(URLUtility.isValidURLString("http://www.google.co.jp/search?q=C"), Is.True);
Assert.That(URLUtility.isValidURLString("ftp://github.com/"), Is.False);
Assert.That(URLUtility.isValidURLString(""), Is.False);
Assert.That(URLUtility.isValidURLString(null), Is.False);
Assert.That(URLUtility.isValidURLString("Test String"), Is.False);
}
public static string ComposeURL(Protocol protocol, bool secure, string host, int port)
{
string result = URLUtility.ProtocolPrefix(protocol, secure) + "://" + host;
// Make this removal optional
if (port != 80)
{
result += ":" + port;
}
return(result);
}
/// <summary>
/// Called when the spider is about to process a NON-HTML
/// URL.
/// </summary>
/// <param name="url">The URL that the spider found.</param>
/// <param name="stream">An InputStream to read the page contents from.</param>
public void SpiderProcessURL(Uri url, Stream stream)
{
byte[] buffer = new byte[1024];
int length;
String filename = URLUtility.convertFilename(this.path, url, true);
Stream os = new FileStream(filename, FileMode.Create);
do
{
length = stream.Read(buffer, 0, buffer.Length);
if (length > 0)
{
os.Write(buffer, 0, length);
}
} while (length > 0);
os.Close();
}
/// <summary>
/// Used internally, to add a URL to the spider's workload.
/// </summary>
/// <param name="u">The URL to add.</param>
/// <param name="type">What type of link this is.</param>
private void AddURL(String u, Spider.URLType type)
{
if (u == null)
{
return;
}
try
{
Uri url = URLUtility.constructURL(this.baseURL, u, true);
url = this.spider.Workload.ConvertURL(url.ToString());
if ((String.Compare(url.Scheme, "http", true) == 0) ||
(String.Compare(url.Scheme, "https", true) == 0))
{
if (this.spider.Report.SpiderFoundURL(url, this.baseURL, type))
{
try
{
this.spider.AddURL(url, this.baseURL, this.depth + 1);
}
catch (WorkloadException e)
{
throw new IOException(e.Message);
}
}
}
}
catch (UriFormatException)
{
spider.Logging.Log(Logger.Level.INFO, "Malformed URL found:" + u);
}
catch (WorkloadException)
{
spider.Logging.Log(Logger.Level.INFO, "Invalid URL found:" + u);
}
}
/// <summary>
/// This method is called when an anchor(A) tag is found.
/// </summary>
/// <param name="href">The link found.</param>
private void HandleA(String href)
{
String cmp = null;
if (href != null)
{
href = href.Trim();
cmp = href.ToLower();
}
if ((cmp != null) && !URLUtility.containsInvalidURLCharacters(href))
{
if (!cmp.StartsWith("javascript:") &&
!cmp.StartsWith("rstp:") &&
!cmp.StartsWith("rtsp:") &&
!cmp.StartsWith("news:") &&
!cmp.StartsWith("irc:") &&
!cmp.StartsWith("mailto:"))
{
AddURL(href, Spider.URLType.HYPERLINK);
}
}
}
public void Initialize(PwDatabase db)
{
Debug.Assert(m_dicCustomIconsStats != null); if (m_dicCustomIconsStats == null)
{
throw new InvalidOperationException();
}
Debug.Assert(m_dicStandardIconsStats != null); if (m_dicStandardIconsStats == null)
{
throw new InvalidOperationException();
}
GroupHandler gh = delegate(PwGroup pg)
{
if (!(pg.CustomIconUuid.Equals(PwUuid.Zero)))
{
if (!(m_dicCustomIconsStats.ContainsKey(pg.CustomIconUuid)))
{
m_dicCustomIconsStats.Add(pg.CustomIconUuid, new IconStats());
}
m_dicCustomIconsStats[pg.CustomIconUuid].nbInGroups += 1;
m_dicCustomIconsStats[pg.CustomIconUuid].listGroups.Add(pg);
}
else
{
if (!(m_dicStandardIconsStats.ContainsKey(pg.IconId)))
{
m_dicStandardIconsStats.Add(pg.IconId, new IconStats());
}
m_dicStandardIconsStats[pg.IconId].nbInGroups += 1;
m_dicStandardIconsStats[pg.IconId].listGroups.Add(pg);
}
return(true);
};
String urlFieldValue;
Uri siteUri;
List <Uri> uriList;
EntryHandler eh = delegate(PwEntry pe)
{
if (!(pe.CustomIconUuid.Equals(PwUuid.Zero)))
{
if (!(m_dicCustomIconsStats.ContainsKey(pe.CustomIconUuid)))
{
m_dicCustomIconsStats.Add(pe.CustomIconUuid, new IconStats());
}
m_dicCustomIconsStats[pe.CustomIconUuid].nbInEntries += 1;
m_dicCustomIconsStats[pe.CustomIconUuid].listEntries.Add(pe);
uriList = m_dicCustomIconsStats[pe.CustomIconUuid].listUris;
}
else
{
if (!(m_dicStandardIconsStats.ContainsKey(pe.IconId)))
{
m_dicStandardIconsStats.Add(pe.IconId, new IconStats());
}
m_dicStandardIconsStats[pe.IconId].nbInEntries += 1;
m_dicStandardIconsStats[pe.IconId].listEntries.Add(pe);
uriList = m_dicStandardIconsStats[pe.IconId].listUris;
}
// Get Uri
urlFieldValue = pe.Strings.ReadSafe(PwDefs.UrlField);
if (!string.IsNullOrEmpty(urlFieldValue))
{
try {
urlFieldValue = URLUtility.addUrlHttpPrefix(urlFieldValue);
bool urlWellFFormatted = Uri.TryCreate(urlFieldValue, UriKind.Absolute, out siteUri);
if (urlWellFFormatted)
{
if ((siteUri.Scheme.ToLower() == "http") ||
(siteUri.Scheme.ToLower() == "https"))
{
uriList.Add(siteUri);
}
}
}
catch (UriFormatException e)
{
Debug.Write(String.Format("URIFormatException during parsing uri of entry = {0}", e.Message));
}
catch (Exception) { };
}
return(true);
};
gh(db.RootGroup);
db.RootGroup.TraverseTree(TraversalMethod.PreOrder, gh, eh);
m_isInitialized = true;
}
public string GenerateJS(Type modelType, string host, List <string> readOnlyProperties, List <string> properties, List <string> viewIgnoreProperties, bool hasUpdate, bool hasAdd, bool hasDelete, bool minimize)
{
WrappedStringBuilder sb = new WrappedStringBuilder(minimize);
if (!minimize)
{
sb.AppendLine("//Org.Reddragonit.BackBoneDotNet.JSGenerators.ModelListCallGenerators");
}
foreach (MethodInfo mi in modelType.GetMethods(Constants.LOAD_METHOD_FLAGS))
{
if (mi.GetCustomAttributes(typeof(ModelListMethod), false).Length > 0)
{
foreach (ModelListMethod mlm in mi.GetCustomAttributes(typeof(ModelListMethod), false))
{
if (mlm.Host == host || mlm.Host == "*")
{
WrappedStringBuilder sbCurParameters = new WrappedStringBuilder(minimize);
sbCurParameters.Append((minimize ? "function(){return{":"function(){return {"));
sb.Append(string.Format((minimize ?
"{0}=_.extend({0},{{{1}:function("
: "{0} = _.extend({0}, {{{1}:function("), ModelNamespace.GetFullNameForModel(modelType, host), mi.Name));
for (int x = 0; x < (mlm.Paged ? mi.GetParameters().Length - 3 : mi.GetParameters().Length); x++)
{
sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
sbCurParameters.AppendLine((x == 0 ? "" : ",") + string.Format("'{0}':{0}", mi.GetParameters()[x].Name));
}
sbCurParameters.Append("};}");
if (mlm.Paged)
{
if (mi.GetParameters().Length != 3)
{
sb.Append(",");
}
sb.Append("pageStartIndex,pageSize");
}
sb.AppendLine("){");
string urlCode = URLUtility.CreateJavacriptUrlCode(mlm, mi, modelType);
sb.Append(urlCode);
if (mlm.Paged)
{
sb.AppendLine(string.Format((minimize ?
"pageStartIndex=(pageStartIndex==undefined?0:(pageStartIndex==null?0:pageStartIndex));pageSize=(pageSize==undefined?10:(pageSize==null?10:pageSize));var ret=Backbone.Collection.extend({{url:url+'{0}PageStartIndex='+pageStartIndex+'&PageSize='+pageSize,CurrentParameters:{1},currentIndex:pageStartIndex*pageSize,currentPageSize:pageSize,CurrentPage:Math.floor(pageStartIndex/pageSize),parse:function(response){{if(response.Backbone!=undefined){{_.extend(Backbone,response.Backbone);}}response=response.response;this.TotalPages=response.Pager.TotalPages;return response.response;}},MoveToPage:function(pageNumber){{if(pageNumber>=0&&pageNumber<this.TotalPages){{this.currentIndex=pageNumber*this.currentPageSize;{2}this.fetch();this.CurrentPage=pageNumber;}}}},ChangePageSize:function(pageSize){{this.currentPageSize=pageSize;this.MoveToPage(Math.floor(this.currentIndex/pageSize));}},MoveToNextPage:function(){{if(Math.floor(this.currentIndex/this.currentPageSize)+1<this.TotalPages){{this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)+1);}}}},MoveToPreviousPage:function(){{if(Math.floor(this.currentIndex/this.currentPageSize)-1>=0){{this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)-1);}}}},"
:@"pageStartIndex = (pageStartIndex == undefined ? 0 : (pageStartIndex == null ? 0 : pageStartIndex));
pageSize = (pageSize == undefined ? 10 : (pageSize == null ? 10 : pageSize));
var ret = Backbone.Collection.extend({{url:url+'{0}PageStartIndex='+pageStartIndex+'&PageSize='+pageSize,
CurrentParameters:{1},
currentIndex : pageStartIndex*pageSize,
currentPageSize : pageSize,
CurrentPage : Math.floor(pageStartIndex/pageSize),
parse : function(response){{
if(response.Backbone!=undefined){{
_.extend(Backbone,response.Backbone);
}}
response = response.response;
this.TotalPages = response.Pager.TotalPages;
return response.response;
}},
MoveToPage : function(pageNumber){{
if (pageNumber>=0 && pageNumber<this.TotalPages){{
this.currentIndex = pageNumber*this.currentPageSize;
{2}
this.fetch();
this.CurrentPage=pageNumber;
}}
}},
ChangePageSize : function(pageSize){{
this.currentPageSize = pageSize;
this.MoveToPage(Math.floor(this.currentIndex/pageSize));
}},
MoveToNextPage : function(){{
if(Math.floor(this.currentIndex/this.currentPageSize)+1<this.TotalPages){{
this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)+1);
}}
}},
MoveToPreviousPage : function(){{
if(Math.floor(this.currentIndex/this.currentPageSize)-1>=0){{
this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)-1);
}}
}},"), new object[] {
(mlm.Path.Contains("?") ? "&" : "?"),
sbCurParameters.ToString(),
(mlm.Path.Contains("?") ?
(minimize ? "" : "\t\t\t") + "this.url = this.url.substring(0,this.url.indexOf('&PageStartIndex='))+'&PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;" :
(minimize ? "" : "\t\t\t") + "this.url = this.url.substring(0,this.url.indexOf('?'))+'?PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;")
}));
if (mi.GetParameters().Length > 0)
{
sb.Append((minimize ? "" : "\t") + "ChangeParameters: function(");
for (int x = 0; x < mi.GetParameters().Length - 3; x++)
{
sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
}
sb.AppendLine(string.Format((minimize ?
"){{{0}url+='{1}PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;this.CurrentParameters={2};this.currentIndex=0;this.url=url;this.fetch();}},"
:@"){{{0}
url+='{1}PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;
this.CurrentParameters = {2};
this.currentIndex=0;
this.url=url;
this.fetch();
}},"), new object[] { urlCode, (mlm.Path.Contains("?") ? "&" : "?"), sbCurParameters.ToString() }));
}
sb.AppendLine(string.Format((minimize ?
"model:{0}.Model}});"
:@" model:{0}.Model
}});"), ModelNamespace.GetFullNameForModel(modelType, host)));
}
else
{
sb.AppendLine(string.Format((minimize ?
"var ret=Backbone.Collection.extend({{url:url,CurrentParameters:{0},parse : function(response){{if(response.Backbone!=undefined){{_.extend(Backbone,response.Backbone);return response.response;}}else{{return response;}}}},"
:@" var ret = Backbone.Collection.extend({{url:url,CurrentParameters:{0},parse : function(response){{
if(response.Backbone!=undefined){{
_.extend(Backbone,response.Backbone);
return response.response;
}}else{{
return response;
}}
}},"), sbCurParameters.ToString()));
if (mi.GetParameters().Length > 0)
{
sb.Append((minimize ? "" : "\t") + "ChangeParameters: function(");
for (int x = 0; x < mi.GetParameters().Length; x++)
{
sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
}
sb.AppendLine(string.Format((minimize ?
"){{{0}this.CurrentParameters={1};this.currentIndex=0;this.url=url;this.fetch();}},"
:@"){{{0}
this.CurrentParameters = {1};
this.currentIndex=0;
this.url=url;
this.fetch();
}},"), urlCode, sbCurParameters.ToString()));
}
sb.AppendLine("model:" + ModelNamespace.GetFullNameForModel(modelType, host) + ".Model});");
}
sb.AppendLine((minimize ?
"ret=new ret();return ret;}});"
:@"ret = new ret();
return ret;
}});"));
break;
}
}
}
}
return(sb.ToString());
}
