/// <summary>
/// Get the element with a callback.
/// </summary>
public void Get(IAction <Element> onGet)
{
_lock.Take();
_callbacks.Add(onGet);
if (_processing)
{
_lock.Release();
return;
}
_processing = true;
// check if the link has or should be updated
if (_element == null)
{
if (_path == null)
{
ManagerUpdate.Control.AddSingle(OnRetrieved, _getElement.Run());
}
else
{
new ElementParser(_path, Act.New(OnParsed, (ElementParser)null)).Run();
}
}
else
{
if (_nextUpdate < Time.Milliseconds)
{
if (_path == null)
{
ManagerUpdate.Control.AddSingle(OnRetrieved, _getElement.Run());
}
else
{
new ElementParser(_path, Act.New(OnParsed, (ElementParser)null)).Run();
}
}
else
{
_processing = false;
foreach (var callback in _callbacks)
{
callback.ArgA = _element.Clone();
ManagerUpdate.Control.AddSingle(callback);
}
_callbacks.Clear();
}
}
_lock.Release();
}
/// <summary>
/// Start processing the specified address. This stops current processing.
/// </summary>
protected void PreConnect()
{
// stop timeout
_timer.Run = false;
// if the next url hasn't been set - get one from the crawler
Url next = _urls.Dequeue() ? _urls.Current : _session.UrlControl.NextUrl;
// was the next url retrieved?
if (next == Url.Empty)
{
// set the timer to move to the next url on timeout
_timer.OnDone = _stopped;
_timer.Reset(500);
return;
}
// does the next host match the last?
if (next.Host != Url.Host)
{
// no, get a new identity
Identity = GetIdentity.Run();
}
// set the next url
Url = next;
// ensure the url is valid
Uri uri;
if (!Uri.TryCreate(Url.ToString(), UriKind.Absolute, out uri))
{
// start again
_stopped.Run();
return;
}
// connecting
Connecting = true;
// create a web request
_request = System.Net.WebRequest.CreateHttp(Url.ToString());
_request.AuthenticationLevel = System.Net.Security.AuthenticationLevel.None;
// ensure no caching
_request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore);
// set the request cookies
_request.CookieContainer = Cookies;
// set the request identity
Identity.UpdateRequest(_request);
// reset timeout
_request.Timeout = Randomize.Range(_session.CrawlerMinConnectTimeout, _session.CrawlerMaxConnectTimeout);
// start the timer
_time.Start();
// connect to the url
_connect.Run();
}
//-------------------------------------------//
/// <summary>
/// Construct a new Table instance.
/// </summary>
protected Table(string keyspaceName = null)
{
_keyspaceName = keyspaceName;
_initialize = true;
_lock = new Lock();
_currentCollections = new ArrayRig <RowCollection>();
// if the name wasn't set
Name = Name == null?this.GetType().Name.ToLowercase() : Name.ToLowercase();
// populate the columns
Columns = new ArrayRig <Column>();
// iterate properties
foreach (PropertyInfo info in this.GetType().GetProperties(
BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static))
{
if (info.PropertyType.BaseType == typeof(Column))
{
// get the column attribute
ColumnAttribute attribute = info.GetCustomAttribute <ColumnAttribute>(true);
// if the attribute isn't found - assume default values
if (attribute == null)
{
attribute = new ColumnAttribute(Column.ColumnClass.Data, info.Name);
}
// get the column value if it's been set
Column column = (Column)info.GetValue(this);
// if the column hasn't been assigned
if (column == null)
{
// create a new instance of the column
IFunc <ColumnAttribute, Column> activator = Dynamic.Constructor <IFunc <ColumnAttribute, Column> >(info.PropertyType);
if (attribute.Name == null)
{
attribute.Name = info.Name;
}
activator.ArgA = attribute;
column = activator.Run();
info.SetValue(this, column);
}
// add the column to the collection
Columns.Add(column);
}
}
// iterate fields
foreach (FieldInfo info in this.GetType().GetFields(
BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static))
{
if (info.FieldType.BaseType == typeof(Column))
{
// get the column attribute
ColumnAttribute attribute = info.GetCustomAttribute <ColumnAttribute>(true);
// if the attribute isn't found - assume default values
if (attribute == null)
{
attribute = new ColumnAttribute(Column.ColumnClass.Data, info.Name);
}
// get the column value if it's been set
Column column = (Column)info.GetValue(this);
// if the column hasn't been assigned
if (column == null)
{
// create a new instance of the column
IFunc <ColumnAttribute, Column> activator = Dynamic.Constructor <FuncSet <ColumnAttribute, Column> >(info.FieldType);
if (attribute.Name == null)
{
attribute.Name = info.Name;
}
activator.ArgA = attribute;
column = activator.Run();
info.SetValue(this, column);
}
// add the column to the collection
Columns.Add(column);
}
}
}