예제 #1
0
        /// <summary>
        /// Get the element with a callback.
        /// </summary>
        public void Get(IAction <Element> onGet)
        {
            _lock.Take();

            _callbacks.Add(onGet);

            if (_processing)
            {
                _lock.Release();
                return;
            }

            _processing = true;

            // check if the link has or should be updated
            if (_element == null)
            {
                if (_path == null)
                {
                    ManagerUpdate.Control.AddSingle(OnRetrieved, _getElement.Run());
                }
                else
                {
                    new ElementParser(_path, Act.New(OnParsed, (ElementParser)null)).Run();
                }
            }
            else
            {
                if (_nextUpdate < Time.Milliseconds)
                {
                    if (_path == null)
                    {
                        ManagerUpdate.Control.AddSingle(OnRetrieved, _getElement.Run());
                    }
                    else
                    {
                        new ElementParser(_path, Act.New(OnParsed, (ElementParser)null)).Run();
                    }
                }
                else
                {
                    _processing = false;
                    foreach (var callback in _callbacks)
                    {
                        callback.ArgA = _element.Clone();
                        ManagerUpdate.Control.AddSingle(callback);
                    }
                    _callbacks.Clear();
                }
            }

            _lock.Release();
        }
예제 #2
0
파일: Crawler.cs 프로젝트: flippynips/EFZ
        /// <summary>
        /// Start processing the specified address. This stops current processing.
        /// </summary>
        protected void PreConnect()
        {
            // stop timeout
            _timer.Run = false;

            // if the next url hasn't been set - get one from the crawler
            Url next = _urls.Dequeue() ? _urls.Current : _session.UrlControl.NextUrl;

            // was the next url retrieved?
            if (next == Url.Empty)
            {
                // set the timer to move to the next url on timeout
                _timer.OnDone = _stopped;
                _timer.Reset(500);

                return;
            }

            // does the next host match the last?
            if (next.Host != Url.Host)
            {
                // no, get a new identity
                Identity = GetIdentity.Run();
            }

            // set the next url
            Url = next;

            // ensure the url is valid
            Uri uri;

            if (!Uri.TryCreate(Url.ToString(), UriKind.Absolute, out uri))
            {
                // start again
                _stopped.Run();
                return;
            }

            // connecting
            Connecting = true;

            // create a web request
            _request = System.Net.WebRequest.CreateHttp(Url.ToString());
            _request.AuthenticationLevel = System.Net.Security.AuthenticationLevel.None;
            // ensure no caching
            _request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore);

            // set the request cookies
            _request.CookieContainer = Cookies;

            // set the request identity
            Identity.UpdateRequest(_request);

            // reset timeout
            _request.Timeout = Randomize.Range(_session.CrawlerMinConnectTimeout, _session.CrawlerMaxConnectTimeout);
            // start the timer
            _time.Start();

            // connect to the url
            _connect.Run();
        }
예제 #3
0
파일: Table.cs 프로젝트: flippynips/EFZ
        //-------------------------------------------//

        /// <summary>
        /// Construct a new Table instance.
        /// </summary>
        protected Table(string keyspaceName = null)
        {
            _keyspaceName = keyspaceName;
            _initialize   = true;
            _lock         = new Lock();

            _currentCollections = new ArrayRig <RowCollection>();

            // if the name wasn't set
            Name = Name == null?this.GetType().Name.ToLowercase() : Name.ToLowercase();

            // populate the columns
            Columns = new ArrayRig <Column>();

            // iterate properties
            foreach (PropertyInfo info in this.GetType().GetProperties(
                         BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static))
            {
                if (info.PropertyType.BaseType == typeof(Column))
                {
                    // get the column attribute
                    ColumnAttribute attribute = info.GetCustomAttribute <ColumnAttribute>(true);

                    // if the attribute isn't found - assume default values
                    if (attribute == null)
                    {
                        attribute = new ColumnAttribute(Column.ColumnClass.Data, info.Name);
                    }

                    // get the column value if it's been set
                    Column column = (Column)info.GetValue(this);

                    // if the column hasn't been assigned
                    if (column == null)
                    {
                        // create a new instance of the column
                        IFunc <ColumnAttribute, Column> activator = Dynamic.Constructor <IFunc <ColumnAttribute, Column> >(info.PropertyType);
                        if (attribute.Name == null)
                        {
                            attribute.Name = info.Name;
                        }
                        activator.ArgA = attribute;
                        column         = activator.Run();
                        info.SetValue(this, column);
                    }

                    // add the column to the collection
                    Columns.Add(column);
                }
            }

            // iterate fields
            foreach (FieldInfo info in this.GetType().GetFields(
                         BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static))
            {
                if (info.FieldType.BaseType == typeof(Column))
                {
                    // get the column attribute
                    ColumnAttribute attribute = info.GetCustomAttribute <ColumnAttribute>(true);

                    // if the attribute isn't found - assume default values
                    if (attribute == null)
                    {
                        attribute = new ColumnAttribute(Column.ColumnClass.Data, info.Name);
                    }

                    // get the column value if it's been set
                    Column column = (Column)info.GetValue(this);

                    // if the column hasn't been assigned
                    if (column == null)
                    {
                        // create a new instance of the column
                        IFunc <ColumnAttribute, Column> activator = Dynamic.Constructor <FuncSet <ColumnAttribute, Column> >(info.FieldType);
                        if (attribute.Name == null)
                        {
                            attribute.Name = info.Name;
                        }
                        activator.ArgA = attribute;
                        column         = activator.Run();
                        info.SetValue(this, column);
                    }

                    // add the column to the collection
                    Columns.Add(column);
                }
            }
        }