protected override void executeBegin(webLoaderSettings settings) { webClient = new WebClient(); switch (action) { case webRequestActionType.Download: webClient.DownloadFileAsync(urlObject, localPath, this); break; case webRequestActionType.FTPUpload: case webRequestActionType.FTPDownload: logSystem.log("Not implemented :: " + this.GetType().Name + " :: ", logType.FatalError); break; case webRequestActionType.localFile: if (File.Exists(url)) { fileStream = File.OpenRead(url); fileStream.BeginRead(result.loadedBytes, 0, result.loadedBytes.Length, getResponse, this); } break; default: logSystem.log("Action :: " + action + " not supported by " + GetType().Name, logType.ExecutionError); callExecutionError(webRequestEventType.error); break; } }
/// <summary> /// Pokreze izvrsavanje bloka /// </summary> /// <param name="settings">Podesavanja webLoader-a</param> /// <param name="__syncMode">mod sinhronizacije</param> /// <returns>Referencu prema webResultBlock-u</returns> public webResultBlock executeBlock(webLoaderSettings settings, executionSyncMode __syncMode, webRequestBlockEvent __onFinished = null, webRequestBlockEvent __onAborted = null, webRequestBlockEvent __onProgress = null) { __settings = settings; deploySettings(settings, __onFinished, __onAborted, __onProgress); if (__syncMode != executionSyncMode.unknown) { executeSyncMode = __syncMode; } output.Clear(); if (executeSyncMode == executionSyncMode.synced) { startExecution(settings); } else { executeThread = new Thread(startExecution); executeThread.Start(settings); } return(output); }
protected webResult _executeRequest(webLoaderSettings settings, executionSyncMode __syncMode, webRequestEvent __onExecutedOk, webRequestEvent __onError, webRequestEvent __onRetry) { deploySettings(settings, __onExecutedOk, __onError, __onRetry); if (settings.doUseCache && webCacheSystem.hasCache(url)) { status = webRequestEventType.executing; Thread lc = new Thread(loadCache); lc.Start(); syncWaitingLoop(settings, __syncMode); if (status == webRequestEventType.executedOk) { return(result); } else { } lc.Abort(); } status = webRequestEventType.executing; executeBegin(settings); syncWaitingLoop(settings, __syncMode); if (doLogNewLoad) { aceLog.log("Load: " + url + " (" + status.toString() + ")"); } return(result); }
protected void startWebRequest(Object obj) { List <Object> input = obj as List <Object>; webLoaderSettings settings = input[0] as webLoaderSettings; webRequest wR = input[1] as webRequest; wR.executeRequest(settings, executionSyncMode.asynced, onItemUpdate, onItemUpdate, onItemUpdate); }
public void deploySettings(webLoaderSettings settings, webRequestBlockEvent __onFinished = null, webRequestBlockEvent __onAborted = null, webRequestBlockEvent __onProgress = null) { executeMode = settings.blockExecuteMode; onBlockFinishedEvent_addHandler(__onFinished); onBlockAbortedEvent_addHandler(__onAborted); onExecutionProgress_addHandler(__onProgress); }
/// <summary> /// Izvrsavanje /// </summary> /// <param name="settings">Podesavanja loadera</param> /// <param name="__syncMode">Tip izvrsavanja</param> /// <param name="__onExecutedOk">kada je executedOk</param> /// <param name="__onError">za sve vrste gresaka</param> /// <param name="__onRetry">kada se desi retry</param> public override webResult executeRequest(webLoaderSettings settings, executionSyncMode __syncMode, webRequestEvent __onExecutedOk, webRequestEvent __onError, webRequestEvent __onRetry) { url = imbUrlOps.getStandardizedUrl(url, urlShema.http); settings.url = url; return(_executeRequest(settings, __syncMode, __onExecutedOk, __onError, __onRetry)); }
/// <summary> /// /// </summary> /// <param name="settings"></param> /// <param name="__syncMode"></param> protected void monitoringLoop(webLoaderSettings settings) { while (isWorking) { Thread.Sleep(tickDelay); } callBlockFinishedEvent(); }
/// <summary> /// Addhock request execution Executes the request /// </summary> /// <param name="settings">The settings.</param> /// <returns></returns> public virtual webResult executeRequest(webLoaderSettings settings = null) { if (settings == null) { settings = new webLoaderSettings(); } settings.url = imbUrlOps.getStandardizedUrl(url, urlShema.http); return(_executeRequest(settings, executionSyncMode.synced, null, null, null)); }
protected override void executeBegin(webLoaderSettings settings) { switch (action) { case webRequestActionType.CheckUrlOnly: case webRequestActionType.ipResolve: Dns.BeginGetHostEntry(url, getResponse, this); break; case webRequestActionType.whoIs: if (urlObject != null) { String tLD = urlObject.guessTopLevelDomain(); imbTopLevelDomain tmpTLD = null; if (imbDomainManager.DomainDictionary.ContainsKey(tLD)) { String serverUrl; if (String.IsNullOrEmpty(customServer)) { tmpTLD = imbDomainManager.DomainDictionary[tLD]; imbWhoIsServer usedServer = tmpTLD.shuffledServer; usedServer.callCount++; serverUrl = usedServer.url; } else { serverUrl = customServer; } tcpClient = new TcpClient(); tcpClient.BeginConnect(serverUrl, 43, getResponse, this); } else { callExecutionError(webRequestEventType.error, "TLD not supported :: TLD [" + tmpTLD.domainName + "]"); } } else { callExecutionError(webRequestEventType.error, "URL (uri) object is null :: URL [" + url + "]"); } break; default: logSystem.log("Action :: " + action + " not supported by " + GetType().Name, logType.ExecutionError); callExecutionError(webRequestEventType.error); break; } }
protected override void executeEnd(webLoaderSettings settings) { //switch (action) //{ // case webRequestActionType.CheckUrlOnly: // break; // default: // break; //} //result httpRequest.ContentLength; callExecutionDone(); }
protected void retry(webLoaderSettings settings) { timeCounter.reset(); contentCriteriaCheckDelayCounter.reset(); contentCriteriaTryCounter.reset(); loadCooloffCounter.reset(); status = webRequestEventType.executing; callRetryCalled(); result = null; executeBegin(settings); }
/// <summary> /// Prekida izvrsavanje /// </summary> /// <returns></returns> public webResultBlock abortExecution(webLoaderSettings settings) { executeThread.Abort(); foreach (webRequest wR in this) { if (wR.isActive) { wR.abortRequest(settings); } } callBlockAbortedEvent(); return(output); }
protected override void executeBegin(webLoaderSettings settings) { switch (action) { case webRequestActionType.openUrl: case webRequestActionType.GetHeaders: case webRequestActionType.Text: case webRequestActionType.XML: case webRequestActionType.HTMLasXML: break; default: logSystem.log("Action :: " + action + " not supported by " + GetType().Name, logType.ExecutionError); callExecutionError(webRequestEventType.error); break; } }
/// <summary> /// Trenutni rezultat requesta /// </summary> protected void syncWaitingLoop(webLoaderSettings settings, executionSyncMode __syncMode) { DateTime start = DateTime.Now; if (__syncMode == executionSyncMode.synced) { while (isActive) { Thread.Sleep(settings.tickDuration); TimeSpan ts = DateTime.Now.Subtract(start); if (ts.TotalMilliseconds > settings.timeout) { aceLog.saveAllLogs(true); abortRequest(settings); } checkRequest(settings); } } }
/// <summary> /// Primenjuje podesavanja i postavlja event handlere /// </summary> /// <param name="settings">Podesavanja koja treba da primenu</param> /// <param name="__onExecutedOk">Poziva se nakon finalizacije</param> /// <param name="__onError">Poziva se za timeout ili neku drugu gresku</param> /// <param name="__onRetry">Poziva se ako dodje do ponavljanja requesta</param> public void deploySettings(webLoaderSettings __settings, webRequestEvent __onExecutedOk, webRequestEvent __onError, webRequestEvent __onRetry) { status = webRequestEventType.scheduled; settings = __settings; result = null; // if (settings.action != webRequestActionType.None) action = settings.action; // if (settings.requestType != webRequestType.unknown) requestType = settings.requestType; // imbTypologyExtensions.setObjectBySource(this, settings, new string[]{"url", "action", "requestType"}); //timeCounter. //retryWaitCounter.setup(settings.retryWaitDefault, true); //retryCounter.setup(settings.retryLimitDefault, true); //loadCooloffCounter.setup(settings.loadCoolOffDefault, true); //contentCriteriaCheckDelayCounter.setup(settings.contentCriteriaCheckDelayDefault, true); onRetryCalled_addHandler(__onRetry); onExecutionTimeout_addHandler(__onError); onExecutionError_addHandler(__onError); onExecutionDone_addHandler(__onExecutedOk); if (String.IsNullOrEmpty(url)) { logSystem.log("Url is null/empty!!!", logType.Notification); } else { Uri urlObj = null; if (Uri.TryCreate(url, UriKind.Absolute, out urlObj)) { urlObject = urlObj; } else { } } }
protected void startExecution(Object obj) { webLoaderSettings settings = obj as webLoaderSettings; switch (executeMode) { case webRequestBlockExecuteMode.parallelSingleThread: foreach (webRequest wR in this) { wR.executeRequest(settings, executionSyncMode.asynced, onItemUpdate, onItemUpdate, onItemUpdate); } break; case webRequestBlockExecuteMode.parallelThreads: foreach (webRequest wR in this) { Thread newThread = new Thread(startWebRequest); List <Object> input = new List <object>(); input.Add(settings); input.Add(wR); newThread.Start(input); } break; case webRequestBlockExecuteMode.oneByOne: foreach (webRequest wR in this) { wR.executeRequest(settings, executionSyncMode.synced, onItemUpdate, onItemUpdate, onItemUpdate); } break; } if (executeSyncMode == executionSyncMode.synced) { monitoringLoop(settings); } }
protected override void executeBegin(webLoaderSettings settings) { if (urlObject == null) { logSystem.log( "Url object is null [" + action.ToString() + "] - aborting webRequest [" + settings.requestType.ToString() + "]", logType.ExecutionError); getResponse(null); return; } httpRequest = WebRequest.Create(urlObject); httpRequest.Timeout = settings.timeout; if (proxyToUse != null) { httpRequest.Proxy = proxyToUse; } if (method == httpRequestMethod.unknown) { httpRequest.Method = "GET"; } else { httpRequest.Method = method.ToString(); } switch (action) { case webRequestActionType.openUrl: break; case webRequestActionType.GetHeaders: httpRequest.Method = "HEAD"; break; case webRequestActionType.Text: //webClient = new WebClient(); //webClient.OpenReadAsync(urlObject, this); //break; case webRequestActionType.XML: case webRequestActionType.HTMLasXML: break; default: logSystem.log("Action :: " + action + " not supported by " + GetType().Name, logType.ExecutionError); callExecutionError(webRequestEventType.error); return; break; } switch (action) { //case webRequestActionType.Text: // webClient = new WebClient(); // webClient.OpenReadAsync(urlObject, this); // break; default: httpRequest.BeginGetResponse(getResponse, this); break; } }
/// <summary> /// Abort the current request /// </summary> /// <param name="settings"></param> public void abortRequest(webLoaderSettings settings) { cancelAllActivities(); deploySettings(settings, null, null, null); }
/// <summary> /// Calls all counters and check the status of request /// </summary> /// <param name="settings"></param> public void checkRequest(webLoaderSettings settings) { if (status == webRequestEventType.scheduled) { return; } #region isEXECUTED & Retry if (isExecutedOrFailed) { if (retryCounter.isRunning) { Boolean tryRetry = (status == webRequestEventType.error); if (settings.doRetryOnContentError) { if (status == webRequestEventType.errorContent) { tryRetry = true; } } if (settings.doRetryOnTimeoutError) { if (status == webRequestEventType.errorTimeout) { tryRetry = true; } } if (tryRetry) { if (!retryCounter.check(1, true)) { // moze jos jedan retry retryWaitCounter.reset(); status = webRequestEventType.waitingForRetry; cancelAllActivities(); } else { return; } } } else { return; } } // ako ceka retry uraditi retryWaitcheck if (status == webRequestEventType.waitingForRetry) { if (retryWaitCounter.check(1, true)) { retry(settings); } return; } #endregion if (status == webRequestEventType.loaded) { if (loadCooloffCounter.isRunning) { status = webRequestEventType.waitingCooloff; } else { checkContentCriteria(); } } if (status == webRequestEventType.waitingCooloff) { if (loadCooloffCounter.check(1, true)) { checkContentCriteria(); } } if (status == webRequestEventType.waitingContentCriteria) { if (contentCriteriaCheckDelayCounter.check(1, true)) { if (contentCriteriaTryCounter.check(1, false)) { status = webRequestEventType.errorContent; } else { if (result.document.checkCriteria(criteriaValue, criteriaType)) { status = webRequestEventType.finalization; } else { // nije ispunjen uslov } } } } if (status == webRequestEventType.finalization) { executeEnd(settings); return; } /// Provera vremena if (timeCounter.check(1, false)) { status = webRequestEventType.errorTimeout; callExecutionTimeout(); } }
protected override void executeEnd(webLoaderSettings settings) { status = webRequestEventType.executedOk; }
public abstract webResult executeRequest(webLoaderSettings settings, executionSyncMode __syncMode, webRequestEvent __onExecutedOk, webRequestEvent __onError, webRequestEvent __onRetry);
/// <summary> /// Izvrsavanje /// </summary> /// <param name="settings">Podesavanja loadera</param> /// <param name="__syncMode">Tip izvrsavanja</param> /// <param name="__onExecutedOk">kada je executedOk</param> /// <param name="__onError">za sve vrste gresaka</param> /// <param name="__onRetry">kada se desi retry</param> public override webResult executeRequest(webLoaderSettings settings, executionSyncMode __syncMode, webRequestEvent __onExecutedOk, webRequestEvent __onError, webRequestEvent __onRetry) { return(_executeRequest(settings, __syncMode, __onExecutedOk, __onError, __onRetry)); }
/// <summary> /// Pocinje izvrsavanje requesta /// </summary> /// <param name="settings"></param> protected abstract void executeBegin(webLoaderSettings settings);
/// <summary> /// zavrsava izvrsavanje requesta /// </summary> /// <param name="settings"></param> protected abstract void executeEnd(webLoaderSettings settings);
public webRequestContext(webLoaderSettings __settings) { settings = __settings; }
protected override void executeEnd(webLoaderSettings settings) { callExecutionDone(); }
protected override void executeEnd(webLoaderSettings settings) { logSystem.log("Not implemented :: " + this.GetType().Name + " :: ", logType.FatalError); }