/// <summary>
/// Sets the failed domain.
/// </summary>
/// <param name="wProfile">The w profile.</param>
public void SetFailedDomain(webSiteProfile wProfile, modelSpiderSiteRecord wRecord = null)
{
if ((wRecord == null) && (wProfile == null))
{
var axe = new aceGeneralException("Supplied wProfile and wRecord are null - this task should never run on the first place", null, this, "SetFailedDomain(webSiteProfile null)");
throw axe;
}
else
{
string domain = "[unknown]";
if (wProfile != null)
{
domain = wProfile.domain;
}
if (wRecord != null)
{
domain = wRecord.domain;
}
if (!failedSample.ContainsKey(domain))
{
failedSample.Add(domain, wProfile);
domainFailList.Append(domain, true);
}
}
}
public bool CheckStage(modelSpiderSiteRecord wRecord, spiderObjectiveSolutionSet oSet, spiderTask task)
{
bool okToLeave = false;
if (task.Count() == 0)
{
wRecord.logBuilder.log("> Spider task [i:" + task.iteration + "] have no tasks defined. Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------- OBJECTIVE SOLUTION SET
okToLeave = operation_executeObjectiveSolutionSet(oSet, wRecord);
if (okToLeave)
{
return(okToLeave);
}
// <----------------------------- SPIDER LIMITS OVERRIDERS ---------------|
if (stageIteration > wRecord.tRecord.instance.settings.limitIterations)
{
wRecord.log("> Spider settings (limit iterations) trigered abort at [" + stageIteration + "] Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------------------------------------------------|
// <----------------------------- SPIDER LIMITS OVERRIDERS ---------------|
if (wRecord.web.webPages.Count() > wRecord.tRecord.instance.settings.limitTotalPageLoad)
{
wRecord.log("> Spider settings (limit pages load) trigered abort at [" + wRecord.web.webPages.Count() + "] Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------------------------------------------------|
if (stageIteration > stageIterationLimit)
{
wRecord.logBuilder.log("> Stage [" + name + "] iteration limit reached [ " + stageIterationLimit + " ] -- aborting [" + objectives.Count + "] objectives and move on");
okToLeave = true;
return(okToLeave);
}
if (stageIteration > GLOBAL_stageIterationLimit)
{
Exception ex = new aceGeneralException("spiderStage [" + name + "] reached the " + nameof(GLOBAL_stageIterationLimit) + "(" + GLOBAL_stageIterationLimit.ToString() + ")");
throw ex;
}
stageIteration++;
return(okToLeave);
}
/// <summary>
/// Vraca kursor na njegovu glavnu zonu
/// </summary>
public void switchToMainZone()
{
if (mainZone == textCursorZone.unknownZone)
{
Exception ex =
new aceGeneralException(
"Main zone je ostao nepoznat - nikada nije doslo do dodeljivanja nove vrednosti currentZone propertiju!");
throw ex;
}
currentZone = mainZone;
}
protected void deploy(Exception __axe, crawlerErrorEnum __errorType, string __specificMessage, object __relevantInstance, IModelRecord __relevantRecord, crawlerDomainTask __task = null, int stacks = 1)
{
specificMessage = __specificMessage;
relevantRecord = __relevantRecord;
if (relevantRecord == null)
{
if (__relevantInstance is IModelRecord)
{
relevantRecord = (IModelRecord)__relevantInstance;
}
}
if (__errorType == crawlerErrorEnum.none)
{
__errorType = crawlerErrorEnum.exceptionError;
}
if (__axe != null)
{
if (__axe is aceGeneralException)
{
axe = (aceGeneralException)__axe;
axe.SetLogSerializable(this);
}
else if (__axe is Exception)
{
if (__axe.Message.Contains("thread"))
{
isThreadCancelError = true;
}
axe = new aceGeneralException(__axe.Message, __axe, __relevantInstance, "Crawler error: " + __errorType.ToString(), stacks + 2);
axe.SetLogSerializable(this);
}
else
{
axe = null;
}
}
if (__relevantRecord != null)
{
relevantRecord = relevantRecord;
// relevantRecord.SetLogSerializable(this);
}
if (__task != null)
{
task = __task;
__task.SetLogSerializable(this);
__errorType |= crawlerErrorEnum.domainTaskError;
}
}
protected bool basicKnownWordTest(string input)
{
if (hunspellEngine == null)
{
var ex = new aceGeneralException("hunspellEngine not ready at basic language object", null, this, "Hunspell:" + languageEnglishName + " not ready");
throw ex;
//devNoteManager.note(this, ex, isb.ToString(), "basicLanguage", devNoteType.nlp);
return(false);
}
return(hunspellEngine.Spell(input));
}
protected bool basicKnownWordTest(string input)
{
checkHuspell();
if (hunspellEngine == null)
{
var ex = new aceGeneralException("hunspellEngine not ready at basic language object", null, this, "Hunspell:" + languageEnglishName + " not ready");
throw ex;
return(false);
}
return(hunspellEngine.Spell(input));
}
public aceSubEnum(IEnumerable __source, String __path)
{
try
{
source = __source;
path = __path;
Type sourceCollectionType = __source.GetType();
sourceType = sourceCollectionType.GetGenericArguments().FirstOrDefault();
targetPi = sourceType.getProperty(__path);
} catch (Exception ex)
{
var axe = new aceGeneralException("The source collection is not applicable", ex, this, "aceSubEnum()");
throw axe;
}
}
public override void onStartUp()
{
try
{
imbWEMManager.prepare(null);
} catch (Exception ex)
{
aceGeneralException axe = new aceGeneralException("Web Exploration Module failed to prepare on WBI console start-up: " + ex.Message, ex, this, "WEM prepare call failed: " + ex.Message);
throw axe;
}
base.onStartUp();
output.log("Plugin: " + itm.name);
output.log("Plugin: " + wem.name);
imbNLP.PartOfSpeech.nlpTools.nlpTypeManager.main.prepare();
//
// imbLanguageFrameworkManager.Prepare();
//manager.workspaceFolderPath = workspace.folder.path;
//log("Preparing Semantic Lexicon manager", true);
//manager.prepare();
//log("Preparing Semantic Lexicon cache", true);
//manager.prepareCache(output, workspace.folder);
//manager.constructor.startConstruction(workspace.folder[ACFolders.constructor].path.removeEndsWith("\\"));
// Program app = appManager.Application as Program;
// imbWEMManager.index.isFullTrustMode = imbWEMManager.settings.indexEngine.doIndexFullTrustMode;
//if (app.args.Any())
//{
// string line = app.args.toCsvInLine(" ");
// executeCommand(line);
//}
//else
//{
// var script = workspace.loadScript("autoexec.ace");
// executeScript(script);
//}
}
/// <summary>
/// Prepares this instance.
/// </summary>
public override void prepare()
{
if (isReady)
{
return;
}
imbLanguageFrameworkManager.log.log("Semantic lexicon manager prepare start");
constructionSettings.Poke();
settings.Poke();
if (constructor == null)
{
constructor = new lexiconConstructor(constructionSettings);
}
var missing = settings.sourceFiles.checkMissingFiles();
foreach (string mfile in missing)
{
imbLanguageFrameworkManager.log.log("Source file missing: " + mfile);
}
if (missing.Any())
{
aceGeneralException axe = new aceGeneralException("Lexicon source files missing", null, missing, "Lexicon source files missing");
throw axe;
}
consoleSettings.Poke();
console = new lexiconConsole(consoleSettings.defaultSession, consoleSettings);
semanticLexiconExtensions.SetBrightStarDB();
//String conString = "type=rest;storesdirectory=G:\\BrightStarDB\\;storename=lex";
string conString = @"type=rest;endpoint=http://localhost:8090/brightstar;storename=lex";
lexiconContext = new semanticLexiconContext(conString);
imbLanguageFrameworkManager.log.log("Semantic lexicon manager prepare finished");
}
/// <summary>
/// Invokes the command
/// </summary>
/// <returns></returns>
public Boolean invoke()
{
if (hasComment)
{
console.output.AppendLine("Comment: " + commentLine);
return(true);
}
if (!hasCommand)
{
errorMessage = "Command line has no command and no comment recognized";
return(false);
}
Object[] array = carg.getInvokeArray();
if (marg.method != null)
{
try
{
marg.method.Invoke(marg.executor, array);
return(true);
} catch (Exception ex)
{
axe = new aceGeneralException("Command " + command + " execution error", ex, console, console.consoleTitle + " execution of " + command + " error");
errorMessage = axe.Message;
return(false);
}
}
else
{
errorMessage = "Method not assigned";
return(false);
}
return(false);
}
/// <summary>
/// Appends the exception.
/// </summary>
/// <param name="title">The title.</param>
/// <param name="ex">The ex.</param>
/// <param name="exceptionLevel">The exception level.</param>
public void AppendException(String title, Exception ex, Int32 exceptionLevel = 0)
{
lock (AppendExceptionLock)
{
throw new aceGeneralException("Exception: " + ex.Message, ex, this, title);
open("Exception", title);
if (ex == null)
{
AppendLine("Exception: [null]");
return;
}
ex.describe(this, "");
callerInfo ci = null;
StackFrame[] frames = null;
if (ex is aceGeneralException)
{
aceGeneralException axe = (aceGeneralException)ex;
ci = axe.callInfo;
}
else
{
StackTrace st = new StackTrace(ex, true);
StackFrame sf = st.GetFrame(0);
if (sf == null)
{
aceGeneralException axe = new aceGeneralException("WrapperException for " + ex.GetType().Name, ex, this, title);
ci = axe.callInfo;
}
else
{
ci = callerInfo.getCallerInfo(sf, true);
}
frames = st.GetFrames();
}
if (exceptionLevel == 0)
{
AppendTable(ci.AppendDataFields());
}
AppendTable(ci.AppendDataFieldsOfMethod());
open("parents", "Parent StackFrames");
List <String> sfi_strings = new List <string>();
Int32 c = 0;
if (frames != null)
{
foreach (StackFrame sfi in frames)
{
if (c != 0)
{
open("frame", "StackFrame(" + c.ToString() + ")");
callerInfo sfi_ci = callerInfo.getCallerInfo(sfi, true);
AppendTable(sfi_ci.AppendDataFieldsOfMethod());
close();
}
c++;
//sfi.GetFileLineNumb
}
}
close();
if (ex.InnerException != null)
{
AppendException("InnerException [level:" + exceptionLevel + "]", ex.InnerException, exceptionLevel + 1);
}
close();
}
}
/// <summary>
/// vrsi osnovnu klasifikaciju tokena
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="newToken"></param>
internal static void setTokenCaseAndGeneral <T>(T newToken) where T : IContentToken, new()
{
if (newToken == null)
{
var isb = new StringBuilder();
isb.AppendLine("setTokenCaseAndGeneral error");
isb.AppendLine("Received newToken is : " + newToken.toStringSafe());
aceGeneralException ex = new aceGeneralException("newToken is null in setTokenCaseAndGeneral");
throw ex;
//devNoteManager.note(ex, isb.ToString(), "setTokenCaseAndGeneral error", devNoteType.tokenization);
return;
}
// logSystem.log(" -- -- -- setTokenCaseAndGeneral: " + newToken.content, logType.Notification);
/*
* if (String.IsNullOrEmpty(newToken.content))
* {
* newToken.flags.Add(contentTokenFlag.empty);
* return;
* }
* if (newToken.content.isNumber())
* {
* newToken.flags.Add(contentTokenFlag.number);
* newToken.flags.Add(contentTokenFlag.numberDigital);
* if (newToken.content.isOrdinalNumber())
* {
* newToken.flags.Add(contentTokenFlag.numberOrdinal);
* } else if (newToken.content.isFormatedNumber())
* {
* newToken.flags.Add(contentTokenFlag.numberFormated);
* } else if (newToken.content.isDecimalNumber())
* {
* newToken.flags.Add(contentTokenFlag.numberDecimal);
* }
* } else
* {
* if (_select_caseCamel.IsMatch(newToken.content))
* {
* newToken.flags.Add(contentTokenFlag.caseFirstUpper);
* newToken.flags.Add(contentTokenFlag.languageWord);
* }
* else if (_select_caseUpper.IsMatch(newToken.content))
* {
* newToken.flags.Add(contentTokenFlag.caseAllUpper);
* newToken.flags.Add(contentTokenFlag.languageWord);
* }
* else if (_select_caseLower.IsMatch(newToken.content))
* {
* newToken.flags.Add(contentTokenFlag.caseLower);
* newToken.flags.Add(contentTokenFlag.languageWord);
* }
* else
* {
* newToken.flags.Add(contentTokenFlag.caseIrregular);
* newToken.flags.Add(contentTokenFlag.languageWord);
* }
* }
*/
// logSystem.log(" -- -- -- setTokenCaseAndGeneral done: ", logType.Notification);
}
/// <summary>
/// 2013C: Ovo je bitno da bude pozvano kako bi uspesno referencirao ovu Biblioteku!! -- > TREBA DA GA POZOVE manager.onApplicationReady()
/// </summary>
public static void Prepare()
{
aceLog.consoleControl.setAsOutput(log, "lang_mng");
//basicLanguageCollection.isReadyGlobal = true;
// Int32 ln = languages.loadItems(false, 5);
// log.log("Hunspell language definitions loaded [" + ln + "]");
if (_serbian == null)
{
_serbian = new extendedLanguage();
// _serbian.morphologies_verbs.buildDefaultItem();
// _serbian.morphologies_nouns.buildDefaultItem();
if (!File.Exists(PATH_hunspell_aff))
{
throw new aceGeneralException("Hunspell AFF file missing!");
}
if (!File.Exists(PATH_hunspell_dict))
{
throw new aceGeneralException("Hunspell DICT file missing!");
}
basicLanguage language = new basicLanguage();
language.affixFilePath = PATH_hunspell_aff;
language.dictFilePath = PATH_hunspell_dict;
language.languageNativeName = "Srpski";
language.languageEnglishName = "Serbian";
language.iso2Code = "sr";
_serbian.basic = language;
if (language.checkHuspell(true))
{
log.log("Hunspell language module: " + language.languageEnglishName + " ready");
}
else
{
aceGeneralException axe = new aceGeneralException("Serbian language Hunspell module failed", null, english, "Serbian Hunspell failed");
throw axe;
}
_serbian.basic = language;
_serbian.loadAlfabet("extended\\alfabet.txt");
}
if (_english == null)
{
_english = new extendedLanguage
{
basic = new basicLanguage("en")
};
_english.basic.affixFilePath = PATH_hunspell_aff_en;
_english.basic.dictFilePath = PATH_hunspell_dict_en;
_english.basic.languageNativeName = "Engleski";
_english.basic.languageEnglishName = "English";
_english.basic.iso2Code = "en";
if (_english.basic.checkHuspell(true))
{
log.log("Hunspell language module: " + _english.basic.languageEnglishName + " ready");
}
else
{
aceGeneralException axe = new aceGeneralException("English language Hunspell module failed", null, english, "English Hunspell failed");
throw axe;
}
}
//languages.loadItems(false, -1, 0, System.Data.LoadOption.OverwriteChanges);
//dictionaryManager.prepare();
//elementsManager.prepare();
languageManagerApertium.manager.prepare();
languageManagerAlphabet.manager.prepare();
// languageManagerDictionary.manager.prepare();
// languageManagerElements.manager.prepare();
languageManagerHunspell.manager.prepare();
// languageManagerLexicon.manager.prepare();
// languageManagerMorphology.manager.prepare();
languageManagerUnitex.manager.prepare();
languageManagerWordnet.manager.prepare();
// languageManagerDict.manager.prepare();
languageManagerDBNamedEntities.manager.prepare();
semanticLexicon.semanticLexiconManager.manager.prepare();
if (serbian.basic.hunspellEngine.Spell("Proba"))
{
log.log("Hunspell language module: " + serbian.basic.languageEnglishName + " Spell check working");
}
if (english.basic.hunspellEngine.Spell("Test"))
{
log.log("Hunspell language module: " + english.basic.languageEnglishName + " Spell check working");
}
aceLog.consoleControl.removeFromOutput(log);
}
/// <summary>
///
/// </summary>
/// <param name="sf"></param>
/// <param name="getCode">Kompleksnija operacija - daje vise informacija</param>
/// <returns></returns>
public static callerInfo getCallerInfo(StackFrame sf, Boolean getCode = true)
{
callerInfo output = new callerInfo();
if (sf == null)
{
Exception ex = new aceGeneralException("Exception had no *StackFrame* --- don-t use explicit throwing after the Exception was caught");
throw ex;
}
else
{
}
MethodBase mb = sf.GetMethod();
//MethodBodyReader.GetInstructions(output.methodBase);
output.methodBase = mb;
output.methodBody = mb.GetMethodBody();
output.parameters = mb.GetParameters();
output.methodInfo = output.methodBase as MethodInfo;
output.callerType = mb.DeclaringType;
output.callerTypeInfo = output.callerType;
output.methodName = output.callerType.FullName + "." + mb.Name + "()";
if (output.callerTypeInfo == null)
{
output.className = "callerTypeInfo is null";
}
else
{
output.className = output.callerType.FullName; // + "." + mb.Name + "()";
}
output.line = sf.GetFileLineNumber();
output.column = sf.GetFileColumnNumber();
output.Filepath = sf.GetFileName();
if (output.methodBody != null)
{
output.localVariables = output.methodBody.LocalVariables.ToList();
}
if (getCode)
{
if (output.callerType != null)
{
output.methodInfo = output.callerType.GetMethod(mb.Name, new[] { output.callerType }); // output.callerTypeInfo.findMethod(mb); //.Name, output.parameters);
if (!String.IsNullOrEmpty(output.Filepath))
{
output.sourceCodeFile = output.Filepath.openFileToList(false, Encoding.UTF8);// open.openFile(output.filepath);
output.sourceCodeLine = output.sourceCodeFile[output.line - 1];
output.sourceCodeInvokePart = output.sourceCodeLine.Substring(output.column);
String ln = methodParameters.Match(output.sourceCodeInvokePart).Value;
output.sourceCodeInvokeParamsPart = ln;
var propCalls = methodParametersEach.Matches(ln);
output.sourceCodeParameterCalls = new List <String>();
foreach (Match m in propCalls)
{
output.sourceCodeParameterCalls.Add(m.Value.Trim("()".ToCharArray()));
}
//if (output.sourceCodeParameterCalls.Any()) output.sourceCodeParameterCalls.RemoveAt(0);
}
else
{
output.sourceCodeLine = " { unknown file location }";
}
}
}
if (output.callerType != null)
{
output.label = String.Format(labelFormat, output.callerType.FullName, output.methodName,
output.sourceCodeInvokeParamsPart);
}
else
{
output.label = "Output caller type info is null";
}
output.sufix = String.Format(sufixFormat, output.Filepath, output.line, output.column);
return(output);
}
/// <summary>
/// Ending procedure
/// </summary>
/// <exception cref="aceGeneralException">Reporter.reportDomainFinished() exception</exception>
public void Closing()
{
if (closeCalled)
{
return;
}
closeCalled = true;
// wRecord.children.FinishAllStarted();
wRecord.recordFinish();
//if (imbWEMManager.settings.TFIDF.doExploitStandardCC && wRecord.tRecord.instance.settings.doEnableDLC_TFIDF)
//{
// imbWEMManager.index.domainIndexTable.SetSiteTFCompiled(wRecord.context.targets.dlTargetPageTokens, wRecord.domain);
//}
if (imbWEMManager.settings.indexEngine.doIndexUpdateOnDLC)
{
imbWEMManager.index.deployWRecord(wRecord);
}
if (reporter != null)
{
try
{
reporter.reportDomainFinished(wRecord);
}
catch (Exception ex)
{
var axe = new aceGeneralException(ex.Message, ex, reporter, "Reporter.reportDomainFinished() exception");
var clog = reporter.CreateAndSaveError(axe, wRecord, this, crawlerErrorEnum.DReportError);
wRecord.log(clog.Message);
throw axe;
}
}
parent.parent.plugins.eventDLCFinished(parent.parent, this, wRecord);
evaluator.plugins.eventDLCFinished(evaluator as spiderEvaluatorBase, this, wRecord);
imbWEMManager.index.plugins.eventDLCFinished(imbWEMManager.index.experimentEntry, this, wRecord);
parent.parent.reportPlugins.eventDLCFinished(evaluator, this, wRecord);
if (isStageAborted)
{
status = crawlerDomainTaskStatusEnum.aborted;
}
else
{
status = crawlerDomainTaskStatusEnum.done;
}
if (imbWEMManager.settings.executionLog.doRemoveWRecordOnFinished)
{
// var tRecord = wRecord.tRecord;
// var wGeneralRecord = wRecord.wGeneralRecord;
tRecord.children.Remove(wRecord);
tRecord.tGeneralRecord.children.Remove(wGeneralRecord);
}
if (wRecord.iteration < 2)
{
wRecord.log("Domain [" + wRecord.domain + "] considered as failed since less then two iterations were made on it");
parent.parent.webLoaderControler.SetFailedDomain(wRecord.wProfile, wRecord);
}
iterationStatus = crawlerDomainTaskIterationPhase.iterationProcessFinished;
}
/// <summary>
/// Samples the accept and prepare -- central sample set operation
/// </summary>
/// <param name="filepath">The filepath.</param>
/// <param name="fileHasPriority">if set to <c>true</c> [file has priority].</param>
/// <param name="group_tags">The group tags.</param>
/// <param name="limit">The limit.</param>
/// <param name="skip">The skip.</param>
/// <param name="fromDomainIndex">Index of from domain.</param>
/// <param name="fromPageIndex">Index of from page.</param>
/// <param name="samplefilename">The samplefilename.</param>
public void sampleAcceptAndPrepare(string filepath = "", bool fileHasPriority = false, string group_tags = "", int limit = 0,
int skip = 0, indexDomainContentEnum fromDomainIndex = indexDomainContentEnum.none, indexPageEvaluationEntryState fromPageIndex = indexPageEvaluationEntryState.none,
string samplefilename = "")
{
List <string> domains = new List <string>();
List <string> pages = new List <string>();
sampleAcceptAndPrepareStates processState = sampleAcceptAndPrepareStates.started;
analyticConsoleState state = console.state as analyticConsoleState;
string sourcePath = filepath;
if (!filepath.isNullOrEmpty())
{
processState |= sampleAcceptAndPrepareStates.filepathArgumentSupplied;
sourcePath = folder.findFile(filepath, SearchOption.AllDirectories, false);
if (sourcePath.isNullOrEmpty())
{
processState |= sampleAcceptAndPrepareStates.sourcePathDiscovered;
}
else
{
processState |= sampleAcceptAndPrepareStates.sourcePathNOTFOUND;
}
}
if (state.aRecord == null)
{
console.log("You should define job before calling this command!", true);
return;
}
int startSampleCount = 0;
if (state.sampleList != null)
{
startSampleCount = state.sampleList.Count();
processState |= sampleAcceptAndPrepareStates.existingSampleDetected;
}
else
{
state.sampleList = new webSiteSimpleSample();
}
state.sampleTags = group_tags;
state.sampleFile = filepath;
console.response.log("Sample with group_tags=" + group_tags + ", samplename=" + filepath + ", fileHasPriority=" + fileHasPriority + ".");
// ==============================================================================================
if (!state.sampleList.Any())
{
if (fromPageIndex != indexPageEvaluationEntryState.none) // -------------------------- LOADING FROM THE PAGE INDEX
{
processState |= sampleAcceptAndPrepareStates.fromPageIndexImportCalled;
List <indexDomain> dSample = new List <indexDomain>();
var pageList = imbWEMManager.index.pageIndexTable.GetPagesAndDomains(fromPageIndex, out dSample);
if (imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleTake)
{
dSample.Randomize();
}
state.sampleList.Add(dSample);
}
}
// ==============================================================================================
if (!state.sampleList.Any())
{
if (fromDomainIndex != indexDomainContentEnum.none) // -------------------------- LOADING FROM THE DOMAIN INDEX
{
processState |= sampleAcceptAndPrepareStates.fromDomainIndexImportCalled;
var list = imbWEMManager.index.domainIndexTable.GetDomainUrls(fromDomainIndex);
if (imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleTake)
{
list.Randomize();
}
foreach (string str in list)
{
state.sampleList.Add(str);
}
}
}
if (!state.sampleList.Any()) // -------------------------- LOADING THE EXTERNAL SAMPLE FILE
{
if (!sourcePath.isNullOrEmpty())
{
processState |= sampleAcceptAndPrepareStates.filesourceFoundForImportFromFile;
var domainList = sourcePath.openFileToList(true);
if (imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleTake)
{
domainList.Randomize();
}
state.sampleList.Add(domainList, skip, limit);
console.response.log("Sample external file list [" + samplefilename + "] found at [" + sourcePath + "] containing [" + domainList.Count() + "] domains.");
}
else
{
processState |= sampleAcceptAndPrepareStates.filesourceNOTFOUND_ForImportFromFile;
}
}
if (!state.sampleList.Any()) // -------------- LOADING THE INTERNAL SAMPLE FILE
{
if (!filepath.isNullOrEmptyString())
{
if (fileHasPriority && sampleExist(filepath))
{
processState |= sampleAcceptAndPrepareStates.internalSampleFilesourceLoaded;
state.sampleList = loadSample(filepath, imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleTake);
}
else
{
if (state.sciProject != null) // -------------------------- LOADING FROM THE DATABASE
{
if (!group_tags.isNullOrEmpty())
{
processState |= sampleAcceptAndPrepareStates.groupTagsSpecified_databaseImportCalled;
// state.sampleList = state.sciProject.getSamples(group_tags.getTokens(), limit, "stamp", 0, imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleTake);
}
}
else
{
processState |= sampleAcceptAndPrepareStates.noSciProjectFound;
}
}
}
else
{
// sample = state.sampleList;
}
}
if (!state.sampleList.Any())
{
processState |= sampleAcceptAndPrepareStates.sampleListStillEmpty;
var ace = new aceGeneralException("Sample creation failed: " + processState.ToString(), null, this, "Sample import failed :: ");
throw ace;
return;
}
int AddedSampleCount = state.sampleList.Count - startSampleCount;
console.log("Added to the sample list [" + AddedSampleCount + "] at current job record [" + state.aRecord.job.name + "]", true);
// ==============================================================================================
if (!filepath.isNullOrEmpty()) // -------------------------- EXPORTING INTO LOCAL XML FILE
{
processState |= sampleAcceptAndPrepareStates.sampleListExported;
var fi = saveSample(filepath, state.sampleList);
console.output.log("Sample list exported to: " + fi.Name);
state.sampleFile = fi.Name;
}
console.output.AppendLine("--- loged sample import procedure states: [" + processState.ToString() + "]");
}
