/// <summary>
/// Add item to queue. It is possible to create a named queue.
/// It is the same like BotCycle.Add() but not so efficient.
/// </summary>
/// <param name="queue_name"></param>
/// <param name="item"></param>
static public bool Add(string queue_name, InputItem item)
{
if (queue_name == null)
{
queue_name = item.GetType().Name;
}
InputItemQueue iiq = Session.GetInputItemQueue(queue_name);
return(InputItem.Add2Queue(iiq, BotCycle.GetCurrentInputItemForThisThread(), item));
}
public static void Start()
{
Log.Main.Inform("Version compiled: " + Cliver.Bot.Program.GetCustomizationCompiledTime().ToString());
Log.Main.Inform("Command line parameters: " + string.Join("|", Environment.GetCommandLineArgs()));
if (This != null)
{
throw new Exception("Previous session was not closed.");
}
new Session();
if (This == null)
{
return;
}
BotCycle.Start();
}
public static void Start()
{
Log.Initialize(Log.Mode.SESSIONS, Cliver.Bot.Properties.Log.Default.PreWorkDir, Cliver.Bot.Properties.Log.Default.WriteLog, Cliver.Bot.Properties.Log.Default.DeleteLogsOlderDays);
Log.Main.Inform("Version compiled: " + Cliver.Bot.Program.GetCustomizationCompiledTime().ToString());
Log.Main.Inform("Command line parameters: " + string.Join("|", Environment.GetCommandLineArgs()));
if (This != null)
{
throw new Exception("Previous session was not closed.");
}
new Session();
if (This == null)
{
return;
}
BotCycle.Start();
}
internal static void Start()
{
lock (threads2bot_cycle)//locked until threads2bot_cycle[t] = bc; to have a correct thread number
{
if (threads2bot_cycle.Count >= Settings.Engine.MaxBotThreadNumber)
{
return;
}
BotCycle bc = null;
Thread t = ThreadRoutines.Start(() =>
{
bc = Activator.Create <BotCycle>(false);
bc.bot_cycle();
}
);
if (!SleepRoutines.WaitForCondition(() => { return(bc != null && bc.Id >= 0); }, 100000))
{
throw new Exception("Could not start BotCycle thread");
}
threads2bot_cycle[t] = bc;
}
}
public static void Start()
{
try
{
if (!Settings.Log.WriteLog)
{
Log.DefaultLevel = Log.Level.NONE;
}
Log.Initialize(Log.Mode.FOLDER_PER_SESSION, new List <string> {
Settings.Log.PreWorkDir
}, Settings.Log.DeleteLogsOlderDays);
Log.Main.Inform("Version compiled: " + Program.GetCustomizationCompiledTime().ToString());
Log.Main.Inform("Command line parameters: " + string.Join("|", Environment.GetCommandLineArgs()));
if (This != null)
{
throw new Exception("Previous session was not closed.");
}
Activator.Create <Session>(true);
if (This == null)
{
return;
}
BotCycle.Start();
Session.State = SessionState.RUNNING;
This.Storage.WriteState(SessionState.RUNNING, new { });
}
catch (ThreadAbortException)
{
Close();
throw;
}
catch (Exception e)
{
Session.__ErrorClose(e, true);
}
}
void close()
{
lock (This_)
{
BotCycle.Abort();
if (This.items_xtw != null)
{
This.items_xtw.WriteEndElement();
This.items_xtw.WriteEndDocument();
This.items_xtw.Close();
}
if (This.input_item_queue_name2input_item_queues.Count > 0)
{
if (This.IsUnprocessedInputItem)
{
This.set_session_state(SessionState.ABORTED);
}
else if (This.IsItem2Restore)
{
This.set_session_state(SessionState.UNCOMPLETED);
}
else
{
This.set_session_state(SessionState.COMPLETED);
}
}
This.workflow_xtw.WriteEndElement();
This.workflow_xtw.WriteEndDocument();
This.workflow_xtw.Close();
try
{
CustomizationApi.SessionClosing();
}
catch (Exception e)
{
LogMessage.Error(e);
}
try
{
if (Closing != null)
{
Closing.Invoke();
}
}
catch (Exception e)
{
LogMessage.Error(e);
}
InputItemQueue.Close();
FileWriter.ClearSession();
Cache.ClearSession();
Proxies.ClearSession();
WebRoutine.ClearSession();
Log.Main.Write("Closing session.");
Cliver.Log.ClearSession();
This_ = null;
}
}
/// <summary>
/// Method to process custom InputItem.
/// When it is not defined within custom InputItem class, it must be defined in CustomBot as a function with InputItem Type parameter, where the function name is "PROCESSOR".
/// </summary>
/// <param name="bc"></param>
virtual public void PROCESSOR(BotCycle bc)
{
//it will be invoked by default if no overriding PROCESSOR implementation
bc.Bot.PROCESSOR(this);
}
void close()
{
lock (This_)
{
try
{
Log.Main.Write("Closing the bot session: " + Session.State.ToString());
BotCycle.Abort();
if (This.IsUnprocessedInputItem)
{
State = SessionState.BROKEN;
}
else if (This.IsItem2Restore)
{
State = SessionState.UNCOMPLETED;
}
else
{
State = SessionState.COMPLETED;
}
This.Storage.WriteState(State, new { });
try
{
__Closing();
}
catch (Exception e)
{
Session.State = SessionState.FATAL_ERROR;
This.Storage.WriteState(State, new { });
LogMessage.Error(e);
__ErrorClosing(e.Message);
}
try
{
Closing?.Invoke();
}
catch (Exception e)
{
Session.State = SessionState.FATAL_ERROR;
This.Storage.WriteState(State, new { });
LogMessage.Error(e);
__ErrorClosing(e.Message);
}
InputItemQueue.Close();
FileWriter.ClearSession();
}
catch (ThreadAbortException)
{
}
catch (Exception e)
{
Session.State = SessionState.FATAL_ERROR;
This.Storage.WriteState(State, new { });
LogMessage.Error(e);
__ErrorClosing(e.Message);
}
finally
{
Storage.Close();
switch (State)
{
case SessionState.NULL:
case SessionState.STARTING:
case SessionState.COMPLETED:
case SessionState.FATAL_ERROR:
Directory.Move(Dir, Dir + "_" + TimeMark + "_" + State);
break;
case SessionState.RESTORING:
case SessionState.RUNNING:
case SessionState.CLOSING:
case SessionState.UNCOMPLETED:
case SessionState.BROKEN:
case SessionState.NONFATAL_ERROR:
break;
default:
throw new Exception("Unknown option: " + State);
}
This_ = null;
Cliver.Log.Head.Close(false);
}
}
try
{
Closed?.Invoke();
}
catch (Exception e)
{
LogMessage.Error(e);
__ErrorClosing(e.Message);
}
}
/// <summary>
/// Add item as dynamic object to queue. It is possible to create a named queue.
/// It is the same like BotCycle.Add() but not so efficient and safe.
/// </summary>
/// <typeparam name="ItemT"></typeparam>
/// <param name="queue_name"></param>
/// <param name="anonymous_object"></param>
/// <returns></returns>
static public bool Add <ItemT>(string queue_name, object anonymous_object) where ItemT : InputItem
{
InputItemQueue iiq = Session.GetInputItemQueue(queue_name);
return(InputItem.Add2Queue <ItemT>(iiq, BotCycle.GetCurrentInputItemForThisThread(), anonymous_object));
}
/// <summary>
/// Method to process custom InputItem.
/// When it is not defined within custom InputItem class, it must be defined in CustomBot as a function with InputItem Type parameter, where the function name is "PROCESSOR".
/// </summary>
/// <param name="bc"></param>
virtual public void __Processor(BotCycle bc)
{
//it will be invoked by default if no overriding PROCESSOR implementation
bc.__Processor(this);
}
public override void PROCESSOR(BotCycle bc)
{
CustomBot cb = (CustomBot)bc.Bot;
if (!cb.HR.Get(Url))
throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
DataSifter.Capture gc = cb.category.Parse(cb.HR.HtmlResult);
string[] urls = Spider.GetAbsoluteUrls(gc.ValuesOf("CategoryUrl"), cb.HR.ResponseUrl, cb.HR.HtmlResult);
foreach (string url in urls)
cb.BotCycle.Add(new ListItem(url));
}
public override void PROCESSOR(BotCycle bc)
{
CustomBot cb = (CustomBot)bc.Bot;
if (!cb.HR.Get(Url))
throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
DataSifter.Capture gc = cb.product.Parse(cb.HR.HtmlResult);
decimal stock = (decimal)Fhr.CrawlerHost.Product.StockValue.NOT_SET;
if (gc.ValueOf("Stock") != null)
if (!decimal.TryParse(gc.ValueOf("Stock"), out stock))
stock = (decimal)Fhr.CrawlerHost.Product.StockValue.CANNOT_PARSE;
Fhr.CrawlerHost.Product product = new Fhr.CrawlerHost.Product(
id: gc.ValueOf("Id"),
url: Url,
name: gc.ValueOf("Name"),
sku: gc.ValueOf("Sku"),
price: gc.ValueOf("Price"),
category_branch: gc.ValuesOf("Category"),
image_urls: Spider.GetAbsoluteUrls(gc.ValuesOf("ImageUrl"), Url, cb.HR.HtmlResult),
stock: stock,
description: gc.ValueOf("Description")
);
if (!Fhr.CrawlerHost.CrawlerApi.SaveProductAsJson(product))
throw new ProcessorException(ProcessorExceptionType.ERROR, "Product was not saved.");
}
public override void PROCESSOR(BotCycle bc)
{
CustomBot cb = (CustomBot)bc.Bot;
if (!cb.HR.Get(Url))
throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
DataSifter.Capture gc = cb.list.Parse(cb.HR.HtmlResult);
{
string url = gc.ValueOf("NextPageUrl");
if (url != null)
cb.BotCycle.Add(new ListItem(Spider.GetAbsoluteUrl(url, cb.HR.ResponseUrl)));
}
string[] urls = Spider.GetAbsoluteUrls(gc.ValuesOf("ProductUrl"), cb.HR.ResponseUrl, cb.HR.HtmlResult);
foreach (string url in urls)
{
cb.BotCycle.Add(new ProductItem(url));
}
}
/// <summary>
/// Add item as dynamic object to queue. By default name of queue is name of item type.
/// It is the same as BotCycle.Add() but not so efficient and safe.
/// </summary>
/// <typeparam name="ItemT"></typeparam>
/// <param name="anonymous_object"></param>
/// <returns></returns>
public bool Add <ItemT>(object anonymous_object) where ItemT : InputItem
{
return(InputItem.Add2Queue <ItemT>(this, BotCycle.GetCurrentInputItemForThisThread(), anonymous_object));
}
/// <summary>
/// Add item as dynamic object to queue. It is possible to create a named queue.
/// It is the same like BotCycle.Add() but not so efficient and safe.
/// </summary>
/// <param name="queue_name"></param>
/// <param name="item"></param>
public bool Add(InputItem item)
{
return(InputItem.Add2Queue(this, BotCycle.GetCurrentInputItemForThisThread(), item));
}