/// <summary>
/// Attempts to stop the Crawler.
/// </summary>
/// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
/// <exception cref="InvalidOperationException">Thrown if the Crawler has not yet been initialized.</exception>
public SerializedException Stop()
{
SerializedException sx = null;
try
{
if (Crawler.InstanceExists())
{
if (crawler == null)
{
crawler = Crawler.Instance();
AttachObservers();
}
crawler.StopImmediately();
}
else
{
throw new InvalidOperationException("The Crawler has not been initialized and cannot be stopped.");
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace);
}
return(sx);
}
/// <exception cref="System.Exception"/>
public virtual void TestSerializedExceptionDeSer()
{
// without cause
YarnException yarnEx = new YarnException("Yarn_Exception");
SerializedException serEx = SerializedException.NewInstance(yarnEx);
Exception throwable = serEx.DeSerialize();
NUnit.Framework.Assert.AreEqual(yarnEx.GetType(), throwable.GetType());
NUnit.Framework.Assert.AreEqual(yarnEx.Message, throwable.Message);
// with cause
IOException ioe = new IOException("Test_IOException");
RuntimeException runtimeException = new RuntimeException("Test_RuntimeException",
ioe);
YarnException yarnEx2 = new YarnException("Test_YarnException", runtimeException);
SerializedException serEx2 = SerializedException.NewInstance(yarnEx2);
Exception throwable2 = serEx2.DeSerialize();
Sharpen.Runtime.PrintStackTrace(throwable2);
NUnit.Framework.Assert.AreEqual(yarnEx2.GetType(), throwable2.GetType());
NUnit.Framework.Assert.AreEqual(yarnEx2.Message, throwable2.Message);
NUnit.Framework.Assert.AreEqual(runtimeException.GetType(), throwable2.InnerException
.GetType());
NUnit.Framework.Assert.AreEqual(runtimeException.Message, throwable2.InnerException
.Message);
NUnit.Framework.Assert.AreEqual(ioe.GetType(), throwable2.InnerException.InnerException
.GetType());
NUnit.Framework.Assert.AreEqual(ioe.Message, throwable2.InnerException.InnerException
.Message);
}
/// <summary>
/// Attempts to retrieve the user's statistics from the server.
/// </summary>
/// <param name="stats">The statistics of the user.</param>
/// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException GetUserStatistics(ref UserStatistics stats)
{
SerializedException sx = null;
try
{
//WebServiceProxy proxy = WebServiceProxy.Instance();
UserStatistics userstats = null;
sx = proxy.SendUserStatistics(globals.Client_Info, out userstats);
if (sx != null)
{
log.LogError("An error occured while retrieving the statistics:" + sx.Message);
globals.FileLog.LogWarning("CrawlWave.Client: Failed to retrieve user's statistics: " + sx.Message);
stats = userstats;
}
}
catch (Exception e)
{
log.LogWarning("An error occured while retrieving the statistics: " + e.Message);
globals.FileLog.LogWarning("CrawlWave.Client: Failed to retrieve user's statistics: " + e.ToString());
sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace);
}
finally
{
GC.Collect();
}
return(sx);
}
public void Serialize_Exception_PreservesValues()
{
// Arrange
ArgumentException exception;
try
{
ThrowArgumentNullException();
throw new InvalidOperationException();
}
catch (ArgumentNullException ex)
{
exception = ex;
}
var exceptionObject = SerializedException.FromException(exception);
var stateSerializer = new StateSerializer();
stateSerializer.Serialize("Foo", exceptionObject);
stateSerializer.Dispose();
var stateDeserializer = new StateDeserializer(stateSerializer.GetState());
// Act
var result = stateDeserializer.Deserialize <SerializedException>("Foo");
// Assert
Assert.Equal(exception.Message, result.Message);
Assert.Equal(exception.StackTrace, result.StackTrace);
Assert.Equal(typeof(ArgumentNullException), result.ExceptionType);
Assert.Equal(exception.ParamName, result.AdditionalProperties["ParamName"]);
}
private void CheckConnected(SerializedException ex)
{
if (ex.ContainsType("CommunicationException", "TimeoutException"))
{
ValidateConnection();
}
}
internal MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, URL
localPath, SerializedException ex)
{
this.rsrc = rsrc;
this.tag = tag;
this.localPath = localPath;
this.ex = ex;
}
public virtual void SetException(SerializedException exception)
{
MaybeInitBuilder();
if (exception == null)
{
builder.ClearException();
}
this.exception = exception;
}
/// <summary>
/// Selects and returns the statistics for a certain user.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the statistics.</param>
/// <param name="stats">The <see cref="UserStatistics"/> of the user.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException SelectUserStatistics(ClientInfo ci, ref UserStatistics stats)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
SqlCommand cmd = new SqlCommand("cw_select_user_statistic", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@user_id", SqlDbType.Int);
cmd.Parameters[0].Value = ci.UserID;
SqlDataAdapter da = new SqlDataAdapter(cmd);
DataSet ds = new DataSet();
da.Fill(ds);
da.Dispose();
cmd.Dispose();
if (ds.Tables[0].Rows.Count > 0)
{
stats.RegistrationDate = (DateTime)ds.Tables[0].Rows[0][2];
stats.LastActive = (DateTime)ds.Tables[0].Rows[0][6];
foreach (DataRow dr in ds.Tables[0].Rows)
{
stats.NumClients++;
stats.UrlsAssigned += (long)dr[4];
stats.UrlsReturned += (long)dr[5];
DateTime la = (DateTime)dr[6];
if (la > stats.LastActive)
{
stats.LastActive = la;
}
}
}
ds.Dispose();
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectUserStatistics failed for user " + ci.UserID.ToString() + ":" + e.ToString());
}
}
finally
{
UpdateClientLastActive(ci);
LogClientAction(ci, CWClientActions.LogSendUserStatistics);
}
return(sx);
}
/// <summary>
/// Performs the registration of a new user by storing his info in the database.
/// </summary>
/// <param name="ID">The ID that will be assigned to the new user, passed by reference.</param>
/// <param name="username">The username requested from the new user.</param>
/// <param name="password">The hash of the new user's password.</param>
/// <param name="email">The user's email address.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException StoreUserRegistrationInfo(ref int ID, string username, byte[] password, string email)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
SqlCommand cmd = new SqlCommand("cw_insert_user", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@username", SqlDbType.NVarChar, 20);
cmd.Parameters.Add("@password", SqlDbType.UniqueIdentifier);
cmd.Parameters.Add("@email", SqlDbType.NVarChar, 50);
cmd.Parameters.Add("@user_id", SqlDbType.Int);
cmd.Parameters[3].Direction = ParameterDirection.ReturnValue;
cmd.Parameters[0].Value = username;
cmd.Parameters[1].Value = new Guid(password);
cmd.Parameters[2].Value = email;
try
{
cmd.ExecuteNonQuery();
ID = (int)cmd.Parameters[3].Value;
ClientInfo ci;
ci.UserID = ID;
ci.ClientID = Guid.Empty;
ci.Version = "0.0.0.0";
LogClientAction(ci, CWClientActions.LogRegisterUser);
}
catch (Exception se)
{
//the user already exists, throw an appropriate exception
throw new CWUserExistsException("User registration failed: " + se.Message);
}
finally
{
cmd.Dispose();
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("StoreUserRegistrationInfo failed: " + e.ToString());
}
}
return(sx);
}
/// <summary>
/// Performs the update of the Client's version. It queries an available server for
/// the latest version and if a new version exists it goes on with the update.
/// </summary>
private void UpdateClient()
{
try
{
while (!InternetUtils.ConnectedToInternet())
{
Thread.Sleep(updateBackoff.Next());
}
//proxy = WebServiceProxy.Instance();
proxy = CrawlWaveServerProxy.Instance(globals);
string latest = String.Empty;
SerializedException sx = proxy.SendLatestVersion(globals.Client_Info, out latest);
if (sx != null)
{
globals.SystemLog.LogError("CrawlWave Client Scheduler failed to retrieve the latest version of the Client:" + sx.Message);
return;
}
Version latestVersion = new Version(latest);
if (GetClientVersion() < latestVersion)
{
//we must update the client. First of all download the update.
updating = true;
byte [] buffer = new byte[0];
sx = proxy.SendUpdatedVersion(globals.Client_Info, latest, out buffer);
if (sx != null || buffer.Length == 0)
{
globals.SystemLog.LogError("CrawlWave Client Scheduler failed to retrieve the latest version of the Client: " + sx.Message);
updating = false;
return;
}
//save the compressed file to disk. If necessary launch the installer.
string updateFileName = globals.AppPath + latest + ".zip";
FileStream outputStream = new FileStream(updateFileName, FileMode.Create);
outputStream.Write(buffer, 0, buffer.Length);
outputStream.Close();
string mustLaunchInstaller = ExtractUpdatedFiles(updateFileName);
if (mustLaunchInstaller != String.Empty)
{
//Launch Installer and exit
Process.Start(mustLaunchInstaller);
}
}
}
catch
{}
finally
{
updating = false;
}
}
/// <summary>
/// Clears the banned hosts list and initializes it with the latest version.
/// </summary>
private void InitializeBannedHosts()
{
try
{
ICrawlWaveServer server = proxy;
try
{
server.IsAlive();
}
catch
{
if (globals.Settings.LogLevel <= CWLogLevel.LogWarning)
{
globals.FileLog.LogWarning("HostBanFilter failed to connect to the server.");
}
return;
}
DataSet ds = new DataSet();
SerializedException sx = server.SendBannedHosts(globals.Client_Info, out ds);
if (sx != null)
{
if (globals.Settings.LogLevel <= CWLogLevel.LogWarning)
{
globals.FileLog.LogWarning("HostBanFilter failed to download a list of banned hosts.");
}
return;
}
else
{
if (ds.Tables[0].Rows.Count > 0)
{
lock (hostTable.SyncRoot)
{
hostTable.Clear();
foreach (DataRow dr in ds.Tables[0].Rows)
{
Guid g = (Guid)(dr[0]);
try
{
hostTable.Add(g.ToByteArray(), null);
}
catch
{
continue;
}
}
}
}
}
ds.Dispose();
}
catch
{}
}
/// <summary>
/// Attempts to terminate the application.
/// </summary>
/// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public void Terminate(ref SerializedException sx)
{
try
{
//System.Windows.Forms.Application.Exit();
Client.Instance().MustTerminate = true;
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace);
}
}
/// <summary>
/// Selects and returns the latest version of the client updates available.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client performing the call.</param>
/// <param name="version">The latest version update available.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException SelectLatestVersion(ClientInfo ci, ref string version)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
//Load the values from the database
SqlCommand cmd = new SqlCommand("cw_select_client_versions", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
DataSet ds = new DataSet();
SqlDataAdapter da = new SqlDataAdapter(cmd);
da.Fill(ds);
da.Dispose();
cmd.Dispose();
dbcon.Close();
Version latestVersion = new Version(0, 0, 0, 0);
Version currentVersion;
foreach (DataRow dr in ds.Tables[0].Rows)
{
try
{
currentVersion = new Version(((string)dr[0]).Trim());
if (currentVersion > latestVersion)
{
latestVersion = currentVersion;
}
}
catch
{
continue;
}
}
ds.Dispose();
version = latestVersion.ToString();
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectBannedHosts failed: " + e.ToString());
}
}
finally
{
UpdateClientLastActive(ci);
}
return(sx);
}
public virtual SerializedException GetException()
{
YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder p = viaProto
? proto : builder;
if (this.exception != null)
{
return(this.exception);
}
if (!p.HasException())
{
return(null);
}
this.exception = ConvertFromProtoFormat(p.GetException());
return(this.exception);
}
/// <summary>
/// the serialized exception that is associated with an error that causes this item not to process
/// </summary>
/// <param name="ex">the thrown exception</param>
/// <param name="recoverable">indicates whether the error that caused the fail is recoverable.</param>
public virtual void Fail(SerializedException ex, bool recoverable)
{
if (!Done && (!recoverable || !(Parent?.DemandRequeue((TMe)this) ?? false)))
{
base.Fail(ex);
Success = false;
Done = true;
Parent?.DecreaseOpenItems(false, ex);
OnException(ex);
}
else
{
FailCount++;
}
}
protected void LogException(SerializedException ex, bool debug)
{
if (CollectExceptions)
{
exceptionCollector.Add(ex);
}
if (debug)
{
LogEnvironment.LogDebugEvent(null, ex.ToString(), (int)LogSeverity.Error, null);
}
else
{
LogEnvironment.LogEvent(ex.ToString(), LogSeverity.Error);
}
}
public void SerializedExceptionCanBeDotNetSerialized()
{
var payload = new JsonObject()
.Add("foo", "bar")
.Add("baz", 42);
var original = new SerializedException(payload);
var json = ExceptionSerializer.LegacyToJson(original);
var deserialized = ExceptionSerializer.LegacyFromJson(json);
Assert.AreEqual(typeof(SerializedException), deserialized.GetType());
Assert.AreEqual(
original.SerializedValue.ToString(),
((SerializedException)deserialized).SerializedValue.ToString()
);
}
/// <summary>
/// Selects and returns a byte array containing a Client Update version.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the data.</param>
/// <param name="version">The requested version.</param>
/// <param name="data">A byte array that will contain the binary update file.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException SelectUpdatedVersion(ClientInfo ci, string version, byte[] data)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
SqlCommand cmd = new SqlCommand("cw_select_updated_version", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@version", SqlDbType.NChar, 15);
cmd.Parameters[0].Value = version;
SqlDataAdapter da = new SqlDataAdapter(cmd);
DataSet ds = new DataSet();
da.Fill(ds);
da.Dispose();
cmd.Dispose();
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
if (ds.Tables[0].Rows.Count == 0)
{
throw new CWException("Version unavailable or not recognized.");
}
else
{
data = (byte[])ds.Tables[0].Rows[0][0];
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectUpdatedVersion failed: " + e.ToString());
}
}
finally
{
UpdateClientLastActive(ci);
LogClientAction(ci, CWClientActions.LogSendUpdatedVersion);
}
return(sx);
}
internal static LocalResourceStatus CreateLocalResourceStatus()
{
LocalResourceStatus ret = recordFactory.NewRecordInstance <LocalResourceStatus>();
NUnit.Framework.Assert.IsTrue(ret is LocalResourceStatusPBImpl);
ret.SetResource(CreateResource());
ret.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(new Path("file:///local/foo/bar"
)));
ret.SetStatus(ResourceStatusType.FetchSuccess);
ret.SetLocalSize(4443L);
Exception e = new Exception("Dingos.");
e.SetStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz"
, 10), new StackTraceElement("sbb", "one", "onm", 10) });
ret.SetException(SerializedException.NewInstance(e));
return(ret);
}
/// <summary>
/// Performs the registration of a new client for a registered user of the system.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client that wishes to be
/// registered to the system.</param>
/// <param name="info">The <see cref="CWComputerInfo"/> of the computer running the
/// client.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException StoreClientRegistrationInfo(ref ClientInfo ci, CWComputerInfo info)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
ci.ClientID = Guid.NewGuid();
SqlCommand cmd = new SqlCommand("cw_insert_client", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier);
cmd.Parameters.Add("@user_id", SqlDbType.Int);
cmd.Parameters.Add("@info_cpu", SqlDbType.NVarChar, 50);
cmd.Parameters.Add("@info_ram", SqlDbType.SmallInt);
cmd.Parameters.Add("@info_hdd", SqlDbType.Int);
cmd.Parameters.Add("@info_net", SqlDbType.TinyInt);
cmd.Parameters[0].Value = ci.ClientID;
cmd.Parameters[1].Value = ci.UserID;
cmd.Parameters[2].Value = info.CPUType;
cmd.Parameters[3].Value = info.RAMSize;
cmd.Parameters[4].Value = info.HDDSpace;
cmd.Parameters[5].Value = (byte)info.ConnectionSpeed;
cmd.ExecuteNonQuery();
cmd.Dispose();
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("StoreClientRegistrationInfo failed: " + e.ToString());
}
}
finally
{
LogClientAction(ci, CWClientActions.LogRegisterClient);
}
return(sx);
}
/// <summary>Create the payload for the HeartBeat.</summary>
/// <remarks>
/// Create the payload for the HeartBeat. Mainly the list of
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalResourceStatus
/// "/>
/// es
/// </remarks>
/// <returns>
/// a
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalizerStatus
/// "/>
/// that can be sent via heartbeat.
/// </returns>
/// <exception cref="System.Exception"/>
private LocalizerStatus CreateStatus()
{
IList <LocalResourceStatus> currentResources = new AList <LocalResourceStatus>();
// TODO: Synchronization??
for (IEnumerator <LocalResource> i = pendingResources.Keys.GetEnumerator(); i.HasNext
();)
{
LocalResource rsrc = i.Next();
LocalResourceStatus stat = recordFactory.NewRecordInstance <LocalResourceStatus>();
stat.SetResource(rsrc);
Future <Path> fPath = pendingResources[rsrc];
if (fPath.IsDone())
{
try
{
Path localPath = fPath.Get();
stat.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(localPath));
stat.SetLocalSize(FileUtil.GetDU(new FilePath(localPath.GetParent().ToUri())));
stat.SetStatus(ResourceStatusType.FetchSuccess);
}
catch (ExecutionException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e.InnerException));
}
catch (CancellationException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e));
}
// TODO shouldn't remove until ACK
i.Remove();
}
else
{
stat.SetStatus(ResourceStatusType.FetchPending);
}
currentResources.AddItem(stat);
}
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId(localizerId);
status.AddAllResources(currentResources);
return(status);
}
/// <summary>
/// Attempts to start the Crawler and enable the logging of events.
/// </summary>
/// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException Start()
{
SerializedException sx = null;
try
{
if (crawler == null)
{
crawler = Crawler.Instance();
AttachObservers();
}
crawler.Start();
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace);
}
return(sx);
}
public override Exception DeSerialize()
{
SerializedException cause = GetCause();
YarnProtos.SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder;
Type realClass = null;
try
{
realClass = Sharpen.Runtime.GetType(p.GetClassName());
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException(e);
}
Type classType = null;
if (typeof(YarnException).IsAssignableFrom(realClass))
{
classType = typeof(YarnException);
}
else
{
if (typeof(IOException).IsAssignableFrom(realClass))
{
classType = typeof(IOException);
}
else
{
if (typeof(RuntimeException).IsAssignableFrom(realClass))
{
classType = typeof(RuntimeException);
}
else
{
classType = typeof(Exception);
}
}
}
return(InstantiateException(realClass.AsSubclass(classType), GetMessage(), cause
== null ? null : cause.DeSerialize()));
}
/// <summary>
/// Attempts to perform the registration of a new user.
/// </summary>
/// <param name="UserName">The user's username.</param>
/// <param name="Password">The user's password.</param>
/// <param name="Email">The user's email address.</param>
/// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException RegisterUser(string UserName, string Password, string Email)
{
SerializedException sx = null;
try
{
//WebServiceProxy proxy = WebServiceProxy.Instance();
int ID = 0;
byte [] password = MD5Hash.md5(Password);
sx = proxy.RegisterUser(ref ID, UserName, password, Email);
if (sx != null)
{
if (sx.Type == "CrawlWave.Common.CWUserExistsException")
{
log.LogWarning("User already exists, attempting to register client.");
}
}
globals.Settings.UserID = ID;
globals.Settings.UserName = UserName;
globals.Settings.Password = password;
globals.Settings.Email = Email;
CWComputerInfo info = ComputerInfo.GetComputerInfo();
globals.Settings.HardwareInfo = ComputerInfo.GetSHA1HashCode(info);
globals.Settings.SaveSettings();
//proxy.ForceInitializeProxies();
ClientInfo ci = new ClientInfo();
ci.UserID = globals.Settings.UserID;
sx = proxy.RegisterClient(ref ci, info);
globals.Settings.ClientID = ci.ClientID;
globals.Settings.SaveSettings();
}
catch (Exception ex)
{
sx = new SerializedException(ex.GetType().ToString(), ex.Message, ex.StackTrace);
}
return(sx);
}
/// <summary>
/// Selects and returns a list of all the CrawlWave Servers.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the data.</param>
/// <param name="data">A <see cref="DataSet"/> that will contain the list of servers.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException SelectServers(ClientInfo ci, ref DataSet data)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
SqlCommand cmd = new SqlCommand("cw_select_servers", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
SqlDataAdapter da = new SqlDataAdapter(cmd);
data = new DataSet();
da.Fill(data);
da.Dispose();
cmd.Dispose();
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectServers failed: " + e.ToString());
}
}
finally
{
UpdateClientLastActive(ci);
LogClientAction(ci, CWClientActions.LogSendServers);
}
return(sx);
}
/// <summary>
/// Decreases the number of open items and raises the PackageFinished event when done
/// </summary>
/// <param name="success">indicates whether the processing of the current task was successful</param>
/// <param name="ex">the exception that has ocurred during processing</param>
public void DecreaseOpenItems(bool success, SerializedException ex)
{
lock (localResourceLock)
{
if (openTaskCount > 0)
{
Successful &= success;
openTaskCount--;
if (ex != null)
{
LogEnvironment.LogEvent(ex.ToString(), LogSeverity.Error);
}
if (openTaskCount == 0)
{
OnPackageFinished(new PackageFinishedEventArgs
{
Package = this,
Tasks = subTasks.Cast <IProcessTask>().ToArray()
});
}
}
}
}
private YarnProtos.SerializedExceptionProto ConvertToProtoFormat(SerializedException
t)
{
return(((SerializedExceptionPBImpl)t).GetProto());
}
/// <summary>
/// Stores the results that the clients return after crawling a set of Urls.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client returning the data.</param>
/// <param name="data">An array of <see cref="UrlCrawlData"/> objects containing the data of the crawled urls.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException StoreCrawlResults(ClientInfo ci, UrlCrawlData[] data)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
try
{
//store the new robots.txt files in the database, nothing else needs to
//be done since the urls will be marked as not assigned when their data
//is processed by DBUpdater
if ((data != null) && (data.Length > 0))
{
SqlCommand cmd = new SqlCommand("cw_update_or_insert_robot", dbcon);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@host_id", SqlDbType.UniqueIdentifier);
cmd.Parameters.Add("@disallowed", SqlDbType.NVarChar, 1000);
foreach (UrlCrawlData urlData in data)
{
if ((urlData.FlagFetchRobots) || (urlData.Redirected))
{
string url = urlData.Url;
cmd.Parameters[0].Value = new Guid(MD5Hash.md5(InternetUtils.HostName(url)));
cmd.Parameters[1].Value = urlData.RobotsDisallowedPaths;
try
{
cmd.ExecuteNonQuery();
}
catch
{
continue;
}
}
}
cmd.Dispose();
SqlCommand statscmd = new SqlCommand("cw_update_client_statistics", dbcon);
statscmd.CommandType = CommandType.StoredProcedure;
statscmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier);
statscmd.Parameters.Add("@assigned", SqlDbType.BigInt);
statscmd.Parameters.Add("@returned", SqlDbType.BigInt);
statscmd.Parameters.Add("@type", SqlDbType.TinyInt);
statscmd.Parameters[0].Value = ci.ClientID;
statscmd.Parameters[1].Value = DBNull.Value;
statscmd.Parameters[2].Value = data.Length;
statscmd.Parameters[3].Value = 1;
statscmd.ExecuteNonQuery();
statscmd.Dispose();
}
}
catch (Exception ex)
{
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("StoreCrawlResults failed: " + ex.ToString());
}
throw ex;
}
finally
{
//save xml file on disk
try
{
SaveXMLFile(ci, data);
}
catch (Exception se)
{
sx = new SerializedException(se.GetType().ToString(), se.Message, se.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("StoreCrawlResults failed to save XML data on disk: " + se.ToString());
}
}
}
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
}
finally
{
UpdateClientLastActive(ci);
LogClientAction(ci, CWClientActions.LogGetCrawlResults);
}
return(sx);
}
public virtual void SetException(SerializedException ex)
{
this.ex = ex;
}
/// <summary>
/// Selects and returns a set of urls that are ready to be crawled.
/// </summary>
/// <param name="ci">The <see cref="ClientInfo"/> of the client requesting urls to crawl.</param>
/// <param name="data">An array of <see cref="InternetUrlToCrawl"/> objects containing the selected urls.</param>
/// <returns>Null if the operation succeeds, or <see cref="SerializedException"/>
/// encapsulating the error that occured if the operation fails.</returns>
public SerializedException SelectUrlsToCrawl(ClientInfo ci, ref InternetUrlToCrawl[] data)
{
SerializedException sx = null;
try
{
if (!ConnectToDatabase())
{
throw new CWDBConnectionFailedException();
}
//we must use a transaction to make sure that if something goes wrong the
//changes to the database will be rolled back.
SqlTransaction transaction = dbcon.BeginTransaction(IsolationLevel.Serializable); //perhaps | repeatableread
try
{
//first select the urls to crawl
SqlCommand cmd = new SqlCommand("cw_select_urls_to_crawl", dbcon, transaction);
cmd.CommandType = CommandType.StoredProcedure;
cmd.CommandTimeout = 120;
SqlDataAdapter da = new SqlDataAdapter(cmd);
DataSet ds = new DataSet();
da.Fill(ds);
da.Dispose();
cmd.Dispose();
//now delete them from the table of urls to crawl
data = new InternetUrlToCrawl[ds.Tables[0].Rows.Count];
if (data.Length > 0)
{
int i = 0;
foreach (DataRow dr in ds.Tables[0].Rows)
{
try
{
InternetUrlToCrawl url = new InternetUrlToCrawl((int)dr[0], (string)dr[1]);
if (dr[2] != DBNull.Value)
{
url.CRC = (long)dr[2];
}
if (dr[3] != DBNull.Value)
{
url.FlagDomain = (DomainFlagValue)((byte)dr[3]);
}
if (dr[4] != DBNull.Value)
{
url.RobotsDisallowedPaths = (string)dr[4];
}
else
{
RobotsTxtEntry entry = settings.Robots.GetEntry(InternetUtils.HostName(url));
if (entry != null)
{
url.RobotsDisallowedPaths = ConcatenatePaths(entry.DisallowedPaths);
}
else
{
url.FlagFetchRobots = true;
}
}
data[i++] = url;
}
catch
{
continue;
}
}
SqlCommand statscmd = new SqlCommand("cw_update_client_statistics", dbcon, transaction);
statscmd.CommandType = CommandType.StoredProcedure;
statscmd.CommandTimeout = 120;
statscmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier);
statscmd.Parameters.Add("@assigned", SqlDbType.BigInt);
statscmd.Parameters.Add("@returned", SqlDbType.BigInt);
statscmd.Parameters.Add("@type", SqlDbType.TinyInt);
statscmd.Parameters[0].Value = ci.ClientID;
statscmd.Parameters[1].Value = data.Length;
statscmd.Parameters[2].Value = DBNull.Value;
statscmd.Parameters[3].Value = 0;
statscmd.ExecuteNonQuery();
statscmd.Dispose();
transaction.Commit();
}
}
catch (Exception ex)
{
transaction.Rollback();
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectUrlsToCrawl failed, Transaction was rolled back: " + ex.ToString());
}
throw ex;
}
finally
{
UpdateClientLastActive(ci);
LogClientAction(ci, CWClientActions.LogSendUrlsToCrawl);
if (!DisconnectFromDatabase())
{
throw new CWDBConnectionFailedException("Disconnect from database failure.");
}
}
}
catch (Exception e)
{
sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString());
if (settings.LogLevel <= CWLogLevel.LogWarning)
{
settings.Log.LogWarning("SelectUrlsToCrawl failed: " + e.ToString());
}
}
return(sx);
}
