/// <summary> /// Attempts to stop the Crawler. /// </summary> /// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> /// <exception cref="InvalidOperationException">Thrown if the Crawler has not yet been initialized.</exception> public SerializedException Stop() { SerializedException sx = null; try { if (Crawler.InstanceExists()) { if (crawler == null) { crawler = Crawler.Instance(); AttachObservers(); } crawler.StopImmediately(); } else { throw new InvalidOperationException("The Crawler has not been initialized and cannot be stopped."); } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace); } return(sx); }
/// <exception cref="System.Exception"/> public virtual void TestSerializedExceptionDeSer() { // without cause YarnException yarnEx = new YarnException("Yarn_Exception"); SerializedException serEx = SerializedException.NewInstance(yarnEx); Exception throwable = serEx.DeSerialize(); NUnit.Framework.Assert.AreEqual(yarnEx.GetType(), throwable.GetType()); NUnit.Framework.Assert.AreEqual(yarnEx.Message, throwable.Message); // with cause IOException ioe = new IOException("Test_IOException"); RuntimeException runtimeException = new RuntimeException("Test_RuntimeException", ioe); YarnException yarnEx2 = new YarnException("Test_YarnException", runtimeException); SerializedException serEx2 = SerializedException.NewInstance(yarnEx2); Exception throwable2 = serEx2.DeSerialize(); Sharpen.Runtime.PrintStackTrace(throwable2); NUnit.Framework.Assert.AreEqual(yarnEx2.GetType(), throwable2.GetType()); NUnit.Framework.Assert.AreEqual(yarnEx2.Message, throwable2.Message); NUnit.Framework.Assert.AreEqual(runtimeException.GetType(), throwable2.InnerException .GetType()); NUnit.Framework.Assert.AreEqual(runtimeException.Message, throwable2.InnerException .Message); NUnit.Framework.Assert.AreEqual(ioe.GetType(), throwable2.InnerException.InnerException .GetType()); NUnit.Framework.Assert.AreEqual(ioe.Message, throwable2.InnerException.InnerException .Message); }
/// <summary> /// Attempts to retrieve the user's statistics from the server. /// </summary> /// <param name="stats">The statistics of the user.</param> /// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException GetUserStatistics(ref UserStatistics stats) { SerializedException sx = null; try { //WebServiceProxy proxy = WebServiceProxy.Instance(); UserStatistics userstats = null; sx = proxy.SendUserStatistics(globals.Client_Info, out userstats); if (sx != null) { log.LogError("An error occured while retrieving the statistics:" + sx.Message); globals.FileLog.LogWarning("CrawlWave.Client: Failed to retrieve user's statistics: " + sx.Message); stats = userstats; } } catch (Exception e) { log.LogWarning("An error occured while retrieving the statistics: " + e.Message); globals.FileLog.LogWarning("CrawlWave.Client: Failed to retrieve user's statistics: " + e.ToString()); sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace); } finally { GC.Collect(); } return(sx); }
public void Serialize_Exception_PreservesValues() { // Arrange ArgumentException exception; try { ThrowArgumentNullException(); throw new InvalidOperationException(); } catch (ArgumentNullException ex) { exception = ex; } var exceptionObject = SerializedException.FromException(exception); var stateSerializer = new StateSerializer(); stateSerializer.Serialize("Foo", exceptionObject); stateSerializer.Dispose(); var stateDeserializer = new StateDeserializer(stateSerializer.GetState()); // Act var result = stateDeserializer.Deserialize <SerializedException>("Foo"); // Assert Assert.Equal(exception.Message, result.Message); Assert.Equal(exception.StackTrace, result.StackTrace); Assert.Equal(typeof(ArgumentNullException), result.ExceptionType); Assert.Equal(exception.ParamName, result.AdditionalProperties["ParamName"]); }
private void CheckConnected(SerializedException ex) { if (ex.ContainsType("CommunicationException", "TimeoutException")) { ValidateConnection(); } }
internal MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, URL localPath, SerializedException ex) { this.rsrc = rsrc; this.tag = tag; this.localPath = localPath; this.ex = ex; }
public virtual void SetException(SerializedException exception) { MaybeInitBuilder(); if (exception == null) { builder.ClearException(); } this.exception = exception; }
/// <summary> /// Selects and returns the statistics for a certain user. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the statistics.</param> /// <param name="stats">The <see cref="UserStatistics"/> of the user.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException SelectUserStatistics(ClientInfo ci, ref UserStatistics stats) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } SqlCommand cmd = new SqlCommand("cw_select_user_statistic", dbcon); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add("@user_id", SqlDbType.Int); cmd.Parameters[0].Value = ci.UserID; SqlDataAdapter da = new SqlDataAdapter(cmd); DataSet ds = new DataSet(); da.Fill(ds); da.Dispose(); cmd.Dispose(); if (ds.Tables[0].Rows.Count > 0) { stats.RegistrationDate = (DateTime)ds.Tables[0].Rows[0][2]; stats.LastActive = (DateTime)ds.Tables[0].Rows[0][6]; foreach (DataRow dr in ds.Tables[0].Rows) { stats.NumClients++; stats.UrlsAssigned += (long)dr[4]; stats.UrlsReturned += (long)dr[5]; DateTime la = (DateTime)dr[6]; if (la > stats.LastActive) { stats.LastActive = la; } } } ds.Dispose(); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectUserStatistics failed for user " + ci.UserID.ToString() + ":" + e.ToString()); } } finally { UpdateClientLastActive(ci); LogClientAction(ci, CWClientActions.LogSendUserStatistics); } return(sx); }
/// <summary> /// Performs the registration of a new user by storing his info in the database. /// </summary> /// <param name="ID">The ID that will be assigned to the new user, passed by reference.</param> /// <param name="username">The username requested from the new user.</param> /// <param name="password">The hash of the new user's password.</param> /// <param name="email">The user's email address.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException StoreUserRegistrationInfo(ref int ID, string username, byte[] password, string email) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } SqlCommand cmd = new SqlCommand("cw_insert_user", dbcon); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add("@username", SqlDbType.NVarChar, 20); cmd.Parameters.Add("@password", SqlDbType.UniqueIdentifier); cmd.Parameters.Add("@email", SqlDbType.NVarChar, 50); cmd.Parameters.Add("@user_id", SqlDbType.Int); cmd.Parameters[3].Direction = ParameterDirection.ReturnValue; cmd.Parameters[0].Value = username; cmd.Parameters[1].Value = new Guid(password); cmd.Parameters[2].Value = email; try { cmd.ExecuteNonQuery(); ID = (int)cmd.Parameters[3].Value; ClientInfo ci; ci.UserID = ID; ci.ClientID = Guid.Empty; ci.Version = "0.0.0.0"; LogClientAction(ci, CWClientActions.LogRegisterUser); } catch (Exception se) { //the user already exists, throw an appropriate exception throw new CWUserExistsException("User registration failed: " + se.Message); } finally { cmd.Dispose(); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("StoreUserRegistrationInfo failed: " + e.ToString()); } } return(sx); }
/// <summary> /// Performs the update of the Client's version. It queries an available server for /// the latest version and if a new version exists it goes on with the update. /// </summary> private void UpdateClient() { try { while (!InternetUtils.ConnectedToInternet()) { Thread.Sleep(updateBackoff.Next()); } //proxy = WebServiceProxy.Instance(); proxy = CrawlWaveServerProxy.Instance(globals); string latest = String.Empty; SerializedException sx = proxy.SendLatestVersion(globals.Client_Info, out latest); if (sx != null) { globals.SystemLog.LogError("CrawlWave Client Scheduler failed to retrieve the latest version of the Client:" + sx.Message); return; } Version latestVersion = new Version(latest); if (GetClientVersion() < latestVersion) { //we must update the client. First of all download the update. updating = true; byte [] buffer = new byte[0]; sx = proxy.SendUpdatedVersion(globals.Client_Info, latest, out buffer); if (sx != null || buffer.Length == 0) { globals.SystemLog.LogError("CrawlWave Client Scheduler failed to retrieve the latest version of the Client: " + sx.Message); updating = false; return; } //save the compressed file to disk. If necessary launch the installer. string updateFileName = globals.AppPath + latest + ".zip"; FileStream outputStream = new FileStream(updateFileName, FileMode.Create); outputStream.Write(buffer, 0, buffer.Length); outputStream.Close(); string mustLaunchInstaller = ExtractUpdatedFiles(updateFileName); if (mustLaunchInstaller != String.Empty) { //Launch Installer and exit Process.Start(mustLaunchInstaller); } } } catch {} finally { updating = false; } }
/// <summary> /// Clears the banned hosts list and initializes it with the latest version. /// </summary> private void InitializeBannedHosts() { try { ICrawlWaveServer server = proxy; try { server.IsAlive(); } catch { if (globals.Settings.LogLevel <= CWLogLevel.LogWarning) { globals.FileLog.LogWarning("HostBanFilter failed to connect to the server."); } return; } DataSet ds = new DataSet(); SerializedException sx = server.SendBannedHosts(globals.Client_Info, out ds); if (sx != null) { if (globals.Settings.LogLevel <= CWLogLevel.LogWarning) { globals.FileLog.LogWarning("HostBanFilter failed to download a list of banned hosts."); } return; } else { if (ds.Tables[0].Rows.Count > 0) { lock (hostTable.SyncRoot) { hostTable.Clear(); foreach (DataRow dr in ds.Tables[0].Rows) { Guid g = (Guid)(dr[0]); try { hostTable.Add(g.ToByteArray(), null); } catch { continue; } } } } } ds.Dispose(); } catch {} }
/// <summary> /// Attempts to terminate the application. /// </summary> /// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public void Terminate(ref SerializedException sx) { try { //System.Windows.Forms.Application.Exit(); Client.Instance().MustTerminate = true; } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace); } }
/// <summary> /// Selects and returns the latest version of the client updates available. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client performing the call.</param> /// <param name="version">The latest version update available.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException SelectLatestVersion(ClientInfo ci, ref string version) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } //Load the values from the database SqlCommand cmd = new SqlCommand("cw_select_client_versions", dbcon); cmd.CommandType = CommandType.StoredProcedure; DataSet ds = new DataSet(); SqlDataAdapter da = new SqlDataAdapter(cmd); da.Fill(ds); da.Dispose(); cmd.Dispose(); dbcon.Close(); Version latestVersion = new Version(0, 0, 0, 0); Version currentVersion; foreach (DataRow dr in ds.Tables[0].Rows) { try { currentVersion = new Version(((string)dr[0]).Trim()); if (currentVersion > latestVersion) { latestVersion = currentVersion; } } catch { continue; } } ds.Dispose(); version = latestVersion.ToString(); } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectBannedHosts failed: " + e.ToString()); } } finally { UpdateClientLastActive(ci); } return(sx); }
public virtual SerializedException GetException() { YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder; if (this.exception != null) { return(this.exception); } if (!p.HasException()) { return(null); } this.exception = ConvertFromProtoFormat(p.GetException()); return(this.exception); }
/// <summary> /// the serialized exception that is associated with an error that causes this item not to process /// </summary> /// <param name="ex">the thrown exception</param> /// <param name="recoverable">indicates whether the error that caused the fail is recoverable.</param> public virtual void Fail(SerializedException ex, bool recoverable) { if (!Done && (!recoverable || !(Parent?.DemandRequeue((TMe)this) ?? false))) { base.Fail(ex); Success = false; Done = true; Parent?.DecreaseOpenItems(false, ex); OnException(ex); } else { FailCount++; } }
protected void LogException(SerializedException ex, bool debug) { if (CollectExceptions) { exceptionCollector.Add(ex); } if (debug) { LogEnvironment.LogDebugEvent(null, ex.ToString(), (int)LogSeverity.Error, null); } else { LogEnvironment.LogEvent(ex.ToString(), LogSeverity.Error); } }
public void SerializedExceptionCanBeDotNetSerialized() { var payload = new JsonObject() .Add("foo", "bar") .Add("baz", 42); var original = new SerializedException(payload); var json = ExceptionSerializer.LegacyToJson(original); var deserialized = ExceptionSerializer.LegacyFromJson(json); Assert.AreEqual(typeof(SerializedException), deserialized.GetType()); Assert.AreEqual( original.SerializedValue.ToString(), ((SerializedException)deserialized).SerializedValue.ToString() ); }
/// <summary> /// Selects and returns a byte array containing a Client Update version. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the data.</param> /// <param name="version">The requested version.</param> /// <param name="data">A byte array that will contain the binary update file.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException SelectUpdatedVersion(ClientInfo ci, string version, byte[] data) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } SqlCommand cmd = new SqlCommand("cw_select_updated_version", dbcon); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add("@version", SqlDbType.NChar, 15); cmd.Parameters[0].Value = version; SqlDataAdapter da = new SqlDataAdapter(cmd); DataSet ds = new DataSet(); da.Fill(ds); da.Dispose(); cmd.Dispose(); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } if (ds.Tables[0].Rows.Count == 0) { throw new CWException("Version unavailable or not recognized."); } else { data = (byte[])ds.Tables[0].Rows[0][0]; } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectUpdatedVersion failed: " + e.ToString()); } } finally { UpdateClientLastActive(ci); LogClientAction(ci, CWClientActions.LogSendUpdatedVersion); } return(sx); }
internal static LocalResourceStatus CreateLocalResourceStatus() { LocalResourceStatus ret = recordFactory.NewRecordInstance <LocalResourceStatus>(); NUnit.Framework.Assert.IsTrue(ret is LocalResourceStatusPBImpl); ret.SetResource(CreateResource()); ret.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(new Path("file:///local/foo/bar" ))); ret.SetStatus(ResourceStatusType.FetchSuccess); ret.SetLocalSize(4443L); Exception e = new Exception("Dingos."); e.SetStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz" , 10), new StackTraceElement("sbb", "one", "onm", 10) }); ret.SetException(SerializedException.NewInstance(e)); return(ret); }
/// <summary> /// Performs the registration of a new client for a registered user of the system. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client that wishes to be /// registered to the system.</param> /// <param name="info">The <see cref="CWComputerInfo"/> of the computer running the /// client.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException StoreClientRegistrationInfo(ref ClientInfo ci, CWComputerInfo info) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } ci.ClientID = Guid.NewGuid(); SqlCommand cmd = new SqlCommand("cw_insert_client", dbcon); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier); cmd.Parameters.Add("@user_id", SqlDbType.Int); cmd.Parameters.Add("@info_cpu", SqlDbType.NVarChar, 50); cmd.Parameters.Add("@info_ram", SqlDbType.SmallInt); cmd.Parameters.Add("@info_hdd", SqlDbType.Int); cmd.Parameters.Add("@info_net", SqlDbType.TinyInt); cmd.Parameters[0].Value = ci.ClientID; cmd.Parameters[1].Value = ci.UserID; cmd.Parameters[2].Value = info.CPUType; cmd.Parameters[3].Value = info.RAMSize; cmd.Parameters[4].Value = info.HDDSpace; cmd.Parameters[5].Value = (byte)info.ConnectionSpeed; cmd.ExecuteNonQuery(); cmd.Dispose(); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("StoreClientRegistrationInfo failed: " + e.ToString()); } } finally { LogClientAction(ci, CWClientActions.LogRegisterClient); } return(sx); }
/// <summary>Create the payload for the HeartBeat.</summary> /// <remarks> /// Create the payload for the HeartBeat. Mainly the list of /// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalResourceStatus /// "/> /// es /// </remarks> /// <returns> /// a /// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalizerStatus /// "/> /// that can be sent via heartbeat. /// </returns> /// <exception cref="System.Exception"/> private LocalizerStatus CreateStatus() { IList <LocalResourceStatus> currentResources = new AList <LocalResourceStatus>(); // TODO: Synchronization?? for (IEnumerator <LocalResource> i = pendingResources.Keys.GetEnumerator(); i.HasNext ();) { LocalResource rsrc = i.Next(); LocalResourceStatus stat = recordFactory.NewRecordInstance <LocalResourceStatus>(); stat.SetResource(rsrc); Future <Path> fPath = pendingResources[rsrc]; if (fPath.IsDone()) { try { Path localPath = fPath.Get(); stat.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(localPath)); stat.SetLocalSize(FileUtil.GetDU(new FilePath(localPath.GetParent().ToUri()))); stat.SetStatus(ResourceStatusType.FetchSuccess); } catch (ExecutionException e) { stat.SetStatus(ResourceStatusType.FetchFailure); stat.SetException(SerializedException.NewInstance(e.InnerException)); } catch (CancellationException e) { stat.SetStatus(ResourceStatusType.FetchFailure); stat.SetException(SerializedException.NewInstance(e)); } // TODO shouldn't remove until ACK i.Remove(); } else { stat.SetStatus(ResourceStatusType.FetchPending); } currentResources.AddItem(stat); } LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>(); status.SetLocalizerId(localizerId); status.AddAllResources(currentResources); return(status); }
/// <summary> /// Attempts to start the Crawler and enable the logging of events. /// </summary> /// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException Start() { SerializedException sx = null; try { if (crawler == null) { crawler = Crawler.Instance(); AttachObservers(); } crawler.Start(); } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.StackTrace); } return(sx); }
public override Exception DeSerialize() { SerializedException cause = GetCause(); YarnProtos.SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; Type realClass = null; try { realClass = Sharpen.Runtime.GetType(p.GetClassName()); } catch (TypeLoadException e) { throw new YarnRuntimeException(e); } Type classType = null; if (typeof(YarnException).IsAssignableFrom(realClass)) { classType = typeof(YarnException); } else { if (typeof(IOException).IsAssignableFrom(realClass)) { classType = typeof(IOException); } else { if (typeof(RuntimeException).IsAssignableFrom(realClass)) { classType = typeof(RuntimeException); } else { classType = typeof(Exception); } } } return(InstantiateException(realClass.AsSubclass(classType), GetMessage(), cause == null ? null : cause.DeSerialize())); }
/// <summary> /// Attempts to perform the registration of a new user. /// </summary> /// <param name="UserName">The user's username.</param> /// <param name="Password">The user's password.</param> /// <param name="Email">The user's email address.</param> /// <returns>Null if the operation succeeds, or a <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException RegisterUser(string UserName, string Password, string Email) { SerializedException sx = null; try { //WebServiceProxy proxy = WebServiceProxy.Instance(); int ID = 0; byte [] password = MD5Hash.md5(Password); sx = proxy.RegisterUser(ref ID, UserName, password, Email); if (sx != null) { if (sx.Type == "CrawlWave.Common.CWUserExistsException") { log.LogWarning("User already exists, attempting to register client."); } } globals.Settings.UserID = ID; globals.Settings.UserName = UserName; globals.Settings.Password = password; globals.Settings.Email = Email; CWComputerInfo info = ComputerInfo.GetComputerInfo(); globals.Settings.HardwareInfo = ComputerInfo.GetSHA1HashCode(info); globals.Settings.SaveSettings(); //proxy.ForceInitializeProxies(); ClientInfo ci = new ClientInfo(); ci.UserID = globals.Settings.UserID; sx = proxy.RegisterClient(ref ci, info); globals.Settings.ClientID = ci.ClientID; globals.Settings.SaveSettings(); } catch (Exception ex) { sx = new SerializedException(ex.GetType().ToString(), ex.Message, ex.StackTrace); } return(sx); }
/// <summary> /// Selects and returns a list of all the CrawlWave Servers. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client requesting the data.</param> /// <param name="data">A <see cref="DataSet"/> that will contain the list of servers.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException SelectServers(ClientInfo ci, ref DataSet data) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } SqlCommand cmd = new SqlCommand("cw_select_servers", dbcon); cmd.CommandType = CommandType.StoredProcedure; SqlDataAdapter da = new SqlDataAdapter(cmd); data = new DataSet(); da.Fill(data); da.Dispose(); cmd.Dispose(); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectServers failed: " + e.ToString()); } } finally { UpdateClientLastActive(ci); LogClientAction(ci, CWClientActions.LogSendServers); } return(sx); }
/// <summary> /// Decreases the number of open items and raises the PackageFinished event when done /// </summary> /// <param name="success">indicates whether the processing of the current task was successful</param> /// <param name="ex">the exception that has ocurred during processing</param> public void DecreaseOpenItems(bool success, SerializedException ex) { lock (localResourceLock) { if (openTaskCount > 0) { Successful &= success; openTaskCount--; if (ex != null) { LogEnvironment.LogEvent(ex.ToString(), LogSeverity.Error); } if (openTaskCount == 0) { OnPackageFinished(new PackageFinishedEventArgs { Package = this, Tasks = subTasks.Cast <IProcessTask>().ToArray() }); } } } }
private YarnProtos.SerializedExceptionProto ConvertToProtoFormat(SerializedException t) { return(((SerializedExceptionPBImpl)t).GetProto()); }
/// <summary> /// Stores the results that the clients return after crawling a set of Urls. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client returning the data.</param> /// <param name="data">An array of <see cref="UrlCrawlData"/> objects containing the data of the crawled urls.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException StoreCrawlResults(ClientInfo ci, UrlCrawlData[] data) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } try { //store the new robots.txt files in the database, nothing else needs to //be done since the urls will be marked as not assigned when their data //is processed by DBUpdater if ((data != null) && (data.Length > 0)) { SqlCommand cmd = new SqlCommand("cw_update_or_insert_robot", dbcon); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add("@host_id", SqlDbType.UniqueIdentifier); cmd.Parameters.Add("@disallowed", SqlDbType.NVarChar, 1000); foreach (UrlCrawlData urlData in data) { if ((urlData.FlagFetchRobots) || (urlData.Redirected)) { string url = urlData.Url; cmd.Parameters[0].Value = new Guid(MD5Hash.md5(InternetUtils.HostName(url))); cmd.Parameters[1].Value = urlData.RobotsDisallowedPaths; try { cmd.ExecuteNonQuery(); } catch { continue; } } } cmd.Dispose(); SqlCommand statscmd = new SqlCommand("cw_update_client_statistics", dbcon); statscmd.CommandType = CommandType.StoredProcedure; statscmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier); statscmd.Parameters.Add("@assigned", SqlDbType.BigInt); statscmd.Parameters.Add("@returned", SqlDbType.BigInt); statscmd.Parameters.Add("@type", SqlDbType.TinyInt); statscmd.Parameters[0].Value = ci.ClientID; statscmd.Parameters[1].Value = DBNull.Value; statscmd.Parameters[2].Value = data.Length; statscmd.Parameters[3].Value = 1; statscmd.ExecuteNonQuery(); statscmd.Dispose(); } } catch (Exception ex) { if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("StoreCrawlResults failed: " + ex.ToString()); } throw ex; } finally { //save xml file on disk try { SaveXMLFile(ci, data); } catch (Exception se) { sx = new SerializedException(se.GetType().ToString(), se.Message, se.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("StoreCrawlResults failed to save XML data on disk: " + se.ToString()); } } } if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); } finally { UpdateClientLastActive(ci); LogClientAction(ci, CWClientActions.LogGetCrawlResults); } return(sx); }
public virtual void SetException(SerializedException ex) { this.ex = ex; }
/// <summary> /// Selects and returns a set of urls that are ready to be crawled. /// </summary> /// <param name="ci">The <see cref="ClientInfo"/> of the client requesting urls to crawl.</param> /// <param name="data">An array of <see cref="InternetUrlToCrawl"/> objects containing the selected urls.</param> /// <returns>Null if the operation succeeds, or <see cref="SerializedException"/> /// encapsulating the error that occured if the operation fails.</returns> public SerializedException SelectUrlsToCrawl(ClientInfo ci, ref InternetUrlToCrawl[] data) { SerializedException sx = null; try { if (!ConnectToDatabase()) { throw new CWDBConnectionFailedException(); } //we must use a transaction to make sure that if something goes wrong the //changes to the database will be rolled back. SqlTransaction transaction = dbcon.BeginTransaction(IsolationLevel.Serializable); //perhaps | repeatableread try { //first select the urls to crawl SqlCommand cmd = new SqlCommand("cw_select_urls_to_crawl", dbcon, transaction); cmd.CommandType = CommandType.StoredProcedure; cmd.CommandTimeout = 120; SqlDataAdapter da = new SqlDataAdapter(cmd); DataSet ds = new DataSet(); da.Fill(ds); da.Dispose(); cmd.Dispose(); //now delete them from the table of urls to crawl data = new InternetUrlToCrawl[ds.Tables[0].Rows.Count]; if (data.Length > 0) { int i = 0; foreach (DataRow dr in ds.Tables[0].Rows) { try { InternetUrlToCrawl url = new InternetUrlToCrawl((int)dr[0], (string)dr[1]); if (dr[2] != DBNull.Value) { url.CRC = (long)dr[2]; } if (dr[3] != DBNull.Value) { url.FlagDomain = (DomainFlagValue)((byte)dr[3]); } if (dr[4] != DBNull.Value) { url.RobotsDisallowedPaths = (string)dr[4]; } else { RobotsTxtEntry entry = settings.Robots.GetEntry(InternetUtils.HostName(url)); if (entry != null) { url.RobotsDisallowedPaths = ConcatenatePaths(entry.DisallowedPaths); } else { url.FlagFetchRobots = true; } } data[i++] = url; } catch { continue; } } SqlCommand statscmd = new SqlCommand("cw_update_client_statistics", dbcon, transaction); statscmd.CommandType = CommandType.StoredProcedure; statscmd.CommandTimeout = 120; statscmd.Parameters.Add("@client_id", SqlDbType.UniqueIdentifier); statscmd.Parameters.Add("@assigned", SqlDbType.BigInt); statscmd.Parameters.Add("@returned", SqlDbType.BigInt); statscmd.Parameters.Add("@type", SqlDbType.TinyInt); statscmd.Parameters[0].Value = ci.ClientID; statscmd.Parameters[1].Value = data.Length; statscmd.Parameters[2].Value = DBNull.Value; statscmd.Parameters[3].Value = 0; statscmd.ExecuteNonQuery(); statscmd.Dispose(); transaction.Commit(); } } catch (Exception ex) { transaction.Rollback(); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectUrlsToCrawl failed, Transaction was rolled back: " + ex.ToString()); } throw ex; } finally { UpdateClientLastActive(ci); LogClientAction(ci, CWClientActions.LogSendUrlsToCrawl); if (!DisconnectFromDatabase()) { throw new CWDBConnectionFailedException("Disconnect from database failure."); } } } catch (Exception e) { sx = new SerializedException(e.GetType().ToString(), e.Message, e.ToString()); if (settings.LogLevel <= CWLogLevel.LogWarning) { settings.Log.LogWarning("SelectUrlsToCrawl failed: " + e.ToString()); } } return(sx); }