public TorrentQueue(State st) { this.AppState = st; this.slots = new Amib.Threading.SmartThreadPool(); this.set_queue_size(App.Settings.QueueSize); this.last_queue_enable_setting = App.Settings.EnableQueue; }
public void Initialize(Scene scene, IConfigSource config) { m_scene = scene; m_scene.RegisterModuleInterface <IHttpRequestModule>(this); m_proxyurl = config.Configs["Startup"].GetString("HttpProxy"); m_proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions"); m_pendingRequests = new Dictionary <UUID, HttpRequestObject>(); m_completedRequests = new Queue <HttpRequestObject>(); m_objectQueueSize = new Dictionary <UUID, short>(); m_objectPriorities = new Dictionary <UUID, TimestampedItem <Amib.Threading.WorkItemPriority> >(); m_lastPriorityMaintenance = Util.GetLongTickCount(); _threadPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_NORMAL_THREADS, 2); _threadPool.Name = "HttpRequestWorkerNormal"; _slowPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_SLOW_THREADS, 0); _slowPool.Name = "HttpRequestWorkerSlow"; ReadBlacklistFromConfig(config.Configs["HttpRequest"]); MainConsole.Instance.Commands.AddCommand( "Comms", true, "httprequest queuelength", "httprequest queuelength", "Report on the current size of the request queue.", "Displays the current size of the request queue.", HttpRequestConsoleCommand); MainConsole.Instance.Commands.AddCommand( "Comms", true, "httprequest debug", "httprequest debug <debuglevel>", "Enable/disable debugging of the request queue.", "Enable/disable debugging of the request queue.", HttpRequestConsoleCommand); }
public void TestPurgeCreateConsistent() { Amib.Threading.SmartThreadPool pool = new Amib.Threading.SmartThreadPool(1000, 20); pool.Start(); var userId = UUID.Parse("01EAE367-3A88-48B2-A226-AB3234EE506B"); InventoryFolderBase folder1 = new InventoryFolderBase(); folder1.ID = UUID.Parse("F1EAE367-3A88-48B2-A226-AB3234EE506B"); folder1.Name = "Folder1"; folder1.Level = InventoryFolderBase.FolderLevel.TopLevel; folder1.Owner = userId; folder1.ParentID = UUID.Zero; folder1.Type = (short)OpenMetaverse.FolderType.Root; try { _storage.GetFolderAttributes(folder1.ID); } catch (InventoryObjectMissingException e) { _storage.CreateFolder(folder1); } for (int i = 0; i < 100; i++) { UUID assetId = UUID.Random(); UUID itemId = UUID.Random(); InventoryItemBase item = new InventoryItemBase { AssetID = assetId, AssetType = (int)OpenMetaverse.AssetType.Texture, BasePermissions = UInt32.MaxValue, CreationDate = Util.UnixTimeSinceEpoch(), CreatorId = userId.ToString(), CurrentPermissions = unchecked((uint)-1), Description = "A good item, of goodness", EveryOnePermissions = Int32.MaxValue + (uint)1, Flags = unchecked((uint)Int32.MinValue), Folder = folder1.ID, GroupID = UUID.Zero, GroupOwned = false, GroupPermissions = 0x123, ID = itemId, InvType = (int)OpenMetaverse.InventoryType.Texture, Name = "Item of Goodness", NextPermissions = 0xF, Owner = userId, SalePrice = 100, SaleType = 1 }; pool.QueueWorkItem(() => { _storage.CreateItem(item); }); pool.QueueWorkItem(() => { _storage.PurgeItem(item); }); pool.QueueWorkItem(() => { _storage.CreateItem(item); }); pool.QueueWorkItem(() => { _storage.PurgeItem(item); }); pool.WaitForIdle(); InventoryFolderBase newFolder = _storage.GetFolder(folder1.ID); //either the item should completely exist or not if (newFolder.Items.Exists((InventoryItemBase litem) => { return litem.ID == itemId; })) { Assert.DoesNotThrow(delegate() { InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero); }); //cleanup _storage.PurgeItem(item); } else { var ex = Assert.Throws<InventoryObjectMissingException>(delegate() { InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero); }); Assert.AreEqual("Item was not found in the index", ex.ErrorDetails); } } InventoryFolderBase finalFolder = _storage.GetFolder(folder1.ID); Assert.AreEqual(0, finalFolder.Items.Count); }
public void TestItemPurge() { UUID userId = UUID.Random(); InventoryFolderBase folder1 = new InventoryFolderBase(); folder1.ID = UUID.Random(); folder1.Name = "Folder1"; folder1.Level = InventoryFolderBase.FolderLevel.TopLevel; folder1.Owner = userId; folder1.ParentID = UUID.Zero; folder1.Type = (short)OpenMetaverse.FolderType.Root; _storage.CreateFolder(folder1); Amib.Threading.SmartThreadPool pool = new Amib.Threading.SmartThreadPool(1000, 20); pool.Start(); for (int i = 0; i < 1000; i++) { pool.QueueWorkItem(() => { UUID assetId = UUID.Random(); UUID itemId = UUID.Random(); InventoryItemBase item = new InventoryItemBase { AssetID = assetId, AssetType = (int)OpenMetaverse.AssetType.Texture, BasePermissions = UInt32.MaxValue, CreationDate = Util.UnixTimeSinceEpoch(), CreatorId = userId.ToString(), CurrentPermissions = unchecked((uint)-1), Description = "A good item, of goodness", EveryOnePermissions = Int32.MaxValue + (uint)1, Flags = unchecked((uint)Int32.MinValue), Folder = folder1.ID, GroupID = UUID.Zero, GroupOwned = false, GroupPermissions = 0x123, ID = itemId, InvType = (int)OpenMetaverse.InventoryType.Texture, Name = "Item of Goodness", NextPermissions = 0xF, Owner = userId, SalePrice = 100, SaleType = 1 }; _storage.PurgeItem(item); _storage.CreateItem(item); _storage.PurgeItem(item); _storage.CreateItem(item); _storage.PurgeItem(item); var ex = Assert.Throws<InventoryObjectMissingException>(delegate() { InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero); }); Assert.AreEqual("Item was not found in the index", ex.ErrorDetails); }); } pool.WaitForIdle(); InventoryFolderBase newFolder = _storage.GetFolder(folder1.ID); Assert.AreEqual(0, newFolder.Items.Count); Assert.AreEqual(5001, newFolder.Version); }
public void Initialize(Scene scene, IConfigSource config) { m_scene = scene; m_scene.RegisterModuleInterface<IHttpRequestModule>(this); m_proxyurl = config.Configs["Startup"].GetString("HttpProxy"); m_proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions"); m_pendingRequests = new Dictionary<UUID, HttpRequestObject>(); m_completedRequests = new Queue<HttpRequestObject>(); m_objectQueueSize = new Dictionary<UUID, short>(); m_objectPriorities = new Dictionary<UUID, TimestampedItem<Amib.Threading.WorkItemPriority>>(); m_lastPriorityMaintenance = Util.GetLongTickCount(); _threadPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_NORMAL_THREADS, 2); _threadPool.Name = "HttpRequestWorkerNormal"; _slowPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_SLOW_THREADS, 0); _slowPool.Name = "HttpRequestWorkerSlow"; ReadBlacklistFromConfig(config.Configs["HttpRequest"]); MainConsole.Instance.Commands.AddCommand( "Comms", true, "httprequest queuelength", "httprequest queuelength", "Report on the current size of the request queue.", "Displays the current size of the request queue.", HttpRequestConsoleCommand); MainConsole.Instance.Commands.AddCommand( "Comms", true, "httprequest debug", "httprequest debug <debuglevel>", "Enable/disable debugging of the request queue.", "Enable/disable debugging of the request queue.", HttpRequestConsoleCommand); }
static void Main(string[] args) { WebRequest.DefaultWebProxy = null; ServicePointManager.DefaultConnectionLimit = 1000; int single_id = -1; bool server = true; bool is_t = false; bool autorun = true; foreach (string arg in args) { if (arg.StartsWith("--thread:")) { // --thread:board:id board = arg.Split(':')[1]; single_id = Convert.ToInt32(arg.Split(':')[2]); } if (arg == "--noserver") { server = false; } if (arg.StartsWith("--board:")) { board = arg.Split(':')[1]; } if (arg == "--thumbonly") { is_t = true; } if (arg == "--verbose") { verbose = true; } if (arg == "--idle") { autorun = false; } if (arg.StartsWith("--port:")) { Int32.TryParse(arg.Split(':')[1], out port); } if (arg.StartsWith("--savedir")) { program_dir = args[Array.IndexOf(args, arg) + 1]; } if (arg == "-help" || arg == "--help" || arg == "/help" || arg == "-h") { Console.Write(Properties.Resources.help_text); return; } } ThreadStore.SetUp(new Thread_Storage.FolderStorageEngine()); thumb_stp = new Amib.Threading.SmartThreadPool() { MaxThreads = 10, MinThreads = 0 }; thumb_stp.Start(); file_stp = new Amib.Threading.SmartThreadPool() { MaxThreads = 5, MinThreads = 0 }; file_stp.Start(); if (string.IsNullOrEmpty(program_dir)) { program_dir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "chanarchiver"); } Directory.CreateDirectory(program_dir); file_save_dir = Path.Combine(program_dir, "files"); Directory.CreateDirectory(file_save_dir); thumb_save_dir = Path.Combine(program_dir, "thumbs"); Directory.CreateDirectory(thumb_save_dir); post_files_dir = Path.Combine(program_dir, "posts"); Directory.CreateDirectory(post_files_dir); api_cache_dir = Path.Combine(program_dir, "aniwrap_cache"); Directory.CreateDirectory(api_cache_dir); temp_files_dir = Path.Combine(program_dir, "temp"); Directory.CreateDirectory(temp_files_dir); board_settings_dir = Path.Combine(program_dir, "settings"); Directory.CreateDirectory(board_settings_dir); invalid_files_dir = Path.Combine(program_dir, "invalid_files"); Directory.CreateDirectory(invalid_files_dir); html_templates_dir = Path.Combine(program_dir, "html_templates", Version); Directory.CreateDirectory(html_templates_dir); HtmlTemplates.Init(); Settings.Load(); Settings.ThumbnailOnly = is_t; if (Settings.CacheAPIFilesInMemory) { aw = new AniWrap.AniWrap(); } else { aw = new AniWrap.AniWrap(api_cache_dir); } Console.Title = "ChanArchiver"; print("ChanArchiver ", ConsoleColor.Cyan); Console.Write(Version); Console.WriteLine(" stable"); if (Environment.OSVersion.Platform == PlatformID.Unix) { string[] ffmpegs_paths = { get_ffmpeg_path_unix(), "/bin/ffmpeg", "/usr/bin/ffmpeg", }; foreach (string s in ffmpegs_paths) { if (File.Exists(s)) { ffmpeg_path = s; Console.Write("Detected ffmpeg path:"); print(string.Format("'{0}'\n", s), ConsoleColor.Yellow); break; } } } else { ffmpeg_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "ffmpeg.exe"); Console.WriteLine("Make sure to place ffmpeg.exe in your home folder"); } swf_watch = new FileSystemWatcher(file_save_dir); swf_watch.EnableRaisingEvents = true; swf_watch.Filter = "*.swf"; swf_watch.IncludeSubdirectories = false; swf_watch.Created += new FileSystemEventHandler(handle_new_swf_file); Console.Write("Downloading board data..."); ValidBoards = aw.GetAvailableBoards(); Console.Write("loaded {0} board.\n", ValidBoards.Count); Console.Write("Saving files in "); print(string.Format("'{0}'\n", program_dir), ConsoleColor.Red); load_settings(autorun); if (Settings.ThumbnailOnly) { print("Warning:", ConsoleColor.Yellow); Console.WriteLine("ChanArchiver is running in thumbnail only mode"); } if (server) { Console.WriteLine("ChanArchiver is running in server mode"); start_server(); } if (!string.IsNullOrWhiteSpace(board)) { if (single_id > 0) { archive_single(board, single_id, Settings.ThumbnailOnly); } else { archive_board(board, BoardWatcher.BoardMode.FullBoard); } } FileIndex.Load(); optimize_directory_struct(thumb_save_dir); optimize_directory_struct(file_save_dir); ArchivesProvider.Load(); interactive_console(); save_settings(); FileSystemStats.Dispose(); }