public TorrentQueue(State st)
{
this.AppState = st;
this.slots = new Amib.Threading.SmartThreadPool();
this.set_queue_size(App.Settings.QueueSize);
this.last_queue_enable_setting = App.Settings.EnableQueue;
}
public TorrentQueue(State st)
{
this.AppState = st;
this.slots = new Amib.Threading.SmartThreadPool();
this.set_queue_size(App.Settings.QueueSize);
this.last_queue_enable_setting = App.Settings.EnableQueue;
}
public void Initialize(Scene scene, IConfigSource config)
{
m_scene = scene;
m_scene.RegisterModuleInterface <IHttpRequestModule>(this);
m_proxyurl = config.Configs["Startup"].GetString("HttpProxy");
m_proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions");
m_pendingRequests = new Dictionary <UUID, HttpRequestObject>();
m_completedRequests = new Queue <HttpRequestObject>();
m_objectQueueSize = new Dictionary <UUID, short>();
m_objectPriorities = new Dictionary <UUID, TimestampedItem <Amib.Threading.WorkItemPriority> >();
m_lastPriorityMaintenance = Util.GetLongTickCount();
_threadPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_NORMAL_THREADS, 2);
_threadPool.Name = "HttpRequestWorkerNormal";
_slowPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_SLOW_THREADS, 0);
_slowPool.Name = "HttpRequestWorkerSlow";
ReadBlacklistFromConfig(config.Configs["HttpRequest"]);
MainConsole.Instance.Commands.AddCommand(
"Comms",
true,
"httprequest queuelength",
"httprequest queuelength",
"Report on the current size of the request queue.",
"Displays the current size of the request queue.",
HttpRequestConsoleCommand);
MainConsole.Instance.Commands.AddCommand(
"Comms",
true,
"httprequest debug",
"httprequest debug <debuglevel>",
"Enable/disable debugging of the request queue.",
"Enable/disable debugging of the request queue.",
HttpRequestConsoleCommand);
}
public void TestPurgeCreateConsistent()
{
Amib.Threading.SmartThreadPool pool = new Amib.Threading.SmartThreadPool(1000, 20);
pool.Start();
var userId = UUID.Parse("01EAE367-3A88-48B2-A226-AB3234EE506B");
InventoryFolderBase folder1 = new InventoryFolderBase();
folder1.ID = UUID.Parse("F1EAE367-3A88-48B2-A226-AB3234EE506B");
folder1.Name = "Folder1";
folder1.Level = InventoryFolderBase.FolderLevel.TopLevel;
folder1.Owner = userId;
folder1.ParentID = UUID.Zero;
folder1.Type = (short)OpenMetaverse.FolderType.Root;
try
{
_storage.GetFolderAttributes(folder1.ID);
}
catch (InventoryObjectMissingException e)
{
_storage.CreateFolder(folder1);
}
for (int i = 0; i < 100; i++)
{
UUID assetId = UUID.Random();
UUID itemId = UUID.Random();
InventoryItemBase item = new InventoryItemBase
{
AssetID = assetId,
AssetType = (int)OpenMetaverse.AssetType.Texture,
BasePermissions = UInt32.MaxValue,
CreationDate = Util.UnixTimeSinceEpoch(),
CreatorId = userId.ToString(),
CurrentPermissions = unchecked((uint)-1),
Description = "A good item, of goodness",
EveryOnePermissions = Int32.MaxValue + (uint)1,
Flags = unchecked((uint)Int32.MinValue),
Folder = folder1.ID,
GroupID = UUID.Zero,
GroupOwned = false,
GroupPermissions = 0x123,
ID = itemId,
InvType = (int)OpenMetaverse.InventoryType.Texture,
Name = "Item of Goodness",
NextPermissions = 0xF,
Owner = userId,
SalePrice = 100,
SaleType = 1
};
pool.QueueWorkItem(() =>
{
_storage.CreateItem(item);
});
pool.QueueWorkItem(() =>
{
_storage.PurgeItem(item);
});
pool.QueueWorkItem(() =>
{
_storage.CreateItem(item);
});
pool.QueueWorkItem(() =>
{
_storage.PurgeItem(item);
});
pool.WaitForIdle();
InventoryFolderBase newFolder = _storage.GetFolder(folder1.ID);
//either the item should completely exist or not
if (newFolder.Items.Exists((InventoryItemBase litem) => { return litem.ID == itemId; }))
{
Assert.DoesNotThrow(delegate()
{
InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero);
});
//cleanup
_storage.PurgeItem(item);
}
else
{
var ex =
Assert.Throws<InventoryObjectMissingException>(delegate()
{
InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero);
});
Assert.AreEqual("Item was not found in the index", ex.ErrorDetails);
}
}
InventoryFolderBase finalFolder = _storage.GetFolder(folder1.ID);
Assert.AreEqual(0, finalFolder.Items.Count);
}
public void TestItemPurge()
{
UUID userId = UUID.Random();
InventoryFolderBase folder1 = new InventoryFolderBase();
folder1.ID = UUID.Random();
folder1.Name = "Folder1";
folder1.Level = InventoryFolderBase.FolderLevel.TopLevel;
folder1.Owner = userId;
folder1.ParentID = UUID.Zero;
folder1.Type = (short)OpenMetaverse.FolderType.Root;
_storage.CreateFolder(folder1);
Amib.Threading.SmartThreadPool pool = new Amib.Threading.SmartThreadPool(1000, 20);
pool.Start();
for (int i = 0; i < 1000; i++)
{
pool.QueueWorkItem(() =>
{
UUID assetId = UUID.Random();
UUID itemId = UUID.Random();
InventoryItemBase item = new InventoryItemBase
{
AssetID = assetId,
AssetType = (int)OpenMetaverse.AssetType.Texture,
BasePermissions = UInt32.MaxValue,
CreationDate = Util.UnixTimeSinceEpoch(),
CreatorId = userId.ToString(),
CurrentPermissions = unchecked((uint)-1),
Description = "A good item, of goodness",
EveryOnePermissions = Int32.MaxValue + (uint)1,
Flags = unchecked((uint)Int32.MinValue),
Folder = folder1.ID,
GroupID = UUID.Zero,
GroupOwned = false,
GroupPermissions = 0x123,
ID = itemId,
InvType = (int)OpenMetaverse.InventoryType.Texture,
Name = "Item of Goodness",
NextPermissions = 0xF,
Owner = userId,
SalePrice = 100,
SaleType = 1
};
_storage.PurgeItem(item);
_storage.CreateItem(item);
_storage.PurgeItem(item);
_storage.CreateItem(item);
_storage.PurgeItem(item);
var ex =
Assert.Throws<InventoryObjectMissingException>(delegate()
{
InventoryItemBase itemCopy = _storage.GetItem(itemId, UUID.Zero);
});
Assert.AreEqual("Item was not found in the index", ex.ErrorDetails);
});
}
pool.WaitForIdle();
InventoryFolderBase newFolder = _storage.GetFolder(folder1.ID);
Assert.AreEqual(0, newFolder.Items.Count);
Assert.AreEqual(5001, newFolder.Version);
}
public void Initialize(Scene scene, IConfigSource config)
{
m_scene = scene;
m_scene.RegisterModuleInterface<IHttpRequestModule>(this);
m_proxyurl = config.Configs["Startup"].GetString("HttpProxy");
m_proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions");
m_pendingRequests = new Dictionary<UUID, HttpRequestObject>();
m_completedRequests = new Queue<HttpRequestObject>();
m_objectQueueSize = new Dictionary<UUID, short>();
m_objectPriorities = new Dictionary<UUID, TimestampedItem<Amib.Threading.WorkItemPriority>>();
m_lastPriorityMaintenance = Util.GetLongTickCount();
_threadPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_NORMAL_THREADS, 2);
_threadPool.Name = "HttpRequestWorkerNormal";
_slowPool = new Amib.Threading.SmartThreadPool(Amib.Threading.SmartThreadPool.DefaultIdleTimeout, MAX_SLOW_THREADS, 0);
_slowPool.Name = "HttpRequestWorkerSlow";
ReadBlacklistFromConfig(config.Configs["HttpRequest"]);
MainConsole.Instance.Commands.AddCommand(
"Comms",
true,
"httprequest queuelength",
"httprequest queuelength",
"Report on the current size of the request queue.",
"Displays the current size of the request queue.",
HttpRequestConsoleCommand);
MainConsole.Instance.Commands.AddCommand(
"Comms",
true,
"httprequest debug",
"httprequest debug <debuglevel>",
"Enable/disable debugging of the request queue.",
"Enable/disable debugging of the request queue.",
HttpRequestConsoleCommand);
}
static void Main(string[] args)
{
WebRequest.DefaultWebProxy = null;
ServicePointManager.DefaultConnectionLimit = 1000;
int single_id = -1;
bool server = true;
bool is_t = false;
bool autorun = true;
foreach (string arg in args)
{
if (arg.StartsWith("--thread:"))
{
// --thread:board:id
board = arg.Split(':')[1];
single_id = Convert.ToInt32(arg.Split(':')[2]);
}
if (arg == "--noserver")
{
server = false;
}
if (arg.StartsWith("--board:"))
{
board = arg.Split(':')[1];
}
if (arg == "--thumbonly")
{
is_t = true;
}
if (arg == "--verbose")
{
verbose = true;
}
if (arg == "--idle")
{
autorun = false;
}
if (arg.StartsWith("--port:"))
{
Int32.TryParse(arg.Split(':')[1], out port);
}
if (arg.StartsWith("--savedir"))
{
program_dir = args[Array.IndexOf(args, arg) + 1];
}
if (arg == "-help" || arg == "--help" || arg == "/help" || arg == "-h")
{
Console.Write(Properties.Resources.help_text);
return;
}
}
ThreadStore.SetUp(new Thread_Storage.FolderStorageEngine());
thumb_stp = new Amib.Threading.SmartThreadPool() { MaxThreads = 10, MinThreads = 0 };
thumb_stp.Start();
file_stp = new Amib.Threading.SmartThreadPool() { MaxThreads = 5, MinThreads = 0 };
file_stp.Start();
if (string.IsNullOrEmpty(program_dir))
{
program_dir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "chanarchiver");
}
Directory.CreateDirectory(program_dir);
file_save_dir = Path.Combine(program_dir, "files"); Directory.CreateDirectory(file_save_dir);
thumb_save_dir = Path.Combine(program_dir, "thumbs"); Directory.CreateDirectory(thumb_save_dir);
post_files_dir = Path.Combine(program_dir, "posts"); Directory.CreateDirectory(post_files_dir);
api_cache_dir = Path.Combine(program_dir, "aniwrap_cache"); Directory.CreateDirectory(api_cache_dir);
temp_files_dir = Path.Combine(program_dir, "temp"); Directory.CreateDirectory(temp_files_dir);
board_settings_dir = Path.Combine(program_dir, "settings"); Directory.CreateDirectory(board_settings_dir);
invalid_files_dir = Path.Combine(program_dir, "invalid_files"); Directory.CreateDirectory(invalid_files_dir);
html_templates_dir = Path.Combine(program_dir, "html_templates", Version); Directory.CreateDirectory(html_templates_dir);
HtmlTemplates.Init();
Settings.Load();
Settings.ThumbnailOnly = is_t;
if (Settings.CacheAPIFilesInMemory)
{
aw = new AniWrap.AniWrap();
}
else
{
aw = new AniWrap.AniWrap(api_cache_dir);
}
Console.Title = "ChanArchiver";
print("ChanArchiver ", ConsoleColor.Cyan);
Console.Write(Version);
Console.WriteLine(" stable");
if (Environment.OSVersion.Platform == PlatformID.Unix)
{
string[] ffmpegs_paths = { get_ffmpeg_path_unix(), "/bin/ffmpeg", "/usr/bin/ffmpeg", };
foreach (string s in ffmpegs_paths)
{
if (File.Exists(s))
{
ffmpeg_path = s;
Console.Write("Detected ffmpeg path:");
print(string.Format("'{0}'\n", s), ConsoleColor.Yellow);
break;
}
}
}
else
{
ffmpeg_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "ffmpeg.exe");
Console.WriteLine("Make sure to place ffmpeg.exe in your home folder");
}
swf_watch = new FileSystemWatcher(file_save_dir);
swf_watch.EnableRaisingEvents = true;
swf_watch.Filter = "*.swf";
swf_watch.IncludeSubdirectories = false;
swf_watch.Created += new FileSystemEventHandler(handle_new_swf_file);
Console.Write("Downloading board data...");
ValidBoards = aw.GetAvailableBoards();
Console.Write("loaded {0} board.\n", ValidBoards.Count);
Console.Write("Saving files in ");
print(string.Format("'{0}'\n", program_dir), ConsoleColor.Red);
load_settings(autorun);
if (Settings.ThumbnailOnly)
{
print("Warning:", ConsoleColor.Yellow);
Console.WriteLine("ChanArchiver is running in thumbnail only mode");
}
if (server)
{
Console.WriteLine("ChanArchiver is running in server mode");
start_server();
}
if (!string.IsNullOrWhiteSpace(board))
{
if (single_id > 0)
{
archive_single(board, single_id, Settings.ThumbnailOnly);
}
else
{
archive_board(board, BoardWatcher.BoardMode.FullBoard);
}
}
FileIndex.Load();
optimize_directory_struct(thumb_save_dir);
optimize_directory_struct(file_save_dir);
ArchivesProvider.Load();
interactive_console();
save_settings();
FileSystemStats.Dispose();
}
