public async void ListDownloadTest()
{
// Downlad galleries
listDownloader = new ListDownloader();
listDownloader.ListDownloadProgress += InitializationTest_onProgress;
List <Gallery> galleries = new List <Gallery>();
galleries.AddRange(await listDownloader.Download());
// Check whether list is downloaded
Assert.True(galleries.Count > 456900);
Assert.DoesNotContain(null, galleries);
// Check whether gallery information is correct
Gallery gallery = galleries.Find(v => v.Id == 1277807);
Assert.Equal(GalleryCrawlMethod.Normal, gallery.GalleryCrawlMethod);
Assert.Equal(1277807, gallery.Id);
Assert.Equal("doujinshi", gallery.Type);
Assert.Equal("Oidemase!! 2-jigen Fuuzoku Gakuen Dai 2 Kukaku", gallery.Name);
Assert.Equal(new string[] { "nyuu" }, gallery.Artists);
Assert.Equal(new string[] { "nyuu koubou" }, gallery.Groups);
Assert.Equal(new string[] { "elf yamada", "haruhi suzumiya", "lyfa", "masamune izumi", "muramasa senju", "sagiri izumi", "shino asada", "suguha kirigaya" }.Sort(), gallery.Characters.Sort());
Assert.Equal(new string[] { "eromanga sensei", "nier automata", "ranma 12", "sword art online", "the idolmaster", "the melancholy of haruhi suzumiya", "to love-ru", "urusei yatsura", "yuru camp" }.Sort(), gallery.Parodies.Sort());
Assert.Equal(new string[] { "female:anal", "female:body writing", "female:bunny girl", "female:mind control", "female:piercing", "full color" }.Sort(), gallery.Tags.Sort());
logger.WriteLine("Has correct gallery");
}
public GalleriesFixture()
{
ListDownloader listDownloader = new ListDownloader();
var listDownloadTask = listDownloader.Download();
listDownloadTask.Wait();
galleries = new List <Gallery>(listDownloadTask.Result);
}
private void createObjects()
{
CreateStations();
ListDownloader downloader;
foreach (var station in StationsToSave)
{
downloader = new ListDownloader(_appDbContext);
var ReceivedData = downloader.GetStationInfo(station.Link);
ReceivedData.Name = station.Name;
var rgx = new Regex(@"(?:0?[0-9]|1[0-9]|2[0-3]):[0-5][0-9]?");
var stringToCut = rgx.Match(ReceivedData.NextStations[ReceivedData.NextStations.Count - 1]).ToString();
ReceivedData.Direction = ReceivedData.NextStations[ReceivedData.NextStations.Count - 1].Replace(stringToCut + " ", "");
StationsToPass.Add(ReceivedData);
}
}
static void Main(string[] args)
{
try
{
#region Tool modes
String toolProfileDownload = "pd";
String toolProfileCrawl = "pc";
String toolListProfileDownload = "lpd";
String toolLocationCrawl = "lc";
#endregion
#region Call timings and limits
int minimumDelayBetweenPages = 200;
int minimumDelayBetweenAPICalls = 250;
int maximumCrawlDepth = 2;
#endregion
CommandLineArgument toolArg = new CommandLineArgument("t", "tool");
CommandLineArgument userNameArg = new CommandLineArgument("u", "username");
CommandLineArgument hashKeyArg = new CommandLineArgument("h", "hashkey");
CommandLineArgument storeDirectoryArg = new CommandLineArgument("s", "store_directory");
CommandLineArgument capturePhotosArg = new CommandLineArgument("c", "capture_photos");
CommandLineArgument captureConnectionsArg = new CommandLineArgument("n", "capture_connections");
CommandLineArgument delayBetweenPagesArg = new CommandLineArgument("dbp", "delay_between_pages");
CommandLineArgument delayBetweenAPICallsArg = new CommandLineArgument("dba", "delay_between_api_calls");
CommandLineArgument crawlDepthArg = new CommandLineArgument("d", "crawl_depth");
CommandLineArgument usernameListPathArg = new CommandLineArgument("l", "username_list_path");
CommandLineArgument locationCriteriaArg = new CommandLineArgument("o", "location_criteria");
CommandLineArgument allowEmptyLocationsArg = new CommandLineArgument("e", "allow_empty_locations");
List <CommandLineArgument> argumentList = new List <CommandLineArgument>()
{
toolArg,
userNameArg,
hashKeyArg,
storeDirectoryArg,
capturePhotosArg,
captureConnectionsArg,
delayBetweenPagesArg,
delayBetweenAPICallsArg,
crawlDepthArg,
usernameListPathArg,
locationCriteriaArg,
allowEmptyLocationsArg
};
#region Print argument help if no arguments.
if (args.Length == 0)
{
ArgumentHelp();
}
#endregion
#region Set default arguments
//Default to profile download tool
toolArg.Value = toolProfileDownload;
//Blank hashkey
hashKeyArg.Value = "";
//Get store directory from app file if available, otherwise use current working directory.
String storeDirectoryApp = ConfigurationManager.AppSettings["store_directory"];
storeDirectoryArg.Value = !String.IsNullOrEmpty(storeDirectoryApp) ? storeDirectoryApp
: Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
capturePhotosArg.Value = false.ToString();
captureConnectionsArg.Value = false.ToString();
//Keep delays to calls to minimum
delayBetweenPagesArg.Value = minimumDelayBetweenPages.ToString();
delayBetweenAPICallsArg.Value = minimumDelayBetweenAPICalls.ToString();
crawlDepthArg.Value = 2.ToString(); //Keep default crawl depth to minimum
//Default to no location criteria
locationCriteriaArg.Value = null;
//Default to allow empty locations
allowEmptyLocationsArg.Value = true.ToString();
#endregion
//Parse arugments
if (args.Length > 0)
{
int argumentIndex = 0;
while (argumentIndex < args.Length)
{
CommandLineArgument matchedArguments = argumentList
.Where(x => x.ShortName.Equals(args[argumentIndex]) || x.LongName.Equals(args[argumentIndex]))
.FirstOrDefault();
if (matchedArguments != null)
{
if (argumentIndex < args.Length - 1)
{
//Found an argument, get the value and skip to next position
matchedArguments.Value = args[argumentIndex + 1];
argumentIndex = argumentIndex + 2;
}
else
{
//Argument with no value
Console.WriteLine(String.Format("Argument with no value: {0}", args[argumentIndex]));
Environment.Exit(1);
}
}
else
{
//Invalid argument, ignore for now
Console.WriteLine(String.Format("Invalid argument: {0}", args[argumentIndex]));
if (argumentIndex < args.Length)
{
argumentIndex++;
}
}
}
}
#region Process defaults that may be in the .app file.
if (String.IsNullOrEmpty(usernameListPathArg.Value))
{
//Default to list.txt in store directory for list path
usernameListPathArg.Value = Path.Combine(storeDirectoryArg.Value, "list.txt");
}
if (String.IsNullOrEmpty(hashKeyArg.Value))
{
//Get hashkey from app file if available.
hashKeyArg.Value = ConfigurationManager.AppSettings["hashkey"];
}
#endregion
//Verify arguments and launch tools
if (String.IsNullOrEmpty(toolArg.Value) || toolArg.Value.Equals(toolProfileDownload)) //Default to profile download
{
#region Profile Download
if (String.IsNullOrEmpty(userNameArg.Value))
{
ArgumentError(userNameArg);
return;
}
ProfileDownloader pd = new ProfileDownloader(
userNameArg.Value,
storeDirectoryArg.Value,
hashKeyArg.Value,
capturePhotosArg.AsBool != null ? (bool)capturePhotosArg.AsBool : false,
captureConnectionsArg.AsBool != null ? (bool)captureConnectionsArg.AsBool : false);
pd.Download();
#endregion
}
else if (toolArg.Value.Equals(toolProfileCrawl))
{
#region Profile Crawl
if (String.IsNullOrEmpty(userNameArg.Value))
{
ArgumentError(userNameArg);
return;
}
ProfileCrawler pc = new ProfileCrawler(
storeDirectoryArg.Value,
hashKeyArg.Value,
userNameArg.Value,
delayBetweenPagesArg.AsInt != null ? Math.Max((int)delayBetweenPagesArg.AsInt, minimumDelayBetweenPages) : minimumDelayBetweenPages,
delayBetweenAPICallsArg.AsInt != null ? Math.Max((int)delayBetweenAPICallsArg.AsInt, minimumDelayBetweenAPICalls) : minimumDelayBetweenAPICalls,
crawlDepthArg.AsInt != null ? Math.Min((int)crawlDepthArg.AsInt, maximumCrawlDepth) : 2,
locationCriteriaArg.Value,
allowEmptyLocationsArg.AsBool != null ? (bool)allowEmptyLocationsArg.AsBool : true,
capturePhotosArg.AsBool != null ? (bool)capturePhotosArg.AsBool : false
);
pc.Crawl();
#endregion
}
else if (toolArg.Value.Equals(toolListProfileDownload))
{
#region Download Profile List
if (String.IsNullOrEmpty(usernameListPathArg.Value))
{
ArgumentError(usernameListPathArg);
return;
}
ListDownloader ld = new ListDownloader(
usernameListPathArg.Value,
storeDirectoryArg.Value,
hashKeyArg.Value,
capturePhotosArg.AsBool != null ? (bool)capturePhotosArg.AsBool : false,
captureConnectionsArg.AsBool != null ? (bool)captureConnectionsArg.AsBool : false
);
ld.Download();
#endregion
}
else if (toolArg.Value.Equals(toolLocationCrawl))
{
#region Crawl for locations in connections from directory of existing profiles
LocationCrawler lc = new LocationCrawler(
storeDirectoryArg.Value,
hashKeyArg.Value
);
lc.Crawl();
#endregion
}
else
{
//Not a recognized tool
ArgumentError(toolArg);
return;
}
Environment.Exit(0);
}
catch (Exception appExc)
{
Console.WriteLine(String.Format("Error: {0}{1}{2}", appExc.Message, Environment.NewLine, appExc.StackTrace));
Environment.Exit(1);
}
}
