public static Int64 GetObjectIDForSpotifyID(string sSpotifyID)
{
Int64 iObjectID;
iObjectID = CacheDatabase.GetObjectIDForSpotifyID(sSpotifyID);
return(iObjectID);
}
public static void UpdateObjectIDForSpotifyObject(SpotifyObjectBase oSpotifyObject)
{
if (oSpotifyObject.ID == 0)
{
oSpotifyObject.ID = CacheDatabase.GetObjectIDForSpotifyID(oSpotifyObject.SpotifyID);
}
}
void MakeEnvironment(string dbName,
Action <CacheDatabase, WorkerConfiguration, WorkManager, int> action,
bool depthSearch = false, int memLimit = 100000)
{
var database = new CacheDatabase(dbName);
database.EnsureNew();
var config = new WorkerConfiguration();
config.DepthSearch = depthSearch;
var wm = new WorkManager(config, database, memLimit);
try
{
action(database, config, wm, memLimit);
}
finally
{
database.Dispose();
database.Delete();
}
}
public DownloadCache(DirectoryInfo directory = null, bool createIfDoesNotExist = true)
{
directory ??= StorageHelper.GetTempDirectory();
CacheDatabase = new(Path.Combine(directory.FullName, CACHE_FILENAME));
if (!CacheDatabase.Exists && createIfDoesNotExist)
{
using BinaryWriter cache = new(CacheDatabase.Open(FileMode.CreateNew));
cache.Write(FILE_MAGIC);
}
}
public void SetUp()
{
var options = new DbContextOptionsBuilder <CurrencyDbContext>()
.UseInMemoryDatabase(databaseName: "CurrencyListDatabase")
.ConfigureWarnings(x => x.Ignore(InMemoryEventId.TransactionIgnoredWarning))
.Options;
_dbContext = new CurrencyDbContext(options);
_holidayCheckerMock = new Mock <IHolidayChecker>();
_cacheDatabase = new CacheDatabase(_dbContext, _holidayCheckerMock.Object);
}
public void CanGetTableNames()
{
CacheConnectionStrings connectionStrings = _serviceRegistry.Get <CacheConnectionStrings>();
CacheSchemaExtractorConfig config = new CacheSchemaExtractorConfig {
TableNameFilter = "%EnsLib.Workflow%", ConnectionString = connectionStrings.WA
};
CacheDatabase db = new CacheDatabase(connectionStrings.WA);
CacheSchemaExtractor schemaExtractor = new CacheSchemaExtractor(config);
string[] tableNames = schemaExtractor.GetTableNames();
Expect.IsNotNull(tableNames, "tableNames was null");
Expect.IsTrue(tableNames.Length > 0);
}
public HostProgram(Configuration config, PluginManager plugins)
{
this.config = config;
this.plugins = plugins;
database = new CacheDatabase(config.CacheFilename);
workmanager = new WorkManager(config.WorkerConfig, database, plugins, () =>
{
return(workermanager.Clients.Count(x => x.Online && string.IsNullOrEmpty(x.AssignedUrl))
< workermanager.Clients.Count(x => x.Online));
});
workermanager = new WorkerManager(workmanager, config, new FreeWorkerPicker(), plugins);
webgui = new WebGUI(new IPEndPoint(IPAddress.Parse(config.WebGUI.IP), config.WebGUI.Port), new HostResponder(config, workmanager, workermanager));
workmanager.HostMessageReceived += Workmanager_HostMessageReceived;
}
public WorkerProgram(Configuration config, PluginManager plugins)
{
this.config = config;
this.plugins = plugins;
database = new CacheDatabase(config.CacheFilename);
workmanager = new WorkManager(config.WorkerConfig, database, plugins, () =>
{
return(crawler.CurrentTasks.Count(x => x.Value == null) < crawler.CurrentTasks.Count);
});
crawler = new Crawler(workmanager, config.WorkerConfig, plugins);
webgui = new WebGUI(new IPEndPoint(IPAddress.Parse(config.WebGUI.IP), config.WebGUI.Port), new WorkerResponder(config, crawler));
// register events
workmanager.HostMessageReceived += Workmanager_HostMessageReceived;
crawler.StateChanged += Crawler_StateChanged;
}
public WorkManager(WorkerConfiguration config, CacheDatabase database, int newMemoryLimitCount,
PluginManager plugins = null, Func <bool> areWorkersActive = null)
: this(config, database, plugins, areWorkersActive) => MemoryLimitCount = newMemoryLimitCount;
public void FilenamesTest()
{
var config = new Configuration();
var database = new CacheDatabase("filenames_test");
database.EnsureNew();
var manager = new WorkManager(config.WorkerConfig, database);
var crawler = new Crawler(manager, config.WorkerConfig);
// CHECK FILES
var f1 = crawler.GetFilename("https://www.youtube.com/playlist?list=PLJcysJ8SvKlA4NQGW_Qh-UF7sdpC0Mo", "text/html");
Assert.Equal("playlist.htm", f1);
var f2 = crawler.GetFilename("https://www.facebook.com/", "text/html");
Assert.EndsWith(".htm", f2);
var f3 = crawler.GetFilename("https://www.reddit.com/", "application/zip");
Assert.EndsWith(".zip", f3);
var f4 = crawler.GetFilename("https://stackoverflow.com/questions/2435594/net-path", "application/javascript");
Assert.Equal("net-path.js", f4);
var f5 = crawler.GetFilename("https://outlook.live.com/mail/inbox", "text/css");
Assert.Equal("inbox.css", f5);
var f6 = crawler.GetFilename("https://pbs.twimg.com/media/D8NsdsdsAIA8vP.png:large", "image/png");
Assert.Equal("D8NsdsdsAIA8vP.png", f6);
var f7 = crawler.GetFilename("https://pbs.twimg.com/D8NsdsdsAIA8vP.png", "image/png");
Assert.Equal("D8NsdsdsAIA8vP.png", f7);
var f8 = crawler.GetFilename("https://pbs.twimg.com/D8NsdsdsAIA8vP", "image/png");
Assert.Equal("D8NsdsdsAIA8vP.png", f8);
var f9 = crawler.GetFilename("https://cryshana.me/viewer/bsrhroae1pd.jpg?d=true", "image/jpeg");
Assert.Equal("bsrhroae1pd.jpg", f9);
// CHECK DIRECTORIES
var d1 = crawler.GetDirectoryPath("https://www.youtube.com/playlist?list=PLJcysJ8SvKlA4NQGW_Qh-UF7sdpC0Mo");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\www.youtube.com", d1);
var d2 = crawler.GetDirectoryPath("https://www.facebook.com/");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\www.facebook.com", d2);
var d3 = crawler.GetDirectoryPath("https://www.reddit.com/");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\www.reddit.com", d3);
var d4 = crawler.GetDirectoryPath("https://stackoverflow.com/questions/2435594/net-path");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\stackoverflow.com\\questions\\2435594", d4);
var d5 = crawler.GetDirectoryPath("https://outlook.live.com/mail/inbox");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\outlook.live.com\\mail", d5);
var d6 = crawler.GetDirectoryPath("https://pbs.twimg.com/media/D8NsdsdsAIA8vP.png:large");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\pbs.twimg.com\\media", d6);
var d7 = crawler.GetDirectoryPath("https://pbs.twimg.com/D8Nwd7lUcAIA8vP.png");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\pbs.twimg.com", d7);
var d8 = crawler.GetDirectoryPath("https://pbs.twimg.com/D8Nwd7lUcAIA8vP");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\pbs.twimg.com", d8);
var d9 = crawler.GetDirectoryPath("https://cryshana.me/viewer/bsrhroae1pd.jpg?d=true");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\cryshana.me\\viewer", d9);
var d10 = crawler.GetDirectoryPath("https://cryshana.me");
Assert.Equal($"{config.WorkerConfig.DownloadsPath}\\cryshana.me", d10);
database.Dispose();
database.Delete();
}
