public ExpirationManagerFacts()
{
_storage = ConnectionUtils.CreateStorage();
_queueProviders = _storage.QueueProviders;
_token = new CancellationToken(true);
}
private void queue_ContentChanged(object sender, QueueChangedEventArgs <QueueModel> args)
{
if (args.Action == QueueChangedActionEnum.Enqueue)
{
pool.QueueWorkItem(() =>
{
try
{
QueueModel qm;
if (queue.Dequeue(out qm))
{
var result = NodeVisitor.Cooperater.GetResult(qm.Url);
if (result != null)
{
var cm = new ContentModel();
cm.FeedId = qm.FeedId;
cm.Url = qm.Url;
cm.Metas = result.Metas;
cm.CDate = DateTime.Now;
var connectString = string.Format(@"LiteDb/Content/{0}.db", DateTime.Now.ToString("yyyyMM"));
var storage = new LiteDbStorage(connectString, "contents");
if (storage.Insert(cm) == -1)
{
File.AppendAllText(path + @"\" + EncryptHelper.GetMD5Hash(qm.Url) + ".json", JsonConvert.SerializeObject(cm));
}
}
}
}
catch {
//save failed
}
});
}
}
public async Task WriteTwoTypesGetSameValues()
{
//ARRANGE
LiteDbStorage storage = new LiteDbStorage();
var storeItems = new Dictionary <string, object>()
{
["createPoco"] = new PocoItem()
{
Id = "1"
},
["createPocoStoreItem"] = new PocoStoreItem()
{
Id = "2"
},
};
//ACT
await storage.WriteAsync(storeItems);
//ASSERT
var readStoreItems = new Dictionary <string, object>(await storage.ReadAsync(storeItems.Keys.ToArray()));
var createPoco = readStoreItems["createPoco"] as PocoItem;
Assert.IsNotNull(createPoco, "createPoco should not be null");
Assert.AreEqual(createPoco.Id, "1", "createPoco.id should be 1");
var createPocoStoreItem = readStoreItems["createPocoStoreItem"] as PocoStoreItem;
Assert.IsNotNull(createPocoStoreItem, "createPocoStoreItem should not be null");
Assert.AreEqual(createPocoStoreItem.Id, "2", "createPocoStoreItem.id should be 2");
//Assert.IsNotNull(createPocoStoreItem.ETag, "createPocoStoreItem.eTag should not be null");
}
public void GetMonitoringApi_ReturnsNonNullInstance()
{
LiteDbStorage storage = ConnectionUtils.CreateStorage();
IMonitoringApi api = storage.GetMonitoringApi();
Assert.NotNull(api);
}
public async Task WriteTwoTypesGetSameTypes()
{
//ARRANGE
LiteDbStorage storage = new LiteDbStorage();
var storeItems = new Dictionary <string, object>()
{
["createPoco"] = new PocoItem()
{
Id = "1"
},
["createPocoStoreItem"] = new PocoStoreItem()
{
Id = "2"
},
};
//ACT
await storage.WriteAsync(storeItems);
//ASSERT
var readStoreItems = new Dictionary <string, object>(await storage.ReadAsync(storeItems.Keys.ToArray()));
Assert.IsInstanceOfType(readStoreItems["createPoco"], typeof(PocoItem));
Assert.IsInstanceOfType(readStoreItems["createPocoStoreItem"], typeof(PocoStoreItem));
}
public void GetConnection_ReturnsNonNullInstance()
{
LiteDbStorage storage = ConnectionUtils.CreateStorage();
using (IStorageConnection connection = storage.GetConnection())
{
Assert.NotNull(connection);
}
}
public LiteDbTestObjectRepository(LiteDbStorage dbLiteStorage) : base(dbLiteStorage, NullLogger.Instance)
{
LiteStorage = dbLiteStorage;
IsReadOnly = true;
AddType((TestEntity x) => new TestModel(x));
AddType((ParentEntity x) => new ParentModel(x));
AddType((ChildEntity x) => new ChildModel(x));
Initialize();
}
/// <summary>
/// Configure Hangfire to use LiteDB storage
/// </summary>
/// <param name="configuration">Configuration</param>
/// <param name="connectionString">Connection string for LiteDB database, for example 'LiteDB://*****:*****@host:port'</param>
/// <param name="storageOptions">Storage options</param>
/// <returns></returns>
public static LiteDbStorage LiteDbStorage(this IGlobalConfiguration configuration,
string connectionString,
LiteDbStorageOptions storageOptions)
{
var storage = new LiteDbStorage(connectionString, storageOptions);
configuration.UseStorage(storage);
return(storage);
}
public static LiteDbStorage GetLiteDbStorage(string connectionString)
{
Log.Information("HangfireLiteDb: {0}", connectionString);
var options = new LiteDbStorageOptions()
{
QueuePollInterval = TimeSpan.FromSeconds(10)
};
var storage = new LiteDbStorage(connectionString, options);
return(storage);
}
public async Task Initialize()
{
storage = new LiteDbStorage();
var dict = new Dictionary <string, object>()
{
{ "delete1", new PocoStoreItem()
{
Id = "1", Count = 1
} },
};
await storage.WriteAsync(dict);
}
public async Task BatchCreateBigObjectsShouldSucceed()
{
LiteDbStorage storage = new LiteDbStorage();
string[] stringArray = GenerateExtraBytes(23);
var storeItemsList = new List <Dictionary <string, object> >(new[]
{
new Dictionary <string, object> {
["createPoco"] = new PocoItem()
{
Id = "1", Count = 0, ExtraBytes = stringArray
}
},
new Dictionary <string, object> {
["createPoco"] = new PocoItem()
{
Id = "1", Count = 1, ExtraBytes = stringArray
}
},
new Dictionary <string, object> {
["createPoco"] = new PocoItem()
{
Id = "1", Count = 2, ExtraBytes = stringArray
}
},
});
// TODO: this code as a generic test doesn't make much sense - for now just eliminating the custom exception
// Writing large objects in parallel might raise an InvalidOperationException
//await Task.WhenAll(
// storeItemsList.Select(storeItems =>
// Task.Run(async () => await storage.WriteAsync(storeItems))));
Parallel.Invoke(
async() => { await storage.WriteAsync(storeItemsList[0]); },
async() => { await storage.WriteAsync(storeItemsList[1]); },
async() => { await storage.WriteAsync(storeItemsList[2]); }
); //close parallel.invoke
var readStoreItems = new Dictionary <string, object>(await storage.ReadAsync(new[] { "createPoco" }));
Assert.IsInstanceOfType(readStoreItems["createPoco"], typeof(PocoItem));
var createPoco = readStoreItems["createPoco"] as PocoItem;
Assert.AreEqual(createPoco.Id, "1", "createPoco.id should be 1");
}
public static LiteDbStorage GetLiteDbStorage()
{
var connectionString = BotSettings.HangfireLiteDb;
Log.Information($"HangfireLiteDb: {connectionString}");
var options = new LiteDbStorageOptions()
{
QueuePollInterval = TimeSpan.FromSeconds(10)
};
var storage = new LiteDbStorage(connectionString, options);
return(storage);
}
public bool SaveContent([FromBody] ContentModel content, string shard = "")
{
try
{
if (string.IsNullOrEmpty(shard))
{
shard = DateTime.Now.ToString("yyyyMM");
}
var storage = new LiteDbStorage(@"LiteDb/Content/" + shard + ".db", "contents");
return(storage.Insert(content) != -1);
}
catch
{
return(false);
}
}
public async Task UpdateTwoTypesGetUpdatedValues()
{
LiteDbStorage storage = new LiteDbStorage();
var originalPocoItem = new PocoItem()
{
Id = "1", Count = 1
};
var originalPocoStoreItem = new PocoStoreItem()
{
Id = "1", Count = 1
};
// first write should work
var dict = new Dictionary <string, object>()
{
{ "pocoItem", originalPocoItem },
{ "pocoStoreItem", originalPocoStoreItem },
};
await storage.WriteAsync(dict);
var loadedStoreItems = new Dictionary <string, object>(await storage.ReadAsync(new[] { "pocoItem", "pocoStoreItem" }));
var updatePocoItem = loadedStoreItems["pocoItem"] as PocoItem;
var updatePocoStoreItem = loadedStoreItems["pocoStoreItem"] as PocoStoreItem;
//Assert.IsNotNull(updatePocoStoreItem.ETag, "updatePocoItem.eTag should not be null");
// 2nd write should work, because we have new etag, or no etag
updatePocoItem.Count++;
updatePocoStoreItem.Count++;
await storage.WriteAsync(loadedStoreItems);
var reloadedStoreItems = new Dictionary <string, object>(await storage.ReadAsync(new[] { "pocoItem", "pocoStoreItem" }));
var reloeadedUpdatePocoItem = reloadedStoreItems["pocoItem"] as PocoItem;
var reloadedUpdatePocoStoreItem = reloadedStoreItems["pocoStoreItem"] as PocoStoreItem;
//Assert.IsNotNull(reloadedUpdatePocoStoreItem.ETag, "updatePocoItem.eTag should not be null");
//Assert.AreNotEqual(updatePocoStoreItem.ETag, reloadedUpdatePocoStoreItem.ETag, "updatePocoItem.eTag should not be different");
Assert.AreEqual(2, reloeadedUpdatePocoItem.Count, "updatePocoItem.Count should be 2");
Assert.AreEqual(2, reloadedUpdatePocoStoreItem.Count, "updatePocoStoreItem.Count should be 2");
}
protected override ObjectRepositoryBase CreateRepository()
{
var db = new LiteDatabase(_memory);
var dbStorage = new LiteDbStorage(db);
var objectRepo = new LiteDbTestObjectRepository(dbStorage);
objectRepo.OnException += ex => Console.WriteLine(ex.ToString());
objectRepo.WaitForInitialize().GetAwaiter().GetResult();
if (firstTime)
{
firstTime = false;
objectRepo.Add(_testModel);
objectRepo.Add(_parentModel);
objectRepo.Add(_childModel);
GetStorage(objectRepo).SaveChanges().GetAwaiter().GetResult();
}
return(objectRepo);
}
private ObjectRepository ConfigureObjectRepository(IServiceCollection services)
{
var liteDb = Configuration.GetConnectionString("LiteDb");
if (String.IsNullOrEmpty(liteDb))
{
throw new Exception("Connection string for 'LiteDb' should been specified.");
}
var connectionString = new ConnectionString(liteDb)
{
Upgrade = true
};
DbFileName = connectionString.Filename;
var liteDbDatabase = new LiteDatabase(connectionString);
IStorage storage = new LiteDbStorage(liteDbDatabase);
var objectRepository = new ObjectRepository(storage, NullLoggerFactory.Instance);
services.AddSingleton(storage);
services.AddSingleton(objectRepository);
return(objectRepository);
}
public async Task HandleCrazyKeys()
{
LiteDbStorage storage = new LiteDbStorage();
var key = "!@#$%^&*()~/\\><,.?';\"`~";
var storeItem = new PocoStoreItem()
{
Id = "1"
};
var dict = new Dictionary <string, object>()
{
{ key, storeItem }
};
await storage.WriteAsync(dict);
var storeItems = await storage.ReadAsync(new[] { key });
storeItem = storeItems.FirstOrDefault(si => si.Key == key).Value as PocoStoreItem;
Assert.IsNotNull(storeItem);
Assert.AreEqual("1", storeItem.Id);
}
public void ConfigureServices(IServiceCollection services)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
services.AddResponseCompression(x => x.EnableForHttps = true);
services.AddMvc().AddNewtonsoftJson(options =>
{
options.SerializerSettings.NullValueHandling = NullValueHandling.Include;
options.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
options.SerializerSettings.Formatting = IsProduction ? Formatting.None : Formatting.Indented;
});
services.AddSingleton <Chrome>();
services.AddSingleton <ScrapeService>();
ObjectRepository objectRepository;
{
var liteDb = Configuration.GetConnectionString("LiteDb");
var azureDb = Configuration.GetConnectionString("AzureStorage");
IStorage storage = null;
if (!String.IsNullOrEmpty(liteDb))
{
var connectionString = new ConnectionString(liteDb);
DbFileName = connectionString.Filename;
var liteDbDatabase = new LiteDatabase(connectionString);
liteDbDatabase.Engine.Shrink();
storage = new LiteDbStorage(liteDbDatabase);
}
else if (!String.IsNullOrEmpty(azureDb))
{
var cloudStorageAccount = CloudStorageAccount.Parse(azureDb);
storage = new AzureTableContext(cloudStorageAccount.CreateCloudTableClient());
}
else
{
throw new Exception("Connection string for either 'AzureStorage' or 'LiteDb' should been specified.");
}
objectRepository = new ObjectRepository(storage, NullLoggerFactory.Instance);
services.AddSingleton(storage);
services.AddSingleton(objectRepository);
}
var scrapers = GetType().Assembly.GetTypes().Where(v => v.IsSubclassOf(typeof(GenericScraper))).ToList();
foreach (var s in scrapers)
{
services.AddSingleton(typeof(GenericScraper), s);
}
services.AddWebEncoders(o =>
{
var textEncoderSettings = new TextEncoderSettings();
textEncoderSettings.AllowRange(UnicodeRanges.All);
o.TextEncoderSettings = textEncoderSettings;
});
services.AddTransient(x => new TableViewModelFactory(x.GetRequiredService <ObjectRepository>()));
services.AddSingleton <ScriptService>();
services.AddSingleton <SmsRuleProcessor>();
services.AddSingleton <UpdateService>();
services.AddLogging();
services.AddSession();
services.AddControllers().AddNewtonsoftJson();
services.AddHangfire(x => { });
services.AddDataProtection().AddKeyManagementOptions(options =>
{
options.XmlRepository = new ObjectRepositoryXmlStorage(objectRepository);
});
services.AddAuthorization();
services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme).AddCookie(x =>
{
x.AccessDeniedPath = "/Auth";
x.LoginPath = "/Auth";
x.LogoutPath = "/Auth/Logout";
});
}
