public NameValueCollection GetConfiguration()
{
var connectionString = _configurationProvider.Get(ConfigurationKeys.QuartzSqlConnection);
var instanceName = _configurationProvider.Get(ConfigurationKeys.InstanceName);
var collection = new NameValueCollection
{
{ InstanceNameKey, instanceName },
{ InstanceIdKey, InstanceId },
{ ThreadPoolTypeKey, "Quartz.Simpl.SimpleThreadPool, Quartz" },
{ ThreadPoolThreadCountKey, ThreadCount.ToString() },
{ QuartzSerializerTypeKey, "json" },
{ JobStoreMisforeThresholdKey, "60000" },
{ JobStoreTypeKey, "Quartz.Impl.AdoJobStore.JobStoreTX, Quartz" },
{ JobStoreUsePropertiesKey, "true" },
{ ClusteredKey, "true" },
{ JobStoreDataSourceKey, DataSource },
{ JobStoreTablePrefixKey, "QRTZ_" },
{ LockHandlerTypeKey, "Quartz.Impl.AdoJobStore.UpdateLockRowSemaphore, Quartz" },
{ QuartzConnectionString, connectionString },
{ QuartzDefaultProvider, Provider },
{ JobSuccessMessageKey, JobSuccessMessageValue },
{ JobFailureMessageKey, JobFailureMessageValue },
{ HistoryPluginTypeKey, HistoryPluginTypeValue }
};
return(collection);
}
public async Task <ResponseModel> Write(string collectionId, HttpRequest request)
{
var type = request.Query["type"].ToString();
var typeId = type.ToHash();
var path = Path.Combine(_config.Get("data_dir"), $"{typeId}.{collectionId}.rocks");
var id = request.Query.ContainsKey("id") ?
BitConverter.GetBytes(long.Parse(request.Query["id"])) :
BitConverter.GetBytes(Guid.NewGuid().ToString().ToHash());
var requestStream = new MemoryStream();
await request.Body.CopyToAsync(requestStream);
_writeLock.WaitOne();
DoWrite(collectionId.ToHash(), typeId, id, requestStream.ToArray());
_writeLock.Release();
var response = new MemoryStream();
response.Write(id);
return(new ResponseModel {
Stream = response, MediaType = "application/octet-stream"
});
}
public ScoredResult Reduce(Query queryExpression)
{
var endpoint = _config.Get("postings_endpoint");
var url = string.Format("{0}{1}?skip={2}&take={3}",
endpoint, _collectionName, queryExpression.Skip, queryExpression.Take);
var request = (HttpWebRequest)WebRequest.Create(url);
var query = queryExpression.ToStream();
request.ContentType = "application/query";
request.Accept = "application/postings";
request.Method = WebRequestMethods.Http.Put;
request.ContentLength = query.Length;
using (var requestBody = request.GetRequestStream())
{
requestBody.Write(query, 0, query.Length);
using (var response = (HttpWebResponse)request.GetResponse())
{
var result = new Dictionary <long, float>();
int total = 0;
using (var body = response.GetResponseStream())
{
var mem = new MemoryStream();
body.CopyTo(mem);
var buf = mem.ToArray();
if (response.ContentLength != buf.Length)
{
throw new DataMisalignedException();
}
var read = 0;
while (read < buf.Length)
{
var docId = BitConverter.ToInt64(buf, read);
read += sizeof(long);
var score = BitConverter.ToSingle(buf, read);
read += sizeof(float);
result.Add(docId, score);
}
total = int.Parse(response.Headers["X-Total"]);
}
return(new ScoredResult {
Documents = result, Total = total
});
}
}
}
public ILEmitter Emit(ILEmitter il)
{
var config = _configuration.Get(_variable.OwnerType);
return(il.Ldc_I8(config.HashSeed)
.Stloc(typeof(long), out var hash)
.Emit(this.Emit(hash)));
}
private Stream CreateReadableWritableDataStream(ulong collectionId)
{
var fileName = Path.Combine(_config.Get("data_dir"), string.Format(DataFileNameFormat, collectionId));
var stream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
return(Stream.Synchronized(stream));
}
public static byte[] TakeScreenshot(
this IWebElement element,
IWebDriver driver,
IConfigurationProvider configuration)
{
var pageSize = driver.PageSize();
var browserSize = driver.ViewPort();
var screenshotConfiguration = configuration.Get(ScreenshotTakerConfiguration.Default);
byte[] imageBytes;
if (pageSize.Width < browserSize.Width && pageSize.Height < browserSize.Height)
{
imageBytes = ((ITakesScreenshot)driver).GetScreenshot().AsByteArray;
}
else
{
using (var screenshotTaker = new ScreenshotTaker(driver, screenshotConfiguration))
imageBytes = screenshotTaker.TakeImage();
}
var elementArea = element.Area();
elementArea.Inflate(screenshotConfiguration.Margin, screenshotConfiguration.Margin);
return(imageBytes.Crop(elementArea));
}
private async Task Send(dynamic eventMessage)
{
var time = Stopwatch.StartNew();
var token = _config.Get("slack_token");
var url = $"{_postMessageUrl}?token={token}&channel={eventMessage.channel}&text={Uri.EscapeDataString((string)eventMessage.text)}";
var req = (HttpWebRequest)WebRequest.Create(url);
req.ContentType = "application/x-www-form-urlencoded";
req.Method = "POST";
using (var res = (HttpWebResponse)await req.GetResponseAsync())
{
this.LogJ($"sent {url}");
if (res.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"HTTP {res.StatusCode} {res.StatusCode}");
}
using (var body = res.GetResponseStream())
{
var responseMessage = Deserialize <dynamic>(body);
this.LogJ($"response {responseMessage}");
}
}
}
public async Task <string> Delete()
{
var customers = kenticoSiteProvider.GetSites();
var now = GetCurrentTime();
var tasks = new List <Task <BaseResponseDto <object> > >();
foreach (var customer in customers)
{
var config = configurationProvider.Get <MailingListConfiguration>(customer.Name);
if (config.DeleteMailingListsPeriod != null)
{
var deleteOlderThan = now.AddDays(-config.DeleteMailingListsPeriod.Value);
tasks.Add(new MailingListClient(new StaticMicroProperties(customer.Name)).RemoveMailingList(deleteOlderThan));
}
}
var results = await Task.WhenAll(tasks).ConfigureAwait(false);
if (results.Any(r => !r.Success))
{
return(CreateErrorMessageFromResponses(results, customers));
}
return("Done");
}
private ArrayHasher(HasherResolver resolver, IConfigurationProvider configuration, IVariable variable)
{
_variable = variable;
_arrayHashEmitter = new ArrayHashEmitter(resolver, variable);
_configuration = configuration.Get(variable.OwnerType);
_hasCustomComparer = configuration.HasCustomComparer(variable.VariableType.GetElementType());
}
public override void OnActionExecuted(ActionExecutedContext context)
{
long? docCount = null;
string fileName = Path.Combine(_config.Get("data_dir"), "6604389855880847730.docs");
if (System.IO.File.Exists(fileName))
{
docCount = new FileInfo(fileName).Length / 111;
}
ViewData["doc_count"] = docCount;
ViewBag.DefaultCollection = _config.Get("default_collection");
ViewBag.Collection = context.HttpContext.Request.Query.ContainsKey("collection") ?
context.HttpContext.Request.Query["collection"].ToString() :
ViewBag.DefaultCollection;
base.OnActionExecuted(context);
}
//public async Task Concat(IDictionary<long, IList<long>> offsets)
//{
// foreach (var offset in offsets)
// {
// var canonical = offset.Key;
// var batch = new Dictionary<long, IList<long>>();
// batch.Add(canonical, offset.Value);
// await ExecuteConcat(batch);
// }
//}
public async Task Concat(IDictionary <long, IList <long> > offsets)
{
if (offsets == null)
{
throw new ArgumentNullException(nameof(offsets));
}
var timer = Stopwatch.StartNew();
var requestMessage = new MemoryStream();
foreach (var offset in offsets)
{
// write key
requestMessage.Write(BitConverter.GetBytes(offset.Key));
// write count
requestMessage.Write(BitConverter.GetBytes(offset.Value.Count));
foreach (var offs in offset.Value)
{
// write data
requestMessage.Write(BitConverter.GetBytes(offs));
}
}
var messageBuf = requestMessage.ToArray();
var compressed = QuickLZ.compress(messageBuf, 3);
var endpoint = string.Format("{0}{1}?concat=true",
_config.Get("postings_endpoint"), _collectionName);
var request = (HttpWebRequest)WebRequest.Create(endpoint);
request.Method = WebRequestMethods.Http.Post;
request.ContentType = "application/postings";
using (var requestBody = await request.GetRequestStreamAsync())
{
requestBody.Write(compressed, 0, compressed.Length);
using (var response = (HttpWebResponse)await request.GetResponseAsync())
{
this.Log(string.Format("{0} concat operation took {1}", _collectionName, timer.Elapsed));
}
}
}
public RocksDbStore(IConfigurationProvider config)
{
_dir = config.Get("data_dir");
var options = new DbOptions().SetCreateIfMissing(true);
_db = RocksDbSharp.RocksDb.Open(options, _dir);
}
public async Task <IActionResult> Callback([FromQuery] string code)
{
var accessToken = await _securityProvider.GetAccessToken(code);
var baseAddress = _configurationProvider.Get("uiBaseAddress");
return(Redirect($"{baseAddress}#token={accessToken}"));
}
public ActionResult Index()
{
var applicationConfiguration = _configurationProvider.Get <Core.Configuration.ApplicationConfiguration>();
ViewBag.isUsersAcceptanceNeed = applicationConfiguration.IsUsersAcceptanceNeed;
return(View());
}
private bool IsValidToken(string accessToken)
{
if (string.IsNullOrWhiteSpace(accessToken))
{
return(false);
}
return(_config.Get("admin_password").Equals(accessToken));
}
public static StringHasher Create(IConfigurationProvider configuration, IVariable variable)
{
if (variable.VariableType == typeof(string))
{
var stringComparisonType = configuration.Get(variable.OwnerType).StringComparisonType;
return(new StringHasher(stringComparisonType, variable));
}
return(null);
}
private StringsComparison(
MethodInfo compareMethod,
EmitterDelegate checkForIntermediateResultEmitter,
IConfigurationProvider configuration,
IVariable variable)
{
_compareMethod = compareMethod;
_checkForIntermediateResultEmitter = checkForIntermediateResultEmitter;
_variable = variable;
_stringComparisonType = (int)configuration.Get(_variable.OwnerType).StringComparisonType;
}
private static void CheckConfigurationType(IConfigurationProvider provider, Type type, List <string> errors)
{
try
{
provider.Get(type);
}
catch (Exception error)
{
errors.Add(error.InnerException?.Message ?? error.Message);
}
}
public IWebDriver Activate(IConfigurationProvider provider)
{
var configuration = provider.Get <FirefoxConfiguration>(null);
if (configuration == null)
{
return(new FirefoxDriver());
}
return(new FirefoxDriver(GetOptions(configuration)));
}
// GET: Admin/Home
public ActionResult Index()
{
var model = new IndexViewModel();
var appConfiguration = _configurationProvider.Get <ApplicationConfiguration>();
model.ShowAcceptance = appConfiguration.IsUsersAcceptanceNeed;
model.ShowNotifyAboutNewVersion = VersionChecker.Instance.NeedToNotify();
return(View(model));
}
public override void OnActionExecuted(ActionExecutedContext context)
{
ViewData["doc_count"] = context.HttpContext.Request.Query.ContainsKey("collection") ?
_sessionFactory.GetDocCount(context.HttpContext.Request.Query["collection"].ToString()) :
0;
ViewBag.DefaultCollection = _config.Get("default_collection");
ViewBag.Collection = context.HttpContext.Request.Query.ContainsKey("collection") ?
context.HttpContext.Request.Query["collection"].ToString() :
ViewBag.DefaultCollection;
base.OnActionExecuted(context);
}
public override void OnActionExecuted(ActionExecutedContext context)
{
long? docCount = null;
string fileName = Path.Combine(_config.Get("data_dir"), "6604389855880847730.docs");
if (System.IO.File.Exists(fileName))
{
docCount = new FileInfo(fileName).Length / 111;
}
ViewData["doc_count"] = docCount;
base.OnActionExecuted(context);
}
public LogAppender(IConfigurationProvider provider)
{
configuration = provider.Get(CoreConfiguration.Default);
patternLayout = new PatternLayout(configuration.LogPattern);
if (!File.Exists(LogPath))
{
File.Create(LogPath).Dispose();
}
Task.Run(() => Writer());
}
public TermIndexSession(
string collectionName,
ulong collectionId,
SessionFactory sessionFactory,
ITokenizer tokenizer,
IConfigurationProvider config) : base(collectionName, collectionId, sessionFactory)
{
_config = config;
_tokenizer = tokenizer;
_dirty = new ConcurrentDictionary <long, VectorNode>();
var numThreads = int.Parse(_config.Get("write_thread_count"));
}
/// <summary>
/// Returns a string value from an IConfigurationProvider as a Dictionary, assuming its delimited in Key=Value; notation
/// </summary>
/// <param name="provider">The provider to use as a source</param>
/// <param name="key">The Key to search for</param>
/// <returns>A dictionary representing the key value pairs found in the configuration value</returns>
public static Dictionary <string, string> GetDictionary(this IConfigurationProvider provider, string key)
{
string v = provider.Get(key);
if (v is null)
{
return(null);
}
else
{
return(v.ToDictionary());
}
}
public Nullable <T> Get <T>(string key) where T : struct
{
T?result = null;
if (settings.AllKeys.Contains(key))
{
object safeValue = settings[key];
if (safeValue != null)
{
try {
Type t = typeof(T);
if (t.GetTypeInfo().IsEnum)
{
T enumVal;
if (Enum.TryParse <T>(safeValue.ToString(), out enumVal))
{
safeValue = enumVal;
}
}
else
{
safeValue = Convert.ChangeType(safeValue, t);
}
} catch (FormatException) {
//change safeValue to null if failing to cast string to type T.
safeValue = null;
}
}
result = (T?)safeValue;
}
if (result == null && chainedProvider != null)
{
result = chainedProvider.Get <T>(key);
}
return(result);
}
public AppSettingsProvider(String section, IConfigurationProvider provider) : base(provider)
{
if (String.IsNullOrEmpty(section))
{
settings = base.settings;
}
else
{
settings = new NameValueCollection()
{
{ section, provider.Get(section) }
};
}
}
public IWebDriver Activate(IConfigurationProvider provider)
{
var configuration = provider.Get <EdgeConfiguration>(null);
if (configuration == null)
{
return(new EdgeDriver());
}
return(new EdgeDriver(
configuration.WebDriverDirectory,
GetOptions(configuration),
configuration.CommandTimeout));
}
public ILEmitter Emit(ILEmitter il, Label gotoNext)
{
var arrayType = _variable.VariableType;
var(arrayX, arrayY) = _arrayComparisonEmitter.EmitLoad(_variable, il, gotoNext);
if (_configuration.Get(_variable.OwnerType).IgnoreCollectionOrder)
{
var elementType = arrayType.GetElementType();
var hasCustomComparer = _configuration.HasCustomComparer(elementType);
il.EmitArraySorting(hasCustomComparer, elementType, arrayX, arrayY);
}
return(_arrayComparisonEmitter.EmitCompareArrays(il, arrayType, _variable.OwnerType, arrayX, arrayY, gotoNext));
}
public ActionResult DownloadAndIndexWat(
string accessToken,
string commonCrawlId,
string collectionName,
int skip,
int take
)
{
if (!IsValidToken(accessToken))
{
return(StatusCode((int)HttpStatusCode.MethodNotAllowed));
}
CCHelper.DownloadAndIndexWat(
commonCrawlId,
_config.Get("data_dir"),
collectionName,
skip,
take,
SessionFactory.Model,
_logger);
return(View());
}
public AccountService(ILearnWithQBUow uow, IConfigurationProvider configurationProvider)
{
this.uow = uow;
this.stripeConfiguration = configurationProvider.Get<IStripeConfiguration>();
}
public AccountService(IWeddingBiddersUow uow, IConfigurationProvider configurationProvider)
{
this.uow = uow;
this.stripeConfiguration = configurationProvider.Get<IStripeConfiguration>();
}