public ObjectResult Get(string collectionId, string query) { //TODO: add pagination var contentType = Request.ContentType; var accepts = Request.Headers["Accept"]; if (query == null) { using (var r = new StreamReader(Request.Body)) { query = r.ReadToEnd(); } } var queryParser = _plugins.Get <IQueryParser>(contentType); var reader = _plugins.Get <IReader>(); if (queryParser == null || reader == null) { throw new NotSupportedException(); } var parsedQuery = queryParser.Parse(query); parsedQuery.CollectionId = collectionId.ToHash(); var payload = reader.Read(parsedQuery); return(new ObjectResult(payload)); }
public ObjectResult Get(string collectionId, string q) { //TODO: add pagination var mediaType = Request.ContentType ?? string.Empty; if (q == null) { using (var r = new StreamReader(Request.Body)) { q = r.ReadToEnd(); } } var queryParser = _plugins.Get <IQueryParser>(mediaType); var reader = _plugins.Get <IReader>(); var tokenizer = _plugins.Get <ITokenizer>(mediaType); if (queryParser == null || reader == null || tokenizer == null) { throw new NotSupportedException(); } var parsedQuery = queryParser.Parse(q, tokenizer); parsedQuery.CollectionId = collectionId.ToHash(); var payload = reader.Read(parsedQuery, int.MaxValue).ToList(); return(new ObjectResult(payload)); }
public async Task <IActionResult> Page(string url, string collectionId) { if (string.IsNullOrWhiteSpace(url)) { return(View("Index")); } if (!url.StartsWith("http://") && !url.StartsWith("https://")) { return(View("Index")); } var collectionName = collectionId ?? "www"; try { var uri = new Uri(url); var document = new Dictionary <string, object>(); var parsed = GetWebString(uri); document["site"] = uri.Host; document["url"] = uri.ToString(); document["body"] = parsed.body; document["title"] = parsed.title; document["created"] = DateTime.Now.ToBinary(); var writers = _plugins.All <IWriter>(Request.ContentType).ToList(); var reader = _plugins.Get <IReader>(); var mediaType = Request.ContentType ?? string.Empty; var queryParser = _plugins.Get <IQueryParser>(mediaType); var tokenizer = _plugins.Get <ITokenizer>(mediaType); if (writers == null || reader == null || queryParser == null || tokenizer == null) { return(StatusCode(415)); // Media type not supported } var deleteQuery = queryParser.Parse(string.Format("url:{0}", uri.ToString()), tokenizer); deleteQuery.CollectionId = collectionName.ToHash(); var oldData = reader.Read(deleteQuery).ToList(); foreach (var writer in writers) { await Task.Run(() => { writer.Update(collectionName, new[] { document }, oldData); }); } return(Redirect("/add/thankyou")); } catch (Exception ex) { // TODO: add logging framework System.IO.File.WriteAllText(string.Format("_{0}_{1}.log", DateTime.Now.ToBinary(), WebUtility.UrlEncode(url)), ex.ToString()); return(View("Error")); } }
public async Task <IActionResult> Get() { var mediaType = Request.Headers["Accept"].ToArray()[0]; var reader = _plugins.Get <IHttpReader>(mediaType); if (reader == null) { reader = _plugins.Get <IHttpReader>("application/json"); if (reader == null) { throw new NotSupportedException(); // Media type not supported } } var timer = Stopwatch.StartNew(); var result = await reader.Read(Request, _model); _logger.LogInformation($"processed {mediaType} request in {timer.Elapsed}"); Response.Headers.Add("X-Total", result.Total.ToString()); if (result.Total == 0) { return(new EmptyResult()); } return(new FileContentResult(result.Body, result.MediaType)); }
public async Task <IActionResult> Post(string collectionId) { if (collectionId == null) { throw new ArgumentNullException(nameof(collectionId)); } var writer = _plugins.Get <IWriter>(Request.ContentType); if (writer == null) { throw new NotSupportedException(); // Media type not supported } try { Result result = await writer.Write(collectionId, Request); return(new FileContentResult(result.Data.ToArray(), result.MediaType)); } catch (Exception ew) { _log.WriteLine(ew); throw ew; } }
public async Task <IActionResult> Post(string collectionName) { if (collectionName == null) { throw new ArgumentNullException(nameof(collectionName)); } var writer = _plugins.Get <IWriter>(Request.ContentType); if (writer == null) { throw new NotSupportedException(); // Media type not supported } try { ResponseModel result = await writer.Write(collectionName, Request); if (result.Stream != null) { var buf = result.Stream.ToArray(); return(new FileContentResult(buf, result.MediaType)); } else { return(Ok()); } } catch (Exception ew) { this.Log(ew); throw ew; } }
public IActionResult Post(string collectionName) { if (collectionName == null) { throw new ArgumentNullException(nameof(collectionName)); } var writer = _plugins.Get <IWriter>(Request.ContentType); if (writer == null) { writer = _plugins.Get <IWriter>(Request.ContentType.Split(';', StringSplitOptions.RemoveEmptyEntries)[0]); } if (writer == null) { throw new NotSupportedException(); // Media type not supported } try { writer.Write(collectionName, _model, Request); return(StatusCode(201)); } catch (Exception ew) { this.Log(ew); throw ew; } }
public ActionResult Index(string q, string cid) { if (string.IsNullOrWhiteSpace(q)) { return(View()); } string query = q.Trim(); string collectionId = cid ?? "www"; string htmlEncodedQuery = WebUtility.HtmlEncode(query); ViewData["q"] = query; if (!query.Contains(":")) { query = string.Format("title:{0}\nbody:{0}\nurl:{0}", query); } var timer = new Stopwatch(); timer.Start(); var queryParser = _plugins.Get <IQueryParser>(); var reader = _plugins.Get <IReader>(); var tokenizer = _plugins.Get <ITokenizer>(); if (queryParser == null || reader == null || tokenizer == null) { throw new System.NotSupportedException(); } var parsedQuery = queryParser.Parse(query, tokenizer); parsedQuery.CollectionId = collectionId.ToHash(); var documents = reader.Read(parsedQuery, 10) .GroupBy(x => x["_url"]) .SelectMany(x => x.OrderByDescending(y => y["_created"]).Take(1)) .Select(x => new SearchResultModel { Document = x }).ToList(); ViewData["collectionName"] = collectionId; ViewData["time_ms"] = timer.ElapsedMilliseconds; ViewData["last_processed_url"] = _crawlQueue.LastProcessed.uri; ViewData["last_processed_title"] = _crawlQueue.LastProcessed.title; return(View(documents)); }
public ActionResult Index(string q, string collectionId) { if (string.IsNullOrWhiteSpace(q)) { return(View("MultilineQuery")); } string collectionName = collectionId ?? "www"; var htmlEncodedQuery = WebUtility.HtmlEncode(q); ViewData["q"] = htmlEncodedQuery; if (!q.Contains(":")) { q = string.Format("title:{0}\nbody:{0}", q); } _timer.Restart(); var queryParser = _plugins.Get <IQueryParser>(); var reader = _plugins.Get <IReader>(); var tokenizer = _plugins.Get <ITokenizer>(); if (queryParser == null || reader == null || tokenizer == null) { throw new System.NotSupportedException(); } var parsedQuery = queryParser.Parse(q, tokenizer); parsedQuery.CollectionId = collectionName.ToHash(); var documents = reader.Read(parsedQuery).Select(x => new SearchResultModel { Document = x }).ToList(); ViewData["collectionName"] = collectionName; ViewData["time_ms"] = _timer.ElapsedMilliseconds; ViewBag.Title = htmlEncodedQuery + "Did you gogo?"; return(View(documents)); }
public async Task <IActionResult> Delete(string collectionId, string q) { var mediaType = Request.ContentType ?? string.Empty; var queryParser = _plugins.Get <IQueryParser>(mediaType); var reader = _plugins.Get <IReader>(); var writers = _plugins.All <IWriter>(mediaType).ToList(); var tokenizer = _plugins.Get <ITokenizer>(mediaType); if (queryParser == null || writers == null || writers.Count == 0 || tokenizer == null) { throw new NotSupportedException(); } var parsedQuery = queryParser.Parse(q, tokenizer); parsedQuery.CollectionId = collectionId.ToHash(); var oldData = reader.Read(parsedQuery).ToList(); foreach (var writer in writers) { await Task.Run(() => { writer.Remove(collectionId, oldData); }); } return(StatusCode(202)); // marked for deletion }
public IActionResult Post(string accessToken) { if (!IsValidToken(accessToken)) { return(StatusCode((int)HttpStatusCode.MethodNotAllowed)); } if (string.IsNullOrWhiteSpace(Request.ContentType)) { throw new NotSupportedException(); } var writer = _plugins.Get <IHttpWriter>(Request.ContentType); if (writer == null) { writer = _plugins.Get <IHttpWriter>(Request.ContentType.Split(';', StringSplitOptions.RemoveEmptyEntries)[0]); } if (writer == null) { throw new NotSupportedException(); // Media type not supported } try { writer.Write(Request, _model); return(Ok()); } catch (Exception ew) { _logger.LogError(ew.ToString()); throw ew; } }
public async Task <IActionResult> Index(string q, string collectionId) { if (string.IsNullOrWhiteSpace(q)) { return(View()); } string cid = collectionId ?? "www"; ViewData["q"] = q; var reader = _plugins.Get <IReader>("application/json"); if (reader == null) { throw new System.NotSupportedException(); } var timer = new Stopwatch(); timer.Start(); var result = await reader.Read(cid, Request); ViewData["time_ms"] = timer.ElapsedMilliseconds; ViewData["collectionName"] = cid; ViewData["total"] = result.Total; if (result.Total == 0) { return(View(new SearchResultModel[0])); } var documents = result.Documents.Select(x => new SearchResultModel { Document = x }); return(View(documents)); }
public async Task <IActionResult> Index(string q, string qf, string collection, string newCollection, string[] fields) { var formatter = _serviceProvider.GetService <IQueryFormatter>(); var formatted = qf ?? formatter.Format(collection, Request); ViewData["qf"] = formatted; ViewData["q"] = q; var reader = _plugins.Get <IReader>("application/json"); if (reader == null) { throw new NotSupportedException(); } var timer = new Stopwatch(); timer.Start(); var result = await reader.Read(collection, Request); ViewData["time_ms"] = timer.ElapsedMilliseconds; ViewData["collection"] = collection; ViewData["total"] = result.Total; ViewData["newCollection"] = newCollection; if (result.Total == 0) { return(View(new SearchResultModel[0])); } var documents = result.Documents.Select(x => new SearchResultModel { Document = x }); return(View(documents)); }