public ObjectResult Get(string collectionId, string query)
{
//TODO: add pagination
var contentType = Request.ContentType;
var accepts = Request.Headers["Accept"];
if (query == null)
{
using (var r = new StreamReader(Request.Body))
{
query = r.ReadToEnd();
}
}
var queryParser = _plugins.Get <IQueryParser>(contentType);
var reader = _plugins.Get <IReader>();
if (queryParser == null || reader == null)
{
throw new NotSupportedException();
}
var parsedQuery = queryParser.Parse(query);
parsedQuery.CollectionId = collectionId.ToHash();
var payload = reader.Read(parsedQuery);
return(new ObjectResult(payload));
}
public ObjectResult Get(string collectionId, string q)
{
//TODO: add pagination
var mediaType = Request.ContentType ?? string.Empty;
if (q == null)
{
using (var r = new StreamReader(Request.Body))
{
q = r.ReadToEnd();
}
}
var queryParser = _plugins.Get <IQueryParser>(mediaType);
var reader = _plugins.Get <IReader>();
var tokenizer = _plugins.Get <ITokenizer>(mediaType);
if (queryParser == null || reader == null || tokenizer == null)
{
throw new NotSupportedException();
}
var parsedQuery = queryParser.Parse(q, tokenizer);
parsedQuery.CollectionId = collectionId.ToHash();
var payload = reader.Read(parsedQuery, int.MaxValue).ToList();
return(new ObjectResult(payload));
}
public async Task <IActionResult> Page(string url, string collectionId)
{
if (string.IsNullOrWhiteSpace(url))
{
return(View("Index"));
}
if (!url.StartsWith("http://") && !url.StartsWith("https://"))
{
return(View("Index"));
}
var collectionName = collectionId ?? "www";
try
{
var uri = new Uri(url);
var document = new Dictionary <string, object>();
var parsed = GetWebString(uri);
document["site"] = uri.Host;
document["url"] = uri.ToString();
document["body"] = parsed.body;
document["title"] = parsed.title;
document["created"] = DateTime.Now.ToBinary();
var writers = _plugins.All <IWriter>(Request.ContentType).ToList();
var reader = _plugins.Get <IReader>();
var mediaType = Request.ContentType ?? string.Empty;
var queryParser = _plugins.Get <IQueryParser>(mediaType);
var tokenizer = _plugins.Get <ITokenizer>(mediaType);
if (writers == null || reader == null || queryParser == null || tokenizer == null)
{
return(StatusCode(415)); // Media type not supported
}
var deleteQuery = queryParser.Parse(string.Format("url:{0}", uri.ToString()), tokenizer);
deleteQuery.CollectionId = collectionName.ToHash();
var oldData = reader.Read(deleteQuery).ToList();
foreach (var writer in writers)
{
await Task.Run(() =>
{
writer.Update(collectionName, new[] { document }, oldData);
});
}
return(Redirect("/add/thankyou"));
}
catch (Exception ex)
{
// TODO: add logging framework
System.IO.File.WriteAllText(string.Format("_{0}_{1}.log", DateTime.Now.ToBinary(), WebUtility.UrlEncode(url)), ex.ToString());
return(View("Error"));
}
}
public async Task <IActionResult> Get()
{
var mediaType = Request.Headers["Accept"].ToArray()[0];
var reader = _plugins.Get <IHttpReader>(mediaType);
if (reader == null)
{
reader = _plugins.Get <IHttpReader>("application/json");
if (reader == null)
{
throw new NotSupportedException(); // Media type not supported
}
}
var timer = Stopwatch.StartNew();
var result = await reader.Read(Request, _model);
_logger.LogInformation($"processed {mediaType} request in {timer.Elapsed}");
Response.Headers.Add("X-Total", result.Total.ToString());
if (result.Total == 0)
{
return(new EmptyResult());
}
return(new FileContentResult(result.Body, result.MediaType));
}
public async Task <IActionResult> Post(string collectionId)
{
if (collectionId == null)
{
throw new ArgumentNullException(nameof(collectionId));
}
var writer = _plugins.Get <IWriter>(Request.ContentType);
if (writer == null)
{
throw new NotSupportedException(); // Media type not supported
}
try
{
Result result = await writer.Write(collectionId, Request);
return(new FileContentResult(result.Data.ToArray(), result.MediaType));
}
catch (Exception ew)
{
_log.WriteLine(ew);
throw ew;
}
}
public async Task <IActionResult> Post(string collectionName)
{
if (collectionName == null)
{
throw new ArgumentNullException(nameof(collectionName));
}
var writer = _plugins.Get <IWriter>(Request.ContentType);
if (writer == null)
{
throw new NotSupportedException(); // Media type not supported
}
try
{
ResponseModel result = await writer.Write(collectionName, Request);
if (result.Stream != null)
{
var buf = result.Stream.ToArray();
return(new FileContentResult(buf, result.MediaType));
}
else
{
return(Ok());
}
}
catch (Exception ew)
{
this.Log(ew);
throw ew;
}
}
public IActionResult Post(string collectionName)
{
if (collectionName == null)
{
throw new ArgumentNullException(nameof(collectionName));
}
var writer = _plugins.Get <IWriter>(Request.ContentType);
if (writer == null)
{
writer = _plugins.Get <IWriter>(Request.ContentType.Split(';', StringSplitOptions.RemoveEmptyEntries)[0]);
}
if (writer == null)
{
throw new NotSupportedException(); // Media type not supported
}
try
{
writer.Write(collectionName, _model, Request);
return(StatusCode(201));
}
catch (Exception ew)
{
this.Log(ew);
throw ew;
}
}
public ActionResult Index(string q, string cid)
{
if (string.IsNullOrWhiteSpace(q))
{
return(View());
}
string query = q.Trim();
string collectionId = cid ?? "www";
string htmlEncodedQuery = WebUtility.HtmlEncode(query);
ViewData["q"] = query;
if (!query.Contains(":"))
{
query = string.Format("title:{0}\nbody:{0}\nurl:{0}", query);
}
var timer = new Stopwatch();
timer.Start();
var queryParser = _plugins.Get <IQueryParser>();
var reader = _plugins.Get <IReader>();
var tokenizer = _plugins.Get <ITokenizer>();
if (queryParser == null || reader == null || tokenizer == null)
{
throw new System.NotSupportedException();
}
var parsedQuery = queryParser.Parse(query, tokenizer);
parsedQuery.CollectionId = collectionId.ToHash();
var documents = reader.Read(parsedQuery, 10)
.GroupBy(x => x["_url"])
.SelectMany(x => x.OrderByDescending(y => y["_created"]).Take(1))
.Select(x => new SearchResultModel {
Document = x
}).ToList();
ViewData["collectionName"] = collectionId;
ViewData["time_ms"] = timer.ElapsedMilliseconds;
ViewData["last_processed_url"] = _crawlQueue.LastProcessed.uri;
ViewData["last_processed_title"] = _crawlQueue.LastProcessed.title;
return(View(documents));
}
public ActionResult Index(string q, string collectionId)
{
if (string.IsNullOrWhiteSpace(q))
{
return(View("MultilineQuery"));
}
string collectionName = collectionId ?? "www";
var htmlEncodedQuery = WebUtility.HtmlEncode(q);
ViewData["q"] = htmlEncodedQuery;
if (!q.Contains(":"))
{
q = string.Format("title:{0}\nbody:{0}", q);
}
_timer.Restart();
var queryParser = _plugins.Get <IQueryParser>();
var reader = _plugins.Get <IReader>();
var tokenizer = _plugins.Get <ITokenizer>();
if (queryParser == null || reader == null || tokenizer == null)
{
throw new System.NotSupportedException();
}
var parsedQuery = queryParser.Parse(q, tokenizer);
parsedQuery.CollectionId = collectionName.ToHash();
var documents = reader.Read(parsedQuery).Select(x => new SearchResultModel {
Document = x
}).ToList();
ViewData["collectionName"] = collectionName;
ViewData["time_ms"] = _timer.ElapsedMilliseconds;
ViewBag.Title = htmlEncodedQuery + "Did you gogo?";
return(View(documents));
}
public async Task <IActionResult> Delete(string collectionId, string q)
{
var mediaType = Request.ContentType ?? string.Empty;
var queryParser = _plugins.Get <IQueryParser>(mediaType);
var reader = _plugins.Get <IReader>();
var writers = _plugins.All <IWriter>(mediaType).ToList();
var tokenizer = _plugins.Get <ITokenizer>(mediaType);
if (queryParser == null || writers == null || writers.Count == 0 || tokenizer == null)
{
throw new NotSupportedException();
}
var parsedQuery = queryParser.Parse(q, tokenizer);
parsedQuery.CollectionId = collectionId.ToHash();
var oldData = reader.Read(parsedQuery).ToList();
foreach (var writer in writers)
{
await Task.Run(() =>
{
writer.Remove(collectionId, oldData);
});
}
return(StatusCode(202)); // marked for deletion
}
public IActionResult Post(string accessToken)
{
if (!IsValidToken(accessToken))
{
return(StatusCode((int)HttpStatusCode.MethodNotAllowed));
}
if (string.IsNullOrWhiteSpace(Request.ContentType))
{
throw new NotSupportedException();
}
var writer = _plugins.Get <IHttpWriter>(Request.ContentType);
if (writer == null)
{
writer = _plugins.Get <IHttpWriter>(Request.ContentType.Split(';', StringSplitOptions.RemoveEmptyEntries)[0]);
}
if (writer == null)
{
throw new NotSupportedException(); // Media type not supported
}
try
{
writer.Write(Request, _model);
return(Ok());
}
catch (Exception ew)
{
_logger.LogError(ew.ToString());
throw ew;
}
}
public async Task <IActionResult> Index(string q, string collectionId)
{
if (string.IsNullOrWhiteSpace(q))
{
return(View());
}
string cid = collectionId ?? "www";
ViewData["q"] = q;
var reader = _plugins.Get <IReader>("application/json");
if (reader == null)
{
throw new System.NotSupportedException();
}
var timer = new Stopwatch();
timer.Start();
var result = await reader.Read(cid, Request);
ViewData["time_ms"] = timer.ElapsedMilliseconds;
ViewData["collectionName"] = cid;
ViewData["total"] = result.Total;
if (result.Total == 0)
{
return(View(new SearchResultModel[0]));
}
var documents = result.Documents.Select(x => new SearchResultModel {
Document = x
});
return(View(documents));
}
public async Task <IActionResult> Index(string q, string qf, string collection, string newCollection, string[] fields)
{
var formatter = _serviceProvider.GetService <IQueryFormatter>();
var formatted = qf ?? formatter.Format(collection, Request);
ViewData["qf"] = formatted;
ViewData["q"] = q;
var reader = _plugins.Get <IReader>("application/json");
if (reader == null)
{
throw new NotSupportedException();
}
var timer = new Stopwatch();
timer.Start();
var result = await reader.Read(collection, Request);
ViewData["time_ms"] = timer.ElapsedMilliseconds;
ViewData["collection"] = collection;
ViewData["total"] = result.Total;
ViewData["newCollection"] = newCollection;
if (result.Total == 0)
{
return(View(new SearchResultModel[0]));
}
var documents = result.Documents.Select(x => new SearchResultModel {
Document = x
});
return(View(documents));
}
