private IQueryable <RecordIndex.Result> AddDataFormatsToQuery(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (input.F.DataFormats != null && input.F.DataFormats.Any() && input.F.DataFormats[0].IsNotBlank())
{
var formatTypes = new List <string>();
foreach (var format in input.F.DataFormats)
{
var formatTypesList = DataFormats.Known.Find(x => x.Name.Equals(format)).Formats;
foreach (var formatType in formatTypesList)
{
formatTypes.Add(formatType.Name);
}
}
if (input.F.DataFormats.Contains("Other"))
{
query = query.Where(r =>
r.DataFormat.In(formatTypes) || r.DataFormat.Equals(null) || r.DataFormat.Equals(""));
}
else
{
query = query.Where(r => r.DataFormat.In(formatTypes));
}
}
return(query);
}
private IQueryable <Record> SortRecords(IQueryable <Record> recordQuery, RecordQueryInputModel input)
{
var sortedRecords = recordQuery;
Expression <Func <Record, Object> > orderByFunc;
switch (input.O)
{
case MostRelevant:
break;
case TitleAZ:
orderByFunc = record => record.Gemini.Title;
sortedRecords = recordQuery.OrderBy(orderByFunc);
break;
case TitleZA:
orderByFunc = record => record.Gemini.Title;
sortedRecords = recordQuery.OrderByDescending(orderByFunc);
break;
case NewestToOldest:
orderByFunc = record => record.Gemini.DatasetReferenceDate;
sortedRecords = recordQuery.OrderByDescending(orderByFunc);
break;
case OldestToNewest:
orderByFunc = record => record.Gemini.DatasetReferenceDate;
sortedRecords = recordQuery.OrderBy(orderByFunc);
break;
}
return(sortedRecords);
}
IQueryable <Record> RecordQueryImpl(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (input.Q.IsNotBlank())
{
query = query
.Search(r => r.Title, input.Q, 10, SearchOptions.Guess, EscapeQueryOptions.RawQuery)
.Search(r => r.TitleN, input.Q)
.Search(r => r.Abstract, input.Q, 1, SearchOptions.Guess, EscapeQueryOptions.RawQuery)
.Search(r => r.AbstractN, input.Q)
.Search(r => r.KeywordsN, input.Q);
}
if (input.F != null)
{
query = AddKeywordsToQuery(input, query);
query = AddDataFormatsToQuery(input, query);
query = AddMetadataDateToQuery(input, query);
query = AddManagerToQuery(input, query);
query = AddResourceTypesToQuery(input, query);
}
var recordQuery = query.As <Record>(); // ravendb method to project from the index result type to the actual document type
recordQuery = SortRecords(recordQuery, input);
// allow N to be negative
if (input.N >= 0)
{
recordQuery = recordQuery.Skip(input.P * input.N).Take(input.N);
}
return(recordQuery);
}
public IQueryable<Record> AsyncQuery(IAsyncDocumentSession adb, RecordQueryInputModel input)
{
var query = adb.Query<RecordIndex.Result, RecordIndex>()
.Statistics(out stats);
return RecordQueryImpl(input, query);
}
public IQueryable <Record> AsyncQuery(IAsyncDocumentSession adb, RecordQueryInputModel input)
{
var query = adb.Query <RecordIndex.Result, RecordIndex>()
.Statistics(out stats);
return(RecordQueryImpl(input, query));
}
private IQueryable <RecordIndex.Result> AddMetadataDateToQuery(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (input.F.MetadataDate != null)
{
query = query.Where(r => r.MetadataDate >= input.F.MetadataDate);
}
return(query);
}
private IQueryable <RecordIndex.Result> AddResourceTypesToQuery(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (input.F.ResourceTypes != null && input.F.ResourceTypes.Any() && input.F.ResourceTypes[0].IsNotBlank())
{
query = query.Where(r => r.ResourceType.In(input.F.ResourceTypes));
}
return(query);
}
private IQueryable <RecordIndex.Result> AddManagerToQuery(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (!string.IsNullOrWhiteSpace(input.F.Manager))
{
query = query.SearchMultiple(r => r.Manager, input.F.Manager, 1, SearchOptions.And);
}
return(query);
}
/// <summary>
/// A general-purpose query that returns records.
/// Can be materialised as-is, or customised further (see SearchQuery method).
/// </summary>
public IQueryable <Record> Query(RecordQueryInputModel input)
{
var query = _db.Query <RecordIndex.Result, RecordIndex>()
.Statistics(out stats)
.Customize(x => x.Highlight("Title", 202, 1, out titleLites))
.Customize(x => x.Highlight("TitleN", 202, 1, out titleNLites))
.Customize(x => x.Highlight("Abstract", 202, 1, out abstractLites))
.Customize(x => x.Highlight("AbstractN", 202, 1, out abstractNLites))
.Customize(x => x.SetHighlighterTags("<b>", "</b>"));
return(RecordQueryImpl(input, query));
}
private IQueryable <RecordIndex.Result> AddKeywordsToQuery(RecordQueryInputModel input, IQueryable <RecordIndex.Result> query)
{
if (input.F.Keywords != null && input.F.Keywords.Any() && input.F.Keywords[0].IsNotBlank())
{
foreach (var keyword in ParameterHelper.ParseMetadataKeywords(input.F.Keywords))
{
string k = keyword.Vocab + "/" + keyword.Value;
query = query.Where(r => r.Keywords.Contains(k));
}
}
return(query);
}
/// <summary>
/// A general-purpose query that returns records.
/// Can be materialised as-is, or customised further (see SearchQuery method).
/// </summary>
public IQueryable<Record> Query(RecordQueryInputModel input)
{
var query = _db.Query<RecordIndex.Result, RecordIndex>()
.Statistics(out stats)
.Customize(x => x.Highlight("Title", 202, 1, out titleLites))
.Customize(x => x.Highlight("TitleN", 202, 1, out titleNLites))
.Customize(x => x.Highlight("Abstract", 202, 1, out abstractLites))
.Customize(x => x.Highlight("AbstractN", 202, 1, out abstractNLites))
.Customize(x => x.SetHighlighterTags("<b>", "</b>"));
return RecordQueryImpl(input, query);
}
public List<string> SeabedHabitatMapsWithNoGui()
{
var query = new RecordQueryInputModel
{
N = 1000,
K = new [] { "vocab.jncc.gov.uk/jncc-category/Seabed Habitat Maps" },
};
var records = recordQueryer.Query(query).ToList();
var q = from r in records
where !(from k in r.Gemini.Keywords
where k.Vocab == "http://vocab.jncc.gov.uk/mesh-gui"
select k.Value).Any()
select r.Id.ToString();
return q.ToList();
}
public void export_sanity_check()
{
var controller = new ExportController(Db, new RecordQueryer(Db));
var input = new RecordQueryInputModel
{
Q = "",
K = new[] { "vocab.jncc.gov.uk/jncc-category/Seabed Habitat Maps" },
P = 0,
N = -1
};
var result = controller.Get(input);
var task = (PushStreamContent) result.Content;
string content = task.ReadAsStringAsync().Result;
var matches = Regex.Matches(content, @"""""Value"""":""""GB\d{6}"""""); // match a mesh identifier e.g. GB000272
matches.Count.Should().Be(189); // the number of mesh records
}
/// <summary>
/// A general-purpose query that returns records.
/// Can be materialised as-is, or customised further (see SearchQuery method).
/// </summary>
public IQueryable <Record> Query(RecordQueryInputModel input)
{
var highlightingOptions = new HighlightingOptions
{
PreTags = new[] { "<b>" },
PostTags = new[] { "</b>" }
};
var query = _db.Query <RecordIndex.Result, RecordIndex>()
.Statistics(out stats)
.Highlight("Title", 1000, 1, highlightingOptions, out titleLites) //todo: longer titles not shown even with huge fragment length?
.Highlight("TitleN", 202, 1, highlightingOptions, out titleNLites)
.Highlight("Abstract", 202, 1, highlightingOptions, out abstractLites)
.Highlight("AbstractN", 202, 1, highlightingOptions, out abstractNLites);
;
return(RecordQueryImpl(input, query));
}
/// <summary>
/// A query for the Google-style search results page.
/// </summary>
public SearchOutputModel Search(RecordQueryInputModel input)
{
// materializing the query will populate our stats
var results = from r in Query(input).ToList()
let titleFragments = titleLites.GetFragments("records/" + r.Id).Concat(titleNLites.GetFragments("records/" + r.Id))
let abstractFragments = abstractLites.GetFragments("records/" + r.Id).Concat(abstractNLites.GetFragments("records/" + r.Id))
let title = titleFragments.Select(f => f.TruncateNicely(200)).FirstOrDefault()
?? r.Gemini.Title.TruncateNicely(200)
let snippet = abstractFragments.Select(f => f.TruncateNicely(200)).FirstOrDefault()
?? r.Gemini.Abstract.TruncateNicely(200)
let format = DataFormatQueries.GetDataFormatInfo(r.Gemini.DataFormat)
select new ResultOutputModel
{
Id = r.Id,
Title = title, // could be better; always want the whole title, highlighted
Snippet = snippet,
Format = new FormatOutputModel
{
Group = format.Group,
Glyph = format.Glyph,
Name = format.Name,
},
Keywords = MakeKeywordOutputModelList(r.Gemini.Keywords).ToList(),
//.OrderBy(k => k.Vocab != "http://vocab.jncc.gov.uk/jncc-broad-category") // show first
//.ThenBy(k => k.Vocab.IsBlank())
//.ThenBy(k => k.Vocab).ToList(),
TopCopy = r.TopCopy,
Date = r.Gemini.DatasetReferenceDate,
ResourceType = r.Gemini.ResourceType.FirstCharToUpper(),
Box = r.Gemini.BoundingBox,
};
return(new SearchOutputModel
{
Total = stats.TotalResults,
Results = results.ToList(),
Speed = stats.DurationMilliseconds,
Query = input
});
}
/// <summary>
/// A query for the Google-style search results page.
/// </summary>
public SearchOutputModel Search(RecordQueryInputModel input)
{
// materializing the query will populate our stats
var results = from r in Query(input).ToList()
let titleFragments = titleLites.GetFragments("records/" + r.Id).Concat(titleNLites.GetFragments("records/" + r.Id))
let abstractFragments = abstractLites.GetFragments("records/" + r.Id).Concat(abstractNLites.GetFragments("records/" + r.Id))
let title = titleFragments.Select(f => f.TruncateNicely(200)).FirstOrDefault()
?? r.Gemini.Title.TruncateNicely(200)
let snippet = abstractFragments.Select(f => f.TruncateNicely(200)).FirstOrDefault()
?? r.Gemini.Abstract.TruncateNicely(200)
let format = DataFormatQueries.GetDataFormatInfo(r.Gemini.DataFormat)
select new ResultOutputModel
{
Id = r.Id,
Title = title, // could be better; always want the whole title, highlighted
Snippet = snippet,
Format = new FormatOutputModel
{
Group = format.Group,
Glyph = format.Glyph,
Name = format.Name,
},
Keywords = MakeKeywordOutputModelList(r.Gemini.Keywords).ToList(),
//.OrderBy(k => k.Vocab != "http://vocab.jncc.gov.uk/jncc-broad-category") // show first
//.ThenBy(k => k.Vocab.IsBlank())
//.ThenBy(k => k.Vocab).ToList(),
TopCopy = r.TopCopy,
Date = r.Gemini.DatasetReferenceDate,
ResourceType = r.Gemini.ResourceType.FirstCharToUpper(),
Box = r.Gemini.BoundingBox,
};
return new SearchOutputModel
{
Total = stats.TotalResults,
Results = results.ToList(),
Speed = stats.DurationMilliseconds,
Query = input
};
}
/// <summary>
/// A query for the Google-style search results page.
/// </summary>
public SearchOutputModel Search(RecordQueryInputModel input)
{
// materializing the query will populate our stats
var results = from r in Query(input).ToList()
let titleFragments = titleLites.GetFragments(r.Id).Concat(titleNLites.GetFragments(r.Id))
let abstractFragments = abstractLites.GetFragments(r.Id).Concat(abstractNLites.GetFragments(r.Id))
let title = titleFragments.FirstOrDefault() ?? r.Gemini.Title
let snippet = abstractFragments.Select(f => f.TruncateNicely(200)).FirstOrDefault()
?? r.Gemini.Abstract.TruncateNicely(200)
let format = DataFormatQueries.GetDataFormatInfo(r.Gemini.DataFormat)
select new ResultOutputModel
{
Id = Helpers.RemoveCollection(r.Id),
Title = title, // could be better; always want the whole title, highlighted
Snippet = snippet,
Format = new FormatOutputModel
{
Group = format.Group,
Glyph = format.Glyph,
Name = format.Name,
},
Keywords = MakeKeywordOutputModelList(r.Gemini.Keywords).ToList(),
TopCopy = r.TopCopy,
Date = r.Gemini.DatasetReferenceDate,
ResourceType = r.Gemini.ResourceType.FirstCharToUpper(),
Box = r.Gemini.BoundingBox,
};
return(new SearchOutputModel
{
Total = stats.TotalResults,
Results = results.ToList(),
Speed = stats.DurationInMs,
Query = input
});
}
static List<Record> GetRecords(IDocumentSession db, string keyword)
{
var recordQueryer = new RecordQueryer(db);
var query = new RecordQueryInputModel
{
K = new[] { keyword },
N = 1024,
};
int count = recordQueryer.Query(query).Count();
var records = recordQueryer.Query(query).ToList();
if (records.Count != count)
throw new Exception("Too many records.");
return records;
}
void RemovePagingParametersFromRecordQuery(RecordQueryInputModel input)
{
input.P = 0;
input.N = -1;
}
IQueryable<Record> RecordQueryImpl(RecordQueryInputModel input, IQueryable<RecordIndex.Result> query)
{
if (input.Q.IsNotBlank())
{
query = query
.Search(r => r.Title, input.Q, boost: 10)
.Search(r => r.TitleN, input.Q)
.Search(r => r.Abstract, input.Q)
.Search(r => r.AbstractN, input.Q)
.Search(r => r.KeywordsN, input.Q);
}
if (input.K != null && input.K.Any())
{
foreach (var keyword in ParameterHelper.ParseMetadataKeywords(input.K))
{
string k = keyword.Vocab + "/" + keyword.Value;
query = query.Where(r => r.Keywords.Contains(k));
}
}
if (input.D != null)
{
query = query.Where(r => r.MetadataDate >= input.D);
}
var recordQuery = query.As<Record>(); // ravendb method to project from the index result type to the actual document type
// allow N to be negative
if (input.N >= 0)
{
recordQuery = recordQuery.Skip(input.P * input.N).Take(input.N);
}
return recordQuery;
}
public string RenameKeyword(string keyword, string newValue)
{
var query = new RecordQueryInputModel
{
K = new[] { keyword },
N = 1024,
};
int count = recordQueryer.Query(query).Count();
var records = recordQueryer.Query(query).ToList();
if (records.Count != count)
throw new Exception("Too many records.");
foreach (var record in records)
{
var kword = ParameterHelper.ParseMetadataKeywords(new [] { keyword }).Single();
var keywordToChange = record.Gemini.Keywords.Single(k => k.Vocab == kword.Vocab && k.Value == kword.Value);
keywordToChange.Value = newValue;
}
db.SaveChanges();
return String.Format("{0} records updated.", count);
}