public PagedResponse <ReadTopicDto> Execute(TopicSearch search)
{
var query = _context.Topics.AsQueryable();
if (!string.IsNullOrEmpty(search.Topic) || !string.IsNullOrWhiteSpace(search.Topic))
{
query = query.Where(x => x.Name.ToLower().Contains(search.Topic.ToLower()));
}
var skipCount = search.PerPage * (search.Page - 1);
var response = new PagedResponse <ReadTopicDto>
{
CurrentPage = search.Page,
ItemsPerPage = search.PerPage,
TotalCount = query.Count(),
Items = query.Skip(skipCount).Take(search.PerPage).Select(x => new ReadTopicDto
{
Id = x.Id,
Name = x.Name,
NumberOfPosts = x.Posts.Count()
}).ToList()
};
return(response);
}
public async Task <IEnumerable <Topic> > FindList(TopicSearch search)
{
var query = _dbContext.Topic.AsQueryable();
if (search.BeginTime.HasValue)
{
query = query.Where(t => t.CreationTime >= search.BeginTime.Value);
}
if (search.EndTime.HasValue)
{
query = query.Where(t => t.CreationTime < search.EndTime.Value);
}
return(await query.ToListAsync());
}
public void SaveTopic(TopicSearch topicSearch)
{
var topicsDirectory = Path.Combine(_directory, "topics");
if (!Directory.Exists(topicsDirectory))
{
Directory.CreateDirectory(topicsDirectory);
}
var topicKey = SanitizeTopic(topicSearch.Topic);
var destination = Path.Combine(topicsDirectory, $"{topicKey}.json");
if (File.Exists(destination))
{
File.Delete(destination);
}
File.WriteAllText(destination, JsonConvert.SerializeObject(topicSearch, Formatting.Indented));
}
public IEnumerable <TopicDto> Execute(TopicSearch request)
{
var query = Context.Topics.AsQueryable();
if (request.Keyword != null)
{
query = query.Where(c => c.Subject
.ToLower()
.Contains(request.Keyword.ToLower()));
}
if (request.OnlyActive.HasValue)
{
query = query.Where(c => c.isDeleted != request.OnlyActive);
}
return(query.Select(c => new TopicDto
{
Id = c.Id,
Subject = c.Subject,
}));
}
public async Task <PagedModel <Topic> > Search(TopicSearch search)
{
var query = _dbContext.Topic.AsQueryable();
if (search.BeginTime.HasValue)
{
query = query.Where(t => t.CreationTime >= search.BeginTime.Value);
}
if (search.EndTime.HasValue)
{
query = query.Where(t => t.CreationTime < search.EndTime.Value);
}
var count = await query.CountAsync();
if (count == 0)
{
return(new PagedModel <Topic>(search.PageNow, search.PageSize, 0, Enumerable.Empty <Topic>()));
}
var records = await query.Skip(search.StartIndex).Take(search.PageSize).ToListAsync();
return(new PagedModel <Topic>(search.PageSize, search.PageNow, count, records));
}
public IActionResult Get([FromQuery] TopicSearch query) => Ok(_getCommand.Execute(query));
public IActionResult Get([FromQuery] TopicSearch search, [FromServices] IGetTopicsQuery query)
{
return(Ok(_executor.ExecuteQuery(query, search)));
}
// GET: Topics
public ActionResult Index(TopicSearch search)
{
var result = _getCommand.Execute(search);
return(View(result));
}
public static void Run(string indexDirectory, string query)
{
var workspace = Helpers.GetWorkspace(indexDirectory);
Log.Logger.Information("Querying for {query}...", query);
Log.Logger.Information("Discovering captions...");
var captionEntries = workspace.GetCaptions();
if (captionEntries.Count == 0)
{
Log.Logger.Information("No captions are present.");
Environment.Exit(1);
}
var videoCaptions = workspace.GetCaptions();
var videosWithCaptions = workspace.GetVideos().Where(x => videoCaptions.ContainsKey(x.Id)).ToList();
Log.Information("Searching captions for {count} videos...", videosWithCaptions.Count);
var topic = new TopicSearch();
topic.Topic = query;
foreach (var video in videosWithCaptions)
{
var captionText = string.Join(" ", captionEntries[video.Id].Select(x => $"[@{x.Start}] {x.Value}"));
var terms = new List <SpanQuery>();
foreach (var term in query.Trim().Split(" "))
{
terms.Add(new SpanTermQuery(new Term("content", term)));
}
var spanNearQuery = new SpanNearQuery(terms.ToArray(), 25, false);
var queryScorer = new QueryScorer(spanNearQuery);
var highlighter = new Highlighter(new MarkerFormatter(), queryScorer)
{
TextFragmenter = new NullFragmenter()
};
var tokenStream = new StandardAnalyzer(LuceneVersion.LUCENE_48).GetTokenStream("content", captionText);
var searchResult = highlighter.GetBestFragment(tokenStream, captionText);
if (string.IsNullOrEmpty(searchResult))
{
continue;
}
var model = new TopicSearch.VideoResult();
model.Id = video.Id;
model.Segments = new List <TopicSearch.VideoResult.Segment>();
foreach (Match match in Regex.Matches(searchResult, @"\[\@([0-9\.]*)\].+?(?=\[\@[0-9\.]*\]|$)"))
{
var segment = match.Groups[0].Value;
if (!segment.Contains("!!!! "))
{
continue;
}
segment = segment.Replace("!!!! ", "");
segment = Regex.Replace(segment, @"\[\@[0-9\.]*\]", "");
segment = segment.Trim();
var timeStamp = (int)Math.Max(Math.Floor(decimal.Parse(match.Groups[1].Value)), 0);
model.Segments.Add(new TopicSearch.VideoResult.Segment
{
Text = segment,
Location = timeStamp
});
}
topic.Results.Add(model);
}
Log.Information("Found {total} videos, with {total} segments.", topic.Results.Count, topic.Results.Sum(x => x.Segments.Count));
Console.WriteLine(JsonConvert.SerializeObject(topic, Formatting.Indented));
Log.Logger.Information("Done!");
}
public static void Run(string indexDirectory, List <string> query)
{
var workspace = Helpers.GetWorkspace(indexDirectory);
if (query == null || query.Count == 0 || query.Any(string.IsNullOrEmpty))
{
Log.Error("No query given.");
Environment.Exit(1);
}
Log.Logger.Information("Indexing topic(s) {@topic}...", query);
Log.Logger.Information("Discovering captions...");
var captionEntries = workspace.GetCaptions();
if (captionEntries.Count == 0)
{
Log.Logger.Information("No captions are present.");
return;
}
foreach (var q in query)
{
ParseTopic(q, out var topicName, out var topicAliases);
Log.Information("Indexing {topic}...", topicName);
var topic = workspace.FindTopic(topicName);
if (topic == null)
{
topic = new TopicSearch();
topic.Topic = topicName;
topic.Aliases = topicAliases;
topic.Results = new List <TopicSearch.VideoResult>();
}
else
{
// If this topic has updated aliases, we need to reindex everything.
string Hash(string t, List <string> a)
{
var hash = (t ?? "").CalculateMD5Hash();
if (a != null)
{
hash += string.Join("", a.Select(x => x.CalculateMD5Hash()));
}
return(hash);
}
if (Hash(topic.Topic, topic.Aliases) != Hash(topicName, topicAliases))
{
topic.Indexed = null;
}
}
var indexedVideos = new List <string>();
if (!string.IsNullOrEmpty(topic.Indexed))
{
indexedVideos = topic.Indexed.Split(",").ToList();
}
else
{
indexedVideos = new List <string>();
}
foreach (var captions in captionEntries)
{
if (indexedVideos.Contains(captions.Key))
{
continue;
}
indexedVideos.Add(captions.Key);
var segments = SearchEngine.Search(captions.Value, topicAliases ?? new List <string> {
topicName
});
if (segments.Count == 0)
{
continue;
}
topic.Results.Add(new TopicSearch.VideoResult
{
Id = captions.Key,
Segments = segments.Select(x => new TopicSearch.VideoResult.Segment
{
Text = x.Text,
Location = x.Location
}).ToList()
});
}
topic.Indexed = string.Join(",", indexedVideos);
workspace.SaveTopic(topic);
}
Log.Information("Done!");
}
