private IDocumentHandler FindDocumentHandler(IDocumentSource source)
{
var registry = This.GetService <IDocumentHandlerRegistryService>();
var found = registry.Find(source);
return(found[0]); // TODO: handle multiple results
}
public void SetSource(IDocumentSource source)
{
if (source == null) throw new ArgumentNullException("source");
if (documentSource != null) throw new InvalidOperationException("This document's source can only be set once.");
documentSource = source;
}
public bool CanHandle(IDocumentSource source)
{
if (source == null) throw new ArgumentNullException("source");
var ok = CanHandleSource(source);
if (ok) Source = source;
return ok;
}
void OnSourceChanged(IDocumentSource oldValue, IDocumentSource newValue)
{
if (oldValue != null)
oldValue.DocumentChanged -= Source_DocumentChanged;
if (newValue != null)
newValue.DocumentChanged += Source_DocumentChanged;
SetRootVisual();
}
private static IDataView BuildDataView(
MLContext context,
IDocumentSource source)
{
var environment = context.Data.GetEnvironment();
var schema = SchemaDefinition.Create(typeof(IDocument), SchemaDefinition.Direction.Read);
var data = environment.CreateStreamingDataView(source.GetDocuments(), schema);
return(data);
}
public void CreateDocument(IDocumentSource source)
{
if (source == null) throw new ArgumentNullException("source");
if (source != Source) throw new ArgumentException(
"The same source object must be passed to CanHandle and CreateDocument methods", "source");
var doc = CreateDocumentFromSource();
doc.Handler = this;
doc.SetSource(source);
Document = doc;
}
public void SetSource(IDocumentSource source)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (documentSource != null)
{
throw new InvalidOperationException("This document's source can only be set once.");
}
documentSource = source;
}
void OnSourceChanged(IDocumentSource oldValue, IDocumentSource newValue)
{
if (oldValue != null)
{
oldValue.DocumentChanged -= Source_DocumentChanged;
}
if (newValue != null)
{
newValue.DocumentChanged += Source_DocumentChanged;
}
SetRootVisual();
}
public bool CanHandle(IDocumentSource source)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
var ok = CanHandleSource(source);
if (ok)
{
Source = source;
}
return(ok);
}
public IDocumentHandler[] Find(IDocumentSource source)
{
List <IDocumentHandler> result = new List <IDocumentHandler>();
foreach (var current in list.OrderByDescending(e => e.Priority).Select(e => e.Value))
{
var function = current;
IDocumentHandler handler = null;
try
{
handler = function();
}
catch (Exception ex)
{
This.Logger.Error(string.Format("Handler creation error: {0}", ex.Message), ex);
}
if (handler == null)
{
continue;
}
var canHandle = false;
try
{
canHandle = handler.CanHandle(source);
}
catch (Exception ex)
{
This.Logger.Error(string.Format("Handler.CanHandle invocation error: {0}", ex.Message), ex);
}
if (canHandle)
{
result.Add(handler);
}
}
if (result.Count == 0)
{
return new IDocumentHandler[] { new DefaultDocumentHandler() }
}
;
else
{
return(result.ToArray());
}
}
public IDocument CreateDocument(IDocumentSource source)
{
if (source == null) throw new ArgumentNullException("source");
var handler = FindDocumentHandler(source);
handler.CreateDocument(source);
var document = handler.Document;
if (document == null)
{
This.Logger.Warning(string.Format(
"Document creation failed for source {0}", source.Uri));
return null;
}
OnDocumentCreated(document);
return document;
}
public IReadOnlyList <ITopic> GroupDocuments(
IDocumentSource source,
int numberOfGroups)
{
var context = new MLContext();
var data = BuildDataView(context, source);
var pipeline = BuildPipeline(context, numberOfGroups);
var model = pipeline.Fit(data);
var transformedData = model.Transform(data);
var ldaData = transformedData.GetColumn <float[]>(context, "LDA").ToList();
var topics = BuildTopics(source, numberOfGroups, ldaData);
return(topics);
}
public void CreateDocument(IDocumentSource source)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (source != Source)
{
throw new ArgumentException(
"The same source object must be passed to CanHandle and CreateDocument methods", "source");
}
var doc = CreateDocumentFromSource();
doc.Handler = this;
doc.SetSource(source);
Document = doc;
}
protected override bool CanHandleSource(IDocumentSource source)
{
if (source == null || !(source is FileDocumentSource))
{
return(false);
}
handler = plugin.CreateHandler();
var ok = handler.CanHandleFile(((FileDocumentSource)source).Uri);
if (ok)
{
This.Logger.Info(string.Format(
"This file will be handled by plugin [{0}]", plugin.PluginInfo.Name));
}
return(ok);
}
/// <summary>
/// Creates an instance of <see cref="Indexer"/> that will be indexing documents from
/// the specified <paramref name="source"/> with the <paramref name="textParser"/> and
/// merging them to the <paramref name="index"/>
/// </summary>
/// <param name="index"></param>
/// <param name="source"></param>
/// <param name="textParser"></param>
public Indexer([NotNull] IIndex index, [NotNull] IDocumentSource source, [NotNull] ITextParser textParser)
{
if (index == null)
{
throw new ArgumentNullException("index");
}
if (source == null)
{
throw new ArgumentNullException("source");
}
if (textParser == null)
{
throw new ArgumentNullException("textParser");
}
this.Index = index;
this.Source = source;
this.TextParser = textParser;
this.textParserReader = textParser.ExtractWords;
this.wordComparer = new StringComparisonComparer(index.WordComparison);
}
public IDocument CreateDocument(IDocumentSource source)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
var handler = FindDocumentHandler(source);
handler.CreateDocument(source);
var document = handler.Document;
if (document == null)
{
This.Logger.Warning(string.Format(
"Document creation failed for source {0}", source.Uri));
return(null);
}
OnDocumentCreated(document);
return(document);
}
private static List <Topic> BuildTopics(
IDocumentSource source,
int numberOfGroups,
List <float[]> ldaData)
{
var topics = new List <Topic>(numberOfGroups);
for (var i = 0; i < numberOfGroups; i++)
{
topics.Add(new Topic());
}
var results = source.GetDocuments().Zip(
ldaData,
(
d,
s) => new {
Document = d,
Scores = s
});
foreach (var result in results)
{
var maxScore = float.MinValue;
var maxIndex = 0;
for (var i = 0; i < numberOfGroups; i++)
{
if (result.Scores[i] > maxScore)
{
maxScore = result.Scores[i];
maxIndex = i;
}
}
topics[maxIndex].Add(result.Document, maxScore);
}
return(topics);
}
/// <summary>
/// Creates <see cref="Indexer"/> for the specified <paramref name="source"/> and
/// adds it to the <see cref="Indexers"/> list
/// </summary>
/// <param name="source">The source, providing documents to include in the index</param>
/// <param name="textParser">
/// The <see cref="ITextParser"/> used to extract words from documents of this source.
/// In case if this parameter is not specified, the <see cref="DefaultTextParser"/> property value is used.
/// </param>
/// <param name="autoStartIndexing">Specifies, whether to start indexing this source immediately</param>
/// <returns>Returns an <see cref="Indexer"/> created for the <paramref name="source"/></returns>
/// <exception cref="ArgumentException">Thrown when the <paramref name="source"/> is already included in this set</exception>
public Indexer Add([NotNull] IDocumentSource source, ITextParser textParser = null, bool autoStartIndexing = true)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (ContainsSource(source))
{
throw new ArgumentException("Source is already included in this IndexerSet", "source");
}
var indexer = new Indexer(Index, source, textParser ?? DefaultTextParser);
lock (lockObject)
indexers = indexers.Add(indexer);
if (autoStartIndexing)
{
indexer.StartIndexing();
}
return(indexer);
}
public GamerChWikiIdolSource(IDocumentSource document)
{
m_doc = document;
}
private IDocumentHandler FindDocumentHandler(IDocumentSource source)
{
var registry = This.GetService<IDocumentHandlerRegistryService>();
var found = registry.Find(source);
return found[0]; // TODO: handle multiple results
}
protected abstract bool CanHandleSource(IDocumentSource source);
private bool ContainsSource(IDocumentSource source)
{
return(indexers.Any(indexer => indexer.Source == source));
}
public GamerChWikiSongSource(IDocumentSource document)
{
m_doc = document;
}
public GamerChWikiSongSource(IDocumentSource document)
{
m_doc = document;
}
public GamerChWikiIdolSource(IDocumentSource document)
{
m_doc = document;
}
protected override bool CanHandleSource(IDocumentSource source)
{
return(source != null && source is FileDocumentSource);
}
public PathFinder(IDocumentSource documentSource)
{
_documentSource = documentSource;
}
protected abstract bool CanHandleSource(IDocumentSource source);
