public IQueryable <TEntity> Apply(IQueryable <TEntity> query, FilterRequestModel filterModel)
{
var filterDefinitions = FilterParser.Parse(filterModel.Filter);
var filterExpression = ExpressionProvider.CreateFilterExpression(filterDefinitions);
return(query.Where(filterExpression));
}
private IFilter RenderFilterDefinition(FilterDefinition <TDocument> filter)
{
var documentSerializer = BsonSerializer.SerializerRegistry.GetSerializer <TDocument>();
var filterBson = filter.Render(documentSerializer, BsonSerializer.SerializerRegistry);
return(_filterParser.Parse(filterBson));
}
public virtual Query <TEntity> Parse(HttpRequest request, ActionDescriptor action, bool isCollectionCall)
{
var query = new Query <TEntity>
(
_fieldsParser.Parse <TEntity>(request, isCollectionCall),
_orderByParser.Parse <TEntity>(request),
_pagingParser.Parse(request),
_filterParser.Parse(request, action, _webFilterConverter),
GetVerb(request)
)
{
TypeFilter = _typeFilterParser.Parse(request)
};
if (query.Fields.Contains((ISelection c) => c.Count))
{
query.Options.NeedsCount = true;
query.Options.NeedsEnumeration = query.Fields.Children.Count != 1;
}
if (query.Verb == HttpVerbs.Get)
{
query.Options.NeedsDataTracking = false;
}
return(query);
}
private IEnumerable <BsonDocument> Filter(FilterDefinition <TDocument> filter)
{
var documentSerializer = BsonSerializer.SerializerRegistry.GetSerializer <TDocument>();
var filterBson = filter.Render(documentSerializer, BsonSerializer.SerializerRegistry);
var parsedFilter = _filterParser.Parse(filterBson);
return(_documents.Where(parsedFilter.Filter));
}
private Filter GetFilter(string value)
{
var filter = _filterParser.Parse(value);
if (filter == null)
{
throw new InvalidOperationException(string.Format(ErrorMessages.TheParameterIsNotValid, Common.Constants.SearchParameterNames.Filter));
}
return(filter);
}
/// <summary>
/// Webページをインデックス化
/// </summary>
/// <param name="wc"></param>
/// <param name="indexWriter"></param>
/// <param name="threadName"></param>
/// <returns></returns>
private bool AddWebDocument(WebContents wc, IndexWriter indexWriter, string threadName)
{
string extension = wc.Extention;
//ドキュメント追加
Document doc = new Document();
if (extension.ToLower() == ".html" ||
extension.ToLower() == ".htm" ||
extension.ToLower() == "")
{
if (wc.Contents.Length > 0)
{
if (_txtExtractMode == TextExtractModes.Tika)
{
byte[] data = System.Text.Encoding.Unicode.GetBytes(wc.Contents);
_parser.parse(new java.io.ByteArrayInputStream(data), _handler, _metadata, new ParseContext());
string content = _handler.toString();
doc.Add(new Field(Content, content, _hilightFieldType));
}
else
{
doc.Add(new Field(Content, IFilterParser.Parse(wc.Contents), _hilightFieldType));
}
}
}
else
{
//FIXME ダウンロードしてインデックス化
//バイト数セット
//インデックス後削除
}
doc.Add(new StringField(Path, wc.Url, FieldStore.YES));
doc.Add(new StringField(Title, wc.Title, FieldStore.YES));
doc.Add(new StringField(Extension, extension.ToLower(), FieldStore.YES));
long l = long.Parse(wc.UpdateDate.ToString("yyyyMMddHHmmss"));
doc.Add(new LongPoint(UpdateDate, l));
doc.Add(new StoredField(UpdateDate, l));
//doc.Add(new StringField(UpdateDate,
// DateTools.DateToString(_sdf.parse(wc.UpdateDate.ToString("yyyy/MM/dd")), DateToolsResolution.DAY),
// FieldStore.YES));
indexWriter.AddDocument(doc);
return(true);
}
public IFilter Parse(BsonValue filter)
{
if (!filter.IsBsonArray)
{
throw new ArgumentOutOfRangeException(nameof(filter));
}
var childrenFilters = new List <IFilter>();
var filterStatements = filter.AsBsonArray;
foreach (var filterStatement in filterStatements)
{
var childFilter = _rootFilterParser.Parse(filterStatement);
childrenFilters.Add(childFilter);
}
return(CreateFilter(childrenFilters));
}
public async Task <SearchResult> Search(SearchParameter searchParameter)
{
if (searchParameter == null)
{
throw new ArgumentNullException(nameof(searchParameter));
}
var filter = searchParameter.Filter;
if (string.IsNullOrWhiteSpace(filter))
{
filter = "select$*";
}
var interpreter = _filterParser.Parse(filter);
var events = interpreter.Execute(_context.Events);
int totalResult = await events.CountAsync().ConfigureAwait(false);
var pagedResult = events.Skip(searchParameter.StartIndex).Take(searchParameter.Count);
return(new SearchResult(totalResult, await pagedResult.ToListAsync().ConfigureAwait(false)));
}
// Dynamically build SQL where clause.
// See: https://stackoverflow.com/a/39183597
public static IQueryable <T> Filter <T>(this IQueryable <T> source, IFilterParser parser, string filter)
{
var pred = parser.Parse <T>("x", filter);
return(source.Where(pred as Expression <Func <T, bool> >));
}
public async Task <ApiActionResult> Execute(string id, JObject jObj, string schemaId, string locationPattern)
{
// 1. Check parameters.
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentNullException(nameof(id));
}
if (jObj == null)
{
throw new ArgumentNullException(nameof(jObj));
}
if (string.IsNullOrWhiteSpace(schemaId))
{
throw new ArgumentNullException(nameof(schemaId));
}
_parametersValidator.ValidateLocationPattern(locationPattern);
// 2. Check representation exists
var representation = await _representationStore.GetRepresentation(id);
if (representation == null)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.NotFound,
string.Format(ErrorMessages.TheResourceDoesntExist, id)));
}
// 3. Get patch operations.
ErrorResponse errorResponse;
var operations = _patchRequestParser.Parse(jObj, out errorResponse);
if (operations == null)
{
return(_apiResponseFactory.CreateError(
(HttpStatusCode)errorResponse.Status,
errorResponse));
}
// 4. Process operations.
foreach (var operation in operations)
{
// 4.1 Check path is filled-in.
if (operation.Type == PatchOperations.remove &&
string.IsNullOrWhiteSpace(operation.Path))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.ThePathNeedsToBeSpecified, HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.InvalidSyntax)));
}
// 4.2 Check value is filled-in.
if ((operation.Type == PatchOperations.add || operation.Type == PatchOperations.replace) &&
operation.Value == null)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheValueNeedsToBeSpecified, HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.InvalidSyntax)));
}
// 4.3 Process filter & get values.
IEnumerable <RepresentationAttribute> attrs = null;
IEnumerable <RepresentationAttribute> filteredAttrs = null;
if (!string.IsNullOrWhiteSpace(operation.Path))
{
// 4.3.1 Process filter.
var filter = _filterParser.Parse(operation.Path);
var filtered = filter.Evaluate(representation);
if (filtered == null || !filtered.Any())
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheFilterIsNotCorrect, HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.InvalidFilter)
));
}
// 4.3.2 Get targeted attributes.
var target = _filterParser.GetTarget(operation.Path);
var filterRepresentation = _filterParser.Parse(target);
attrs = filterRepresentation.Evaluate(representation);
if (operation.Type == PatchOperations.remove)
{
// 4.3.3 If operation = remove then values are not retrieved.
filteredAttrs = filtered;
}
else
{
// 4.3.4 if operation = replace or add then retrieve values.
var name = filtered.First().SchemaAttribute.Name;
var token = operation.Value.SelectToken(name);
if (token == null)
{
token = new JObject();
token[name] = operation.Value;
}
var value = _jsonParser.GetRepresentation(token, filtered.First().SchemaAttribute, CheckStrategies.Standard);
if (!value.IsParsed)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(value.ErrorMessage, HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.InvalidSyntax)));
}
filteredAttrs = new[] { value.RepresentationAttribute };
}
}
// 4.4 If there's no filter then parse the value with the schema.
if (filteredAttrs == null)
{
if (operation.Value != null)
{
var repr = await _representationRequestParser.Parse(operation.Value, schemaId, CheckStrategies.Standard);
if (!repr.IsParsed)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(repr.ErrorMessage, HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.InvalidSyntax)));
}
filteredAttrs = repr.Representation.Attributes;
attrs = representation.Attributes;
}
}
foreach (var filteredAttr in filteredAttrs)
{
var attr = attrs.FirstOrDefault(a => a.SchemaAttribute.Name == filteredAttr.SchemaAttribute.Name);
// 4.5.1 Check mutability.
if (filteredAttr.SchemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.Immutable ||
filteredAttr.SchemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.ReadOnly)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheImmutableAttributeCannotBeUpdated, filteredAttr.SchemaAttribute.Name), HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.Mutability)));
}
// 4.5.2 Check uniqueness
if (filteredAttr.SchemaAttribute.Uniqueness == Common.Constants.SchemaAttributeUniqueness.Server) // TH : SELECT THE VALUE AND CHECK THE UNIQUENESS.
{
var filter = _filterParser.Parse(filteredAttr.FullPath);
var uniqueAttrs = await _representationStore.SearchValues(representation.ResourceType, filter);
if (uniqueAttrs.Any())
{
if (uniqueAttrs.Any(a => a.CompareTo(filteredAttr) == 0))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheAttributeMustBeUnique, filteredAttr.SchemaAttribute.Name), HttpStatusCode.BadRequest, Common.Constants.ScimTypeValues.Uniqueness)));
}
}
}
switch (operation.Type)
{
// 4.5.3.1 Remove attributes.
case PatchOperations.remove:
if (attr == null)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheAttributeDoesntExist, filteredAttr.SchemaAttribute.Name), HttpStatusCode.BadRequest)
));
}
if (filteredAttr.SchemaAttribute.MultiValued)
{
// 4.5.3.1.1 Remove attribute from array
if (!Remove(attr, filteredAttr))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheRepresentationCannotBeRemoved, HttpStatusCode.BadRequest)
));
}
}
else
{
// 4.5.3.1.2 Remove attribute from complex representation.
if (attr.Parent != null)
{
var complexParent = attr.Parent as ComplexRepresentationAttribute;
if (complexParent == null)
{
continue;
}
complexParent.Values = complexParent.Values.Where(v => !v.Equals(attr));
}
else
{
representation.Attributes = representation.Attributes.Where(v => !v.Equals(attr));
}
}
break;
// 4.5.3.2 Add attribute.
case PatchOperations.add:
if (string.IsNullOrWhiteSpace(operation.Path) && attr == null)
{
representation.Attributes = representation.Attributes.Concat(new[] { filteredAttr });
continue;
}
if (attr == null)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheAttributeDoesntExist, filteredAttr.SchemaAttribute.Name), HttpStatusCode.BadRequest)
));
}
if (!filteredAttr.SchemaAttribute.MultiValued)
{
if (!Set(attr, filteredAttr))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheRepresentationCannotBeSet, HttpStatusCode.BadRequest)
));
}
}
else
{
if (!Add(attr, filteredAttr))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheRepresentationCannotBeAdded, HttpStatusCode.BadRequest)
));
}
}
break;
// 4.5.3.3 Replace attribute
case PatchOperations.replace:
if (attr == null)
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheAttributeDoesntExist, filteredAttr.SchemaAttribute.Name), HttpStatusCode.BadRequest)
));
}
if (attr.SchemaAttribute.MultiValued)
{
if (!SetEnum(attr, filteredAttr))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheRepresentationCannotBeSet, HttpStatusCode.BadRequest)
));
}
}
else
{
if (!Set(attr, filteredAttr))
{
return(_apiResponseFactory.CreateError(
HttpStatusCode.BadRequest,
_errorResponseFactory.CreateError(ErrorMessages.TheRepresentationCannotBeSet, HttpStatusCode.BadRequest)
));
}
}
break;
}
}
}
// 5. Save the representation.
representation.Version = Guid.NewGuid().ToString();
await _representationStore.UpdateRepresentation(representation);
// 6. Returns the JSON representation.
var response = await _responseParser.Parse(representation, locationPattern.Replace("{id}", id), schemaId, OperationTypes.Modification);
return(_apiResponseFactory.CreateResultWithContent(HttpStatusCode.OK,
response.Object,
response.Location,
representation.Version,
representation.Id));
}
public void When_Passing_Null_Or_Empty_Parameter_Then_Exception_Is_Thrown()
{
// ARRANGE
InitializeFakeObjects();
// ACTS & ASSERTS
Assert.Throws <ArgumentNullException>(() => _filterParser.Parse(null));
Assert.Throws <ArgumentNullException>(() => _filterParser.Parse(string.Empty));
}
public async Task When_Filtering_Groups_Then_Correct_Results_Are_Returned()
{
// ARRANGE
InitializeFakeObjects();
_schemaStoreStub.Setup(s => s.GetSchema(It.IsAny <string>()))
.Returns(_schemaStore.GetSchema(Common.Constants.SchemaUrns.Group));
var firstObj = JObject.Parse(@"{'schemas': ['urn:ietf:params:scim:schemas:core:2.0:Group']," +
"'displayName': 'Group A'," +
"'members': [" +
"{" +
"'type': 'Group1'," +
"'value': 'bulkId:ytrewq'" +
"}" +
"]}");
var secondObj = JObject.Parse(@"{'schemas': ['urn:ietf:params:scim:schemas:core:2.0:Group']," +
"'displayName': 'Group A'," +
"'members': [" +
"{" +
"'type': 'Group3'," +
"'value': 'bulkId:ytrewq'" +
"}" +
"]}");
var thirdObj = JObject.Parse(@"{'schemas': ['urn:ietf:params:scim:schemas:core:2.0:Group']," +
"'displayName': 'Group A'," +
"'members': [" +
"{" +
"'type': 'Group2'," +
"'value': 'bulkId:ytrewq'" +
"}" +
"]}");
var firstGroup = await _requestParser.Parse(firstObj, Common.Constants.SchemaUrns.Group, CheckStrategies.Strong);
var secondGroup = await _requestParser.Parse(secondObj, Common.Constants.SchemaUrns.Group, CheckStrategies.Strong);
var thirdGroup = await _requestParser.Parse(thirdObj, Common.Constants.SchemaUrns.Group, CheckStrategies.Strong);
var groups = new[] { firstGroup.Representation, secondGroup.Representation, thirdGroup.Representation };
var searchOrderAscending = new SearchParameter
{
Filter = null,
SortBy = _filterParser.Parse("members.type"),
SortOrder = SortOrders.Ascending
};
var searchOrderDescending = new SearchParameter
{
Filter = null,
SortBy = _filterParser.Parse("members.type"),
SortOrder = SortOrders.Descending
};
var searchOrderAscFilterPaginate = new SearchParameter
{
Filter = _filterParser.Parse("displayName sw Group"),
Count = 2,
StartIndex = 2,
Attributes = new [] { _filterParser.Parse("members.type") },
SortBy = _filterParser.Parse("members.type"),
SortOrder = SortOrders.Ascending
};
var searchOrderAscExcludeAttrsPaginate = new SearchParameter
{
Filter = _filterParser.Parse("displayName sw Group"),
Count = 2,
StartIndex = 2,
ExcludedAttributes = new[] { _filterParser.Parse("members.value") },
SortBy = _filterParser.Parse("members.type"),
SortOrder = SortOrders.Ascending
};
// ACTS
var ascendingResult = _responseParser.Filter(groups, searchOrderAscending, 0);
var descendingResult = _responseParser.Filter(groups, searchOrderDescending, 0);
var filteredResult = _responseParser.Filter(groups, searchOrderAscFilterPaginate, 0);
var secondFilteredResult = _responseParser.Filter(groups, searchOrderAscExcludeAttrsPaginate, 0);
// ASSERTS
Assert.NotNull(ascendingResult);
Assert.NotNull(descendingResult);
Assert.NotNull(filteredResult);
Assert.NotNull(secondFilteredResult);
}
private async Task <UpdateResponse> UpdateAttribute(RepresentationAttribute source, RepresentationAttribute target, string resourceType)
{
var result = new UpdateResponse();
var complexSource = source as ComplexRepresentationAttribute;
var complexTarget = target as ComplexRepresentationAttribute;
if (complexTarget != null)
{
var schemaAttribute = complexTarget.SchemaAttribute;
if (schemaAttribute.MultiValued)
{
// Check mutability
if (schemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.Immutable)
{
if (complexTarget.CompareTo(complexSource) != 0)
{
result.SetError(_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheImmutableAttributeCannotBeUpdated, schemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Mutability));
return(result);
}
}
// Check uniqueness
if (schemaAttribute.Uniqueness == Common.Constants.SchemaAttributeUniqueness.Server)
{
var filter = _filterParser.Parse(complexTarget.FullPath);
var uniqueAttrs = await _representationStore.SearchValues(resourceType, filter);
if (uniqueAttrs.Any())
{
if (uniqueAttrs.Any(a => a.CompareTo(complexTarget) == 0))
{
result.SetError(_errorResponseFactory.CreateError(
string.Format(ErrorMessages.TheAttributeMustBeUnique, complexTarget.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Uniqueness));
return(result);
}
}
}
}
complexSource.Values = complexTarget.Values;
return(result);
}
// Check mutability
if (target.SchemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.Immutable)
{
if (source.CompareTo(target) != 0)
{
result.SetError(_errorResponseFactory.CreateError(string.Format(ErrorMessages.TheImmutableAttributeCannotBeUpdated, target.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Mutability));
return(result);
}
}
// Check uniqueness
if (target.SchemaAttribute.Uniqueness == Common.Constants.SchemaAttributeUniqueness.Server)
{
var filter = _filterParser.Parse(target.FullPath);
var uniqueAttrs = await _representationStore.SearchValues(resourceType, filter);
if (uniqueAttrs.Any())
{
if (uniqueAttrs.Any(a => a.CompareTo(target) == 0))
{
result.SetError(_errorResponseFactory.CreateError(
string.Format(ErrorMessages.TheAttributeMustBeUnique, target.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Uniqueness));
return(result);
}
}
}
// Assign the values
AssignValues(source, target);
return(result);
}
public static IEnumerable <T> Filter <T>(this IEnumerable <T> source, IFilterParser parser, string filter)
{
var pred = parser.Parse <T>("x", filter);
return(source.Where(pred.Compile() as Func <T, bool>));
}
private bool UpdateAttribute(RepresentationAttribute source, RepresentationAttribute target, IEnumerable <Representation> allRepresentations, out ErrorResponse error)
{
error = null;
var complexSource = source as ComplexRepresentationAttribute;
var complexTarget = target as ComplexRepresentationAttribute;
if (complexTarget != null)
{
var schemaAttribute = complexTarget.SchemaAttribute;
if (schemaAttribute.MultiValued)
{
// Check mutability
if (schemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.Immutable)
{
if (complexTarget.CompareTo(complexSource) != 0)
{
error = _errorResponseFactory.CreateError(string.Format(ErrorMessages.TheImmutableAttributeCannotBeUpdated, schemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Mutability);
return(false);
}
}
// Check uniqueness
if (schemaAttribute.Uniqueness == Common.Constants.SchemaAttributeUniqueness.Server && allRepresentations != null && allRepresentations.Any())
{
var filter = _filterParser.Parse(complexTarget.FullPath);
var uniqueAttrs = new List <RepresentationAttribute>();
foreach (var records in allRepresentations.Select(r => filter.Evaluate(r)))
{
uniqueAttrs.AddRange(records);
}
if (uniqueAttrs.Any())
{
if (uniqueAttrs.Any(a => a.CompareTo(complexTarget) == 0))
{
error = _errorResponseFactory.CreateError(
string.Format(ErrorMessages.TheAttributeMustBeUnique, complexTarget.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Uniqueness);
return(false);
}
}
}
}
complexSource.Values = complexTarget.Values;
return(true);
}
// Check mutability
if (target.SchemaAttribute.Mutability == Common.Constants.SchemaAttributeMutability.Immutable)
{
if (source.CompareTo(target) != 0)
{
error = _errorResponseFactory.CreateError(string.Format(ErrorMessages.TheImmutableAttributeCannotBeUpdated, target.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Mutability);
return(false);
}
}
// Check uniqueness
if (target.SchemaAttribute.Uniqueness == Common.Constants.SchemaAttributeUniqueness.Server && allRepresentations != null && allRepresentations.Any())
{
var filter = _filterParser.Parse(target.FullPath);
var uniqueAttrs = new List <RepresentationAttribute>();
foreach (var records in allRepresentations.Select(r => filter.Evaluate(r)))
{
uniqueAttrs.AddRange(records);
}
if (uniqueAttrs.Any())
{
if (uniqueAttrs.Any(a => a.CompareTo(target) == 0))
{
error = _errorResponseFactory.CreateError(
string.Format(ErrorMessages.TheAttributeMustBeUnique, target.SchemaAttribute.Name),
HttpStatusCode.BadRequest,
Common.Constants.ScimTypeValues.Uniqueness);
return(false);
}
}
}
// Assign the values
return(AssignValues(source, target));
}
/// <summary>
/// 指定したテキスト抽出器でテキスト化したものをインデックス化
/// テキスト抽出器の種類は以下のとおり
/// ・Apache Tika
/// ・IFilter
/// </summary>
/// <param name="path"></param>
/// <param name="indexWriter"></param>
private bool AddDocument(string path, IndexWriter indexWriter, string threadName, Dictionary <string, DocInfo> docDic)
{
string filename = System.IO.Path.GetFileName(path);
string extension = System.IO.Path.GetExtension(path);
FileInfo fi = new FileInfo(path);
if (extension == "" ||
!_targetExtensionDic.ContainsKey(extension.ToLower()))
{
//拡張子なし or 対象拡張子外
AppObject.Logger.Info(threadName + ":" + "Out of target extension. Skipped: " + path);
Interlocked.Increment(ref _skippedCount);
return(false);
}
if (extension.ToLower() != ".mp4" && fi.Length > this.FileSizeLimit)
{
//サイズオーバー(mp4は対象外)
AppObject.Logger.Info(threadName + ":" + "File size over. Skipped: " + path);
Interlocked.Increment(ref _skippedCount);
return(false);
}
//存在するドキュメントか？
if (docDic != null && docDic.ContainsKey(path))
{
DocInfo di = docDic[path];
di.Exists = true;
docDic[path] = di;
//更新日時チェック(秒単位で比較)
if (di.UpdateDate < DateTimeUtil.Truncate(fi.LastWriteTime, TimeSpan.FromSeconds(1)))
{
//更新されている場合Delete+Insert
Term t = new Term(LuceneIndexBuilder.Path, di.Path);
indexWriter.DeleteDocuments(t);
}
else
{
//更新されていない。
AppObject.Logger.Info(threadName + ":" + "No updated. Skipped: " + path);
Interlocked.Increment(ref _skippedCount);
return(false);
}
}
//ドキュメント追加
Document doc = new Document();
if (extension.ToLower() == ".md")
{
//Markdown形式
string content = ReadToString(path);
doc.Add(new Field(Content, content, _hilightFieldType));
}
else if (extension.ToLower() == ".txt")
{
//TXTファイル
var sjis = Encoding.GetEncoding("Shift_JIS");
if (FileUtil.GetTextEncoding(path) == sjis)
{
string content = "";
using (var reader = new StreamReader(path, sjis)) {
content = reader.ReadToEnd();
}
doc.Add(new Field(Content, content, _hilightFieldType));
}
else
{
if (_txtExtractMode == TextExtractModes.Tika)
{
var content = _txtExtractor.Extract(path);
doc.Add(new Field(Content, content.Text, _hilightFieldType));
}
else
{
doc.Add(new Field(Content, IFilterParser.Parse(path), _hilightFieldType));
}
}
}
else
{
if (_txtExtractMode == TextExtractModes.Tika)
{
var content = _txtExtractor.Extract(path);
doc.Add(new Field(Content, content.Text, _hilightFieldType));
}
else
{
doc.Add(new Field(Content, IFilterParser.Parse(path), _hilightFieldType));
}
}
doc.Add(new StringField(Path, path, FieldStore.YES));
doc.Add(new StringField(Title, filename.ToLower(), FieldStore.YES));
doc.Add(new StringField(Extension, extension.ToLower(), FieldStore.YES));
//NOTE:Date型のFieldは存在しないのでlongで保持
long l = long.Parse(fi.LastWriteTime.ToString("yyyyMMddHHmmss"));
doc.Add(new LongPoint(UpdateDate, l));
doc.Add(new StoredField(UpdateDate, l));
//doc.Add(new StringField(UpdateDate,
// DateTools.DateToString(_sdf.parse(fi.LastWriteTime.ToString("yyyy/MM/dd")), DateToolsResolution.DAY),
// FieldStore.YES));
indexWriter.AddDocument(doc);
return(true);
}
public IFilter Parse(BsonValue filter)
{
var childFilter = _rootFilterParser.Parse(filter);
return(new NotFilter(childFilter));
}