/// <summary>
/// Defines a document type that can be stored in the document store by providing functions for accessing key, serializer and deserializers.
/// </summary>
/// <typeparam name="T">The type of the objects that are converted using these functions.</typeparam>
/// <param name="key">A function that returns a string key, this key is limited to 64 characters.</param>
/// <param name="serializer">A function that serializes objects to documents, typically inject Newtonsoft.Json or XmlSerializer.</param>
/// <param name="deserializer">A function that deserializes documents back to objects, typically inject Newtonsoft.Json or XmlSerializer.</param>
public void DefineDocument <T>(Func <T, string> key, Func <T, string> serializer, Func <string, T> deserializer)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key), "Document must provide a key function.");
}
if (serializer == null)
{
throw new ArgumentNullException(nameof(serializer), "Document must provide a serializer. E.g. e => JsonConvert.Serialize(e);");
}
if (deserializer == null)
{
throw new ArgumentNullException(nameof(deserializer), "Document must provide a deserializer. E.g. e => JsonConvert.Deserialize<T>(e);");
}
var type = typeof(T);
if (documentTypes.ContainsKey(type))
{
throw new ArgumentException("Define document can only be called once per type.", type.Name);
}
var definition = new DocumentDefinition {
Type = type,
KeyFunc = TypedToUntypedLambdaSerializer(key),
Serializer = TypedToUntypedLambdaSerializer(serializer),
Deserializer = TypedToUntypedLambdaDeserializer(deserializer),
};
documentTypes.Add(type, definition);
}
public async Task Load()
{
DocumentDefinition definition = new DocumentDefinition();
parser.Setup(item => item.ParseDocument(It.IsAny <DirectoryInfo>(), It.IsAny <FileInfo>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(definition));
List <ProcessingProgress> list = new List <ProcessingProgress>();
var subscription = manager.ProgressUpdate.Subscribe(
progress =>
{
lock (list)
{
list.Add(progress);
}
});
var result = await manager.LoadAll(new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @".\pdf")));
Assert.IsNotNull(result);
Assert.IsNotNull(result.Path);
Assert.AreEqual(1, result.Document.Length);
Assert.AreEqual(2, list.Count);
Assert.AreEqual(1, list[0].Total);
Assert.AreEqual(0, list[0].Current);
Assert.AreEqual(1, list[1].Total);
Assert.AreEqual(1, list[1].Current);
subscription.Dispose();
}
public DocumentDefinitionDetails(DocumentDefinition document)
{
InitializeComponent();
var viewModel = new DocumentDefinitionDetailsViewModel(Navigation);
viewModel.Document = document;
BindingContext = viewModel;
}
public void Setup()
{
parser = new Mock <IDocumentParser>();
testClient = new Mock <ISvmTestClient>();
dataset = new Mock <IArffDataSet>();
review = new Mock <IArffDataRow>();
dataset.Setup(item => item.AddDocument()).Returns(review.Object);
testClient.Setup(item => item.CreateTestDataset()).Returns(dataset.Object);
instance = new LearnedClassifier(parser.Object, testClient.Object);
file = new FileInfo("test.doc");
definition = new DocumentDefinition();
}
public void Profile_GetExistingDocument_Ok()
{
DocumentDefinition doc = _profile.GetDocument("place");
Assert.NotNull(doc);
Assert.Equal("place", doc.Id);
Assert.Equal("SELECT COUNT(*) FROM place;", doc.CountSql);
Assert.Equal("SELECT title AS plttl, description AS pldsc, " +
"details AS pldtl, id AS m_targetid FROM place " +
"ORDER BY place.id " +
"LIMIT {1} OFFSET {0};", doc.DataSql);
Assert.Equal(2, doc.TextFilterChains.Count);
Assert.Equal(2, doc.Tokenizers.Count);
}
/// <summary>
/// Private method to create a media object from a ContentDto
/// </summary>
/// <param name="dto"></param>
/// <param name="docSql"></param>
/// <returns></returns>
private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql)
{
var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
var media = MediaFactory.BuildEntity(dto, contentType);
var docDef = new DocumentDefinition(dto, contentType);
var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });
media.Properties = properties[dto.VersionId];
//on initial construction we don't want to have dirty properties tracked
// http://issues.umbraco.org/issue/U4-1946
((Entity)media).ResetDirtyProperties(false);
return(media);
}
/// <summary>
/// Private method to create a media object from a ContentDto
/// </summary>
/// <param name="d"></param>
/// <param name="versionId"></param>
/// <param name="docSql"></param>
/// <returns></returns>
private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql)
{
var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId);
var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId);
var media = factory.BuildEntity(dto);
var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
var properties = GetPropertyCollection(docSql, new[] { docDef });
media.Properties = properties[dto.NodeId];
//on initial construction we don't want to have dirty properties tracked
// http://issues.umbraco.org/issue/U4-1946
((Entity)media).ResetDirtyProperties(false);
return(media);
}
/// <summary>
/// Créé une nouvelle instance de ElasticStore.
/// </summary>
/// <param name="dataSourceName">Nom de la datasource.</param>
public ElasticStore(string dataSourceName)
{
try {
_definition = DocumentDescriptor.GetDefinition(typeof(TDocument));
_documentTypeName = _definition.DocumentTypeName;
_dataSourceName = dataSourceName ?? throw new ArgumentNullException("dataSourceName");
_indexName = ElasticManager.Instance.LoadSearchSettings(_dataSourceName).IndexName;
_standardHandler = new StandardFacetHandler <TDocument>(_definition);
_portfolioHandler = new PortfolioFacetHandler <TDocument>(_definition);
} catch (Exception e) {
if (_log.IsErrorEnabled)
{
_log.Error("Echec d'instanciation du store.", e);
}
throw new NotSupportedException("Search Broker<" + typeof(TDocument).FullName + "> " + e.Message, e);
}
}
/// <summary>
/// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion.
/// </summary>
/// <param name="dto"></param>
/// <param name="versionId"></param>
/// <param name="docSql"></param>
/// <returns></returns>
private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql)
{
var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId);
var content = factory.BuildEntity(dto);
//Check if template id is set on DocumentDto, and get ITemplate if it is.
if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
{
content.Template = _templateRepository.Get(dto.TemplateId.Value);
}
var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType);
var properties = GetPropertyCollection(docSql, new[] { docDef });
content.Properties = properties[dto.NodeId];
//on initial construction we don't want to have dirty properties tracked
// http://issues.umbraco.org/issue/U4-1946
((Entity)content).ResetDirtyProperties(false);
return(content);
}
/// <summary>
/// Document format/rules validation method
/// </summary>
/// <param name="property">String property being validated</param>
/// <param name="rule">DocumentDefition rule being used</param>
/// <param name="message">ErrorMessage</param>
/// <returns>Validation context</returns>
public ValidationResult <TModel> Rule(Expression <Func <TModel, string> > property, DocumentDefinition rule, string message = null)
{
if (this._breakIfIsInvalid && !this.Success)
{
return(this);
}
this._breakIfIsInvalid = false;
if (!GetMember(property, out var member))
{
return(this);
}
var index = 0;
foreach (var value in this.Values)
{
var propertyValue = value.GetPropertyValue <string>(member.Name);
this.AddItem(rule.Validate(propertyValue), message, member.Name, member.Name, index);
index++;
}
return(this);
}
/// <summary>
/// \u7B7E\u7F72\u6587\u4EF6 \u7B7E\u7F72\u6587\u4EF6
/// </summary>
/// <exception cref="IO.Swagger.Client.ApiException">Thrown when fails to make API call</exception>
/// <param name="clientId"></param>
/// <param name="body"> (optional)</param>
/// <returns>DocumentSummary</returns>
public DocumentSummary CreateDocument(string clientId, DocumentDefinition body = null)
{
ApiResponse <DocumentSummary> localVarResponse = CreateDocumentWithHttpInfo(clientId, body);
return(localVarResponse.Data);
}
public async Task <DocumentDefinition> ParseDocument(DirectoryInfo repositoryPath, FileInfo file, CancellationToken token)
{
logger.Debug("ParseDocument: {0}", file);
Guard.NotNull(() => file, file);
Guard.NotNull(() => repositoryPath, repositoryPath);
Guard.IsValid(() => file, file, info => info.Exists, "invalid file");
var parser = textParserFactory.ConstructParsers(file);
if (parser == NullTextParser.Instance)
{
logger.Debug("Null parser: {0}", file);
return(null);
}
var text = parser.Parse();
DocumentDefinition definition = new DocumentDefinition();
var bytes = File.ReadAllBytes(file.FullName);
token.ThrowIfCancellationRequested();
definition.Crc32 = Crc32CAlgorithm.Compute(bytes);
string path = string.IsNullOrEmpty(repositoryPath.FullName) || repositoryPath.FullName[repositoryPath.FullName.Length - 1] == Path.DirectorySeparatorChar
? repositoryPath.FullName
: $"{repositoryPath.FullName}{Path.DirectorySeparatorChar}";
var directory = path.GetRelativePath(file.DirectoryName);
if (directory == string.Empty)
{
if (file.Directory != null)
{
definition.Labels = new[] { file.Directory.Name };
}
}
else
{
definition.Labels = new[] { directory.Split(Path.DirectorySeparatorChar).First() };
}
definition.Labels = definition.Labels.Select(item => item.CreateLetterText()).ToArray();
definition.Path = file.FullName;
if (!string.IsNullOrWhiteSpace(text))
{
var result = await textSplitter.Splitter.Process(new ParseRequest(text)).ConfigureAwait(false);
token.ThrowIfCancellationRequested();
var review = result.GetReview(textSplitter.DataLoader);
token.ThrowIfCancellationRequested();
var words = review.Items.Where(
item =>
item.POS.WordType != WordType.Number &&
item.POS.WordType != WordType.SeparationSymbol &&
item.POS.WordType != WordType.Symbol &&
item.POS.WordType != WordType.Conjunction &&
item.POS.WordType != WordType.Sentence &&
!item.IsStopWord)
.Select(item => item.Text)
.ToArray();
foreach (var word in words)
{
token.ThrowIfCancellationRequested();
string underlyingWord;
if (!wordsTable.TryGetAddItem(word, word, out underlyingWord))
{
underlyingWord = word;
}
var total = definition.WordsTable.GetSafe(underlyingWord);
total++;
definition.WordsTable[underlyingWord] = total;
}
}
return(definition);
}
/// <summary>
/// \u7B7E\u7F72\u6587\u4EF6 \u7B7E\u7F72\u6587\u4EF6
/// </summary>
/// <exception cref="IO.Swagger.Client.ApiException">Thrown when fails to make API call</exception>
/// <param name="clientId"></param>
/// <param name="body"> (optional)</param>
/// <returns>Task of DocumentSummary</returns>
public async System.Threading.Tasks.Task <DocumentSummary> CreateDocumentAsync(string clientId, DocumentDefinition body = null)
{
ApiResponse <DocumentSummary> localVarResponse = await CreateDocumentAsyncWithHttpInfo(clientId, body);
return(localVarResponse.Data);
}
public static bool ValidateDocument(this string value, DocumentDefinition definition)
{
return(definition.Validate(value));
}
/// <summary>
/// \u7B7E\u7F72\u6587\u4EF6 \u7B7E\u7F72\u6587\u4EF6
/// </summary>
/// <exception cref="IO.Swagger.Client.ApiException">Thrown when fails to make API call</exception>
/// <param name="clientId"></param>
/// <param name="body"> (optional)</param>
/// <returns>Task of ApiResponse (DocumentSummary)</returns>
public async System.Threading.Tasks.Task <ApiResponse <DocumentSummary> > CreateDocumentAsyncWithHttpInfo(string clientId, DocumentDefinition body = null)
{
// verify the required parameter 'clientId' is set
if (clientId == null)
{
throw new ApiException(400, "Missing required parameter 'clientId' when calling CreateDocument");
}
var localVarPath = "/clients/{clientId}/documents";
var localVarPathParams = new Dictionary <String, String>();
var localVarQueryParams = new Dictionary <String, String>();
var localVarHeaderParams = new Dictionary <String, String>(Configuration.DefaultHeader);
var localVarFormParams = new Dictionary <String, String>();
var localVarFileParams = new Dictionary <String, FileParameter>();
Object localVarPostBody = null;
// to determine the Content-Type header
String[] localVarHttpContentTypes = new String[] {
"application/json"
};
String localVarHttpContentType = Configuration.ApiClient.SelectHeaderContentType(localVarHttpContentTypes);
// to determine the Accept header
String[] localVarHttpHeaderAccepts = new String[] {
"application/json"
};
String localVarHttpHeaderAccept = Configuration.ApiClient.SelectHeaderAccept(localVarHttpHeaderAccepts);
if (localVarHttpHeaderAccept != null)
{
localVarHeaderParams.Add("Accept", localVarHttpHeaderAccept);
}
// set "format" to json by default
// e.g. /pet/{petId}.{format} becomes /pet/{petId}.json
localVarPathParams.Add("format", "json");
if (clientId != null)
{
localVarPathParams.Add("clientId", Configuration.ApiClient.ParameterToString(clientId)); // path parameter
}
if (body.GetType() != typeof(byte[]))
{
localVarPostBody = Configuration.ApiClient.Serialize(body); // http body (model) parameter
}
else
{
localVarPostBody = body; // byte array
}
// make the HTTP request
IRestResponse localVarResponse = (IRestResponse)await Configuration.ApiClient.CallApiAsync(localVarPath,
Method.POST, localVarQueryParams, localVarPostBody, localVarHeaderParams, localVarFormParams, localVarFileParams,
localVarPathParams, localVarHttpContentType);
int localVarStatusCode = (int)localVarResponse.StatusCode;
if (localVarStatusCode >= 400)
{
throw new ApiException(localVarStatusCode, "Error calling CreateDocument: " + localVarResponse.Content, localVarResponse.Content);
}
else if (localVarStatusCode == 0)
{
throw new ApiException(localVarStatusCode, "Error calling CreateDocument: " + localVarResponse.ErrorMessage, localVarResponse.ErrorMessage);
}
return(new ApiResponse <DocumentSummary>(localVarStatusCode,
localVarResponse.Headers.ToDictionary(x => x.Name, x => x.Value.ToString()),
(DocumentSummary)Configuration.ApiClient.Deserialize(localVarResponse, typeof(DocumentSummary))));
}
/// <summary>
/// Créé une nouvelle instance de ElasticFacetHandler.
/// </summary>
/// <param name="document">Définition du document.</param>
public ElasticFacetHandler(DocumentDefinition document)
{
_document = document;
}
public void Profile_GetNonExistingDocument_Null()
{
DocumentDefinition doc = _profile.GetDocument("not-existing");
Assert.Null(doc);
}
/// <summary>
/// Créé une nouvelle instance de PortfolioFacetHandler.
/// </summary>
/// <param name="document">Définition du document.</param>
public PortfolioFacetHandler(DocumentDefinition document)
{
_document = document;
}
/// <summary>
/// Créé une nouvelle instance de StandardFacetHandler.
/// </summary>
/// <param name="document">Définition du document.</param>
public StandardFacetHandler(DocumentDefinition document)
{
_document = document;
}
