/// <summary>
/// Indexes the files located in the specified folder.
/// </summary>
/// <param name="p_Args">Parsed command-line arguments.</param>
private static void IndexFiles(ProgramArguments p_Args)
{
string folder = Path.GetFullPath(p_Args.folder);
if (!folder.EndsWith(Path.DirectorySeparatorChar))
{
folder += Path.DirectorySeparatorChar;
}
Console.WriteLine($"Pushing files \"{p_Args.include}\" from folder \"{folder}\"...");
ulong orderingId = RequestOrderingUtilities.CreateOrderingId();
ICoveoPlatformConfig platformConfig = new CoveoPlatformConfig(GetPushApiUrl(p_Args), GetPlatformApiUrl(p_Args), p_Args.apikey, p_Args.organizationid);
using (ICoveoPlatformClient platformClient = new CoveoPlatformClient(platformConfig)) {
IList <PushDocument> documentBatch = new List <PushDocument>();
foreach (FileInfo fileInfo in new DirectoryInfo(folder).EnumerateFiles(p_Args.include, p_Args.recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (!fileInfo.FullName.StartsWith(folder))
{
throw new Exception("Unexpected file gathered from outside the source folder.");
}
Console.WriteLine(fileInfo.FullName.Substring(folder.Length));
PushDocument document = new PushDocument(new Uri(fileInfo.FullName).AbsoluteUri)
{
ModifiedDate = fileInfo.LastWriteTimeUtc
};
document.AddMetadata("title", fileInfo.Name);
document.AddMetadata("fileextension", fileInfo.Extension);
if (fileInfo.Length > 0)
{
PushDocumentHelper.SetBinaryContentFromFileAndCompress(document, fileInfo.FullName);
}
documentBatch.Add(document);
if (documentBatch.Count >= p_Args.batchSize)
{
// Push this batch of documents.
SendBatch(platformClient, documentBatch, p_Args.sourceid, orderingId);
}
}
// Send the (partial) final batch of documents.
SendBatch(platformClient, documentBatch, p_Args.sourceid, orderingId);
// Delete the already indexed files that no longer exist.
platformClient.DocumentManager.DeleteDocumentsOlderThan(p_Args.sourceid, orderingId, null);
}
}
private static PushDocument CreateDocumentFromRecord(GsaFeedRecord p_Record,
bool p_DownloadContent, string p_ParentId, string p_fileExt)
{
IDictionary <string, JToken> metadata = p_Record.ConvertMetadata();
if (p_Record.DisplayUrl == null)
{
p_Record.DisplayUrl = p_Record.Url;
}
p_Record.Url = p_Record.Url.Replace("&", "|");
metadata.Add("clickableuri", p_Record.DisplayUrl);
metadata.Add(nameof(p_Record.DisplayUrl), p_Record.DisplayUrl);
metadata.Add(nameof(p_Record.Lock), p_Record.Lock);
metadata.Add(nameof(p_Record.MimeType), p_Record.MimeType);
metadata.Add(nameof(p_Record.PageRank), p_Record.PageRank);
metadata.Add(nameof(p_Record.Scoring), p_Record.Scoring);
metadata.Add(nameof(p_Record.Url), p_Record.Url);
metadata.Add(nameof(p_Record.AuthMethod), p_Record.AuthMethod.ToString());
metadata.Add(nameof(p_Record.CrawlImmediately), p_Record.CrawlImmediately);
metadata.Add(nameof(p_Record.CrawlOnce), p_Record.CrawlOnce);
PushDocument document = new PushDocument(p_Record.Url)
{
ModifiedDate = p_Record.LastModified ?? DateTime.MinValue,
Metadata = metadata,
ParentId = p_ParentId,
FileExtension = p_fileExt
};
if (p_Record.Acl != null)
{
DocumentPermissionSet currentDocSet = new DocumentPermissionSet();
PermissionIdentity denyGroup = new PermissionIdentity(p_Record.Url + DISALLOW_GROUP, PermissionIdentityType.VirtualGroup);
PermissionIdentity allowGroup = new PermissionIdentity(p_Record.Url + ALLOW_GROUP, PermissionIdentityType.VirtualGroup);
currentDocSet.DeniedPermissions.Add(denyGroup);
currentDocSet.AllowedPermissions.Add(allowGroup);
DocumentPermissionLevel currentDocLevel = new DocumentPermissionLevel();
currentDocLevel.PermissionSets.Add(currentDocSet);
if (p_Record.Acl.ParentAcl != null)
{
GsaFeedAcl currentAcl = p_Record.Acl;
List <DocumentPermissionLevel> allLevels = new List <DocumentPermissionLevel>();
allLevels.Add(currentDocLevel);
int currentLevelIndex = 0;
while (currentAcl.ParentAcl != null)
{
GsaFeedAcl curParentAcl = currentAcl.ParentAcl;
DocumentPermissionSet curParentDocSet = new DocumentPermissionSet();
PermissionIdentity parentDenyGroup = new PermissionIdentity(curParentAcl.DocumentUrl + DISALLOW_GROUP, PermissionIdentityType.VirtualGroup);
PermissionIdentity parentAllowGroup = new PermissionIdentity(curParentAcl.DocumentUrl + ALLOW_GROUP, PermissionIdentityType.VirtualGroup);
//We sill always need the parents in a different set
curParentDocSet.DeniedPermissions.Add(parentDenyGroup);
curParentDocSet.AllowedPermissions.Add(parentAllowGroup);
switch (curParentAcl.InheritanceType)
{
case GsaFeedAclInheritance.BothPermit:
//The parent and the document are in two different sets
allLevels.ElementAt(currentLevelIndex).PermissionSets.Add(curParentDocSet);
break;
case GsaFeedAclInheritance.ChildOverrides:
//The parent is in a lower level than the current document
DocumentPermissionLevel parentLowerDocLevel = new DocumentPermissionLevel();
parentLowerDocLevel.PermissionSets.Add(curParentDocSet);
//We are adding our self after the children
currentLevelIndex++;
allLevels.Insert(currentLevelIndex, parentLowerDocLevel);
break;
case GsaFeedAclInheritance.ParentOverrides:
//The parent is in a higher level than the current document
//on doit ajouter avant l'enfant
DocumentPermissionLevel parentHigherDocLevel = new DocumentPermissionLevel();
parentHigherDocLevel.PermissionSets.Add(curParentDocSet);
allLevels.Insert(currentLevelIndex, parentHigherDocLevel);
break;
case GsaFeedAclInheritance.LeafNode:
//The document is not suppose to have inheritance from a leaf node
ConsoleUtilities.WriteLine("> Warning: You are trying to have inheritance on a LeafNode. Document in error: {0}", ConsoleColor.Yellow, p_Record.Url);
curParentAcl.ParentAcl = null;
break;
}
currentAcl = curParentAcl;
}
//Now we push the permissions
foreach (DocumentPermissionLevel documentPermissionLevel in allLevels)
{
document.Permissions.Add(documentPermissionLevel);
}
}
else
{
//We might need to add the parent level before, so we will not default this action.
document.Permissions.Add(currentDocLevel);
}
}
if (p_DownloadContent)
{
string content = s_HttpDownloader.Download(p_Record.Url);
PushDocumentHelper.SetCompressedEncodedContent(document, Compression.GetCompressedBinaryData(content));
}
else
{
if (p_Record.Content.Encoding == GsaFeedContentEncoding.Base64Compressed)
{
PushDocumentHelper.SetCompressedEncodedContent(document, p_Record.Content.Value.Trim(Convert.ToChar("\n")));
}
else
{
PushDocumentHelper.SetContent(document, p_Record.Content.GetDecodedValue());
}
}
return(document);
}