private static void ProcessFeed(HttpListenerRequest p_HttpRequest)
{
s_Response.statusCode = HttpStatusCode.OK;
s_Response.reason = "";
if (p_HttpRequest != null && p_HttpRequest.HttpMethod == HttpMethod.Post.Method.ToUpperInvariant())
{
ConsoleUtilities.WriteLine("Processing the GSA Feed.", ConsoleColor.Green);
string sourceId = s_Configuration.SourceId;
string orgId = s_Configuration.OrganizationId;
string apiKey = s_Configuration.ApiKey;
// Read in the query string parameters to optionally allow the post to direct the push.
// The SourceId is not optional.
foreach (string q_key in p_HttpRequest.QueryString.AllKeys)
{
if (string.Equals(q_key, "sourceid", StringComparison.OrdinalIgnoreCase))
{
sourceId = p_HttpRequest.QueryString.Get(q_key);
}
else if (string.Equals(q_key, "orgid", StringComparison.OrdinalIgnoreCase))
{
orgId = p_HttpRequest.QueryString.Get(q_key);
}
else if (string.Equals(q_key, "apikey", StringComparison.OrdinalIgnoreCase))
{
apiKey = p_HttpRequest.QueryString.Get(q_key);
}
else if (string.Equals(q_key, "providerid", StringComparison.OrdinalIgnoreCase))
{
s_Configuration.ProviderId = p_HttpRequest.QueryString.Get(q_key);
}
}
if (sourceId != null)
{
string feedFilePath = p_HttpRequest.Headers.Get("FeedFilePath");
GsaFeedParser parser = new GsaFeedParser(feedFilePath);
GsaFeedHeader header = parser.ParseFeedHeader();
if (header == null)
{
m_Logger.Fatal("Malformed XML exception, aborting.");
return;
}
string datasource = header.DataSource;
string feedtype = header.FeedType.ToString();
ICoveoPushApiConfig clientConfig = new CoveoPushApiConfig(s_Configuration.PushApiEndpointUrl, s_Configuration.PlatformApiEndpointUrl, apiKey, orgId);
ICoveoPushApiClient client = new CoveoPushApiClient(clientConfig);
m_Logger.Info("--------------------------------------------------------");
m_Logger.Info("Organization: " + orgId);
m_Logger.Info("Security provider: " + s_Configuration.ProviderId);
m_Logger.Info("Push source: " + sourceId);
m_Logger.Info("Datasource: " + datasource + ", Type: " + feedtype);
m_Logger.Info("Feed File: " + feedFilePath);
m_Logger.Info("Records:");
int addedDocs = 0;
int deletedDocs = 0;
int ignoredDocs = 0;
ulong orderingIdRef = 0;
ulong firstOrderingIdRef = 0;
try
{
client.ActivityManager.UpdateSourceStatus(sourceId, header.FeedType == GsaFeedType.Full ? SourceStatusType.Refresh : SourceStatusType.Incremental);
} catch (Exception e) {
m_Logger.Error("Failed to update source status: " + e.Message);
}
//foreach (GsaFeedAcl acl in parser.ParseFeedAcl()) {
// Console.WriteLine(acl.ToString());
//}
foreach (GsaFeedRecord record in parser.ParseFeedRecords())
{
ConsoleUtilities.WriteLine("{4}>>> {0}|{1}|{2}|{3}",
record.Action == GsaFeedRecordAction.Delete ? ConsoleColor.Yellow : ConsoleColor.Cyan,
record.Action.ToString().ToUpperInvariant(),
record.Url, record.MimeType ?? "None",
record.LastModified?.ToUniversalTime().ToString(CultureInfo.InvariantCulture) ?? "Unspecified",
DateTime.Now);
m_Logger.Info("Processing: " + record.Action.ToString().ToUpperInvariant() + " " + record.Url);
if ((record.Action == GsaFeedRecordAction.Add) || (record.Action == GsaFeedRecordAction.Unspecified))
{
try
{
//We need to push the acl virtual groups
PushGroupFromAcl(client.PermissionManager, s_Configuration.ProviderId, record.Acl);
orderingIdRef = client.DocumentManager.AddOrUpdateDocument(sourceId,
CreateDocumentFromRecord(record, header.FeedType == GsaFeedType.MetadataAndUrl),
null);
addedDocs++;
m_Logger.Info("Success: " + record.Action.ToString().ToUpperInvariant() + " " + record.Url);
}
catch (Exception e)
{
m_Logger.Error("Failed to add item: " + record.Url);
m_Logger.Error("Reason: " + e.Message);
}
}
else if (record.Action == GsaFeedRecordAction.Delete)
{
record.Url = record.Url.Replace("&", "|");
orderingIdRef = client.DocumentManager.DeleteDocument(sourceId, record.Url, null);
deletedDocs++;
}
else
{
m_Logger.Error("No action was specified for the record " + record.Url);
ignoredDocs++;
}
if (firstOrderingIdRef == 0)
{
firstOrderingIdRef = orderingIdRef;
}
// Note that each record may contain attachments which also need to be pushed with the metadata of their parent
// record. This version supports an ATTACHMENTS tag to specify the path where the attachment files can be found.
if (record.Attachments != null)
{
// Create a duplicate of the parent record and update the body based on the file input
GsaFeedRecord attachRecord = record;
string recordBaseUrl = record.Url;
// Get a list of files that should be pushed as part of this record.
string dirPath = record.Attachments.Value;
m_Logger.Info("Processing attachment folder: " + dirPath);
//if (Directory.Exists(dirPath))
//{
try
{
string[] fileList = Directory.GetFiles(dirPath);
foreach (string filePath in fileList)
{
m_Logger.Info("Processing: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + filePath);
string sourceFileName = Path.GetFileName(@filePath);
string sourceFileExt = Path.GetExtension(@filePath);
string sourceFile = @filePath;
string targetFile = Path.Combine(@s_Configuration.TempFolder, String.Format("{0}_{1}.{2}", sourceFileName, Guid.NewGuid(), sourceFileExt));
int buffLength = 24 * 1024; //24KB
byte[] buff = new byte[buffLength];
if ((attachRecord.Action == GsaFeedRecordAction.Delete))
{
attachRecord.Url = recordBaseUrl + "|" + sourceFileName;
attachRecord.Url = attachRecord.Url.Replace("&", "|");
orderingIdRef = client.DocumentManager.DeleteDocument(sourceId, attachRecord.Url, null);
deletedDocs++;
}
else
{
try
{
using (FileStream sourceStream = File.OpenRead(sourceFile))
{
using (FileStream targetStream = File.OpenWrite(targetFile))
{
using (ZlibStream zipStream = new ZlibStream(targetStream, Ionic.Zlib.CompressionMode.Compress))
{
int offset = 0;
int count = buffLength;
int numRead = 0;
do
{
numRead = sourceStream.Read(buff, offset, count);
zipStream.Write(buff, 0, numRead);
}while (numRead == buffLength);
}
}
}
}
catch (Exception e)
{
m_Logger.Error("Failed to process attachment: " + sourceFile + ", " + e.Message);
m_Logger.Error("Trace: " + e.StackTrace);
// Skip to the next attachment
continue;
}
byte[] fileBytes = File.ReadAllBytes(@targetFile);
attachRecord.Content.Value = Convert.ToBase64String(fileBytes);
attachRecord.Content.Encoding = GsaFeedContentEncoding.Base64Compressed;
if ((attachRecord.Action == GsaFeedRecordAction.Add))
{
// Update the record url so it doesn't overwrite the parent, but leave the displayUrl as is
attachRecord.Url = recordBaseUrl + "|" + sourceFileName;
//We need to push the acl virtual groups
PushGroupFromAcl(client.PermissionManager, s_Configuration.ProviderId, attachRecord.Acl);
int parentIdx = attachRecord.Metadata.Values.FindIndex(item => item.Name == "uid");
if (parentIdx >= 0)
{
try
{
string parentId = attachRecord.Metadata.Values[parentIdx].Content;
client.DocumentManager.AddOrUpdateDocument(sourceId,
CreateDocumentFromRecord(attachRecord, header.FeedType == GsaFeedType.MetadataAndUrl, parentId, sourceFileExt),
null);
addedDocs++;
m_Logger.Info("Success: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + filePath);
}
catch (Exception e)
{
m_Logger.Error("Failed to add item: " + recordBaseUrl);
m_Logger.Error("Reason: " + e.Message);
}
}
else
{
try
{
m_Logger.Warning("Metadata field \"uid\" missing in record. Attachment will not be bound to the parent.");
client.DocumentManager.AddOrUpdateDocument(sourceId,
CreateDocumentFromRecord(attachRecord, header.FeedType == GsaFeedType.MetadataAndUrl),
null);
addedDocs++;
m_Logger.Info("Success: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + attachRecord.Url);
}
catch (Exception e)
{
m_Logger.Error("Failed to add item: " + recordBaseUrl);
m_Logger.Error("Reason: " + e.Message);
}
}
}
}
// Remove this file from the temp space
File.Delete(filePath);
}
// Remove this directory from the temp space
if (Directory.GetFiles(dirPath).Length > 0)
{
m_Logger.Error("Attachment folder is not empty: " + dirPath);
}
else
{
Directory.Delete(dirPath);
}
}
catch (Exception e)
{
m_Logger.Error("Failed to read attachment: " + dirPath + ", " + e.Message);
m_Logger.Error("Trace: " + e.StackTrace);
}
//} else {
// m_Logger.Warning("Attachment folder does not exist: " + dirPath);
//}
}
}
if (header.FeedType == GsaFeedType.Full)
{
m_Logger.Info("Full feed detected - Deleting old documents. Reference ordering Id: " + firstOrderingIdRef);
client.DocumentManager.DeleteDocumentsOlderThan(sourceId, firstOrderingIdRef, 5);
}
try
{
client.ActivityManager.UpdateSourceStatus(sourceId, SourceStatusType.Idle);
}
catch (Exception e)
{
}
m_Logger.Info("The feed was processed.");
m_Logger.Info(" ");
m_Logger.Info("Statistics:");
m_Logger.Info("> Added documents: " + addedDocs);
m_Logger.Info("> Deleted documents: " + deletedDocs);
m_Logger.Info("> Ignored documents: " + ignoredDocs);
m_Logger.Info(" ");
// The local XML files are no longer needed
var files = new DirectoryInfo(s_Configuration.TempFolder).GetFiles("*.*");
foreach (var file in files)
{
if (DateTime.UtcNow - file.CreationTimeUtc > TimeSpan.FromDays(10))
{
File.Delete(file.FullName);
}
}
}
}
else
{
if (p_HttpRequest == null)
{
m_Logger.Error("No HTTP request to process.");
}
else
{
m_Logger.Error("Invalid received request: " + p_HttpRequest.HttpMethod + " - " + p_HttpRequest.Url);
}
}
}
