private static void ProcessGroups(HttpListenerRequest p_HttpRequest) { if (p_HttpRequest != null && p_HttpRequest.HttpMethod == HttpMethod.Post.Method.ToUpperInvariant()) { ConsoleUtilities.WriteLine("Processing the GSA XML Groups.", ConsoleColor.Green); string datasource = p_HttpRequest.QueryString.Get(DATASOURCE); string filename = String.Format("{0}_{1}_Groups.xml", DateTime.Now.Ticks, Guid.NewGuid()); string feedFilePath = Path.Combine(@s_Configuration.TempFolder, filename); using (FileStream output = File.OpenWrite(feedFilePath)) { p_HttpRequest.InputStream.CopyTo(output); } GsaFeedParser parser = new GsaFeedParser(feedFilePath); ICoveoPushApiConfig clientConfig = new CoveoPushApiConfig(s_Configuration.PushApiEndpointUrl, s_Configuration.PlatformApiEndpointUrl, s_Configuration.ApiKey, s_Configuration.OrganizationId); ICoveoPushApiClient client = new CoveoPushApiClient(clientConfig); Console.WriteLine(); Console.WriteLine("--------------------------------------------------------"); Console.WriteLine("Organization: {0}", s_Configuration.OrganizationId); Console.WriteLine("Security provider: {0}", s_Configuration.ProviderId); Console.WriteLine("Push source: {0}", datasource); Console.WriteLine("Feed File: '{0}'", feedFilePath); Console.WriteLine("Records:"); int nbOfGroups = 0; foreach (GsaFeedMembership gsaFeedMembership in parser.ParseFeedGroups()) { nbOfGroups++; PushMemberOfGroup(client.PermissionManager, s_Configuration.ProviderId, gsaFeedMembership); } Console.WriteLine(); ConsoleUtilities.WriteLine("The XML Groups was processed.", ConsoleColor.Green); Console.WriteLine(); Console.WriteLine("Statistics:"); ConsoleUtilities.WriteLine("> Added/Updated groups: {0}", ConsoleColor.Cyan, nbOfGroups); //File.Delete(feedFilePath); } else { if (p_HttpRequest == null) { ConsoleUtilities.WriteError("No HTTP request to process."); s_Response.statusCode = HttpStatusCode.NoContent; s_Response.reason = "No HTTP request to process."; } else { ConsoleUtilities.WriteError("Invalid received request: {1} - {0}.", p_HttpRequest.HttpMethod, p_HttpRequest.Url); s_Response.statusCode = HttpStatusCode.BadRequest; s_Response.reason = "Invalid received request: " + p_HttpRequest.HttpMethod + " - " + p_HttpRequest.Url; } } }
private static void ProcessFeed(HttpListenerRequest p_HttpRequest) { s_Response.statusCode = HttpStatusCode.OK; s_Response.reason = ""; if (p_HttpRequest != null && p_HttpRequest.HttpMethod == HttpMethod.Post.Method.ToUpperInvariant()) { ConsoleUtilities.WriteLine("Processing the GSA Feed.", ConsoleColor.Green); string sourceId = s_Configuration.SourceId; string orgId = s_Configuration.OrganizationId; string apiKey = s_Configuration.ApiKey; // Read in the query string parameters to optionally allow the post to direct the push. // The SourceId is not optional. foreach (string q_key in p_HttpRequest.QueryString.AllKeys) { if (string.Equals(q_key, "sourceid", StringComparison.OrdinalIgnoreCase)) { sourceId = p_HttpRequest.QueryString.Get(q_key); } else if (string.Equals(q_key, "orgid", StringComparison.OrdinalIgnoreCase)) { orgId = p_HttpRequest.QueryString.Get(q_key); } else if (string.Equals(q_key, "apikey", StringComparison.OrdinalIgnoreCase)) { apiKey = p_HttpRequest.QueryString.Get(q_key); } else if (string.Equals(q_key, "providerid", StringComparison.OrdinalIgnoreCase)) { s_Configuration.ProviderId = p_HttpRequest.QueryString.Get(q_key); } } if (sourceId != null) { string feedFilePath = p_HttpRequest.Headers.Get("FeedFilePath"); GsaFeedParser parser = new GsaFeedParser(feedFilePath); GsaFeedHeader header = parser.ParseFeedHeader(); if (header == null) { m_Logger.Fatal("Malformed XML exception, aborting."); return; } string datasource = header.DataSource; string feedtype = header.FeedType.ToString(); ICoveoPushApiConfig clientConfig = new CoveoPushApiConfig(s_Configuration.PushApiEndpointUrl, s_Configuration.PlatformApiEndpointUrl, apiKey, orgId); ICoveoPushApiClient client = new CoveoPushApiClient(clientConfig); m_Logger.Info("--------------------------------------------------------"); m_Logger.Info("Organization: " + orgId); m_Logger.Info("Security provider: " + s_Configuration.ProviderId); m_Logger.Info("Push source: " + sourceId); m_Logger.Info("Datasource: " + datasource + ", Type: " + feedtype); m_Logger.Info("Feed File: " + feedFilePath); m_Logger.Info("Records:"); int addedDocs = 0; int deletedDocs = 0; int ignoredDocs = 0; ulong orderingIdRef = 0; ulong firstOrderingIdRef = 0; try { client.ActivityManager.UpdateSourceStatus(sourceId, header.FeedType == GsaFeedType.Full ? SourceStatusType.Refresh : SourceStatusType.Incremental); } catch (Exception e) { m_Logger.Error("Failed to update source status: " + e.Message); } //foreach (GsaFeedAcl acl in parser.ParseFeedAcl()) { // Console.WriteLine(acl.ToString()); //} foreach (GsaFeedRecord record in parser.ParseFeedRecords()) { ConsoleUtilities.WriteLine("{4}>>> {0}|{1}|{2}|{3}", record.Action == GsaFeedRecordAction.Delete ? ConsoleColor.Yellow : ConsoleColor.Cyan, record.Action.ToString().ToUpperInvariant(), record.Url, record.MimeType ?? "None", record.LastModified?.ToUniversalTime().ToString(CultureInfo.InvariantCulture) ?? "Unspecified", DateTime.Now); m_Logger.Info("Processing: " + record.Action.ToString().ToUpperInvariant() + " " + record.Url); if ((record.Action == GsaFeedRecordAction.Add) || (record.Action == GsaFeedRecordAction.Unspecified)) { try { //We need to push the acl virtual groups PushGroupFromAcl(client.PermissionManager, s_Configuration.ProviderId, record.Acl); orderingIdRef = client.DocumentManager.AddOrUpdateDocument(sourceId, CreateDocumentFromRecord(record, header.FeedType == GsaFeedType.MetadataAndUrl), null); addedDocs++; m_Logger.Info("Success: " + record.Action.ToString().ToUpperInvariant() + " " + record.Url); } catch (Exception e) { m_Logger.Error("Failed to add item: " + record.Url); m_Logger.Error("Reason: " + e.Message); } } else if (record.Action == GsaFeedRecordAction.Delete) { record.Url = record.Url.Replace("&", "|"); orderingIdRef = client.DocumentManager.DeleteDocument(sourceId, record.Url, null); deletedDocs++; } else { m_Logger.Error("No action was specified for the record " + record.Url); ignoredDocs++; } if (firstOrderingIdRef == 0) { firstOrderingIdRef = orderingIdRef; } // Note that each record may contain attachments which also need to be pushed with the metadata of their parent // record. This version supports an ATTACHMENTS tag to specify the path where the attachment files can be found. if (record.Attachments != null) { // Create a duplicate of the parent record and update the body based on the file input GsaFeedRecord attachRecord = record; string recordBaseUrl = record.Url; // Get a list of files that should be pushed as part of this record. string dirPath = record.Attachments.Value; m_Logger.Info("Processing attachment folder: " + dirPath); //if (Directory.Exists(dirPath)) //{ try { string[] fileList = Directory.GetFiles(dirPath); foreach (string filePath in fileList) { m_Logger.Info("Processing: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + filePath); string sourceFileName = Path.GetFileName(@filePath); string sourceFileExt = Path.GetExtension(@filePath); string sourceFile = @filePath; string targetFile = Path.Combine(@s_Configuration.TempFolder, String.Format("{0}_{1}.{2}", sourceFileName, Guid.NewGuid(), sourceFileExt)); int buffLength = 24 * 1024; //24KB byte[] buff = new byte[buffLength]; if ((attachRecord.Action == GsaFeedRecordAction.Delete)) { attachRecord.Url = recordBaseUrl + "|" + sourceFileName; attachRecord.Url = attachRecord.Url.Replace("&", "|"); orderingIdRef = client.DocumentManager.DeleteDocument(sourceId, attachRecord.Url, null); deletedDocs++; } else { try { using (FileStream sourceStream = File.OpenRead(sourceFile)) { using (FileStream targetStream = File.OpenWrite(targetFile)) { using (ZlibStream zipStream = new ZlibStream(targetStream, Ionic.Zlib.CompressionMode.Compress)) { int offset = 0; int count = buffLength; int numRead = 0; do { numRead = sourceStream.Read(buff, offset, count); zipStream.Write(buff, 0, numRead); }while (numRead == buffLength); } } } } catch (Exception e) { m_Logger.Error("Failed to process attachment: " + sourceFile + ", " + e.Message); m_Logger.Error("Trace: " + e.StackTrace); // Skip to the next attachment continue; } byte[] fileBytes = File.ReadAllBytes(@targetFile); attachRecord.Content.Value = Convert.ToBase64String(fileBytes); attachRecord.Content.Encoding = GsaFeedContentEncoding.Base64Compressed; if ((attachRecord.Action == GsaFeedRecordAction.Add)) { // Update the record url so it doesn't overwrite the parent, but leave the displayUrl as is attachRecord.Url = recordBaseUrl + "|" + sourceFileName; //We need to push the acl virtual groups PushGroupFromAcl(client.PermissionManager, s_Configuration.ProviderId, attachRecord.Acl); int parentIdx = attachRecord.Metadata.Values.FindIndex(item => item.Name == "uid"); if (parentIdx >= 0) { try { string parentId = attachRecord.Metadata.Values[parentIdx].Content; client.DocumentManager.AddOrUpdateDocument(sourceId, CreateDocumentFromRecord(attachRecord, header.FeedType == GsaFeedType.MetadataAndUrl, parentId, sourceFileExt), null); addedDocs++; m_Logger.Info("Success: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + filePath); } catch (Exception e) { m_Logger.Error("Failed to add item: " + recordBaseUrl); m_Logger.Error("Reason: " + e.Message); } } else { try { m_Logger.Warning("Metadata field \"uid\" missing in record. Attachment will not be bound to the parent."); client.DocumentManager.AddOrUpdateDocument(sourceId, CreateDocumentFromRecord(attachRecord, header.FeedType == GsaFeedType.MetadataAndUrl), null); addedDocs++; m_Logger.Info("Success: " + attachRecord.Action.ToString().ToUpperInvariant() + " " + attachRecord.Url); } catch (Exception e) { m_Logger.Error("Failed to add item: " + recordBaseUrl); m_Logger.Error("Reason: " + e.Message); } } } } // Remove this file from the temp space File.Delete(filePath); } // Remove this directory from the temp space if (Directory.GetFiles(dirPath).Length > 0) { m_Logger.Error("Attachment folder is not empty: " + dirPath); } else { Directory.Delete(dirPath); } } catch (Exception e) { m_Logger.Error("Failed to read attachment: " + dirPath + ", " + e.Message); m_Logger.Error("Trace: " + e.StackTrace); } //} else { // m_Logger.Warning("Attachment folder does not exist: " + dirPath); //} } } if (header.FeedType == GsaFeedType.Full) { m_Logger.Info("Full feed detected - Deleting old documents. Reference ordering Id: " + firstOrderingIdRef); client.DocumentManager.DeleteDocumentsOlderThan(sourceId, firstOrderingIdRef, 5); } try { client.ActivityManager.UpdateSourceStatus(sourceId, SourceStatusType.Idle); } catch (Exception e) { } m_Logger.Info("The feed was processed."); m_Logger.Info(" "); m_Logger.Info("Statistics:"); m_Logger.Info("> Added documents: " + addedDocs); m_Logger.Info("> Deleted documents: " + deletedDocs); m_Logger.Info("> Ignored documents: " + ignoredDocs); m_Logger.Info(" "); // The local XML files are no longer needed var files = new DirectoryInfo(s_Configuration.TempFolder).GetFiles("*.*"); foreach (var file in files) { if (DateTime.UtcNow - file.CreationTimeUtc > TimeSpan.FromDays(10)) { File.Delete(file.FullName); } } } } else { if (p_HttpRequest == null) { m_Logger.Error("No HTTP request to process."); } else { m_Logger.Error("Invalid received request: " + p_HttpRequest.HttpMethod + " - " + p_HttpRequest.Url); } } }