public override kCura.EventHandler.Response Execute()
{
// Update Security Protocol
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <PreInstallEventHandler>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Audit Log Elastic Search, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Get database context of the instance
IDBContext instanceContext = Helper.GetDBContext(-1);
// Current Relativity version
string currentRelativityVersion = "";
try
{
// Get Relativity version
currentRelativityVersion = instanceContext.ExecuteSqlStatementAsScalar("SELECT [Value] FROM [eddsdbo].[Relativity] WHERE [Key] = 'Version'").ToString();
_logger.LogDebug("Audit Log Elastic Search, current Relativity version: {version}", currentRelativityVersion);
_logger.LogDebug("Audit Log Elastic Search, minimum required Relativity version: {version}", this.MinRelativityVersion);
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Pre Install EventHandler Relativity version error");
response.Success = false;
response.Message = "Pre Install EventHandler Relativity version error";
return(response);
}
// Check Relativity version requirements
if (new Version(currentRelativityVersion) < new Version(this.MinRelativityVersion))
{
_logger.LogError("Audit Log Elastic Search, old Relativity instance (version: {version})", currentRelativityVersion);
response.Success = false;
response.Message = string.Format("This application requires Relativity {0} or later", this.MinRelativityVersion);
return(response);
}
// Log end of Pre Install EventHandler
_logger.LogDebug("Audit Log Elastic Search, Pre Install EventHandler successfully finished");
return(response);
}
/*
* Occurs after the user has selected items and pressed go.
* In this function you can validate the items selected and return a warning/error message.
*/
public override kCura.EventHandler.Response ValidateSelection()
{
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Azure Translator, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Check if all Instance Settings are in place
IDictionary <string, string> instanceSettings = this.GetInstanceSettings(ref response, new string[] { "SourceField", "DestinationField", "LogField", "Cost1MCharacters", "AzureServiceRegion", "AzureSubscriptionKey", "AzureTranslatorEndpoint", "TranslateFrom", "TranslateTo" });
// Check if there was not error
if (!response.Success)
{
return(response);
}
/*
* Calculate the translation costs
*/
// TODO: redo in Object Manager API
try
{
// Construct and execute SQL Query to get the characters count
string sqlText = "SELECT SUM(LEN(CAST([" + instanceSettings["SourceField"].Replace(" ", "") + "] AS NVARCHAR(MAX)))) FROM [EDDSDBO].[Document] AS [Document] JOIN [Resource].[" + this.MassActionTableName + "] AS [MassActionTableName] ON [Document].[ArtifactID] = [MassActionTableName].[ArtifactID]";
_logger.LogDebug("Azure Translator, translation cost SQL Parameter and Query: {query}", sqlText);
long count = (long)this.Helper.GetDBContext(workspaceId).ExecuteSqlStatementAsScalar(sqlText);
// Calculate translation cost
float cost = (count / 1000000f) * float.Parse(instanceSettings["Cost1MCharacters"]);
_logger.LogDebug("Azure Translator, translation cost: {price}CHF, ({chars} chars)", cost.ToString("0.00"), count.ToString());
response.Message = string.Format("Translation cost is {0}CHF, ({1} chars)", cost.ToString("0.00"), count.ToString());
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, translation cost error ({SourceField}, {Cost1MCharacters})", instanceSettings["SourceField"], instanceSettings["Cost1MCharacters"]);
response.Success = false;
response.Message = "Translation cost error";
}
return(response);
}
/*
* Custom method to get required Relativity Instance Settings
*/
private IDictionary <string, string> GetInstanceSettings(ref kCura.EventHandler.Response response, string[] instanceSettingsNames)
{
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();
// Output Dictionary
IDictionary <string, string> instanceSettingsValues = new Dictionary <string, string>();
// Get and validate instance settings
foreach (string name in instanceSettingsNames)
{
try
{
instanceSettingsValues.Add(name, this.Helper.GetInstanceSettingBundle().GetString("Azure.Translator", name));
if (instanceSettingsValues[name].Length <= 0)
{
_logger.LogError("Azure Translator, Instance Settings empty error: {section}/{name}", "Azure.Translator", name);
response.Success = false;
response.Message = "Instance Settings error";
return(instanceSettingsValues);
}
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, Instance Settings error: {section}/{name}", "Azure.Translator", name);
response.Success = false;
response.Message = "Instance Settings error";
return(instanceSettingsValues);
}
_logger.LogDebug("Azure Translator, Instance Setting: {name}=>{value}", name, instanceSettingsValues[name]);
}
// Check Cost1MCharacters Instance Settings is a number
try
{
float.Parse(instanceSettingsValues["Cost1MCharacters"]);
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, Instance Settings error: {section}/{name}", "Azure.Translator", "Cost1MCharacters");
response.Success = false;
response.Message = "Instance Settings error";
return(instanceSettingsValues);
}
return(instanceSettingsValues);
}
public override kCura.EventHandler.Response Execute()
{
// Update Security Protocol
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <PreUninstallEventHandler>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Audit Log Elastic Search, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Get database context of the instance
IDBContext instanceContext = Helper.GetDBContext(-1);
// Update line of the application management and set current workspace to disabled
try
{
// Update the application management table
SqlParameter workspaceIdParam = new SqlParameter("@workspaceId", workspaceId);
instanceContext.ExecuteNonQuerySQLStatement("UPDATE [eddsdbo].[" + this.TableName + "] SET [Status] = 0, [LastUpdated] = CURRENT_TIMESTAMP WHERE [CaseArtifactID] = @workspaceId", new SqlParameter[] { workspaceIdParam });
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Pre Uninstall EventHandler application management table update error");
response.Success = false;
response.Message = "Pre Uninstall EventHandler application management table update error";
return(response);
}
// Log end of Pre Uninstall EventHandler
_logger.LogDebug("Audit Log Elastic Search, Pre Uninstall EventHandler successfully finished");
return(response);
}
public override kCura.EventHandler.Response Execute()
{
// Update Security Protocol
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <PostWorkspaceCreateEventHandlerBase>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Audit Log Elastic Search, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Get database context of the instance
IDBContext instanceContext = Helper.GetDBContext(-1);
// Add line to the application management table for current workspace
try
{
// Insert to the application management table
SqlParameter workspaceIdParam = new SqlParameter("@workspaceId", workspaceId);
instanceContext.ExecuteNonQuerySQLStatement("INSERT INTO [eddsdbo].[" + this.TableName + "] ([CaseArtifactID], [AuditRecordID], [Status], [LastUpdated]) VALUES (@workspaceId, 0, 1, CURRENT_TIMESTAMP)", new SqlParameter[] { workspaceIdParam });
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Post Workspace Create EventHandler application management table insert error");
response.Success = false;
response.Message = "Post Workspace Create EventHandler application management table insert error";
return(response);
}
// Log end of Post Workspace Create EventHandler
_logger.LogDebug("Audit Log Elastic Search, Post Workspace Create EventHandler successfully finished");
return(response);
}
public override kCura.EventHandler.Response Execute()
{
// Update Security Protocol
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <PostInstallEventHandler>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Audit Log Elastic Search, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Get database context of the instance
IDBContext instanceContext = Helper.GetDBContext(-1);
// Existing application management table name
string tableExisting = "";
try
{
// Get application management table
tableExisting = instanceContext.ExecuteSqlStatementAsScalar("SELECT ISNULL((SELECT '" + this.TableName + "' FROM [INFORMATION_SCHEMA].[TABLES] WHERE [TABLE_SCHEMA] = 'eddsdbo' AND [TABLE_NAME] = '" + this.TableName + "'), '')").ToString();
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Post Install EventHandler application management table error");
response.Success = false;
response.Message = "Post Install EventHandler management application table error";
return(response);
}
// If application management table not present, create it
if (tableExisting != this.TableName)
{
instanceContext.BeginTransaction();
try
{
// Create application management table
instanceContext.ExecuteNonQuerySQLStatement("CREATE TABLE [eddsdbo].[" + this.TableName + "] ([CaseArtifactID] [int] NOT NULL, [AuditRecordID] [bigint] NOT NULL, [Status] [bit] NOT NULL, [LastUpdated] [datetime] NOT NULL, [AgentArtifactID] [int] NULL)");
instanceContext.CommitTransaction();
}
catch (Exception e)
{
instanceContext.RollbackTransaction();
_logger.LogError(e, "Audit Log Elastic Search, Post Install EventHandler application management table creation error");
response.Success = false;
response.Message = "Post Install EventHandler application management table creation error";
return(response);
}
}
// Add line to the application management table for current workspace
try
{
// Insert to the application management table
SqlParameter workspaceIdParam = new SqlParameter("@workspaceId", workspaceId);
instanceContext.ExecuteNonQuerySQLStatement("INSERT INTO [eddsdbo].[" + this.TableName + "] ([CaseArtifactID], [AuditRecordID], [Status], [LastUpdated]) VALUES (@workspaceId, 0, 1, CURRENT_TIMESTAMP)", new SqlParameter[] { workspaceIdParam });
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Post Install EventHandler application management table insert error");
response.Success = false;
response.Message = "Post Install EventHandler application management table insert error";
return(response);
}
// Log end of Post Install EventHandler
_logger.LogDebug("Audit Log Elastic Search, Post Install EventHandler successfully finished");
return(response);
}
/*
* Custom method to translate document using Azure Translator
*/
private async Task <int> TranslateDocument(int workspaceId, int documentArtifactId, string sourceField, string destinationField, string logField, string azureServiceRegion, string azureSubscriptionKey, string azureTranslatorEndpoint, string translateFrom, string translateTo)
{
/*
* Custom local function to split string into chunks of defined size with delimiter priority
*/
List <string> SplitMulti(string str, char[] delimiters, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
{
List <string> chunks = new List <string>()
{
str
};
List <string> subChunks;
foreach (char delimiter in delimiters)
{
for (int i = 0; i < chunks.Count; i++)
{
// Check if chunks are all below threshold otherwise split by space
if (chunks[i].Length > maxChunkThreshold)
{
subChunks = SplitSized(chunks[i], delimiter, minChunkThreshold, maxChunkThreshold, smallChunkThreshold);
chunks.RemoveAt(i);
chunks.InsertRange(i, subChunks);
}
}
}
return(chunks);
}
/*
* Custom local function to split string into chunks of defined size
*/
List <string> SplitSized(string str, char delimiter, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
{
string[] split = str.Split(delimiter);
List <string> chunks = new List <string>();
string hlp = split[0];
for (int i = 1; i < split.Length; i++)
{
// Rearange split text into bigger chunks
if (((hlp.Length + split[i].Length) < minChunkThreshold || split[i].Length < smallChunkThreshold) && ((hlp.Length + split[i].Length) < maxChunkThreshold))
{
hlp = hlp + delimiter + split[i];
}
else
{
chunks.Add(hlp + delimiter);
hlp = split[i];
}
}
chunks.Add(hlp);
return(chunks);
}
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();
// Get Relativity Object Manager API
IObjectManager objectManager = this.Helper.GetServicesManager().CreateProxy <IObjectManager>(ExecutionIdentity.CurrentUser);
// Get document text
Stream streamToTranslate;
try
{
// Construct objects and retreive document content
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = sourceField
};
IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);
streamToTranslate = await keplerStream.GetStreamAsync();
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, document for translation retrieval error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Copy stream to text as that is used for further on
string textToTranslate = new StreamReader(streamToTranslate).ReadToEnd();
// Log original document
_logger.LogDebug("Azure Translator, original document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textToTranslate.Length.ToString());
// Split document text to chunks as Azure Translator request is limited by 10K characters
List <string> partsToTranslate = SplitMulti(textToTranslate, new char[] { '.', ' ' }, 9000, 9900, 20);
_logger.LogDebug("Azure Translator, document split into {n} parts (ArtifactID: {id})", partsToTranslate.Count.ToString(), documentArtifactId.ToString());
// Force TLS 1.2 or higher as Azure requires it
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12 & ~(SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11);
// URL encode request paramenters
translateFrom = WebUtility.HtmlEncode(translateFrom.ToLower());
translateTo = WebUtility.HtmlEncode(translateTo.ToLower());
// Do Azure Translator call for every text part
List <string> partsTranslated = new List <string>();
for (int i = 0; i < partsToTranslate.Count; i++)
{
// Build translation request
HttpRequestMessage request = new HttpRequestMessage();
request.Method = HttpMethod.Post;
request.RequestUri = new Uri(azureTranslatorEndpoint + "translate?api-version=3.0&to=" + translateTo + (translateFrom == "auto" ? "" : "&from=" + translateFrom) + "&includeAlignment=true"); // https://docs.microsoft.com/azure/cognitive-services/translator/reference/v3-0-translate
request.Content = new StringContent(JsonConvert.SerializeObject(new object[] { new { Text = partsToTranslate[i] } }), Encoding.UTF8, "application/json");
request.Headers.Add("Ocp-Apim-Subscription-Key", azureSubscriptionKey);
request.Headers.Add("Ocp-Apim-Subscription-Region", azureServiceRegion);
// Send the request
HttpClient client = new HttpClient();
HttpResponseMessage response = await client.SendAsync(request).ConfigureAwait(false);
// Check the response
if (response.StatusCode != HttpStatusCode.OK)
{
_logger.LogError("Azure Translator, HTTP reposnse error (ArtifactID: {id}, status: {status})", documentArtifactId.ToString(), response.StatusCode.ToString());
return(documentArtifactId);
}
// Read the response
string partTranslated = await response.Content.ReadAsStringAsync();
_logger.LogDebug("Azure Translator, translation result JSON check (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), partTranslated.Length.ToString());
client.Dispose();
// Parse JSON
TranslationResult[] translationResults = JsonConvert.DeserializeObject <TranslationResult[]>(partTranslated);
// Check the translation result
if (translationResults.Length > 1 || translationResults[0].Translations.Length > 1)
{
_logger.LogError("Azure Translator, unexpected document translation results (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
if (translationResults.Length == 0 || translationResults[0].Translations.Length == 0)
{
_logger.LogError("Azure Translator, empty document translation results (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Log the translation result
_logger.LogDebug("Azure Translator, translation check (ArtifactID: {id}, part: {part}, length: {length})", documentArtifactId.ToString(), i.ToString(), translationResults[0].Translations[0].Text.Length.ToString());
// Get the translation result
partsTranslated.Add(translationResults[0].Translations[0].Text);
}
// Construct translated text
string textTranslated = string.Join(string.Empty, partsTranslated);
Stream streamTranslated = new MemoryStream();
StreamWriter streamWriter = new StreamWriter(streamTranslated);
streamWriter.Write(textTranslated);
streamWriter.Flush();
streamTranslated.Position = 0;
// Log translated document
_logger.LogDebug("Azure Translator, translated document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textTranslated.Length.ToString());
// Update document translated text
try
{
// Construct objects and do document update
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = destinationField
};
UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
{
Object = relativityObject,
Field = relativityField
};
KeplerStream keplerStream = new KeplerStream(streamTranslated);
await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, document for translation update error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Update document translation log
try
{
Stream streamCurrentLog;
// Construct objects and get current translation log
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = logField
};
IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);
streamCurrentLog = await keplerStream.GetStreamAsync();
// Add new translation log
Stream streamUpdatedLog = new MemoryStream();
StreamWriter streamLogWriter = new StreamWriter(streamUpdatedLog);
streamLogWriter.Write(new StreamReader(streamCurrentLog).ReadToEnd());
streamLogWriter.Write("Azure Translator;" + this.Helper.GetAuthenticationManager().UserInfo.EmailAddress + ";" + DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss") + ";" + translateFrom + ";" + translateTo + ";" + textToTranslate.Length.ToString() + ";" + textTranslated.Length.ToString() + "\n");
streamLogWriter.Flush();
streamUpdatedLog.Position = 0;
// Write updated translation log
UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
{
Object = relativityObject,
Field = relativityField
};
keplerStream = new KeplerStream(streamUpdatedLog);
await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, translation log update error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Return 0 as all went without error
return(0);
}
/*
* This function is called in batches based on the size defined in configuration.
*/
public override kCura.EventHandler.Response DoBatch()
{
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();
// Init general response
kCura.EventHandler.Response response = new kCura.EventHandler.Response()
{
Success = true,
Message = ""
};
// Get current Workspace ID
int workspaceId = this.Helper.GetActiveCaseID();
_logger.LogDebug("Azure Translator, current Workspace ID: {workspaceId}", workspaceId.ToString());
// Check if all Instance Settings are in place
IDictionary <string, string> instanceSettings = this.GetInstanceSettings(ref response, new string[] { "SourceField", "DestinationField", "LogField", "Cost1MCharacters", "AzureServiceRegion", "AzureSubscriptionKey", "AzureTranslatorEndpoint", "TranslateFrom", "TranslateTo" });
// Check if there was not error
if (!response.Success)
{
return(response);
}
// Update general status
this.ChangeStatus("Translating documents");
// For each document create translation task
List <Task <int> > translationTasks = new List <Task <int> >();
int runningTasks = 0;
int concurrentTasks = 16;
for (int i = 0; i < this.BatchIDs.Count; i++)
{
// Translate documents in Azure and update Relativity using Object Manager API
translationTasks.Add(TranslateDocument(workspaceId, this.BatchIDs[i], instanceSettings["SourceField"], instanceSettings["DestinationField"], instanceSettings["LogField"], instanceSettings["AzureServiceRegion"], instanceSettings["AzureSubscriptionKey"], instanceSettings["AzureTranslatorEndpoint"], instanceSettings["TranslateFrom"], instanceSettings["TranslateTo"]));
// Update progreass bar
this.IncrementCount(1);
// Allow only certain number of tasks to run concurrently
do
{
runningTasks = 0;
foreach (Task <int> translationTask in translationTasks)
{
if (!translationTask.IsCompleted)
{
runningTasks++;
}
}
if (runningTasks >= concurrentTasks)
{
Thread.Sleep(100);
}
} while (runningTasks >= concurrentTasks);
}
// Update general status
this.ChangeStatus("Waiting to finish the document translation");
// Wait for all translations to finish
_logger.LogDebug("Azure Translator, waiting for all documents finish translating ({n} document(s))", this.BatchIDs.Count.ToString());
Task.WaitAll(translationTasks.ToArray());
// Update general status
this.ChangeStatus("Checking the results of the document translation");
// Check results
List <string> translationErrors = new List <string>();
for (int i = 0; i < translationTasks.Count; i++)
{
// If translation was not done add to the error List
_logger.LogDebug("Azure Translator, translation task result: {result} (task: {task})", translationTasks[i].Result.ToString(), translationTasks[i].Id.ToString());
if (translationTasks[i].Result != 0)
{
translationErrors.Add(translationTasks[i].Result.ToString());
}
}
// If there are any errors adjust response
if (translationErrors.Count > 0)
{
_logger.LogError("Azure Translator, not all documents have been translated: ({documents})", string.Join(", ", translationErrors));
response.Success = false;
response.Message = "Not all documents have been translated";
}
return(response);
}
public override void Execute()
{
// Update Security Protocol
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <Agent>();
// Get current Agent ID
int agentArtifactId = this.AgentID;
_logger.LogDebug("Audit Log Elastic Search, current Agent ID: {agentArtifactId}", agentArtifactId.ToString());
// Display initial message
this.RaiseMessageNoLogging("Getting Instance Settings.", 10);
// Get ES URI Instance Settings
List <Uri> elasticUris = new List <Uri>();
try
{
string[] uris = this.Helper.GetInstanceSettingBundle().GetString("Relativity.AuditLogElasticSearch", "ElasticSearchUris").Split(';');
foreach (string uri in uris)
{
if (Uri.IsWellFormedUriString(uri, UriKind.Absolute))
{
elasticUris.Add(new Uri(uri));
}
else
{
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchUri), single URI error ({uri})", agentArtifactId.ToString(), uri);
this.RaiseMessageNoLogging(string.Format("Instance Settings error (ElasticSearchUri), single URI error ({0}).", uri), 1);
return;
}
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchUri)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchUri).", 1);
return;
}
// Get ES authentication API Key Instance Settings
string[] elasticApiKey = new string[] { "", "" };
try
{
string apiKey = this.Helper.GetInstanceSettingBundle().GetString("Relativity.AuditLogElasticSearch", "ElasticSearchApiKey");
if (apiKey.Length > 0)
{
if (apiKey.Split(':').Length == 2)
{
elasticApiKey = apiKey.Split(':');
}
else
{
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchApiKey), API Key format error ({apiKey})", agentArtifactId.ToString(), apiKey);
this.RaiseMessageNoLogging(string.Format("Instance Settings error (ElasticSearchApiKey), API Key format error ({0}).", apiKey), 1);
return;
}
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchApiKey)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchApiKey).", 1);
return;
}
// Get ES index prefix Instance Settings (must by lowercase)
string elasticIndexPrefix = "";
try
{
elasticIndexPrefix = this.Helper.GetInstanceSettingBundle().GetString("Relativity.AuditLogElasticSearch", "ElasticSearchIndexPrefix").ToLower();
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchIndexPrefix)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchIndexPrefix).", 1);
return;
}
// Get ES index number of replicas Instance Settings
int elasticIndexReplicas = 1;
try
{
elasticIndexReplicas = this.Helper.GetInstanceSettingBundle().GetInt("Relativity.AuditLogElasticSearch", "ElasticSearchIndexReplicas").Value;
if (elasticIndexReplicas < 0)
{
elasticIndexReplicas = 1;
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchIndexReplicas)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchIndexReplicas).", 1);
return;
}
// Get ES index number of shards Instance Settings
int elasticIndexShards = 1;
try
{
elasticIndexShards = this.Helper.GetInstanceSettingBundle().GetInt("Relativity.AuditLogElasticSearch", "ElasticSearchIndexShards").Value;
if (elasticIndexShards < 0)
{
elasticIndexShards = 1;
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchIndexShards)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchIndexShards).", 1);
return;
}
// Get ES synchronization threshold for one agent run
int elasticSyncSize = 1000;
try
{
elasticSyncSize = this.Helper.GetInstanceSettingBundle().GetInt("Relativity.AuditLogElasticSearch", "ElasticSearchSyncSize").Value;
if (elasticSyncSize < 1000)
{
elasticSyncSize = 1000;
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), Instance Settings error (ElasticSearchSyncSize)", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Instance Settings error (ElasticSearchSyncSize).", 1);
return;
}
// Get database context of the instance
IDBContext instanceContext = Helper.GetDBContext(-1);
// Check if management table exists
try
{
int exists = instanceContext.ExecuteSqlStatementAsScalar <int>("IF OBJECT_ID('[eddsdbo].[" + this.tableName + "]', 'U') IS NOT NULL SELECT 1 ELSE SELECT 0");
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), application management table " + (exists == 1 ? "exists" : "does not exist"), agentArtifactId.ToString());
if (exists != 1)
{
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), application management table does not exist", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Application management table does not exist.", 1);
return;
}
}
catch (Exception e)
{
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), application management table existence check error", agentArtifactId.ToString());
}
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), selecting Workspace", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Selecting Workspace.", 10);
// Check what needs to be done
int workspaceId = -1;
long auditRecordId = -1;
int status = -1;
instanceContext.BeginTransaction();
try
{
// Get workspace that was synchronized latest
DataTable dataTable = instanceContext.ExecuteSqlStatementAsDataTable(@"
SELECT TOP(1)
[CaseArtifactID],
[AuditRecordID],
[Status]
FROM [eddsdbo].[" + this.tableName + @"]
WHERE [AgentArtifactID] IS NULL
ORDER BY
[Status] ASC,
[LastUpdated] ASC
");
// If there is no workspace check if table is empty and if it is, delete it
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), Workspace selection row count: {count}", agentArtifactId.ToString(), dataTable.Rows.Count.ToString());
if (dataTable.Rows.Count == 0)
{
int count = instanceContext.ExecuteSqlStatementAsScalar <int>("SELECT COUNT(*) FROM [eddsdbo].[" + this.tableName + "]");
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), application management table row count: {count}", agentArtifactId.ToString(), count.ToString());
// If there are no rows in the application management table better to drop it
if (count == 0)
{
instanceContext.ExecuteNonQuerySQLStatement("DROP TABLE [eddsdbo].[" + this.tableName + "]");
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), application management table was deleted", agentArtifactId.ToString());
}
}
// Else we have workspace to work with
else
{
DataRow dataRow = dataTable.Rows[0];
workspaceId = Convert.ToInt32(dataRow["CaseArtifactID"]);
auditRecordId = Convert.ToInt64(dataRow["AuditRecordID"]);
status = Convert.ToInt32(dataRow["Status"]);
// Update the application management table with Agent ID lock
SqlParameter agentArtifactIdParam = new SqlParameter("@agentArtifactId", agentArtifactId);
SqlParameter workspaceIdParam = new SqlParameter("@workspaceId", workspaceId);
instanceContext.ExecuteNonQuerySQLStatement("UPDATE [eddsdbo].[" + this.tableName + "] SET [AgentArtifactID] = @agentArtifactId WHERE [CaseArtifactID] = @workspaceId", new SqlParameter[] { agentArtifactIdParam, workspaceIdParam });
}
instanceContext.CommitTransaction();
}
catch (Exception e)
{
instanceContext.RollbackTransaction();
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), application management table querying error", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Application management table querying error.", 1);
return;
}
// If we have Workspace ID we have to do something
if (workspaceId > 0)
{
// Construct ES index name
string elasticIndexName = elasticIndexPrefix + workspaceId.ToString();
// Construct connector to ES cluster
Nest.ElasticClient elasticClient = null;
try
{
Elasticsearch.Net.StaticConnectionPool pool = new Elasticsearch.Net.StaticConnectionPool(elasticUris, true);
elasticClient = new Nest.ElasticClient(new Nest.ConnectionSettings(pool).DefaultIndex(elasticIndexName).ApiKeyAuthentication(elasticApiKey[0], elasticApiKey[1]).EnableHttpCompression());
}
catch (Exception e)
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}) Elastic Search connection call error ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Elastic Search connection call error ({0}, {1}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName), 1);
return;
}
// Check ES cluster connection
Nest.PingResponse pingResponse = elasticClient.Ping();
if (pingResponse.IsValid)
{
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), Ping succeeded ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
}
else
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Ping failed, check cluster health and connection settings ({elasticUris}, {indexName}, {elasticError})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName, pingResponse.DebugInformation);
this.RaiseMessageNoLogging(string.Format("Elastic Search ping failed, check cluster health and connection settings ({0}, {1}, {2}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName, pingResponse.DebugInformation), 1);
return;
}
switch (status)
{
// If the status is 0 we will be deleting ES index
case 0:
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), deleting ES index ({indexName})", agentArtifactId.ToString(), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Deleting ES index ({0}).", elasticIndexName), 10);
// Delete ES index
try
{
Nest.DeleteIndexResponse deleteIndexResponse = elasticClient.Indices.Delete(elasticIndexName);
if (deleteIndexResponse.Acknowledged)
{
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), Elastic Search index deleted ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
}
else
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Elastic Search index deletion error ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Elastic Search index deletion error ({0}, {1}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName), 1);
return;
}
}
catch (Exception e)
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}) Elastic Search deletion call error ({indexName})", agentArtifactId.ToString(), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Elastic Search deletion call error ({0}).", elasticIndexName), 1);
return;
}
// Delete related row from the application management table
try
{
SqlParameter workspaceIdParam = new SqlParameter("@workspaceId", workspaceId);
instanceContext.ExecuteNonQuerySQLStatement("DELETE FROM [eddsdbo].[" + this.tableName + "] WHERE [Status] = 0 AND [CaseArtifactID] = @workspaceId", new SqlParameter[] { workspaceIdParam });
}
catch (Exception e)
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}), application management table delete error", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Application management table delete error.", 1);
return;
}
break;
// If the status is 1 we will be synchronizing Audit Log with ES index
case 1:
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), synchronizing Audit Log of Workspace ({workspaceId}) to ES index ({indexName})", agentArtifactId.ToString(), workspaceId.ToString(), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Synchronizing Audit Log of Workspace ({0}) to ES index ({1})", workspaceId.ToString(), elasticIndexName), 10);
// If there is no records synchronized yet, we have to create ES index first
if (auditRecordId == 0)
{
// Create ES index
try {
Nest.CreateIndexResponse createIndexResponse = elasticClient.Indices.Create(elasticIndexName, c => c.Settings(s => s.NumberOfShards(elasticIndexShards).NumberOfReplicas(elasticIndexReplicas)).Map <AuditRecord>(m => m.AutoMap()));
if (createIndexResponse.Acknowledged)
{
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), Elastic Search index created ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
}
else
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Elastic Search index creation error ({elasticUris}, {indexName}, {serverError})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName, createIndexResponse.ServerError.ToString());
this.RaiseMessageNoLogging(string.Format("Elastic Search index creation error ({0}, {1}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName), 1);
return;
}
}
catch (Exception e)
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}) Elastic Search index creation call error ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Elastic Search index creation call error ({0}, {1}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName), 1);
return;
}
}
// Get database context of the given workspace
IDBContext workspaceContext = Helper.GetDBContext(workspaceId);
// Synchronize until threshold is reached
int syncCount = 0;
while (syncCount < elasticSyncSize)
{
try
{
// Get Audit Log to synchronize
SqlParameter auditRecordIdParam = new SqlParameter("@auditRecordId", auditRecordId);
DataTable dataTable = workspaceContext.ExecuteSqlStatementAsDataTable(@"
SELECT TOP (1000)
[AuditRecord].[ID],
[AuditRecord].[TimeStamp],
[AuditRecord].[ArtifactID],
[AuditRecord].[Action] AS [ActionID],
[AuditAction].[Action],
[AuditRecord].[UserID],
[AuditUser].[FullName] AS [User],
[AuditRecord].[ExecutionTime],
[AuditRecord].[Details],
[AuditRecord].[RequestOrigination],
[AuditRecord].[RecordOrigination]
FROM [EDDSDBO].[AuditRecord] WITH (NOLOCK)
JOIN [EDDSDBO].[AuditUser] WITH (NOLOCK) ON [AuditRecord].[UserID] = [AuditUser].[UserID]
JOIN [EDDSDBO].[AuditAction] WITH (NOLOCK) ON [AuditRecord].[Action] = [AuditAction].[AuditActionID]
WHERE [AuditRecord].[ID] > @auditRecordId
ORDER BY [AuditRecord].[ID] ASC
", new SqlParameter[] { auditRecordIdParam });
// If there is nothing to synchronize end
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), Audit Log row count to synchronize: {count}", agentArtifactId.ToString(), dataTable.Rows.Count.ToString());
if (dataTable.Rows.Count == 0)
{
// Log end of Agent execution
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), completed, nothing to synchronize", agentArtifactId.ToString());
this.RaiseMessageNoLogging("Completed.", 10);
return;
}
// Else synchronize workspace Audit Log with ES index
else
{
// Synchronizing workspace Audit Log with ES index
List <AuditRecord> auditRecords = new List <AuditRecord>();
long newAuditRecordId = auditRecordId;
for (int i = 0; i < dataTable.Rows.Count; i++)
{
// Read Audit Log data
AuditRecord auditRecord = new AuditRecord();
DataRow dataRow = dataTable.Rows[i];
auditRecord.AuditRecordId = Convert.ToInt64(dataRow["ID"]);
auditRecord.TimeStamp = Convert.ToDateTime(dataRow["TimeStamp"]);
auditRecord.ArtifactId = Convert.ToInt32(dataRow["ArtifactID"]);
auditRecord.ActionId = Convert.ToInt32(dataRow["ActionID"]);
auditRecord.Action = Convert.ToString(dataRow["Action"]);
auditRecord.UserId = Convert.ToInt32(dataRow["UserID"]);
auditRecord.User = Convert.ToString(dataRow["User"]);
auditRecord.ExecutionTime = dataRow["ExecutionTime"] is DBNull ? default : Convert.ToInt32(dataRow["ExecutionTime"]);
auditRecord.Details = dataRow["Details"] is DBNull ? default : Convert.ToString(dataRow["Details"]);
auditRecord.RequestOrigination = dataRow["RequestOrigination"] is DBNull ? default : Convert.ToString(dataRow["RequestOrigination"]);
auditRecord.RecordOrigination = dataRow["RecordOrigination"] is DBNull ? default : Convert.ToString(dataRow["RecordOrigination"]);
auditRecords.Add(auditRecord);
// Record last Audit Log ID
if (newAuditRecordId < auditRecord.AuditRecordId)
{
newAuditRecordId = auditRecord.AuditRecordId;
}
// Index data in threshold is reached or we are at the last row
if (auditRecords.Count >= 500 || i + 1 >= dataTable.Rows.Count)
{
try
{
Nest.BulkResponse bulkResponse = elasticClient.Bulk(b => b.Index(elasticIndexName).IndexMany(auditRecords, (descriptor, s) => descriptor.Id(s.AuditRecordId.ToString())));
if (!bulkResponse.Errors)
{
auditRecords.Clear();
_logger.LogDebug("Audit Log Elastic Search, Agent ({agentArtifactId}), documents synchronized to Elastic Serach index ({indexName})", agentArtifactId.ToString(), elasticIndexName);
}
else
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
foreach (Nest.BulkResponseItemBase itemWithError in bulkResponse.ItemsWithErrors)
{
_logger.LogError("Audit Log Elastic Search, Agent ({agentArtifactId}), Elastic Serach bulk index error to index {indexName} ({elasticUris}) on document {docIs}:{docError}", agentArtifactId.ToString(), elasticIndexName, string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), itemWithError.Id, itemWithError.Error.ToString());
}
this.RaiseMessageNoLogging(string.Format("Elastic Serach bulk index error to index {0} ({1}).", elasticIndexName, string.Join(";", elasticUris.Select(x => x.ToString()).ToArray())), 1);
return;
}
}
catch (Exception e)
{
this.releaseAgentLock(agentArtifactId, auditRecordId, workspaceId);
_logger.LogError(e, "Audit Log Elastic Search, Agent ({agentArtifactId}) Elastic Search bulk index call error ({elasticUris}, {indexName})", agentArtifactId.ToString(), string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName);
this.RaiseMessageNoLogging(string.Format("Elastic Search bulk index call error ({0}, {1}).", string.Join(";", elasticUris.Select(x => x.ToString()).ToArray()), elasticIndexName), 1);
return;
}
}
}
// After successful indexing assign new Audit Log ID
auditRecordId = newAuditRecordId;
}
}