private void TextStreamer(object obj)
{
foreach (RelativityObjectSlim ros in _streamQueue.GetConsumingEnumerable())
{
if (_cancellationToken.IsCancellationRequested)
{
break;
}
RelativityObjectRef documentObjectRef = new RelativityObjectRef {
ArtifactID = ros.ArtifactID
};
try
{
foreach (int index in _longTextIds)
{
object longText = ros.Values[index];
if (longText != null && longText.ToString().Equals(_SHIBBOLETH))
{
IKeplerStream keplerStream = _objectManager
.StreamLongTextAsync(_workspaceId, documentObjectRef, _queryRequest.Fields.ElementAt(index)).Result;
Stream realStream = keplerStream.GetStreamAsync().Result;
ros.Values[index] = realStream;
}
}
}
catch (Exception exception)
{
// Close any streams we already opened in this ROS
CloseOpenStreamsInRos(ros);
// Send error to handler and cancel
SendErrorAndCancel("StreamLongTextAsync failed", exception);
// Skip rest of loop
break;
}
try
{
_openStreamQueue.Add(ros, _cancellationToken);
}
catch (OperationCanceledException) { }
}
// Close all open streams in the Open Stream Queue if we are canceling
if (_cancellationToken.IsCancellationRequested)
{
// close open streams left in open stream queue
_openStreamQueue.CompleteAdding();
foreach (RelativityObjectSlim rosToClose in _openStreamQueue.GetConsumingEnumerable())
{
CloseOpenStreamsInRos(rosToClose);
}
}
}
/// <summary>
/// THIS is the method that needs to be what each
/// thread executes--streams the text to a file.
/// </summary>
public static void StreamToFile()
{
// buffer size for file IO
const int BUFFER_SIZE = 5000;
while (!_writeJobs.IsCompleted)
{
WriteJob op;
bool success = _writeJobs.TryTake(out op);
if (!success)
{
Thread.Sleep(100);
continue;
}
// stream
// create ref to object
var relativityObj = new RelativityObjectRef
{
ArtifactID = op.DocumentId
};
// create ref to field
var longTextFieldRef = new FieldRef
{
ArtifactID = op.LongTextFieldId
};
// instantiate object manager service
using (IObjectManager objMgr = ServiceFactory.CreateProxy <IObjectManager>())
using (IKeplerStream ks = objMgr.StreamLongTextAsync(
op.WorkspaceId, relativityObj, longTextFieldRef).Result)
using (Stream s = ks.GetStreamAsync().Result)
using (var reader = new StreamReader(s))
using (var writer = new StreamWriter(op.Path, append: false))
{
char[] buffer = new char[BUFFER_SIZE];
int copied;
do
{
copied = reader.Read(buffer, 0, BUFFER_SIZE);
writer.Write(
copied == BUFFER_SIZE
? buffer
: buffer.Take(copied).ToArray());
writer.Flush();
} while (copied > 0);
}
}
}
/*
* Custom method to translate document using Azure Translator
*/
private async Task <int> TranslateDocument(int workspaceId, int documentArtifactId, string sourceField, string destinationField, string logField, string azureServiceRegion, string azureSubscriptionKey, string azureTranslatorEndpoint, string translateFrom, string translateTo)
{
/*
* Custom local function to split string into chunks of defined size with delimiter priority
*/
List <string> SplitMulti(string str, char[] delimiters, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
{
List <string> chunks = new List <string>()
{
str
};
List <string> subChunks;
foreach (char delimiter in delimiters)
{
for (int i = 0; i < chunks.Count; i++)
{
// Check if chunks are all below threshold otherwise split by space
if (chunks[i].Length > maxChunkThreshold)
{
subChunks = SplitSized(chunks[i], delimiter, minChunkThreshold, maxChunkThreshold, smallChunkThreshold);
chunks.RemoveAt(i);
chunks.InsertRange(i, subChunks);
}
}
}
return(chunks);
}
/*
* Custom local function to split string into chunks of defined size
*/
List <string> SplitSized(string str, char delimiter, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
{
string[] split = str.Split(delimiter);
List <string> chunks = new List <string>();
string hlp = split[0];
for (int i = 1; i < split.Length; i++)
{
// Rearange split text into bigger chunks
if (((hlp.Length + split[i].Length) < minChunkThreshold || split[i].Length < smallChunkThreshold) && ((hlp.Length + split[i].Length) < maxChunkThreshold))
{
hlp = hlp + delimiter + split[i];
}
else
{
chunks.Add(hlp + delimiter);
hlp = split[i];
}
}
chunks.Add(hlp);
return(chunks);
}
// Get logger
Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();
// Get Relativity Object Manager API
IObjectManager objectManager = this.Helper.GetServicesManager().CreateProxy <IObjectManager>(ExecutionIdentity.CurrentUser);
// Get document text
Stream streamToTranslate;
try
{
// Construct objects and retreive document content
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = sourceField
};
IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);
streamToTranslate = await keplerStream.GetStreamAsync();
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, document for translation retrieval error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Copy stream to text as that is used for further on
string textToTranslate = new StreamReader(streamToTranslate).ReadToEnd();
// Log original document
_logger.LogDebug("Azure Translator, original document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textToTranslate.Length.ToString());
// Split document text to chunks as Azure Translator request is limited by 10K characters
List <string> partsToTranslate = SplitMulti(textToTranslate, new char[] { '.', ' ' }, 9000, 9900, 20);
_logger.LogDebug("Azure Translator, document split into {n} parts (ArtifactID: {id})", partsToTranslate.Count.ToString(), documentArtifactId.ToString());
// Force TLS 1.2 or higher as Azure requires it
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12 & ~(SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11);
// URL encode request paramenters
translateFrom = WebUtility.HtmlEncode(translateFrom.ToLower());
translateTo = WebUtility.HtmlEncode(translateTo.ToLower());
// Do Azure Translator call for every text part
List <string> partsTranslated = new List <string>();
for (int i = 0; i < partsToTranslate.Count; i++)
{
// Build translation request
HttpRequestMessage request = new HttpRequestMessage();
request.Method = HttpMethod.Post;
request.RequestUri = new Uri(azureTranslatorEndpoint + "translate?api-version=3.0&to=" + translateTo + (translateFrom == "auto" ? "" : "&from=" + translateFrom) + "&includeAlignment=true"); // https://docs.microsoft.com/azure/cognitive-services/translator/reference/v3-0-translate
request.Content = new StringContent(JsonConvert.SerializeObject(new object[] { new { Text = partsToTranslate[i] } }), Encoding.UTF8, "application/json");
request.Headers.Add("Ocp-Apim-Subscription-Key", azureSubscriptionKey);
request.Headers.Add("Ocp-Apim-Subscription-Region", azureServiceRegion);
// Send the request
HttpClient client = new HttpClient();
HttpResponseMessage response = await client.SendAsync(request).ConfigureAwait(false);
// Check the response
if (response.StatusCode != HttpStatusCode.OK)
{
_logger.LogError("Azure Translator, HTTP reposnse error (ArtifactID: {id}, status: {status})", documentArtifactId.ToString(), response.StatusCode.ToString());
return(documentArtifactId);
}
// Read the response
string partTranslated = await response.Content.ReadAsStringAsync();
_logger.LogDebug("Azure Translator, translation result JSON check (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), partTranslated.Length.ToString());
client.Dispose();
// Parse JSON
TranslationResult[] translationResults = JsonConvert.DeserializeObject <TranslationResult[]>(partTranslated);
// Check the translation result
if (translationResults.Length > 1 || translationResults[0].Translations.Length > 1)
{
_logger.LogError("Azure Translator, unexpected document translation results (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
if (translationResults.Length == 0 || translationResults[0].Translations.Length == 0)
{
_logger.LogError("Azure Translator, empty document translation results (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Log the translation result
_logger.LogDebug("Azure Translator, translation check (ArtifactID: {id}, part: {part}, length: {length})", documentArtifactId.ToString(), i.ToString(), translationResults[0].Translations[0].Text.Length.ToString());
// Get the translation result
partsTranslated.Add(translationResults[0].Translations[0].Text);
}
// Construct translated text
string textTranslated = string.Join(string.Empty, partsTranslated);
Stream streamTranslated = new MemoryStream();
StreamWriter streamWriter = new StreamWriter(streamTranslated);
streamWriter.Write(textTranslated);
streamWriter.Flush();
streamTranslated.Position = 0;
// Log translated document
_logger.LogDebug("Azure Translator, translated document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textTranslated.Length.ToString());
// Update document translated text
try
{
// Construct objects and do document update
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = destinationField
};
UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
{
Object = relativityObject,
Field = relativityField
};
KeplerStream keplerStream = new KeplerStream(streamTranslated);
await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, document for translation update error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Update document translation log
try
{
Stream streamCurrentLog;
// Construct objects and get current translation log
RelativityObjectRef relativityObject = new RelativityObjectRef
{
ArtifactID = documentArtifactId
};
FieldRef relativityField = new FieldRef
{
Name = logField
};
IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);
streamCurrentLog = await keplerStream.GetStreamAsync();
// Add new translation log
Stream streamUpdatedLog = new MemoryStream();
StreamWriter streamLogWriter = new StreamWriter(streamUpdatedLog);
streamLogWriter.Write(new StreamReader(streamCurrentLog).ReadToEnd());
streamLogWriter.Write("Azure Translator;" + this.Helper.GetAuthenticationManager().UserInfo.EmailAddress + ";" + DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss") + ";" + translateFrom + ";" + translateTo + ";" + textToTranslate.Length.ToString() + ";" + textTranslated.Length.ToString() + "\n");
streamLogWriter.Flush();
streamUpdatedLog.Position = 0;
// Write updated translation log
UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
{
Object = relativityObject,
Field = relativityField
};
keplerStream = new KeplerStream(streamUpdatedLog);
await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
}
catch (Exception e)
{
_logger.LogError(e, "Azure Translator, translation log update error (ArtifactID: {id})", documentArtifactId.ToString());
return(documentArtifactId);
}
// Return 0 as all went without error
return(0);
}