コード例 #1
0
        private void TextStreamer(object obj)
        {
            foreach (RelativityObjectSlim ros in _streamQueue.GetConsumingEnumerable())
            {
                if (_cancellationToken.IsCancellationRequested)
                {
                    break;
                }

                RelativityObjectRef documentObjectRef = new RelativityObjectRef {
                    ArtifactID = ros.ArtifactID
                };

                try
                {
                    foreach (int index in _longTextIds)
                    {
                        object longText = ros.Values[index];
                        if (longText != null && longText.ToString().Equals(_SHIBBOLETH))
                        {
                            IKeplerStream keplerStream = _objectManager
                                                         .StreamLongTextAsync(_workspaceId, documentObjectRef, _queryRequest.Fields.ElementAt(index)).Result;
                            Stream realStream = keplerStream.GetStreamAsync().Result;
                            ros.Values[index] = realStream;
                        }
                    }
                }
                catch (Exception exception)
                {
                    // Close any streams we already opened in this ROS
                    CloseOpenStreamsInRos(ros);
                    // Send error to handler and cancel
                    SendErrorAndCancel("StreamLongTextAsync failed", exception);
                    // Skip rest of loop
                    break;
                }

                try
                {
                    _openStreamQueue.Add(ros, _cancellationToken);
                }
                catch (OperationCanceledException) { }
            }

            // Close all open streams in the Open Stream Queue if we are canceling

            if (_cancellationToken.IsCancellationRequested)
            {
                // close open streams left in open stream queue
                _openStreamQueue.CompleteAdding();
                foreach (RelativityObjectSlim rosToClose in _openStreamQueue.GetConsumingEnumerable())
                {
                    CloseOpenStreamsInRos(rosToClose);
                }
            }
        }
コード例 #2
0
        /// <summary>
        /// THIS is the method that needs to be what each
        /// thread executes--streams the text to a file.
        /// </summary>
        public static void StreamToFile()
        {
            // buffer size for file IO
            const int BUFFER_SIZE = 5000;

            while (!_writeJobs.IsCompleted)
            {
                WriteJob op;
                bool     success = _writeJobs.TryTake(out op);
                if (!success)
                {
                    Thread.Sleep(100);
                    continue;
                }

                // stream

                // create ref to object
                var relativityObj = new RelativityObjectRef
                {
                    ArtifactID = op.DocumentId
                };

                // create ref to field
                var longTextFieldRef = new FieldRef
                {
                    ArtifactID = op.LongTextFieldId
                };

                // instantiate object manager service
                using (IObjectManager objMgr = ServiceFactory.CreateProxy <IObjectManager>())
                    using (IKeplerStream ks = objMgr.StreamLongTextAsync(
                               op.WorkspaceId, relativityObj, longTextFieldRef).Result)
                        using (Stream s = ks.GetStreamAsync().Result)
                            using (var reader = new StreamReader(s))
                                using (var writer = new StreamWriter(op.Path, append: false))
                                {
                                    char[] buffer = new char[BUFFER_SIZE];

                                    int copied;
                                    do
                                    {
                                        copied = reader.Read(buffer, 0, BUFFER_SIZE);
                                        writer.Write(
                                            copied == BUFFER_SIZE
                            ? buffer
                            : buffer.Take(copied).ToArray());
                                        writer.Flush();
                                    } while (copied > 0);
                                }
            }
        }
コード例 #3
0
        /*
         * Custom method to translate document using Azure Translator
         */
        private async Task <int> TranslateDocument(int workspaceId, int documentArtifactId, string sourceField, string destinationField, string logField, string azureServiceRegion, string azureSubscriptionKey, string azureTranslatorEndpoint, string translateFrom, string translateTo)
        {
            /*
             * Custom local function to split string into chunks of defined size with delimiter priority
             */
            List <string> SplitMulti(string str, char[] delimiters, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
            {
                List <string> chunks = new List <string>()
                {
                    str
                };
                List <string> subChunks;

                foreach (char delimiter in delimiters)
                {
                    for (int i = 0; i < chunks.Count; i++)
                    {
                        // Check if chunks are all below threshold otherwise split by space
                        if (chunks[i].Length > maxChunkThreshold)
                        {
                            subChunks = SplitSized(chunks[i], delimiter, minChunkThreshold, maxChunkThreshold, smallChunkThreshold);
                            chunks.RemoveAt(i);
                            chunks.InsertRange(i, subChunks);
                        }
                    }
                }

                return(chunks);
            }

            /*
             * Custom local function to split string into chunks of defined size
             */
            List <string> SplitSized(string str, char delimiter, int minChunkThreshold, int maxChunkThreshold, int smallChunkThreshold)
            {
                string[]      split  = str.Split(delimiter);
                List <string> chunks = new List <string>();

                string hlp = split[0];

                for (int i = 1; i < split.Length; i++)
                {
                    // Rearange split text into bigger chunks
                    if (((hlp.Length + split[i].Length) < minChunkThreshold || split[i].Length < smallChunkThreshold) && ((hlp.Length + split[i].Length) < maxChunkThreshold))
                    {
                        hlp = hlp + delimiter + split[i];
                    }
                    else
                    {
                        chunks.Add(hlp + delimiter);
                        hlp = split[i];
                    }
                }
                chunks.Add(hlp);

                return(chunks);
            }

            // Get logger
            Relativity.API.IAPILog _logger = this.Helper.GetLoggerFactory().GetLogger().ForContext <MassOperationHandler>();

            // Get Relativity Object Manager API
            IObjectManager objectManager = this.Helper.GetServicesManager().CreateProxy <IObjectManager>(ExecutionIdentity.CurrentUser);

            // Get document text
            Stream streamToTranslate;

            try
            {
                // Construct objects and retreive document content
                RelativityObjectRef relativityObject = new RelativityObjectRef
                {
                    ArtifactID = documentArtifactId
                };
                FieldRef relativityField = new FieldRef
                {
                    Name = sourceField
                };
                IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);

                streamToTranslate = await keplerStream.GetStreamAsync();
            }
            catch (Exception e)
            {
                _logger.LogError(e, "Azure Translator, document for translation retrieval error (ArtifactID: {id})", documentArtifactId.ToString());
                return(documentArtifactId);
            }

            // Copy stream to text as that is used for further on
            string textToTranslate = new StreamReader(streamToTranslate).ReadToEnd();

            // Log original document
            _logger.LogDebug("Azure Translator, original document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textToTranslate.Length.ToString());

            // Split document text to chunks as Azure Translator request is limited by 10K characters
            List <string> partsToTranslate = SplitMulti(textToTranslate, new char[] { '.', ' ' }, 9000, 9900, 20);

            _logger.LogDebug("Azure Translator, document split into {n} parts (ArtifactID: {id})", partsToTranslate.Count.ToString(), documentArtifactId.ToString());

            // Force TLS 1.2 or higher as Azure requires it
            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12 & ~(SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11);

            // URL encode request paramenters
            translateFrom = WebUtility.HtmlEncode(translateFrom.ToLower());
            translateTo   = WebUtility.HtmlEncode(translateTo.ToLower());

            // Do Azure Translator call for every text part
            List <string> partsTranslated = new List <string>();

            for (int i = 0; i < partsToTranslate.Count; i++)
            {
                // Build translation request
                HttpRequestMessage request = new HttpRequestMessage();
                request.Method     = HttpMethod.Post;
                request.RequestUri = new Uri(azureTranslatorEndpoint + "translate?api-version=3.0&to=" + translateTo + (translateFrom == "auto" ? "" : "&from=" + translateFrom) + "&includeAlignment=true");  // https://docs.microsoft.com/azure/cognitive-services/translator/reference/v3-0-translate
                request.Content    = new StringContent(JsonConvert.SerializeObject(new object[] { new { Text = partsToTranslate[i] } }), Encoding.UTF8, "application/json");
                request.Headers.Add("Ocp-Apim-Subscription-Key", azureSubscriptionKey);
                request.Headers.Add("Ocp-Apim-Subscription-Region", azureServiceRegion);

                // Send the request
                HttpClient          client   = new HttpClient();
                HttpResponseMessage response = await client.SendAsync(request).ConfigureAwait(false);

                // Check the response
                if (response.StatusCode != HttpStatusCode.OK)
                {
                    _logger.LogError("Azure Translator, HTTP reposnse error (ArtifactID: {id}, status: {status})", documentArtifactId.ToString(), response.StatusCode.ToString());
                    return(documentArtifactId);
                }

                // Read the response
                string partTranslated = await response.Content.ReadAsStringAsync();

                _logger.LogDebug("Azure Translator, translation result JSON check (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), partTranslated.Length.ToString());
                client.Dispose();

                // Parse JSON
                TranslationResult[] translationResults = JsonConvert.DeserializeObject <TranslationResult[]>(partTranslated);

                // Check the translation result
                if (translationResults.Length > 1 || translationResults[0].Translations.Length > 1)
                {
                    _logger.LogError("Azure Translator, unexpected document translation results (ArtifactID: {id})", documentArtifactId.ToString());
                    return(documentArtifactId);
                }
                if (translationResults.Length == 0 || translationResults[0].Translations.Length == 0)
                {
                    _logger.LogError("Azure Translator, empty document translation results (ArtifactID: {id})", documentArtifactId.ToString());
                    return(documentArtifactId);
                }

                // Log the translation result
                _logger.LogDebug("Azure Translator, translation check (ArtifactID: {id}, part: {part}, length: {length})", documentArtifactId.ToString(), i.ToString(), translationResults[0].Translations[0].Text.Length.ToString());
                // Get the translation result
                partsTranslated.Add(translationResults[0].Translations[0].Text);
            }

            // Construct translated text
            string       textTranslated   = string.Join(string.Empty, partsTranslated);
            Stream       streamTranslated = new MemoryStream();
            StreamWriter streamWriter     = new StreamWriter(streamTranslated);

            streamWriter.Write(textTranslated);
            streamWriter.Flush();
            streamTranslated.Position = 0;

            // Log translated document
            _logger.LogDebug("Azure Translator, translated document (ArtifactID: {id}, length: {length})", documentArtifactId.ToString(), textTranslated.Length.ToString());

            // Update document translated text
            try
            {
                // Construct objects and do document update
                RelativityObjectRef relativityObject = new RelativityObjectRef
                {
                    ArtifactID = documentArtifactId
                };
                FieldRef relativityField = new FieldRef
                {
                    Name = destinationField
                };
                UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
                {
                    Object = relativityObject,
                    Field  = relativityField
                };
                KeplerStream keplerStream = new KeplerStream(streamTranslated);
                await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
            }
            catch (Exception e)
            {
                _logger.LogError(e, "Azure Translator, document for translation update error (ArtifactID: {id})", documentArtifactId.ToString());
                return(documentArtifactId);
            }

            // Update document translation log
            try
            {
                Stream streamCurrentLog;
                // Construct objects and get current translation log
                RelativityObjectRef relativityObject = new RelativityObjectRef
                {
                    ArtifactID = documentArtifactId
                };
                FieldRef relativityField = new FieldRef
                {
                    Name = logField
                };
                IKeplerStream keplerStream = await objectManager.StreamLongTextAsync(workspaceId, relativityObject, relativityField);

                streamCurrentLog = await keplerStream.GetStreamAsync();

                // Add new translation log
                Stream       streamUpdatedLog = new MemoryStream();
                StreamWriter streamLogWriter  = new StreamWriter(streamUpdatedLog);
                streamLogWriter.Write(new StreamReader(streamCurrentLog).ReadToEnd());
                streamLogWriter.Write("Azure Translator;" + this.Helper.GetAuthenticationManager().UserInfo.EmailAddress + ";" + DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss") + ";" + translateFrom + ";" + translateTo + ";" + textToTranslate.Length.ToString() + ";" + textTranslated.Length.ToString() + "\n");
                streamLogWriter.Flush();
                streamUpdatedLog.Position = 0;

                // Write updated translation log
                UpdateLongTextFromStreamRequest updateRequest = new UpdateLongTextFromStreamRequest
                {
                    Object = relativityObject,
                    Field  = relativityField
                };
                keplerStream = new KeplerStream(streamUpdatedLog);
                await objectManager.UpdateLongTextFromStreamAsync(workspaceId, updateRequest, keplerStream);
            }
            catch (Exception e)
            {
                _logger.LogError(e, "Azure Translator, translation log update error (ArtifactID: {id})", documentArtifactId.ToString());
                return(documentArtifactId);
            }

            // Return 0 as all went without error
            return(0);
        }