public ImportProcessor(IFileTypeManager fileTypeManager, TokenVisitor tokenVisitor,
ImportOptions settings, List <AnalysisBand> analysisBands, SegmentBuilder segmentBuilder)
{
_fileTypeManager = fileTypeManager;
_analysisBands = analysisBands;
_settings = settings;
_segmentBuilder = segmentBuilder;
_tokenVisitor = tokenVisitor;
}
public ExportProcessor(string projectId, IFileTypeManager fileTypeManager, TokenVisitor tokenVisitor,
ExportOptions settings, List <AnalysisBand> analysisBands)
{
_projectId = projectId;
_fileTypeManager = fileTypeManager;
_analysisBands = analysisBands;
_settings = settings;
_tokenVisitor = tokenVisitor;
}
public void TokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
GlobalCode ast = null;
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateProvider(sourceUnit.SourceLexer.AllTokens, original);
var composer = new WhitespaceComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code;
var result = code.ToString();
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "original.txt"), original);
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "result.txt"), result);
//Assert.AreEqual(original.Length, result.Length);
//for (int i = 0; i < original.Length; i++)
//{
// Assert.AreEqual(original[i], result[i]);
//}
Assert.AreEqual(original, result);
var tokens = provider.GetTokens(new Span(0, original.Length)).AsArray();
Assert.AreEqual(tokens.Length, composer.Processed.Count);
for (int i = 0; i < tokens.Length; i++)
{
Assert.AreEqual(tokens[i].Token, composer.Processed[i].Token);
Assert.AreEqual(tokens[i].Span, composer.Processed[i].Span);
}
}
public ContentReader(string projectId, TokenVisitor tokenVisitor, ExportOptions exportOptions,
List <AnalysisBand> analysisBands, string filePath, string outputFilePath, string targetLanguage)
{
_projectId = projectId;
_tokenVisitor = tokenVisitor;
_exportOptions = exportOptions;
_analysisBands = analysisBands;
_originalFilePath = filePath;
_contextTypes = new List <string>();
_outputFilePath = outputFilePath;
_targetLanguage = targetLanguage;
ConfirmationStatistics = new ConfirmationStatistics();
TranslationOriginStatistics = new TranslationOriginStatistics();
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnPlaceholder();
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnBody(Text);
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnAnnotation(Text);
}
public override void ProcessParagraphUnit(IParagraphUnit paragraphUnit)
{
if (paragraphUnit.IsStructure)
{
base.ProcessParagraphUnit(paragraphUnit);
return;
}
foreach (var segmentPair in paragraphUnit.SegmentPairs)
{
SegmentPairInfo segmentPairInfo = null;
try
{
segmentPairInfo = SegmentPairProcessor.GetSegmentPairInfo(segmentPair);
}
catch
{
// catch all; ignore
}
var status = segmentPair.Properties.ConfirmationLevel.ToString();
var match = Enumerators.GetTranslationOriginType(segmentPair.Target.Properties.TranslationOrigin, _analysisBands);
var targetSegment = segmentPair.Target;
//capture if segment contains tracked changes
//var hasTrackedChanges = false;
var segmentIdentifier = string.Empty;
if (SegmentExists(paragraphUnit.Properties.ParagraphUnitId.Id, segmentPair.Properties.Id.Id, ref segmentIdentifier))
{
var noOverwrite = !_importOptions.OverwriteTranslations && segmentPair.Target.Any();
var excludeFilter = false;
if (_importOptions.ExcludeFilterIds != null)
{
excludeFilter = (segmentPair.Properties.IsLocked && _importOptions.ExcludeFilterIds.Exists(a => a == "Locked")) ||
_importOptions.ExcludeFilterIds.Exists(a => a == status) ||
_importOptions.ExcludeFilterIds.Exists(a => a == match);
}
if (noOverwrite || excludeFilter)
{
if (!string.IsNullOrEmpty(_importOptions.StatusTranslationNotUpdatedId))
{
var success = Enum.TryParse <ConfirmationLevel>(_importOptions.StatusTranslationNotUpdatedId, true, out var result);
var statusTranslationNotUpdated = success ? result : ConfirmationLevel.Unspecified;
segmentPair.Target.Properties.ConfirmationLevel = statusTranslationNotUpdated;
segmentPair.Properties.ConfirmationLevel = statusTranslationNotUpdated;
status = segmentPair.Properties.ConfirmationLevel.ToString();
match = Enumerators.GetTranslationOriginType(segmentPair.Target.Properties.TranslationOrigin, _analysisBands);
}
AddWordCounts(status, ConfirmationStatistics.WordCounts.Excluded, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Excluded, segmentPairInfo);
AddWordCounts(status, ConfirmationStatistics.WordCounts.Total, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Total, segmentPairInfo);
continue;
}
var tokenVisitor = new TokenVisitor();
tokenVisitor.VisitSegment(targetSegment);
var originalText = tokenVisitor.PlainText.ToString();
try
{
if (segmentPair.Target.Properties.TranslationOrigin == null)
{
targetSegment.Properties.TranslationOrigin = _segmentBuilder.ItemFactory.CreateTranslationOrigin();
}
else
{
var currentTranslationOrigin = (ITranslationOrigin)targetSegment.Properties.TranslationOrigin.Clone();
targetSegment.Properties.TranslationOrigin.OriginBeforeAdaptation = currentTranslationOrigin;
}
SetTranslationOrigin(targetSegment);
var updatedSegment = _updatedSegments[segmentIdentifier];
if (updatedSegment.TranslationTokens.Count > 0)
{
targetSegment = _segmentBuilder.GetUpdatedSegment(targetSegment, updatedSegment.TranslationTokens, segmentPair.Source);
}
if (updatedSegment.BackTranslationTokens.Count > 0)
{
var backTranslations = JsonConvert.SerializeObject(updatedSegment.BackTranslationTokens);
segmentPair.Target.Properties.TranslationOrigin.SetMetaData("back-translation", backTranslations);
}
}
catch (Exception ex)
{
throw new Exception("Problem when merging content of segment " + segmentPair.Properties.Id.Id, ex);
}
tokenVisitor.Process(targetSegment);
var updatedText = tokenVisitor.PlainText.ToString();
if (string.Compare(originalText, updatedText, StringComparison.Ordinal) != 0)
{
if (!string.IsNullOrEmpty(_importOptions.StatusTranslationUpdatedId))
{
var success = Enum.TryParse <ConfirmationLevel>(_importOptions.StatusTranslationUpdatedId, true,
out var result);
var statusTranslationUpdated = success ? result : ConfirmationLevel.Unspecified;
targetSegment.Properties.ConfirmationLevel = statusTranslationUpdated;
segmentPair.Properties.ConfirmationLevel = statusTranslationUpdated;
status = segmentPair.Properties.ConfirmationLevel.ToString();
match = Enumerators.GetTranslationOriginType(segmentPair.Target.Properties.TranslationOrigin, _analysisBands);
}
AddWordCounts(status, ConfirmationStatistics.WordCounts.Processed, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Processed, segmentPairInfo);
AddWordCounts(status, ConfirmationStatistics.WordCounts.Total, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Total, segmentPairInfo);
}
else
{
if (!string.IsNullOrEmpty(_importOptions.StatusTranslationNotUpdatedId))
{
var success = Enum.TryParse <ConfirmationLevel>(_importOptions.StatusTranslationNotUpdatedId, true, out var result);
var statusTranslationNotUpdated = success ? result : ConfirmationLevel.Unspecified;
targetSegment.Properties.ConfirmationLevel = statusTranslationNotUpdated;
}
status = targetSegment.Properties.ConfirmationLevel.ToString();
match = Enumerators.GetTranslationOriginType(targetSegment.Properties.TranslationOrigin, _analysisBands);
AddWordCounts(status, ConfirmationStatistics.WordCounts.Excluded, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Excluded, segmentPairInfo);
AddWordCounts(status, ConfirmationStatistics.WordCounts.Total, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Total, segmentPairInfo);
}
}
else
{
if (!string.IsNullOrEmpty(_importOptions.StatusSegmentNotImportedId))
{
var success = Enum.TryParse <ConfirmationLevel>(_importOptions.StatusTranslationNotUpdatedId, true, out var result);
var statusTranslationNotUpdated = success ? result : ConfirmationLevel.Unspecified;
segmentPair.Target.Properties.ConfirmationLevel = statusTranslationNotUpdated;
segmentPair.Properties.ConfirmationLevel = statusTranslationNotUpdated;
status = segmentPair.Properties.ConfirmationLevel.ToString();
match = Enumerators.GetTranslationOriginType(segmentPair.Target.Properties.TranslationOrigin, _analysisBands);
}
AddWordCounts(status, ConfirmationStatistics.WordCounts.NotProcessed, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.NotProcessed, segmentPairInfo);
AddWordCounts(status, ConfirmationStatistics.WordCounts.Total, segmentPairInfo);
AddWordCounts(match, TranslationOriginStatistics.WordCounts.Total, segmentPairInfo);
}
}
base.ProcessParagraphUnit(paragraphUnit);
}
public override void Accept(TokenVisitor visitor)
{
visitor.OnRuby(RubyParentText, RubyText);
}
public override void Accept(TokenVisitor visitor)
{
visitor.OnNormal(this.Text);
}
public abstract void Accept(TokenVisitor visitor);
public void EmptyTokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
if (original.Contains("namespace\\"))
{
return; // TODO - current namespace cannot be decided from AST
}
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
GlobalCode ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateEmptyProvider();
var composer = new EmptyComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code.ToString();
var expectedStr = PrepareString(original);
var actualStr = PrepareString(code);
Assert.AreEqual(expectedStr, actualStr);
var expected = FilterTokens(sourceUnit.SourceLexer.AllTokens);
var actual = FilterTokens(composer.Processed);
Assert.AreEqual(expected.Length, actual.Length);
for (int i = 0; i < Math.Min(expected.Length, actual.Length); i++)
{
if (expected[i].Token == Tokens.T_SEMI && actual[i].Token == Tokens.T_CASE)
{
}
if (expected[i].Token == Tokens.T_LOGICAL_OR && actual[i].Token == Tokens.T_BOOLEAN_OR ||
expected[i].Token == Tokens.T_LOGICAL_AND && actual[i].Token == Tokens.T_BOOLEAN_AND)
{
}
else
{
Assert.AreEqual(expected[i].Token, actual[i].Token);
}
}
sourceUnit = new TestSourceUnit(code, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
var newAst = sourceUnit.Ast;
var serializer = new JsonNodeWriter();
var serializerVisitor = new TreeSerializer(serializer);
ast.VisitMe(visitor);
expectedStr = serializer.ToString();
serializer = new JsonNodeWriter();
serializerVisitor = new TreeSerializer(serializer);
newAst.VisitMe(visitor);
actualStr = serializer.ToString();
Assert.AreEqual(expectedStr, actualStr);
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnPlaceholder();
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnAnnotation(Text);
}
void IToken.Accept(TokenVisitor visitor)
{
visitor.OnBody(Text);
}
public override void Accept(TokenVisitor visitor)
{
visitor.OnRuby(RubyParentText, RubyText);
}
public override void Accept(TokenVisitor visitor)
{
visitor.OnNormal(this.Text);
}
public abstract void Accept(TokenVisitor visitor);
