public static CExtendedCodeUnit FromToken(Token token, CTokenizerParms args)
{
CElementPosition pos = new CElementPosition(token.line, token.col, token.line, token.col + token.val.Length);
CCodeUnit simple_unit = new CCodeUnit(pos, token.val);
CExtendedCodeUnit unit = new CExtendedCodeUnit(simple_unit, args.GetFileID().SourceFileID);
return(unit);
}
/// <summary>
/// Получить набор синтаксических единиц для заданного файла
/// </summary>
/// <param name="filename"></param>
protected override void LoadOneFile(string filename)
{
CTokenizerParms args = new CTokenizerParms(filename, m_LoadOptions.FileEncoding, new CSourceFileID());
CSyntacticUnitsCollection current_collection = m_Extension.Syntacticize(args);
m_CountOfSyntacticUnits += current_collection.Size();
//Interlocked.Add(ref m_CountOfSyntacticUnits, current_collection.Size());
m_SynUnitsCollection.AddRange(current_collection);
}
private void IncreaseCountOfLines(string filename)
{
// Нам нужно реальное количество строк в файле!
CSourceFileID FileID = new CSourceFileID();
CTokenizerParms args = new CTokenizerParms(filename, FileID);
ISourceFileContentLoader loader = new CEntireRowSourceFileContentLoader(args);
CCodeUnitsCollection collection = loader.Load();
Interlocked.Add(ref m_CountOfLines, collection.Size());
}
public static ISourceFileContentLoader Create(CTokenizerParms args, ICloneExtension ext)
{
if (CCodeUnit.IsUseTokens())
{
return(new CTokenSourceFileContentLoader(args, ext));
}
else
{
return(new CEntireRowSourceFileContentLoader(args));
}
}
public void LoadTest2()
{
string filename = Path.GetFullPath("UnitTestSampleFile2.cs");
Assert.IsTrue(File.Exists(filename));
CTokenizerParms args = new CTokenizerParms(filename, new CSourceFileID());
CEntireRowSourceFileContentLoader target = new CEntireRowSourceFileContentLoader(args);
CCodeUnitsCollection actual = target.Load();
Assert.AreEqual(76, actual.Size());
}
/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора токенов
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CCodeUnitsCollection Tokenize(CTokenizerParms args)
{
CCodeUnitsCollection result_collection = new CCodeUnitsCollection();
Scanner lex = new Scanner(args.GetPath());
CTokensIterator iter = new CTokensIterator(lex);
foreach (Token token in iter)
{
result_collection.Add(ExtCodeUnitCreator.FromToken(token, args));
}
return(result_collection);
}
public void LoadTest3()
{
string filename = Path.GetFullPath("MimeTypeDetection.cs");
Assert.IsTrue(File.Exists(filename));
CTokenizerParms args = new CTokenizerParms(filename, new CSourceFileID());
CTokenSourceFileContentLoader target = new CTokenSourceFileContentLoader(args, CAvailableExtentions.GetExtention(LANGUAGES.LANGUAGE_C_SHARP));
CCodeUnitsCollection actual = target.Load();
Assert.AreEqual(796, actual.Size());
}
public void SyntacticizeTest2()
{
string filename = Path.GetFullPath("UnitTestSampleFile2.cs");
Assert.IsTrue(File.Exists(filename));
CCsICloneExtension target = new CCsICloneExtension();
CTokenizerParms args = new CTokenizerParms(filename, new CSourceFileID());
CSyntacticUnitsCollection actual = target.Syntacticize(args);
Assert.AreEqual(12, actual.Size());
Assert.AreEqual(37, actual[0].Size());
}
private void IncreaseCountOfTokens(string filename)
{
try
{
CSourceFileID FileID = new CSourceFileID();
CTokenizerParms args = new CTokenizerParms(filename, FileID);
ISourceFileContentLoader loader = new CTokenSourceFileContentLoader(args, m_Extension);
CCodeUnitsCollection collection = loader.Load();
Interlocked.Add(ref m_CountOfTokens, collection.Size());
}
catch
{
// Если поддержка токенизации отсутствует, то ничего не делаем
}
}
/// <summary>
/// Загрузить один файл
/// </summary>
/// <param name="filename"></param>
protected override void LoadOneFile(string filename)
{
CSourceFileID FileID = new CSourceFileID();
CTokenizerParms parms = new CTokenizerParms(filename, LoadFilesOptions.FileEncoding, FileID);
ISourceFileContentLoader loader = CSourceFileContentLoaderFactory.Create(parms, m_ext);
CSourceFile source_file = new CSourceFile(FileID, filename, LoadFilesOptions.FileEncoding, new CCodeFragment(loader.Load()));
m_FilesCollection.IncreaseCountOfLinesInOriginFiles(source_file.Size());
if (LoadFilesOptions.IsPreProcessingFile)
{
source_file.Content.Content = new CSimpleCodePreProcessingAlgorithm().PreProcessing(LoadFilesOptions.PreProcessingOptions, source_file.Content.Content);
}
m_FilesCollection.Add(source_file.SourceFileID, source_file);
IncreaseCountOfLinesInModifiedFiles(source_file.Size());
}
/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора синтаксических единиц
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CSyntacticUnitsCollection Syntacticize(CTokenizerParms args)
{
CSyntacticUnitsCollection syntactic_collection = new CSyntacticUnitsCollection();
Parser parser = new Parser(new Scanner(args.GetPath()), string.Empty);
CounterCollection counters = parser.Parse();
if (parser.errors.count > 0)
{
//throw new System.Exception(args.GetPath() + " parse error");
}
else
{
SyntUnitCreator synt_creator = new SyntUnitCreator(args);
foreach (MethodBlock mb in counters.MethodCounters)
{
syntactic_collection.Add(synt_creator.FromMethodBlock(mb));
}
}
return(syntactic_collection);
}
/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора токенов
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CCodeUnitsCollection Tokenize(CTokenizerParms args)
{
throw new System.NotImplementedException();
/*
* ICharStream input_stream = new ANTLRFileStream(args.GetPath(), args.GetEncoding());
* Lexer lex = GetLexer(input_stream);
* CommonTokenStream tokens = new CommonTokenStream(lex);
*
* CCodeUnitsCollection result_collection = new CCodeUnitsCollection();
* foreach (IToken token in tokens.GetTokens())
* {
* CElementPosition pos = new CElementPosition(token.Line, token.CharPositionInLine, token.Line, token.CharPositionInLine + token.Text.Length);
* CCodeUnit simple_unit = new CCodeUnit(pos, token.Text);
* CExtendedCodeUnit unit = new CExtendedCodeUnit(simple_unit, args.GetSourceFileID());
* result_collection.Add(unit);
* }
* return result_collection;
*/
}
public void GetSourceFileIDTest()
{
CTokenizerParms target = new CTokenizerParms(_Path, _Encoding, FileID);
Assert.AreEqual(FileID, target.GetFileID());
}
public SyntUnitCreator(CTokenizerParms args)
{
m_args = args;
iter = new CTokensIterator(new Scanner(args.GetPath()));
}
public CTokenSourceFileContentLoader(CTokenizerParms args, ICloneExtension ext)
{
m_args = args;
m_ext = ext;
}
public void CTokenizerParmsConstructorTest1()
{
CTokenizerParms target = new CTokenizerParms(string.Empty, _Encoding, FileID);
}
public CEntireRowSourceFileContentLoader(CTokenizerParms args)
{
m_args = args;
}
public void GetPathTest()
{
CTokenizerParms target = new CTokenizerParms(_Path, _Encoding, FileID);
Assert.AreEqual(_Path, target.GetPath());
}
/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора синтаксических единиц
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CSyntacticUnitsCollection Syntacticize(CTokenizerParms args)
{
throw new System.NotImplementedException();
}
