public void CheckGSTAndTotalExcludingGST()
{
string content = @"Hi Yvaine,
<expense>
<total>1024.01</total>
</expense>
Regards,
Ivan";
IContentExtractor extractor = new ContentExtractor();
RelevantDataBO actual;
var result = extractor.GetRelevantData(content, out actual);
Assert.True(result);
Assert.NotNull(actual);
Assert.NotNull(actual.Expense);
var expected = new RelevantDataBO()
{
Expense = new ExpenseBO()
{
Total = 1024.01M,
TotalExcludingGST = 890.44M,
GST = 133.57M
}
};
Assert.Equal(expected.Expense.Total, actual.Expense.Total);
Assert.Equal(expected.Expense.TotalExcludingGST, actual.Expense.TotalExcludingGST);
Assert.Equal(expected.Expense.GST, actual.Expense.GST);
}
private static void ValidateFileFrameworks(IEnumerable <IPackageFile> files)
{
var set = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var file in files.Where(t => t.Path != null).Select(t => PathUtility.GetPathWithDirectorySeparator(t.Path)))
{
set.Add(file);
}
var managedCodeConventions = new ManagedCodeConventions(new RuntimeGraph());
var collection = new ContentItemCollection();
collection.Load(set.Select(path => path.Replace('\\', '/')).ToArray());
var patterns = managedCodeConventions.Patterns;
var frameworkPatterns = new List <PatternSet>()
{
patterns.RuntimeAssemblies,
patterns.CompileRefAssemblies,
patterns.CompileLibAssemblies,
patterns.NativeLibraries,
patterns.ResourceAssemblies,
patterns.MSBuildFiles,
patterns.ContentFiles,
patterns.ToolsAssemblies,
patterns.EmbedAssemblies,
patterns.MSBuildTransitiveFiles
};
var warnPaths = new HashSet <string>();
var frameworksMissingPlatformVersion = new HashSet <string>();
foreach (var pattern in frameworkPatterns)
{
IEnumerable <ContentItemGroup> targetedItemGroups = ContentExtractor.GetContentForPattern(collection, pattern);
foreach (ContentItemGroup group in targetedItemGroups)
{
foreach (ContentItem item in group.Items)
{
var framework = (NuGetFramework)item.Properties["tfm"];
if (framework == null)
{
continue;
}
if (framework.HasPlatform && framework.PlatformVersion == FrameworkConstants.EmptyVersion)
{
frameworksMissingPlatformVersion.Add(framework.GetShortFolderName());
}
}
}
}
if (frameworksMissingPlatformVersion.Any())
{
throw new PackagingException(NuGetLogCode.NU1012, string.Format(CultureInfo.CurrentCulture, Strings.MissingTargetPlatformVersionsFromIncludedFiles, string.Join(", ", frameworksMissingPlatformVersion.OrderBy(str => str))));
}
}
public void SuccessTest(string content)
{
IContentExtractor extractor = new ContentExtractor();
RelevantDataBO actual;
var result = extractor.GetRelevantData(content, out actual);
Assert.False(result);
}
static void Main(string[] args)
{
var pages = ContentExtractor.Read(@"Example.PDF");
var page = pages[0];
page.DetermineTableStructures();
page.DetermineParagraphs();
page.FillContent();
Console.WriteLine(page);
Console.ReadLine();
}
public async Task Extract_IntegrationTest()
{
// Arrange
var config = Configuration.Default.WithDefaultLoader();
var address = "https://lenta.ru/articles/2020/05/13/coronausa/";
var context = BrowsingContext.New(config);
var document = (IHtmlDocument)await context.OpenAsync(address);
var extractor = new ContentExtractor();
// Act
extractor.Extract(document);
// Assert
Assert.Pass();
}
internal (HashSet <NuGetFramework>, HashSet <NuGetFramework>, HashSet <NuGetFramework>) Categorize(IEnumerable <string> files, IEnumerable <NuGetFramework> tfmsFromNuspec)
{
var managedCodeConventions = new ManagedCodeConventions(new RuntimeGraph());
Func <object, object, bool> isCompatible = managedCodeConventions.Properties["tfm"].CompatibilityTest;
var collection = new ContentItemCollection();
collection.Load(files);
List <ContentItemGroup> libItems = new();
List <ContentItemGroup> refItems = new();
ContentExtractor.GetContentForPattern(collection, managedCodeConventions.Patterns.CompileLibAssemblies, libItems);
ContentExtractor.GetContentForPattern(collection, managedCodeConventions.Patterns.CompileRefAssemblies, refItems);
var tfmsFromFilesSet = new HashSet <NuGetFramework>();
tfmsFromFilesSet.AddRange(ContentExtractor.GetGroupFrameworks(libItems));
tfmsFromFilesSet.AddRange(ContentExtractor.GetGroupFrameworks(refItems));
var tfmsFromFiles = tfmsFromFilesSet.Where(t => t != Net00).ToList();
var noExactMatchesFromFile = new HashSet <NuGetFramework>();
var noExactMatchesFromNuspec = new HashSet <NuGetFramework>();
var compatNotExactMatches = new HashSet <NuGetFramework>();
noExactMatchesFromFile.AddRange(tfmsFromFiles.Where(t => !tfmsFromNuspec.Contains(t)));
noExactMatchesFromNuspec.AddRange(tfmsFromNuspec.Where(t => !tfmsFromFiles.Contains(t)));
foreach (var fileTFM in noExactMatchesFromFile)
{
foreach (var nuspecTFM in noExactMatchesFromNuspec)
{
if (isCompatible(fileTFM, nuspecTFM))
{
compatNotExactMatches.Add(fileTFM);
break;
}
}
}
if (compatNotExactMatches.Count != 0)
{
noExactMatchesFromFile.RemoveWhere(p => compatNotExactMatches.Contains(p));
}
return(compatNotExactMatches, noExactMatchesFromFile, noExactMatchesFromNuspec);
}
internal IEnumerable <PackagingLogMessage> Validate(IEnumerable <string> files)
{
var managedCodeConventions = new ManagedCodeConventions(new RuntimeGraph());
var collection = new ContentItemCollection();
collection.Load(files);
List <ContentItemGroup> libItems = new();
List <ContentItemGroup> refItems = new();
List <ContentItemGroup> buildItems = new();
ContentExtractor.GetContentForPattern(collection, managedCodeConventions.Patterns.CompileLibAssemblies, libItems);
ContentExtractor.GetContentForPattern(collection, managedCodeConventions.Patterns.CompileRefAssemblies, refItems);
ContentExtractor.GetContentForPattern(collection, managedCodeConventions.Patterns.MSBuildFiles, buildItems);
var libFrameworks = ContentExtractor.GetGroupFrameworks(libItems).ToArray();
var refFrameworks = ContentExtractor.GetGroupFrameworks(refItems).ToArray();
var buildFrameworks = ContentExtractor.GetGroupFrameworks(buildItems).ToArray();
if (libFrameworks.Length == 0 && refFrameworks.Length == 0)
{
//if you can't find the ref and lib folder, then find the build folder
if (buildFrameworks.Length != 0)
{
//if you can find any folders other than native or any, raise an NU5127
if (buildFrameworks.Any(t => (FrameworkConstants.DotNetAll.Satisfies(t) || FrameworkNameValidatorUtility.IsValidFrameworkName(t)) &&
t.GetShortFolderName() != FrameworkConstants.FrameworkIdentifiers.DotNet &&
t.GetShortFolderName() != FrameworkConstants.FrameworkIdentifiers.Native))
{
var possibleFrameworks = buildFrameworks.
Where(t => t.IsSpecificFramework &&
t.GetShortFolderName() != FrameworkConstants.FrameworkIdentifiers.DotNet &&
t.GetShortFolderName() != FrameworkConstants.FrameworkIdentifiers.Native).
Select(t => t.GetShortFolderName()).ToArray();
(var tfmNames, var suggestedDirectories) = GenerateWarningString(possibleFrameworks);
var issue = new List <PackagingLogMessage>();
issue.Add(PackagingLogMessage.CreateWarning(string.Format(MessageFormat, tfmNames, suggestedDirectories),
NuGetLogCode.NU5127));
return(issue);
}
}
}
return(Array.Empty <PackagingLogMessage>());
}
public void NoCostCentre()
{
string content = @"Hi Yvaine,
Please create an expense claim for the below. Relevant details are marked up as
requested...
<expense>
<total>1024.01</total><payment_method>personal card</payment_method>
</expense>
From: Ivan Castle
Sent: Friday, 16 February 2018 10:32 AM
To: Antoine Lloyd
Subject: test
Hi Antoine,
Please create a reservation at the <vendor>Viaduct Steakhouse</vendor> our
<description>development team’s project end celebration dinner</description> on
<date>Thursday 27 April 2017</date>. We expect to arrive around
7.15pm. Approximately 12 people but I’ll confirm exact numbers closer to the day.
Regards,
Ivan";
IContentExtractor extractor = new ContentExtractor();
RelevantDataBO actual;
var result = extractor.GetRelevantData(content, out actual);
Assert.True(result);
Assert.NotNull(actual);
Assert.NotNull(actual.Expense);
var expected = new RelevantDataBO()
{
Date = new System.DateTime(2017, 4, 27),
Description = "development team’s project end celebration dinner",
Vendor = "Viaduct Steakhouse",
Expense = new ExpenseBO()
{
CostCentre = "UNKNOWN",
PaymentMethod = "personal card",
Total = 1024.01M,
TotalExcludingGST = 890.44M,
GST = 133.57M
}
};
AssertResponse(expected, actual);
}
public void InvalidDate()
{
string content = @"abc
<expense>
<total>1024.01</total>
</expense>
adfd
<date>Monday 27 April 2017</date>.
Regards,
Ivan";
IContentExtractor extractor = new ContentExtractor();
RelevantDataBO actual;
var result = extractor.GetRelevantData(content, out actual);
Assert.True(result);
Assert.NotNull(actual);
Assert.Equal(DateTime.MinValue, actual.Date);
}
public static void Main(string[] args)
{
var inputPath = ArgsUtil.GetArgument(args, 0, DefaultFileInput);
var firstOutputPath = ArgsUtil.GetArgument(args, 1, DefaultFileOutput1);
var secondOutputPath = ArgsUtil.GetArgument(args, 2, DefaultFileOutput2);
var allLines = File.ReadAllLines(inputPath);
var contentExtractor = new ContentExtractor();
var extracted = contentExtractor.Extract(allLines);
var output = new OutputGenerator(extracted);
File.WriteAllLines(firstOutputPath, output.GetFrequencyWithAlphabeticLines());
Console.WriteLine("Created: " + firstOutputPath);
File.WriteAllLines(secondOutputPath, output.GetByStreetNameLines());
Console.WriteLine("Created: " + secondOutputPath);
}
public void ExtractPopulatesCorrectly()
{
var allLines = new[]
{
"FirstName,LastName,Address,PhoneNumber",
"TestName1,LastName1,1 Address,11111111",
"TestName2,LastName2,2 Address,22222222",
"TestName3,LastName3,3 Address,33333333",
"TestName4,LastName4,4 Address,4444444"
};
var contentExtractor = new ContentExtractor();
var extracted = contentExtractor.Extract(allLines);
var allContentItems = extracted.AllContentItems;
Assert.That(allContentItems.Count, Is.EqualTo(4));
Assert.That(allContentItems.Count(item => item.FirstName.Equals("TestName1")), Is.EqualTo(1));
Assert.That(allContentItems.Count(item => item.LastName.Equals("LastName2")), Is.EqualTo(1));
Assert.That(allContentItems.Count(item => item.Address.Equals("3 Address")), Is.EqualTo(1));
Assert.That(allContentItems.Count(item => item.PhoneNumber.Equals("4444444")), Is.EqualTo(1));
}
private static void Execute(string[] args, Assembly assembly)
{
string extractionDirectoryPath = Utilities.GetResourceExtractionDirectoryPath(assembly);
string logFilePath = InstallerDiagnostics.GetExeLogFilePath(extractionDirectoryPath, assembly);
progressMessageLogger = (string message) => InstallerDiagnostics.LogProgressMessage(message, logFilePath);
uiMode = GetUImode(args);
AppDomain.CurrentDomain.UnhandledException +=
(object sender, UnhandledExceptionEventArgs e) => ErrorHandler.CurrentDomain_UnhandledException(logFilePath, e, uiMode);
InstallerDiagnostics.CreateSelfServiceEventSource();
Utilities.ValidateNoProcessWithSameNameIsRunning(Process.GetCurrentProcess(), logFilePath, uiMode);
ContentExtractor contentExtractor = new ContentExtractor(assembly, AssemblyRootNamespace, extractionDirectoryPath);
UIHelper.ShowMainView(uiMode);
InstallerDiagnostics.LogProgressMessage(Messages.ProgressMessageUnpackingResources, logFilePath);
contentExtractor.UnpackResources();
InstallerDiagnostics.LogProgressMessage(Messages.ProgressMessageValidatingNet45Installed, logFilePath);
Utilities.ValidateNet45prereq(logFilePath, uiMode);
InstallerDiagnostics.LogProgressMessage(Messages.ProgressMessageUnpackingContent, logFilePath);
contentExtractor.UnzipContent();
UIHelper.HideMainView();
args = GetUpdatedArgsForSetup(args, contentExtractor, assembly);
InstallerDiagnostics.LogProgressMessage(Messages.ProgressMessageRunningMainSetupApp, logFilePath);
Utilities.RunProcessAsyncAndExitWhenCompleted(contentExtractor.GetSetupExePath(), args);
// We need this to begin running a standard application message loop on the main thread.
// So that main thread does not exit and waits until setup run finishes.
Application application = new Application();
application.Run();
}
private static string[] GetUpdatedArgsForSetup(string[] args, ContentExtractor contentExtractor, Assembly assembly)
{
// Customized file path.
string customizedFilesPath = Utilities.GetCustomizedFilesPath(contentExtractor.GetContentExtractionPath());
args = Utilities.GetArgsWithNonExistingParameterAppended(args, CustomizedFilesArgument, customizedFilesPath);
// Setup bootstrapper version.
string bootStrapperVersion = FileVersionInfo.GetVersionInfo(assembly.Location).FileVersion;
args = Utilities.GetArgsWithNonExistingParameterAppended(args, BootStrapperVersionArgument, bootStrapperVersion);
// Try to lookup settings file that was downloaded from AX and put side by side with bootstrapper.
string configurationSettingsFilePath = Utilities.GetSettingsFilePathIfExists(assembly.Location, progressMessageLogger);
if (!string.IsNullOrWhiteSpace(configurationSettingsFilePath))
{
args = Utilities.GetArgsWithNonExistingParameterAppended(args, SettingsArgument, configurationSettingsFilePath);
}
return(args);
}
private static void CheckFile(string pdfFilePath)
{
Console.WriteLine("Reading " + System.IO.Path.GetFileName(pdfFilePath));
PageCollection pages = ContentExtractor.ReadPdfFileAndRefreshContent(pdfFilePath);
string fileContent = string.Empty;
foreach (Page page in pages)
{
fileContent += "======================================================\r\n";
fileContent += page.ToString();
}
string txtFileName = pdfFilePath + ".txt";
if (File.Exists(txtFileName))
{
Console.WriteLine("Testing " + System.IO.Path.GetFileName(pdfFilePath));
string txtFileContent = File.ReadAllText(txtFileName);
if (txtFileContent != fileContent)
{
string[] txtFileLines = txtFileContent.Replace("\r\n", "\r").Split('\r');
string[] txtLines = fileContent.Replace("\r\n", "\r").Split('\r');
string diff = MHDiff.GetDiff(txtFileLines, txtLines);
Console.WriteLine("Files are different");
Console.WriteLine(diff);
throw new Exception("Wrong content in file " + pdfFilePath);
}
}
else
{
Console.WriteLine(System.IO.Path.GetFileName(pdfFilePath) + " NOT TESTED!!!");
Console.WriteLine("Creating txt file " + txtFileName);
File.WriteAllText(txtFileName, fileContent);
}
}
private void ShowDocument(string fileName)
{
_pages = ContentExtractor.Read(fileName);
lblPages.Text = _pages.Count.ToString();
DrawPage(0);
}
internal static IEnumerable <PackagingLogMessage> ValidateFiles(IEnumerable <string> files)
{
var set = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var file in files.Select(t => PathUtility.GetPathWithDirectorySeparator(t)))
{
set.Add(file);
}
var managedCodeConventions = new ManagedCodeConventions(new RuntimeGraph());
var collection = new ContentItemCollection();
collection.Load(set.Select(path => path.Replace('\\', '/')).ToArray());
var patterns = managedCodeConventions.Patterns;
var frameworkPatterns = new List <PatternSet>()
{
patterns.RuntimeAssemblies,
patterns.CompileRefAssemblies,
patterns.CompileLibAssemblies,
patterns.NativeLibraries,
patterns.ResourceAssemblies,
patterns.MSBuildFiles,
patterns.ContentFiles,
patterns.ToolsAssemblies,
patterns.EmbedAssemblies,
patterns.MSBuildTransitiveFiles
};
var warnPaths = new HashSet <string>();
foreach (var pattern in frameworkPatterns)
{
IEnumerable <ContentItemGroup> targetedItemGroups = ContentExtractor.GetContentForPattern(collection, pattern);
foreach (ContentItemGroup group in targetedItemGroups)
{
foreach (ContentItem item in group.Items)
{
var exists = item.Properties.TryGetValue("tfm_raw", out var frameworkRaw);
string frameworkString = (string)frameworkRaw;
if (!exists || string.IsNullOrEmpty(frameworkString))
{
continue;
}
if (!FrameworkVersionHasDesiredDots(frameworkString))
{
warnPaths.Add(item.Path);
}
}
}
}
var messages = new List <PackagingLogMessage>();
if (warnPaths.Count > 0)
{
messages.Add(
PackagingLogMessage.CreateWarning(
string.Format(CultureInfo.CurrentCulture, AnalysisResources.InvalidUndottedFrameworkInFilesWarning, string.Join(", ", warnPaths)),
NuGetLogCode.NU5501
)
);
}
return(messages);
}
public override string Execute <TOwner>(IUIComponent <TOwner> component) => ContentExtractor.Get(component, Source);
