public void Import (ImporterOptions options) { ZipArchive zip; using (var stream = File.OpenRead (options.InputZipArchive)) { zip = new ZipArchive (stream); foreach (var ent in zip.Entries) { options.DiagnosticWriter.WriteLine (ent.FullName); if (!ent.Name.EndsWith (".java", StringComparison.OrdinalIgnoreCase)) continue; var java = new StreamReader (ent.Open ()).ReadToEnd (); if (!ParseJava (java)) break; } } foreach (var pkg in api.Packages) { foreach (var t in pkg.Types) { // Our API definitions don't contain non-public members, so remove those (but it does contain non-public types). t.Members = t.Members.Where (m => m != null && m.Visibility != "").ToList (); // Constructor "type" is the full name of the class. foreach (var c in t.Members.OfType<JavaConstructor> ()) c.Type = (pkg.Name.Length > 0 ? (pkg.Name + '.') : string.Empty) + t.Name; // Pupulated enum fields need type to be filled var cls = t as JavaClass; if (cls != null && cls.Extends == "java.lang.Enum") { cls.ExtendsGeneric = "java.lang.Enum<" + pkg.Name + "." + t.Name + ">"; foreach (var m in cls.Members.OfType<JavaField> ()) { if (m.Type == null) { m.Type = pkg.Name + "." + t.Name; m.TypeGeneric = pkg.Name + "." + t.Name; } } foreach (var m in cls.Members.OfType<JavaMethod> ()) { if (m.Name == "valueOf") m.Return = pkg.Name + "." + t.Name; else if (m.Name == "values") m.Return = pkg.Name + "." + t.Name + "[]"; } } t.Members = t.Members.OfType<JavaMethodBase> () .OrderBy (m => m.Name + "(" + string.Join (",", m.Parameters.Select (p => p.Type)) + ")") .ToArray (); } pkg.Types = pkg.Types.OrderBy (t => t.Name).ToArray (); } api.Packages = api.Packages.OrderBy (p => p.Name).ToArray (); if (options.OutputTextFile != null) api.WriteParameterNamesText (options.OutputTextFile); if (options.OutputXmlFile != null) api.WriteParameterNamesXml (options.OutputXmlFile); }
/// <summary> /// Tests CSV import using logic from the NuGet package 'codingfreaks.libs.Csv'. /// </summary> /// <remarks> /// Uses a BMA import file <see cref="SourceFileUri"/>. /// </remarks> private static async Task TestCsvImporterAsync() { var lastPercentage = 0; // prepare import options var options = new ImporterOptions { AutoDetectEncoding = true, Culture = new CultureInfo("de-DE"), Logger = Console.WriteLine, ItemsPerWorker = 500, LogMappingPerformance = true }; // get a new importer var importer = new Importer <CsvImporterSample>(options); // define a progress to visualize it later in the console var progress = new Progress <OperationProgress>(p => { if (p.Percentage % 10 == 0 && p.Percentage > lastPercentage) { Interlocked.Exchange(ref lastPercentage, p.Percentage.Value); Console.WriteLine($"{p.Percentage}% imported"); } }); importer.ItemImported += (s, e) => { }; // run the importer and wait for it to finish var result = await importer.ImportAsync(SourceFileUri, progress); // handle results Console.Clear(); if (result.Failed) { } Console.WriteLine(result.ItemsCount); Console.WriteLine(result.Finished - result.Started); var resultItems = result.Items.ToList(); Console.WriteLine(resultItems.Count); }
public bool ValidateWorksheetMetadata(ImporterOptions importerOptions) { bool isValid = false; if (_spreadsheet == null) { throw new InvalidOperationException("Spreadsheet object must be loaded by call to LoadSpreadsheetMetadata prioir to calling ValidateSpreadsheetMetadata"); } string lastColumnName = importerOptions.RangeExpression.Substring(importerOptions.RangeExpression.IndexOf(RangeExpressionColumnSeparator, StringComparison.InvariantCulture) + 1); int expectedColumnCount = _sheetsApiHelper.ExcelColumnNameToNumber(lastColumnName); string sanitizedWorksheetName = importerOptions.WorkSheetName.Replace(MultiWordWorksheetNameDelimiter, String.Empty); Sheet sheet = Enumerable.FirstOrDefault <Sheet>(_spreadsheet.Sheets, w => w.Properties.Title == sanitizedWorksheetName); //validate expected column count matches actual if (sheet?.Properties.GridProperties.ColumnCount != null) { var actualColumnCount = sheet.Properties.GridProperties.ColumnCount.Value; //modify actualColumnCount lower for chart at the end (as in Experience) if (sheet.Charts != null && sheet.Charts.Any()) { int?anchorCellColumnIndex = sheet.Charts.First().Position.OverlayPosition.AnchorCell.ColumnIndex; if (anchorCellColumnIndex != null) { actualColumnCount = (int)anchorCellColumnIndex; } } if (actualColumnCount == expectedColumnCount) { isValid = true; } } return(isValid); }
/* * * The DroidDoc format from API Level 16 to 23, the format is: * * - All pages have ToC links and body (unlike standard JavaDoc which is based on HTML frames). * - The actual doc section is a div element whose id is "doc-col". * - The "doc-col" div element has a section div element whose id is "jd-header" and another one with id "jd-content". * - "jd-header" div element contains the type signature (modifiers, name, and inheritance). * - Here we care only about type name and kind (whether it is a class or interface). * - Generic arguments are insignificant. * - In the following terms I explain the "correct" (or "expected") document structure, but in fact * Google completely broke it and it is impossible to retrieve the document tree like this. * We workaround this issue by changing the strategy "iterate children of 'jd-content'" * with "iterate descendants of 'jd-content'"... It occurs only in API Level 15 or later. * - "jd-content" div element contains a collection of sections. Each section consists of: * - an "h2" element whose value text indicates the section name ("Public Constructors", "Protected Methods" etc.) * - There was an issue in javax/xml/validation/SchemaFactory.html in API Level 15 that the method details contain * "h2" and confuses the parser. To workaround this, we accept only limited kind of values. * - the content, which follows the h2 element. * - The section content is a collection of members. Each member consists of: * - an anchor ("A") element with "name" attribute, and * - a div element which contains an h4 child element whose class contains "jd-details-title". * - The h4 element contains the member signature. We parse it and retrieve the method name and list of parameters. * - Parameters are tokenized by ", ". * - Note that the splitter contains a white space which disambiguates any use of generic arguments (we don't want to split "Foo<K,V> bar" as "Foo<K" and "V> bar") * * API Level 10 to 15 has slightly different format: * * - There is no "doc-col" element. But "jd-header" and "jd-content" are still alive. * */ public void Import(ImporterOptions options) { options.DiagnosticWriter.WriteLine(options.DocumentDirectory); string referenceDocsTopDir = Path.Combine(options.DocumentDirectory, "reference"); var htmlFiles = Directory.GetDirectories(referenceDocsTopDir).SelectMany(d => Directory.GetFiles(d, "*.html", SearchOption.AllDirectories)); var api = new JavaApi(); foreach (var htmlFile in htmlFiles) { // skip irrelevant files. if (excludes.Any(x => htmlFile.EndsWith(x, StringComparison.OrdinalIgnoreCase))) { continue; } var packageName = Path.GetDirectoryName(htmlFile).Substring(referenceDocsTopDir.Length + 1).Replace('/', '.'); if (options.FrameworkOnly && non_frameworks.Any(n => packageName.StartsWith(n, StringComparison.Ordinal))) { continue; } options.DiagnosticWriter.WriteLine("-- " + htmlFile); var doc = new HtmlLoader().GetJavaDocFile(htmlFile); var header = doc.Descendants().FirstOrDefault(e => e.Attribute("id")?.Value == "jd-header"); var content = doc.Descendants().FirstOrDefault(e => e.Attribute("id")?.Value == "jd-content"); if (header == null || content == null) { continue; } var apiSignatureTokens = header.Value.Replace('\r', ' ').Replace('\n', ' ').Replace('\t', ' ').Trim(); if (apiSignatureTokens.Contains("extends ")) { apiSignatureTokens = apiSignatureTokens.Substring(0, apiSignatureTokens.IndexOf("extends ", StringComparison.Ordinal)).Trim(); } if (apiSignatureTokens.Contains("implements ")) { apiSignatureTokens = apiSignatureTokens.Substring(0, apiSignatureTokens.IndexOf("implements ", StringComparison.Ordinal)).Trim(); } bool isClass = apiSignatureTokens.Contains("class"); options.DiagnosticWriter.WriteLine(apiSignatureTokens); var javaPackage = api.Packages.FirstOrDefault(p => p.Name == packageName); if (javaPackage == null) { javaPackage = new JavaPackage(api) { Name = packageName }; api.Packages.Add(javaPackage); } var javaType = isClass ? (JavaType) new JavaClass(javaPackage) : new JavaInterface(javaPackage); javaType.Name = apiSignatureTokens.Substring(apiSignatureTokens.LastIndexOf(' ') + 1); javaPackage.Types.Add(javaType); string sectionType = null; var sep = new string [] { ", " }; var ssep = new char [] { ' ' }; foreach (var child in content.Descendants()) { if (child.Name == "h2") { var value = child.Value; switch (value) { case "Public Constructors": case "Protected Constructors": case "Public Methods": case "Protected Methods": sectionType = value; break; } continue; } if (sectionType == null) { continue; } if (child.Name != "a" || child.Attribute("name") == null) { continue; } var h4 = child.XPathSelectElement("following-sibling::div/h4[contains(@class, 'jd-details-title')]"); if (h4 == null) { continue; } string sigTypeOnly = child.Attribute("name").Value; string sigTypeAndName = h4.Value.Replace('\n', ' ').Replace('\r', ' ').Trim(); if (!sigTypeAndName.Contains('(')) { continue; } JavaMethodBase javaMethod = null; string name = sigTypeAndName.Substring(0, sigTypeAndName.IndexOf('(')).Split(ssep, StringSplitOptions.RemoveEmptyEntries).Last(); switch (sectionType) { case "Public Constructors": case "Protected Constructors": javaMethod = new JavaConstructor(javaType) { Name = name }; break; case "Public Methods": case "Protected Methods": string mname = sigTypeAndName.Substring(0, sigTypeAndName.IndexOf('(')); javaMethod = new JavaMethod(javaType) { Name = name }; break; } javaType.Members.Add(javaMethod); var paramTypes = SplitTypes(sigTypeOnly.Substring(sigTypeOnly.IndexOf('(') + 1).TrimEnd(')'), 0).ToArray(); var parameters = sigTypeAndName.Substring(sigTypeAndName.IndexOf('(') + 1).TrimEnd(')') .Split(sep, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()) .ToArray(); foreach (var p in paramTypes.Zip(parameters, (to, tn) => new { Type = to, TypeAndName = tn }) .Select(pp => new { Type = pp.Type, Name = pp.TypeAndName.Split(' ') [1] })) { javaMethod.Parameters.Add(new JavaParameter(javaMethod) { Name = p.Name, Type = p.Type }); } } javaType.Members = javaType.Members.OfType <JavaMethodBase> () .OrderBy(m => m.Name + "(" + string.Join(",", m.Parameters.Select(p => p.Type)) + ")") .ToArray(); } foreach (var pkg in api.Packages) { pkg.Types = pkg.Types.OrderBy(t => t.Name).ToArray(); } api.Packages = api.Packages.OrderBy(p => p.Name).ToArray(); if (options.OutputTextFile != null) { api.WriteParameterNamesText(options.OutputTextFile); } if (options.OutputXmlFile != null) { api.WriteParameterNamesXml(options.OutputXmlFile); } }
/// <summary> /// Constructor /// </summary> /// <param name="module">The module that will own all references</param> public Importer(ModuleDef module) { this.module = module; this.recursionCounter = new RecursionCounter(); this.options = 0; }
/// <summary> /// Constructor /// </summary> /// <param name="module">The module that will own all references</param> /// <param name="options">Importer options</param> public Importer(ModuleDef module, ImporterOptions options) { this.module = module; this.recursionCounter = new RecursionCounter(); this.options = options; }
protected RowImporterBase(ISheetsApiHelper sheetsApiHelper, IOptions <ImporterOptions> importerOptionsAccessor, ILogger <RowImporterBase <T> > logger) { _sheetsApiHelper = sheetsApiHelper; _importerOptions = importerOptionsAccessor.Value; _logger = logger; }