public void Import (ImporterOptions options)
{
ZipArchive zip;
using (var stream = File.OpenRead (options.InputZipArchive)) {
zip = new ZipArchive (stream);
foreach (var ent in zip.Entries) {
options.DiagnosticWriter.WriteLine (ent.FullName);
if (!ent.Name.EndsWith (".java", StringComparison.OrdinalIgnoreCase))
continue;
var java = new StreamReader (ent.Open ()).ReadToEnd ();
if (!ParseJava (java))
break;
}
}
foreach (var pkg in api.Packages) {
foreach (var t in pkg.Types) {
// Our API definitions don't contain non-public members, so remove those (but it does contain non-public types).
t.Members = t.Members.Where (m => m != null && m.Visibility != "").ToList ();
// Constructor "type" is the full name of the class.
foreach (var c in t.Members.OfType<JavaConstructor> ())
c.Type = (pkg.Name.Length > 0 ? (pkg.Name + '.') : string.Empty) + t.Name;
// Pupulated enum fields need type to be filled
var cls = t as JavaClass;
if (cls != null && cls.Extends == "java.lang.Enum") {
cls.ExtendsGeneric = "java.lang.Enum<" + pkg.Name + "." + t.Name + ">";
foreach (var m in cls.Members.OfType<JavaField> ()) {
if (m.Type == null) {
m.Type = pkg.Name + "." + t.Name;
m.TypeGeneric = pkg.Name + "." + t.Name;
}
}
foreach (var m in cls.Members.OfType<JavaMethod> ()) {
if (m.Name == "valueOf")
m.Return = pkg.Name + "." + t.Name;
else if (m.Name == "values")
m.Return = pkg.Name + "." + t.Name + "[]";
}
}
t.Members = t.Members.OfType<JavaMethodBase> ()
.OrderBy (m => m.Name + "(" + string.Join (",", m.Parameters.Select (p => p.Type)) + ")")
.ToArray ();
}
pkg.Types = pkg.Types.OrderBy (t => t.Name).ToArray ();
}
api.Packages = api.Packages.OrderBy (p => p.Name).ToArray ();
if (options.OutputTextFile != null)
api.WriteParameterNamesText (options.OutputTextFile);
if (options.OutputXmlFile != null)
api.WriteParameterNamesXml (options.OutputXmlFile);
}
/// <summary>
/// Tests CSV import using logic from the NuGet package 'codingfreaks.libs.Csv'.
/// </summary>
/// <remarks>
/// Uses a BMA import file <see cref="SourceFileUri"/>.
/// </remarks>
private static async Task TestCsvImporterAsync()
{
var lastPercentage = 0;
// prepare import options
var options = new ImporterOptions
{
AutoDetectEncoding = true,
Culture = new CultureInfo("de-DE"),
Logger = Console.WriteLine,
ItemsPerWorker = 500,
LogMappingPerformance = true
};
// get a new importer
var importer = new Importer <CsvImporterSample>(options);
// define a progress to visualize it later in the console
var progress = new Progress <OperationProgress>(p =>
{
if (p.Percentage % 10 == 0 && p.Percentage > lastPercentage)
{
Interlocked.Exchange(ref lastPercentage, p.Percentage.Value);
Console.WriteLine($"{p.Percentage}% imported");
}
});
importer.ItemImported += (s, e) =>
{
};
// run the importer and wait for it to finish
var result = await importer.ImportAsync(SourceFileUri, progress);
// handle results
Console.Clear();
if (result.Failed)
{
}
Console.WriteLine(result.ItemsCount);
Console.WriteLine(result.Finished - result.Started);
var resultItems = result.Items.ToList();
Console.WriteLine(resultItems.Count);
}
public bool ValidateWorksheetMetadata(ImporterOptions importerOptions)
{
bool isValid = false;
if (_spreadsheet == null)
{
throw new InvalidOperationException("Spreadsheet object must be loaded by call to LoadSpreadsheetMetadata prioir to calling ValidateSpreadsheetMetadata");
}
string lastColumnName = importerOptions.RangeExpression.Substring(importerOptions.RangeExpression.IndexOf(RangeExpressionColumnSeparator, StringComparison.InvariantCulture) + 1);
int expectedColumnCount = _sheetsApiHelper.ExcelColumnNameToNumber(lastColumnName);
string sanitizedWorksheetName = importerOptions.WorkSheetName.Replace(MultiWordWorksheetNameDelimiter, String.Empty);
Sheet sheet = Enumerable.FirstOrDefault <Sheet>(_spreadsheet.Sheets, w => w.Properties.Title == sanitizedWorksheetName);
//validate expected column count matches actual
if (sheet?.Properties.GridProperties.ColumnCount != null)
{
var actualColumnCount = sheet.Properties.GridProperties.ColumnCount.Value;
//modify actualColumnCount lower for chart at the end (as in Experience)
if (sheet.Charts != null && sheet.Charts.Any())
{
int?anchorCellColumnIndex = sheet.Charts.First().Position.OverlayPosition.AnchorCell.ColumnIndex;
if (anchorCellColumnIndex != null)
{
actualColumnCount = (int)anchorCellColumnIndex;
}
}
if (actualColumnCount == expectedColumnCount)
{
isValid = true;
}
}
return(isValid);
}
/*
*
* The DroidDoc format from API Level 16 to 23, the format is:
*
* - All pages have ToC links and body (unlike standard JavaDoc which is based on HTML frames).
* - The actual doc section is a div element whose id is "doc-col".
* - The "doc-col" div element has a section div element whose id is "jd-header" and another one with id "jd-content".
* - "jd-header" div element contains the type signature (modifiers, name, and inheritance).
* - Here we care only about type name and kind (whether it is a class or interface).
* - Generic arguments are insignificant.
* - In the following terms I explain the "correct" (or "expected") document structure, but in fact
* Google completely broke it and it is impossible to retrieve the document tree like this.
* We workaround this issue by changing the strategy "iterate children of 'jd-content'"
* with "iterate descendants of 'jd-content'"... It occurs only in API Level 15 or later.
* - "jd-content" div element contains a collection of sections. Each section consists of:
* - an "h2" element whose value text indicates the section name ("Public Constructors", "Protected Methods" etc.)
* - There was an issue in javax/xml/validation/SchemaFactory.html in API Level 15 that the method details contain
* "h2" and confuses the parser. To workaround this, we accept only limited kind of values.
* - the content, which follows the h2 element.
* - The section content is a collection of members. Each member consists of:
* - an anchor ("A") element with "name" attribute, and
* - a div element which contains an h4 child element whose class contains "jd-details-title".
* - The h4 element contains the member signature. We parse it and retrieve the method name and list of parameters.
* - Parameters are tokenized by ", ".
* - Note that the splitter contains a white space which disambiguates any use of generic arguments (we don't want to split "Foo<K,V> bar" as "Foo<K" and "V> bar")
*
* API Level 10 to 15 has slightly different format:
*
* - There is no "doc-col" element. But "jd-header" and "jd-content" are still alive.
*
*/
public void Import(ImporterOptions options)
{
options.DiagnosticWriter.WriteLine(options.DocumentDirectory);
string referenceDocsTopDir = Path.Combine(options.DocumentDirectory, "reference");
var htmlFiles = Directory.GetDirectories(referenceDocsTopDir).SelectMany(d => Directory.GetFiles(d, "*.html", SearchOption.AllDirectories));
var api = new JavaApi();
foreach (var htmlFile in htmlFiles)
{
// skip irrelevant files.
if (excludes.Any(x => htmlFile.EndsWith(x, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
var packageName = Path.GetDirectoryName(htmlFile).Substring(referenceDocsTopDir.Length + 1).Replace('/', '.');
if (options.FrameworkOnly && non_frameworks.Any(n => packageName.StartsWith(n, StringComparison.Ordinal)))
{
continue;
}
options.DiagnosticWriter.WriteLine("-- " + htmlFile);
var doc = new HtmlLoader().GetJavaDocFile(htmlFile);
var header = doc.Descendants().FirstOrDefault(e => e.Attribute("id")?.Value == "jd-header");
var content = doc.Descendants().FirstOrDefault(e => e.Attribute("id")?.Value == "jd-content");
if (header == null || content == null)
{
continue;
}
var apiSignatureTokens = header.Value.Replace('\r', ' ').Replace('\n', ' ').Replace('\t', ' ').Trim();
if (apiSignatureTokens.Contains("extends "))
{
apiSignatureTokens = apiSignatureTokens.Substring(0, apiSignatureTokens.IndexOf("extends ", StringComparison.Ordinal)).Trim();
}
if (apiSignatureTokens.Contains("implements "))
{
apiSignatureTokens = apiSignatureTokens.Substring(0, apiSignatureTokens.IndexOf("implements ", StringComparison.Ordinal)).Trim();
}
bool isClass = apiSignatureTokens.Contains("class");
options.DiagnosticWriter.WriteLine(apiSignatureTokens);
var javaPackage = api.Packages.FirstOrDefault(p => p.Name == packageName);
if (javaPackage == null)
{
javaPackage = new JavaPackage(api)
{
Name = packageName
};
api.Packages.Add(javaPackage);
}
var javaType = isClass ? (JavaType) new JavaClass(javaPackage) : new JavaInterface(javaPackage);
javaType.Name = apiSignatureTokens.Substring(apiSignatureTokens.LastIndexOf(' ') + 1);
javaPackage.Types.Add(javaType);
string sectionType = null;
var sep = new string [] { ", " };
var ssep = new char [] { ' ' };
foreach (var child in content.Descendants())
{
if (child.Name == "h2")
{
var value = child.Value;
switch (value)
{
case "Public Constructors":
case "Protected Constructors":
case "Public Methods":
case "Protected Methods":
sectionType = value;
break;
}
continue;
}
if (sectionType == null)
{
continue;
}
if (child.Name != "a" || child.Attribute("name") == null)
{
continue;
}
var h4 = child.XPathSelectElement("following-sibling::div/h4[contains(@class, 'jd-details-title')]");
if (h4 == null)
{
continue;
}
string sigTypeOnly = child.Attribute("name").Value;
string sigTypeAndName = h4.Value.Replace('\n', ' ').Replace('\r', ' ').Trim();
if (!sigTypeAndName.Contains('('))
{
continue;
}
JavaMethodBase javaMethod = null;
string name = sigTypeAndName.Substring(0, sigTypeAndName.IndexOf('(')).Split(ssep, StringSplitOptions.RemoveEmptyEntries).Last();
switch (sectionType)
{
case "Public Constructors":
case "Protected Constructors":
javaMethod = new JavaConstructor(javaType)
{
Name = name
};
break;
case "Public Methods":
case "Protected Methods":
string mname = sigTypeAndName.Substring(0, sigTypeAndName.IndexOf('('));
javaMethod = new JavaMethod(javaType)
{
Name = name
};
break;
}
javaType.Members.Add(javaMethod);
var paramTypes = SplitTypes(sigTypeOnly.Substring(sigTypeOnly.IndexOf('(') + 1).TrimEnd(')'), 0).ToArray();
var parameters = sigTypeAndName.Substring(sigTypeAndName.IndexOf('(') + 1).TrimEnd(')')
.Split(sep, StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.ToArray();
foreach (var p in paramTypes.Zip(parameters, (to, tn) => new { Type = to, TypeAndName = tn })
.Select(pp => new { Type = pp.Type, Name = pp.TypeAndName.Split(' ') [1] }))
{
javaMethod.Parameters.Add(new JavaParameter(javaMethod)
{
Name = p.Name, Type = p.Type
});
}
}
javaType.Members = javaType.Members.OfType <JavaMethodBase> ()
.OrderBy(m => m.Name + "(" + string.Join(",", m.Parameters.Select(p => p.Type)) + ")")
.ToArray();
}
foreach (var pkg in api.Packages)
{
pkg.Types = pkg.Types.OrderBy(t => t.Name).ToArray();
}
api.Packages = api.Packages.OrderBy(p => p.Name).ToArray();
if (options.OutputTextFile != null)
{
api.WriteParameterNamesText(options.OutputTextFile);
}
if (options.OutputXmlFile != null)
{
api.WriteParameterNamesXml(options.OutputXmlFile);
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="module">The module that will own all references</param>
public Importer(ModuleDef module)
{
this.module = module;
this.recursionCounter = new RecursionCounter();
this.options = 0;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="module">The module that will own all references</param>
/// <param name="options">Importer options</param>
public Importer(ModuleDef module, ImporterOptions options)
{
this.module = module;
this.recursionCounter = new RecursionCounter();
this.options = options;
}
protected RowImporterBase(ISheetsApiHelper sheetsApiHelper, IOptions <ImporterOptions> importerOptionsAccessor, ILogger <RowImporterBase <T> > logger)
{
_sheetsApiHelper = sheetsApiHelper;
_importerOptions = importerOptionsAccessor.Value;
_logger = logger;
}
