Skip to content

Commit a65b8ab

Browse files
jonpryorjonpryor
committed
[generator] Add generator --doc-comment-style=summary,remarks
Context: #687 (comment) Context: dotnet/android#5253 (comment) "Full Javadoc Import" is slow, causing CI `generator` invocation to take ~12x longer, and `CoreCompile` to take ~7x longer. What happens if we instead do a "Summary Javadoc Import", and only emit `<summary/>`, `<param/>`, `<returns/>`, `<exception/>`? Add a `generator --doc-comment-style=summary,remarks` option, so that we can emit only "summary" docs instead of *all* docs, and see what the build time changes are.
1 parent ab03227 commit a65b8ab

File tree

10 files changed

+259
-56
lines changed

10 files changed

+259
-56
lines changed

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,26 +42,37 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
4242

4343
AuthorDeclaration.Rule = "@author" + BlockValues;
4444
AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
45+
if (!grammar.ShouldImport (ImportJavadoc.AuthorTag))
46+
return;
4547
// Ignore; not sure how best to convert to Xmldoc
4648
FinishParse (context, parseNode);
4749
};
4850

4951
ApiSinceDeclaration.Rule = "@apiSince" + BlockValues;
5052
ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
53+
if (!grammar.ShouldImport (ImportJavadoc.SinceTag)) {
54+
return;
55+
}
5156
var p = new XElement ("para", "Added in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
5257
FinishParse (context, parseNode).Remarks.Add (p);
5358
parseNode.AstNode = p;
5459
};
5560

5661
DeprecatedDeclaration.Rule = "@deprecated" + BlockValues;
5762
DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
63+
if (!grammar.ShouldImport (ImportJavadoc.DeprecatedTag)) {
64+
return;
65+
}
5866
var p = new XElement ("para", "This member is deprecated. ", AstNodeToXmlContent (parseNode.ChildNodes [1]));
5967
FinishParse (context, parseNode).Remarks.Add (p);
6068
parseNode.AstNode = p;
6169
};
6270

6371
DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValues;
6472
DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
73+
if (!grammar.ShouldImport (ImportJavadoc.DeprecatedTag)) {
74+
return;
75+
}
6576
var p = new XElement ("para", "This member was deprecated in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
6677
FinishParse (context, parseNode).Remarks.Add (p);
6778
parseNode.AstNode = p;
@@ -75,6 +86,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
7586

7687
ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValues;
7788
ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
89+
if (!grammar.ShouldImport (ImportJavadoc.ExceptionTag)) {
90+
return;
91+
}
7892
// TODO: convert `nonSpaceTerm` into a proper CREF
7993
var e = new XElement ("exception",
8094
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
@@ -85,6 +99,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
8599

86100
ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValues;
87101
ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
102+
if (!grammar.ShouldImport (ImportJavadoc.ParamTag)) {
103+
return;
104+
}
88105
var p = new XElement ("param",
89106
new XAttribute ("name", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
90107
AstNodeToXmlContent (parseNode.ChildNodes [2]));
@@ -95,6 +112,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
95112
ReturnDeclaration.Rule = "@return" + BlockValues;
96113
// ReturnDeclaration.Flags = TermFlags.IsMultiline;
97114
ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
115+
if (!grammar.ShouldImport (ImportJavadoc.ReturnTag)) {
116+
return;
117+
}
98118
var r = new XElement ("returns",
99119
AstNodeToXmlContent (parseNode.ChildNodes [1]));
100120
FinishParse (context, parseNode).Returns.Add (r);
@@ -103,6 +123,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
103123

104124
SeeDeclaration.Rule = "@see" + BlockValues;
105125
SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
126+
if (!grammar.ShouldImport (ImportJavadoc.SeeTag)) {
127+
return;
128+
}
106129
// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
107130
var e = new XElement ("altmember",
108131
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))));
@@ -112,13 +135,19 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
112135

113136
SinceDeclaration.Rule = "@since" + BlockValues;
114137
SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
138+
if (!grammar.ShouldImport (ImportJavadoc.SinceTag)) {
139+
return;
140+
}
115141
var p = new XElement ("para", "Added in ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
116142
FinishParse (context, parseNode).Remarks.Add (p);
117143
parseNode.AstNode = p;
118144
};
119145

120146
ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValues;
121147
ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
148+
if (!grammar.ShouldImport (ImportJavadoc.ExceptionTag)) {
149+
return;
150+
}
122151
// TODO: convert `nonSpaceTerm` into a proper CREF
123152
var e = new XElement ("exception",
124153
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
@@ -130,16 +159,25 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
130159
// Ignore serialization informatino
131160
SerialDeclaration.Rule = "@serial" + BlockValues;
132161
SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
162+
if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) {
163+
return;
164+
}
133165
FinishParse (context, parseNode);
134166
};
135167

136168
SerialDataDeclaration.Rule = "@serialData" + BlockValues;
137169
SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
170+
if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) {
171+
return;
172+
}
138173
FinishParse (context, parseNode);
139174
};
140175

141176
SerialFieldDeclaration.Rule = "@serialField" + BlockValues;
142177
SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
178+
if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) {
179+
return;
180+
}
143181
FinishParse (context, parseNode);
144182
};
145183

@@ -152,13 +190,19 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
152190

153191
UnknownTagDeclaration.Rule = unknownTagTerminal + BlockValues;
154192
UnknownTagDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
193+
if (!grammar.ShouldImport (ImportJavadoc.Remarks)) {
194+
return;
195+
}
155196
Console.WriteLine ($"# Unsupported @block-tag value: {parseNode.ChildNodes [0].AstNode}");
156197
FinishParse (context, parseNode);
157198
};
158199

159200
// Ignore Version
160201
VersionDeclaration.Rule = "@version" + BlockValues;
161202
VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
203+
if (!grammar.ShouldImport (ImportJavadoc.VersionTag)) {
204+
return;
205+
}
162206
FinishParse (context, parseNode);
163207
};
164208
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,17 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
4545

4646
TopLevelInlineDeclaration.Rule = inlineDeclarations;
4747
TopLevelInlineDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
48-
var remarks = FinishParse (context, parseNode).Remarks;
48+
var remarks = FinishParse (context, parseNode).Remarks;
49+
var addRemarks = grammar.ShouldImport (ImportJavadoc.Remarks) ||
50+
(grammar.ShouldImport (ImportJavadoc.Summary) && remarks.Count == 0);
51+
if (!addRemarks) {
52+
parseNode.AstNode = "";
53+
return;
54+
}
4955
foreach (var p in GetParagraphs (parseNode.ChildNodes)) {
5056
remarks.Add (p);
5157
}
52-
parseNode.AstNode = "";
58+
parseNode.AstNode = "";
5359
};
5460

5561
var fontstyle_tt = CreateHtmlToCrefElement (grammar, "tt", "c", InlineDeclarations);
@@ -58,6 +64,13 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
5864
var preText = new PreBlockDeclarationBodyTerminal ();
5965
PreBlockDeclaration.Rule = CreateStartElement ("pre", grammar) + preText + CreateEndElement ("pre", grammar);
6066
PreBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
67+
if (!grammar.ShouldImport (ImportJavadoc.Remarks)) {
68+
return;
69+
}
70+
if (grammar.XmldocStyle == XmldocStyle.Summary) {
71+
parseNode.AstNode = "";
72+
return;
73+
}
6174
var c = new XElement ("code",
6275
new XAttribute ("lang", "text/java"),
6376
parseNode.ChildNodes [1].Token.Value);
@@ -71,6 +84,13 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
7184
CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
7285
;
7386
PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
87+
var remarks = FinishParse (context, parseNode).Remarks;
88+
var addRemarks = grammar.ShouldImport (ImportJavadoc.Remarks) ||
89+
(grammar.ShouldImport (ImportJavadoc.Summary) && remarks.Count == 0);
90+
if (!addRemarks) {
91+
parseNode.AstNode = "";
92+
return;
93+
}
7494
var p = new XElement ("para",
7595
parseNode.ChildNodes
7696
.Select (c => AstNodeToXmlContent (c)));
@@ -276,6 +296,8 @@ public override void Init (GrammarData grammarData)
276296

277297
class UnknownHtmlElementStartTerminal : Terminal {
278298

299+
bool addingRemarks;
300+
279301
public UnknownHtmlElementStartTerminal (string name)
280302
: base (name)
281303
{
@@ -285,6 +307,8 @@ public UnknownHtmlElementStartTerminal (string name)
285307
public override void Init (GrammarData grammarData)
286308
{
287309
base.Init (grammarData);
310+
var g = grammarData.Grammar as SourceJavadocToXmldocGrammar;
311+
addingRemarks = g?.ShouldImport (ImportJavadoc.Remarks) ?? false;
288312
}
289313

290314
public override Token? TryMatch (ParsingContext context, ISourceStream source)
@@ -296,7 +320,9 @@ public override void Init (GrammarData grammarData)
296320
int stop = start;
297321
while (source.Text [stop] != '>' && stop < source.Text.Length)
298322
stop++;
299-
Console.Error.WriteLine ($"# Unsupported HTML element: {source.Text.Substring (start, stop - start)}");
323+
if (addingRemarks) {
324+
Console.Error.WriteLine ($"# Unsupported HTML element: {source.Text.Substring (start, stop - start)}");
325+
}
300326
return source.CreateToken (this.OutputTerminal, "<");
301327
}
302328
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,16 @@ public partial class SourceJavadocToXmldocGrammar : Grammar {
1616
public readonly InlineTagsBnfTerms InlineTagsTerms;
1717
public readonly HtmlBnfTerms HtmlTerms;
1818

19-
public SourceJavadocToXmldocGrammar ()
19+
public readonly XmldocStyle XmldocStyle;
20+
21+
public SourceJavadocToXmldocGrammar (XmldocStyle style)
2022
{
2123
BlockTagsTerms = new BlockTagsBnfTerms ();
2224
InlineTagsTerms = new InlineTagsBnfTerms ();
2325
HtmlTerms = new HtmlBnfTerms ();
2426

27+
XmldocStyle = style;
28+
2529
BlockTagsTerms.CreateRules (this);
2630
InlineTagsTerms.CreateRules (this);
2731
HtmlTerms.CreateRules (this);
@@ -49,6 +53,12 @@ public SourceJavadocToXmldocGrammar ()
4953
this.Root = root;
5054
}
5155

56+
internal bool ShouldImport (ImportJavadoc value)
57+
{
58+
var v = (ImportJavadoc) XmldocStyle;
59+
return v.HasFlag (value);
60+
}
61+
5262
internal static void ConcatChildNodes (AstContext context, ParseTreeNode parseNode)
5363
{
5464
switch (parseNode.ChildNodes.Count) {

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,59 @@
1111

1212
namespace Java.Interop.Tools.JavaSource {
1313

14+
[Flags]
15+
internal enum ImportJavadoc {
16+
None,
17+
Summary = 1 << 0,
18+
Remarks = 1 << 1,
19+
AuthorTag = 1 << 2,
20+
DeprecatedTag = 1 << 3,
21+
ExceptionTag = 1 << 4,
22+
ParamTag = 1 << 5,
23+
ReturnTag = 1 << 6,
24+
SeeTag = 1 << 7,
25+
SerialTag = 1 << 8,
26+
SinceTag = 1 << 9,
27+
VersionTag = 1 << 10,
28+
}
29+
30+
[Flags]
31+
public enum XmldocStyle {
32+
None,
33+
Remarks = ImportJavadoc.Summary
34+
| ImportJavadoc.Remarks
35+
| ImportJavadoc.AuthorTag
36+
| ImportJavadoc.DeprecatedTag
37+
| ImportJavadoc.ExceptionTag
38+
| ImportJavadoc.ParamTag
39+
| ImportJavadoc.ReturnTag
40+
| ImportJavadoc.SeeTag
41+
| ImportJavadoc.SerialTag
42+
| ImportJavadoc.SinceTag
43+
| ImportJavadoc.VersionTag
44+
,
45+
Summary = ImportJavadoc.Summary
46+
| ImportJavadoc.ExceptionTag
47+
| ImportJavadoc.ParamTag
48+
| ImportJavadoc.ReturnTag
49+
,
50+
}
51+
1452
public class SourceJavadocToXmldocParser : Irony.Parsing.Parser {
1553

16-
public SourceJavadocToXmldocParser ()
17-
: base (CreateGrammar ())
54+
public SourceJavadocToXmldocParser (XmldocStyle style = XmldocStyle.Remarks)
55+
: base (CreateGrammar (style))
1856
{
57+
XmldocStyle = style;
1958
}
2059

60+
public XmldocStyle XmldocStyle { get; }
61+
2162
public XElement[]? ExtraRemarks { get; set; }
2263

23-
static Grammar CreateGrammar ()
64+
static Grammar CreateGrammar (XmldocStyle style)
2465
{
25-
return new SourceJavadocToXmldocGrammar () {
66+
return new SourceJavadocToXmldocGrammar (style) {
2667
LanguageFlags = LanguageFlags.Default | LanguageFlags.CreateAst,
2768
};
2869
}
@@ -56,7 +97,9 @@ IEnumerable<XNode> CreateParseIterator (ParseTree parseTree)
5697
var summary = CreateSummaryNode (info);
5798
if (summary != null)
5899
yield return summary;
59-
if (info.Remarks.Count > 0 || ExtraRemarks?.Length > 0) {
100+
var style = (ImportJavadoc) XmldocStyle;
101+
if (style.HasFlag (ImportJavadoc.Remarks) &&
102+
(info.Remarks.Count > 0 || ExtraRemarks?.Length > 0)) {
60103
yield return new XElement ("remarks", info.Remarks, ExtraRemarks);
61104
}
62105
foreach (var n in info.Returns) {

‎tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammarFixture.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class SourceJavadocToXmldocGrammarFixture {
1919

2020
public static Parser CreateParser (Func<SourceJavadocToXmldocGrammar, NonTerminal> root)
2121
{
22-
var g = new SourceJavadocToXmldocGrammar () {
22+
var g = new SourceJavadocToXmldocGrammar (XmldocStyle.Remarks) {
2323
LanguageFlags = LanguageFlags.Default | LanguageFlags.CreateAst,
2424
};
2525
g.Root = root (g);

0 commit comments

Comments
 (0)