Skip to content

Commit 9c016bc

Browse files
jonpryorjonpryor
committed
[generator] Add --with-javadoc-xml=FILE support.
Commit 69e1b80 added `tools/java-source-utils`, which can parse Java source code and extract Javadoc comments into an XML file: $ java -jar "java-source-utils.jar" -v \ $HOME/android-toolchain/sdk/platforms/android-29/android-stubs-src.jar \ --output-javadoc android-javadoc.xml What can we *do* with the generated `android-javadoc.xml`? `android-javadoc.xml` contains parameter names, and thus can be used with `class-parse --docspath`; see commit 806082f. What we *really* want to do is make the Javadoc information *useful* to consumers of the binding assembly. This means that we want a C# XML Documentation file for the binding assembly. The most straightforward way to get a C# XML Documentation File is to emit [C# XML Documentation Comments][0] into the binding source code! Add a new `generator --with-javadoc-xml=FILE` option. When specified, `FILE` will be treated as an XML file containing the output from `java-source-utils.jar --output-javadoc` (see 69e1b80)`, and all `<javadoc/>` elements within the XML file will be associated with C# types and members to emit, based on the `//@jni-signature` and `//@name` attributes, as appropriate. When the bindings are written to disk, the Javadoc comments will be translated to C# XML Documentation comments, in a "best effort" basis. (THIS WILL BE INCOMPLETE.) To perform the Javadoc-to-C# XML Documentation comments conversion, add a new Irony-based grammar to `Java.Interop.Tools.JavaSource.dll`, in the new `Java.Interop.Tools.JavaSource.SourceJavadocToXmldocParser` type, which parses the Javadoc content and translates to XML. TODO: * Properties? [0]: https://docs.microsoft.com/en-us/dotnet/csharp/codedoc
1 parent ee7afee commit 9c016bc

30 files changed

+1196
-0
lines changed

‎src/Java.Interop.Tools.JavaSource/IronyExtensions.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Text;
6+
using Irony.Ast;
7+
using Irony.Parsing;
8+
9+
namespace Java.Interop.Tools.JavaSource {
10+
11+
static class IronyExtensions {
12+
13+
public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter, BnfTerm of)
14+
{
15+
star.Rule = grammar.MakeStarRule (star, delimiter, of);
16+
}
17+
18+
public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm of)
19+
{
20+
star.Rule = grammar.MakeStarRule (star, of);
21+
}
22+
}
23+
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.ObjectModel;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Xml.Linq;
7+
8+
using Irony.Ast;
9+
using Irony.Parsing;
10+
11+
namespace Java.Interop.Tools.JavaSource {
12+
13+
sealed class JavadocInfo {
14+
public readonly ICollection<XNode> Parameters = new Collection<XNode> ();
15+
public readonly ICollection<XNode> Paragraphs = new Collection<XNode> ();
16+
public readonly ICollection<XNode> Exceptions = new Collection<XNode> ();
17+
}
18+
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Xml.Linq;
6+
7+
using Irony.Ast;
8+
using Irony.Parsing;
9+
10+
namespace Java.Interop.Tools.JavaSource {
11+
12+
public partial class SourceJavadocToXmldocGrammar {
13+
14+
// https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags
15+
public class BlockTagsBnfTerms {
16+
17+
internal BlockTagsBnfTerms ()
18+
{
19+
}
20+
21+
internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
22+
{
23+
AllBlockTerms.Rule = AuthorDeclaration
24+
| ApiSinceDeclaration
25+
| DeprecatedDeclaration
26+
| DeprecatedSinceDeclaration
27+
| ExceptionDeclaration
28+
| ParamDeclaration
29+
| ReturnDeclaration
30+
| SeeDeclaration
31+
| SerialDataDeclaration
32+
| SerialFieldDeclaration
33+
| SinceDeclaration
34+
| ThrowsDeclaration
35+
| VersionDeclaration
36+
;
37+
BlockValue.Rule = Cdata | grammar.InlineTagsTerms.AllInlineTerms;
38+
39+
AuthorDeclaration.Rule = "@author" + BlockValue;
40+
AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
41+
// Ignore; not sure how best to convert to Xmldoc
42+
parseNode.AstNode = "";
43+
};
44+
45+
ApiSinceDeclaration.Rule = "@apiSince" + BlockValue;
46+
ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
47+
parseNode.AstNode = new XElement ("para", "Added in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
48+
};
49+
50+
DeprecatedDeclaration.Rule = "@deprecated" + BlockValue;
51+
DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
52+
parseNode.AstNode = new XElement ("para", "This member is deprecated. " + parseNode.ChildNodes [1].AstNode?.ToString ());
53+
};
54+
55+
DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValue;
56+
DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
57+
parseNode.AstNode = new XElement ("para", "This member was deprecated in API level " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
58+
};
59+
60+
var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") {
61+
AstConfig = new AstNodeConfig {
62+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value,
63+
},
64+
};
65+
66+
ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValue;
67+
ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
68+
// TODO: convert `nonSpaceTerm` into a proper CREF
69+
parseNode.AstNode = new XElement ("exception",
70+
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
71+
parseNode.ChildNodes [2].AstNode);
72+
};
73+
74+
ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValue;
75+
ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
76+
parseNode.AstNode = new XElement ("param",
77+
new XAttribute ("name", parseNode.ChildNodes [1].AstNode?.ToString ()),
78+
parseNode.ChildNodes [2].AstNode);
79+
};
80+
81+
ReturnDeclaration.Rule = "@return" + BlockValue;
82+
ReturnDeclaration.Flags = TermFlags.IsMultiline;
83+
ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
84+
var returns = parseNode.ChildNodes [1].AstNode.ToString ();
85+
parseNode.AstNode = new XElement ("returns", returns);
86+
};
87+
88+
SeeDeclaration.Rule = "@see" + BlockValue;
89+
SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
90+
// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
91+
parseNode.AstNode = new XElement ("altmember",
92+
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()));
93+
};
94+
95+
SinceDeclaration.Rule = "@since" + BlockValue;
96+
SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
97+
parseNode.AstNode = new XElement ("para", "Added in " + parseNode.ChildNodes [1].AstNode?.ToString () + ".");
98+
};
99+
100+
ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValue;
101+
ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
102+
// TODO: convert `nonSpaceTerm` into a proper CREF
103+
parseNode.AstNode = new XElement ("exception",
104+
new XAttribute ("cref", parseNode.ChildNodes [1].AstNode?.ToString ()),
105+
parseNode.ChildNodes [2].AstNode);
106+
};
107+
108+
// Ignore serialization informatino
109+
SerialDeclaration.Rule = "@serial" + BlockValue;
110+
SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
111+
parseNode.AstNode = "";
112+
};
113+
114+
SerialDataDeclaration.Rule = "@serialData" + BlockValue;
115+
SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
116+
parseNode.AstNode = "";
117+
};
118+
119+
SerialFieldDeclaration.Rule = "@serialField" + BlockValue;
120+
SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
121+
parseNode.AstNode = "";
122+
};
123+
124+
// Ignore Version
125+
VersionDeclaration.Rule = "@version" + BlockValue;
126+
VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
127+
parseNode.AstNode = "";
128+
};
129+
}
130+
131+
public readonly NonTerminal AllBlockTerms = new NonTerminal (nameof (AllBlockTerms), ConcatChildNodes);
132+
133+
public readonly Terminal Cdata = new RegexBasedTerminal (nameof (BlockValue), "[^<]*") {
134+
AstConfig = new AstNodeConfig {
135+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
136+
},
137+
};
138+
139+
public readonly NonTerminal BlockValue = new NonTerminal (nameof (BlockValue), ConcatChildNodes);
140+
public readonly NonTerminal AuthorDeclaration = new NonTerminal (nameof (AuthorDeclaration));
141+
public readonly NonTerminal ApiSinceDeclaration = new NonTerminal (nameof (ApiSinceDeclaration));
142+
public readonly NonTerminal DeprecatedDeclaration = new NonTerminal (nameof (DeprecatedDeclaration));
143+
public readonly NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration));
144+
public readonly NonTerminal ExceptionDeclaration = new NonTerminal (nameof (ExceptionDeclaration));
145+
public readonly NonTerminal ParamDeclaration = new NonTerminal (nameof (ParamDeclaration));
146+
public readonly NonTerminal ReturnDeclaration = new NonTerminal (nameof (ReturnDeclaration));
147+
public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration));
148+
public readonly NonTerminal SerialDeclaration = new NonTerminal (nameof (SerialDeclaration));
149+
public readonly NonTerminal SerialDataDeclaration = new NonTerminal (nameof (SerialDataDeclaration));
150+
public readonly NonTerminal SerialFieldDeclaration = new NonTerminal (nameof (SerialFieldDeclaration));
151+
public readonly NonTerminal SinceDeclaration = new NonTerminal (nameof (SinceDeclaration));
152+
public readonly NonTerminal ThrowsDeclaration = new NonTerminal (nameof (ThrowsDeclaration));
153+
public readonly NonTerminal VersionDeclaration = new NonTerminal (nameof (VersionDeclaration));
154+
}
155+
}
156+
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Xml.Linq;
6+
7+
using Irony.Ast;
8+
using Irony.Parsing;
9+
10+
namespace Java.Interop.Tools.JavaSource {
11+
12+
using static IronyExtensions;
13+
14+
public partial class SourceJavadocToXmldocGrammar {
15+
16+
public class HtmlBnfTerms {
17+
internal HtmlBnfTerms ()
18+
{
19+
}
20+
21+
internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
22+
{
23+
AllHtmlTerms.Rule = InlineDeclaration
24+
| PBlockDeclaration
25+
;
26+
27+
InlineDeclaration.Rule = ParsedCharacterData
28+
| FontStyleDeclaration
29+
/*
30+
| PhraseDeclaration
31+
| SpecialDeclaration
32+
| FormCtrlDeclaration
33+
*/
34+
| grammar.InlineTagsTerms.AllInlineTerms
35+
| UnknownHtmlElementStart
36+
;
37+
InlineDeclarations.MakeStarRule (grammar, InlineDeclaration);
38+
39+
var fontstyle_tt = new NonTerminal ("<tt>") {
40+
Rule = CreateStartElement ("tt", grammar) + InlineDeclarations + CreateEndElement ("tt", grammar),
41+
AstConfig = new AstNodeConfig {
42+
NodeCreator = (context, parseNode) => {
43+
parseNode.AstNode = new XElement ("c",
44+
parseNode.ChildNodes
45+
.Select (c => c.AstNode ?? ""));
46+
},
47+
},
48+
};
49+
50+
var fontstyle_i = new NonTerminal ("<i>", ConcatChildNodes) {
51+
Rule = CreateStartElement ("i", grammar) + InlineDeclarations + CreateEndElement ("i", grammar),
52+
};
53+
54+
FontStyleDeclaration.Rule = fontstyle_tt | fontstyle_i;
55+
56+
PBlockDeclaration.Rule =
57+
CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
58+
;
59+
PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
60+
parseNode.AstNode = new XElement ("para",
61+
parseNode.ChildNodes
62+
.Select (c => c.AstNode ?? ""));
63+
};
64+
}
65+
66+
public readonly NonTerminal AllHtmlTerms = new NonTerminal (nameof (AllHtmlTerms), ConcatChildNodes);
67+
68+
69+
// https://www.w3.org/TR/html401/struct/global.html#h-7.5.3
70+
// public readonly Terminal ParsedCharacterData = new RegexBasedTerminal (nameof (ParsedCharacterData), "[^<{@}]*") {
71+
// public readonly Terminal ParsedCharacterData = new WikiTextTerminal (nameof (ParsedCharacterData)) {*
72+
public readonly Terminal ParsedCharacterData = new CharacterDataTerminal ("#PCDATA") {
73+
// Priority = TerminalPriority.Low,
74+
AstConfig = new AstNodeConfig {
75+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
76+
},
77+
};
78+
79+
// https://www.w3.org/TR/html4/sgml/dtd.html#inline
80+
public readonly NonTerminal InlineDeclaration = new NonTerminal (nameof (InlineDeclaration), ConcatChildNodes);
81+
public readonly NonTerminal InlineDeclarations = new NonTerminal (nameof (InlineDeclarations), ConcatChildNodes);
82+
// https://www.w3.org/TR/html4/sgml/dtd.html#fontstyle
83+
public readonly NonTerminal FontStyleDeclaration = new NonTerminal (nameof (FontStyleDeclaration), ConcatChildNodes);
84+
// https://www.w3.org/TR/html4/sgml/dtd.html#phrase
85+
public readonly NonTerminal PhraseDeclaration = new NonTerminal (nameof (PhraseDeclaration), ConcatChildNodes);
86+
// https://www.w3.org/TR/html4/sgml/dtd.html#special
87+
public readonly NonTerminal SpecialDeclaration = new NonTerminal (nameof (SpecialDeclaration), ConcatChildNodes);
88+
// https://www.w3.org/TR/html4/sgml/dtd.html#formctrl
89+
public readonly NonTerminal FormCtrlDeclaration = new NonTerminal (nameof (FormCtrlDeclaration), ConcatChildNodes);
90+
// https://www.w3.org/TR/html4/sgml/dtd.html#block
91+
public readonly NonTerminal BlockDeclaration = new NonTerminal (nameof (BlockDeclaration), ConcatChildNodes);
92+
public readonly NonTerminal PBlockDeclaration = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes);
93+
94+
public readonly Terminal UnknownHtmlElementStart = new UnknownHtmlElementStartTerminal (nameof (UnknownHtmlElementStart)) {
95+
AstConfig = new AstNodeConfig {
96+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
97+
},
98+
};
99+
100+
static NonTerminal CreateStartElement (string startElement, Grammar grammar)
101+
{
102+
var start = new NonTerminal ("<" + startElement + ">", nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
103+
Rule = grammar.ToTerm ("<" + startElement + ">") | "<" + startElement.ToUpperInvariant () + ">",
104+
};
105+
return start;
106+
}
107+
108+
static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false)
109+
{
110+
var end = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
111+
Rule = grammar.ToTerm ("</" + endElement + ">") | "</" + endElement.ToUpperInvariant () + ">",
112+
};
113+
if (optional) {
114+
end.Rule |= grammar.Empty;
115+
}
116+
return end;
117+
}
118+
}
119+
}
120+
121+
// Based in part on WikiTextTerminal
122+
class CharacterDataTerminal : Terminal {
123+
124+
private char[] _stopChars;
125+
126+
public CharacterDataTerminal (string name)
127+
: base (name)
128+
{
129+
base.Priority = TerminalPriority.Low;
130+
}
131+
132+
public override void Init (GrammarData grammarData)
133+
{
134+
base.Init (grammarData);
135+
var stopCharSet = new Irony.CharHashSet ();
136+
foreach(var term in grammarData.Terminals) {
137+
var firsts = term.GetFirsts ();
138+
if (firsts == null)
139+
continue;
140+
foreach (var first in firsts) {
141+
if (string.IsNullOrEmpty (first))
142+
continue;
143+
stopCharSet.Add (first [0]);
144+
}
145+
}
146+
_stopChars = stopCharSet.ToArray();
147+
}
148+
149+
public override Token TryMatch (ParsingContext context, ISourceStream source)
150+
{
151+
var stopIndex = source.Text.IndexOfAny (_stopChars, source.Location.Position);
152+
if (stopIndex == source.Location.Position)
153+
return null;
154+
if (stopIndex < 0)
155+
stopIndex = source.Text.Length;
156+
source.PreviewPosition = stopIndex;
157+
158+
// preserve leading whitespace, if present.
159+
int start = source.Location.Position;
160+
while (start > 0 && char.IsWhiteSpace (source.Text, start-1)) {
161+
start--;
162+
}
163+
var content = source.Text.Substring (start, stopIndex - start);
164+
165+
return source.CreateToken (this.OutputTerminal, content);
166+
}
167+
}
168+
169+
class UnknownHtmlElementStartTerminal : Terminal {
170+
171+
public UnknownHtmlElementStartTerminal (string name)
172+
: base (name)
173+
{
174+
base.Priority = TerminalPriority.Low-1;
175+
}
176+
177+
public override void Init (GrammarData grammarData)
178+
{
179+
base.Init (grammarData);
180+
}
181+
182+
public override Token TryMatch (ParsingContext context, ISourceStream source)
183+
{
184+
if (source.Text [source.Location.Position] != '<')
185+
return null;
186+
source.PreviewPosition += 1;
187+
return source.CreateToken (this.OutputTerminal, "<");
188+
}
189+
}
190+
}

0 commit comments

Comments
 (0)