Markdown integration

build.gradle

@@ -48,6 +48,7 @@ dependencies {
     compile 'org.apache.commons:commons-lang3:3.4'
     compile 'org.apache.logging.log4j:log4j-api:2.3'
     compile 'org.apache.logging.log4j:log4j-core:2.3'
+    compile 'com.atlassian.commonmark:commonmark:0.9.0'
 
     // Get the jdk files we need to run javaDoc. We need to use these during compile, testCompile,
     // test execution, and gatkDoc generation, but we don't want them as part of the runtime

src/main/java/org/broadinstitute/barclay/argparser/CommandLineArgumentParser.java

@@ -12,6 +12,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.barclay.utils.Utils;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.text.TextContentRenderer;
 
 import java.io.BufferedReader;
 import java.io.FileReader;
@@ -74,6 +76,11 @@ public final class CommandLineArgumentParser implements CommandLineParser {
     // Keeps a map of tagged arguments for just-in-time retrieval at field population time
     private TaggedArgumentParser tagParser = new TaggedArgumentParser();
 
+    // default parser/renderer for Markdown formatted documentation
+    // TODO: allow customization of Markdown parser
+    private Parser markdownParser = Parser.builder().build();
+    private TextContentRenderer markdownToText = TextContentRenderer.builder().build();
+
     // Return the plugin instance corresponding to the targetDescriptor class
     @Override
     public <T> T getPluginDescriptor(Class<T> targetDescriptor) {
@@ -136,7 +143,8 @@ private boolean inArgumentMap(ArgumentDefinition arg){
     private String getUsagePreamble() {
         String usagePreamble = "";
         if (null != programProperties) {
-            usagePreamble += programProperties.summary();
+            // render as plain text to markdown
+            usagePreamble += markdownToText.render(markdownParser.parse(programProperties.summary()));
         } else if (positionalArguments == null) {
             usagePreamble += defaultUsagePreamble;
         } else {
@@ -805,7 +813,8 @@ private void printArgumentParamUsage(final StringBuilder sb, final String name,
     private String makeArgumentDescription(final ArgumentDefinition argumentDefinition) {
         final StringBuilder sb = new StringBuilder();
         if (!argumentDefinition.doc.isEmpty()) {
-            sb.append(argumentDefinition.doc);
+            // render markdown to text
+            markdownToText.render(markdownParser.parse(argumentDefinition.doc), sb);
             sb.append("  ");
         }
         if (argumentDefinition.isCollection) {

src/main/java/org/broadinstitute/barclay/help/DefaultDocWorkUnitHandler.java

@@ -8,6 +8,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.barclay.argparser.*;
+import org.broadinstitute.barclay.utils.Utils;
+import org.commonmark.renderer.Renderer;
 
 import java.lang.reflect.*;
 import java.util.*;
@@ -33,6 +35,14 @@ public DefaultDocWorkUnitHandler(final HelpDoclet doclet) {
         super(doclet);
     }
 
+    /**
+     * @param doclet the HelpDoclet driving this documentation run. Can not be null.
+     * @param markdownRenderer renderer for rendering markdown formatted strings. Can not be null.
+     */
+    public DefaultDocWorkUnitHandler(final HelpDoclet doclet, final Renderer markdownRenderer) {
+        super(doclet, markdownRenderer);
+    }
+
     /**
      * Return the template to be used for the particular workUnit. Must be present in the location
      * specified is the -settings-dir doclet parameter.
@@ -55,16 +65,19 @@ public String getTemplateName(final DocWorkUnit workUnit) {
      */
     @Override
     public String getSummaryForWorkUnit(final DocWorkUnit workUnit) {
-        String summary = workUnit.getDocumentedFeature().summary();
-        if (summary == null || summary.isEmpty()) {
+        // get the documented feature already rendered by the super method
+        String summary = super.getSummaryForWorkUnit(workUnit);
+        if (summary.isEmpty()) {
             final CommandLineProgramProperties commandLineProperties = workUnit.getCommandLineProperties();
             if (commandLineProperties != null) {
-                summary = commandLineProperties.oneLineSummary();
+                summary = renderMarkdown(commandLineProperties.oneLineSummary());
             }
-            if (summary == null || summary.isEmpty()) {
+            if (summary.isEmpty()) {
                 // If no summary was found from annotations, use the javadoc if there is any
+                // it is not rendered, because we do not expect Markdown in the Javadoc
                 summary = Arrays.stream(workUnit.getClassDoc().firstSentenceTags())
-                        .map(tag -> tag.text())
+                        // render possibly markdown formatted javadoc
+                        .map(tag -> renderMarkdown(tag.text()))
                         .collect(Collectors.joining());
             }
         }
@@ -106,15 +119,15 @@ public String getGroupNameForWorkUnit(final DocWorkUnit workUnit) {
      */
     @Override
     public String getGroupSummaryForWorkUnit( final DocWorkUnit workUnit){
-        String groupSummary = workUnit.getDocumentedFeature().groupSummary();
+        // get the documented feature already rendered by the super method
+        String groupSummary = super.getGroupSummaryForWorkUnit(workUnit);
         final CommandLineProgramGroup clpGroup = workUnit.getCommandLineProgramGroup();
-        if (groupSummary == null || groupSummary.isEmpty()) {
+        if (groupSummary.isEmpty()) {
             if (clpGroup != null) {
-                groupSummary = clpGroup.getDescription();
+                groupSummary = renderMarkdown(clpGroup.getDescription());
             }
-            if (groupSummary == null || groupSummary.isEmpty()) {
+            if (groupSummary.isEmpty()) {
                 logger.warn("No group summary declared for: " + workUnit.getClazz().getCanonicalName());
-                groupSummary = "";
             }
         }
         return groupSummary;
@@ -130,7 +143,8 @@ public String getGroupSummaryForWorkUnit( final DocWorkUnit workUnit){
     protected String getDescription(final DocWorkUnit currentWorkUnit) {
         return Arrays.stream(currentWorkUnit.getClassDoc().inlineTags())
                 .filter(t -> getTagPrefix() == null || !t.name().startsWith(getTagPrefix()))
-                .map(t -> t.text())
+                // render possibly markdown formatted javadoc
+                .map(t -> renderMarkdown(t.text()))
                 .collect(Collectors.joining());
     }
 
@@ -226,10 +240,10 @@ protected void addHighLevelBindings(final DocWorkUnit workUnit)
     {
         workUnit.setProperty("name", workUnit.getName());
         workUnit.setProperty("group", workUnit.getGroupName());
-        workUnit.setProperty("summary", workUnit.getSummary());
+        workUnit.setProperty("summary", renderMarkdown(workUnit.getSummary()));
         workUnit.setProperty("beta", workUnit.getBetaFeature());
 
-        workUnit.setProperty("description", getDescription(workUnit));
+        workUnit.setProperty("description", renderMarkdown(getDescription(workUnit)));
 
         workUnit.setProperty("version", getDoclet().getBuildVersion());
         workUnit.setProperty("timestamp", getDoclet().getBuildTimeStamp());
@@ -245,6 +259,7 @@ protected void addCustomBindings(final DocWorkUnit currentWorkUnit) {
         final String tagFilterPrefix = getTagPrefix();
         Arrays.stream(currentWorkUnit.getClassDoc().inlineTags())
                 .filter(t -> t.name().startsWith(tagFilterPrefix))
+                // TODO: should we render also text in custom tags?
                 .forEach(t -> currentWorkUnit.setProperty(t.name().substring(tagFilterPrefix.length()), t.text()));
     }
 
@@ -442,8 +457,8 @@ private void processPositionalArguments(
             PositionalArguments posArgs = positionalField.getAnnotation(PositionalArguments.class);
             argBindings.put("kind", "positional");
             argBindings.put("name", NAME_FOR_POSITIONAL_ARGS);
-            argBindings.put("summary", posArgs.doc());
-            argBindings.put("fulltext", posArgs.doc());
+            argBindings.put("summary", renderMarkdown(posArgs.doc()));
+            argBindings.put("fulltext", renderMarkdown(posArgs.doc()));
             argBindings.put("otherArgumentRequired", "NA");
             argBindings.put("synonyms", "NA");
             argBindings.put("exclusiveOf", "NA");
@@ -721,8 +736,8 @@ protected Map<String, Object> docForArgument(final FieldDoc fieldDoc, final Comm
         root.put("type", argumentTypeString(def.field.getGenericType()));
 
         // summary and fulltext
-        root.put("summary", def.doc != null ? def.doc : "");
-        root.put("fulltext", fieldDoc.commentText());
+        root.put("summary", renderMarkdown(def.doc));
+        root.put("fulltext", renderMarkdown(fieldDoc.commentText()));
 
         // Does this argument interact with any others?
         if (def.isControlledByPlugin()) {
@@ -779,7 +794,7 @@ private List<Map<String, Object>> docForEnumArgument(final Class<?> enumClass) {
                             static final long serialVersionUID = 0L;
                             {
                                 put("name", fieldDoc.name());
-                                put("summary", fieldDoc.commentText());
+                                put("summary", renderMarkdown(fieldDoc.commentText()));
                             }
                         }
                 );

src/main/java/org/broadinstitute/barclay/help/DocWorkUnitHandler.java

@@ -1,6 +1,10 @@
 package org.broadinstitute.barclay.help;
 
+import org.apache.commons.lang3.StringUtils;
 import org.broadinstitute.barclay.utils.Utils;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.Renderer;
+import org.commonmark.renderer.html.HtmlRenderer;
 
 import java.io.*;
 import java.util.List;
@@ -11,14 +15,33 @@
  * used for each documented feature, and populates the template property map for that template.
  */
 public abstract class DocWorkUnitHandler {
+    /**
+     * Default Markdown to HTML renderer.
+     */
+    // TODO: add extensions to default renderer?
+    protected static final Renderer DEFAULT_RENDERER = HtmlRenderer.builder().build();
+
     private final HelpDoclet doclet;
+    private final Renderer markdownRenderer;
+
+    // default parser for Markdown formatted documentation
+    // TODO: allow customization of Markdown parser
+    private Parser markdownParser = Parser.builder().build();
 
     /**
      * @param doclet the HelpDoclet driving this documentation run. Can not be null.
      */
     public DocWorkUnitHandler(final HelpDoclet doclet) {
-        Utils.nonNull("Doclet cannot be null");
-        this.doclet = doclet;
+        this(doclet, DEFAULT_RENDERER);
+    }
+
+    /**
+     * @param doclet the HelpDoclet driving this documentation run. Can not be null.
+     * @param markdownRenderer renderer for rendering markdown formatted strings. Can not be null.
+     */
+    public DocWorkUnitHandler(final HelpDoclet doclet, final Renderer markdownRenderer) {
+       this.doclet = Utils.nonNull(doclet, "Doclet cannot be null");
+       this.markdownRenderer = Utils.nonNull(markdownRenderer, "Renderer cannot be null");
     }
 
     /**
@@ -28,6 +51,24 @@ public HelpDoclet getDoclet() {
         return doclet;
     }
 
+    /**
+     * @return rendered markdown string for documentation; empty String if the {@code markdownString} is null or empty.
+     */
+    public final String renderMarkdown(final String markdownString) {
+        if (markdownString == null || markdownString.isEmpty()) {
+            return "";
+        }
+        // render the String and remove the end of line added by the parser/renderer
+        final String renderedString = StringUtils.stripEnd(markdownRenderer.render(markdownParser.parse(markdownString)), "\n");
+        // Markdown parser/renderer adds always a <p></p> around the parsed String even if it is a single line
+        // but we do not want this in single line documentation
+        // TODO: commonmark should have a way to remove this behaviour from the parser/renderer
+        if (!renderedString.contains("\n")) {
+            return StringUtils.removePattern(renderedString, "^<p>|</p>$");
+        }
+        return renderedString;
+    }
+
     /**
      * Actually generate the documentation map by populating the associated workUnit's properties.
      *
@@ -57,12 +98,12 @@ public String getDestinationFilename(final DocWorkUnit workUnit) {
 
     /**
      * Apply any fallback rules to determine the summary line that should be used for the work unit.
-     * Default implementation uses the value from the DocumentedFeature annotation.
+     * Default implementation uses the value from the DocumentedFeature annotation, rendered with {@link #renderMarkdown(String)}.
      * @param workUnit
      * @return Summary for this work unit.
      */
     public String getSummaryForWorkUnit(final DocWorkUnit workUnit) {
-        return workUnit.getDocumentedFeature().summary();
+        return renderMarkdown(workUnit.getDocumentedFeature().summary());
     }
 
     /**
@@ -77,12 +118,12 @@ public String getGroupNameForWorkUnit(final DocWorkUnit workUnit) {
 
     /**
      * Apply any fallback rules to determine the group summary line that should be used for the work unit.
-     * Default implementation uses the value from the DocumentedFeature annotation.
+     * Default implementation uses the value from the DocumentedFeature annotation, rendered with {@link #renderMarkdown(String)}.
      * @param workUnit
      * @return Group summary to be used for this work unit.
      */
     public String getGroupSummaryForWorkUnit(final DocWorkUnit workUnit) {
-        return workUnit.getDocumentedFeature().groupSummary();
+        return renderMarkdown(workUnit.getDocumentedFeature().groupSummary());
     }
 
 }

src/test/java/org/broadinstitute/barclay/argparser/CommandLineArgumentParserTest.java

@@ -715,6 +715,30 @@ private static String captureSystemStream(Runnable runnable, PrintStream stream,
         return out.toString();
     }
 
+    @CommandLineProgramProperties(
+            summary = "Tool with [Markdown](https://daringfireball.net/projects/markdown/) formatted docs",
+            oneLineSummary = "Tool with Markdown formatted docs",
+            programGroup = TestProgramGroup.class
+    )
+    public class MarkdownFormattedDocs {
+        @Argument(doc = "`String` **BOLD** __argument__", optional = true)
+        public String bold = "";
+
+        @Argument(doc = "`String` *ITALIC* _argument_", optional = true)
+        public String italic = "";
+    }
+
+    @Test
+    public void testMarkdownToTextUsage() {
+        final CommandLineArgumentParser clp = new CommandLineArgumentParser(new MarkdownFormattedDocs());
+        final String usage = clp.usage(false, false);
+        // [text](link) is rendered as quoted text plus link in parenthesis
+        Assert.assertTrue(usage.contains("Tool with \"Markdown\" (https://daringfireball.net/projects/markdown/) formatted docs"));
+        // Italic/Bold is rendered as normal text, backtick is rendered as quoted string
+        Assert.assertTrue(usage.contains("\"String\" BOLD argument"));
+        Assert.assertTrue(usage.contains("\"String\" ITALIC argument"));
+    }
+
 
     @CommandLineProgramProperties(
             summary = "tool with nullable arguments",

src/test/java/org/broadinstitute/barclay/help/ClpWithMarkdownDocs.java

@@ -0,0 +1,60 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015 Daniel Gómez-Sánchez
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.broadinstitute.barclay.help;
+
+import org.broadinstitute.barclay.argparser.Argument;
+import org.broadinstitute.barclay.argparser.ArgumentCollection;
+import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
+import org.broadinstitute.barclay.argparser.TestProgramGroup;
+
+/**
+ * Documentation of this tool is written in Markdown and rendered as HTML.
+ *
+ * ## Javadoc formatted with Markdown
+ *
+ * The purpose of this paragraph is to test embedded [Markdown](https://daringfireball.net/projects/markdown/) formatting:
+ *
+ * - First element of list (**bold**)
+ * - Second element of list (_italic_)
+ *
+ * @author Daniel Gomez-Sanchez (magicDGS)
+ */
+@CommandLineProgramProperties(
+        summary = "Test tool summary with [Markdown](https://daringfireball.net/projects/markdown/) formatted docs",
+        oneLineSummary = "Tool where the documentation is formatted with [Markdown](https://daringfireball.net/projects/markdown/) `String`.",
+        programGroup = TestProgramGroup.class)
+@DocumentedFeature(groupName = MarkdownDocumentedFeature.markdownGroup, groupSummary = MarkdownDocumentedFeature.markdownGroupSummary)
+public class ClpWithMarkdownDocs {
+
+    @Argument(doc = "This is a **bold** `String` __argument__")
+    public String bold = "";
+
+    @Argument(doc = "This is a *italic* `String` _argument_")
+    public String italic = "";
+
+    @ArgumentCollection(doc = "Argument collection with [Markdown](https://daringfireball.net/projects/markdown/) formatted docs")
+    public MarkdownArgumentCollection argumentCollection = new MarkdownArgumentCollection();
+
+}