diff --git a/MANIFEST.in b/MANIFEST.in index 08f31b1..7061606 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ include CONTRIBUTING.rst include LICENSE.txt include README.rst include requirements/base.in -recursive-include code_annotations *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.yaml *.yml +recursive-include code_annotations *.tpl *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.yaml *.yml diff --git a/README.rst b/README.rst index dd91285..37e6f32 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,6 @@ code-annotations ============================= -|pypi-badge| |CI| |codecov-badge| |doc-badge| |pyversions-badge| -|license-badge| - Extensible tools for parsing annotations in codebases Overview @@ -55,28 +52,3 @@ Have a question about this repository, or about Open edX in general? Please refer to this `list of resources`_ if you need any assistance. .. _list of resources: https://open.edx.org/getting-help - - -.. |pypi-badge| image:: https://img.shields.io/pypi/v/code-annotations.svg - :target: https://pypi.python.org/pypi/code-annotations/ - :alt: PyPI - -.. |CI| image:: https://github.com/openedx/code-annotations/workflows/Python%20CI/badge.svg?branch=master - :target: https://github.com/openedx/code-annotations/actions?query=workflow%3A%22Python+CI%22 - :alt: CI - -.. |codecov-badge| image:: http://codecov.io/github/edx/code-annotations/coverage.svg?branch=master - :target: http://codecov.io/github/edx/code-annotations?branch=master - :alt: Codecov - -.. |doc-badge| image:: https://readthedocs.org/projects/code-annotations/badge/?version=latest - :target: http://code-annotations.readthedocs.io/en/latest/ - :alt: Documentation - -.. |pyversions-badge| image:: https://img.shields.io/pypi/pyversions/code-annotations.svg - :target: https://pypi.python.org/pypi/code-annotations/ - :alt: Supported Python versions - -.. |license-badge| image:: https://img.shields.io/github/license/edx/code-annotations.svg - :target: https://github.com/openedx/code-annotations/blob/master/LICENSE.txt - :alt: License diff --git a/code_annotations/base.py b/code_annotations/base.py index dddfcb7..6f7964d 100644 --- a/code_annotations/base.py +++ b/code_annotations/base.py @@ -14,6 +14,9 @@ from code_annotations.exceptions import ConfigurationException from code_annotations.helpers import VerboseEcho +PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_TEMPLATE_DIR = os.path.join(PACKAGE_DIR, "report_templates") + class AnnotationConfig: """ @@ -58,10 +61,17 @@ def __init__(self, config_file_path, report_path_override=None, verbosity=1, sou self.echo(f"Configured for source path: {self.source_path}") self._configure_coverage(raw_config.get('coverage_target', None)) - self.report_template_dir = raw_config.get('report_template_dir') - self.rendered_report_dir = raw_config.get('rendered_report_dir') - self.rendered_report_file_extension = raw_config.get('rendered_report_file_extension') - self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix') + + self.rendered_report_format = raw_config.get('rendered_report_format', 'rst') + self.report_template_dir = raw_config.get( + 'report_template_dir', + os.path.join(DEFAULT_TEMPLATE_DIR, self.rendered_report_format) + ) + self.rendered_report_dir = raw_config.get('rendered_report_dir', 'annotation_reports') + self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix', None) + self.trim_filename_prefixes = raw_config.get('trim_filename_prefixes', []) + self.third_party_package_location = raw_config.get('third_party_package_location', "site-packages") + self.rendered_report_file_extension = f".{self.rendered_report_format}" self._configure_annotations(raw_config) self._configure_extensions() diff --git a/code_annotations/cli.py b/code_annotations/cli.py index 853da13..9e56f02 100644 --- a/code_annotations/cli.py +++ b/code_annotations/cli.py @@ -241,10 +241,15 @@ def generate_docs(config_file, verbosity, report_files): try: config = AnnotationConfig(config_file, verbosity) + if not report_files: + raise ConfigurationException( + "No report files provided. Please provide one or more report files to generate docs from." + ) + for key in ( "report_template_dir", "rendered_report_dir", - "rendered_report_file_extension", + "rendered_report_format", "rendered_report_source_link_prefix", ): if not getattr(config, key): diff --git a/code_annotations/generate_docs.py b/code_annotations/generate_docs.py index abf8319..e32dc37 100644 --- a/code_annotations/generate_docs.py +++ b/code_annotations/generate_docs.py @@ -27,8 +27,7 @@ def __init__(self, config, report_files): self.config = config self.echo = self.config.echo self.report_files = report_files - self.create_time = datetime.datetime.utcnow().isoformat() - + self.create_time = datetime.datetime.now(tz=datetime.timezone.utc) self.full_report = self._aggregate_reports() self.jinja_environment = jinja2.Environment( @@ -62,6 +61,12 @@ def _add_report_file_to_full_report(self, report_file, report): loaded_report = yaml.safe_load(report_file) for filename in loaded_report: + trimmed_filename = filename + for prefix in self.config.trim_filename_prefixes: + if filename.startswith(prefix): + trimmed_filename = filename[len(prefix):] + break + if filename in report: for loaded_annotation in loaded_report[filename]: found = False @@ -74,9 +79,9 @@ def _add_report_file_to_full_report(self, report_file, report): break if not found: - report[filename].append(loaded_annotation) + report[trimmed_filename].append(loaded_annotation) else: - report[filename] = loaded_report[filename] + report[trimmed_filename] = loaded_report[filename] def _aggregate_reports(self): """ @@ -91,11 +96,12 @@ def _aggregate_reports(self): return report - def _write_doc_file(self, doc_filename, doc_data): + def _write_doc_file(self, doc_title, doc_filename, doc_data): """ Write out a single report file with the given data. This is rendered using the configured top level template. Args: + doc_title: Title to use for the document. doc_filename: Filename to write to. doc_data: Dict of reporting data to use, in the {'file name': [list, of, annotations,]} style. """ @@ -110,14 +116,16 @@ def _write_doc_file(self, doc_filename, doc_data): with open(full_doc_filename, 'w') as output: output.write(self.top_level_template.render( + doc_title=doc_title, create_time=self.create_time, report=doc_data, all_choices=self.all_choices, all_annotations=self.config.annotation_tokens, group_mapping=self.group_mapping, slugify=slugify, - source_link_prefix=self.config.rendered_report_source_link_prefix) - ) + source_link_prefix=self.config.rendered_report_source_link_prefix, + third_party_package_location=self.config.third_party_package_location, + )) def _generate_per_choice_docs(self): """ @@ -130,7 +138,7 @@ def _generate_per_choice_docs(self): if isinstance(annotation['annotation_data'], list) and choice in annotation['annotation_data']: choice_report[filename].append(annotation) - self._write_doc_file(f'choice_{choice}', choice_report) + self._write_doc_file(f"All References to Choice '{choice}'", f'choice_{choice}', choice_report) def _generate_per_annotation_docs(self): """ @@ -143,13 +151,15 @@ def _generate_per_annotation_docs(self): if report_annotation['annotation_token'] == annotation: annotation_report[filename].append(report_annotation) - self._write_doc_file(f'annotation_{annotation}', annotation_report) + self._write_doc_file( + f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report + ) def render(self): """ Perform the rendering of all documentation using the configured Jinja2 templates. """ # Generate the top level list of all annotations - self._write_doc_file('index', self.full_report) + self._write_doc_file("Complete Annotation List", 'index', self.full_report) self._generate_per_choice_docs() self._generate_per_annotation_docs() diff --git a/code_annotations/report_templates/annotation.tpl b/code_annotations/report_templates/annotation.tpl deleted file mode 100644 index d0ce6c8..0000000 --- a/code_annotations/report_templates/annotation.tpl +++ /dev/null @@ -1,5 +0,0 @@ -{% if annotation.extra and annotation.extra.object_id %} -`<{{ annotation.extra.object_id }}> line {{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} -{% else %} -`{{ filename }}:{{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} -{% endif %} diff --git a/code_annotations/report_templates/annotation_group.tpl b/code_annotations/report_templates/annotation_group.tpl deleted file mode 100644 index ed719b0..0000000 --- a/code_annotations/report_templates/annotation_group.tpl +++ /dev/null @@ -1,3 +0,0 @@ -.. _index.rst#{{ slugify(filename + '-' + annotation.report_group_id |string) }}: -.. admonition:: Group "{{ group_mapping[annotation.annotation_token] }}" - diff --git a/code_annotations/report_templates/base.tpl b/code_annotations/report_templates/base.tpl deleted file mode 100644 index b460430..0000000 --- a/code_annotations/report_templates/base.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{% block content %}{% endblock %} - -{% for choice in all_choices %} -.. _choice_{{ slugify(choice) }}: {{ slugify('choice_' + choice) + '.rst' }} -{% endfor %} - -{% for annotation in all_annotations %} -.. _annotation_{{ slugify(annotation) }}: {{ slugify('annotation_' + annotation) + '.rst' }} -{% endfor %} - - -{% block footer %} -Built at {{ create_time }} -{% endblock %} diff --git a/code_annotations/report_templates/html/annotation.tpl b/code_annotations/report_templates/html/annotation.tpl new file mode 100644 index 0000000..8f4055b --- /dev/null +++ b/code_annotations/report_templates/html/annotation.tpl @@ -0,0 +1,14 @@ +{% if is_third_party %} +{# no links for third party code since we don't know where to link to #} +{% if annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} + {% if loop.changed(annotation.line_number)%}{{ filename }}:{{ annotation.line_number }}
{% endif %}: + {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} + +{% elif annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +`{{ filename }}:{{ annotation.line_number }}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} diff --git a/code_annotations/report_templates/html/annotation_data.tpl b/code_annotations/report_templates/html/annotation_data.tpl new file mode 100644 index 0000000..a95251c --- /dev/null +++ b/code_annotations/report_templates/html/annotation_data.tpl @@ -0,0 +1,8 @@ +{% if annotation.annotation_data is sequence and annotation.annotation_data is not string %} +{% for a in annotation.annotation_data %} +{{ a }}{% if not loop.last %}, {% endif %} +{% endfor %} + +{% else %} +{{ annotation.annotation_data }} +{% endif %} diff --git a/code_annotations/report_templates/html/annotation_list.tpl b/code_annotations/report_templates/html/annotation_list.tpl new file mode 100644 index 0000000..83434f2 --- /dev/null +++ b/code_annotations/report_templates/html/annotation_list.tpl @@ -0,0 +1,27 @@ +{% extends "base.tpl" %} +{% block content %} +Annotations found in {{ report|length }} files. + +{% for filename in report %} +{% set is_third_party = third_party_package_location in filename %} + +

{{ filename }}

+
+ {{ report[filename]|length }} annotations {% if is_third_party %}(installed package){% endif %}
+
+ + {% for annotation in report[filename] %} + {% if loop.changed(annotation.report_group_id) %} + {% if not loop.first %}{% endif %} +
+ {% endif %} + {% endfor %} + + +{% endfor %} + +{% endblock %} diff --git a/code_annotations/report_templates/html/base.tpl b/code_annotations/report_templates/html/base.tpl new file mode 100644 index 0000000..b292122 --- /dev/null +++ b/code_annotations/report_templates/html/base.tpl @@ -0,0 +1,92 @@ + + + {{ doc_title }} + + + + +

{{ doc_title }}

+ +
+
+
+

Home

+ +

Annotations

+ + + +

Choices

+ + +
+
+

Files in this page

+ + + {% block content %}{% endblock %} +
+
+
+{% block footer %} + +{% endblock %} + + diff --git a/code_annotations/report_templates/rst/annotation.tpl b/code_annotations/report_templates/rst/annotation.tpl new file mode 100644 index 0000000..2acccb7 --- /dev/null +++ b/code_annotations/report_templates/rst/annotation.tpl @@ -0,0 +1,13 @@ +{% if is_third_party %} +{# no links for third party code since we don't know where to link to #} +{% if annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +{{ filename }}:{{ annotation.line_number }}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} + +{% elif annotation.extra and annotation.extra.object_id %} +`{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +`{{ filename }}:{{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} diff --git a/code_annotations/report_templates/annotation_data.tpl b/code_annotations/report_templates/rst/annotation_data.tpl similarity index 100% rename from code_annotations/report_templates/annotation_data.tpl rename to code_annotations/report_templates/rst/annotation_data.tpl diff --git a/code_annotations/report_templates/rst/annotation_group.tpl b/code_annotations/report_templates/rst/annotation_group.tpl new file mode 100644 index 0000000..495a7b8 --- /dev/null +++ b/code_annotations/report_templates/rst/annotation_group.tpl @@ -0,0 +1,2 @@ +.. _index.rst#{{ slugify(filename + '-' + annotation.report_group_id |string) }}: +.. admonition:: {{ group_mapping[annotation.annotation_token] or annotation.annotation_token }} diff --git a/code_annotations/report_templates/annotation_list.tpl b/code_annotations/report_templates/rst/annotation_list.tpl similarity index 72% rename from code_annotations/report_templates/annotation_list.tpl rename to code_annotations/report_templates/rst/annotation_list.tpl index 624fd77..20226ab 100644 --- a/code_annotations/report_templates/annotation_list.tpl +++ b/code_annotations/report_templates/rst/annotation_list.tpl @@ -1,17 +1,22 @@ {% extends "base.tpl" %} {% block content %} -Complete Annotation List ------------------------- Annotations found in {{ report|length }} files. {% for filename in report %} -{{ filename }} has {{ report[filename]|length }} annotations. +{% set is_third_party = third_party_package_location in filename %} + +{{ filename }} +{{ "-" * filename|length }} + + .. note:: {{ report[filename]|length }} annotations {% if is_third_party %}(installed package){% endif %} + {% for annotation in report[filename] %} {% if annotation.report_group_id %} {% if loop.changed(annotation.report_group_id) %} {% include 'annotation_group.tpl' %} + {% endif %} * {% include 'annotation.tpl' %} diff --git a/code_annotations/report_templates/rst/base.tpl b/code_annotations/report_templates/rst/base.tpl new file mode 100644 index 0000000..77499d4 --- /dev/null +++ b/code_annotations/report_templates/rst/base.tpl @@ -0,0 +1,41 @@ +{{ "#" * doc_title|length }} +{{ doc_title }} +{{ "#" * doc_title|length }} + +.. sidebar:: Table of Contents + + `Home `_ + + Annotations + + {% for a in all_annotations %} + * annotation_{{ slugify(a) }}_ + {% endfor %} + + Choices + + {% for choice in all_choices %} + * choice_{{ slugify(choice) }}_ + {% endfor %} + + +.. contents:: + +{% block content %}{% endblock %} + + +{# backlinks for all choices #} +{% for choice in all_choices %} +.. _choice_{{ slugify(choice) }}: {{ slugify('choice_' + choice) + '.rst' }} +{% endfor %} + + +{# backlinks for all annotations #} +{% for annotation in all_annotations %} +.. _annotation_{{ slugify(annotation) }}: {{ slugify('annotation_' + annotation) + '.rst' }} +{% endfor %} + + +{% block footer %} +Built at {{ create_time.strftime('%Y-%m-%d %H:%M:%S %Z') }} +{% endblock %} diff --git a/docs/configuration.rst b/docs/configuration.rst index 83cc836..6469bb6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -26,6 +26,15 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi - py3 javascript: - js + # Options only used for human readable reports + report_template_dir: /my_custom_templates/report_templates/ + rendered_report_dir: rendered_reports + rendered_report_format: html + rendered_report_source_link_prefix: https://github.com/my_org/my_project/blob/master/ + trim_filename_prefixes: + - /my_org/my_project/ + - /my_project/venv/lib/python3.11/ + third_party_package_location: site-packages ``source_path`` The file or directory to be searched for annotations. If a directory, it will be searched recursively. This can be @@ -68,3 +77,21 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi extensions are turned on here. The key is the extension name, as given in the ``setup.py`` or ``setup.cfg`` of the package that installed the extension. The values are a list of file extensions that, when found, will be passed to the extension for annotation searching. See :doc:`extensions` for details. + +``report_template_dir`` (optional) + When running the ``generate_docs`` comman you can specify a custom template directory here to override the default templates if you would like a different look. + +``rendered_report_dir`` (optional) + When running the ``generate_docs`` command, this option specifies the directory where the rendered report will be written. The default is ``annotation_reports`` in the current working directory. + +``rendered_report_format`` (optional) + When running the ``generate_docs`` command, this option specifies the format of the rendered report. Options are ``html`` and ``rst``. The default is ``rst``. + +``rendered_report_source_link_prefix`` (optional) + When running the ``generate_docs`` command, this option specifies the URL prefix to use when linking to source files in the rendered report. When specified, "local" source files (those not found in site-packages) will be appended to this setting to create hyperlinks to the lines in source files online. For Github links this is the correct format: ``https://github.com/openedx/edx-platform/blob/master/``. The default is an None. + +``trim_filename_prefixes`` (optional) + When running the ``generate_docs`` command, this option specifies a list of prefixes to remove from the beginning of filenames in the rendered report. This is useful for making the report more readable by removing long, repetitive prefixes of the type often found in a Django search. The default is an empty list. + +``third_party_package_location`` (optional) + When running the ``generate_docs`` command, this option specifies the location of third party packages that may have been found in a Django search. This is used to determine if a file is a third party file or not. The default is ``site-packages``. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0fcf005..2ed3b1b 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -126,3 +126,17 @@ Add more structure to your annotations Annotations can be more than simple messages. They can enforce the use of choices from a fixed list, and can be grouped to provide more context-aware information. See :doc:`configuration` and :doc:`writing_annotations` for more information on how to use those options. + +Get a human readable report +--------------------------- +The output generated by the search commands is a YAML file. To get a human readable report generated from one of those files in either rst or html format, you can use the ``generate_docs`` command. + +There are several configuration options available for this command, including the ability to specify the output format, the output directory, and create links to the source files on sites like Github. For more information, see :doc:`configuration`. Once your configuration is set, you can run: + +.. code-block:: bash + + $ code_annotations generate_docs --config_file /path/to/your/config /path/to/your/report.yaml + +Which will generate files in the output directory you configured. From there you can open the files in your browser to see the report, if you chose HTML, or use a tool like `restview`_ to render the RST files to your browser. + +.. _restview: https://pypi.python.org/pypi/restview diff --git a/tests/test_configurations/.annotations_test_success_with_report_docs b/tests/test_configurations/.annotations_test_success_with_report_docs index 946c852..ff4858f 100644 --- a/tests/test_configurations/.annotations_test_success_with_report_docs +++ b/tests/test_configurations/.annotations_test_success_with_report_docs @@ -1,10 +1,11 @@ source_path: tests/extensions/javascript_test_files/ report_path: test_reports safelist_path: .annotation_safe_list.yml -report_template_dir: code_annotations/report_templates/ +report_template_dir: code_annotations/report_templates/rst rendered_report_dir: test_reports/ -rendered_report_file_extension: .rst +rendered_report_format: rst rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/ + coverage_target: 50.0 annotations: ".. no_pii:": diff --git a/tests/test_configurations/.annotations_test_success_with_report_docs_html b/tests/test_configurations/.annotations_test_success_with_report_docs_html new file mode 100644 index 0000000..27800be --- /dev/null +++ b/tests/test_configurations/.annotations_test_success_with_report_docs_html @@ -0,0 +1,26 @@ +source_path: tests/extensions/javascript_test_files/ +report_path: test_reports +safelist_path: .annotation_safe_list.yml +report_template_dir: code_annotations/report_templates/rst +rendered_report_dir: test_reports/ +rendered_report_format: html +rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/ +trim_filename_prefixes: + - tacos + - simple +third_party_package_location: site-packages + +coverage_target: 50.0 +annotations: + ".. no_pii:": + ".. ignored:": + choices: [irrelevant, terrible, silly-silly] + "pii_group": + - ".. pii:": + - ".. pii_types:": + choices: [id, name, other] + - ".. pii_retirement:": + choices: [retained, local_api, consumer_api, third_party] +extensions: + python: + - pyt diff --git a/tests/test_generate_docs.py b/tests/test_generate_docs.py index 94c9937..f352cb3 100644 --- a/tests/test_generate_docs.py +++ b/tests/test_generate_docs.py @@ -49,6 +49,40 @@ def test_generate_report_simple(): assert os.path.exists(created_doc) +def test_generate_report_simple_html(): + find_result = call_script( + ( + 'static_find_annotations', + '--config_file', + 'tests/test_configurations/.annotations_test_python_only', + '--source_path=tests/extensions/python_test_files/simple_success.pyt', + '--no_lint', + ), + delete_test_reports=False) + + assert find_result.exit_code == EXIT_CODE_SUCCESS + assert "Writing report..." in find_result.output + report_file = get_report_filename_from_output(find_result.output) + + report_result = call_script( + ( + 'generate_docs', + report_file, + '--config_file', + 'tests/test_configurations/.annotations_test_success_with_report_docs_html', + '-vv' + ), + delete_test_docs=False + ) + + assert find_result.exit_code == EXIT_CODE_SUCCESS + assert "Report rendered in" in report_result.output + + # All file types are created + for created_doc in ('test_reports/index.html', 'test_reports/choice-id.html', 'test_reports/annotation-pii.html'): + assert os.path.exists(created_doc) + + def _do_find(source_path, new_report_path): """ Do a static annotation search with report, rename the report to a distinct name. @@ -167,4 +201,4 @@ def test_generate_report_missing_key(): )) assert report_result.exit_code == EXIT_CODE_FAILURE - assert "No report_template_dir key in tests/test_configurations/" in report_result.output + assert "No rendered_report_source_link_prefix key in" in report_result.output