clean: lint and format, add pre-commit, update README

.circleci/config.yml

@@ -54,11 +54,12 @@ jobs:
       - attach_workspace:
           at: .
       - run:
-          name: Lint code
-          command: poetry run flake8 << pipeline.parameters.python-module >>
-      - run:
-          name: Lint tests
-          command: poetry run flake8 tests/
+          name: Lint and format code and sort imports
+          # ruff check --select I . : check linting and imports sorting without fixing (to fix, use --fix)
+          # ruff format --check . : check code formatting without fixing (to fix, remove --check)
+          command: |
+            poetry run ruff check --select I .
+            poetry run ruff format --check .
 
   tests:
     docker:

.gitignore

@@ -41,7 +41,13 @@ js-built
 /udata/static/
 /jsdoc/
 *.rdb
+
+# Python packaging
+.ruff_cache
 venv
+.venv
+.python-version
+uv.lock
 
 # Installer logs
 pip-log.txt
@@ -74,4 +80,4 @@ Procfile
 .idea
 
 # Config
-config.toml
+config.toml

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+
+  # https://github.com/pre-commit/pre-commit-hooks#pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-added-large-files
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.5 # Ruff version
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --extend-select, I]
+      # Run the formatter
+      - id: ruff-format

README.md

@@ -16,7 +16,7 @@ You can now access the raw postgrest API on http://localhost:8080.
 
 Now you can launch the proxy (ie the app):
 
-```
+```shell
 poetry install
 poetry run adev runserver -p8005 api_tabular/app.py        # Api related to apified CSV files by udata-hydra
 poetry run adev runserver -p8005 api_tabular/metrics.py    # Api related to udata's metrics
@@ -186,6 +186,25 @@ returns
 ```
 
 Pagination is made through queries with `page` and `page_size`:
-```
+```shell
 curl http://localhost:8005/api/resources/aaaaaaaa-1111-bbbb-2222-cccccccccccc/data/?page=2&page_size=30
 ```
+
+
+## Contributing
+
+### Pre-commit hook
+
+This repository uses a [pre-commit](https://pre-commit.com/) hook which lint and format code before each commit.
+Please install it with:
+```shell
+poetry run pre-commit install
+```
+
+### Lint and format code
+
+To lint, format and sort imports, this repository uses [Ruff](https://astral.sh/ruff/).
+You can run the following command to lint and format the code:
+```shell
+poetry run ruff check --fix && poetry run ruff format
+```

api_tabular/__init__.py

@@ -1,5 +1,4 @@
 import os
-
 from pathlib import Path
 
 import toml

api_tabular/app.py

@@ -1,19 +1,24 @@
 import os
-import sentry_sdk
-import aiohttp_cors
 
-from aiohttp import web, ClientSession
+import aiohttp_cors
+import sentry_sdk
+from aiohttp import ClientSession, web
 from aiohttp_swagger import setup_swagger
-
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
+
 from api_tabular import config
+from api_tabular.error import QueryException
 from api_tabular.query import (
     get_resource,
     get_resource_data,
     get_resource_data_streamed,
 )
-from api_tabular.utils import build_sql_query_string, build_link_with_page, url_for, build_swagger_file
-from api_tabular.error import QueryException
+from api_tabular.utils import (
+    build_link_with_page,
+    build_sql_query_string,
+    build_swagger_file,
+    url_for,
+)
 
 routes = web.RouteTableDef()
 
@@ -27,26 +32,24 @@
 @routes.get(r"/api/resources/{rid}/", name="meta")
 async def resource_meta(request):
     resource_id = request.match_info["rid"]
-    resource = await get_resource(
-        request.app["csession"], resource_id, ["created_at", "url"]
-    )
+    resource = await get_resource(request.app["csession"], resource_id, ["created_at", "url"])
     return web.json_response(
         {
             "created_at": resource["created_at"],
             "url": resource["url"],
             "links": [
                 {
-                    "href": url_for(request, 'profile', rid=resource_id, _external=True),
+                    "href": url_for(request, "profile", rid=resource_id, _external=True),
                     "type": "GET",
                     "rel": "profile",
                 },
                 {
-                    "href": url_for(request, 'data', rid=resource_id, _external=True),
+                    "href": url_for(request, "data", rid=resource_id, _external=True),
                     "type": "GET",
                     "rel": "data",
                 },
                 {
-                    "href": url_for(request, 'swagger', rid=resource_id, _external=True),
+                    "href": url_for(request, "swagger", rid=resource_id, _external=True),
                     "type": "GET",
                     "rel": "swagger",
                 },
@@ -58,19 +61,15 @@ async def resource_meta(request):
 @routes.get(r"/api/resources/{rid}/profile/", name="profile")
 async def resource_profile(request):
     resource_id = request.match_info["rid"]
-    resource = await get_resource(
-        request.app["csession"], resource_id, ["profile:csv_detective"]
-    )
+    resource = await get_resource(request.app["csession"], resource_id, ["profile:csv_detective"])
     return web.json_response(resource)
 
 
 @routes.get(r"/api/resources/{rid}/swagger/", name="swagger")
 async def resource_swagger(request):
     resource_id = request.match_info["rid"]
-    resource = await get_resource(
-        request.app["csession"], resource_id, ["profile:csv_detective"]
-    )
-    swagger_string = build_swagger_file(resource['profile']['columns'], resource_id)
+    resource = await get_resource(request.app["csession"], resource_id, ["profile:csv_detective"])
+    swagger_string = build_swagger_file(resource["profile"]["columns"], resource_id)
     return web.Response(body=swagger_string)
 
 
@@ -82,9 +81,7 @@ async def resource_data(request):
     page_size = int(request.query.get("page_size", config.PAGE_SIZE_DEFAULT))
 
     if page_size > config.PAGE_SIZE_MAX:
-        raise QueryException(
-            400, None, "Invalid query string", "Page size exceeds allowed maximum"
-        )
+        raise QueryException(400, None, "Invalid query string", "Page size exceeds allowed maximum")
     if page > 1:
         offset = page_size * (page - 1)
     else:
@@ -95,20 +92,16 @@ async def resource_data(request):
     except ValueError:
         raise QueryException(400, None, "Invalid query string", "Malformed query")
 
-    resource = await get_resource(
-        request.app["csession"], resource_id, ["parsing_table"]
-    )
-    response, total = await get_resource_data(
-        request.app["csession"], resource, sql_query
-    )
+    resource = await get_resource(request.app["csession"], resource_id, ["parsing_table"])
+    response, total = await get_resource_data(request.app["csession"], resource, sql_query)
 
     next = build_link_with_page(request, query_string, page + 1, page_size)
     prev = build_link_with_page(request, query_string, page - 1, page_size)
     body = {
         "data": response,
         "links": {
-            "profile": url_for(request, 'profile', rid=resource_id, _external=True),
-            "swagger": url_for(request, 'swagger', rid=resource_id, _external=True),
+            "profile": url_for(request, "profile", rid=resource_id, _external=True),
+            "swagger": url_for(request, "swagger", rid=resource_id, _external=True),
             "next": next if page_size + offset < total else None,
             "prev": prev if page > 1 else None,
         },
@@ -127,9 +120,7 @@ async def resource_data_csv(request):
     except ValueError:
         raise QueryException(400, None, "Invalid query string", "Malformed query")
 
-    resource = await get_resource(
-        request.app["csession"], resource_id, ["parsing_table"]
-    )
+    resource = await get_resource(request.app["csession"], resource_id, ["parsing_table"])
 
     response_headers = {
         "Content-Disposition": f'attachment; filename="{resource_id}.csv"',
@@ -138,9 +129,7 @@ async def resource_data_csv(request):
     response = web.StreamResponse(headers=response_headers)
     await response.prepare(request)
 
-    async for chunk in get_resource_data_streamed(
-        request.app["csession"], resource, sql_query
-    ):
+    async for chunk in get_resource_data_streamed(request.app["csession"], resource, sql_query):
         await response.write(chunk)
 
     await response.write_eof()
@@ -168,16 +157,19 @@ async def on_cleanup(app):
         app,
         defaults={
             "*": aiohttp_cors.ResourceOptions(
-                    allow_credentials=True,
-                    expose_headers="*",
-                    allow_headers="*"
-                )
-        }
+                allow_credentials=True, expose_headers="*", allow_headers="*"
+            )
+        },
     )
     for route in list(app.router.routes()):
         cors.add(route)
 
-    setup_swagger(app, swagger_url=config.DOC_PATH, ui_version=3, swagger_from_file="ressource_app_swagger.yaml")
+    setup_swagger(
+        app,
+        swagger_url=config.DOC_PATH,
+        ui_version=3,
+        swagger_from_file="ressource_app_swagger.yaml",
+    )
 
     return app
 

api_tabular/error.py

@@ -1,24 +1,22 @@
 import json
+from typing import Union
+
 import sentry_sdk
 from aiohttp import web
+
 from api_tabular import config
-from typing import Union
 
 
 class QueryException(web.HTTPException):
     """Re-raise an exception from postgrest as aiohttp exception"""
 
     def __init__(self, status, error_code, title, detail) -> None:
         self.status_code = status
-        error_body = {
-            "errors": [{"code": error_code, "title": title, "detail": detail}]
-        }
+        error_body = {"errors": [{"code": error_code, "title": title, "detail": detail}]}
         super().__init__(content_type="application/json", text=json.dumps(error_body))
 
 
-def handle_exception(
-    status: int, title: str, detail: Union[str, dict], resource_id: str = None
-):
+def handle_exception(status: int, title: str, detail: Union[str, dict], resource_id: str = None):
     event_id = None
     e = Exception(detail)
     if config.SENTRY_DSN:

api_tabular/metrics.py

@@ -1,18 +1,18 @@
 import os
-import sentry_sdk
-import aiohttp_cors
 
+import aiohttp_cors
+import sentry_sdk
+from aiohttp import ClientSession, web
 from aiohttp_swagger import setup_swagger
-from aiohttp import web, ClientSession
-
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
+
 from api_tabular import config
+from api_tabular.error import QueryException, handle_exception
 from api_tabular.utils import (
-    build_sql_query_string,
     build_link_with_page,
+    build_sql_query_string,
     process_total,
 )
-from api_tabular.error import QueryException, handle_exception
 
 routes = web.RouteTableDef()
 
@@ -46,14 +46,12 @@ async def get_object_data_streamed(
     res = await session.head(f"{url}&limit=1&", headers=headers)
     total = process_total(res.headers.get("Content-Range"))
     for i in range(0, total, batch_size):
-        async with session.get(
-            url=f"{url}&limit={batch_size}&offset={i}", headers=headers
-        ) as res:
+        async with session.get(url=f"{url}&limit={batch_size}&offset={i}", headers=headers) as res:
             if not res.ok:
                 handle_exception(res.status, "Database error", await res.json(), None)
             async for chunk in res.content.iter_chunked(1024):
                 yield chunk
-            yield b'\n'
+            yield b"\n"
 
 
 @routes.get(r"/api/{model}/data/")
@@ -67,9 +65,7 @@ async def metrics_data(request):
     page_size = int(request.query.get("page_size", config.PAGE_SIZE_DEFAULT))
 
     if page_size > config.PAGE_SIZE_MAX:
-        raise QueryException(
-            400, None, "Invalid query string", "Page size exceeds allowed maximum"
-        )
+        raise QueryException(400, None, "Invalid query string", "Page size exceeds allowed maximum")
     if page > 1:
         offset = page_size * (page - 1)
     else:
@@ -111,9 +107,7 @@ async def metrics_data_csv(request):
     response = web.StreamResponse(headers=response_headers)
     await response.prepare(request)
 
-    async for chunk in get_object_data_streamed(
-        request.app["csession"], model, sql_query
-    ):
+    async for chunk in get_object_data_streamed(request.app["csession"], model, sql_query):
         await response.write(chunk)
 
     return response
@@ -140,16 +134,16 @@ async def on_cleanup(app):
         app,
         defaults={
             "*": aiohttp_cors.ResourceOptions(
-                    allow_credentials=True,
-                    expose_headers="*",
-                    allow_headers="*"
-                )
-        }
+                allow_credentials=True, expose_headers="*", allow_headers="*"
+            )
+        },
     )
     for route in list(app.router.routes()):
         cors.add(route)
 
-    setup_swagger(app, swagger_url=config.DOC_PATH, ui_version=3, swagger_from_file="metrics_swagger.yaml")
+    setup_swagger(
+        app, swagger_url=config.DOC_PATH, ui_version=3, swagger_from_file="metrics_swagger.yaml"
+    )
 
     return app