Override config with env and rename PGREST_ENDPOINT

CHANGELOG.md

@@ -3,4 +3,5 @@
 - Add endpoint to stream a CSV response [#5](https://github.com/etalab/api-tabular/pull/5)
 - Make URL in links absolute [#7](https://github.com/etalab/api-tabular/pull/7)
 - Add health route [#16](https://github.com/etalab/api-tabular/pull/16)
-- Add SERVER NAME and SCHEME config [#17](https://github.com/etalab/api-tabular/pull/17)
+- Add SERVER NAME and SCHEME config [#17](https://github.com/etalab/api-tabular/pull/17)
+- Override config with env [#18](https://github.com/etalab/api-tabular/pull/18)

api_tabular/__init__.py

@@ -17,15 +17,21 @@ def __init__(self):
     def configure(self):
         # load default settings
         configuration = toml.load(Path(__file__).parent / "config_default.toml")
-        if "PGREST_ENDPOINT" in os.environ:
-            configuration["PG_RST_URL"] = f"http://{os.getenv('PGREST_ENDPOINT')}"
 
-        configuration["PG_RST_URL"]
         # override with local settings
         local_settings = os.environ.get("CSVAPI_SETTINGS", Path.cwd() / "config.toml")
         if Path(local_settings).exists():
             configuration.update(toml.load(local_settings))
 
+        # override with os env settings
+        for config_key in configuration:
+            if config_key in os.environ:
+                configuration[config_key] = os.getenv(config_key)
+
+        # Make sure PGREST_ENDPOINT has a scheme
+        if not configuration["PGREST_ENDPOINT"].startswith("http"):
+            configuration["PGREST_ENDPOINT"] = f"http://{configuration['PGREST_ENDPOINT']}"
+
         self.configuration = configuration
         self.check()
 

api_tabular/config_default.toml

@@ -1,4 +1,4 @@
-PG_RST_URL = "http://localhost:8080"
+PGREST_ENDPOINT = "http://localhost:8080"
 SERVER_NAME = 'localhost:8005'
 SCHEME = 'http'
 SENTRY_DSN = ""

api_tabular/metrics.py

@@ -24,7 +24,7 @@
 
 async def get_object_data(session: ClientSession, model: str, sql_query: str):
     headers = {"Prefer": "count=exact"}
-    url = f"{config.PG_RST_URL}/{model}?{sql_query}"
+    url = f"{config.PGREST_ENDPOINT}/{model}?{sql_query}"
     async with session.get(url, headers=headers) as res:
         if not res.ok:
             handle_exception(res.status, "Database error", await res.json(), None)
@@ -41,7 +41,7 @@ async def get_object_data_streamed(
     batch_size: int = config.BATCH_SIZE,
 ):
     headers = {"Accept": accept_format, "Prefer": "count=exact"}
-    url = f"{config.PG_RST_URL}/{model}?{sql_query}"
+    url = f"{config.PGREST_ENDPOINT}/{model}?{sql_query}"
     res = await session.head(f"{url}&limit=1&", headers=headers)
     total = process_total(res.headers.get("Content-Range"))
     for i in range(0, total, batch_size):

api_tabular/query.py

@@ -6,7 +6,7 @@
 
 async def get_resource(session: ClientSession, resource_id: str, columns: list):
     q = f"select={','.join(columns)}&resource_id=eq.{resource_id}&order=created_at.desc"
-    url = f"{config.PG_RST_URL}/tables_index?{q}"
+    url = f"{config.PGREST_ENDPOINT}/tables_index?{q}"
     async with session.get(url) as res:
         record = await res.json()
         if not res.ok:
@@ -18,7 +18,7 @@ async def get_resource(session: ClientSession, resource_id: str, columns: list):
 
 async def get_resource_data(session: ClientSession, resource: dict, sql_query: str):
     headers = {"Prefer": "count=exact"}
-    url = f"{config.PG_RST_URL}/{resource['parsing_table']}?{sql_query}"
+    url = f"{config.PGREST_ENDPOINT}/{resource['parsing_table']}?{sql_query}"
     async with session.get(url, headers=headers) as res:
         if not res.ok:
             handle_exception(
@@ -36,7 +36,7 @@ async def get_resource_data_streamed(
     accept_format: str = "text/csv",
     batch_size: int = config.BATCH_SIZE,
 ):
-    url = f"{config.PG_RST_URL}/{model['parsing_table']}?{sql_query}"
+    url = f"{config.PGREST_ENDPOINT}/{model['parsing_table']}?{sql_query}"
     res = await session.head(f"{url}&limit=1&", headers={"Prefer": "count=exact"})
     total = process_total(res.headers.get("Content-Range"))
     for i in range(0, total, batch_size):

tests/conftest.py

@@ -9,7 +9,7 @@
 from api_tabular import config
 from api_tabular.app import app_factory
 
-PG_RST_URL = "https://example.com"
+PGREST_ENDPOINT = "https://example.com"
 RESOURCE_ID = "60963939-6ada-46bc-9a29-b288b16d969b"
 DATE = "2023-01-01T00:00:00.000000+00:00"
 TABLES_INDEX_PATTERN = re.compile(
@@ -19,7 +19,7 @@
 
 @pytest.fixture(autouse=True)
 def setup():
-    config.override(PG_RST_URL=PG_RST_URL)
+    config.override(PGREST_ENDPOINT=PGREST_ENDPOINT)
 
 
 @pytest.fixture(autouse=True)

tests/test_api.py

@@ -2,7 +2,7 @@
 
 from api_tabular.utils import external_url
 
-from .conftest import RESOURCE_ID, DATE, PG_RST_URL, TABLES_INDEX_PATTERN
+from .conftest import RESOURCE_ID, DATE, PGREST_ENDPOINT, TABLES_INDEX_PATTERN
 
 pytestmark = pytest.mark.asyncio
 
@@ -57,7 +57,7 @@ async def test_api_resource_profile_not_found(client, mock_get_resource_empty):
 async def test_api_resource_data(client, rmock):
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?limit=20&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?limit=20&order=__id.asc",
         payload={"such": "data"},
         headers={"Content-Range": "0-10/10"},
     )
@@ -79,7 +79,7 @@ async def test_api_resource_data_with_args(client, rmock):
     args = "page=1"
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?limit=20&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?limit=20&order=__id.asc",
         payload={"such": "data"},
         headers={"Content-Range": "0-10/10"},
     )
@@ -101,7 +101,7 @@ async def test_api_resource_data_with_args_case(client, rmock):
     args = "COLUM_NAME__EXACT=BIDULE&page=1"
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?COLUM_NAME=eq.BIDULE&limit=20&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?COLUM_NAME=eq.BIDULE&limit=20&order=__id.asc",
         payload={"such": "data"},
         headers={"Content-Range": "0-10/10"},
     )
@@ -158,7 +158,7 @@ async def test_api_resource_data_not_found(client, mock_get_resource_empty):
 
 async def test_api_resource_data_table_error(client, rmock):
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
-    rmock.get(f"{PG_RST_URL}/xxx?limit=20&order=__id.asc", status=502, payload={"such": "error"})
+    rmock.get(f"{PGREST_ENDPOINT}/xxx?limit=20&order=__id.asc", status=502, payload={"such": "error"})
     res = await client.get(f"/api/resources/{RESOURCE_ID}/data/")
     assert res.status == 502
     assert await res.json() == {
@@ -171,7 +171,7 @@ async def test_api_resource_data_table_error(client, rmock):
 async def test_api_percent_encoding_arabic(client, rmock):
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?%D9%85%D9%88%D8%A7%D8%B1%D8%AF=eq.%D9%85%D9%88%D8%A7%D8%B1%D8%AF&limit=20&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?%D9%85%D9%88%D8%A7%D8%B1%D8%AF=eq.%D9%85%D9%88%D8%A7%D8%B1%D8%AF&limit=20&order=__id.asc",  # noqa
         status=200,
         payload={"such": "data"},
         headers={"Content-Range": "0-10/10"},
@@ -195,7 +195,7 @@ async def test_api_percent_encoding_arabic(client, rmock):
 async def test_api_with_unsupported_args(client, rmock):
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?limit=20&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?limit=20&order=__id.asc",
         status=200,
         payload={"such": "data"},
         headers={"Content-Range": "0-10/10"},
@@ -217,7 +217,7 @@ async def test_api_with_unsupported_args(client, rmock):
 async def test_api_pagination(client, rmock):
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?limit=1&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?limit=1&order=__id.asc",
         status=200,
         payload=[{"such": "data"}],
         headers={"Content-Range": "0-2/2"},
@@ -238,7 +238,7 @@ async def test_api_pagination(client, rmock):
 
     rmock.get(TABLES_INDEX_PATTERN, payload=[{"__id": 1, "id": "test-id", "parsing_table": "xxx"}])
     rmock.get(
-        f"{PG_RST_URL}/xxx?limit=1&offset=1&order=__id.asc",
+        f"{PGREST_ENDPOINT}/xxx?limit=1&offset=1&order=__id.asc",
         status=200,
         payload=[{"such": "data"}],
         headers={"Content-Range": "0-2/2"},