From 6b93a651f3ff29e4c5037c3a28f68ccb9050ae7a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 3 Apr 2025 12:21:07 +0000 Subject: [PATCH 1/5] adds pytest-xdist to nox sessions for experimentation --- noxfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/noxfile.py b/noxfile.py index 625b0e39..c01ae285 100644 --- a/noxfile.py +++ b/noxfile.py @@ -48,6 +48,7 @@ "asyncmock", "pytest", "pytest-cov", + "pytest-xdist", "pytest-asyncio", ] UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] @@ -79,6 +80,7 @@ "mock", "pytest", "google-cloud-testutils", + "pytest-xdist", ] SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] From 810f0956c58d62c601fc4e9782ee5e0b38fc7914 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 4 Apr 2025 10:21:44 +0000 Subject: [PATCH 2/5] adds calculate_duration decorator and xdist --- noxfile.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/noxfile.py b/noxfile.py index c01ae285..9b089055 100644 --- a/noxfile.py +++ b/noxfile.py @@ -108,6 +108,26 @@ CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +def _calculate_duration(func): + """This decorator prints the execution time for the decorated function.""" + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.monotonic() + result = func(*args, **kwargs) + end = time.monotonic() + total_seconds = round(end - start) + hours = total_seconds // 3600 # Integer division to get hours + remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds + minutes = remaining_seconds // 60 + seconds = remaining_seconds % 60 + human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}" + print(f"Session ran in {total_seconds} seconds ({human_time})") + return result + + return wrapper + + nox.options.sessions = [ "unit", "system", @@ -126,6 +146,7 @@ @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint(session): """Run linters. @@ -142,6 +163,7 @@ def lint(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) @@ -152,6 +174,7 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def format(session): """ Run isort to sort imports. Then run black @@ -172,6 +195,7 @@ def format(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") @@ -207,6 +231,7 @@ def install_unittest_dependencies(session, *constraints): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@_calculate_duration @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], @@ -239,6 +264,7 @@ def unit(session, protobuf_implementation, install_extras=True): # Run py.test against the unit tests. session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=sqlalchemy_bigquery", @@ -286,6 +312,7 @@ def install_systemtest_dependencies(session, *constraints): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system(session): """Run the system test suite.""" constraints_path = str( @@ -313,6 +340,7 @@ def system(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -321,6 +349,7 @@ def system(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -329,6 +358,7 @@ def system(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system_noextras(session): """Run the system test suite.""" constraints_path = str( @@ -358,6 +388,7 @@ def system_noextras(session): if system_test_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -366,6 +397,7 @@ def system_noextras(session): if system_test_folder_exists: session.run( "py.test", + "-n=auto", "--quiet", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, @@ -374,6 +406,7 @@ def system_noextras(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@_calculate_duration def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( @@ -391,6 +424,7 @@ def compliance(session): session.install( "mock", "pytest", + "pytest-xdist", "pytest-rerunfailures", "google-cloud-testutils", "-c", @@ -408,6 +442,7 @@ def compliance(session): session.run( "py.test", + "-n=auto", "-vv", f"--junitxml=compliance_{session.python}_sponge_log.xml", "--reruns=3", @@ -428,6 +463,7 @@ def compliance(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def cover(session): """Run the final coverage report. @@ -441,6 +477,7 @@ def cover(session): @nox.session(python="3.10") +@_calculate_duration def docs(session): """Build the docs for this library.""" @@ -478,6 +515,7 @@ def docs(session): @nox.session(python="3.10") +@_calculate_duration def docfx(session): """Build the docfx yaml files for this library.""" @@ -526,6 +564,7 @@ def docfx(session): @nox.session(python="3.12") +@_calculate_duration @nox.parametrize( "protobuf_implementation", ["python", "upb", "cpp"], @@ -614,6 +653,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, @@ -625,6 +665,7 @@ def prerelease_deps(session, protobuf_implementation): if os.path.exists(system_test_folder_path): session.run( "py.test", + "-n=auto", "--verbose", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, From 8d8771c487a31c9281568fd62ed44b938ffaeaa3 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 4 Apr 2025 10:28:25 +0000 Subject: [PATCH 3/5] adds import to support calculate_duration --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 9b089055..3c4850b2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,10 +18,10 @@ from __future__ import absolute_import +from functools import wraps import os import pathlib import re -import re import shutil from typing import Dict, List import warnings From 0028b3f2fd33dbcc81b6ca01c23ff0e6050acd8f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 4 Apr 2025 10:33:15 +0000 Subject: [PATCH 4/5] adds import #2 to support calculate_duration --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 3c4850b2..0fb05f86 100644 --- a/noxfile.py +++ b/noxfile.py @@ -23,6 +23,7 @@ import pathlib import re import shutil +import time from typing import Dict, List import warnings From 5e7dc74e1db2d561cc9474e8c0c23df909b67ff7 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 4 Apr 2025 13:31:47 +0000 Subject: [PATCH 5/5] adds two functions to avoid NotImplementedErrors --- noxfile.py | 6 +++--- sqlalchemy_bigquery/base.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 0fb05f86..26b5540b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -443,11 +443,11 @@ def compliance(session): session.run( "py.test", - "-n=auto", + "-n=2", "-vv", f"--junitxml=compliance_{session.python}_sponge_log.xml", - "--reruns=3", - "--reruns-delay=60", + "--reruns=0", + "--reruns-delay=1", "--only-rerun=Exceeded rate limits", "--only-rerun=Already Exists", "--only-rerun=Not found", diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 0204bc92..61c41b2d 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -69,6 +69,25 @@ TABLE_VALUED_ALIAS_ALIASES = "bigquery_table_valued_alias_aliases" +from sqlalchemy.testing.provision import create_db, drop_db +@create_db.for_db("bigquery") +def _bigquery_create_db(cfg, eng, ident): + # This and _bigquery_drop_db are adequate to get past the errors generated when using xdist: + # NotImplementedError: no DB creation routine for cfg + # NotImplementedError: no DB drop routine for cfg + # + # HOWEVER... we need to populate this in some way to create dbs that + # work. THere are examples of create_db style functions in various + # dialects provision.py files. + # This sqlalchemy issue speaks to this problem: + # https://github.com/sqlalchemy/sqlalchemy/discussions/10948 + pass + +@drop_db.for_db("bigquery") +def _bigquery_drop_db(cfg, eng, ident): + pass # _drop_dbs_w_ident(eng.url.database, eng.driver, ident) + # SEE note above for create_db + def assert_(cond, message="Assertion failed"): # pragma: NO COVER if not cond: