diff --git a/packages/sqlalchemy-spanner/create_test_config.py b/packages/sqlalchemy-spanner/create_test_config.py index 3ca1ab9bce88..1917e199a90d 100644 --- a/packages/sqlalchemy-spanner/create_test_config.py +++ b/packages/sqlalchemy-spanner/create_test_config.py @@ -15,6 +15,7 @@ # limitations under the License. import configparser +import os import sys @@ -41,7 +42,8 @@ def set_test_config( config.add_section("db") config["db"]["default"] = url - with open("test.cfg", "w") as configfile: + config_filename = os.getenv("SQLALCHEMY_SPANNER_CONFIG", "test.cfg") + with open(config_filename, "w") as configfile: config.write(configfile) diff --git a/packages/sqlalchemy-spanner/create_test_database.py b/packages/sqlalchemy-spanner/create_test_database.py index 804bd2936ad2..4c004fb05c97 100644 --- a/packages/sqlalchemy-spanner/create_test_database.py +++ b/packages/sqlalchemy-spanner/create_test_database.py @@ -14,18 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os +import pathlib import re import time +import uuid +from create_test_config import set_test_config from google.api_core import datetime_helpers from google.api_core.exceptions import AlreadyExists, ResourceExhausted from google.cloud.spanner_v1 import Client from google.cloud.spanner_v1.database import Database from google.cloud.spanner_v1.instance import Instance -from create_test_config import set_test_config - USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None PROJECT = os.getenv( @@ -69,26 +71,39 @@ def delete_stale_test_instances(): def delete_stale_test_databases(): - """Delete test databases that are older than 10 minutes. - - In this test suite, active databases typically finish running in ~5 minutes. - To prevent concurrent Kokoro runs from accidentally deleting each other's - active databases we use a 10-minute safety threshold. Without an aggressive - cutoff we quickly bump up against Cloud Spanner's limit of 100 databases per instance. + """Delete test databases that are older than 4 hours. + + Uses a .stale_cleanup_done sentinel file gate to ensure this global sweep + runs exactly once at the start of a test run across parallel/parametrized sessions, + preventing concurrent sessions from interfering with each other. """ - cutoff = (int(time.time()) - 10 * 60) * 1000 + marker = ".stale_cleanup_done" + if os.path.exists(marker): + return + + try: + pathlib.Path(marker).touch(exist_ok=False) + except FileExistsError: + return # Another parallel process already performed cleanup + + cutoff = (int(time.time()) - 4 * 60 * 60) * 1000 instance = CLIENT.instance("sqlalchemy-dialect-test") if not instance.exists(): return database_pbs = instance.list_databases() for database_pb in database_pbs: database = Database.from_pb(database_pb, instance) - # Parse creation time from database ID first (e.g. "sqlalchemy-test-1779989493809") - # to be 100% independent of emulator metadata or GCP Client API create_time gaps! + + # Parse creation time from database ID first (e.g. "sp_test_1787069488_a3f") create_time = None - match = re.match(r"sqlalchemy-test-(\d+)", database.database_id) + match = re.match(r"sp_test_(\d+)", database.database_id) if match: - create_time = int(match.group(1)) + ts_str = match.group(1) + ts_val = int(ts_str) + if len(ts_str) == 10: + create_time = ts_val * 1000 + else: + create_time = ts_val elif database_pb.create_time is not None: create_time = datetime_helpers.to_milliseconds(database_pb.create_time) @@ -123,8 +138,12 @@ def create_test_instance(): except AlreadyExists: pass # instance was already created - unique_resource_id = "%s%d" % ("-", 1000 * time.time()) - database_id = "sqlalchemy-test" + unique_resource_id + # Generate a session-isolated unique database ID within Spanner 30-char limit + # Format: sp_test_{timestamp_in_seconds}_{rand_hex3} (compliant with Spanner naming: ^[a-z][a-z0-9_]{1,29}$) + creation_timestamp = time.time() + timestamp_part = str(int(creation_timestamp)) + rand_part = uuid.uuid4().hex[:8] + database_id = f"sp_test_{timestamp_part}_{rand_part}" try: database = instance.database(database_id) @@ -135,6 +154,44 @@ def create_test_instance(): set_test_config(PROJECT, instance_id, database_id) + # Record metadata for duration tracking on teardown + meta_path = os.path.join(os.path.dirname(__file__), ".db_session_info.json") + with open(meta_path, "w") as f: + json.dump({"database_id": database_id, "creation_time": creation_timestamp}, f) + + +def main(argv): + config_filename = argv[0] if argv else os.getenv("SQLALCHEMY_SPANNER_CONFIG", "test.cfg") + os.environ["SQLALCHEMY_SPANNER_CONFIG"] = config_filename + + delete_stale_test_databases() + create_test_instance() + + instance_id = "sqlalchemy-dialect-test" + instance = CLIENT.instance(instance_id) + + # Generate a session-isolated unique database ID within Spanner 30-char limit + # Format: sp_test_{timestamp_in_seconds}_{rand_hex8} (compliant with Spanner naming: ^[a-z][a-z0-9_]{1,29}$) + creation_timestamp = time.time() + timestamp_part = str(int(creation_timestamp)) + rand_part = uuid.uuid4().hex[:8] + database_id = f"sp_test_{timestamp_part}_{rand_part}" + + try: + database = instance.database(database_id) + created_op = database.create() + created_op.result(1800) + except AlreadyExists: + pass # database was already created + + set_test_config(PROJECT, instance_id, database_id) + + # Record metadata for duration tracking on teardown + meta_path = os.path.join(os.path.dirname(__file__), ".db_session_info.json") + with open(meta_path, "w") as f: + json.dump({"database_id": database_id, "creation_time": creation_timestamp}, f) + -delete_stale_test_databases() -create_test_instance() +if __name__ == "__main__": + import sys + main(sys.argv[1:]) diff --git a/packages/sqlalchemy-spanner/drop_test_database.py b/packages/sqlalchemy-spanner/drop_test_database.py index b7facf1f9ea1..c9cee6b06635 100644 --- a/packages/sqlalchemy-spanner/drop_test_database.py +++ b/packages/sqlalchemy-spanner/drop_test_database.py @@ -15,18 +15,18 @@ # limitations under the License. import configparser +import json import os import re import time +from create_test_config import set_test_config from google.api_core import datetime_helpers from google.api_core.exceptions import AlreadyExists, ResourceExhausted from google.cloud.spanner_v1 import Client from google.cloud.spanner_v1.database import Database from google.cloud.spanner_v1.instance import Instance -from create_test_config import set_test_config - USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None PROJECT = os.getenv( @@ -43,21 +43,76 @@ CLIENT = Client(project=PROJECT) +def format_duration(seconds): + mins = int(seconds // 60) + secs = int(seconds % 60) + if mins > 0: + return f"{mins} minutes and {secs} seconds" + else: + return f"{secs} seconds" + + def delete_test_database(): """Delete the currently configured test database.""" config = configparser.ConfigParser() - if os.path.exists("test.cfg"): - config.read("test.cfg") + config_env_val = os.getenv("SQLALCHEMY_SPANNER_CONFIG") + if config_env_val: + config_filename = config_env_val + if not os.path.exists(config_filename): + print(f"[Spanner DB] Config file {config_filename} specified in SQLALCHEMY_SPANNER_CONFIG does not exist. Skipping database drop.") + return + elif os.path.exists("test.cfg"): + config_filename = "test.cfg" else: - config.read("setup.cfg") + config_filename = "setup.cfg" + + config.read(config_filename) + db_url = config.get("db", "default") + if not db_url.startswith("spanner"): + print(f"[Spanner DB] Database URL {db_url} is not a Spanner URL. Skipping database drop.") + return instance_id = re.findall(r"instances(.*?)databases", db_url) database_id = re.findall(r"databases(.*?)$", db_url) instance = CLIENT.instance(instance_id="".join(instance_id).replace("/", "")) - database = instance.database("".join(database_id).replace("/", "")) + database_id_str = "".join(database_id).replace("/", "") + database = instance.database(database_id_str) database.drop() + # Calculate and report active duration with type-validation for compliance + meta_path = os.path.join(os.path.dirname(__file__), ".db_session_info.json") + if os.path.exists(meta_path): + try: + with open(meta_path, "r") as f: + meta = json.load(f) + if isinstance(meta, dict): + creation_time = meta.get("creation_time", time.time()) + db_name = meta.get("database_id", database_id_str) + elapsed_seconds = time.time() - creation_time + duration_str = format_duration(elapsed_seconds) + print(f"[Spanner DB] Database {db_name} was active for {duration_str} before teardown.") + except Exception: + pass + finally: + if os.path.exists(meta_path): + os.remove(meta_path) + + # Clean up session-specific config file + if os.path.exists(config_filename) and config_filename != "setup.cfg": + try: + os.remove(config_filename) + except Exception: + pass + + +def main(argv): + config_filename = argv[0] if argv else os.getenv("SQLALCHEMY_SPANNER_CONFIG", "test.cfg") + os.environ["SQLALCHEMY_SPANNER_CONFIG"] = config_filename + delete_test_database() + -delete_test_database() +if __name__ == "__main__": + import sys + main(sys.argv[1:]) diff --git a/packages/sqlalchemy-spanner/migration_test_cleanup.py b/packages/sqlalchemy-spanner/migration_test_cleanup.py deleted file mode 100644 index b6efb15910e0..000000000000 --- a/packages/sqlalchemy-spanner/migration_test_cleanup.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import sys - -from google.cloud import spanner - - -def main(argv): - db_url = argv[0] - - project = re.findall(r"projects(.*?)instances", db_url) - instance_id = re.findall(r"instances(.*?)databases", db_url) - database_id = re.findall(r"databases(.*?)$", db_url) - - client = spanner.Client(project="".join(project).replace("/", "")) - instance = client.instance(instance_id="".join(instance_id).replace("/", "")) - database = instance.database("".join(database_id).replace("/", "")) - - database.update_ddl( - ["DROP TABLE IF EXISTS account", "DROP TABLE IF EXISTS alembic_version"] - ).result(120) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index a69e9bb2b89b..a09b96f44052 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -21,6 +21,7 @@ import pathlib import re import shutil +import uuid import nox @@ -179,6 +180,10 @@ def lint_setup_py(session): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS[0]) def compliance_test_14(session): """Run SQLAlchemy dialect compliance test suite.""" + config_file = os.environ.setdefault( + "SQLALCHEMY_SPANNER_CONFIG", + f"test_compliance_14_{session.python}_{uuid.uuid4().hex[:6]}.cfg", + ) # Check the value of `RUN_COMPLIANCE_TESTS` env var. It defaults to true. if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": @@ -191,34 +196,52 @@ def compliance_test_14(session): "Credentials or emulator host must be set via environment variable" ) - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) - session.install(".[tracing]") - session.run( - "pip", - "install", - *SQLALCHEMY_14_DEPENDENCIES, - "--force-reinstall", - ) - session.run("python", "create_test_database.py") - session.run( - "py.test", - "--cov=google.cloud.sqlalchemy_spanner", - "--cov=tests", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - "--asyncio-mode=auto", - "tests/test_suite_14.py", - *session.posargs, - # Silence SQLAlchemy 2.0 transition warnings for this 1.4 compatibility session. - env={"SQLALCHEMY_SILENCE_UBER_WARNING": "1"}, - ) + try: + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + session.install(".[tracing]") + session.run( + "pip", + "install", + *SQLALCHEMY_14_DEPENDENCIES, + "--force-reinstall", + ) + session.run("python", "create_test_database.py", config_file) + config = configparser.ConfigParser() + config.read(config_file) + db_url = config.get("db", "default") + session.run( + "py.test", + f"--dburi={db_url}", + "--cov=google.cloud.sqlalchemy_spanner", + "--cov=tests", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + "--asyncio-mode=auto", + "tests/test_suite_14.py", + *session.posargs, + # Silence SQLAlchemy 2.0 transition warnings for this 1.4 compatibility session. + env={"SQLALCHEMY_SILENCE_UBER_WARNING": "1"}, + ) + finally: + if os.path.exists(config_file): + session.run( + "python", "drop_test_database.py", config_file, success_codes=[0, 1] + ) + elif os.path.exists("test.cfg"): + session.run( + "python", "drop_test_database.py", "test.cfg", success_codes=[0, 1] + ) @nox.session(python=DEFAULT_PYTHON_VERSION_FOR_SQLALCHEMY_20) def compliance_test_20(session): """Run SQLAlchemy dialect compliance test suite.""" + config_file = os.environ.setdefault( + "SQLALCHEMY_SPANNER_CONFIG", + f"test_compliance_20_{session.python}_{uuid.uuid4().hex[:6]}.cfg", + ) # Check the value of `RUN_COMPLIANCE_TESTS` env var. It defaults to true. if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": @@ -232,24 +255,38 @@ def compliance_test_20(session): "Credentials or emulator host must be set via environment variable" ) - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) - session.install("-e", ".", "--force-reinstall") - session.run("python", "create_test_database.py") - - session.install(*SQLALCHEMY_20_DEPENDENCIES) - - session.run( - "py.test", - "--cov=google.cloud.sqlalchemy_spanner", - "--cov=tests", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - "--asyncio-mode=auto", - "tests/test_suite_20.py", - *session.posargs, - ) + try: + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + session.install("-e", ".", "--force-reinstall") + session.run("python", "create_test_database.py", config_file) + config = configparser.ConfigParser() + config.read(config_file) + db_url = config.get("db", "default") + + session.install(*SQLALCHEMY_20_DEPENDENCIES) + + session.run( + "py.test", + f"--dburi={db_url}", + "--cov=google.cloud.sqlalchemy_spanner", + "--cov=tests", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + "--asyncio-mode=auto", + "tests/test_suite_20.py", + *session.posargs, + ) + finally: + if os.path.exists(config_file): + session.run( + "python", "drop_test_database.py", config_file, success_codes=[0, 1] + ) + elif os.path.exists("test.cfg"): + session.run( + "python", "drop_test_database.py", "test.cfg", success_codes=[0, 1] + ) @nox.session(python=DEFAULT_PYTHON_VERSION_FOR_SQLALCHEMY_20) @@ -300,50 +337,63 @@ def _migration_test(session): import os import shutil + config_file = os.environ.setdefault( + "SQLALCHEMY_SPANNER_CONFIG", + f"test_migration_{session.python}_{uuid.uuid4().hex[:6]}.cfg", + ) + session.install(*MIGRATION_TEST_DEPENDENCIES) session.install(".") - session.run("python", "create_test_database.py") + try: + session.run("python", "create_test_database.py", config_file) + + config = configparser.ConfigParser() + if os.path.exists(config_file): + config.read(config_file) + else: + config.read("setup.cfg") + db_url = config.get("db", "default") - config = configparser.ConfigParser() - if os.path.exists("test.cfg"): - config.read("test.cfg") - else: - config.read("setup.cfg") - db_url = config.get("db", "default") + session.run("alembic", "init", "test_migration") - session.run("alembic", "init", "test_migration") + # setting testing configurations + if os.path.exists("alembic.ini"): + os.remove("alembic.ini") + with open("alembic.ini", "w") as f: + f.write(ALEMBIC_CONF.format(db_url)) - # setting testing configurations - os.remove("alembic.ini") - with open("alembic.ini", "w") as f: - f.write(ALEMBIC_CONF.format(db_url)) + session.run("alembic", "revision", "-m", "migration_for_test") + files = glob.glob("test_migration/versions/*.py") - session.run("alembic", "revision", "-m", "migration_for_test") - files = glob.glob("test_migration/versions/*.py") + # updating the upgrade-script code + with open(files[0], "rb") as f: + script_code = f.read().decode() - # updating the upgrade-script code - with open(files[0], "rb") as f: - script_code = f.read().decode() + script_code = script_code.replace( + """def upgrade() -> None:\n pass""", UPGRADE_CODE + ) + with open(files[0], "wb") as f: + f.write(script_code.encode()) - script_code = script_code.replace( - """def upgrade() -> None:\n pass""", UPGRADE_CODE - ) - with open(files[0], "wb") as f: - f.write(script_code.encode()) + if os.path.exists("test_migration/env.py"): + os.remove("test_migration/env.py") + shutil.copyfile("test_migration_env.py", "test_migration/env.py") - os.remove("test_migration/env.py") - shutil.copyfile("test_migration_env.py", "test_migration/env.py") + # running the test migration + session.run("alembic", "upgrade", "head") - # running the test migration - session.run("alembic", "upgrade", "head") + finally: + # clearing the migration data + if os.path.exists("alembic.ini"): + os.remove("alembic.ini") + if os.path.exists("test_migration"): + shutil.rmtree("test_migration") - # clearing the migration data - os.remove("alembic.ini") - shutil.rmtree("test_migration") - session.run("python", "migration_test_cleanup.py", db_url) - if os.path.exists("test.cfg"): - os.remove("test.cfg") + if os.path.exists(config_file): + session.run( + "python", "drop_test_database.py", config_file, success_codes=[0, 1] + ) @nox.session(python=ALL_PYTHON) @@ -389,6 +439,11 @@ def unit(session, test_type): def system(session, test_type): """Run SQLAlchemy dialect system test suite.""" + config_file = os.environ.setdefault( + "SQLALCHEMY_SPANNER_CONFIG", + f"test_{test_type}_{session.python}_{uuid.uuid4().hex[:6]}.cfg", + ) + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") and not os.environ.get( "SPANNER_EMULATOR_HOST", "" ): @@ -423,10 +478,17 @@ def system(session, test_type): session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) session.install(".[tracing]") session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES) - session.run("python", "create_test_database.py") + session.run("python", "create_test_database.py", config_file) + config = configparser.ConfigParser() + config.read(config_file) + db_url = config.get("db", "default") session.install(*SQLALCHEMY_20_DEPENDENCIES) session.run( - "py.test", "--quiet", os.path.join("tests", "system"), *session.posargs + "py.test", + f"--dburi={db_url}", + "--quiet", + os.path.join("tests", "system"), + *session.posargs, ) elif test_type == "compliance_14": compliance_test_14(session) @@ -437,8 +499,14 @@ def system(session, test_type): elif test_type == "migration_20": _migration_test(session) finally: - if os.path.exists("test.cfg"): - session.run("python", "drop_test_database.py", success_codes=[0, 1]) + if os.path.exists(config_file): + session.run( + "python", "drop_test_database.py", config_file, success_codes=[0, 1] + ) + elif os.path.exists("test.cfg"): + session.run( + "python", "drop_test_database.py", "test.cfg", success_codes=[0, 1] + ) @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/packages/sqlalchemy-spanner/tests/_helpers.py b/packages/sqlalchemy-spanner/tests/_helpers.py index 40673b2607cb..8feea107acc0 100644 --- a/packages/sqlalchemy-spanner/tests/_helpers.py +++ b/packages/sqlalchemy-spanner/tests/_helpers.py @@ -44,9 +44,12 @@ def get_db_url(): config = configparser.ConfigParser() - if os.path.exists("test.cfg"): + config_filename = os.getenv("SQLALCHEMY_SPANNER_CONFIG", "test.cfg") + if os.path.exists(config_filename): + config.read(config_filename) + elif os.path.exists("test.cfg"): config.read("test.cfg") - else: + elif os.path.exists("setup.cfg"): config.read("setup.cfg") return config.get("db", "default", fallback=DB_URL) diff --git a/packages/sqlalchemy-spanner/tests/conftest.py b/packages/sqlalchemy-spanner/tests/conftest.py index b949e1a4c47e..ec16ad532dde 100644 --- a/packages/sqlalchemy-spanner/tests/conftest.py +++ b/packages/sqlalchemy-spanner/tests/conftest.py @@ -75,7 +75,7 @@ def run( t.create(connection) for value in input_: - ins = t.insert().values(x=literal(value, type_, literal_execute=True)) + ins = t.insert().values(x=literal(value, type_)) connection.execute(ins) if support_whereclause: @@ -85,13 +85,11 @@ def run( == literal( compare, type_, - literal_execute=True, ), t.c.x == literal( input_[0], type_, - literal_execute=True, ), ) else: @@ -100,7 +98,6 @@ def run( == literal( compare if compare is not None else input_[0], type_, - literal_execute=True, ) ) else: diff --git a/packages/sqlalchemy-spanner/tests/test_suite_14.py b/packages/sqlalchemy-spanner/tests/test_suite_14.py index 6494c414069a..9643f898e540 100644 --- a/packages/sqlalchemy-spanner/tests/test_suite_14.py +++ b/packages/sqlalchemy-spanner/tests/test_suite_14.py @@ -1576,7 +1576,7 @@ def run(type_, input_, output, filter_=None, check_scale=False): return run @emits_warning(r".*does \*not\* support Decimal objects natively") - def test_render_literal_numeric(self, literal_round_trip): + def test_render_literal_numeric(self, literal_round_trip_spanner): """ SPANNER OVERRIDE: @@ -1585,14 +1585,14 @@ def test_render_literal_numeric(self, literal_round_trip): following insertions will fail with `Row [] already exists". Overriding the test to avoid the same failure. """ - literal_round_trip( + literal_round_trip_spanner( Numeric(precision=8, scale=4), [decimal.Decimal("15.7563")], [decimal.Decimal("15.7563")], ) @emits_warning(r".*does \*not\* support Decimal objects natively") - def test_render_literal_numeric_asfloat(self, literal_round_trip): + def test_render_literal_numeric_asfloat(self, literal_round_trip_spanner): """ SPANNER OVERRIDE: @@ -1601,13 +1601,13 @@ def test_render_literal_numeric_asfloat(self, literal_round_trip): following insertions will fail with `Row [] already exists". Overriding the test to avoid the same failure. """ - literal_round_trip( + literal_round_trip_spanner( Numeric(precision=8, scale=4, asdecimal=False), [decimal.Decimal("15.7563")], [15.7563], ) - def test_render_literal_float(self, literal_round_trip): + def test_render_literal_float(self, literal_round_trip_spanner): """ SPANNER OVERRIDE: @@ -1616,7 +1616,7 @@ def test_render_literal_float(self, literal_round_trip): following insertions will fail with `Row [] already exists". Overriding the test to avoid the same failure. """ - literal_round_trip( + literal_round_trip_spanner( Float(4), [decimal.Decimal("15.7563")], [15.7563],