From cb0d97d60378d66e9291ed71860a641d58ff8e4d Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 12 Nov 2026 16:31:50 +0100 Subject: [PATCH 1/5] fix(db): warn only about expression indexes a SQLite revision missed; app-appropriate advice (#917) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- docs/module-authoring.md | 16 +++-- docs/reference/diagnostic-codes.md | 2 +- .../diagnostics/_expression_index.py | 31 ++++++-- .../simple_module_db/migration_portability.py | 41 +++++++---- framework/db/simple_module_db/migrations.py | 10 +-- .../test_db_expression_index_warning_scope.py | 71 +++++++++++++++++++ .../db/tests/test_migration_portability.py | 9 ++- .../simple_module_hosting/migrations.py | 8 ++- 8 files changed, 154 insertions(+), 34 deletions(-) create mode 100644 framework/db/tests/test_db_expression_index_warning_scope.py diff --git a/docs/module-authoring.md b/docs/module-authoring.md index ae70c03bb..3540b4868 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -362,13 +362,17 @@ generated file against this checklist before treating it as portable scaffolded `env.py`) rewrite these for you; check any revision written without them, and always write `server_default=sa.false()` on the model. 2. **Expression indexes.** SQLite cannot reflect `Index(..., text("lower(email)"))`, - so autogenerate skips it and `alembic check` cannot see it. Autogenerate on - SQLite logs a banner naming every expression index in your models, and - `make doctor` reports `SM026`. Add each one to a revision by hand + so autogenerate skips it and `alembic check` cannot see it. The revision hook + re-emits the expression indexes of every table a revision creates, so an + initial schema needs nothing. When a revision changes an existing table that + has one, autogenerate on SQLite logs a banner naming it, and `make doctor` + reports `SM026` for every expression index. If the change affects the index, + add it to the revision by hand (`op.create_index(..., [sa.text("lower(email)")], if_not_exists=True)`) and - confirm with the PostgreSQL round trip below. The warning is deliberately not - an error: it fires on every autogenerate, including unrelated revisions, and - the Postgres round trip is the actual gate. Prefer writing an ordered index column as + confirm with the PostgreSQL round trip below; in an app without that make + target, run `alembic upgrade heads` with `SM_DATABASE_URL` pointing at a + PostgreSQL database. The warning is deliberately not an error: SQLite can + neither confirm nor refute the drift, and PostgreSQL is the actual gate. Prefer writing an ordered index column as `sa.desc(sa.column("col"))` over `sa.text("col DESC")` or `sa.desc("col")`: the latter two are expressions to Alembic, and while the host's include filter suppresses the false SQLite change when only the ordering differs, the column diff --git a/docs/reference/diagnostic-codes.md b/docs/reference/diagnostic-codes.md index c32e9b22c..884ec011c 100644 --- a/docs/reference/diagnostic-codes.md +++ b/docs/reference/diagnostic-codes.md @@ -34,7 +34,7 @@ The framework runs a set of static checks over installed modules at app boot. Th | `SM023` | WARNING | A module's `theme.css` contains an unlayered plain rule (anything but an at-rule or a `:root`-style selector). Unlayered CSS outranks every Tailwind utility. | Move the rule to the module's `styles.css`, which is imported into `layer(components)` so utilities still win. | | `SM024` | WARNING | A unique column, constraint or index on a `MultiTenantMixin` table does not include `tenant_id`, so the first tenant to claim a value locks every other tenant out of it. | Make the key per tenant: add `tenant_id` to it (`Index(..., "tenant_id", "slug", unique=True)`). | | `SM025` | WARNING | `multi_tenant` is on but no module registered `app.state.tenant_resolver`, so only the principal's `tenant_id` claim can bind a tenant — with most auth providers every tenant-scoped query then fails closed. Checked at boot (it needs the built app), in every environment, not by `make doctor`. | Install the `tenants` module, or register your own `async (Request) -> str \| None` resolver on `app.state.tenant_resolver`. | -| `SM026` | INFO | The configured database is SQLite and a module model declares an expression index (e.g. `lower(email)`). SQLite cannot reflect those, so autogenerate and `alembic check` cannot verify them and a clean diff says nothing about them. | Add the index to a revision by hand and verify with `make migrations-roundtrip-pg` (PostgreSQL). See [Migrations](/module-authoring#reviewing-a-revision-autogenerated-on-sqlite). | +| `SM026` | INFO | The configured database is SQLite and a module model declares an expression index (e.g. `lower(email)`). SQLite cannot reflect those, so autogenerate and `alembic check` cannot verify them and a clean diff says nothing about them. | Add the index to a revision by hand and verify on PostgreSQL: `make migrations-roundtrip-pg` in this repository, `alembic upgrade heads` against a PostgreSQL `SM_DATABASE_URL` in an app. See [Migrations](/module-authoring#reviewing-a-revision-autogenerated-on-sqlite). | | `SM027` | WARNING | A key in `host/locales/overrides/.json` matches no catalog key, so the override was skipped. Reported by `make doctor` and at dev boot. | Fix the key: overrides replace an existing key by its full dotted path (e.g. `users.login.aside_heading`). See [i18n](/framework/i18n#overriding-module-copy). | | `SM028` | WARNING | A module's `tasks.py` imports `torch`, `tensorflow`, `ultralytics`, `osgeo`, `rasterio`, `cv2` or `sklearn` at module level, so every process that loads the module's tasks (web, each worker, beat) pays for it and a queue-restricted worker cannot avoid it. | Import the package inside the task function. See [background_tasks](/modules/background_tasks#running-a-worker-for-a-subset-of-modules). | | `SM029` | INFO | A module overrides `db_extensions()` (Postgres extensions such as `ltree`) and the configured database is SQLite, which has none. `ensure_db_extensions` raises `UnsupportedDatabaseError` when migrations run. | Point `SM_DATABASE_URL` at PostgreSQL. See [Postgres extensions and column types](/database/extensions). | diff --git a/framework/core/simple_module_core/diagnostics/_expression_index.py b/framework/core/simple_module_core/diagnostics/_expression_index.py index 07afe434e..1e43e0c48 100644 --- a/framework/core/simple_module_core/diagnostics/_expression_index.py +++ b/framework/core/simple_module_core/diagnostics/_expression_index.py @@ -15,6 +15,7 @@ import importlib import logging from collections.abc import Iterable +from pathlib import Path from typing import TYPE_CHECKING, Any from simple_module_core.diagnostics._types import Diagnostic, DiagnosticLevel @@ -94,6 +95,30 @@ def module_all_tables(mod: ModuleBase) -> list[Any]: return list(seen.values()) +_ROUNDTRIP_TARGET = "migrations-roundtrip-pg:" +_DOCS_URL = "https://docs.py.simplemodule.dev/module-authoring" + + +def expression_index_advice(root: Path | None = None) -> str: + """How to verify expression indexes, in terms of what this project has. + + The framework repository has a Postgres round-trip make target; an app made + by ``smpy new`` does not, so pointing it there sends users nowhere (GH #917). + ``root`` defaults to the working directory, where make, alembic and doctor run. + """ + makefile = (root or Path.cwd()) / "Makefile" + try: + has_target = _ROUNDTRIP_TARGET in makefile.read_text(encoding="utf-8") + except OSError: + has_target = False + if has_target: + return "verify them on PostgreSQL with `make migrations-roundtrip-pg`" + return ( + "verify them on PostgreSQL: run `alembic upgrade heads` with `SM_DATABASE_URL` " + f"pointing at a PostgreSQL database. See {_DOCS_URL}" + ) + + def check_expression_indexes_unverifiable( tables: Iterable[Any], dialect: str | None, module_name: str ) -> list[Diagnostic]: @@ -114,16 +139,14 @@ def check_expression_indexes_unverifiable( f"about them" ), module_name=module_name, - suggestion=( - "Add them to a revision by hand and verify on Postgres: " - "`make migrations-roundtrip-pg`" - ), + suggestion=f"Add them to a revision by hand and {expression_index_advice()}", ) ] __all__ = [ "check_expression_indexes_unverifiable", + "expression_index_advice", "find_expression_indexes", "index_is_expression_based", "module_all_tables", diff --git a/framework/db/simple_module_db/migration_portability.py b/framework/db/simple_module_db/migration_portability.py index b7e157f35..8e2f5966b 100644 --- a/framework/db/simple_module_db/migration_portability.py +++ b/framework/db/simple_module_db/migration_portability.py @@ -7,7 +7,8 @@ integer default on a boolean column. :func:`rewrite_boolean_defaults` and :func:`render_boolean_default` emit ``sa.false()`` / ``sa.true()`` instead. * Expression indexes (``lower(email)``) are skipped on SQLite because it cannot - reflect them. :func:`warn_unverifiable_expression_indexes` says so loudly. + reflect them. :func:`warn_unverifiable_expression_indexes` names the ones a + revision may have missed. * ``op.drop_table`` leaves ``CREATE TYPE`` behind on Postgres, so a downgrade followed by an upgrade fails with ``DuplicateObject``. Use :func:`drop_enums_if_postgres` in the revision's ``downgrade``. @@ -24,7 +25,10 @@ import sqlalchemy as sa from alembic.operations.ops import AddColumnOp, AlterColumnOp, CreateTableOp -from simple_module_core.diagnostics._expression_index import find_expression_indexes +from simple_module_core.diagnostics._expression_index import ( + expression_index_advice, + find_expression_indexes, +) from sqlalchemy.dialects import postgresql from sqlalchemy.schema import DefaultClause from sqlalchemy.sql.elements import ColumnElement, False_, TextClause, True_ @@ -116,33 +120,44 @@ def rewrite_boolean_defaults(upgrade_ops: Any) -> None: column.server_default = DefaultClause(replacement) -def warn_unverifiable_expression_indexes(dialect_name: str | None, metadata: sa.MetaData) -> list: - """Warn when autogenerating on SQLite while models declare expression indexes. +def warn_unverifiable_expression_indexes( + dialect_name: str | None, metadata: sa.MetaData, upgrade_ops: Any = None +) -> list: + """Warn when a revision autogenerated on SQLite may miss expression indexes. - A warning rather than an error: autogenerate also runs for ordinary, - unrelated revisions, and failing every one of them for an index that is - already in history would block normal work. The warning is emitted on every - run, names each index, and ``alembic check`` on Postgres - (``make migrations-roundtrip-pg``) is the real gate. Returns the + With ``upgrade_ops`` (what the revision hook passes, after re-emitting the + indexes of new tables), only indexes on tables the revision touches and + does not create are named: a fresh initial schema creates them, and a + revision on another table cannot have dropped them (GH #917). Without it, + every expression index in ``metadata`` is named. + + A warning rather than an error: SQLite can neither confirm nor refute the + drift, and a PostgreSQL run is the real gate. Returns the ``(table, index)`` pairs it warned about. """ if dialect_name != "sqlite": return [] found = find_expression_indexes(metadata.tables.values()) + if upgrade_ops is not None: + ops = list(iter_ops_recursive(upgrade_ops)) + touched = {getattr(op, "table_name", None) for op in ops} + emitted = {getattr(op, "index_name", None) for op in ops} + found = [(t, i) for t, i in found if t in touched and i not in emitted] if not found: return [] listing = "\n".join(f" - {index} on {table}" for table, index in found) logger.warning( "\n%s\n" "WARNING: autogenerating on SQLite. SQLite cannot reflect expression indexes,\n" - "so this revision does NOT account for the following (they are neither\n" - "created nor verified here) and `alembic check` will not notice drift:\n" + "so this revision does NOT account for the following on the tables it\n" + "changes, and `alembic check` will not notice drift:\n" "%s\n" - "Add them to the revision by hand and verify on PostgreSQL before merging\n" - "(`make migrations-roundtrip-pg`). See docs/module-authoring.md, Migrations.\n" + "If the change affects them, add them to the revision by hand and\n" + "%s.\n" "%s", "=" * 78, listing, + expression_index_advice(), "=" * 78, ) return found diff --git a/framework/db/simple_module_db/migrations.py b/framework/db/simple_module_db/migrations.py index 711fc1d4c..0cc9e8ffa 100644 --- a/framework/db/simple_module_db/migrations.py +++ b/framework/db/simple_module_db/migrations.py @@ -195,8 +195,8 @@ def make_process_revision_directives( alone, which keeps this dialect-agnostic rather than special-casing SQLite. It also rewrites ``0``/``1``/``false``/``true`` server defaults on Boolean - columns to ``sa.false()``/``sa.true()`` and, when running on SQLite, warns - that expression indexes cannot be verified (see + columns to ``sa.false()``/``sa.true()`` and, when running on SQLite, names + the expression indexes on changed tables it could not verify (see :mod:`simple_module_db.migration_portability`). Call as:: @@ -213,15 +213,15 @@ def make_process_revision_directives( expression_indexes.setdefault(table.name, []).append(index) def process_revision_directives(context, revision, directives): - warn_unverifiable_expression_indexes( - getattr(getattr(context, "dialect", None), "name", None), metadata - ) + dialect_name = getattr(getattr(context, "dialect", None), "name", None) for script in directives: upgrade_ops = getattr(script, "upgrade_ops", None) if upgrade_ops is not None: rewrite_boolean_defaults(upgrade_ops) if expression_indexes: _inject_create_index_after_create_table(upgrade_ops, expression_indexes) + # After the re-emission, so an index this revision creates is not reported. + warn_unverifiable_expression_indexes(dialect_name, metadata, upgrade_ops) downgrade_ops = getattr(script, "downgrade_ops", None) if downgrade_ops is not None: # A downgrade re-creates dropped tables/columns from reflected diff --git a/framework/db/tests/test_db_expression_index_warning_scope.py b/framework/db/tests/test_db_expression_index_warning_scope.py new file mode 100644 index 000000000..1778f13d2 --- /dev/null +++ b/framework/db/tests/test_db_expression_index_warning_scope.py @@ -0,0 +1,71 @@ +"""The SQLite expression-index warning names only what the revision missed (GH #917). + +The hook used to warn about every expression index in the metadata, before it +had re-emitted them, so a fresh app's initial schema (which does create the +index) and every unrelated module migration told users to hand-edit it in. +""" + +from __future__ import annotations + +import logging + +import sqlalchemy as sa +from alembic.autogenerate import produce_migrations +from alembic.runtime.migration import MigrationContext +from simple_module_core.diagnostics._expression_index import expression_index_advice +from simple_module_db.migrations import make_process_revision_directives, render_item + + +def _metadata(*, extra_column: bool = False) -> sa.MetaData: + md = sa.MetaData() + sa.Table( + "people", + md, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("email", sa.String), + *([sa.Column("nickname", sa.String)] if extra_column else []), + sa.Index("ix_people_email_lower", sa.text("lower(email)"), unique=True), + ) + sa.Table("orders", md, sa.Column("id", sa.Integer, primary_key=True)) + return md + + +def _run_hook(existing: sa.MetaData | None, target: sa.MetaData, caplog) -> str: + engine = sa.create_engine("sqlite://") + with engine.connect() as conn: + if existing is not None: + existing.create_all(conn) + ctx = MigrationContext.configure(conn, opts={"render_item": render_item}) + script = produce_migrations(ctx, target) + with caplog.at_level(logging.WARNING, logger="alembic.env"): + make_process_revision_directives(target)(ctx, None, [script]) + return caplog.text + + +def test_initial_schema_that_creates_the_index_is_silent(caplog) -> None: + assert "ix_people_email_lower" not in _run_hook(None, _metadata(), caplog) + + +def test_revision_on_another_table_is_silent(caplog) -> None: + existing = _metadata() + existing.remove(existing.tables["orders"]) + assert "ix_people_email_lower" not in _run_hook(existing, _metadata(), caplog) + + +def test_revision_altering_the_indexed_table_warns(caplog) -> None: + text = _run_hook(_metadata(), _metadata(extra_column=True), caplog) + assert "ix_people_email_lower on people" in text + + +def test_advice_outside_this_repo_points_at_what_exists(tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + advice = expression_index_advice() + assert "migrations-roundtrip-pg" not in advice + assert "alembic upgrade heads" in advice + assert "PostgreSQL" in advice + + +def test_advice_in_a_checkout_with_the_roundtrip_target(tmp_path, monkeypatch) -> None: + (tmp_path / "Makefile").write_text("migrations-roundtrip-pg:\n\techo\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert "make migrations-roundtrip-pg" in expression_index_advice() diff --git a/framework/db/tests/test_migration_portability.py b/framework/db/tests/test_migration_portability.py index a311587ba..374a4b506 100644 --- a/framework/db/tests/test_migration_portability.py +++ b/framework/db/tests/test_migration_portability.py @@ -103,10 +103,13 @@ def test_expression_index_warning_silent_elsewhere(caplog): assert caplog.text == "" -def test_hook_warns_during_autogenerate_on_sqlite(caplog): +def test_hook_is_silent_when_the_revision_creates_the_index(caplog): + """The re-emitted index is in the revision (GH #917); scope tests in + test_db_expression_index_warning_scope.py.""" with caplog.at_level(logging.WARNING, logger="alembic.env"): - _autogenerate(_metadata(with_index=True)) - assert "ix_flags_email_lower" in caplog.text + code, _, _ = _autogenerate(_metadata(with_index=True)) + assert "ix_flags_email_lower" in code + assert "ix_flags_email_lower" not in caplog.text def test_plain_column_index_is_not_flagged(): diff --git a/framework/hosting/simple_module_hosting/migrations.py b/framework/hosting/simple_module_hosting/migrations.py index 8baab1a29..cd1b3e7a2 100644 --- a/framework/hosting/simple_module_hosting/migrations.py +++ b/framework/hosting/simple_module_hosting/migrations.py @@ -248,7 +248,10 @@ def _note_unverifiable_expression_indexes(engine) -> None: if engine.dialect.name != "sqlite": return try: - from simple_module_core.diagnostics._expression_index import find_expression_indexes + from simple_module_core.diagnostics._expression_index import ( + expression_index_advice, + find_expression_indexes, + ) from simple_module_db.base import all_module_bases found = find_expression_indexes( @@ -260,8 +263,9 @@ def _note_unverifiable_expression_indexes(engine) -> None: if found: logger.info( "Migrations are at head, but SQLite cannot verify expression indexes (%s); " - "check them on PostgreSQL (make migrations-roundtrip-pg). See SM026.", + "%s. See SM026.", ", ".join(f"{index} ({table})" for table, index in found), + expression_index_advice(), ) From 76f89f6bd5492aeacab743e36854399066710e67 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 12 Nov 2026 16:40:50 +0100 Subject: [PATCH 2/5] fix(cli): smpy new shows tool output only on failure, next steps at the end (#917) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 1 + framework/cli/simple_module_cli/new.py | 62 +++++++++++-------- .../tests/test_cli_new_migration_bootstrap.py | 5 +- .../cli/tests/test_cli_new_quiet_install.py | 57 +++++++++++++++++ 4 files changed, 97 insertions(+), 28 deletions(-) create mode 100644 framework/cli/tests/test_cli_new_quiet_install.py diff --git a/CHANGELOG.md b/CHANGELOG.md index baa4aeecd..77b858655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project are documented in this file. The format is b - The quickstart matches what a new app does: an anonymous `curl /api/orders` answers 401 (the scaffold guards the API with `orders.*` permissions), the single-file test example names `tests/web/crud/test_orders_crud.py`, and `smpy users create-admin` is shown without `--password`, so it prompts instead of putting a weak password in shell history (#919). - A `smpy new` app stays installed under plain uv commands. The workspace root now depends on the host (`-host`, a `{ workspace = true }` source), so `uv sync` at the root installs the app instead of uninstalling it. `smpy add … --path host` inside an app, where the host is a workspace member, syncs the whole workspace from the root (`uv sync --all-packages`, as `make install` does) instead of syncing `host/` alone, which removed the root's dev tools and other modules. `smpy create-module` inside an app prints the app's steps (`smpy add`, `make install`, `make migration`, `make migrate`, `make test`, restart `make dev`) instead of `uv sync --extra dev` in the module, which uninstalled the rest of the app; `--no-install` prints `make install`. An existing app can add `dependencies = ["-host"]` and `[tool.uv.sources] -host = { workspace = true }` to its root `pyproject.toml` (#916). - A module installed while `make dev` runs is reported as such. The reload keeps the import path the server started with, so its package cannot be imported until a restart; discovery used to log `No module named '.module'` with a traceback, as if the module were broken. It now logs that the entry point is installed but its package cannot be imported in this process, and to restart `make dev` (`strict` discovery still refuses it). A missing third-party import inside a module is still reported with its traceback (#915). +- The SQLite expression-index warning is no longer printed on every migration. The revision hook now re-emits the indexes of new tables first and then names only expression indexes on tables the revision changes but does not create, so a fresh app's initial schema (which creates `ix_users_user_email_lower`) and a revision on another table print nothing. The warning, `SM026` and the `migrate` note point to `make migrations-roundtrip-pg` only where the Makefile has it; elsewhere they say to run `alembic upgrade heads` against a PostgreSQL `SM_DATABASE_URL` and link the docs. `smpy new` shows the output of `uv sync`, `npm install` and alembic only when a step fails, and prints "Next steps" once, at the end (#917). ## [0.0.45] — 2026-11-11 diff --git a/framework/cli/simple_module_cli/new.py b/framework/cli/simple_module_cli/new.py index 7d84e3c1b..a9e8070cf 100644 --- a/framework/cli/simple_module_cli/new.py +++ b/framework/cli/simple_module_cli/new.py @@ -4,6 +4,7 @@ import shutil import subprocess +from collections.abc import Sequence from enum import StrEnum from pathlib import Path from typing import Annotated @@ -174,19 +175,15 @@ def new_project( rel = path.relative_to(target) if path.is_relative_to(target) else path typer.echo(f" {rel}") typer.echo("\nGenerated secrets were written to .env (gitignored), not .env.example.") - typer.echo("\nNext steps:") - typer.echo(f" cd {target}") if no_install: - typer.echo(" make install") - typer.echo(' make migration msg="initial schema"') - typer.echo(" make migrate") - typer.echo(" make dev") - typer.echo(" make docker-up # or run the full stack in containers") - if "background_tasks" in resolved: - typer.echo(" docker compose up -d redis worker beat # background jobs") + _echo_next_steps( + target, + ["make install", 'make migration msg="initial schema"', "make migrate"], + resolved, + ) return - typer.echo("Installing dependencies...") + typer.echo("\nInstalling dependencies...") # What `make install` runs: a workspace syncs every member, not just the root. uv_sync = ["uv", "sync"] if flat else ["uv", "sync", "--all-packages"] for cmd in (uv_sync, ["npm", "install"]): @@ -197,24 +194,41 @@ def new_project( err=True, ) return - result = subprocess.run(cmd, cwd=target, check=False) - if result.returncode != 0: - typer.echo( - f"WARNING: {' '.join(cmd)} failed (exit {result.returncode}); " - "finish setup manually.", - err=True, - ) + if not _run_quietly(cmd, target): return alembic = _alembic_argv(target, host_dir) + typer.echo("Creating the database...") _bootstrap_initial_migration(target, host_dir, alembic) # `heads` (plural) applies every per-module branch head; `head` (singular) # errors once a second module ships its own migration branch label. - subprocess.run([*alembic, "upgrade", "heads"], cwd=target, check=False) - typer.echo("\nSetup complete. Run `make dev` in the new directory.") - typer.echo("To run the full stack in containers instead: make docker-up") + if not _run_quietly([*alembic, "upgrade", "heads"], target): + return + typer.echo("\nSetup complete.") + _echo_next_steps(target, [], resolved) + + +def _run_quietly(cmd: list[str], cwd: Path) -> bool: + """Run a setup step, showing its output only when it fails (GH #917).""" + result = subprocess.run(cmd, cwd=cwd, check=False, capture_output=True, text=True) + if result.returncode == 0: + return True + typer.echo(f"{result.stdout}{result.stderr}", err=True) + typer.echo( + f"WARNING: {' '.join(cmd)} failed (exit {result.returncode}); finish setup manually.", + err=True, + ) + return False + + +def _echo_next_steps(target: Path, setup: list[str], resolved: Sequence[str]) -> None: + typer.echo("\nNext steps:") + typer.echo(f" cd {target}") + for step in [*setup, "make dev"]: + typer.echo(f" {step}") + typer.echo(" make docker-up # or run the full stack in containers") if "background_tasks" in resolved: - typer.echo("For background jobs, also run: docker compose up -d redis worker beat") + typer.echo(" docker compose up -d redis worker beat # background jobs") def _bootstrap_initial_migration(target: Path, host_dir: Path, alembic: list[str]) -> None: @@ -228,8 +242,4 @@ def _bootstrap_initial_migration(target: Path, host_dir: Path, alembic: list[str versions_dir = host_dir / "migrations" / "versions" if any(p.name != "__init__.py" for p in versions_dir.glob("*.py")): return - subprocess.run( - [*alembic, "revision", "--autogenerate", "-m", "initial schema"], - cwd=target, - check=False, - ) + _run_quietly([*alembic, "revision", "--autogenerate", "-m", "initial schema"], target) diff --git a/framework/cli/tests/test_cli_new_migration_bootstrap.py b/framework/cli/tests/test_cli_new_migration_bootstrap.py index a9aebd48b..ffefcf8b2 100644 --- a/framework/cli/tests/test_cli_new_migration_bootstrap.py +++ b/framework/cli/tests/test_cli_new_migration_bootstrap.py @@ -22,12 +22,13 @@ def test_bootstrap_initial_migration_runs_autogenerate_when_versions_empty( calls: list[tuple[list[str], Path]] = [] - def fake_run(cmd, *, cwd, check): - del check + def fake_run(cmd, *, cwd, check, **kwargs): + del check, kwargs calls.append((list(cmd), Path(cwd))) class _Result: returncode = 0 + stdout = stderr = "" return _Result() diff --git a/framework/cli/tests/test_cli_new_quiet_install.py b/framework/cli/tests/test_cli_new_quiet_install.py new file mode 100644 index 000000000..3d00433c1 --- /dev/null +++ b/framework/cli/tests/test_cli_new_quiet_install.py @@ -0,0 +1,57 @@ +"""`smpy new` keeps tool chatter out of a successful run (GH #917). + +A first run printed ~150 lines (uv's package list, alembic's INFO lines) and +"Next steps: cd " before the install had even started. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from simple_module_cli import new as new_mod +from simple_module_cli.cli import app +from typer.testing import CliRunner + + +def _invoke(tmp_path: Path, monkeypatch, *, fail: str | None = None): + calls: list[list[str]] = [] + + def fake_run(cmd, *, cwd, check, capture_output=False, text=False): + del cwd, check + calls.append(list(cmd)) + assert capture_output and text, f"{cmd} must not stream to the terminal" + failed = fail is not None and fail in cmd + return subprocess.CompletedProcess( + cmd, + 1 if failed else 0, + stdout=f"chatter from {cmd[0]}\n", + stderr="boom\n" if failed else "", + ) + + monkeypatch.setattr(new_mod.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(new_mod.subprocess, "run", fake_run) + result = CliRunner().invoke( + app, ["new", "demo", "--yes", "--db", "sqlite", "--dest", str(tmp_path / "demo")] + ) + return result, calls + + +def test_successful_install_is_quiet_and_ends_with_next_steps(tmp_path, monkeypatch) -> None: + result, calls = _invoke(tmp_path, monkeypatch) + + assert result.exit_code == 0, result.output + assert any("upgrade" in c for c in calls) + assert "chatter from" not in result.output + assert result.output.count("Next steps") == 1 + assert result.output.rstrip().splitlines()[-1].strip() != f"cd {tmp_path / 'demo'}" + assert result.output.index("Next steps") > result.output.index("Installing dependencies") + assert "make dev" in result.output.split("Next steps", 1)[1] + + +def test_a_failing_step_shows_its_output(tmp_path, monkeypatch) -> None: + result, _ = _invoke(tmp_path, monkeypatch, fail="npm") + + assert "chatter from npm" in result.output + assert "boom" in result.output + assert "failed" in result.output From b946053e87d8d80aea23680b806f66f94701aae0 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 12 Nov 2026 16:48:45 +0100 Subject: [PATCH 3/5] fix: address code review findings (round 1, pass 1) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- framework/cli/simple_module_cli/new.py | 23 +++++++++++-------- .../cli/tests/test_cli_new_quiet_install.py | 12 ++++++++++ .../diagnostics/_expression_index.py | 5 ++-- .../simple_module_db/_sqlite_index_compare.py | 5 ++-- .../test_db_expression_index_warning_scope.py | 6 +++++ 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/framework/cli/simple_module_cli/new.py b/framework/cli/simple_module_cli/new.py index a9e8070cf..8077e3411 100644 --- a/framework/cli/simple_module_cli/new.py +++ b/framework/cli/simple_module_cli/new.py @@ -175,12 +175,9 @@ def new_project( rel = path.relative_to(target) if path.is_relative_to(target) else path typer.echo(f" {rel}") typer.echo("\nGenerated secrets were written to .env (gitignored), not .env.example.") + manual_setup = ["make install", 'make migration msg="initial schema"', "make migrate"] if no_install: - _echo_next_steps( - target, - ["make install", 'make migration msg="initial schema"', "make migrate"], - resolved, - ) + _echo_next_steps(target, manual_setup, resolved) return typer.echo("\nInstalling dependencies...") @@ -193,16 +190,21 @@ def new_project( "Install it and finish setup manually.", err=True, ) + _echo_next_steps(target, manual_setup, resolved) return if not _run_quietly(cmd, target): + _echo_next_steps(target, manual_setup, resolved) return alembic = _alembic_argv(target, host_dir) typer.echo("Creating the database...") - _bootstrap_initial_migration(target, host_dir, alembic) # `heads` (plural) applies every per-module branch head; `head` (singular) # errors once a second module ships its own migration branch label. - if not _run_quietly([*alembic, "upgrade", "heads"], target): + if not ( + _bootstrap_initial_migration(target, host_dir, alembic) + and _run_quietly([*alembic, "upgrade", "heads"], target) + ): + _echo_next_steps(target, manual_setup[1:], resolved) return typer.echo("\nSetup complete.") _echo_next_steps(target, [], resolved) @@ -231,15 +233,16 @@ def _echo_next_steps(target: Path, setup: list[str], resolved: Sequence[str]) -> typer.echo(" docker compose up -d redis worker beat # background jobs") -def _bootstrap_initial_migration(target: Path, host_dir: Path, alembic: list[str]) -> None: +def _bootstrap_initial_migration(target: Path, host_dir: Path, alembic: list[str]) -> bool: """Autogenerate the baseline migration if the scaffold ships none. Without a real revision, ``alembic upgrade head`` is a silent no-op against an empty schema — the bundled modules' tables never exist. Runs from ``target`` (the project root) — see :func:`_alembic_argv`. + Returns False when the autogenerate failed. """ versions_dir = host_dir / "migrations" / "versions" if any(p.name != "__init__.py" for p in versions_dir.glob("*.py")): - return - _run_quietly([*alembic, "revision", "--autogenerate", "-m", "initial schema"], target) + return True + return _run_quietly([*alembic, "revision", "--autogenerate", "-m", "initial schema"], target) diff --git a/framework/cli/tests/test_cli_new_quiet_install.py b/framework/cli/tests/test_cli_new_quiet_install.py index 3d00433c1..2ecd30c68 100644 --- a/framework/cli/tests/test_cli_new_quiet_install.py +++ b/framework/cli/tests/test_cli_new_quiet_install.py @@ -55,3 +55,15 @@ def test_a_failing_step_shows_its_output(tmp_path, monkeypatch) -> None: assert "chatter from npm" in result.output assert "boom" in result.output assert "failed" in result.output + + +def test_a_failed_initial_migration_stops_and_prints_the_manual_steps( + tmp_path, monkeypatch +) -> None: + result, calls = _invoke(tmp_path, monkeypatch, fail="--autogenerate") + + assert not any("upgrade" in c for c in calls) + steps = result.output.split("Next steps", 1)[1] + assert "make migration" in steps + assert "make migrate" in steps + assert "Setup complete" not in result.output diff --git a/framework/core/simple_module_core/diagnostics/_expression_index.py b/framework/core/simple_module_core/diagnostics/_expression_index.py index 1e43e0c48..357d5e3f3 100644 --- a/framework/core/simple_module_core/diagnostics/_expression_index.py +++ b/framework/core/simple_module_core/diagnostics/_expression_index.py @@ -14,6 +14,7 @@ import importlib import logging +import re from collections.abc import Iterable from pathlib import Path from typing import TYPE_CHECKING, Any @@ -95,7 +96,7 @@ def module_all_tables(mod: ModuleBase) -> list[Any]: return list(seen.values()) -_ROUNDTRIP_TARGET = "migrations-roundtrip-pg:" +_ROUNDTRIP_TARGET = re.compile(r"^migrations-roundtrip-pg:", re.MULTILINE) _DOCS_URL = "https://docs.py.simplemodule.dev/module-authoring" @@ -108,7 +109,7 @@ def expression_index_advice(root: Path | None = None) -> str: """ makefile = (root or Path.cwd()) / "Makefile" try: - has_target = _ROUNDTRIP_TARGET in makefile.read_text(encoding="utf-8") + has_target = _ROUNDTRIP_TARGET.search(makefile.read_text(encoding="utf-8")) is not None except OSError: has_target = False if has_target: diff --git a/framework/db/simple_module_db/_sqlite_index_compare.py b/framework/db/simple_module_db/_sqlite_index_compare.py index b7e10d2d9..7aa6a754b 100644 --- a/framework/db/simple_module_db/_sqlite_index_compare.py +++ b/framework/db/simple_module_db/_sqlite_index_compare.py @@ -11,8 +11,9 @@ ``make_include_object`` can skip it on SQLite. Anything else still reaches Alembic: a different column list, a uniqueness change, or a real expression such as ``lower(email)``. A change of ordering alone (``DESC`` to ``ASC``) is not -reported on SQLite, which cannot reflect ordering at all; the Postgres round -trip (``make migrations-roundtrip-pg``) is what catches it. +reported on SQLite, which cannot reflect ordering at all; a PostgreSQL +round trip (``make migrations-roundtrip-pg`` in the framework repository) is what +catches it. """ from __future__ import annotations diff --git a/framework/db/tests/test_db_expression_index_warning_scope.py b/framework/db/tests/test_db_expression_index_warning_scope.py index 1778f13d2..b3209820c 100644 --- a/framework/db/tests/test_db_expression_index_warning_scope.py +++ b/framework/db/tests/test_db_expression_index_warning_scope.py @@ -69,3 +69,9 @@ def test_advice_in_a_checkout_with_the_roundtrip_target(tmp_path, monkeypatch) - (tmp_path / "Makefile").write_text("migrations-roundtrip-pg:\n\techo\n", encoding="utf-8") monkeypatch.chdir(tmp_path) assert "make migrations-roundtrip-pg" in expression_index_advice() + + +def test_a_ci_prefixed_target_is_not_the_roundtrip_target(tmp_path, monkeypatch) -> None: + (tmp_path / "Makefile").write_text("ci-migrations-roundtrip-pg:\n\techo\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert "make migrations-roundtrip-pg" not in expression_index_advice() From 89b5520b22094fe31b459ecebc8bdf1f188dd26d Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 12 Nov 2026 16:52:58 +0100 Subject: [PATCH 4/5] fix: address code review findings (round 1, pass 2) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 2 +- docs/module-authoring.md | 12 ++--- docs/reference/diagnostic-codes.md | 2 +- framework/cli/simple_module_cli/new.py | 13 +++-- .../cli/tests/test_cli_new_quiet_install.py | 7 ++- .../diagnostics/_expression_index.py | 6 ++- .../simple_module_db/migration_portability.py | 50 +++++++++++-------- framework/db/simple_module_db/migrations.py | 14 +++++- .../test_db_expression_index_warning_scope.py | 13 ++++- 9 files changed, 76 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b858655..5f462b330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ All notable changes to this project are documented in this file. The format is b - The quickstart matches what a new app does: an anonymous `curl /api/orders` answers 401 (the scaffold guards the API with `orders.*` permissions), the single-file test example names `tests/web/crud/test_orders_crud.py`, and `smpy users create-admin` is shown without `--password`, so it prompts instead of putting a weak password in shell history (#919). - A `smpy new` app stays installed under plain uv commands. The workspace root now depends on the host (`-host`, a `{ workspace = true }` source), so `uv sync` at the root installs the app instead of uninstalling it. `smpy add … --path host` inside an app, where the host is a workspace member, syncs the whole workspace from the root (`uv sync --all-packages`, as `make install` does) instead of syncing `host/` alone, which removed the root's dev tools and other modules. `smpy create-module` inside an app prints the app's steps (`smpy add`, `make install`, `make migration`, `make migrate`, `make test`, restart `make dev`) instead of `uv sync --extra dev` in the module, which uninstalled the rest of the app; `--no-install` prints `make install`. An existing app can add `dependencies = ["-host"]` and `[tool.uv.sources] -host = { workspace = true }` to its root `pyproject.toml` (#916). - A module installed while `make dev` runs is reported as such. The reload keeps the import path the server started with, so its package cannot be imported until a restart; discovery used to log `No module named '.module'` with a traceback, as if the module were broken. It now logs that the entry point is installed but its package cannot be imported in this process, and to restart `make dev` (`strict` discovery still refuses it). A missing third-party import inside a module is still reported with its traceback (#915). -- The SQLite expression-index warning is no longer printed on every migration. The revision hook now re-emits the indexes of new tables first and then names only expression indexes on tables the revision changes but does not create, so a fresh app's initial schema (which creates `ix_users_user_email_lower`) and a revision on another table print nothing. The warning, `SM026` and the `migrate` note point to `make migrations-roundtrip-pg` only where the Makefile has it; elsewhere they say to run `alembic upgrade heads` against a PostgreSQL `SM_DATABASE_URL` and link the docs. `smpy new` shows the output of `uv sync`, `npm install` and alembic only when a step fails, and prints "Next steps" once, at the end (#917). +- The SQLite expression-index warning is no longer printed on every migration. The revision hook now re-emits the indexes of new tables first and then names only expression indexes that the database does not have and the revision does not create (SQLite records index names even though it cannot reflect their definitions), so a fresh app's initial schema (which creates `ix_users_user_email_lower`) and an unrelated revision print nothing, while an expression index added to an existing table is still reported. The warning, `SM026` and the `migrate` note point to `make migrations-roundtrip-pg` only where the Makefile has it; elsewhere they say to run `make migrate` and `alembic check` against a PostgreSQL `SM_DATABASE_URL` and link the docs. `smpy new` shows the output of `uv sync`, `npm install` and alembic only when a step fails, stops after a failed initial migration, and prints "Next steps" once, at the end, listing the steps still to run (#917). ## [0.0.45] — 2026-11-11 diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 3540b4868..5f9a95d93 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -364,14 +364,14 @@ generated file against this checklist before treating it as portable 2. **Expression indexes.** SQLite cannot reflect `Index(..., text("lower(email)"))`, so autogenerate skips it and `alembic check` cannot see it. The revision hook re-emits the expression indexes of every table a revision creates, so an - initial schema needs nothing. When a revision changes an existing table that - has one, autogenerate on SQLite logs a banner naming it, and `make doctor` - reports `SM026` for every expression index. If the change affects the index, - add it to the revision by hand + initial schema needs nothing. When the models declare an expression index the + database does not have (one added to an existing table), autogenerate on + SQLite logs a banner naming it, and `make doctor` reports `SM026` for every + expression index. Add it to the revision by hand (`op.create_index(..., [sa.text("lower(email)")], if_not_exists=True)`) and confirm with the PostgreSQL round trip below; in an app without that make - target, run `alembic upgrade heads` with `SM_DATABASE_URL` pointing at a - PostgreSQL database. The warning is deliberately not an error: SQLite can + target, point `SM_DATABASE_URL` at a PostgreSQL database, run `make migrate`, + then `uv run alembic -c host/alembic.ini check`, which reports a missing index. The warning is deliberately not an error: SQLite can neither confirm nor refute the drift, and PostgreSQL is the actual gate. Prefer writing an ordered index column as `sa.desc(sa.column("col"))` over `sa.text("col DESC")` or `sa.desc("col")`: the latter two are expressions to Alembic, and while the host's include filter diff --git a/docs/reference/diagnostic-codes.md b/docs/reference/diagnostic-codes.md index 884ec011c..b40fbf9de 100644 --- a/docs/reference/diagnostic-codes.md +++ b/docs/reference/diagnostic-codes.md @@ -34,7 +34,7 @@ The framework runs a set of static checks over installed modules at app boot. Th | `SM023` | WARNING | A module's `theme.css` contains an unlayered plain rule (anything but an at-rule or a `:root`-style selector). Unlayered CSS outranks every Tailwind utility. | Move the rule to the module's `styles.css`, which is imported into `layer(components)` so utilities still win. | | `SM024` | WARNING | A unique column, constraint or index on a `MultiTenantMixin` table does not include `tenant_id`, so the first tenant to claim a value locks every other tenant out of it. | Make the key per tenant: add `tenant_id` to it (`Index(..., "tenant_id", "slug", unique=True)`). | | `SM025` | WARNING | `multi_tenant` is on but no module registered `app.state.tenant_resolver`, so only the principal's `tenant_id` claim can bind a tenant — with most auth providers every tenant-scoped query then fails closed. Checked at boot (it needs the built app), in every environment, not by `make doctor`. | Install the `tenants` module, or register your own `async (Request) -> str \| None` resolver on `app.state.tenant_resolver`. | -| `SM026` | INFO | The configured database is SQLite and a module model declares an expression index (e.g. `lower(email)`). SQLite cannot reflect those, so autogenerate and `alembic check` cannot verify them and a clean diff says nothing about them. | Add the index to a revision by hand and verify on PostgreSQL: `make migrations-roundtrip-pg` in this repository, `alembic upgrade heads` against a PostgreSQL `SM_DATABASE_URL` in an app. See [Migrations](/module-authoring#reviewing-a-revision-autogenerated-on-sqlite). | +| `SM026` | INFO | The configured database is SQLite and a module model declares an expression index (e.g. `lower(email)`). SQLite cannot reflect those, so autogenerate and `alembic check` cannot verify them and a clean diff says nothing about them. | Add the index to a revision by hand and verify on PostgreSQL: `make migrations-roundtrip-pg` in this repository, `make migrate` then `alembic check` against a PostgreSQL `SM_DATABASE_URL` in an app. See [Migrations](/module-authoring#reviewing-a-revision-autogenerated-on-sqlite). | | `SM027` | WARNING | A key in `host/locales/overrides/.json` matches no catalog key, so the override was skipped. Reported by `make doctor` and at dev boot. | Fix the key: overrides replace an existing key by its full dotted path (e.g. `users.login.aside_heading`). See [i18n](/framework/i18n#overriding-module-copy). | | `SM028` | WARNING | A module's `tasks.py` imports `torch`, `tensorflow`, `ultralytics`, `osgeo`, `rasterio`, `cv2` or `sklearn` at module level, so every process that loads the module's tasks (web, each worker, beat) pays for it and a queue-restricted worker cannot avoid it. | Import the package inside the task function. See [background_tasks](/modules/background_tasks#running-a-worker-for-a-subset-of-modules). | | `SM029` | INFO | A module overrides `db_extensions()` (Postgres extensions such as `ltree`) and the configured database is SQLite, which has none. `ensure_db_extensions` raises `UnsupportedDatabaseError` when migrations run. | Point `SM_DATABASE_URL` at PostgreSQL. See [Postgres extensions and column types](/database/extensions). | diff --git a/framework/cli/simple_module_cli/new.py b/framework/cli/simple_module_cli/new.py index 8077e3411..5e53e2984 100644 --- a/framework/cli/simple_module_cli/new.py +++ b/framework/cli/simple_module_cli/new.py @@ -200,19 +200,22 @@ def new_project( typer.echo("Creating the database...") # `heads` (plural) applies every per-module branch head; `head` (singular) # errors once a second module ships its own migration branch label. - if not ( - _bootstrap_initial_migration(target, host_dir, alembic) - and _run_quietly([*alembic, "upgrade", "heads"], target) - ): + if not _bootstrap_initial_migration(target, host_dir, alembic): _echo_next_steps(target, manual_setup[1:], resolved) return + if not _run_quietly([*alembic, "upgrade", "heads"], target): + # The revision exists now; regenerating it would create a duplicate. + _echo_next_steps(target, manual_setup[2:], resolved) + return typer.echo("\nSetup complete.") _echo_next_steps(target, [], resolved) def _run_quietly(cmd: list[str], cwd: Path) -> bool: """Run a setup step, showing its output only when it fails (GH #917).""" - result = subprocess.run(cmd, cwd=cwd, check=False, capture_output=True, text=True) + result = subprocess.run( + cmd, cwd=cwd, check=False, capture_output=True, text=True, errors="replace" + ) if result.returncode == 0: return True typer.echo(f"{result.stdout}{result.stderr}", err=True) diff --git a/framework/cli/tests/test_cli_new_quiet_install.py b/framework/cli/tests/test_cli_new_quiet_install.py index 2ecd30c68..5aef92c48 100644 --- a/framework/cli/tests/test_cli_new_quiet_install.py +++ b/framework/cli/tests/test_cli_new_quiet_install.py @@ -17,8 +17,8 @@ def _invoke(tmp_path: Path, monkeypatch, *, fail: str | None = None): calls: list[list[str]] = [] - def fake_run(cmd, *, cwd, check, capture_output=False, text=False): - del cwd, check + def fake_run(cmd, *, cwd, check, capture_output=False, text=False, errors=None): + del cwd, check, errors calls.append(list(cmd)) assert capture_output and text, f"{cmd} must not stream to the terminal" failed = fail is not None and fail in cmd @@ -44,8 +44,7 @@ def test_successful_install_is_quiet_and_ends_with_next_steps(tmp_path, monkeypa assert any("upgrade" in c for c in calls) assert "chatter from" not in result.output assert result.output.count("Next steps") == 1 - assert result.output.rstrip().splitlines()[-1].strip() != f"cd {tmp_path / 'demo'}" - assert result.output.index("Next steps") > result.output.index("Installing dependencies") + assert result.output.index("Next steps") > result.output.index("Setup complete") assert "make dev" in result.output.split("Next steps", 1)[1] diff --git a/framework/core/simple_module_core/diagnostics/_expression_index.py b/framework/core/simple_module_core/diagnostics/_expression_index.py index 357d5e3f3..eb2d0e41d 100644 --- a/framework/core/simple_module_core/diagnostics/_expression_index.py +++ b/framework/core/simple_module_core/diagnostics/_expression_index.py @@ -115,8 +115,10 @@ def expression_index_advice(root: Path | None = None) -> str: if has_target: return "verify them on PostgreSQL with `make migrations-roundtrip-pg`" return ( - "verify them on PostgreSQL: run `alembic upgrade heads` with `SM_DATABASE_URL` " - f"pointing at a PostgreSQL database. See {_DOCS_URL}" + "verify them on PostgreSQL: point `SM_DATABASE_URL` at a PostgreSQL database, " + "run `make migrate`, then `alembic check` with the app's alembic.ini " + "(`uv run alembic -c host/alembic.ini check` in a `smpy new` app), which " + f"reports a missing index. See {_DOCS_URL}" ) diff --git a/framework/db/simple_module_db/migration_portability.py b/framework/db/simple_module_db/migration_portability.py index 8e2f5966b..c00bd337b 100644 --- a/framework/db/simple_module_db/migration_portability.py +++ b/framework/db/simple_module_db/migration_portability.py @@ -8,7 +8,7 @@ :func:`render_boolean_default` emit ``sa.false()`` / ``sa.true()`` instead. * Expression indexes (``lower(email)``) are skipped on SQLite because it cannot reflect them. :func:`warn_unverifiable_expression_indexes` names the ones a - revision may have missed. + revision misses. * ``op.drop_table`` leaves ``CREATE TYPE`` behind on Postgres, so a downgrade followed by an upgrade fails with ``DuplicateObject``. Use :func:`drop_enums_if_postgres` in the revision's ``downgrade``. @@ -121,39 +121,43 @@ def rewrite_boolean_defaults(upgrade_ops: Any) -> None: def warn_unverifiable_expression_indexes( - dialect_name: str | None, metadata: sa.MetaData, upgrade_ops: Any = None + dialect_name: str | None, + metadata: sa.MetaData, + upgrade_ops: Any = None, + existing_index_names: set[str] | None = None, ) -> list: - """Warn when a revision autogenerated on SQLite may miss expression indexes. - - With ``upgrade_ops`` (what the revision hook passes, after re-emitting the - indexes of new tables), only indexes on tables the revision touches and - does not create are named: a fresh initial schema creates them, and a - revision on another table cannot have dropped them (GH #917). Without it, - every expression index in ``metadata`` is named. - - A warning rather than an error: SQLite can neither confirm nor refute the - drift, and a PostgreSQL run is the real gate. Returns the + """Warn when a revision autogenerated on SQLite misses expression indexes. + + SQLite cannot reflect an expression index's definition, so autogenerate + skips them, but it does record their names. The revision hook passes + ``upgrade_ops`` (after re-emitting the indexes of new tables) and the index + names the database already has; then only indexes that are in neither are + named: one added to an existing table, typically. A fresh initial schema + creates them, and a revision on another table leaves them alone, so both + stay silent (GH #917). Without those arguments every expression index in + ``metadata`` is named. + + A warning rather than an error: a PostgreSQL run is the real gate, and an + index whose name exists may still differ in definition. Returns the ``(table, index)`` pairs it warned about. """ if dialect_name != "sqlite": return [] found = find_expression_indexes(metadata.tables.values()) if upgrade_ops is not None: - ops = list(iter_ops_recursive(upgrade_ops)) - touched = {getattr(op, "table_name", None) for op in ops} - emitted = {getattr(op, "index_name", None) for op in ops} - found = [(t, i) for t, i in found if t in touched and i not in emitted] + emitted = {getattr(op, "index_name", None) for op in iter_ops_recursive(upgrade_ops)} + existing = existing_index_names or set() + found = [(t, i) for t, i in found if i not in emitted and i not in existing] if not found: return [] listing = "\n".join(f" - {index} on {table}" for table, index in found) logger.warning( "\n%s\n" "WARNING: autogenerating on SQLite. SQLite cannot reflect expression indexes,\n" - "so this revision does NOT account for the following on the tables it\n" - "changes, and `alembic check` will not notice drift:\n" + "so this revision does NOT create the following, which the models declare\n" + "and the database does not have:\n" "%s\n" - "If the change affects them, add them to the revision by hand and\n" - "%s.\n" + "Add them to the revision by hand and %s.\n" "%s", "=" * 78, listing, @@ -163,6 +167,12 @@ def warn_unverifiable_expression_indexes( return found +def sqlite_index_names(connection: Any) -> set[str]: + """Every index name in a SQLite database, expression indexes included.""" + rows = connection.execute(sa.text("SELECT name FROM sqlite_master WHERE type = 'index'")) + return {row[0] for row in rows} + + def drop_enums_if_postgres(op: Any, *names: str) -> None: """Drop the named enum types on PostgreSQL; a no-op elsewhere. diff --git a/framework/db/simple_module_db/migrations.py b/framework/db/simple_module_db/migrations.py index 0cc9e8ffa..abbb615af 100644 --- a/framework/db/simple_module_db/migrations.py +++ b/framework/db/simple_module_db/migrations.py @@ -36,6 +36,7 @@ from simple_module_db.migration_portability import ( iter_ops_recursive, rewrite_boolean_defaults, + sqlite_index_names, warn_unverifiable_expression_indexes, ) @@ -196,7 +197,7 @@ def make_process_revision_directives( It also rewrites ``0``/``1``/``false``/``true`` server defaults on Boolean columns to ``sa.false()``/``sa.true()`` and, when running on SQLite, names - the expression indexes on changed tables it could not verify (see + the expression indexes the revision misses (see :mod:`simple_module_db.migration_portability`). Call as:: @@ -221,7 +222,9 @@ def process_revision_directives(context, revision, directives): if expression_indexes: _inject_create_index_after_create_table(upgrade_ops, expression_indexes) # After the re-emission, so an index this revision creates is not reported. - warn_unverifiable_expression_indexes(dialect_name, metadata, upgrade_ops) + warn_unverifiable_expression_indexes( + dialect_name, metadata, upgrade_ops, _existing_sqlite_indexes(context) + ) downgrade_ops = getattr(script, "downgrade_ops", None) if downgrade_ops is not None: # A downgrade re-creates dropped tables/columns from reflected @@ -233,6 +236,13 @@ def process_revision_directives(context, revision, directives): return process_revision_directives +def _existing_sqlite_indexes(context) -> set[str]: + connection = getattr(context, "connection", None) + if connection is None or connection.dialect.name != "sqlite": + return set() + return sqlite_index_names(connection) + + def _existing_index_names(container, op_type) -> set[str | None]: """Names of every ``op_type`` index op already present anywhere under ``container``.""" return { diff --git a/framework/db/tests/test_db_expression_index_warning_scope.py b/framework/db/tests/test_db_expression_index_warning_scope.py index b3209820c..020b247f5 100644 --- a/framework/db/tests/test_db_expression_index_warning_scope.py +++ b/framework/db/tests/test_db_expression_index_warning_scope.py @@ -52,8 +52,17 @@ def test_revision_on_another_table_is_silent(caplog) -> None: assert "ix_people_email_lower" not in _run_hook(existing, _metadata(), caplog) -def test_revision_altering_the_indexed_table_warns(caplog) -> None: +def test_revision_altering_a_table_whose_index_exists_is_silent(caplog) -> None: + """SQLite records the index's name even though it cannot reflect it.""" text = _run_hook(_metadata(), _metadata(extra_column=True), caplog) + assert "ix_people_email_lower" not in text + + +def test_an_expression_index_added_to_an_existing_table_warns(caplog) -> None: + """Autogenerate emits nothing for it on SQLite, so the revision misses it.""" + existing = _metadata() + existing.tables["people"].indexes.clear() + text = _run_hook(existing, _metadata(), caplog) assert "ix_people_email_lower on people" in text @@ -61,7 +70,7 @@ def test_advice_outside_this_repo_points_at_what_exists(tmp_path, monkeypatch) - monkeypatch.chdir(tmp_path) advice = expression_index_advice() assert "migrations-roundtrip-pg" not in advice - assert "alembic upgrade heads" in advice + assert "alembic -c host/alembic.ini check" in advice assert "PostgreSQL" in advice From c0baefd21fdf7cea448d38dbc8f648e75c6098e7 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 12 Nov 2026 16:55:34 +0100 Subject: [PATCH 5/5] fix: address code review findings (round 1, pass 3) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- docs/module-authoring.md | 6 ++++-- framework/cli/simple_module_cli/new.py | 16 +++++++++++----- .../cli/tests/test_cli_new_quiet_install.py | 14 ++++++++++++++ .../diagnostics/_expression_index.py | 2 +- .../db/simple_module_db/migration_portability.py | 6 ++++-- .../test_db_expression_index_warning_scope.py | 9 +++++++++ 6 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 5f9a95d93..a94581407 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -371,8 +371,10 @@ generated file against this checklist before treating it as portable (`op.create_index(..., [sa.text("lower(email)")], if_not_exists=True)`) and confirm with the PostgreSQL round trip below; in an app without that make target, point `SM_DATABASE_URL` at a PostgreSQL database, run `make migrate`, - then `uv run alembic -c host/alembic.ini check`, which reports a missing index. The warning is deliberately not an error: SQLite can - neither confirm nor refute the drift, and PostgreSQL is the actual gate. Prefer writing an ordered index column as + then `uv run alembic -c host/alembic.ini check`, which reports a missing + index. The warning is deliberately not an error: + SQLite can neither confirm nor refute the drift, and PostgreSQL is the actual + gate. Prefer writing an ordered index column as `sa.desc(sa.column("col"))` over `sa.text("col DESC")` or `sa.desc("col")`: the latter two are expressions to Alembic, and while the host's include filter suppresses the false SQLite change when only the ordering differs, the column diff --git a/framework/cli/simple_module_cli/new.py b/framework/cli/simple_module_cli/new.py index 5e53e2984..8dc6730e4 100644 --- a/framework/cli/simple_module_cli/new.py +++ b/framework/cli/simple_module_cli/new.py @@ -198,11 +198,11 @@ def new_project( alembic = _alembic_argv(target, host_dir) typer.echo("Creating the database...") - # `heads` (plural) applies every per-module branch head; `head` (singular) - # errors once a second module ships its own migration branch label. if not _bootstrap_initial_migration(target, host_dir, alembic): _echo_next_steps(target, manual_setup[1:], resolved) return + # `heads` (plural) applies every per-module branch head; `head` (singular) + # errors once a second module ships its own migration branch label. if not _run_quietly([*alembic, "upgrade", "heads"], target): # The revision exists now; regenerating it would create a duplicate. _echo_next_steps(target, manual_setup[2:], resolved) @@ -213,9 +213,15 @@ def new_project( def _run_quietly(cmd: list[str], cwd: Path) -> bool: """Run a setup step, showing its output only when it fails (GH #917).""" - result = subprocess.run( - cmd, cwd=cwd, check=False, capture_output=True, text=True, errors="replace" - ) + try: + result = subprocess.run( + cmd, cwd=cwd, check=False, capture_output=True, text=True, errors="replace" + ) + except OSError as exc: # the executable is missing + typer.echo( + f"WARNING: {' '.join(cmd)} could not run ({exc}); finish setup manually.", err=True + ) + return False if result.returncode == 0: return True typer.echo(f"{result.stdout}{result.stderr}", err=True) diff --git a/framework/cli/tests/test_cli_new_quiet_install.py b/framework/cli/tests/test_cli_new_quiet_install.py index 5aef92c48..4ba0dca57 100644 --- a/framework/cli/tests/test_cli_new_quiet_install.py +++ b/framework/cli/tests/test_cli_new_quiet_install.py @@ -66,3 +66,17 @@ def test_a_failed_initial_migration_stops_and_prints_the_manual_steps( assert "make migration" in steps assert "make migrate" in steps assert "Setup complete" not in result.output + + +def test_a_missing_executable_prints_the_manual_steps(tmp_path, monkeypatch) -> None: + def missing(cmd, **kwargs): + raise FileNotFoundError(cmd[0]) + + monkeypatch.setattr(new_mod.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(new_mod.subprocess, "run", missing) + result = CliRunner().invoke( + app, ["new", "demo", "--yes", "--db", "sqlite", "--dest", str(tmp_path / "demo")] + ) + assert result.exit_code == 0, result.output + assert "could not run" in result.output + assert "make install" in result.output.split("Next steps", 1)[1] diff --git a/framework/core/simple_module_core/diagnostics/_expression_index.py b/framework/core/simple_module_core/diagnostics/_expression_index.py index eb2d0e41d..771699dde 100644 --- a/framework/core/simple_module_core/diagnostics/_expression_index.py +++ b/framework/core/simple_module_core/diagnostics/_expression_index.py @@ -107,8 +107,8 @@ def expression_index_advice(root: Path | None = None) -> str: by ``smpy new`` does not, so pointing it there sends users nowhere (GH #917). ``root`` defaults to the working directory, where make, alembic and doctor run. """ - makefile = (root or Path.cwd()) / "Makefile" try: + makefile = (root or Path.cwd()) / "Makefile" has_target = _ROUNDTRIP_TARGET.search(makefile.read_text(encoding="utf-8")) is not None except OSError: has_target = False diff --git a/framework/db/simple_module_db/migration_portability.py b/framework/db/simple_module_db/migration_portability.py index c00bd337b..4d0645314 100644 --- a/framework/db/simple_module_db/migration_portability.py +++ b/framework/db/simple_module_db/migration_portability.py @@ -24,7 +24,7 @@ from typing import Any import sqlalchemy as sa -from alembic.operations.ops import AddColumnOp, AlterColumnOp, CreateTableOp +from alembic.operations.ops import AddColumnOp, AlterColumnOp, CreateIndexOp, CreateTableOp from simple_module_core.diagnostics._expression_index import ( expression_index_advice, find_expression_indexes, @@ -145,7 +145,9 @@ def warn_unverifiable_expression_indexes( return [] found = find_expression_indexes(metadata.tables.values()) if upgrade_ops is not None: - emitted = {getattr(op, "index_name", None) for op in iter_ops_recursive(upgrade_ops)} + emitted = { + op.index_name for op in iter_ops_recursive(upgrade_ops) if isinstance(op, CreateIndexOp) + } existing = existing_index_names or set() found = [(t, i) for t, i in found if i not in emitted and i not in existing] if not found: diff --git a/framework/db/tests/test_db_expression_index_warning_scope.py b/framework/db/tests/test_db_expression_index_warning_scope.py index 020b247f5..7c695cddf 100644 --- a/framework/db/tests/test_db_expression_index_warning_scope.py +++ b/framework/db/tests/test_db_expression_index_warning_scope.py @@ -84,3 +84,12 @@ def test_a_ci_prefixed_target_is_not_the_roundtrip_target(tmp_path, monkeypatch) (tmp_path / "Makefile").write_text("ci-migrations-roundtrip-pg:\n\techo\n", encoding="utf-8") monkeypatch.chdir(tmp_path) assert "make migrations-roundtrip-pg" not in expression_index_advice() + + +def test_dropping_the_index_does_not_count_as_creating_it() -> None: + from alembic.operations.ops import DropIndexOp, UpgradeOps + from simple_module_db.migration_portability import warn_unverifiable_expression_indexes + + ops = UpgradeOps(ops=[DropIndexOp("ix_people_email_lower", table_name="people")]) + found = warn_unverifiable_expression_indexes("sqlite", _metadata(), ops, set()) + assert found == [("people", "ix_people_email_lower")]