diff --git a/recipes/surrealdb-embedded/README.md b/recipes/surrealdb-embedded/README.md new file mode 100644 index 00000000..e4293195 --- /dev/null +++ b/recipes/surrealdb-embedded/README.md @@ -0,0 +1,141 @@ +# surrealdb-embedded + +The native engine behind [SurrealDB](https://surrealdb.com/)'s Python SDK in embedded mode. +With it, a Flet app runs a full SurrealDB database inside itself, with SurrealQL, record +links, full-text and vector search, and keeps it in a file on the device, with no server +and no network. + +This wheel compiles SurrealDB's Rust core for Android and iOS. The SDK itself, +[`surrealdb`](https://pypi.org/project/surrealdb/), is pure Python and installs from PyPI. + +## Install + +```toml +dependencies = [ + "flet", + "surrealdb[embedded]==3.0.0", +] + +[tool.flet.android] +target_arch = ["arm64-v8a", "x86_64"] +``` + +The `embedded` extra pulls in this package at the matching version. The `target_arch` line +is required: there is no 32-bit ARM (`armeabi-v7a`) build, and `flet build apk` targets +that ABI by default, so without the line the build fails resolving it. `x86_64` is only for +the emulator; a release for phones can use `["arm64-v8a"]` alone. + +## Examples + +See runnable Flet apps in [`examples/`](examples): + +- [`notes`](examples/notes) — a notes list in a SurrealKV datastore that survives restarts. + +## Usage in a Flet app + +Open one connection when the app starts and share it. Keep the datastore in +[`FLET_APP_STORAGE_DATA`](https://flet.dev/docs/reference/environment-variables#flet_app_storage_data), +Flet's persistent, app-private storage: + +```python +import os +from pathlib import Path + +from surrealdb import AsyncSurreal, RecordID + +path = Path(os.environ["FLET_APP_STORAGE_DATA"]) / "app.db" +db = AsyncSurreal(f"surrealkv://{path}") +await db.connect() +await db.use("app", "main") +await db.query("DEFINE TABLE IF NOT EXISTS notes SCHEMALESS;") + +await db.create(RecordID("notes", "first"), {"text": "Saved on the device"}) +(rows,) = await db.query("SELECT * FROM notes;") +``` + +`surrealkv://` stores data on disk; `mem://` gives an in-memory database that is gone when +the app closes. + +### Threading + +The async client, `AsyncSurreal`, can be awaited directly in Flet's async event handlers +without blocking the UI. The sync client, `Surreal`, blocks: call it from +[`page.run_thread(...)`](https://flet.dev/docs/controls/page/#flet.Page.run_thread), and end +the worker with [`page.update()`](https://flet.dev/docs/controls/page/#flet.Page.update). + +### Storage + +A SurrealKV datastore is a directory (`LOCK`, `manifest`, `sstables`, `vlog`, `wal`), not a +single file. The engine also creates a `surrealdb/` spill directory under the system temp +directory while it runs. Data written before the app is killed survives: in our runs a +force-stopped app reopened with every committed note intact. + +### App size + +The native library is large: about 26 MB on Android arm64-v8a and 20 MB on an iPhone, +uncompressed, and 32 MB and 27 MB on the x86_64 emulator and simulator. Shipping only the +ABIs you need, through `target_arch`, is the main lever on Android. + +### Other considerations + +Everything above runs offline. The same SDK can also connect to a SurrealDB server over +`ws://` or `http://`; that path does not use this package. + +On a desktop, `flet run` installs PyPI's own `surrealdb-embedded` wheels, so desktop and +device run the same engine version. + +## Things to know + +- **Reading a table that was never defined raises `NotFoundError`,** even if you only + expected an empty result. Define your tables (`DEFINE TABLE IF NOT EXISTS ...`) when you + open the datastore. +- **A datastore can be opened once per process.** A second connection to the same + `surrealkv://` path fails with `ConnectionUnavailableError` because the first holds the + lock. Keep one connection for the life of the app instead of opening one per handler. +- **`query()` returns one result list per statement,** so a single `SELECT` comes back as + `[[row, ...]]`. + +## Build notes (maintainers) + +### Recipe shape + +A maturin (PyO3) package with no recipe-side source changes beyond its `Cargo.toml`. Two +patches: + +- `no-abi3.patch` (both platforms) drops PyO3's `abi3-py39` feature. On iOS, abi3 links + against `libpython3.9`, which no iOS Python tree has; dropping it on Android as well gives + every Python its own wheel, like the other PyO3 recipes. +- `ios-libc.patch` (iOS only) pins `libc` to 0.2.189. The sdist's `Cargo.lock` has 0.2.190, + which restricts Mach APIs that `sysinfo` 0.39.6 still uses on iOS to macOS. + +`AWS_LC_SYS_CMAKE_BUILDER=1` builds AWS-LC with CMake (hence `cmake` in +`requirements.build`), which filters out the `-O3` that cc-rs otherwise appends after +jitterentropy's mandatory `-O0`. `ANDROID_NDK_ROOT` points that CMake build at the NDK. + +`armeabi-v7a` is excluded: `surrealdb-cnf` uses `std::sync::atomic::AtomicU64`, which +32-bit ARM lacks. Upstream ships no 32-bit ARM wheels either. + +### Upgrade hazards + +- **The libc pin is tied to `sysinfo` 0.39.6.** On a bump, drop `ios-libc.patch` first and + see whether the iOS build passes without it; a newer `sysinfo` may no longer need it. +- **The `surrealdb` SDK pins `surrealdb-embedded` exactly** through its `embedded` extra. + Bump this recipe together with each SDK release, or apps cannot resolve both. +- **armeabi-v7a may become buildable** if SurrealDB's crates move to `portable-atomic`; the + build error names the crate that still uses a raw `AtomicU64`. + +### Re-verification checklist + +- Every wheel is tagged for its own Python (`cp312-cp312`, ...), not `cp39-abi3`. +- iOS: `otool -hv` on `_ext.cpython-*.so` reports `DYLIB`. +- The device tests pass on every Python, including `test_persistent_database`, which + reopens a SurrealKV datastore. +- The example's launch count increases across a full restart, on both platforms. + +### Coverage gaps + +The device tests cover loading the engine, sync CRUD and a query, a SurrealKV datastore +surviving a close and reopen, and the async client. CI runs them on the x86_64 Android +emulator and the iPhone simulator; the arm64-v8a slice ran only in the example on an +arm64 emulator, and the iPhone-device slice is built but never run. Full-text, vector and +geo search, live queries and transactions are compiled in but untested here. diff --git a/recipes/surrealdb-embedded/examples/notes/.gitignore b/recipes/surrealdb-embedded/examples/notes/.gitignore new file mode 100644 index 00000000..429a8307 --- /dev/null +++ b/recipes/surrealdb-embedded/examples/notes/.gitignore @@ -0,0 +1,7 @@ +.venv/ +.flet/ +build/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +uv.lock diff --git a/recipes/surrealdb-embedded/examples/notes/README.md b/recipes/surrealdb-embedded/examples/notes/README.md new file mode 100644 index 00000000..4b90176d --- /dev/null +++ b/recipes/surrealdb-embedded/examples/notes/README.md @@ -0,0 +1,34 @@ +# SurrealDB offline notes + +A notes list stored in an embedded SurrealDB datastore on the device. There is no server and +no network: the database engine runs inside the app and keeps its files in the app's +private storage. + +What it demonstrates: + +- **Opening the datastore once.** `open_notebook()` in [`src/notebook.py`](src/notebook.py) + connects to `surrealkv:///notes.db` when the app starts, and every handler + shares that one connection. A second connection to the same path fails on the datastore's + lock. +- **Data that survives restarts.** The status line counts launches in a record of its own. + Close the app completely, open it again, and both the count and your notes are still there. +- **Defining tables before reading them.** SurrealDB raises `NotFoundError` when you select + from a table that was never defined, even an empty one, so the app defines both of its + tables on open. +- **The async API in Flet handlers.** `AsyncSurreal` calls are awaited directly in Flet's + async event handlers, so nothing blocks the UI. + +## Try it + +[Build](https://flet.dev/docs/publish/) the app, then install it on a device or emulator/simulator: + +```bash +# Android +uv run flet build apk + +# iOS +uv run flet build ipa + +# iOS-Simulator +uv run flet build ios-simulator +``` diff --git a/recipes/surrealdb-embedded/examples/notes/pyproject.toml b/recipes/surrealdb-embedded/examples/notes/pyproject.toml new file mode 100644 index 00000000..c3877a54 --- /dev/null +++ b/recipes/surrealdb-embedded/examples/notes/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "surrealdb-notes" +version = "1.0.0" +description = "Offline notes in an embedded SurrealDB datastore that survives restarts." +requires-python = ">=3.12" + +dependencies = [ + "flet==1.1.2", + "surrealdb[embedded]==3.0.0", +] + +[dependency-groups] +dev = ["flet-cli", "flet-desktop", "flet-web"] + +[tool.flet.app] +path = "src" + +[tool.flet.android] +target_arch = ["arm64-v8a", "x86_64"] diff --git a/recipes/surrealdb-embedded/examples/notes/src/main.py b/recipes/surrealdb-embedded/examples/notes/src/main.py new file mode 100644 index 00000000..260cac24 --- /dev/null +++ b/recipes/surrealdb-embedded/examples/notes/src/main.py @@ -0,0 +1,78 @@ +import flet as ft +from notebook import ( + add_note, + count_launch, + database_path, + delete_note, + list_notes, + open_notebook, +) +from surrealdb import AsyncSurreal, RecordID, SurrealError + + +async def main(page: ft.Page): + page.appbar = ft.AppBar(title=ft.Text("Offline notes"), center_title=True) + status = ft.Text("Opening the database…") + entry = ft.TextField(label="New note", multiline=True, expand=True) + save = ft.Button("Save", disabled=True) + notes = ft.Column() + page.add( + ft.SafeArea( + expand=True, + content=ft.Column( + controls=[status, ft.Row([entry, save]), ft.Divider(), notes], + scroll=ft.ScrollMode.AUTO, + ), + ) + ) + + path = database_path() + try: + db: AsyncSurreal = await open_notebook(path) + launches = await count_launch(db) + except (SurrealError, OSError) as exc: + status.value = f"Database failed: {exc}" + status.color = ft.Colors.RED + page.update() + return + + async def refresh(): + """Rebuild the list from the datastore, so it always shows what is on disk.""" + notes.controls = [ + ft.Row( + controls=[ + ft.Text(text, expand=True), + ft.IconButton( + ft.Icons.DELETE_OUTLINE, + on_click=lambda _, n=note: page.run_task(remove, n), + ), + ] + ) + for note, text in await list_notes(db) + ] + count = len(notes.controls) + status.value = f"Launch #{launches} · {count} note{'' if count == 1 else 's'} in {path.name}" + page.update() + + async def remove(note: RecordID): + await delete_note(db, note) + await refresh() + + async def store(_: ft.Event[ft.Button]): + text = (entry.value or "").strip() + if not text: + return + await add_note(db, text) + entry.value = "" + await refresh() + + async def close(_: ft.Event[ft.Page]): + await db.close() + + save.on_click = store + save.disabled = False + page.on_disconnect = close + await refresh() + + +ft.run(main) diff --git a/recipes/surrealdb-embedded/examples/notes/src/notebook.py b/recipes/surrealdb-embedded/examples/notes/src/notebook.py new file mode 100644 index 00000000..e05784bf --- /dev/null +++ b/recipes/surrealdb-embedded/examples/notes/src/notebook.py @@ -0,0 +1,55 @@ +"""The SurrealDB side of the app: one embedded SurrealKV datastore on disk.""" + +import os +from pathlib import Path + +from surrealdb import AsyncSurreal, RecordID + + +def database_path() -> Path: + """Where the datastore lives: Flet's persistent app storage on a device.""" + root = Path(os.getenv("FLET_APP_STORAGE_DATA") or Path.home() / ".surrealdb-notes") + root.mkdir(parents=True, exist_ok=True) + return root / "notes.db" + + +async def open_notebook(path: Path) -> AsyncSurreal: + """Open the datastore once; a second open of the same path fails on its lock.""" + db = AsyncSurreal(f"surrealkv://{path}") + await db.connect() + await db.use("app", "main") + # Reading a table that was never defined raises NotFoundError, even when empty. + await db.query( + "DEFINE TABLE IF NOT EXISTS app_state SCHEMALESS;" + "DEFINE TABLE IF NOT EXISTS notes SCHEMALESS;" + ) + return db + + +async def count_launch(db: AsyncSurreal) -> int: + """Increment and return a counter that only survives if the datastore does.""" + counter = RecordID("app_state", "launches") + previous = await db.select(counter) + launches = (previous["count"] if previous else 0) + 1 + await db.upsert(counter, {"count": launches}) + return launches + + +async def list_notes(db: AsyncSurreal) -> list[tuple[RecordID, str]]: + """Every note, oldest first, as (record id, text).""" + # query() returns one result list per statement. SurrealQL only orders by a + # field the SELECT projects, hence `created`. + (rows,) = await db.query("SELECT id, text, created FROM notes ORDER BY created;") + return [(row["id"], row["text"]) for row in rows] + + +async def add_note(db: AsyncSurreal, text: str) -> None: + """Store one note with a server-side timestamp to sort by.""" + await db.query( + "CREATE notes SET text = $text, created = time::now();", {"text": text} + ) + + +async def delete_note(db: AsyncSurreal, note: RecordID) -> None: + """Remove one note by its record id.""" + await db.delete(note) diff --git a/recipes/surrealdb-embedded/meta.yaml b/recipes/surrealdb-embedded/meta.yaml new file mode 100644 index 00000000..0cbf7239 --- /dev/null +++ b/recipes/surrealdb-embedded/meta.yaml @@ -0,0 +1,30 @@ +package: + name: surrealdb-embedded + version: "3.0.0" + # surrealdb-core's surrealdb-cnf uses std's AtomicU64, which + # 32-bit ARM lacks; upstream ships no 32-bit ARM wheels either. + excluded_arches: [armeabi-v7a] + +requirements: + build: + - cmake + +patches: + - no-abi3.patch +# {% if sdk != 'android' %} + - ios-libc.patch +# {% endif %} + +build: + script_env: + _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' + # cc-rs appends CPython's -O3 after jitterentropy's mandatory -O0. + # AWS-LC's CMake builder filters those inherited optimization flags. + AWS_LC_SYS_CMAKE_BUILDER: "1" +# {% if sdk == 'android' %} + ANDROID_NDK_ROOT: '{NDK_ROOT}' +# {% endif %} + +test: + requires: + - surrealdb==3.0.0 diff --git a/recipes/surrealdb-embedded/patches/ios-libc.patch b/recipes/surrealdb-embedded/patches/ios-libc.patch new file mode 100644 index 00000000..917702e3 --- /dev/null +++ b/recipes/surrealdb-embedded/patches/ios-libc.patch @@ -0,0 +1,19 @@ +Pin libc to 0.2.189 on iOS. + +sysinfo 0.39.6 uses Mach APIs that libc 0.2.190 restricts to macOS, so the +iOS build fails against the locked 0.2.190. + +diff -ruN a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -37,6 +37,10 @@ + surrealdb-types = "4" + tokio = { version = "1", features = ["full"] } + uuid = "1.27" ++ ++[target.'cfg(target_os = "ios")'.dependencies] ++# libc 0.2.190 hides Mach APIs still used by sysinfo 0.39.6 on iOS. ++libc = "=0.2.189" + + [profile.release] + lto = true diff --git a/recipes/surrealdb-embedded/patches/no-abi3.patch b/recipes/surrealdb-embedded/patches/no-abi3.patch new file mode 100644 index 00000000..85d2b19e --- /dev/null +++ b/recipes/surrealdb-embedded/patches/no-abi3.patch @@ -0,0 +1,18 @@ +Build a per-version extension instead of abi3. + +With the abi3-py39 feature, PyO3 links iOS extensions against +`-lpython3.9`, which no iOS Python tree has. Drop abi3 so each wheel links +its own libpython, matching the other PyO3 recipes. + +diff -ruN a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -9,7 +9,7 @@ + crate-type = ["cdylib"] + + [dependencies] +-pyo3 = { version = "0.29", features = ["extension-module", "abi3-py39"] } ++pyo3 = { version = "0.29", features = ["extension-module"] } + pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } + surrealdb-core = { version = "4", default-features = false, features = [ + "kv-mem", diff --git a/recipes/surrealdb-embedded/tests/test_surrealdb_embedded.py b/recipes/surrealdb-embedded/tests/test_surrealdb_embedded.py new file mode 100644 index 00000000..2dc65b95 --- /dev/null +++ b/recipes/surrealdb-embedded/tests/test_surrealdb_embedded.py @@ -0,0 +1,56 @@ +import asyncio +import importlib.util +import sys +from pathlib import Path + +from surrealdb import AsyncSurreal, RecordID, Surreal +from surrealdb_embedded import SyncEmbeddedDB + + +def test_native_engine() -> None: + """Load the native binding and initialize an in-memory engine.""" + native = importlib.util.find_spec("surrealdb_embedded._ext") + assert native is not None + print(f"SURREALDB_RUNTIME python={sys.version} native={native.origin}", flush=True) + with SyncEmbeddedDB("mem://") as db: + db.connect() + + +def test_sync_crud() -> None: + """Run create, select, update, query and delete through the Python SDK.""" + record = RecordID("notes", "smoke") + with Surreal("mem://") as db: + db.use("app", "main") + db.create(record, {"text": "Saved locally", "count": 1}) + assert db.select(record).execute()["text"] == "Saved locally" + db.update(record, {"text": "Updated", "count": 2}) + assert db.select(record).execute()["count"] == 2 + assert db.query("RETURN 6 * 7;").execute() == [42] + db.delete(record).execute() + assert not db.select(record).execute() + + +def test_persistent_database(tmp_path: Path) -> None: + """Keep a SurrealKV record after closing and reopening its datastore.""" + url = f"surrealkv://{tmp_path / 'database'}" + record = RecordID("notes", "persistent") + with Surreal(url) as db: + db.use("app", "main") + db.create(record, {"text": "Survives reopen"}) + with Surreal(url) as db: + db.use("app", "main") + assert db.select(record).execute()["text"] == "Survives reopen" + + +def test_async_sdk() -> None: + """Run the asynchronous binding in a real Python event loop.""" + + async def exercise() -> None: + async with AsyncSurreal("mem://") as db: + await db.use("app", "main") + record = RecordID("notes", "async") + await db.create(record, {"text": "Async native engine"}) + assert (await db.select(record))["text"] == "Async native engine" + await db.delete(record) + + asyncio.run(exercise())