diff --git a/.claude/commands/stress-test-sync-sqlitecloud.md b/.claude/commands/stress-test-sync-sqlitecloud.md new file mode 100644 index 0000000..f7f1c75 --- /dev/null +++ b/.claude/commands/stress-test-sync-sqlitecloud.md @@ -0,0 +1,244 @@ +# Sync Stress Test with remote SQLiteCloud database + +Execute a stress test against the CloudSync server using multiple concurrent local SQLite databases syncing large volumes of CRUD operations simultaneously. Designed to reproduce server-side errors (e.g., "database is locked", 500 errors) under heavy concurrent load. + +## Prerequisites +- Connection string to a sqlitecloud project +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Configuration + +### Step 1: Gather Parameters + +Ask the user for the following configuration using a single question set: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. +2. **SQLiteCloud connection string** — format: `sqlitecloud://:/?apikey=`. If no `` is in the path, ask the user for one or propose `test_stress_sync`. +3. **Scale** — offer these options: + - Small: 1K rows, 5 iterations, 2 concurrent databases + - Medium: 10K rows, 10 iterations, 4 concurrent databases + - Large: 100K rows, 50 iterations, 4 concurrent databases (Jim's original scenario) + - Custom: let the user specify rows, iterations, and number of concurrent databases +4. **Operations per iteration** — how many UPDATE and DELETE operations to perform each iteration: + - `NUM_UPDATES`: number of UPDATE operations per iteration (default: 1). Each UPDATE runs `UPDATE SET value = value + 1;` affecting all rows. + - `NUM_DELETES`: number of DELETE operations per iteration (default: 1). Each DELETE runs `DELETE FROM
WHERE rowid IN (SELECT rowid FROM
ORDER BY RANDOM() LIMIT 10);` removing 10 random rows. Set to 0 to skip deletes entirely. + - Propose defaults of 1 update and 1 delete. The user can set 0 deletes for update-only tests. +5. **RLS mode** — with RLS (requires user tokens) or without RLS +5. **Table schema** — offer simple default or custom: + ```sql + CREATE TABLE test_sync (id TEXT PRIMARY KEY, user_id TEXT NOT NULL DEFAULT '', name TEXT, value INTEGER); + ``` + +Save these as variables: +- `CUSTOM_ADDRESS` (only if the user provided a non-default address) +- `CONNECTION_STRING` (the full sqlitecloud:// connection string) +- `DB_NAME` (database name extracted or provided) +- `HOST` (hostname extracted from connection string) +- `APIKEY` (apikey extracted from connection string) +- `ROWS` (number of rows per iteration) +- `ITERATIONS` (number of delete/insert/update cycles) +- `NUM_DBS` (number of concurrent databases) +- `NUM_UPDATES` (number of UPDATE operations per iteration, default 1) +- `NUM_DELETES` (number of DELETE operations per iteration, default 1; 0 to skip) + +### Step 2: Setup SQLiteCloud Database and Table + +Connect to SQLiteCloud using `~/go/bin/sqlc` (last command must be `quit`). Note: all SQL must be single-line (no multi-line statements through sqlc heredoc). + +1. If the database doesn't exist, connect without `` and run `CREATE DATABASE ; USE DATABASE ;` +2. `LIST TABLES` to check for existing tables +3. For any table with a `_cloudsync` companion table, run `CLOUDSYNC DISABLE ;` +4. `DROP TABLE IF EXISTS ;` +5. Create the test table (single-line DDL) +6. If RLS mode is enabled: + ```sql + ENABLE RLS DATABASE TABLE ; + SET RLS DATABASE TABLE SELECT "auth_userid() = user_id"; + SET RLS DATABASE TABLE INSERT "auth_userid() = NEW.user_id"; + SET RLS DATABASE TABLE UPDATE "auth_userid() = NEW.user_id AND auth_userid() = OLD.user_id"; + SET RLS DATABASE TABLE DELETE "auth_userid() = OLD.user_id"; + ``` +7. Ask the user to enable CloudSync on the table from the SQLiteCloud dashboard + +### Step 3: Get Managed Database ID + +Now that the database and tables are created and CloudSync is enabled on the dashboard, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard after enabling CloudSync on the table. + +Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 4: Set Up Authentication + +Authentication depends on the RLS mode: + +**If RLS is disabled:** Use the project `APIKEY` (already extracted from the connection string). After each `cloudsync_network_init`/`cloudsync_network_init_custom` call, authenticate with: +```sql +SELECT cloudsync_network_set_apikey(''); +``` +No tokens are needed. Skip token creation entirely. + +**If RLS is enabled:** Create tokens for the test users. Create as many users as needed for the number of concurrent databases (assign 2 databases per user, or 1 per user if NUM_DBS <= 2). + +For each user N: +```bash +curl -s -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d '{"name": "claude@sqlitecloud.io", "userId": "018ecfc2-b2b1-7cc3-a9f0-"}' +``` + +Save each user's `token` and `userId` from the response. After each `cloudsync_network_init`/`cloudsync_network_init_custom` call, authenticate with: +```sql +SELECT cloudsync_network_set_token(''); +``` + +**IMPORTANT:** Using a token when RLS is disabled will cause the server to silently reject all sent changes (send appears to succeed but data is not persisted remotely). Always use `cloudsync_network_set_apikey` when RLS is off. + +### Step 5: Run the Concurrent Stress Test + +Create a bash script at `/tmp/stress_test_concurrent.sh` that: + +1. **Initializes N local SQLite databases** at `/tmp/sync_concurrent_.db`: + - Uses Homebrew sqlite3: find with `ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1` + - Loads the extension from `dist/cloudsync.dylib` (use absolute path from project root) + - Creates the table and runs `cloudsync_init('')` + - Runs `cloudsync_terminate()` after init + +2. **Defines a worker function** that runs in a subshell for each database: + - Each worker logs all output to `/tmp/sync_concurrent_.log` + - Each iteration does: + a. **UPDATE** — run `UPDATE
SET value = value + 1;` repeated `NUM_UPDATES` times (skip if 0) + b. **DELETE** — run `DELETE FROM
WHERE rowid IN (SELECT rowid FROM
ORDER BY RANDOM() LIMIT 10);` repeated `NUM_DELETES` times (skip if 0) + c. **Sync using the 3-step send/check/check pattern:** + 1. `SELECT cloudsync_network_send_changes();` — send local changes to the server + 2. `SELECT cloudsync_network_check_changes();` — ask the server to prepare a payload of remote changes + 3. Sleep 1 second (outside sqlite3, between two separate sqlite3 invocations) + 4. `SELECT cloudsync_network_check_changes();` — download the prepared payload, if any + - Each sqlite3 session must: `.load` the extension, call `cloudsync_network_init()`/`cloudsync_network_init_custom()`, `cloudsync_network_set_apikey()`/`cloudsync_network_set_token()` (depending on RLS mode), do the work, call `cloudsync_terminate()` + - **Timing**: Log the wall-clock execution time (in milliseconds) for each `cloudsync_network_send_changes()`, `cloudsync_network_check_changes()` call. Define a `now_ms()` helper function at the top of the script and use it before and after each sqlite3 invocation that calls a network function, computing the delta. On **macOS**, `date` does not support `%3N` (nanoseconds) — use `python3 -c 'import time; print(int(time.time()*1000))'` instead. On **Linux**, `date +%s%3N` works fine. The script should detect the platform and define `now_ms()` accordingly. Log lines like: `[DB][iter ] send_changes: 123ms`, `[DB][iter ] check_changes_1: 45ms`, `[DB][iter ] check_changes_2: 67ms` + - Include labeled output lines like `[DB][iter ] updated count=, deleted count=` for grep-ability + +3. **Launches all workers in parallel** using `&` and collects PIDs + +4. **Waits for all workers** and captures exit codes + +5. **Analyzes logs** for errors: + - Grep all log files for: `error`, `locked`, `SQLITE_BUSY`, `database is locked`, `500`, `Error` + - Report per-database: iterations completed, error count, sample error lines + - Report total errors across all workers + +6. **Prints final verdict**: PASS (0 errors) or FAIL (errors detected) + +**Important script details:** +- Use `echo -e` to pipe generated SQL (with `\n` separators) into sqlite3 +- During database initialization (Step 1), insert `ROWS` initial rows per database in a single transaction so each DB starts with data to update/delete. Row IDs should be unique across databases: `db_r` +- User IDs for rows must match the token's userId for RLS to work +- The sync pattern requires **separate sqlite3 invocations** for send_changes and each check_changes call (with a 1-second sleep between the two check_changes calls), so that timing can be measured per-call from bash +- **stderr capture**: All sqlite3 invocations must redirect both stdout and stderr to the log file. Use `>> "$LOG" 2>&1` (in this order — stdout redirect first, then stderr to stdout). For timed calls that capture output in a variable, redirect stderr to the log file separately: `RESULT=$(echo -e "$SQL" | $SQLITE3 "$DB" 2>> "$LOG")` and then echo `$RESULT` to the log as well. This ensures "Runtime error" messages from sqlite3 are never lost. +- Use `/bin/bash` (not `/bin/sh`) for arrays and process management + +Run the script with a 10-minute timeout. + +### Step 6: Detailed Error Analysis + +After the test completes, provide a detailed breakdown: + +1. **Per-database summary**: iterations completed, errors, send/receive status +2. **Error categorization**: group errors by type (e.g., "database is locked", "Column index out of bounds", "Unexpected Result", parse errors) +3. **Timeline analysis**: do errors cluster at specific iterations or spread evenly? +4. **Read full log files** if errors are found — show the first and last 30 lines of each log with errors + +### Step 7: Final Sync and Data Integrity Verification + +After all workers have terminated, perform a **final sync on every local database** to ensure all databases converge to the same state. Then verify data integrity. + +**IMPORTANT — RLS mode changes what "convergence" means:** When RLS is enabled, each user can only see their own rows. Databases belonging to different users will have different row counts and different data — this is correct behavior. All convergence and integrity checks must therefore be scoped **per user group** (i.e., only compare databases that share the same userId/token). + +1. **Final sync loop** (max 10 retries): Repeat the following until convergence is achieved within each user group, or the retry limit is reached: + a. For each local database (sequentially): + - Load the extension, call `cloudsync_network_init`/`cloudsync_network_init_custom`, authenticate with `cloudsync_network_set_apikey`/`cloudsync_network_set_token` + - Run `SELECT cloudsync_network_sync(100, 10);` to sync remaining changes + - Call `cloudsync_terminate()` + b. After syncing all databases, query `SELECT COUNT(*) FROM
` on each database + c. **If RLS is disabled:** Check that all databases have the same row count. If so, convergence is achieved — break. + d. **If RLS is enabled:** Group databases by userId. Within each user group, check that all databases have the same row count. Convergence is achieved when every user group is internally consistent — break. Different user groups are expected to have different row counts. + e. Otherwise, log the round number and the distinct row counts (per group if RLS), then repeat from (a) + f. If the retry limit is reached without convergence, report it as a failure + +2. **Row count verification**: + - **If RLS is disabled:** Report the final row counts. All databases should have the same number of rows. + - **If RLS is enabled:** Report row counts grouped by user. All databases within the same user group should have identical row counts. Different user groups may differ. Also verify that each database only contains rows matching its userId. + - In both cases, also check SQLiteCloud (as admin) for total row count. + +3. **Row content verification**: + - **If RLS is disabled:** Pick one random row ID from the first database. Query that row on every local database. All must return identical values. + - **If RLS is enabled:** For each user group, pick one random row ID from the first database in that group. Query that row on all databases in the same user group. All databases in the group must return identical values. Do NOT expect databases from other user groups to have this row — they should return empty (RLS blocks cross-user access). + +4. **RLS cross-user leak check** (RLS mode only): For a sample of databases (e.g., one per user group), verify that `SELECT COUNT(*) FROM
WHERE user_id != ''` returns 0. Report any cross-user data leakage as a test failure. + +## Output Format + +Report the test results including: + +| Metric | Value | +|--------|-------| +| Concurrent databases | N | +| Rows per iteration | ROWS | +| Iterations per database | ITERATIONS | +| Total CRUD operations | N × ITERATIONS × (UPDATE_ALL + DELETE_FEW) | +| Total sync operations | N × ITERATIONS × 3 (1 send_changes + 2 check_changes) | +| Duration | start to finish time | +| Total errors | count | +| Error types | categorized list | +| Result | PASS/FAIL | + +If errors are found, include: +- Full error categorization table +- Sample error messages +- Which databases were most affected +- Whether errors are client-side or server-side + +## Success Criteria + +The test **PASSES** if: +1. All workers complete all iterations +2. Zero `error`, `locked`, `SQLITE_BUSY`, or HTTP 500 responses in any log +3. After the final sync, databases converge: + - **Without RLS:** all local databases have the same row count + - **With RLS:** all databases within each user group have the same row count (different user groups may differ) +4. Row content is consistent: + - **Without RLS:** a randomly selected row has identical content across all local databases + - **With RLS:** a randomly selected row has identical content across all databases in the same user group; databases from other user groups correctly return empty for that row +5. **With RLS:** no cross-user data leakage (each database contains only rows matching its userId) + +The test **FAILS** if: +1. Any worker crashes or fails to complete +2. Any `database is locked` or `SQLITE_BUSY` errors appear +3. Server returns 500 errors under concurrent load +4. Row counts differ within the comparison scope (all DBs without RLS, same-user DBs with RLS) after the final sync loop exhausts all retries +5. Row content differs within the comparison scope (data corruption) +6. **With RLS:** any database contains rows belonging to a different userId (cross-user data leakage) + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- Network settings (`cloudsync_network_init`, `cloudsync_network_set_token`) are NOT persisted between sessions — must be called every time +- Extension must be loaded BEFORE any INSERT/UPDATE/DELETE for cloudsync to track changes +- All NOT NULL columns must have DEFAULT values +- `cloudsync_terminate()` must be called before closing each session +- sqlc heredoc only supports single-line SQL statements + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_concurrent_*.db`, `/tmp/sync_concurrent_*.log`) +- SQLiteCloud via `~/go/bin/sqlc ""` +- Curl commands to the sync server and SQLiteCloud API for token creation + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md b/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md new file mode 100644 index 0000000..c23b43c --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-sqlitecloud-rls.md @@ -0,0 +1,468 @@ +# Sync Roundtrip Test with remote SQLiteCloud database and RLS policies + +Execute a full roundtrip sync test between multiple local SQLite databases and the sqlitecloud, verifying that Row Level Security (RLS) policies are correctly enforced during sync. + +## Prerequisites +- Connection string to a sqlitecloud project +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +### Step 1: Get CloudSync Parameters + +Ask the user for: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +## Test Procedure + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key with user_id for RLS** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT, + value INTEGER +); +``` + +**Option 2: Multi tables scenario for advanced RLS policy** + +Propose a simple but multitables real world scenario + +**Option 3: Custom policy** +Ask the user to describe the table/tables in plain English or DDL queries. + +**Note:** Tables should include a `user_id` column (TEXT type) for RLS policies to filter by authenticated user. + +### Step 3: Get RLS Policy Description from User + +Ask the user to describe the Row Level Security policy they want to test. Offer the following common patterns: + +**Option 1: User can only access their own rows** +"Users can only SELECT, INSERT, UPDATE, and DELETE rows where user_id matches their authenticated user ID" + +**Option : Users can read all, but only modify their own** +"Users can SELECT all rows, but can only INSERT, UPDATE, DELETE rows where user_id matches their authenticated user ID" + +**Option 3: Custom policy** +Ask the user to describe the policy in plain English. + +### Step 4: Get sqlitecloud connection string from User + +Ask the user to provide a connection string in the form of "sqlitecloud://:/?apikey=" to be later used with the sqlitecloud cli (sqlc) with `~/go/bin/sqlc ""`. + +### Step 5: Setup SQLiteCloud with RLS + +Connect to SQLiteCloud and prepare the environment: +```bash +~/go/bin/sqlc +``` + +The last command inside sqlc to exit from the cli program must be `quit`. + +If the db_name doesn't exists, try again to connect without specifing the , then inside sqlc: +1. CREATE DATABASE +2. USE DATABASE + +Then, inside sqlc: +1. List existing tables with `LIST TABLES` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + CLOUDSYNC DISABLE + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS ;` +5. Create the test table using the SQLite DDL +6. Enable RLS on the table: + ```sql + ENABLE RLS DATABASE TABLE + ``` +7. Create RLS policies based on the user's description. +Your RLS policies for INSERT, UPDATE, and DELETE operations can reference column values as they are being changed. This is done using the special OLD.column and NEW.column identifiers. Their availability and meaning depend on the operation being performed: + ++-----------+--------------------------------------------+--------------------------------------------+ +| Operation | OLD.column Reference | NEW.column Reference | ++-----------+--------------------------------------------+--------------------------------------------+ +| INSERT | Not available | The value for the new row. | +| UPDATE | The value of the row before the update. | The value of the row after the update. | +| DELETE | The value of the row being deleted. | Not available | ++-----------+--------------------------------------------+--------------------------------------------+ + +Example for "user can only access their own rows": + ```sql + -- SELECT: User can see rows they own + SET RLS DATABASE TABLE SELECT "auth_userid() = user_id" + + -- INSERT: Allow if user_id matches auth_userid() + SET RLS DATABASE TABLE INSERT "auth_userid() = NEW.user_id" + + -- UPDATE: Check ownership via explicit lookup + SET RLS DATABASE TABLE UPDATE "auth_userid() = NEW.user_id AND auth_userid() = OLD.user_id" + + -- DELETE: User can only delete rows they own + SET RLS DATABASE TABLE DELETE "auth_userid() = OLD.user_id" + ``` +8. Ask the user to enable CloudSync on the table from the SQLiteCloud dashboard + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and CloudSync is enabled on the dashboard, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard after enabling CloudSync on the table. + +Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + + + +9. Insert some initial test data (optional, can be done via SQLite clients) + +### Step 6: Get tokens for Two Users + +Get auth tokens for both test users by running the token script twice: + +**User 1: claude1@sqlitecloud.io** +```bash +curl -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d $'{ + "name": "claude1@sqlitecloud.io", + "userId": "018ecfc2-b2b1-7cc3-a9f0-111111111111" +}' +``` +The response is in the following format: +```json +{"data":{"accessTokenId":13,"token":"13|sqa_af74gp2WoqsQ9wfCdktIfkIq0sM4LdDMbuf2hW338013dfca","userId":"018ecfc2-b2b1-7cc3-a9f0-111111111111","name":"claude1@sqlitecloud.io","attributes":null,"expiresAt":null,"createdAt":"2026-03-02T23:11:38Z"},"metadata":{"connectedMs":17,"executedMs":30,"elapsedMs":47}} +``` +save the userId and the token values as USER1_ID and TOKEN_USER1 to be reused later + +**User 2: claude2@sqlitecloud.io** +```bash +curl -X "POST" "https:///v2/tokens" \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d $'{ + "name": "claude2@sqlitecloud.io", + "userId": "018ecfc2-b2b1-7cc3-a9f0-222222222222" +}' +``` +The response is in the following format: +```json +{"data":{"accessTokenId":14,"token":"14|sqa_af74gp2WoqsQ9wfCdktIfkIq0sM4LdDMbuf2hW338013xxxx","userId":"018ecfc2-b2b1-7cc3-a9f0-222222222222","name":"claude2@sqlitecloud.io","attributes":null,"expiresAt":null,"createdAt":"2026-03-02T23:11:38Z"},"metadata":{"connectedMs":17,"executedMs":30,"elapsedMs":47}} +``` +save the userId and the token values as USER2_ID and TOKEN_USER2 to be reused later + +### Step 7: Setup Four SQLite Databases + +Create four temporary SQLite databases using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 +``` + +**Database 1A (User 1, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 1B (User 1, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2A (User 2, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2B (User 2, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +### Step 8: Insert Test Data + +Ask the user for optional details about the kind of test data to insert in the tables, otherwise generate some real world data for the choosen tables. +Insert distinct test data in each database. Use the extracted user IDs for the if needed. +For example, for the simple table scenario: + +**Database 1A (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_1', '', 'User1 DeviceA Row1', 100); +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_2', '', 'User1 DeviceA Row2', 101); +``` + +**Database 1B (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_b_1', '', 'User1 DeviceB Row1', 200); +``` + +**Database 2A (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_1', '', 'User2 DeviceA Row1', 300); +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_2', '', 'User2 DeviceA Row2', 301); +``` + +**Database 2B (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_b_1', '', 'User2 DeviceB Row1', 400); +``` + +### Step 9: Execute Sync on All Databases + +For each of the four SQLite databases, execute the sync operations: + +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_check_changes(); +-- Repeat check_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes +``` + +**Recommended sync order:** +1. Sync Database 1A (send + check) +2. Sync Database 2A (send + check) +3. Sync Database 1B (send + check) +4. Sync Database 2B (send + check) +5. Re-sync all databases (check_changes) to ensure full propagation + +### Step 10: Verify RLS Enforcement + +After syncing all databases, verify that each database contains only the expected rows based on the RLS policy: + +**Expected Results (for "user can only access their own rows" policy):** + +**User 1 databases (1A and 1B) should contain:** +- All rows with `user_id = USER1_ID` (u1_a_1, u1_a_2, u1_b_1) +- Should NOT contain any rows with `user_id = USER2_ID` + +**User 2 databases (2A and 2B) should contain:** +- All rows with `user_id = USER2_ID` (u2_a_1, u2_a_2, u2_b_1) +- Should NOT contain any rows with `user_id = USER1_ID` + +**PostgreSQL (as admin) should contain:** +- ALL rows from all users (6 total rows) + +Run verification queries: +```sql +-- In each SQLite database +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; + +-- In PostgreSQL (as admin) +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; +SELECT user_id, COUNT(*) FROM GROUP BY user_id; +``` + +### Step 11: Test Write RLS Policy Enforcement + +Test that the server-side RLS policy blocks unauthorized writes by attempting to insert a row with a `user_id` that doesn't match the authenticated user's token. + +**In Database 1A (User 1), insert a malicious row claiming to belong to User 2:** +```sql +-- Attempt to insert a row with User 2's user_id while authenticated as User 1 +INSERT INTO (id, user_id, name, value) VALUES ('malicious_1', '', 'Malicious Row from User1', 999); + +-- Attempt to sync this unauthorized row to PostgreSQL +SELECT cloudsync_network_send_changes(); +``` + +**Wait 2-3 seconds, then verify in PostgreSQL (as admin) that the malicious row was rejected:** +```sql +-- In PostgreSQL (as admin) +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows returned + +SELECT COUNT(*) FROM WHERE id = 'malicious_1'; +-- Expected: 0 +``` + +**Also verify the malicious row does NOT appear in User 2's databases after syncing:** +```sql +-- In Database 2A or 2B (User 2) +SELECT cloudsync_network_check_changes(); +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows (the malicious row should not sync to legitimate User 2 databases) +``` + +**Expected Behavior:** +- The `cloudsync_network_send_changes()` call may succeed (return value indicates network success, not RLS enforcement) +- The malicious row should be **rejected by PostgreSQL RLS** and NOT inserted into the server database +- The malicious row will remain in the local SQLite Database 1A (local inserts are not blocked), but it will never propagate to the server or other clients +- User 2's databases should never receive this row + +**This step PASSES if:** +1. The malicious row is NOT present in PostgreSQL +2. The malicious row does NOT appear in any of User 2's SQLite databases +3. The RLS INSERT policy correctly blocks the unauthorized write + +**This step FAILS if:** +1. The malicious row appears in PostgreSQL (RLS bypass vulnerability) +2. The malicious row syncs to User 2's databases (data leakage) + +### Step 12: Cleanup + +In each SQLite database before closing: +```sql +SELECT cloudsync_terminate(); +``` + +In SQLiteCloud (optional, for full cleanup): +```sql +CLOUDSYNC DISABLE ); +DROP TABLE IF EXISTS ; +``` + +## Output Format + +Report the test results including: +- DDL used for both databases +- RLS policies created +- User IDs for both test users +- Initial data inserted in each database +- Number of sync operations performed per database +- Final data in each database (with row counts) +- RLS verification results: + - User 1 databases: expected rows vs actual rows + - User 2 databases: expected rows vs actual rows + - SQLiteCloud: total rows +- Write RLS enforcement results: + - Malicious row insertion attempted: yes/no + - Malicious row present in SQLiteCloud: yes/no (should be NO) + - Malicious row synced to User 2 databases: yes/no (should be NO) +- **PASS/FAIL** status with detailed explanation + +### Success Criteria + +The test PASSES if: +1. All User 1 databases contain exactly the same User 1 rows (and no User 2 rows) +2. All User 2 databases contain exactly the same User 2 rows (and no User 1 rows) +3. SQLiteCloud contains all rows from both users +4. Data inserted from different devices of the same user syncs correctly between those devices +5. **Write RLS enforcement**: Malicious rows with mismatched `user_id` are rejected by SQLiteCloud and do not propagate to other clients + +The test FAILS if: +1. Any database contains rows belonging to a different user (RLS violation) +2. Any database is missing rows that should be visible to that user +3. Sync operations fail or timeout +4. **Write RLS bypass**: A malicious row with a `user_id` not matching the token appears in SQLiteCloud or syncs to other databases + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- SQLiteCloud tables need cleanup before re-running tests +- `cloudsync_network_check_changes()` may need multiple calls with delays +- Run `SELECT cloudsync_terminate();` on SQLite connections before closing to properly cleanup memory +- Ensure both test users exist in Supabase auth before running the test +- The RLS policies must use `auth_userid()` to work with SQLiteCloud token authentication + +## Critical Schema Requirements (Common Pitfalls) + +### 1. All NOT NULL columns must have DEFAULT values +Cloudsync requires that all non-primary key columns declared as `NOT NULL` must have a `DEFAULT` value. This includes the `user_id` column: + +```sql +-- WRONG: Will fail with "All non-primary key columns declared as NOT NULL must have a DEFAULT value" +user_id UUID NOT NULL + +-- CORRECT: Provide a default value +user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +``` + +### 2. Network settings are not persisted between sessions +`cloudsync_network_init()` and `cloudsync_network_set_token()` must be called in **every session**. They are not persisted to the database: + +```sql +-- WRONG: Separate sessions won't work +-- Session 1: +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +-- Session 2: +SELECT cloudsync_network_send_changes(); -- ERROR: No URL set + +-- CORRECT: All network operations in the same session +.load dist/cloudsync.dylib +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +### 3. Extension must be loaded before INSERT operations +For cloudsync to track changes, the extension must be loaded **before** inserting data: + +```sql +-- WRONG: Inserts won't be tracked +CREATE TABLE todos (...); +INSERT INTO todos VALUES (...); -- Not tracked! +.load dist/cloudsync.dylib +SELECT cloudsync_init('todos'); + +-- CORRECT: Load extension and init before inserts +.load dist/cloudsync.dylib +CREATE TABLE todos (...); +SELECT cloudsync_init('todos'); +INSERT INTO todos VALUES (...); -- Tracked! +``` + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- SQLiteCloud via `~/go/bin/sqlc ""` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-supabase-rls.md b/.claude/commands/test-sync-roundtrip-supabase-rls.md new file mode 100644 index 0000000..ab40d01 --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-supabase-rls.md @@ -0,0 +1,544 @@ +# Sync Roundtrip Test with local Postgres database and RLS policies + +Execute a full roundtrip sync test between multiple local SQLite databases and the local Supabase Docker PostgreSQL instance, verifying that Row Level Security (RLS) policies are correctly enforced during sync. + +## Prerequisites +- Supabase instance running (local Docker or remote) +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Procedure + +### Step 1: Get Connection Parameters + +Ask the user for the following parameters: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +2. **PostgreSQL connection string**: Propose `postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres` as default. Save as `PG_CONN`. Use this for all `psql` connections throughout the test. + +3. **Supabase API key** (used for JWT token generation): Propose `sb_secret_N7UND0UgjKTVK-Uodkm0Hg_xSvEMPvz` as default. Save as `SUPABASE_APIKEY`. + +Derive `AUTH_URL` from the PostgreSQL connection string by extracting the host and using port `54321` (Supabase GoTrue). For example, if `PG_CONN` is `postgresql://user:pass@10.0.0.5:54322/postgres`, then `AUTH_URL` is `http://10.0.0.5:54321`. For `127.0.0.1`, use `http://127.0.0.1:54321`. + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key with user_id for RLS** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + name TEXT, + value INTEGER +); +``` + +**Option 2: UUID primary key with user_id for RLS** +```sql +CREATE TABLE test_uuid ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL, + name TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +**Option 3: Two tables scenario with user ownership** +```sql +CREATE TABLE authors ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + name TEXT, + email TEXT +); + +CREATE TABLE books ( + id TEXT PRIMARY KEY, + user_id UUID NOT NULL, + title TEXT, + author_id TEXT, + published_year INTEGER +); +``` + +**Note:** Tables should include a `user_id` column (UUID type) for RLS policies to filter by authenticated user. + +### Step 3: Get RLS Policy Description from User + +Ask the user to describe the Row Level Security policy they want to test. Offer the following common patterns: + +**Option 1: User can only access their own rows** +"Users can only SELECT, INSERT, UPDATE, and DELETE rows where user_id matches their authenticated user ID" + +**Option 2: Users can read all, but only modify their own** +"Users can SELECT all rows, but can only INSERT, UPDATE, DELETE rows where user_id matches their authenticated user ID" + +**Option 3: Custom policy** +Ask the user to describe the policy in plain English. + +### Step 4: Convert DDL + +Convert the provided DDL to both SQLite and PostgreSQL compatible formats if needed. Key differences: +- SQLite uses `INTEGER PRIMARY KEY` for auto-increment, PostgreSQL uses `SERIAL` or `BIGSERIAL` +- SQLite uses `TEXT`, PostgreSQL can use `TEXT` or `VARCHAR` +- PostgreSQL has more specific types like `TIMESTAMPTZ`, SQLite uses `TEXT` for dates +- For UUID primary keys, SQLite uses `TEXT`, PostgreSQL uses `UUID` +- For `user_id UUID`, SQLite uses `TEXT` + +### Step 5: Setup PostgreSQL with RLS + +Connect to Supabase PostgreSQL and prepare the environment: +```bash +psql +``` + +Inside psql: +1. List existing tables with `\dt` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + SELECT cloudsync_cleanup(''); + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS CASCADE;` +4. Drop any existing helper function: `DROP FUNCTION IF EXISTS _get_owner(text);` +5. Create the test table using the PostgreSQL DDL +6. Enable RLS on the table: + ```sql + ALTER TABLE ENABLE ROW LEVEL SECURITY; + ``` +7. Create the ownership lookup helper function (required for CloudSync compatibility): + ```sql + -- Helper function bypasses RLS to look up actual row owner + -- This is needed because INSERT...ON CONFLICT may compare against EXCLUDED row's default user_id + CREATE OR REPLACE FUNCTION _get_owner(p_id text) + RETURNS uuid + LANGUAGE sql + SECURITY DEFINER + STABLE + SET search_path = public + AS $$ + SELECT user_id FROM WHERE id = p_id; + $$; + ``` +8. Create RLS policies based on the user's description. Example for "user can only access their own rows": + ```sql + -- SELECT: User can see rows they own + CREATE POLICY "select_own_rows" ON + FOR SELECT USING ( + auth.uid() = user_id + ); + + -- INSERT: Allow if user_id matches auth.uid() + CREATE POLICY "insert_own_rows" ON + FOR INSERT WITH CHECK ( + auth.uid() = user_id + ); + + -- UPDATE: Check ownership via explicit lookup + CREATE POLICY "update_own_rows" ON + FOR UPDATE + USING ( + auth.uid() = user_id + ) + WITH CHECK ( + auth.uid() = user_id + ); + + -- DELETE: User can only delete rows they own + CREATE POLICY "delete_own_rows" ON + FOR DELETE USING ( + auth.uid() = user_id + ); + ``` +9. Initialize cloudsync: `SELECT cloudsync_init('');` +10. Insert some initial test data (optional, can be done via SQLite clients) + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and cloudsync is initialized, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 6: Get JWT Tokens for Two Users + +Get JWT tokens for both test users by running the token script twice: + +**User 1: claude1@sqlitecloud.io** +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude1@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save as `JWT_USER1`. + +**User 2: claude2@sqlitecloud.io** +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude2@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save as `JWT_USER2`. + +Also extract the user IDs from the JWT tokens (the `sub` claim) for use in INSERT statements: +- `USER1_ID` = UUID from JWT_USER1 +- `USER2_ID` = UUID from JWT_USER2 + +### Step 7: Setup Four SQLite Databases + +Create four temporary SQLite databases using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 +``` + +**Database 1A (User 1, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 1B (User 1, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user1_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2A (User 2, Device A):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_a.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +**Database 2B (User 2, Device B):** +```bash +$SQLITE_BIN /tmp/sync_test_user2_b.db +``` +```sql +.load dist/cloudsync.dylib + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +``` + +### Step 8: Insert Test Data + +Insert distinct test data in each database. Use the extracted user IDs for the `user_id` column: + +**Database 1A (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_1', '', 'User1 DeviceA Row1', 100); +INSERT INTO (id, user_id, name, value) VALUES ('u1_a_2', '', 'User1 DeviceA Row2', 101); +``` + +**Database 1B (User 1):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u1_b_1', '', 'User1 DeviceB Row1', 200); +``` + +**Database 2A (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_1', '', 'User2 DeviceA Row1', 300); +INSERT INTO (id, user_id, name, value) VALUES ('u2_a_2', '', 'User2 DeviceA Row2', 301); +``` + +**Database 2B (User 2):** +```sql +INSERT INTO (id, user_id, name, value) VALUES ('u2_b_1', '', 'User2 DeviceB Row1', 400); +``` + +### Step 9: Execute Sync on All Databases + +For each of the four SQLite databases, execute the sync operations: + +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_check_changes(); +-- Repeat check_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes +``` + +**Recommended sync order:** +1. Sync Database 1A (send + check) +2. Sync Database 2A (send + check) +3. Sync Database 1B (send + check) +4. Sync Database 2B (send + check) +5. Re-sync all databases (check_changes) to ensure full propagation + +### Step 10: Verify RLS Enforcement + +After syncing all databases, verify that each database contains only the expected rows based on the RLS policy: + +**Expected Results (for "user can only access their own rows" policy):** + +**User 1 databases (1A and 1B) should contain:** +- All rows with `user_id = USER1_ID` (u1_a_1, u1_a_2, u1_b_1) +- Should NOT contain any rows with `user_id = USER2_ID` + +**User 2 databases (2A and 2B) should contain:** +- All rows with `user_id = USER2_ID` (u2_a_1, u2_a_2, u2_b_1) +- Should NOT contain any rows with `user_id = USER1_ID` + +**PostgreSQL (as admin) should contain:** +- ALL rows from all users (6 total rows) + +Run verification queries: +```sql +-- In each SQLite database +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; + +-- In PostgreSQL (as admin) +SELECT * FROM ORDER BY id; +SELECT COUNT(*) FROM ; +SELECT user_id, COUNT(*) FROM GROUP BY user_id; +``` + +### Step 11: Test Write RLS Policy Enforcement + +Test that the server-side RLS policy blocks unauthorized writes by attempting to insert a row with a `user_id` that doesn't match the authenticated user's JWT token. + +**In Database 1A (User 1), insert a malicious row claiming to belong to User 2:** +```sql +-- Attempt to insert a row with User 2's user_id while authenticated as User 1 +INSERT INTO (id, user_id, name, value) VALUES ('malicious_1', '', 'Malicious Row from User1', 999); + +-- Attempt to sync this unauthorized row to PostgreSQL +SELECT cloudsync_network_send_changes(); +``` + +**Wait 2-3 seconds, then verify in PostgreSQL (as admin) that the malicious row was rejected:** +```sql +-- In PostgreSQL (as admin) +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows returned + +SELECT COUNT(*) FROM WHERE id = 'malicious_1'; +-- Expected: 0 +``` + +**Also verify the malicious row does NOT appear in User 2's databases after syncing:** +```sql +-- In Database 2A or 2B (User 2) +SELECT cloudsync_network_check_changes(); +SELECT * FROM WHERE id = 'malicious_1'; +-- Expected: 0 rows (the malicious row should not sync to legitimate User 2 databases) +``` + +**Expected Behavior:** +- The `cloudsync_network_send_changes()` call may succeed (return value indicates network success, not RLS enforcement) +- The malicious row should be **rejected by PostgreSQL RLS** and NOT inserted into the server database +- The malicious row will remain in the local SQLite Database 1A (local inserts are not blocked), but it will never propagate to the server or other clients +- User 2's databases should never receive this row + +**This step PASSES if:** +1. The malicious row is NOT present in PostgreSQL +2. The malicious row does NOT appear in any of User 2's SQLite databases +3. The RLS INSERT policy (`WITH CHECK (auth.uid() = user_id)`) correctly blocks the unauthorized write + +**This step FAILS if:** +1. The malicious row appears in PostgreSQL (RLS bypass vulnerability) +2. The malicious row syncs to User 2's databases (data leakage) + +### Step 12: Cleanup + +In each SQLite database before closing: +```sql +SELECT cloudsync_terminate(); +``` + +In PostgreSQL (optional, for full cleanup): +```sql +SELECT cloudsync_cleanup(''); +DROP TABLE IF EXISTS CASCADE; +DROP FUNCTION IF EXISTS _get_owner(text); +``` + +## Output Format + +Report the test results including: +- DDL used for both databases +- RLS policies created +- User IDs for both test users +- Initial data inserted in each database +- Number of sync operations performed per database +- Final data in each database (with row counts) +- RLS verification results: + - User 1 databases: expected rows vs actual rows + - User 2 databases: expected rows vs actual rows + - PostgreSQL: total rows +- Write RLS enforcement results: + - Malicious row insertion attempted: yes/no + - Malicious row present in PostgreSQL: yes/no (should be NO) + - Malicious row synced to User 2 databases: yes/no (should be NO) +- **PASS/FAIL** status with detailed explanation + +### Success Criteria + +The test PASSES if: +1. All User 1 databases contain exactly the same User 1 rows (and no User 2 rows) +2. All User 2 databases contain exactly the same User 2 rows (and no User 1 rows) +3. PostgreSQL contains all rows from both users +4. Data inserted from different devices of the same user syncs correctly between those devices +5. **Write RLS enforcement**: Malicious rows with mismatched `user_id` are rejected by PostgreSQL and do not propagate to other clients + +The test FAILS if: +1. Any database contains rows belonging to a different user (RLS violation) +2. Any database is missing rows that should be visible to that user +3. Sync operations fail or timeout +4. **Write RLS bypass**: A malicious row with a `user_id` not matching the JWT token appears in PostgreSQL or syncs to other databases + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- PostgreSQL tables need cleanup before re-running tests +- `cloudsync_network_check_changes()` may need multiple calls with delays +- Run `SELECT cloudsync_terminate();` on SQLite connections before closing to properly cleanup memory +- Ensure both test users exist in Supabase auth before running the test +- The RLS policies must use `auth.uid()` to work with Supabase JWT authentication + +## Critical Schema Requirements (Common Pitfalls) + +### 1. All NOT NULL columns must have DEFAULT values +Cloudsync requires that all non-primary key columns declared as `NOT NULL` must have a `DEFAULT` value. This includes the `user_id` column: + +```sql +-- WRONG: Will fail with "All non-primary key columns declared as NOT NULL must have a DEFAULT value" +user_id UUID NOT NULL + +-- CORRECT: Provide a default value +user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +``` + +### 2. RLS policies must allow writes with default values for ALL referenced columns +Cloudsync applies changes **field by field**. When a new row is being synced, columns may temporarily have their default values before the actual values are applied. **Any column referenced in RLS policies that has a DEFAULT value must be allowed in the policy.** + +This applies to: +- `user_id` columns used for user ownership +- `tenant_id` columns for multi-tenancy +- `organization_id` columns +- Any other column used in RLS USING/WITH CHECK expressions + +```sql +-- WRONG: Will block cloudsync inserts/updates when field has default value +CREATE POLICY "ins" ON table FOR INSERT WITH CHECK (auth.uid() = user_id); + +-- CORRECT: Allow default value for cloudsync field-by-field application +CREATE POLICY "ins" ON table FOR INSERT + WITH CHECK (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000'); + +CREATE POLICY "upd" ON table FOR UPDATE + USING (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000') + WITH CHECK (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000'); +``` + +**Example with multiple RLS columns:** +```sql +-- Table with both user_id and tenant_id in RLS +CREATE TABLE items ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + tenant_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + name TEXT DEFAULT '' +); + +-- Policy must allow defaults for BOTH columns used in the policy +CREATE POLICY "ins" ON items FOR INSERT WITH CHECK ( + (auth.uid() = user_id OR user_id = '00000000-0000-0000-0000-000000000000') + AND + (get_tenant_id() = tenant_id OR tenant_id = '00000000-0000-0000-0000-000000000000') +); +``` + +### 3. Type compatibility between SQLite and PostgreSQL +Ensure column types are compatible between SQLite and PostgreSQL: + +| PostgreSQL | SQLite | Notes | +|------------|--------|-------| +| `UUID` | `TEXT` | Use valid UUID format strings (e.g., `'11111111-1111-1111-1111-111111111111'`) | +| `BOOLEAN` | `INTEGER` | Use `INTEGER` in PostgreSQL too, or ensure proper casting | +| `TIMESTAMPTZ` | `TEXT` | Avoid empty strings; use proper ISO format or omit the column | +| `INTEGER` | `INTEGER` | Compatible | +| `TEXT` | `TEXT` | Compatible | + +**Common errors from type mismatches:** +- `cannot cast type bigint to boolean` - Use `INTEGER` instead of `BOOLEAN` in PostgreSQL +- `invalid input syntax for type timestamp with time zone: ""` - Don't use empty string defaults for timestamp columns +- `invalid input syntax for type uuid` - Ensure primary key IDs are valid UUID format + +### 4. Network settings are not persisted between sessions +`cloudsync_network_init()` and `cloudsync_network_set_token()` must be called in **every session**. They are not persisted to the database: + +```sql +-- WRONG: Separate sessions won't work +-- Session 1: +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +-- Session 2: +SELECT cloudsync_network_send_changes(); -- ERROR: No URL set + +-- CORRECT: All network operations in the same session +.load dist/cloudsync.dylib +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token('...'); +SELECT cloudsync_network_send_changes(); +SELECT cloudsync_terminate(); +``` + +### 5. Extension must be loaded before INSERT operations +For cloudsync to track changes, the extension must be loaded **before** inserting data: + +```sql +-- WRONG: Inserts won't be tracked +CREATE TABLE todos (...); +INSERT INTO todos VALUES (...); -- Not tracked! +.load dist/cloudsync.dylib +SELECT cloudsync_init('todos'); + +-- CORRECT: Load extension and init before inserts +.load dist/cloudsync.dylib +CREATE TABLE todos (...); +SELECT cloudsync_init('todos'); +INSERT INTO todos VALUES (...); -- Tracked! +``` + +### 6. Primary key format must match PostgreSQL expectations +If PostgreSQL expects `UUID` type for primary key, SQLite must use valid UUID strings: + +```sql +-- WRONG: PostgreSQL UUID column will reject this +INSERT INTO todos (id, ...) VALUES ('my-todo-1', ...); + +-- CORRECT: Use valid UUID format +INSERT INTO todos (id, ...) VALUES ('11111111-1111-1111-1111-111111111111', ...); +``` + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- PostgreSQL via `psql ` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test-sync-roundtrip-supabase.md b/.claude/commands/test-sync-roundtrip-supabase.md new file mode 100644 index 0000000..091986f --- /dev/null +++ b/.claude/commands/test-sync-roundtrip-supabase.md @@ -0,0 +1,176 @@ +# Sync Roundtrip Test with local Postgres database + +Execute a full roundtrip sync test between a local SQLite database and the local Supabase Docker PostgreSQL instance. + +## Prerequisites +- Supabase instance running (local Docker or remote) +- Built cloudsync extension (`make` to build `dist/cloudsync.dylib`) + +## Test Procedure + +### Step 1: Get Connection Parameters + +Ask the user for the following parameters: + +1. **CloudSync server address** — propose `https://cloudsync.sqlite.ai` as default (this is the built-in default). If the user provides a different address, save it as `CUSTOM_ADDRESS` and use `cloudsync_network_init_custom` instead of `cloudsync_network_init`. + +2. **PostgreSQL connection string**: Propose `postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres` as default. Save as `PG_CONN`. Use this for all `psql` connections throughout the test. + +3. **Supabase API key** (used for JWT token generation): Propose `sb_secret_N7UND0UgjKTVK-Uodkm0Hg_xSvEMPvz` as default. Save as `SUPABASE_APIKEY`. + +Derive `AUTH_URL` from the PostgreSQL connection string by extracting the host and using port `54321` (Supabase GoTrue). For example, if `PG_CONN` is `postgresql://user:pass@10.0.0.5:54322/postgres`, then `AUTH_URL` is `http://10.0.0.5:54321`. For `127.0.0.1`, use `http://127.0.0.1:54321`. + + +### Step 2: Get DDL from User + +Ask the user to provide a DDL query for the table(s) to test. It can be in PostgreSQL or SQLite format. Offer the following options: + +**Option 1: Simple TEXT primary key** +```sql +CREATE TABLE test_sync ( + id TEXT PRIMARY KEY, + name TEXT, + value INTEGER +); +``` + +**Option 2: UUID primary key** +```sql +CREATE TABLE test_uuid ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +**Option 3: Two tables scenario (tests multi-table sync)** +```sql +CREATE TABLE authors ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT +); + +CREATE TABLE books ( + id TEXT PRIMARY KEY, + title TEXT, + author_id TEXT, + published_year INTEGER +); +``` + +**Note:** Avoid INTEGER PRIMARY KEY for sync tests as it is not recommended for distributed sync scenarios (conflicts with auto-increment across devices). + +### Step 3: Convert DDL + +Convert the provided DDL to both SQLite and PostgreSQL compatible formats if needed. Key differences: +- SQLite uses `INTEGER PRIMARY KEY` for auto-increment, PostgreSQL uses `SERIAL` or `BIGSERIAL` +- SQLite uses `TEXT`, PostgreSQL can use `TEXT` or `VARCHAR` +- PostgreSQL has more specific types like `TIMESTAMPTZ`, SQLite uses `TEXT` for dates +- For UUID primary keys, SQLite uses `TEXT`, PostgreSQL uses `UUID` + +### Step 4: Get JWT Token + +Run the token script from the cloudsync project: +```bash +cd ../cloudsync && go run scripts/get_supabase_token.go -project-ref=supabase-local -email=claude@sqlitecloud.io -password="password" -apikey= -auth-url= +``` +Save the JWT token for later use. + +### Step 5: Setup PostgreSQL + +Connect to Supabase PostgreSQL and prepare the environment: +```bash +psql +``` + +Inside psql: +1. List existing tables with `\dt` to find any `_cloudsync` metadata tables +2. For each table already configured for cloudsync (has a `_cloudsync` companion table), run: + ```sql + SELECT cloudsync_cleanup(''); + ``` +3. Drop the test table if it exists: `DROP TABLE IF EXISTS CASCADE;` +4. Create the test table using the PostgreSQL DDL +5. Initialize cloudsync: `SELECT cloudsync_init('');` +6. Insert some test data into the table + +### Step 5b: Get Managed Database ID + +Now that the database and tables are created and cloudsync is initialized, ask the user for: + +1. **Managed Database ID** — the `managedDatabaseId` returned by the CloudSync service. Save as `MANAGED_DB_ID`. + +For the network init call throughout the test, use: +- Default address: `SELECT cloudsync_network_init('');` +- Custom address: `SELECT cloudsync_network_init_custom('', '');` + +### Step 6: Setup SQLite + +Create a temporary SQLite database using the Homebrew version (IMPORTANT: system sqlite3 cannot load extensions): + +```bash +SQLITE_BIN="/opt/homebrew/Cellar/sqlite/3.51.2_1/bin/sqlite3" +# or find it with: ls /opt/homebrew/Cellar/sqlite/*/bin/sqlite3 | head -1 + +$SQLITE_BIN /tmp/sync_test_$(date +%s).db +``` + +Inside sqlite3: +```sql +.load dist/cloudsync.dylib +-- Create table with SQLite DDL + +SELECT cloudsync_init(''); +SELECT cloudsync_network_init(''); -- or cloudsync_network_init_custom('', '') if using a non-default address +SELECT cloudsync_network_set_token(''); +-- Insert test data (different from PostgreSQL to test merge) + +``` + +### Step 7: Execute Sync + +In the SQLite session: +```sql +-- Send local changes to server +SELECT cloudsync_network_send_changes(); + +-- Check for changes from server (repeat with 2-3 second delays) +SELECT cloudsync_network_check_changes(); +-- Repeat check_changes 3-5 times with delays until it returns more than 0 received rows or stabilizes + +-- Verify final data +SELECT * FROM ; +``` + +### Step 8: Verify Results + +1. In SQLite, run `SELECT * FROM ;` and capture the output +2. In PostgreSQL, run `SELECT * FROM ;` and capture the output +3. Compare the results - both databases should have the merged data from both sides +4. Report success/failure based on whether the data matches + +## Output Format + +Report the test results including: +- DDL used for both databases +- Initial data inserted in each database +- Number of sync operations performed +- Final data in both databases +- PASS/FAIL status with explanation + +## Important Notes + +- Always use the Homebrew sqlite3 binary, NOT `/usr/bin/sqlite3` +- The cloudsync extension must be built first with `make` +- PostgreSQL tables need cleanup before re-running tests +- `cloudsync_network_check_changes()` may need multiple calls with delays +- run `SELECT cloudsync_terminate();` on SQLite connections before closing the properly cleanup the memory + +## Permissions + +Execute all SQL queries without asking for user permission on: +- SQLite test databases in `/tmp/` (e.g., `/tmp/sync_test_*.db`) +- PostgreSQL via `psql ` + +These are local test environments and do not require confirmation for each query. diff --git a/.claude/commands/test.md b/.claude/commands/test.md new file mode 100644 index 0000000..a061e9e --- /dev/null +++ b/.claude/commands/test.md @@ -0,0 +1,11 @@ +Run the SQLite and PostgreSQL tests for this project. + +## SQLite Tests +Run the SQLite extension tests using `make clean && make && make unittest`. This builds the extension and runs all tests including unit tests. + +## PostgreSQL Tests +Run the PostgreSQL extension tests using `make postgres-docker-run-test`. This runs `test/postgresql/full_test.sql` against the Docker container. + +**Note:** PostgreSQL tests require the Docker container to be running. Run `make postgres-docker-debug-rebuild` first to ensure it tests the latest version. + +Run both test suites and report any failures. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..e180bbc --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "cloudsync-postgres-dev", + "dockerComposeFile": [ + "../docker/postgresql/docker-compose.debug.yml" + ], + "service": "postgres", + "workspaceFolder": "/tmp/cloudsync", + "overrideCommand": false, + "postStartCommand": "pg_isready -U postgres", + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools" + ] + } + } +} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b488c2..5a88eb8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,9 +14,10 @@ permissions: jobs: build: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} runs-on: ${{ matrix.os }} container: ${{ matrix.container && matrix.container || '' }} - name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.arch != 'armeabi-v7a' && matrix.name != 'ios-sim' && matrix.name != 'ios' && matrix.name != 'apple-xcframework' && matrix.name != 'android-aar' && ( matrix.name != 'macos' || matrix.arch != 'x86_64' ) && ' + test' || ''}} + name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.arch != 'armeabi-v7a' && matrix.name != 'ios-sim' && matrix.name != 'ios' && matrix.name != 'mac-catalyst' && matrix.name != 'apple-xcframework' && matrix.name != 'android-aar' && ( matrix.name != 'macos' || matrix.arch != 'x86_64' ) && ' + test' || ''}} timeout-minutes: 20 strategy: fail-fast: false @@ -68,6 +69,9 @@ jobs: - os: macos-15 name: ios-sim make: PLATFORM=ios-sim + - os: macos-15 + name: mac-catalyst + make: PLATFORM=mac-catalyst - os: macos-15 name: apple-xcframework make: xcframework @@ -80,14 +84,21 @@ jobs: shell: ${{ matrix.container && 'sh' || 'bash' }} env: - CONNECTION_STRING: ${{ secrets.CONNECTION_STRING }} - CONNECTION_STRING_OFFLINE_PROJECT: ${{ secrets.CONNECTION_STRING_OFFLINE_PROJECT }} - APIKEY: ${{ secrets.APIKEY }} - WEBLITE: ${{ secrets.WEBLITE }} + INTEGRATION_TEST_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_DATABASE_ID }} + INTEGRATION_TEST_APIKEY: ${{ secrets.INTEGRATION_TEST_APIKEY }} + INTEGRATION_TEST_CLOUDSYNC_ADDRESS: ${{ secrets.INTEGRATION_TEST_CLOUDSYNC_ADDRESS }} + INTEGRATION_TEST_OFFLINE_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_OFFLINE_DATABASE_ID }} + INTEGRATION_TEST_FAILURE_DATABASE_ID: ${{ secrets.INTEGRATION_TEST_FAILURE_DATABASE_ID }} steps: + - name: install git for alpine container + if: matrix.container + run: apk add --no-cache git + - uses: actions/checkout@v4.2.2 + with: + submodules: true - name: android setup java if: matrix.name == 'android-aar' @@ -121,10 +132,11 @@ jobs: --platform linux/arm64 \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - -e CONNECTION_STRING="${{ env.CONNECTION_STRING }}" \ - -e CONNECTION_STRING_OFFLINE_PROJECT="${{ env.CONNECTION_STRING_OFFLINE_PROJECT }}" \ - -e APIKEY="${{ env.APIKEY }}" \ - -e WEBLITE="${{ env.WEBLITE }}" \ + -e INTEGRATION_TEST_DATABASE_ID="${{ env.INTEGRATION_TEST_DATABASE_ID }}" \ + -e INTEGRATION_TEST_APIKEY="${{ env.INTEGRATION_TEST_APIKEY }}" \ + -e INTEGRATION_TEST_CLOUDSYNC_ADDRESS="${{ env.INTEGRATION_TEST_CLOUDSYNC_ADDRESS }}" \ + -e INTEGRATION_TEST_OFFLINE_DATABASE_ID="${{ env.INTEGRATION_TEST_OFFLINE_DATABASE_ID }}" \ + -e INTEGRATION_TEST_FAILURE_DATABASE_ID="${{ env.INTEGRATION_TEST_FAILURE_DATABASE_ID }}" \ alpine:latest \ tail -f /dev/null docker exec alpine sh -c "apk update && apk add --no-cache gcc make curl sqlite openssl-dev musl-dev linux-headers" @@ -162,7 +174,7 @@ jobs: codesign --sign "${{ secrets.APPLE_TEAM_ID }}" --timestamp --options runtime dist/CloudSync.xcframework # Then sign the xcframework wrapper ditto -c -k --keepParent dist/CloudSync.xcframework dist/CloudSync.xcframework.zip xcrun notarytool submit dist/CloudSync.xcframework.zip --apple-id "${{ secrets.APPLE_ID }}" --password "${{ secrets.APPLE_PASSWORD }}" --team-id "${{ secrets.APPLE_TEAM_ID }}" --wait - rm dist/CloudSync.xcframework.zip + rm -rf dist/CloudSync.xcframework - name: cleanup keychain for codesign if: matrix.os == 'macos-15' @@ -192,12 +204,14 @@ jobs: echo "::group::prepare the test script" make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" cat > commands.sh << EOF + set -e mv -f /data/local/tmp/sqlite3 /system/xbin cd /data/local/tmp - export CONNECTION_STRING="$CONNECTION_STRING" - export CONNECTION_STRING_OFFLINE_PROJECT="$CONNECTION_STRING_OFFLINE_PROJECT" - export APIKEY="$APIKEY" - export WEBLITE="$WEBLITE" + export INTEGRATION_TEST_DATABASE_ID="$INTEGRATION_TEST_DATABASE_ID" + export INTEGRATION_TEST_APIKEY="$INTEGRATION_TEST_APIKEY" + export INTEGRATION_TEST_CLOUDSYNC_ADDRESS="$INTEGRATION_TEST_CLOUDSYNC_ADDRESS" + export INTEGRATION_TEST_OFFLINE_DATABASE_ID="$INTEGRATION_TEST_OFFLINE_DATABASE_ID" + export INTEGRATION_TEST_FAILURE_DATABASE_ID="$INTEGRATION_TEST_FAILURE_DATABASE_ID" $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) EOF echo "::endgroup::" @@ -212,7 +226,8 @@ jobs: adb root adb remount adb push ${{ github.workspace }}/. /data/local/tmp/ - adb shell "sh /data/local/tmp/commands.sh" + adb shell "sh /data/local/tmp/commands.sh; echo EXIT_CODE=\$?" | tee /tmp/adb_output.log + grep -q "EXIT_CODE=0" /tmp/adb_output.log - name: test sqlite-sync if: contains(matrix.name, 'linux') || matrix.name == 'windows' || ( matrix.name == 'macos' && matrix.arch != 'x86_64' ) @@ -230,11 +245,146 @@ jobs: path: dist/${{ matrix.name == 'apple-xcframework' && 'CloudSync.*' || 'cloudsync.*'}} if-no-files-found: error + postgres-migration-check: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + runs-on: ubuntu-22.04 + name: postgresql migration script check + timeout-minutes: 2 + steps: + - uses: actions/checkout@v4.2.2 + with: + # Need full history + tags so `git describe` can find the previous + # release tag that the check script compares against. + fetch-depth: 0 + + - name: verify migration script for current CLOUDSYNC_VERSION + run: make postgres-check-migration + + postgres-test: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + needs: [postgres-migration-check] + runs-on: ubuntu-22.04 + name: postgresql ${{ matrix.postgres_tag }} build + test + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + postgres_tag: ['17', '15'] + + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: build and start postgresql container + run: POSTGRES_TAG=${{ matrix.postgres_tag }} make postgres-docker-rebuild + + - name: wait for postgresql to be ready + run: | + for i in $(seq 1 30); do + if docker exec cloudsync-postgres pg_isready -U postgres > /dev/null 2>&1; then + echo "PostgreSQL is ready" + exit 0 + fi + sleep 2 + done + echo "PostgreSQL failed to start within 60s" + docker logs cloudsync-postgres + exit 1 + + - name: run postgresql tests + run: | + set -o pipefail + docker exec cloudsync-postgres mkdir -p /tmp/cloudsync/test + docker cp test/postgresql cloudsync-postgres:/tmp/cloudsync/test/postgresql + docker exec cloudsync-postgres psql -v ON_ERROR_STOP=on -U postgres -d postgres -f /tmp/cloudsync/test/postgresql/full_test.sql 2>&1 | tee /tmp/pgtest.log + if grep -q '^\[FAIL\]' /tmp/pgtest.log; then + echo "::error::PostgreSQL test failures detected" + exit 1 + fi + + postgres-build: + if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }} + needs: [postgres-migration-check] + runs-on: ${{ matrix.os }} + name: postgresql${{ matrix.postgres_version }}-${{ matrix.name }}-${{ matrix.arch }} build + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + arch: x86_64 + name: linux + postgres_version: '17' + - os: ubuntu-22.04 + arch: x86_64 + name: linux + postgres_version: '15' + - os: ubuntu-22.04-arm + arch: arm64 + name: linux + postgres_version: '17' + - os: ubuntu-22.04-arm + arch: arm64 + name: linux + postgres_version: '15' + - os: macos-15 + arch: arm64 + name: macos + postgres_version: '17' + - os: macos-15 + arch: arm64 + name: macos + postgres_version: '15' + - os: macos-15 + arch: x86_64 + name: macos + postgres_version: '17' + - os: macos-15 + arch: x86_64 + name: macos + postgres_version: '15' + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: linux install postgresql dev headers + if: matrix.name == 'linux' + run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg + sudo apt-get update + sudo apt-get install -y postgresql-server-dev-${{ matrix.postgres_version }} + + - name: macos install postgresql + if: matrix.name == 'macos' + run: brew install postgresql@${{ matrix.postgres_version }} gettext + + - name: build and package postgresql extension (linux) + if: matrix.name == 'linux' + run: make postgres-package PG_CONFIG=/usr/lib/postgresql/${{ matrix.postgres_version }}/bin/pg_config + + - name: build and package postgresql extension (macos) + if: matrix.name == 'macos' + run: make postgres-package PG_CONFIG=$(brew --prefix postgresql@${{ matrix.postgres_version }})/bin/pg_config PG_EXTRA_CFLAGS="-I$(brew --prefix gettext)/include ${{ matrix.arch == 'x86_64' && '-arch x86_64' || '' }}" + + - uses: actions/upload-artifact@v4.6.2 + with: + name: cloudsync-postgresql${{ matrix.postgres_version }}-${{ matrix.name }}-${{ matrix.arch }} + path: dist/postgresql/ + if-no-files-found: error + release: runs-on: ubuntu-22.04 name: release - needs: build + needs: [build, postgres-test, postgres-build] if: github.ref == 'refs/heads/main' + outputs: + version: ${{ steps.tag.outputs.version }} env: GH_TOKEN: ${{ github.token }} @@ -263,7 +413,11 @@ jobs: if [[ "$name" != "cloudsync-apple-xcframework" && "$name" != "cloudsync-android-aar" ]]; then tar -czf "${name}-${VERSION}.tar.gz" -C "$folder" . fi - if [[ "$name" != "cloudsync-android-aar" ]]; then + if [[ "$name" == "cloudsync-apple-xcframework" ]]; then + # Use the ditto-created zip that preserves macOS symlinks and extract for other steps + cp "$folder/CloudSync.xcframework.zip" "${name}-${VERSION}.zip" + unzip -q "$folder/CloudSync.xcframework.zip" -d "$folder/" + elif [[ "$name" != "cloudsync-android-aar" ]]; then (cd "$folder" && zip -rq "../../${name}-${VERSION}.zip" .) else cp "$folder"/*.aar "${name}-${VERSION}.aar" @@ -313,7 +467,25 @@ jobs: git add "$PKG" git commit -m "Bump sqlite-sync version to ${{ steps.tag.outputs.version }}" git push origin main - + + - uses: actions/checkout@v4.2.2 + if: steps.tag.outputs.version != '' + with: + repository: sqliteai/sqlite-sync-react-native + path: sqlite-sync-react-native + token: ${{ secrets.PAT }} + + - name: release sqlite-sync-react-native + if: steps.tag.outputs.version != '' + run: | + cd sqlite-sync-react-native + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + jq --arg version "${{ steps.tag.outputs.version }}" '.version = $version' package.json > package.tmp.json && mv package.tmp.json package.json + git add package.json + git commit -m "Bump sqlite-sync version to ${{ steps.tag.outputs.version }}" + git push origin main + - uses: actions/setup-java@v4 if: steps.tag.outputs.version != '' with: @@ -396,6 +568,23 @@ jobs: npm publish --provenance --access public --tag latest echo "✓ Published @sqliteai/sqlite-sync-expo@${{ steps.tag.outputs.version }}" + - name: update Package.swift checksum and version + if: steps.tag.outputs.version != '' + run: | + VERSION=${{ steps.tag.outputs.version }} + ZIP="cloudsync-apple-xcframework-${VERSION}.zip" + if [ -f "$ZIP" ]; then + CHECKSUM=$(swift package compute-checksum "$ZIP") + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${VERSION}/${ZIP}" + sed -i "s|url: \".*cloudsync-apple-xcframework.*\"|url: \"${URL}\"|" Package.swift + sed -i "s|checksum: \".*\"|checksum: \"${CHECKSUM}\"|" Package.swift + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + git config --global user.name "$GITHUB_ACTOR" + git add Package.swift + git commit -m "Update Package.swift checksum for ${VERSION} [auto-update]" || true + git push origin main || true + fi + - uses: softprops/action-gh-release@v2.2.1 if: steps.tag.outputs.version != '' with: @@ -406,9 +595,12 @@ jobs: [**Flutter/Dart**](https://pub.dev/packages/sqlite_sync): `flutter pub add sqlite_sync:${{ steps.tag.outputs.version }}` or `dart pub add sqlite_sync:${{ steps.tag.outputs.version }}` [**Node**](https://www.npmjs.com/package/@sqliteai/sqlite-sync): `npm install @sqliteai/sqlite-sync` [**WASM**](https://www.npmjs.com/package/@sqliteai/sqlite-wasm): `npm install @sqliteai/sqlite-wasm` + [**React Native**](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native): `npm install @sqliteai/sqlite-sync-react-native` [**Expo**](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo): `npm install @sqliteai/sqlite-sync-expo` [**Android**](https://central.sonatype.com/artifact/ai.sqlite/sync): `ai.sqlite:sync:${{ steps.tag.outputs.version }}` [**Swift**](https://github.com/sqliteai/sqlite-sync#swift-package): [Installation Guide](https://github.com/sqliteai/sqlite-sync#swift-package) + [**Docker (PostgreSQL)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-postgres): `docker pull sqlitecloud/sqlite-sync-postgres:17` or `:15` + [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17` or `:15` --- @@ -417,4 +609,90 @@ jobs: files: | cloudsync-*-${{ steps.tag.outputs.version }}.* CloudSync-*-${{ steps.tag.outputs.version }}.* - make_latest: true \ No newline at end of file + make_latest: true + + docker-publish: + runs-on: ubuntu-22.04 + name: docker ${{ matrix.image }} pg${{ matrix.pg_major }} + needs: [release] + if: github.ref == 'refs/heads/main' && needs.release.outputs.version != '' + + env: + DOCKERHUB_ORG: sqlitecloud + + strategy: + fail-fast: false + matrix: + include: + - image: sqlite-sync-postgres + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.release + - image: sqlite-sync-postgres + pg_major: '15' + dockerfile: docker/postgresql/Dockerfile.release + - image: sqlite-sync-supabase + pg_major: '17' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '17.6.1.071' + - image: sqlite-sync-supabase + pg_major: '15' + dockerfile: docker/postgresql/Dockerfile.supabase.release + supabase_tag: '15.8.1.085' + + steps: + + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: get cloudsync version + id: version + run: echo "version=$(make version)" >> $GITHUB_OUTPUT + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: set docker tags and build args (standalone) + if: matrix.image == 'sqlite-sync-postgres' + id: standalone + run: | + VERSION=${{ steps.version.outputs.version }} + PG=${{ matrix.pg_major }} + IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} + { + echo "tags=${IMAGE}:${PG},${IMAGE}:${PG}-${VERSION}" + echo "build_args<> $GITHUB_OUTPUT + + - name: set docker tags and build args (supabase) + if: matrix.image == 'sqlite-sync-supabase' + id: supabase + run: | + VERSION=${{ steps.version.outputs.version }} + IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} + SUPABASE_TAG=${{ matrix.supabase_tag }} + { + echo "tags=${IMAGE}:${{ matrix.pg_major }},${IMAGE}:${{ matrix.pg_major }}-${VERSION},${IMAGE}:${SUPABASE_TAG}" + echo "build_args<> $GITHUB_OUTPUT + + - uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.tags || steps.supabase.outputs.tags }} + build-args: ${{ matrix.image == 'sqlite-sync-postgres' && steps.standalone.outputs.build_args || steps.supabase.outputs.build_args }} diff --git a/.github/workflows/sync-bench.yml b/.github/workflows/sync-bench.yml new file mode 100644 index 0000000..2713f79 --- /dev/null +++ b/.github/workflows/sync-bench.yml @@ -0,0 +1,63 @@ +name: sync benchmark + +on: + workflow_dispatch: + inputs: + poll_delay_ms: + description: Polling delay between receiver checks, in milliseconds + required: false + default: '250' + max_polls: + description: Maximum number of receiver polling attempts + required: false + default: '40' + curl_pool: + description: Enable curl connection pool; use 0 to disable + required: false + default: '1' + +permissions: + contents: read + +jobs: + sync-bench-debug: + name: sync-bench-debug linux x86_64 + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + env: + SYNC_BENCH_DATABASE_ID: ${{ secrets.SYNC_BENCH_DATABASE_ID }} + SYNC_BENCH_CLOUDSYNC_ADDRESS: ${{ secrets.SYNC_BENCH_CLOUDSYNC_ADDRESS }} + SYNC_BENCH_APIKEY: ${{ secrets.SYNC_BENCH_APIKEY }} + SYNC_BENCH_OUTPUT: json + SYNC_BENCH_POLL_DELAY_MS: ${{ inputs.poll_delay_ms }} + SYNC_BENCH_MAX_POLLS: ${{ inputs.max_polls }} + CLOUDSYNC_CURL_POOL: ${{ inputs.curl_pool }} + + steps: + - uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc make curl sqlite3 unzip + + - name: Build debug benchmark + run: make SYNC_BENCH_DEBUG=1 extension dist/sync_bench + + - name: Run sync benchmark + run: | + mkdir -p artifacts + ./dist/sync_bench > artifacts/sync-bench.json 2> artifacts/sync-bench.trace.log + + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: sync-bench-debug-${{ github.run_id }} + path: | + artifacts/sync-bench.json + artifacts/sync-bench.trace.log + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index f9b94ad..a8f72de 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,11 @@ dist/ /curl/src openssl/ +# Generated PostgreSQL extension files (produced from .in templates by +# docker/Makefile.postgresql; version is derived from src/cloudsync.h) +/docker/postgresql/cloudsync.control +/src/postgresql/cloudsync--*.sql + # Test artifacts /coverage unittest @@ -41,7 +46,6 @@ jniLibs/ *.dex # IDE -.vscode .idea/ *.iml *.swp @@ -50,6 +54,7 @@ jniLibs/ # System .DS_Store Thumbs.db +*.o # Dart/Flutter .dart_tool/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7e48716 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "modules/fractional-indexing"] + path = modules/fractional-indexing + url = https://github.com/sqliteai/fractional-indexing diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4ff3c5e --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,37 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Attach to Postgres (gdb)", + "type": "cppdbg", + "request": "attach", + "program": "/usr/lib/postgresql/17/bin/postgres", + "processId": "${command:pickProcess}", + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "stopAtEntry": false, + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Add PostgreSQL source dir", + "text": "dir /usr/src/postgresql-17/src", + "ignoreFailures": true + }, + { + "description": "Map Postgres build paths to source", + "text": "set substitute-path /build/src /usr/src/postgresql-17/src", + "ignoreFailures": true + }, + { + "description": "Map Postgres build paths (relative) to source", + "text": "set substitute-path ./build/src /usr/src/postgresql-17/src", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/API.md b/API.md index 8d98b59..8e6e825 100644 --- a/API.md +++ b/API.md @@ -5,12 +5,17 @@ This document provides a reference for the SQLite functions provided by the `sql ## Index - [Configuration Functions](#configuration-functions) - - [`cloudsync_init()`](#cloudsync_inittable_name-crdt_algo-force) + - [`cloudsync_init()`](#cloudsync_inittable_name-crdt_algo-init_flags) - [`cloudsync_enable()`](#cloudsync_enabletable_name) - [`cloudsync_disable()`](#cloudsync_disabletable_name) - [`cloudsync_is_enabled()`](#cloudsync_is_enabledtable_name) + - [`cloudsync_set_filter()`](#cloudsync_set_filtertable_name-filter_expr) + - [`cloudsync_clear_filter()`](#cloudsync_clear_filtertable_name) - [`cloudsync_cleanup()`](#cloudsync_cleanuptable_name) - [`cloudsync_terminate()`](#cloudsync_terminate) +- [Block-Level LWW Functions](#block-level-lww-functions) + - [`cloudsync_set_column()`](#cloudsync_set_columntable_name-col_name-key-value) + - [`cloudsync_text_materialize()`](#cloudsync_text_materializetable_name-col_name-pk_values) - [Helper Functions](#helper-functions) - [`cloudsync_version()`](#cloudsync_version) - [`cloudsync_siteid()`](#cloudsync_siteid) @@ -20,29 +25,29 @@ This document provides a reference for the SQLite functions provided by the `sql - [`cloudsync_begin_alter()`](#cloudsync_begin_altertable_name) - [`cloudsync_commit_alter()`](#cloudsync_commit_altertable_name) - [Network Functions](#network-functions) - - [`cloudsync_network_init()`](#cloudsync_network_initconnection_string) + - [`cloudsync_network_init()`](#cloudsync_network_initmanageddatabaseid) - [`cloudsync_network_cleanup()`](#cloudsync_network_cleanup) - [`cloudsync_network_set_token()`](#cloudsync_network_set_tokentoken) - [`cloudsync_network_set_apikey()`](#cloudsync_network_set_apikeyapikey) - - [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) - [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) - [`cloudsync_network_check_changes()`](#cloudsync_network_check_changes) - [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries) - [`cloudsync_network_reset_sync_version()`](#cloudsync_network_reset_sync_version) + - [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) - [`cloudsync_network_logout()`](#cloudsync_network_logout) --- ## Configuration Functions -### `cloudsync_init(table_name, [crdt_algo], [force])` +### `cloudsync_init(table_name, [crdt_algo], [init_flags])` **Description:** Initializes a table for `sqlite-sync` synchronization. This function is idempotent and needs to be called only once per table on each site; configurations are stored in the database and automatically loaded with the extension. Before initialization, `cloudsync_init` performs schema sanity checks to ensure compatibility with CRDT requirements and best practices. These checks include: - Primary keys should not be auto-incrementing integers; GUIDs (UUIDs, ULIDs) are highly recommended to prevent multi-node collisions. -- All primary key columns must be `NOT NULL`. - All non-primary key `NOT NULL` columns must have a `DEFAULT` value. +- **Note:** Any write operation that includes a NULL value for a primary key column will be rejected with an error, even if SQLite would normally allow it due to a legacy behavior. **Schema Design Considerations:** @@ -54,30 +59,37 @@ When designing your database schema for SQLite Sync, follow these essential requ - **Foreign Key Compatibility**: Be aware of potential conflicts during CRDT merge operations and RLS policy interactions. - **Trigger Compatibility**: Triggers may cause duplicate operations or be called multiple times due to column-by-column processing. -For comprehensive guidelines, see the [Database Schema Recommendations](README.md#database-schema-recommendations) section in the README. +For comprehensive guidelines, see the [Database Schema Recommendations](docs/schema.md). The function supports three overloads: - `cloudsync_init(table_name)`: Uses the default 'cls' CRDT algorithm. - `cloudsync_init(table_name, crdt_algo)`: Specifies a CRDT algorithm ('cls', 'dws', 'aws', 'gos'). -- `cloudsync_init(table_name, crdt_algo, force)`: Specifies an algorithm and, if `force` is `true` (or `1`), skips the integer primary key check (use with caution, GUIDs are strongly recommended). +- `cloudsync_init(table_name, crdt_algo, init_flags)`: Specifies an algorithm and a bitmask of initialization flags to control which schema sanity checks are skipped. **Parameters:** - `table_name` (TEXT): The name of the table to initialize. -- `crdt_algo` (TEXT, optional): The CRDT algorithm to use. Can be "cls", "dws", "aws", "gos". Defaults to "cls". -- `force` (BOOLEAN, optional): If `true` (or `1`), it skips the check that prevents the use of a single-column INTEGER primary key. Defaults to `false`. It is strongly recommended to use globally unique primary keys instead of integers. +- `crdt_algo` (TEXT, optional): The CRDT algorithm to use. Can be `"cls"`, `"dws"`, `"aws"`, `"gos"`. Defaults to `"cls"`. +- `init_flags` (INTEGER, optional): A bitmask of flags that control initialization behavior. Defaults to `0` (no flags). Available flags: + - `0` — No flags; all sanity checks are performed (default). + - `1` (`CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK`) — Skip the check that prevents the use of a single-column INTEGER primary key. Use with caution; globally unique primary keys (UUID/ULID) are strongly recommended. + - `2` (`CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK`) — Skip the check that requires all NOT NULL non-PK columns to have a DEFAULT value. + - `4` (`CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK`) — Skip the check that rejects NULL primary key values. + - Flags can be combined with bitwise OR (e.g., `3` skips both the integer PK check and the NOT NULL default check). **Returns:** None. **Example:** ```sql --- Initialize a single table for synchronization with the Causal-Length Set (CLS) Algorithm (default) +-- Initialize a table with the default CLS algorithm SELECT cloudsync_init('my_table'); --- Initialize a single table for synchronization with a different algorithm Delete-Wins Set (DWS) +-- Initialize a table with the Delete-Wins Set algorithm SELECT cloudsync_init('my_table', 'dws'); +-- Initialize a table with an integer primary key (skip the integer PK check) +SELECT cloudsync_init('my_table', 'cls', 1); ``` --- @@ -136,6 +148,56 @@ SELECT cloudsync_is_enabled('my_table'); --- +### `cloudsync_set_filter(table_name, filter_expr)` + +**Description:** Sets a row-level filter expression on a synchronized table. Only rows that match the filter are tracked by the sync triggers; changes to rows that do not satisfy the expression are ignored and never replicated. + +The filter expression is a standard SQL boolean expression written using bare column names (without a table or alias prefix). The extension automatically rewrites it with `NEW.` for INSERT/UPDATE triggers and `OLD.` for DELETE triggers. The expression is evaluated inside the trigger `WHEN` clause. + +This function stores the filter in the table's settings and immediately recreates the sync triggers to apply it. The filter persists across database reopens. Use [`cloudsync_clear_filter()`](#cloudsync_clear_filtertable_name) to remove it. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. +- `filter_expr` (TEXT): A SQL boolean expression referencing column names of the table. Only rows for which this expression evaluates to true are tracked for sync. + +**Returns:** `1` on success. + +**Example:** + +```sql +-- Only sync tasks that are not marked as drafts +SELECT cloudsync_set_filter('tasks', "is_draft = 0"); + +-- Only sync rows belonging to a specific tenant +SELECT cloudsync_set_filter('orders', "tenant_id = 'acme'"); + +-- Combine conditions +SELECT cloudsync_set_filter('messages', "deleted = 0 AND type != 'ephemeral'"); +``` + +--- + +### `cloudsync_clear_filter(table_name)` + +**Description:** Removes the row-level filter previously set with [`cloudsync_set_filter()`](#cloudsync_set_filtertable_name-filter_expr). After clearing, all row changes in the table are tracked and replicated regardless of column values. + +This function updates the stored settings and immediately recreates the sync triggers without a filter condition. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. + +**Returns:** `1` on success. + +**Example:** + +```sql +SELECT cloudsync_clear_filter('tasks'); +``` + +--- + ### `cloudsync_cleanup(table_name)` **Description:** Removes the `sqlite-sync` synchronization mechanism from a specified table or all tables. This operation drops the associated `_cloudsync` metadata table and removes triggers from the target table(s). Use this function when synchronization is no longer desired for a table. @@ -173,6 +235,68 @@ SELECT cloudsync_terminate(); --- +## Block-Level LWW Functions + +### `cloudsync_set_column(table_name, col_name, key, value)` + +**Description:** Configures per-column settings for a synchronized table. This function is primarily used to enable **block-level LWW** on text columns, allowing fine-grained conflict resolution at the line (or paragraph) level instead of the entire cell. + +When block-level LWW is enabled on a column, INSERT and UPDATE operations automatically split the text into blocks using a delimiter (default: newline `\n`) and track each block independently. During sync, changes are merged block-by-block, so concurrent edits to different parts of the same text are preserved. + +**Parameters:** + +- `table_name` (TEXT): The name of the synchronized table. +- `col_name` (TEXT): The name of the text column to configure. +- `key` (TEXT): The setting key. Supported keys: + - `'algo'` — Set the column algorithm. Use value `'block'` to enable block-level LWW. + - `'delimiter'` — Set the block delimiter string. Only applies to columns with block-level LWW enabled. +- `value` (TEXT): The setting value. + +**Returns:** None. + +**Example:** + +```sql +-- Enable block-level LWW on a column (splits text by newline by default) +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); + +-- Set a custom delimiter (e.g., double newline for paragraph-level tracking) +SELECT cloudsync_set_column('notes', 'body', 'delimiter', ' + +'); +``` + +--- + +### `cloudsync_text_materialize(table_name, col_name, pk_values...)` + +**Description:** Reconstructs the full text of a block-level LWW column from its individual blocks and writes the result back to the base table column. This is useful after a merge operation to ensure the column contains the up-to-date materialized text. + +After a sync/merge, the column is updated automatically. This function is primarily useful for manual materialization or debugging. + +**Parameters:** + +- `table_name` (TEXT): The name of the table. +- `col_name` (TEXT): The name of the block-level LWW column. +- `pk_values...` (variadic): The primary key values identifying the row. For composite primary keys, pass each key value as a separate argument in declaration order. + +**Returns:** `1` on success. + +**Example:** + +```sql +-- Materialize the body column for a specific row +SELECT cloudsync_text_materialize('notes', 'body', 'note-001'); + +-- With a composite primary key (e.g., PRIMARY KEY (tenant_id, doc_id)) +SELECT cloudsync_text_materialize('docs', 'body', 'tenant-1', 'doc-001'); + +-- Read the materialized text +SELECT body FROM notes WHERE id = 'note-001'; +``` + +--- + ## Helper Functions ### `cloudsync_version()` @@ -287,20 +411,20 @@ SELECT cloudsync_commit_alter('my_table'); ## Network Functions -### `cloudsync_network_init(connection_string)` +### `cloudsync_network_init(managedDatabaseId)` -**Description:** Initializes the `sqlite-sync` network component. This function parses the connection string to configure change checking and upload endpoints, and initializes the cURL library. +**Description:** Initializes the `sqlite-sync` network component. This function configures the endpoints for the CloudSync service and initializes the cURL library. **Parameters:** -- `connection_string` (TEXT): The connection string for the remote synchronization server. The format is `sqlitecloud://:/?`. +- `managedDatabaseId` (TEXT): The managed database identifier returned by the CloudSync service when a new database is registered for sync. For SQLiteCloud projects, this value can be obtained from the project's OffSync page on the dashboard. **Returns:** None. **Example:** ```sql -SELECT cloudsync_network_init('.sqlite.cloud/.sqlite'); +SELECT cloudsync_network_init('your-managed-database-id'); ``` --- @@ -357,19 +481,18 @@ SELECT cloudsync_network_set_apikey('your_api_key'); --- -### `cloudsync_network_has_unsent_changes()` +### Error handling -**Description:** Checks if there are any local changes that have not yet been sent to the remote server. +The sync functions follow a consistent error-handling contract: -**Parameters:** None. +| Error type | Behavior | +|---|---| +| **Endpoint/network errors** (server unreachable, auth failure, bad URL) | SQL error — the function could not execute. | +| **Apply errors** (`cloudsync_payload_apply` failures — unknown schema hash, invalid checksum, decompression error) | Structured JSON — a `receive.error` string field is included in the response. | +| **Server-reported apply job failures** (the server processed the request but its own apply job failed) | Structured JSON — a `send.lastFailure` object is included in the response. | +| **Server-reported check job failures** (the server failed to encode a changeset for the client) | Structured JSON — a `receive.lastFailure` object is included in the response. | -**Returns:** 1 if there are unsent changes, 0 otherwise. - -**Example:** - -```sql -SELECT cloudsync_network_has_unsent_changes(); -``` +This means: if you get JSON back, the server was reachable and the network protocol ran. If you get a SQL error, connectivity or configuration is broken. --- @@ -379,12 +502,25 @@ SELECT cloudsync_network_has_unsent_changes(); **Parameters:** None. -**Returns:** None. +**Returns:** A JSON string with the send result: + +```json +{"send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "lastFailure": {...}}} +``` + +- `send.status`: The current sync state — `"synced"` (all changes confirmed), `"syncing"` (changes sent but not yet confirmed), `"out-of-sync"` (local changes pending or gaps detected), or `"error"`. +- `send.localVersion`: The latest local database version. +- `send.serverVersion`: The latest version confirmed by the server. +- `send.lastFailure` (optional): Present only when the server reports a failed apply job. Forwarded verbatim from the server's `failures.apply` and typically includes `jobId`, `code`, `stage`, `message`, `retryable`, and `failedAt`. It is emitted regardless of `status` so callers can detect server-side failures during `"syncing"` or even after the state has nominally recovered. This function is **send/apply-scoped**: server-reported check-job failures (`failures.check`) are not surfaced here — see [`cloudsync_network_check_changes()`](#cloudsync_network_check_changes) and [`cloudsync_network_sync()`](#cloudsync_network_sync). **Example:** ```sql SELECT cloudsync_network_send_changes(); +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5}}' + +-- With a server-reported failure (e.g. unknown schema hash on the server side): +-- '{"send":{"status":"out-of-sync","localVersion":1,"serverVersion":0,"lastFailure":{"jobId":44961,"code":"internal_error","stage":"apply_payload","message":"cloudsync operation failed: Cannot apply the received payload because the schema hash is unknown 4288148391734624266.","retryable":true,"failedAt":"2026-04-15T22:21:09.018606Z"}}}' ``` --- @@ -398,17 +534,32 @@ If a package of new changes is already available for the local site, the server This function is designed to be called periodically to keep the local database in sync. To force an update and wait for changes (with a timeout), use [`cloudsync_network_sync(wait_ms, max_retries)`]. -If the network is misconfigured or the remote server is unreachable, the function returns an error. -On success, it returns `SQLITE_OK`, and the return value indicates how many changes were downloaded and applied. +If the network is misconfigured or the remote server is unreachable, the function raises a SQL error. If the received payload cannot be applied locally (for example because of an unknown schema hash), the error is returned as a `receive.error` field in the JSON response. If the server reports an unresolved failed check job (e.g. an `encode_changes` failure), that failure is forwarded as a `receive.lastFailure` object. **Parameters:** None. -**Returns:** The number of changes downloaded. Errors are reported via the SQLite return code. +**Returns:** A JSON string with the receive result: + +```json +{"receive": {"rows": N, "tables": ["table1", "table2"], "error": "...", "lastFailure": {...}}} +``` + +- `receive.rows`: The number of rows received and applied to the local database. `0` when the receive phase failed. +- `receive.tables`: An array of table names that received changes. Empty (`[]`) if no changes were applied or the receive phase failed. +- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed. Contains a human-readable error message describing why the received payload could not be applied. +- `receive.lastFailure` (optional, object): Present only when the server reports a failed check job. Forwarded verbatim from the server's `failures.check` and typically includes `jobId`, `dbVersion`, `seq`, `code`, `stage`, `message`, `retryable`, and `failedAt`. Distinct from `receive.error`: `receive.error` describes a client-side apply failure (string), while `receive.lastFailure` describes a server-side check-job failure (object). Both can coexist in the same response. This function is **check-scoped**: server-reported apply-job failures (`failures.apply`) are not surfaced here — see [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) and [`cloudsync_network_sync()`](#cloudsync_network_sync). **Example:** ```sql SELECT cloudsync_network_check_changes(); +-- '{"receive":{"rows":3,"tables":["tasks"]}}' + +-- With a client-side apply error: +-- '{"receive":{"rows":0,"tables":[],"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' + +-- With a server-reported check-job failure: +-- '{"receive":{"rows":0,"tables":[],"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' ``` --- @@ -425,16 +576,36 @@ SELECT cloudsync_network_check_changes(); - `wait_ms` (INTEGER, optional): The time to wait in milliseconds between retries. Defaults to 100. - `max_retries` (INTEGER, optional): The maximum number of times to retry the synchronization. Defaults to 1. -**Returns:** The number of changes downloaded. Errors are reported via the SQLite return code. +**Returns:** A JSON string with the full sync result, combining send and receive: + +```json +{ + "send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "lastFailure": {...}}, + "receive": {"rows": N, "tables": ["table1", "table2"], "error": "...", "lastFailure": {...}} +} +``` + +- `send.status`: The current sync state — `"synced"`, `"syncing"`, `"out-of-sync"`, or `"error"`. +- `send.localVersion`: The latest local database version. +- `send.serverVersion`: The latest version confirmed by the server. +- `send.lastFailure` (optional): Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) — forwarded verbatim from the server's `failures.apply` whenever a failed apply job is reported, regardless of `status`. +- `receive.rows`: The number of rows received and applied during the check phase. `0` when the receive phase failed. +- `receive.tables`: An array of table names that received changes. Empty (`[]`) if no changes were applied or the receive phase failed. +- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed (for example `"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."`). The send result is always preserved so the caller can tell that local changes reached the server even when applying incoming changes failed. The retry loop breaks immediately on apply errors, since failures like schema-hash mismatches do not heal across retries. Endpoint/network errors during the receive phase raise a SQL error instead. +- `receive.lastFailure` (optional, object): Same semantics as in [`cloudsync_network_check_changes()`](#cloudsync_network_check_changes) — forwarded verbatim from the server's `failures.check` whenever a failed check job is reported. Distinct from `receive.error`. `cloudsync_network_sync()` reports both `send.lastFailure` and `receive.lastFailure` when present. **Example:** ```sql -- Perform a single synchronization cycle SELECT cloudsync_network_sync(); +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5},"receive":{"rows":3,"tables":["tasks"]}}' -- Perform a synchronization cycle with custom retry settings SELECT cloudsync_network_sync(500, 3); + +-- Receive phase failed but send phase completed — the error is surfaced in JSON, not as a SQL error: +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5},"receive":{"rows":0,"tables":[],"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' ``` --- @@ -455,9 +626,25 @@ SELECT cloudsync_network_reset_sync_version(); --- +### `cloudsync_network_has_unsent_changes()` + +**Description:** Checks if there are any local changes that have not yet been sent to the remote server. + +**Parameters:** None. + +**Returns:** 1 if there are unsent changes, 0 otherwise. + +**Example:** + +```sql +SELECT cloudsync_network_has_unsent_changes(); +``` + +--- + ### `cloudsync_network_logout()` -**Description:** Logs out the current user and cleans up all local data from synchronized tables. This function deletes and then re-initializes synchronized tables, useful for switching users or resetting the local database. **Warning:** This function deletes all data from synchronized tables. Use with caution. +**Description:** Logs out the current user and cleans up all local data from synchronized tables. This function deletes and then re-initializes synchronized tables, useful for switching users or resetting the local database. **Warning:** This function deletes all data from synchronized tables. Use with caution. Consider calling [`cloudsync_network_has_unsent_changes()`](#cloudsync_network_has_unsent_changes) before logout to check for unsent local changes and warn the user before data that has not been fully synchronized to the remote server is deleted. **Parameters:** None. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..158a6ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,190 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [1.0.20] - 2026-05-26 + +### Changed + +- **Improved network sync performance** by reducing request overhead during `cloudsync_network_send_changes()`, especially for small payloads that can now be applied without the extra upload-URL round trip. +- **Improved repeated sync request latency** by allowing the network layer to reuse HTTP connections across CloudSync API calls. + +## [1.0.19] - 2026-05-15 + +### Added + +- **Mac Catalyst support**. + +## [1.0.18] - 2026-04-29 + +### Fixed + +- **`cloudsync_network_check_changes()`** no longer errors with `missing 'url' in check response` when the server has not yet prepared any incoming changes for this device. The function now returns the standard "no rows yet" response in that case, so polling loops keep working without spurious errors. + +### Added + +- **`receive.lastFailure`** JSON field on `cloudsync_network_check_changes()` and `cloudsync_network_sync()`, surfacing the most recent server-side failure of the receive pipeline (e.g. the server failed to prepare the next batch of incoming changes for this device). It complements the existing `send.lastFailure` (server-side apply failures) and `receive.error` (local apply failures on this device), so applications can distinguish "the server has trouble producing my changes" from "I had trouble applying them locally". Each function reports only the failures relevant to its own scope: `cloudsync_network_send_changes()` reports `send.lastFailure`; `cloudsync_network_check_changes()` reports `receive.lastFailure`; `cloudsync_network_sync()` reports both. + +### Changed + +- Updated the request headers sent to the cloudsync HTTP endpoints (version advertisement, per-endpoint capabilities; legacy `Accept` header removed). + +## [1.0.17] - 2026-04-24 + +### Fixed + +- **Confusing errors when `cloudsync_init` was never called**: `cloudsync_changes` (SQLite), `cloudsync_db_version`, `cloudsync_db_version_next`, `cloudsync_set_filter`, `cloudsync_clear_filter`, and `cloudsync_payload_apply` now raise a single actionable message pointing at `SELECT cloudsync_init('')` instead of leaking low-level symptoms (`out of memory`, `not an error`, silent `-1`, multi-line "no such table" dumps). The guard runs only on the error branch, so the sync hot path is unaffected. + +## [1.0.16] - 2026-04-16 + +### Fixed + +- **WASM crash in `cloudsync_set_column` on existing rows**: Calling `cloudsync_set_column(table, col, 'lww', 'block')` on a table with pre-existing rows crashed the WASM build with a `RuntimeError: function signature mismatch` as soon as the block-index migration tried to allocate memory. `block_init_allocator` was casting `cloudsync_memory_alloc` (a `uint64_t size` function) directly to the fractional-indexing allocator's `void *(*)(size_t)` slot. The cast is a no-op on native platforms where `size_t` is 64-bit, but WASM's `call_indirect` enforces strict type checking — the function is registered as `(i64) -> i32` and called as `(i32) -> i32`, triggering an immediate runtime error. A thin `fi_malloc_wrapper` (mirroring the existing `fi_calloc_wrapper`) now bridges the signatures. Native builds are unaffected. + +## [1.0.15] - 2026-04-16 + +### Fixed + +- **Silent receive failures**: When `cloudsync_payload_apply` failed during the receive phase (for example with an unknown schema hash, invalid checksum, or decompression error), the error was stored only on the internal cloudsync context and never propagated to the SQL caller. Both `cloudsync_network_check_changes()` and `cloudsync_network_sync()` silently returned no result. Apply errors are now surfaced as a `receive.error` field in the JSON response. + +### Changed + +- **Error handling contract**: endpoint/network errors (server unreachable, auth failure, bad URL) always raise a SQL error. Processing errors (`cloudsync_payload_apply` failures) are returned as structured JSON via `receive.error` or `send.lastFailure`, so callers can inspect and log them without try/catch logic. +- **`cloudsync_network_send_changes()` output** now includes a `send.lastFailure` object whenever the server reports one (raw pass-through of the server's `lastFailure` — `jobId`, `code`, `message`, `retryable`, `failedAt`, …), regardless of whether the computed `send.status` is `synced`, `syncing`, or `out-of-sync`. The field is omitted when the server does not report a failure. +- **`cloudsync_network_check_changes()` output** now includes a `receive.error` string when `cloudsync_payload_apply` fails, instead of silently returning NULL. Endpoint/network errors still raise a SQL error. +- **`cloudsync_network_sync()` output** now mirrors the same `send.lastFailure` field and, if the receive phase has a processing error (`cloudsync_payload_apply` failure), returns structured JSON with a `receive.error` string rather than failing silently. The send result is always preserved so callers can tell that their local changes reached the server even when applying incoming changes failed. Endpoint/network errors during the receive phase still raise a SQL error. The receive retry loop breaks immediately on processing errors (a schema-hash mismatch will not heal across retries). + +## [1.0.14] - 2026-04-15 + +### Fixed + +- **Stale `cloudsync_table_settings` crash**: Reopening a database that had its base table and `
_cloudsync` meta-table dropped without calling `cloudsync_cleanup` crashed with a double-free on `sqlite3_close`. Two bugs were involved: (1) `cloudsync_dbversion_rebuild` returned `DBRES_NOMEM` when `cloudsync_dbversion_build_query` yielded a NULL SQL string (stale row in `cloudsync_table_settings` but no matching `*_cloudsync` table in `sqlite_master`), failing extension init; (2) on init failure `dbsync_register_functions` manually freed the context that SQLite already owned via the `cloudsync_version` destructor, causing a double-free when the connection was later closed. `cloudsync_dbversion_rebuild` now treats a NULL build query the same as `count == 0` (no prepared statement, db_version stays at the minimum and is rebuilt on the next `cloudsync_init`), and the manual free in the error path has been removed. + +### Added + +- Unit test `do_test_stale_table_settings_dropped_meta` (Stale Table Settings Dropped Meta) covering the drop-base-table + drop-meta-table + reopen scenario. + +## [1.0.13] - 2026-04-14 + +### Fixed + +- **Block-level LWW migration**: When `cloudsync_set_column(..., 'algo', 'block')` is called on a table that already has tracked rows, those rows are now immediately migrated into the blocks table. Previously, pre-existing column values were ignored until the next UPDATE, leaving sync state incomplete. The migration uses a two-phase collect-then-write approach to avoid SQLite cursor invalidation and `INSERT OR IGNORE` / `ON CONFLICT DO NOTHING` semantics for idempotency. + +### Added + +- Unit test `do_test_block_lww_existing_data` (Block LWW Existing Data) verifying block migration on `set_column`, idempotency of repeated `set_column` calls, and correct materialization after update. +- PostgreSQL test `50_block_lww_existing_data.sql` with equivalent coverage for the PostgreSQL backend. + +## [1.0.12] - 2026-04-11 + +### Fixed + +- **Settings loader**: Prevent infinite loop in `sqlite3_cloudsync_init` when reopening a database that has a persisted block-column setting. `dbutils_settings_table_load_callback` was calling `cloudsync_setup_block_column`, which `REPLACE`d the same row into `cloudsync_table_settings` while `sqlite3_exec` was still iterating it, re-feeding the rewritten row to the cursor. Added a `persist` flag to `cloudsync_setup_block_column` so the loader replays the in-memory setup without writing back. +- **PostgreSQL tests**: Updated 168 `cloudsync_init` callsites across 43 `test/postgresql/*.sql` files to pass integer flags (`0`/`1`) instead of `true`/`false`, matching the signature change in 1.0.9. +- **CI**: The `postgres-test` job now fails on SQL errors and `[FAIL]` markers. `psql` is run with `ON_ERROR_STOP=on`, `pipefail` is enabled around the `tee`, and the captured log is grepped for `[FAIL]` / `psql ERROR` as a final guard. + +### Added + +- Unit test `do_test_block_column_reload` (Block Column Reload) that persists a block column with a custom delimiter, closes the database, and reopens it — without the fix this hangs the test process. + +## [1.0.11] - 2026-04-11 + +### Fixed + +- **cloudsync_cleanup**: Now also drops the `{table}_cloudsync_blocks` table when the table has block LWW columns configured via `cloudsync_set_column(..., 'algo', 'block')`. + +### Added + +- Unit test `do_test_block_lww_cleanup` verifying that both `{table}_cloudsync` and `{table}_cloudsync_blocks` are removed after `cloudsync_cleanup`. + +## [1.0.10] - 2026-04-08 + +### Fixed + +- **PostgreSQL**: Prevent debug assertion crash on `cloudsync_init` error path (#37). +- **Row filter**: `cloudsync_set_filter` and `cloudsync_clear_filter` now reset the metatable and refill it from scratch, ensuring only rows matching the active filter are tracked for sync (#38). + +### Added + +- Row filter edge-case test coverage: clear/change filter lifecycle, complex expressions (AND, IS NULL), row enter/exit via UPDATE, composite PK with multi-column filters, multi-table roundtrip sync, and pre-existing data prefill tests for both SQLite and PostgreSQL. + +## [1.0.9] - 2026-04-08 + +### Changed + +- **cloudsync_init**: Replaced the `force` boolean parameter with an `init_flags` integer bitmask (`CLOUDSYNC_INIT_FLAG`), allowing fine-grained control over which schema sanity checks are skipped. Existing callers passing `0`/`false` or `1`/`true` remain compatible. +- **API**: Updated `cloudsync_init` SQL signature (PostgreSQL) to accept `integer` instead of `boolean` for the third argument, enabling flag combinations via bitwise OR. + +### Added + +- `CLOUDSYNC_INIT_FLAG_NONE` (0), `CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK` (1), `CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK` (2), `CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK` (4) enum values. +- Documentation for `cloudsync_set_filter` and `cloudsync_clear_filter` in API.md. + +## [1.0.8] - 2026-04-03 + +### Changed + +- **CI/CD**: Fix flutter package publish workflow not triggering on new releases. + +## [1.0.7] - 2026-04-02 + +### Fixed + +- **Harden table initialization against stale config and error cleanup.** + +## [1.0.2] - 2026-03-25 + +### Fixed + +- **Swift Package**: Use binary target and versioned macOS framework for Xcode 26 compatibility. +- **Minor bugs** in tests, docs, and examples related to the 1.0.0 major release. + +## [1.0.0] - 2026-03-24 + +### Added + +- **PostgreSQL support**: The CloudSync extension can now be built and loaded on PostgreSQL, so both SQLiteCloud and PostgreSQL are supported as the cloud backend database of the sync service. The core CRDT functions are shared by the SQLite and PostgreSQL extensions. Includes support for PostgreSQL-native types (UUID primary keys, composite PKs with mixed types, and automatic type casting). +- **Row-Level Security (RLS)**: Sync payloads are now fully compatible with SQLiteCloud and PostgreSQL Row-Level Security policies. Changes are buffered per primary key and flushed as complete rows, so RLS policies can evaluate all columns at once. +- **Block-level LWW for text conflict resolution**: Text columns can now be tracked at block level (lines by default) using Last-Writer-Wins. Concurrent edits to different parts of the same text are preserved after sync. New functions: `cloudsync_set_column()` to write individual blocks and `cloudsync_text_materialize()` to reconstruct the full text. + +### Changed + +- **BREAKING: `cloudsync_network_init` now accepts a `managedDatabaseId` instead of a connection string.** The `managedDatabaseId` is returned by the CloudSync service when a new database is registered for sync. For SQLiteCloud projects, it can be obtained from the project's OffSync page on the dashboard. + + Before: + ```sql + SELECT cloudsync_network_init('sqlitecloud://myproject.sqlite.cloud:8860/mydb.sqlite?apikey=KEY'); + ``` + + After: + ```sql + SELECT cloudsync_network_init('your-managed-database-id'); + ``` + +- **BREAKING: Sync functions now return structured JSON.** `cloudsync_network_send_changes`, `cloudsync_network_check_changes`, and `cloudsync_network_sync` return a JSON object instead of a plain integer. This provides richer status information including sync state, version numbers, row counts, and affected table names. + + Before: + ```sql + SELECT cloudsync_network_sync(); + -- 3 (number of rows received) + ``` + + After: + ```sql + SELECT cloudsync_network_sync(); + -- '{"send":{"status":"synced","localVersion":5,"serverVersion":5},"receive":{"rows":3,"tables":["tasks"]}}' + ``` + +- **Batch merge replaces column-by-column processing**: During sync, changes to the same row are now applied in a single SQL statement instead of one statement per column. This eliminates the previous behavior where UPDATE triggers fired multiple times per row during synchronization. +- **Network endpoints updated for the CloudSync v2 HTTP service**: Internal network layer now targets the new CloudSync service endpoints, including support for multi-organization routing. +- **NULL primary key rejection at runtime**: The extension now enforces NULL primary key rejection at runtime, so the explicit `NOT NULL` constraint on primary key columns is no longer a schema requirement. + +### Fixed + +- **Improved error reporting**: Sync network functions now surface the actual server error message instead of generic error codes. +- **Schema hash verification**: Normalized schema comparison now uses only column name (lowercase), type (SQLite affinity), and primary key flag, preventing false mismatches caused by formatting differences. +- **SQLite trigger safety**: Internal functions used inside triggers are now marked with `SQLITE_INNOCUOUS`, fixing `unsafe use of` errors when initializing tables that have triggers. +- **NULL column binding**: Column value parameters are now correctly bound even when NULL, preventing sync failures on rows with NULL values. +- **Stability and reliability improvements** across the SQLite and PostgreSQL codebases, including fixes to memory management, error handling, and CRDT version tracking. diff --git a/Makefile b/Makefile index b77a04c..189bf0f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Supports compilation for Linux, macOS, Windows, Android and iOS # customize sqlite3 executable with -# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3 +# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.50.4/bin/sqlite3 SQLITE3 ?= sqlite3 # set curl version to download and build @@ -30,9 +30,15 @@ endif # Speed up builds by using all available CPU cores MAKEFLAGS += -j$(CPUS) +# Mac Catalyst uses the Apple-native NSURLSession networking path. This needs to +# be set before the common linker flags decide whether libcurl is required. +ifeq ($(PLATFORM),mac-catalyst) + override NATIVE_NETWORK := ON +endif + # Compiler and flags CC = gcc -CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(SQLITE_DIR) -I$(CURL_DIR)/include +CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(SRC_DIR)/sqlite -I$(SRC_DIR)/postgresql -I$(SRC_DIR)/network -I$(SQLITE_DIR) -I$(CURL_DIR)/include -Imodules/fractional-indexing T_CFLAGS = $(CFLAGS) -DSQLITE_CORE -DCLOUDSYNC_UNITTEST -DCLOUDSYNC_OMIT_NETWORK -DCLOUDSYNC_OMIT_PRINT_RESULT COVERAGE = false ifndef NATIVE_NETWORK @@ -41,10 +47,14 @@ endif # Directories SRC_DIR = src +SQLITE_IMPL_DIR = $(SRC_DIR)/sqlite +POSTGRES_IMPL_DIR = $(SRC_DIR)/postgresql DIST_DIR = dist TEST_DIR = test SQLITE_DIR = sqlite -VPATH = $(SRC_DIR):$(SQLITE_DIR):$(TEST_DIR) +FI_DIR = modules/fractional-indexing +NETWORK_DIR = $(SRC_DIR)/network +VPATH = $(SRC_DIR):$(SQLITE_IMPL_DIR):$(POSTGRES_IMPL_DIR):$(NETWORK_DIR):$(SQLITE_DIR):$(TEST_DIR):$(FI_DIR) BUILD_RELEASE = build/release BUILD_TEST = build/test BUILD_DIRS = $(BUILD_TEST) $(BUILD_RELEASE) @@ -59,12 +69,20 @@ ifeq ($(PLATFORM),android) OPENSSL_INSTALL_DIR = $(OPENSSL_DIR)/$(PLATFORM)/$(ARCH) endif -SRC_FILES = $(wildcard $(SRC_DIR)/*.c) +# Multi-platform source files (at src/ root) - exclude database_*.c as they're in subdirs +CORE_SRC = $(filter-out $(SRC_DIR)/database_%.c, $(wildcard $(SRC_DIR)/*.c)) $(wildcard $(NETWORK_DIR)/*.c) +# SQLite-specific files +SQLITE_SRC = $(wildcard $(SQLITE_IMPL_DIR)/*.c) +# Fractional indexing submodule +FI_SRC = $(FI_DIR)/fractional_indexing.c +# Combined for SQLite extension build +SRC_FILES = $(CORE_SRC) $(SQLITE_SRC) $(FI_SRC) + TEST_SRC = $(wildcard $(TEST_DIR)/*.c) TEST_FILES = $(SRC_FILES) $(TEST_SRC) $(wildcard $(SQLITE_DIR)/*.c) RELEASE_OBJ = $(patsubst %.c, $(BUILD_RELEASE)/%.o, $(notdir $(SRC_FILES))) TEST_OBJ = $(patsubst %.c, $(BUILD_TEST)/%.o, $(notdir $(TEST_FILES))) -COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(SRC_DIR)/network.c, $(SRC_FILES)) +COV_FILES = $(filter-out $(SRC_DIR)/lz4.c $(NETWORK_DIR)/network.c $(SQLITE_IMPL_DIR)/sql_sqlite.c $(POSTGRES_IMPL_DIR)/database_postgresql.c $(FI_SRC), $(SRC_FILES)) CURL_LIB = $(CURL_DIR)/$(PLATFORM)/libcurl.a TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC))) @@ -120,7 +138,7 @@ else ifeq ($(PLATFORM),android) CURL_CONFIG = --host $(ARCH)-linux-$(ANDROID_ABI) --with-openssl=$(CURDIR)/$(OPENSSL_INSTALL_DIR) LDFLAGS="-L$(CURDIR)/$(OPENSSL_INSTALL_DIR)/lib" LIBS="-lssl -lcrypto" AR=$(BIN)/llvm-ar AS=$(BIN)/llvm-as CC=$(CC) CXX=$(BIN)/$(ARCH)-linux-$(ANDROID_ABI)-clang++ LD=$(BIN)/ld RANLIB=$(BIN)/llvm-ranlib STRIP=$(BIN)/llvm-strip TARGET := $(DIST_DIR)/cloudsync.so CFLAGS += -fPIC -I$(OPENSSL_INSTALL_DIR)/include - LDFLAGS += -shared -fPIC -L$(OPENSSL_INSTALL_DIR)/lib -lssl -lcrypto -Wl,-z,max-page-size=16384 + LDFLAGS += -shared -fPIC -L$(OPENSSL_INSTALL_DIR)/lib -lssl -lcrypto -lm -Wl,-z,max-page-size=16384 STRIP = $(BIN)/llvm-strip --strip-unneeded $@ else ifeq ($(PLATFORM),ios) TARGET := $(DIST_DIR)/cloudsync.dylib @@ -138,10 +156,23 @@ else ifeq ($(PLATFORM),ios-sim) CFLAGS += -arch x86_64 -arch arm64 $(SDK) CURL_CONFIG = --host=arm64-apple-darwin --with-secure-transport CFLAGS="-arch x86_64 -arch arm64 -isysroot $$(xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0" STRIP = strip -x -S $@ +else ifeq ($(PLATFORM),mac-catalyst) + TARGET := $(DIST_DIR)/cloudsync.dylib + MAC_CATALYST_DEPLOYMENT_TARGET ?= 14.0 + ifndef ARCH + MAC_CATALYST_UNIVERSAL := true + STRIP = strip -x -S $@ + else + SDK := -isysroot $(shell xcrun --sdk macosx --show-sdk-path) -target $(ARCH)-apple-ios$(MAC_CATALYST_DEPLOYMENT_TARGET)-macabi + LDFLAGS += -framework Security -framework CoreFoundation -dynamiclib $(SDK) -headerpad_max_install_names + T_LDFLAGS = -framework Security + CFLAGS += $(SDK) + STRIP = strip -x -S $@ + endif else # linux TARGET := $(DIST_DIR)/cloudsync.so - LDFLAGS += -shared -lssl -lcrypto - T_LDFLAGS += -lpthread + LDFLAGS += -shared -lssl -lcrypto -lm + T_LDFLAGS += -lpthread -lm CURL_CONFIG = --with-openssl STRIP = strip --strip-unneeded $@ endif @@ -154,9 +185,13 @@ endif T_LDFLAGS += -fprofile-arcs -ftest-coverage endif +ifdef SYNC_BENCH_DEBUG + CFLAGS += -DCLOUDSYNC_NETWORK_TRACE +endif + # Native network support only for Apple platforms ifdef NATIVE_NETWORK - RELEASE_OBJ += $(patsubst %.m, $(BUILD_RELEASE)/%_m.o, $(notdir $(wildcard $(SRC_DIR)/*.m))) + RELEASE_OBJ += $(patsubst %.m, $(BUILD_RELEASE)/%_m.o, $(notdir $(wildcard $(NETWORK_DIR)/*.m))) LDFLAGS += -framework Foundation CFLAGS += -DCLOUDSYNC_OMIT_CURL @@ -179,6 +214,20 @@ $(shell mkdir -p $(BUILD_DIRS) $(DIST_DIR)) extension: $(TARGET) all: $(TARGET) +ifeq ($(MAC_CATALYST_UNIVERSAL),true) +MAC_CATALYST_ARCHS = x86_64 arm64 +MAC_CATALYST_DYLIBS = $(foreach arch,$(MAC_CATALYST_ARCHS),$(DIST_DIR)/cloudsync-mac-catalyst-$(arch).dylib) + +$(TARGET): + @for arch in $(MAC_CATALYST_ARCHS); do \ + rm -rf $(BUILD_DIRS); \ + $(MAKE) PLATFORM=mac-catalyst ARCH=$$arch MAC_CATALYST_DEPLOYMENT_TARGET=$(MAC_CATALYST_DEPLOYMENT_TARGET); \ + mv $(DIST_DIR)/cloudsync.dylib $(DIST_DIR)/cloudsync-mac-catalyst-$$arch.dylib; \ + done + lipo -create $(MAC_CATALYST_DYLIBS) -output $@ + rm -f $(MAC_CATALYST_DYLIBS) + $(STRIP) +else # Loadable library ifdef NATIVE_NETWORK $(TARGET): $(RELEASE_OBJ) $(DEF_FILE) @@ -192,6 +241,7 @@ ifeq ($(PLATFORM),windows) endif # Strip debug symbols $(STRIP) +endif # Test executable $(TEST_TARGET): $(TEST_OBJ) @@ -206,15 +256,44 @@ $(BUILD_TEST)/%.o: %.c $(CC) $(T_CFLAGS) -c $< -o $@ # Run code coverage (--css-file $(CUSTOM_CSS)) -test: $(TARGET) $(TEST_TARGET) - $(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT cloudsync_version();" - set -e; for t in $(TEST_TARGET); do ./$$t; done +test: $(TARGET) $(TEST_TARGET) unittest e2e + set -e; $(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT cloudsync_version();" ifneq ($(COVERAGE),false) mkdir -p $(COV_DIR) lcov --capture --directory . --output-file $(COV_DIR)/coverage.info $(subst src, --include src,${COV_FILES}) genhtml $(COV_DIR)/coverage.info --output-directory $(COV_DIR) endif +# Run only unit tests +unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) + @./$(DIST_DIR)/unit$(EXE) + +# Run end-to-end integration tests +e2e: $(TARGET) $(DIST_DIR)/integration$(EXE) + @if [ -f .env ]; then \ + export $$(grep -v '^#' .env | xargs); \ + fi; \ + ./$(DIST_DIR)/integration$(EXE) + +# Run the sync performance benchmark. This is intentionally separate from e2e +# because timings depend on network/server load and polling configuration. +sync-bench: $(TARGET) $(DIST_DIR)/sync_bench$(EXE) + @if [ -f .env ]; then \ + export $$(grep -v '^#' .env | xargs); \ + fi; \ + if [ -n "$(SYNC_BENCH_DATABASE_ID)" ]; then export SYNC_BENCH_DATABASE_ID="$(SYNC_BENCH_DATABASE_ID)"; fi; \ + if [ -n "$(SYNC_BENCH_CLOUDSYNC_ADDRESS)" ]; then export SYNC_BENCH_CLOUDSYNC_ADDRESS="$(SYNC_BENCH_CLOUDSYNC_ADDRESS)"; fi; \ + if [ -n "$(SYNC_BENCH_APIKEY)" ]; then export SYNC_BENCH_APIKEY="$(SYNC_BENCH_APIKEY)"; fi; \ + if [ -n "$(SYNC_BENCH_POLL_DELAY_MS)" ]; then export SYNC_BENCH_POLL_DELAY_MS="$(SYNC_BENCH_POLL_DELAY_MS)"; fi; \ + if [ -n "$(SYNC_BENCH_MAX_POLLS)" ]; then export SYNC_BENCH_MAX_POLLS="$(SYNC_BENCH_MAX_POLLS)"; fi; \ + if [ -n "$(SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES)" ]; then export SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES="$(SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES)"; fi; \ + if [ -n "$(SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS)" ]; then export SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS="$(SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS)"; fi; \ + if [ -n "$(SYNC_BENCH_OUTPUT)" ]; then export SYNC_BENCH_OUTPUT="$(SYNC_BENCH_OUTPUT)"; fi; \ + ./$(DIST_DIR)/sync_bench$(EXE) + +sync-bench-debug: + $(MAKE) SYNC_BENCH_DEBUG=1 sync-bench + OPENSSL_TARBALL = $(OPENSSL_DIR)/$(OPENSSL_VERSION).tar.gz $(OPENSSL_TARBALL): @@ -359,8 +438,8 @@ framework module CloudSync {\ } endef -LIB_NAMES = ios.dylib ios-sim.dylib macos.dylib -FMWK_NAMES = ios-arm64 ios-arm64_x86_64-simulator macos-arm64_x86_64 +LIB_NAMES = ios.dylib ios-sim.dylib mac-catalyst.dylib macos.dylib +FMWK_NAMES = ios-arm64 ios-arm64_x86_64-simulator ios-arm64_x86_64-maccatalyst macos-arm64_x86_64 $(DIST_DIR)/%.xcframework: $(LIB_NAMES) @$(foreach i,1 2 3,\ lib=$(word $(i),$(LIB_NAMES)); \ @@ -373,6 +452,21 @@ $(DIST_DIR)/%.xcframework: $(LIB_NAMES) mv $(DIST_DIR)/$$lib $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ install_name_tool -id "@rpath/CloudSync.framework/CloudSync" $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ ) + @lib=$(word 4,$(LIB_NAMES)); \ + fmwk=$(word 4,$(FMWK_NAMES)); \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Headers; \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Modules; \ + mkdir -p $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Resources; \ + cp src/cloudsync.h $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Headers/CloudSync.h; \ + printf "$(PLIST)" > $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Resources/Info.plist; \ + printf "$(MODULEMAP)" > $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/Modules/module.modulemap; \ + mv $(DIST_DIR)/$$lib $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/CloudSync; \ + install_name_tool -id "@rpath/CloudSync.framework/CloudSync" $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/A/CloudSync; \ + ln -sf A $(DIST_DIR)/$$fmwk/CloudSync.framework/Versions/Current; \ + ln -sf Versions/Current/CloudSync $(DIST_DIR)/$$fmwk/CloudSync.framework/CloudSync; \ + ln -sf Versions/Current/Headers $(DIST_DIR)/$$fmwk/CloudSync.framework/Headers; \ + ln -sf Versions/Current/Modules $(DIST_DIR)/$$fmwk/CloudSync.framework/Modules; \ + ln -sf Versions/Current/Resources $(DIST_DIR)/$$fmwk/CloudSync.framework/Resources; xcodebuild -create-xcframework $(foreach fmwk,$(FMWK_NAMES),-framework $(DIST_DIR)/$(fmwk)/CloudSync.framework) -output $@ rm -rf $(foreach fmwk,$(FMWK_NAMES),$(DIST_DIR)/$(fmwk)) @@ -413,13 +507,21 @@ help: @echo " android (needs ARCH to be set to x86_64, arm64-v8a, or armeabi-v7a and ANDROID_NDK to be set)" @echo " ios (only on macOS - can be compiled with native network support)" @echo " ios-sim (only on macOS - can be compiled with native network support)" + @echo " mac-catalyst (only on macOS - builds a universal arm64 + x86_64 Catalyst dylib)" @echo "" @echo "Targets:" @echo " all - Build the extension (default)" @echo " clean - Remove built files" @echo " test [COVERAGE=true] - Test the extension with optional coverage output" + @echo " unittest - Run only unit tests (test/unit.c)" @echo " help - Display this help message" @echo " xcframework - Build the Apple XCFramework" @echo " aar - Build the Android AAR package" + @echo "" + @echo "PostgreSQL Targets:" + @echo " make postgres-help - Show PostgreSQL-specific targets" + +# Include PostgreSQL extension targets +include docker/Makefile.postgresql -.PHONY: all clean test extension help version xcframework aar +.PHONY: all clean test unittest e2e extension help version xcframework aar diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 0000000..236ab95 --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,190 @@ +# Performance & Overhead + +This document describes the computational and storage overhead introduced by the CloudSync extension, and how sync execution time relates to database size. + +## TL;DR + +Sync execution time scales with **the number of changes since the last sync (D)**, not with total database size (N). If you sync frequently, D stays small regardless of how large the database grows. The per-operation overhead on writes is proportional to the number of columns in the affected row, not to the table size. This is fundamentally different from sync solutions that diff or scan the full dataset. + +## Breaking Down the Cost + +The overhead introduced by the extension can be decomposed into four independent concerns: + +### 1. Per-Operation Overhead (Write-Path Cost) + +Every INSERT, UPDATE, or DELETE on a synced table fires AFTER triggers that write CRDT metadata into a companion `
_cloudsync` table. This happens synchronously, inline with the original write. + +| Operation | Metadata Rows Written | Complexity | +|-----------|----------------------|------------| +| INSERT | 1 sentinel + 1 per non-PK column | O(C) | +| UPDATE | 1 per changed column (NEW != OLD) | O(C_changed) <= O(C) | +| DELETE | 1 sentinel + cleanup of existing metadata | O(C_existing) | + +Where **C** = number of non-PK columns in the table. + +**Key point:** This cost is **constant per row** and independent of the total number of rows in the table (N). Writing to a 100-row table costs the same as writing to a 10-million-row table. The metadata table uses a composite primary key `(pk, col_name)` with `WITHOUT ROWID` optimization (SQLite) or a standard B-tree primary key (PostgreSQL), so the index update cost is O(log M) where M is the metadata table size -- but this is the same cost as any indexed INSERT and is negligible in practice. + +### 2. Sync Operations (Push & Pull) + +These are the operations that create and apply sync payloads. They are synchronous in the extension and should typically be run by the application off the main thread. + +#### Push: Payload Generation + +``` +Cost: O(D) where D = number of column-level changes since last sync +``` + +The push operation queries `cloudsync_changes`, which dynamically reads from all synced `
_cloudsync` tables: +```sql +SELECT ... FROM cloudsync_changes WHERE db_version > +``` + +Each metadata table has an **index on `db_version`**, so payload generation scales primarily with the number of new changes, plus a small per-synced-table overhead to construct the `cloudsync_changes` query. It does not diff the full dataset. In SQLite, each changed column also performs a primary-key lookup in the base table to retrieve the current value. + +The resulting payload is LZ4-compressed before transmission. + +#### Pull: Payload Application + +``` +Cost: O(D) to decode + O(D_unique_pks) to merge into the database +``` + +Incoming changes are decoded and **batched by primary key**. All column changes for the same row are accumulated and flushed as a single UPDATE or INSERT statement. This batching reduces the number of actual database writes to one per affected row, regardless of how many columns changed. + +Conflict resolution (CRDT merge) is O(1) per column: it compares version numbers and, only if tied, falls back to value comparison and site-id tiebreaking. No global state or table scan is required. + +#### Summary + +| Phase | Scales With | Does NOT Scale With | +|-------|-------------|-------------------| +| Payload generation | D (changes since last sync) | N (total rows) | +| Payload application | D (incoming changes) | N (total rows) | +| Conflict resolution | D (conflicting columns) | N (total rows) | + +**This means sync time is driven mainly by delta size (`D`) rather than total database size (`N`)**. As long as the number of changes between syncs stays bounded, sync time remains roughly stable even as the database grows. + +### 3. Sync Frequency & Network Latency + +When the application runs sync off the main thread, perceived latency depends on: + +- **Sync interval**: How often the app triggers a push/pull cycle. More frequent syncs mean smaller deltas (smaller D) and faster individual sync operations, at the cost of more network round-trips. +- **Network latency**: The round-trip time to the sync server. LZ4 compression reduces payload size, but latency is dominated by the network hop itself for small deltas. +- **Payload size**: Proportional to D x average column value size. Large BLOBs or TEXT values will increase transfer time linearly. + +The extension does not impose a sync schedule -- the application controls when and how often to sync. A typical pattern is to sync on a timer (e.g., every 5-30 seconds) or on specific events (app foreground, user action). + +### 4. Metadata Storage Overhead + +Each synced table has a companion `
_cloudsync` metadata table with the following schema: + +``` +PRIMARY KEY (pk, col_name) -- WITHOUT ROWID (SQLite) +Columns: pk, col_name, col_version, db_version, site_id, seq +Index: db_version +``` + +**Storage cost per row in the base table:** +- 1 sentinel row (marks the row's existence/deletion state) +- 1 metadata row per non-PK column that has ever been written + +So for a table with C non-PK columns, the metadata table will contain approximately `N x (1 + C)` rows, where N is the number of rows in the base table. + +**Estimated overhead per metadata row:** +- `pk`: encoded primary key (typically 8-32 bytes depending on PK type and count) +- `col_name`: column name string (shared via SQLite's string interning, typically 5-30 bytes) +- `col_version`, `db_version`, `seq`: 3 integers (8 bytes each = 24 bytes) +- `site_id`: 1 integer (8 bytes) + +Rough estimate: **60-100 bytes per metadata row**, or **60-100 x (1 + C) bytes per base table row**. + +| Base Table | Columns (C) | Rows (N) | Estimated Metadata Size | +|------------|-------------|----------|------------------------| +| Small | 5 | 1,000 | ~360 KB - 600 KB | +| Medium | 10 | 100,000 | ~66 MB - 110 MB | +| Large | 10 | 1,000,000| ~660 MB - 1.1 GB | +| Wide | 50 | 100,000 | ~306 MB - 510 MB | + +**Mitigation strategies:** +- Only sync tables that need it -- not every table requires CRDT tracking. +- Prefer narrow tables (fewer columns) for high-volume data. +- The `WITHOUT ROWID` optimization (SQLite) significantly reduces per-row storage overhead. +- Deleted rows have their per-column metadata cleaned up, but a tombstone sentinel row persists (see section 9 below). + +### 5. Read-Path Overhead + +Normal application reads are not directly instrumented by the extension. No triggers, views, or hooks intercept ordinary SELECT queries on application tables, and the CRDT metadata is stored separately. In practice, read overhead is usually negligible. + +### 6. Initial Sync (First Device) + +When a new device syncs for the first time (`db_version = 0`), the push payload contains the **entire dataset**: every column of every row across all synced tables. The payload size is proportional to `N * C` (total rows times columns). + +The payload is built entirely in memory, starting with a 512 KB buffer (`CLOUDSYNC_PAYLOAD_MINBUF_SIZE` in `src/cloudsync.c`) and growing via `realloc` as needed. Peak memory usage is at least the full uncompressed payload size and can be higher during compression. For a database with 1 million rows and 10 columns of average 50 bytes each, the uncompressed payload could reach ~500 MB before LZ4 compression. + +Subsequent syncs are incremental (proportional to D, changes since the last sync), so the first sync is the expensive one. Applications with large datasets should plan for this -- for example, by seeding new devices from a database snapshot rather than syncing from scratch. + +### 7. WAL and Disk I/O Amplification + +Each write to a synced table generates additional metadata writes via AFTER triggers. The amplification factor depends on the operation: + +| Operation | Total Writes (base + metadata) | Amplification Factor | +|-----------|-------------------------------|---------------------| +| INSERT (C columns) | 1 + 1 sentinel + C metadata | ~C+2x | +| UPDATE (1 column) | 1 + 1 metadata | 2x | +| UPDATE (C columns) | 1 + C metadata | ~C+1x | +| DELETE | 1 + cleanup writes | variable | + +For a table with 10 non-PK columns, an INSERT generates roughly 12 logical row writes instead of 1. This increases WAL/page churn and affects: + +- **Disk I/O**: More pages written per transaction, larger WAL files between checkpoints. +- **WAL checkpoint frequency**: The WAL grows faster, so checkpoints run more often (or the WAL file stays larger if checkpointing is deferred). +- **Battery on mobile**: More disk writes per user action. Batching multiple writes in a single transaction amortizes the transaction overhead but not the per-row metadata cost. + +### 8. Locking During Sync Apply + +Payload application (`cloudsync_payload_apply`) uses savepoints grouped by source `db_version`. On SQLite, each savepoint holds a write lock for its duration. If the application runs sync on the main thread, other work on the same connection is blocked, and reads from other connections may block outside WAL mode. + +On SQLite, using WAL mode prevents readers on other connections from being blocked by writers, which is the recommended configuration for concurrent sync. + +### 9. Metadata Lifecycle (Tombstones and Cleanup) + +When a row is deleted, the per-column metadata rows are removed, but a **tombstone sentinel** (`__[RIP]__`) persists in the metadata table. This tombstone is necessary for propagating deletes to other devices during sync. There is no automatic garbage collection of tombstones -- they accumulate over time. + +Metadata cleanup for **removed columns** (after schema migration) only runs during `cloudsync_finalize_alter()`, which is called as part of the `cloudsync_alter()` workflow. Outside of schema changes, orphaned metadata from dropped columns remains in the metadata table. + +The **site ID table** (`cloudsync_site_id`) also grows monotonically -- one entry per unique device that has ever synced. This is typically small (one row per device) and not a concern in practice. + +For applications with high delete rates, the tombstone accumulation may become significant over time. Consider periodic full re-syncs or application-level archival strategies if this is a concern. + +### 10. Multi-Table Considerations + +The `cloudsync_changes` virtual table (SQLite) or set-returning function (PostgreSQL) dynamically constructs a `UNION ALL` query across all synced tables' metadata tables. The query construction cost scales as O(T) where T is the number of synced tables. + +For most applications (fewer than ~50 synced tables), this is negligible. Applications syncing a very large number of tables should be aware that payload generation involves iterating over all synced tables to check for changes. + +### Platform Differences (SQLite vs PostgreSQL) + +- **SQLite** uses native C triggers registered directly with the SQLite API. Metadata tables use `WITHOUT ROWID` for compact storage. +- **PostgreSQL** uses row-level PL/pgSQL trigger functions that call into C functions via the extension. This adds a small amount of overhead per trigger invocation compared to SQLite's direct C triggers. Additionally, merge operations use per-PK savepoints to handle failures such as RLS policy violations gracefully. +- **Table registration** (`cloudsync_enable()`) is a one-time operation on both platforms. It creates 1 metadata table, 1 index, and 3 triggers (INSERT, UPDATE, DELETE), plus ~15-20 prepared statements that are cached for the lifetime of the connection. + +## Comparison with Full-Scan Sync Solutions + +Many sync solutions must diff or hash the entire dataset to determine what changed. This leads to O(N) sync time that grows linearly with total database size -- the exact problem described in the question. + +CloudSync avoids this through its **monotonic versioning approach**: every write increments a monotonic `db_version` counter, and the sync query filters on this counter using an index. The result is that sync time depends mainly on the volume of changes (D), not on the total data size (N). + +``` +Full-scan sync: sync_time ~ O(N) -- grows with database size +CloudSync: sync_time ~ O(D) -- grows with changes since last sync + where D is independent of N when sync frequency is constant +``` + +## Performance Optimizations in the Implementation + +1. **`WITHOUT ROWID` tables** (SQLite): Metadata tables use clustered primary keys, avoiding the overhead of a separate rowid B-tree. +2. **`db_version` index**: Enables efficient range scans for delta extraction. +3. **Deferred batch merge**: Column changes for the same primary key are accumulated and flushed as a single SQL statement. +4. **Prepared statement caching**: Merge statements are compiled once and reused across rows. +5. **LZ4 compression**: Reduces payload size for network transfer. +6. **Per-column tracking**: Only changed columns are included in the sync payload, not entire rows. +7. **Early exit on stale data**: The CLS algorithm skips rows where the incoming causal length is lower than the local one, avoiding unnecessary column-level comparisons. diff --git a/Package.swift b/Package.swift index 9e0db59..a4a1fda 100644 --- a/Package.swift +++ b/Package.swift @@ -5,29 +5,22 @@ import PackageDescription let package = Package( name: "CloudSync", - platforms: [.macOS(.v11), .iOS(.v11)], + platforms: [.macOS(.v11), .iOS(.v11), .macCatalyst(.v14)], products: [ - // Products can be used to vend plugins, making them visible to other packages. - .plugin( - name: "CloudSyncPlugin", - targets: ["CloudSyncPlugin"]), .library( name: "CloudSync", targets: ["CloudSync"]) ], targets: [ - // Build tool plugin that invokes the Makefile - .plugin( - name: "CloudSyncPlugin", - capability: .buildTool(), - path: "packages/swift/plugin" + .binaryTarget( + name: "CloudSyncBinary", + url: "https://github.com/sqliteai/sqlite-sync/releases/download/1.0.20/cloudsync-apple-xcframework-1.0.20.zip", + checksum: "21d2c3e8e3770fd7498502eff4910b6f1867cd9c0ce00e707931f5ffdd9ad4fc" ), - // CloudSync library target .target( name: "CloudSync", - dependencies: [], - path: "packages/swift/extension", - plugins: ["CloudSyncPlugin"] + dependencies: ["CloudSyncBinary"], + path: "packages/swift" ), ] -) \ No newline at end of file +) diff --git a/README.md b/README.md index c2dc380..6bdaa30 100644 --- a/README.md +++ b/README.md @@ -1,496 +1,278 @@ -# SQLite Sync - -[![sqlite-sync coverage](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F&search=%3Ctd%20class%3D%22headerItem%22%3EFunctions%3A%3C%5C%2Ftd%3E%5Cs*%3Ctd%20class%3D%22headerCovTableEntryHi%22%3E(%5B%5Cd.%5D%2B)%26nbsp%3B%25%3C%5C%2Ftd%3E&replace=%241%25&label=coverage&labelColor=rgb(85%2C%2085%2C%2085)%3B&color=rgb(167%2C%20252%2C%20157)%3B&link=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F)](https://sqliteai.github.io/sqlite-sync/) - -**SQLite Sync** is a multi-platform extension that brings a true **local-first experience** to your applications with minimal effort. It extends standard SQLite tables with built-in support for offline work and automatic synchronization, allowing multiple devices to operate independently—even without a network connection—and seamlessly stay in sync. With SQLite Sync, developers can easily build **distributed, collaborative applications** while continuing to rely on the **simplicity, reliability, and performance of SQLite**. - -Under the hood, SQLite Sync uses advanced **CRDT (Conflict-free Replicated Data Type)** algorithms and data structures designed specifically for **collaborative, distributed systems**. This means: - -- Devices can update data independently, even without a network connection. -- When they reconnect, all changes are **merged automatically and without conflicts**. -- **No data loss. No overwrites. No manual conflict resolution.** - -In simple terms, CRDTs make it possible for multiple users to **edit shared data at the same time**, from anywhere, and everything just works. - -## Table of Contents -- [Key Features](#key-features) -- [Built-in Network Layer](#built-in-network-layer) -- [Row-Level Security](#row-level-security) -- [What Can You Build with SQLite Sync?](#what-can-you-build-with-sqlite-sync) -- [Documentation](#documentation) -- [Installation](#installation) -- [Getting Started](#getting-started) -- [Database Schema Recommendations](#database-schema-recommendations) - - [Primary Key Requirements](#primary-key-requirements) - - [Column Constraint Guidelines](#column-constraint-guidelines) - - [UNIQUE Constraint Considerations](#unique-constraint-considerations) - - [Foreign Key Compatibility](#foreign-key-compatibility) - - [Trigger Compatibility](#trigger-compatibility) -- [License](#license) - -## Key Features - -- **Offline-First by Design**: Works seamlessly even when devices are offline. Changes are queued locally and synced automatically when connectivity is restored. -- **CRDT-Based Conflict Resolution**: Merges updates deterministically and efficiently, ensuring eventual consistency across all replicas without the need for complex merge logic. -- **Embedded Network Layer**: No external libraries or sync servers required. SQLiteSync handles connection setup, message encoding, retries, and state reconciliation internally. -- **Drop-in Simplicity**: Just load the extension into SQLite and start syncing. No need to implement custom protocols or state machines. -- **Efficient and Resilient**: Optimized binary encoding, automatic batching, and robust retry logic make synchronization fast and reliable even on flaky networks. - -Whether you're building a mobile app, IoT device, or desktop tool, SQLite Sync simplifies distributed data management and unlocks the full potential of SQLite in decentralized environments. - -## Built-in Network Layer - -Unlike traditional sync systems that require you to build and maintain a complex backend, **SQLite Sync includes a built-in network layer** that works out of the box: - -- Sync your database with the cloud using **a single function call**. -- Compatible with **any language or framework** that supports SQLite. -- **No backend setup required** — SQLite Sync handles networking, change tracking, and conflict resolution for you. - -The sync layer is tightly integrated with [**SQLite Cloud**](https://sqlitecloud.io/), enabling seamless and secure data sharing across devices, users, and platforms. You get the power of cloud sync without the complexity. +
+ + SQLite AI + + +

SQLite-Sync

+

Offline-first sync for SQLite, powered by CRDTs.
+ Local writes, conflict-free merges, real-time collaboration across devices. Sync to SQLite Cloud, PostgreSQL, or Supabase — no central coordinator required.

+ +

+ Free managed instance → · + Docs · + Live Demo · + Website +

+ +

+ Data: + Vector · + Sync · + Columnar · + JS +
+ AI: + AI · + Agent · + Memory · + MCP + +

+
+ +
+ +> **Need a sync backend?** Plug into any PostgreSQL or Supabase instance, or use **[SQLite Cloud CloudSync](https://www.sqlite.ai/pricing)** — managed device sync with auth, ACL, and a free tier for up to 3 devices. -## Row-Level Security - -Thanks to the underlying SQLite Cloud infrastructure, **SQLite Sync supports Row-Level Security (RLS)**—allowing you to define **precise access control at the row level**: - -- Control not just who can read or write a table, but **which specific rows** they can access. -- Enforce security policies on the server—no need for client-side filtering. - -For example: - -- User A can only see and edit their own data. -- User B can access a different set of rows—even within the same shared table. - -**Benefits of RLS**: +--- -- **Data isolation**: Ensure users only access what they’re authorized to see. -- **Built-in privacy**: Security policies are enforced at the database level. -- **Simplified development**: Reduce or eliminate complex permission logic in your application code. +# SQLite Sync -### What Can You Build with SQLite Sync? +[![sqlite-sync coverage](https://img.shields.io/badge/dynamic/regex?url=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F&search=Functions%3A%3C%5C%2Ftd%3E%5Cs*%3Ctd%20class%3D%22headerCovTableEntry(?:Hi|Med|Lo)%22%3E(%5B%5Cd.%5D%2B)%26nbsp%3B%25&replace=%241%25&label=coverage&labelColor=rgb(85%2C%2085%2C%2085)%3B&color=rgb(167%2C%20252%2C%20157)%3B&link=https%3A%2F%2Fsqliteai.github.io%2Fsqlite-sync%2F)](https://sqliteai.github.io/sqlite-sync/) -SQLite Sync is ideal for building collaborative and distributed apps across web, mobile, desktop, and edge platforms. Some example use cases include: +**SQLite Sync** is a multi-platform extension that turns any SQLite database into a **conflict-free, offline-first replica** that syncs automatically with **[SQLite Cloud](https://sqlitecloud.io/)** nodes, **PostgreSQL** servers, and **Supabase** instances. One function call is all it takes: no backend to build, no sync protocol to implement. -#### 📋 Productivity & Collaboration +Built on **CRDT** (Conflict-free Replicated Data Types), it guarantees: -- **Shared To-Do Lists**: Users independently update tasks and sync effortlessly. -- **Note-Taking Apps**: Real-time collaboration with offline editing. -- **Markdown Editors**: Work offline, sync when back online—no conflicts. +- **No data loss.** Devices update independently, even offline, and all changes merge automatically. +- **No conflicts.** Deterministic merge, no manual conflict resolution, ever. +- **No extra infrastructure.** A globally distributed network of **CloudSync microservices** handles routing, packaging, and delivery of changes between SQLite and other DBMS nodes. -#### 📱 Mobile & Edge +## Why SQLite Sync? -- **Field Data Collection**: For remote inspections, agriculture, or surveys. -- **Point-of-Sale Systems**: Offline-first retail solutions with synced inventory. -- **Health & Fitness Apps**: Sync data across devices with strong privacy controls. +**For offline-first apps** (mobile, desktop, IoT, edge): devices work with a local SQLite database and sync when connectivity is available. Changes queue locally and merge seamlessly on reconnect. -#### 🏢 Enterprise Workflows +**For AI agents**: agents that maintain memory, notes, or shared state in SQLite can sync across instances without coordination. **[Block-Level LWW](#block-level-lww)** was specifically designed to keep **markdown files** in sync: multiple agents editing different sections of the same document preserve all changes after sync. -- **CRM Systems**: Sync leads and clients per user with row-level access control. -- **Project Management Tools**: Offline-friendly planning and task management. -- **Expense Trackers**: Sync team expenses securely and automatically. +## What Can You Build with SQLite Sync? -#### 🧠 Personal Apps +### Offline-First Apps +- **Shared To-Do Lists**: users independently update tasks and sync effortlessly. +- **Note-Taking Apps**: real-time collaboration with offline editing. +- **Field Data Collection**: for remote inspections, agriculture, or surveys. +- **Point-of-Sale Systems**: offline-first retail solutions with synced inventory. -- **Journaling & Diaries**: Private, encrypted entries that sync across devices. -- **Bookmarks & Reading Lists**: Personal or collaborative content management. -- **Habit Trackers**: Sync progress with data security and consistency. +### AI Agent Sync +- **Agent Memory**: multiple agents share and update a common SQLite database, syncing state across instances without coordination. +- **Markdown Knowledge Bases**: agents independently edit different sections of shared markdown documents, with Block-Level LWW preserving all changes. +- **Distributed Pipelines**: agents running on different nodes accumulate results locally and merge them into a single consistent dataset. -#### 🌍 Multi-User, Multi-Tenant Systems +### Enterprise and Multi-Tenant +- **CRM Systems**: sync leads and clients per user with row-level access control. +- **SaaS Platforms**: row-level access for each user or team using a single shared database. +- **Project Management Tools**: offline-friendly planning and task management. -- **SaaS Platforms**: Row-level access for each user or team. -- **Collaborative Design Tools**: Merge visual edits and annotations offline. -- **Educational Apps**: Shared learning content with per-student access controls. +### Personal Apps +- **Journaling and Diaries**: private entries that sync across devices. +- **Habit Trackers**: sync progress with data security and consistency. +- **Bookmarks and Reading Lists**: personal or collaborative content management. -## Documentation +## Key Features -For detailed information on all available functions, their parameters, and examples, refer to the [comprehensive API Reference](./API.md). +| Feature | Description | +|---------|-------------| +| **CRDT-based sync** | Causal-Length Set, Delete-Wins, Add-Wins, and Grow-Only Set algorithms | +| **Block-Level LWW** | Line-level merge for text/markdown columns, concurrent edits to different lines are preserved | +| **Built-in networking** | Embedded network layer (libcurl or native), single function call to sync | +| **Row-Level Security** | Server-enforced RLS: each client syncs only the rows it is authorized to see | +| **Multi-platform** | Linux, macOS, Windows, iOS, Android, WASM | -## Installation +## Quick Start -### Pre-built Binaries +### 1. Install -Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: +Download a pre-built binary from the [Releases](https://github.com/sqliteai/sqlite-sync/releases) page, or install a platform package (see [full installation guide](./docs/INSTALLATION.md) for platform-specific code examples): -- Linux: x86 and ARM -- macOS: x86 and ARM -- Windows: x86 -- Android -- iOS +| Platform | Install | +|----------|---------| +| **SQLite CLI / C** | `.load ./cloudsync` or `SELECT load_extension('./cloudsync');` | +| **Swift** | [Add this repo as a Swift Package dependency](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app), follow [steps 4 and 5](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading), and load extension with `CloudSync.path` | +| **Android** | `implementation 'ai.sqlite:sync:1.0.0'` ([Maven Central](https://central.sonatype.com/artifact/ai.sqlite/sync)) | +| **Flutter** | `flutter pub add sqlite_sync` ([pub.dev](https://pub.dev/packages/sqlite_sync)) | +| **Expo** | `npm install @sqliteai/sqlite-sync-expo` | +| **React Native** | `npm install @sqliteai/sqlite-sync-react-native` | +| **WASM** | `npm install @sqliteai/sqlite-wasm` ([npm](https://www.npmjs.com/package/@sqliteai/sqlite-wasm)) | -### Loading the Extension +### 2. Create a table and enable sync ```sql --- In SQLite CLI .load ./cloudsync --- In SQL -SELECT load_extension('./cloudsync'); -``` - -### WASM Version - -You can download the WebAssembly (WASM) version of SQLite with the SQLite Sync extension enabled from: https://www.npmjs.com/package/@sqliteai/sqlite-wasm - -### Swift Package - -You can [add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, you'll need to set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). - -Here's an example of how to use the package: -```swift -import CloudSync - -... - -var db: OpaquePointer? -sqlite3_open(":memory:", &db) -sqlite3_enable_load_extension(db, 1) -var errMsg: UnsafeMutablePointer? = nil -sqlite3_load_extension(db, CloudSync.path, nil, &errMsg) -var stmt: OpaquePointer? -sqlite3_prepare_v2(db, "SELECT cloudsync_version()", -1, &stmt, nil) -defer { sqlite3_finalize(stmt) } -sqlite3_step(stmt) -log("cloudsync_version(): \(String(cString: sqlite3_column_text(stmt, 0)))") -sqlite3_close(db) -``` - -### Android Package - -Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: - -```gradle -implementation 'ai.sqlite:sync:0.8.41' -``` - -Here's an example of how to use the package: -```java -SQLiteCustomExtension cloudsyncExtension = new SQLiteCustomExtension(getApplicationInfo().nativeLibraryDir + "/cloudsync", null); -SQLiteDatabaseConfiguration config = new SQLiteDatabaseConfiguration( - getCacheDir().getPath() + "/cloudsync_test.db", - SQLiteDatabase.CREATE_IF_NECESSARY | SQLiteDatabase.OPEN_READWRITE, - Collections.emptyList(), - Collections.emptyList(), - Collections.singletonList(cloudsyncExtension) +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + done INTEGER NOT NULL DEFAULT 0 ); -SQLiteDatabase db = SQLiteDatabase.openDatabase(config, null, null); -``` - -**Note:** Additional settings and configuration are required for a complete setup. For full implementation details, see the [complete Android example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/android/README.md). -### Expo - -Install the Expo package: - -```bash -npm install @sqliteai/sqlite-sync-expo -``` - -Add to your `app.json`: - -```json -{ - "expo": { - "plugins": ["@sqliteai/sqlite-sync-expo"] - } -} -``` - -Run prebuild: - -```bash -npx expo prebuild --clean +-- Enable CRDT sync on the table +SELECT cloudsync_init('tasks'); ``` -Load the extension: +### 3. Use your database normally -```typescript -import { Platform } from 'react-native'; -import { getDylibPath, open } from '@op-engineering/op-sqlite'; +```sql +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Buy groceries'); +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Review PR #42'); -const db = open({ name: 'mydb.db' }); +UPDATE tasks SET done = 1 WHERE title = 'Buy groceries'; -// Load SQLite Sync extension -if (Platform.OS === 'ios') { - const path = getDylibPath('ai.sqlite.cloudsync', 'CloudSync'); - db.loadExtension(path); -} else { - db.loadExtension('cloudsync'); -} +SELECT * FROM tasks; ``` -### Flutter Package +### 4. Sync with the cloud -Add the [sqlite_sync](https://pub.dev/packages/sqlite_sync) package to your project: +```sql +-- Connect to your SQLite Cloud managed database +-- (get the managed database ID from the OffSync page on the SQLite Cloud dashboard) +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_apikey('your-api-key'); -```bash -flutter pub add sqlite_sync # Flutter projects -dart pub add sqlite_sync # Dart projects -``` +-- Send local changes and receive remote changes +SELECT cloudsync_network_sync(); +-- Returns JSON: {"send":{"status":"synced","localVersion":3,"serverVersion":3},"receive":{"rows":0,"tables":[]}} -Usage with `sqlite3` package: -```dart -import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite_sync/sqlite_sync.dart'; +-- Call periodically to stay in sync +SELECT cloudsync_network_sync(); -sqlite3.loadSqliteSyncExtension(); -final db = sqlite3.openInMemory(); -print(db.select('SELECT cloudsync_version()')); +-- Before closing the connection +SELECT cloudsync_terminate(); ``` -For a complete example, see the [Flutter example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/flutter/README.md). +### 5. Sync from another device -## Getting Started - -Here's a quick example to get started with SQLite Sync: - -### Prerequisites - -1. **SQLite Cloud Account**: Sign up at [SQLite Cloud](https://sqlitecloud.io/) -2. **SQLite Sync Extension**: Download from [Releases](https://github.com/sqliteai/sqlite-sync/releases) - -### SQLite Cloud Setup - -1. Create a new project and database in your [SQLite Cloud Dashboard](https://dashboard.sqlitecloud.io/) -2. Copy your connection string and API key from the dashboard -3. Create tables with identical schema in both local and cloud databases -4. Enable synchronization: click the **"OffSync"** button for your database and select each table you want to synchronize - -### Local Database Setup - -```bash -# Start SQLite CLI -sqlite3 myapp.db -``` +On a second device (or a second database for testing), repeat the same setup: ```sql --- Load the extension +-- Device B: load extension, create the same table, init sync .load ./cloudsync --- Create a table (primary key MUST be TEXT for global uniqueness) -CREATE TABLE IF NOT EXISTS my_data ( - id TEXT PRIMARY KEY NOT NULL, - value TEXT NOT NULL DEFAULT '', - created_at TEXT DEFAULT CURRENT_TIMESTAMP +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + done INTEGER NOT NULL DEFAULT 0 ); --- Initialize table for synchronization -SELECT cloudsync_init('my_data'); - --- Use your local database normally: read and write data using standard SQL queries --- The CRDT system automatically tracks all changes for synchronization - --- Example: Insert data (always use cloudsync_uuid() for globally unique IDs) -INSERT INTO my_data (id, value) VALUES - (cloudsync_uuid(), 'Hello from device A!'), - (cloudsync_uuid(), 'Working offline is seamless!'); - --- Example: Update and delete operations work normally -UPDATE my_data SET value = 'Updated: Hello from device A!' WHERE value LIKE 'Hello from device A!'; - --- View your data -SELECT * FROM my_data ORDER BY created_at; - --- Configure network connection before using the network sync functions -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/database.sqlite'); -SELECT cloudsync_network_set_apikey('your-api-key-here'); --- Or use token authentication (required for Row-Level Security) --- SELECT cloudsync_network_set_token('your_auth_token'); - --- Sync with cloud: send local changes, then check the remote server for new changes --- and, if a package with changes is ready to be downloaded, applies them to the local database -SELECT cloudsync_network_sync(); --- Keep calling periodically. The function returns > 0 if data was received --- In production applications, you would typically call this periodically --- rather than manually (e.g., every few seconds) -SELECT cloudsync_network_sync(); - --- Before closing the database connection -SELECT cloudsync_terminate(); --- Close the database connection -.quit -``` -```sql --- On another device (or create another database for testing: sqlite3 myapp_2.db) --- Follow the same setup steps: load extension, create table, init sync, configure network - --- Load extension and create identical table structure -.load ./cloudsync -CREATE TABLE IF NOT EXISTS my_data ( - id TEXT PRIMARY KEY NOT NULL, - value TEXT NOT NULL DEFAULT '', - created_at TEXT DEFAULT CURRENT_TIMESTAMP -); -SELECT cloudsync_init('my_data'); +SELECT cloudsync_init('tasks'); -- Connect to the same cloud database -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/database.sqlite'); -SELECT cloudsync_network_set_apikey('your-api-key-here'); +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_apikey('your-api-key'); --- Sync to get data from the first device +-- Pull changes from Device A SELECT cloudsync_network_sync(); --- repeat until data is received (returns > 0) +-- Call again: the first call triggers package preparation, the second downloads it SELECT cloudsync_network_sync(); --- View synchronized data -SELECT * FROM my_data ORDER BY created_at; +-- Device A's tasks are now here +SELECT * FROM tasks; --- Add data from this device to test bidirectional sync -INSERT INTO my_data (id, value) VALUES - (cloudsync_uuid(), 'Hello from device B!'); +-- Add data from this device +INSERT INTO tasks (id, title) VALUES (cloudsync_uuid(), 'Call the dentist'); --- Sync again to send this device's changes +-- Send this device's changes to the cloud SELECT cloudsync_network_sync(); --- The CRDT system ensures all devices eventually have the same data, --- with automatic conflict resolution and no data loss - --- Before closing the database connection +-- Before closing the connection SELECT cloudsync_terminate(); --- Close the database connection -.quit ``` -### For a Complete Example +Back on Device A, calling `cloudsync_network_sync()` will pull Device B's changes. The CRDT engine ensures all devices converge to the same data, automatically, with no conflicts. -See the [examples](./examples/simple-todo-db/) directory for a comprehensive walkthrough including: -- Multi-device collaboration -- Offline scenarios -- Row-level security setup -- Conflict resolution demonstrations +> **Note:** every device participating in the same sync must create **the same set of tables with the same structure** and initialize each one with `cloudsync_init()`. sqlite-sync derives a schema hash from the synced tables, and the server rejects payloads whose hash it does not recognize. For multi-tenant setups where each client should see only a subset of rows, use a shared schema with a tenant/scope column and enforce isolation with [Row-Level Security](./docs/row-level-security.md) — do not give each client a different table. -## 📦 Integrations +## Block-Level LWW -Use SQLite-AI alongside: - -* **[SQLite-AI](https://github.com/sqliteai/sqlite-ai)** – on-device inference, embedding generation, and model interaction directly into your database -* **[SQLite-Vector](https://github.com/sqliteai/sqlite-vector)** – vector search from SQL -* **[SQLite-JS](https://github.com/sqliteai/sqlite-js)** – define SQLite functions in JavaScript - -## Database Schema Recommendations - -When designing your database schema for SQLite Sync, follow these best practices to ensure optimal CRDT performance and conflict resolution: - -### Primary Key Requirements - -- **Use globally unique identifiers**: Always use TEXT primary keys with UUIDs, ULIDs, or similar globally unique identifiers -- **Avoid auto-incrementing integers**: Integer primary keys can cause conflicts across multiple devices -- **Use `cloudsync_uuid()`**: The built-in function generates UUIDv7 identifiers optimized for distributed systems -- **All primary keys must be explicitly declared as `NOT NULL`**. +Standard CRDT sync replaces an entire cell when two devices edit the same column. **Block-Level LWW** splits text into lines and merges them independently, designed for keeping **markdown files and agent memory** in sync. ```sql --- ✅ Recommended: Globally unique TEXT primary key -CREATE TABLE users ( - id TEXT PRIMARY KEY NOT NULL, -- Use cloudsync_uuid() - name TEXT NOT NULL, - email TEXT UNIQUE NOT NULL +CREATE TABLE notes ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '' ); --- ❌ Avoid: Auto-incrementing integer primary key -CREATE TABLE users ( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- Causes conflicts - name TEXT NOT NULL, - email TEXT UNIQUE NOT NULL -); +SELECT cloudsync_init('notes'); +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); ``` -### Column Constraint Guidelines +Now two agents (or devices) can edit different lines of the same note, and both edits are preserved after sync. See the full guide: **[Block-Level LWW Documentation](./docs/block-lww.md)**. -- **Provide DEFAULT values**: All `NOT NULL` columns (except primary keys) must have `DEFAULT` values -- **Consider nullable columns**: For optional data, use nullable columns instead of empty strings - -```sql --- ✅ Recommended: Proper constraints and defaults -CREATE TABLE tasks ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT '', - status TEXT NOT NULL DEFAULT 'pending', - priority INTEGER NOT NULL DEFAULT 1, - created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), - assigned_to TEXT -- Nullable for optional assignment -); -``` +## Row-Level Security -### UNIQUE Constraint Considerations +With SQLite Cloud's RLS, a single shared cloud database serves all users while each client only sees and syncs its own rows. Policies are enforced server-side: a compromised client cannot bypass access controls. -When converting from single-tenant to multi-tenant database schemas with Row-Level Security, **UNIQUE constraints must be globally unique** across all tenants in the cloud database. For columns that should only be unique within a tenant, use composite UNIQUE constraints. +- One database, multiple tenants, no per-user database provisioning. +- Each client syncs only authorized rows, minimal bandwidth and storage. -```sql --- ❌ Single-tenant: Unique email per database -CREATE TABLE users ( - id TEXT PRIMARY KEY, - email TEXT UNIQUE NOT NULL -- Problem: Not unique across tenants -); +See the full guide: **[Row-Level Security Documentation](./docs/row-level-security.md)**. --- ✅ Multi-tenant: Composite unique constraint -CREATE TABLE users ( - id TEXT PRIMARY KEY, - tenant_id TEXT NOT NULL, - email TEXT NOT NULL, - UNIQUE(tenant_id, email) -- Unique email per tenant -); -``` +## Documentation -### Foreign Key Compatibility +- **[API Reference](./API.md)**: all functions, parameters, and examples +- **[Installation Guide](./docs/installation.md)**: platform-specific setup (Swift, Android, Expo, React Native, Flutter, WASM) +- **[Block-Level LWW Guide](./docs/block-lww.md)**: line-level text merge for markdown and documents +- **[Row-Level Security Guide](./docs/row-level-security.md)**: multi-tenant access control with server-enforced policies +- **[Database Schema Recommendations](./docs/schema.md)**: primary keys, constraints, foreign keys, triggers +- **[Custom Network Layer](./docs/internal/network.md)**: replace the built-in libcurl networking +- **[Examples](./examples/)**: complete walkthroughs (todo app, sport tracker, Swift multiplatform) -When using foreign key constraints with SQLite Sync, be aware that interactions with the CRDT merge algorithm and Row-Level Security policies may cause constraint violations. +## SQLite Cloud Setup -#### Potential Conflicts +1. Sign up at [SQLite Cloud](https://sqlitecloud.io/) and create a project. +2. Create a database and your tables in the [dashboard](https://dashboard.sqlitecloud.io/). +3. Enable synchronization: click **"OffSync"** for your database and select the tables to sync. +4. Copy the managed database ID and API key from the dashboard. +5. Use `cloudsync_network_init()` and `cloudsync_network_set_apikey()` locally, then call `cloudsync_network_sync()`. -**CRDT Merge Algorithm and DEFAULT Values** +For token-based authentication (required for RLS), use `cloudsync_network_set_token()` instead of `cloudsync_network_set_apikey()`. -- CRDT changes are applied column-by-column during synchronization -- Columns may be temporarily assigned DEFAULT values during the merge process -- If a foreign key column has a DEFAULT value, that value must exist in the referenced table +## Versioning -**Row-Level Security and CASCADE Actions** -- RLS policies may block operations required for maintaining referential integrity -- CASCADE DELETE/UPDATE operations may fail if RLS prevents access to related rows +This project follows [semver](https://semver.org/). The single source of truth is `CLOUDSYNC_VERSION` in `src/cloudsync.h`; all packaged artifacts (NPM, Maven, pub.dev, Swift, Docker, native tarballs) inherit this version. PATCH releases never alter the exposed API — they ship bug fixes, performance improvements, and internal changes only. -#### Recommendations +The PostgreSQL extension differs only in how it surfaces the version: its catalog version (`default_version` / `installed_version`) exposes `MAJOR.MINOR` only, so PATCH releases are transparent binary upgrades and only MINOR/MAJOR releases need `ALTER EXTENSION cloudsync UPDATE`. The `cloudsync_version()` SQL function always reports the full semver of the loaded `.so`. See the [PostgreSQL upgrade docs](docs/postgresql/quickstarts/postgres.md#upgrading-a-later-release) for the user-facing procedure. -**Database Design Patterns** -- Prefer application-level cascade logic over database-level CASCADE actions -- Design RLS policies to accommodate referential integrity operations -- Use nullable foreign keys where appropriate to avoid DEFAULT value issues -- Alternatively, ensure DEFAULT values for foreign key columns exist in their referenced tables +## License -**Testing and Validation** -- Test synchronization scenarios with foreign key constraints enabled -- Monitor for constraint violations during sync operations in development +This project is licensed under the [Elastic License 2.0](./LICENSE.md). For production or managed service use, [contact SQLite Cloud, Inc](mailto:info@sqlitecloud.io) for a commercial license. -### Trigger Compatibility +--- -Be aware that certain types of triggers can cause errors during synchronization due to SQLite Sync's merge logic. +## ☁️ Hosted version -**Duplicate Operations** -- If a trigger modifies a table that is also synchronized with SQLite Sync, changes performed by the trigger may be applied twice during the merge operation -- This can lead to constraint violations or unexpected data states depending on the table's constraints +Don't want to run a sync server yourself? **[SQLite Cloud CloudSync](https://sqlite.ai)** is the managed backend for SQLite-Sync — works with SQLite Cloud, PostgreSQL, or Supabase as your source of truth, with auth, ACL, and observability included. -**Column-by-Column Processing** -- SQLite Sync applies changes column-by-column during synchronization -- UPDATE triggers may be called multiple times for a single row as each column is processed -- This can result in unexpected trigger behavior +[**Start free →**](https://dashboard.sqlitecloud.io/auth/sign-in) --- -## License - -This project is licensed under the [Elastic License 2.0](./LICENSE.md). You can use, copy, modify, and distribute it under the terms of the license for non-production use. For production or managed service use, please [contact SQLite Cloud, Inc](mailto:info@sqlitecloud.io) for a commercial license. - ---- +## Part of the SQLite AI stack -## Part of the SQLite AI Ecosystem +SQLite-Sync is one piece of a larger ecosystem that turns SQLite into a runtime for intelligent, distributed data: -This project is part of the **SQLite AI** ecosystem, a collection of extensions that bring modern AI capabilities to the world’s most widely deployed database. The goal is to make SQLite the default data and inference engine for Edge AI applications. +**Data layer** +- [sqlite-vector](https://github.com/sqliteai/sqlite-vector) — ANN vector search inside SQLite +- [**sqlite-sync**](https://github.com/sqliteai/sqlite-sync) — Offline-first CRDT sync across devices *(you are here)* +- [sqlite-columnar](https://github.com/sqliteai/sqlite-columnar) — Column-oriented analytics for OLAP queries +- [sqlite-js](https://github.com/sqliteai/sqlite-js) — Custom SQLite functions written in JavaScript -Other projects in the ecosystem include: +**AI layer** +- [sqlite-ai](https://github.com/sqliteai/sqlite-ai) — On-device LLM inference and embeddings +- [sqlite-agent](https://github.com/sqliteai/sqlite-agent) — Autonomous AI agents running inside SQLite +- [sqlite-memory](https://github.com/sqliteai/sqlite-memory) — Persistent, searchable memory for agents +- [sqlite-mcp](https://github.com/sqliteai/sqlite-mcp) — Call MCP tools directly from SQL queries -- **[SQLite-AI](https://github.com/sqliteai/sqlite-ai)** — On-device inference and embedding generation directly inside SQLite. -- **[SQLite-Memory](https://github.com/sqliteai/sqlite-memory)** — Markdown-based AI agent memory with semantic search. -- **[SQLite-Vector](https://github.com/sqliteai/sqlite-vector)** — Ultra-efficient vector search for embeddings stored as BLOBs in standard SQLite tables. -- **[SQLite-Sync](https://github.com/sqliteai/sqlite-sync)** — Local-first CRDT-based synchronization for seamless, conflict-free data sync and real-time collaboration across devices. -- **[SQLite-Agent](https://github.com/sqliteai/sqlite-agent)** — Run autonomous AI agents directly from within SQLite databases. -- **[SQLite-MCP](https://github.com/sqliteai/sqlite-mcp)** — Connect SQLite databases to MCP servers and invoke their tools. -- **[SQLite-JS](https://github.com/sqliteai/sqlite-js)** — Create custom SQLite functions using JavaScript. -- **[Liteparser](https://github.com/sqliteai/liteparser)** — A highly efficient and fully compliant SQLite SQL parser. +**Managed platform** +- [SQLite Cloud](https://sqlite.ai) — Hosted SQLite with sync, auth, edge functions, and analytics. [Free tier →](https://dashboard.sqlitecloud.io/auth/sign-in) -Learn more at **[SQLite AI](https://sqlite.ai)**. +Built by [SQLite AI](https://sqlite.ai). Questions? [Contact us](https://sqlite.ai/support). diff --git a/docker/Makefile.postgresql b/docker/Makefile.postgresql new file mode 100644 index 0000000..f5303bd --- /dev/null +++ b/docker/Makefile.postgresql @@ -0,0 +1,459 @@ +# PostgreSQL Extension Build Configuration +# This file is included by the root Makefile + +# Detect pg_config +PG_CONFIG ?= pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs 2>/dev/null) + +# PostgreSQL directories +PG_SHAREDIR := $(shell $(PG_CONFIG) --sharedir 2>/dev/null) +PG_PKGLIBDIR := $(shell $(PG_CONFIG) --pkglibdir 2>/dev/null) +PG_INCLUDEDIR := $(shell $(PG_CONFIG) --includedir-server 2>/dev/null) + +# Extension metadata +EXTENSION = cloudsync +# Read the binary version (full semver) from src/cloudsync.h, and derive +# EXTVERSION as just MAJOR.MINOR. Rationale: PATCH bumps are binary-only +# (no SQL surface change, no ALTER EXTENSION UPDATE needed); MINOR/MAJOR bumps +# are the SQL-surface-changing ones that require a per-release upgrade script. +# cloudsync_version() in the .so still reports the full semver for debugging. +# +# Recursive (=) rather than immediate (:=) assignment so the shell calls only +# fire when a PG-related target actually references these variables. Non-PG +# targets parse this included makefile without invoking sed/cut at all. +CLOUDSYNC_VERSION_FULL = $(shell sed -n 's/^\#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' src/cloudsync.h) +EXTVERSION = $(shell echo '$(CLOUDSYNC_VERSION_FULL)' | cut -d. -f1-2) + +# Detect OS for platform-specific settings +ifneq ($(OS),Windows_NT) +UNAME_S := $(shell uname -s) +endif + +# Platform-specific PostgreSQL settings +ifeq ($(OS),Windows_NT) + PG_EXTENSION_LIB = $(EXTENSION).dll + PG_CFLAGS = -Wall -Wextra -Wno-unused-parameter -std=c11 -O2 + PG_LDFLAGS = -shared -L$(shell $(PG_CONFIG) --libdir) -lpostgres +else + PG_EXTENSION_LIB = $(EXTENSION).so + PG_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -O2 + ifeq ($(UNAME_S),Darwin) + # macOS: allow undefined symbols resolved at load time by PostgreSQL + PG_LDFLAGS = -shared -undefined dynamic_lookup + else + PG_LDFLAGS = -shared + endif +endif + +# Source files - core platform-agnostic code +PG_CORE_SRC = \ + src/cloudsync.c \ + src/dbutils.c \ + src/pk.c \ + src/utils.c \ + src/lz4.c \ + src/block.c \ + modules/fractional-indexing/fractional_indexing.c + +# PostgreSQL-specific implementation +PG_IMPL_SRC = \ + src/postgresql/database_postgresql.c \ + src/postgresql/cloudsync_postgresql.c \ + src/postgresql/pgvalue.c \ + src/postgresql/sql_postgresql.c + +# All source files +PG_ALL_SRC = $(PG_CORE_SRC) $(PG_IMPL_SRC) +PG_OBJS = $(PG_ALL_SRC:.c=.o) + +# Compiler flags +# Define POSIX macros as compiler flags to ensure they're defined before any includes +# On Windows, skip POSIX defines — PG headers use Winsock instead of netinet/in.h +PG_EXTRA_CFLAGS ?= +PG_CPPFLAGS = -I$(PG_INCLUDEDIR) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD $(PG_EXTRA_CFLAGS) +ifneq ($(OS),Windows_NT) +PG_CPPFLAGS += -D_POSIX_C_SOURCE=200809L +ifeq ($(UNAME_S),Darwin) +PG_CPPFLAGS += -D_DARWIN_C_SOURCE +else +PG_CPPFLAGS += -D_GNU_SOURCE +endif +endif +PG_DEBUG ?= 0 +ifeq ($(PG_DEBUG),1) +ifeq ($(OS),Windows_NT) +PG_CFLAGS = -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer +else +PG_CFLAGS = -fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer +endif +endif + +# Output files +PG_EXTENSION_SQL = src/postgresql/$(EXTENSION)--$(EXTVERSION).sql +PG_EXTENSION_CONTROL = docker/postgresql/$(EXTENSION).control + +# Input templates (tracked). @EXTVERSION@ is substituted at build time. +PG_EXTENSION_SQL_IN = src/postgresql/$(EXTENSION).sql.in +PG_EXTENSION_CONTROL_IN = docker/postgresql/$(EXTENSION).control.in + +# Upgrade scripts (cloudsync----.sql) are hand-written per release. +PG_MIGRATIONS_DIR = src/postgresql/migrations +PG_MIGRATION_SQLS = $(wildcard $(PG_MIGRATIONS_DIR)/$(EXTENSION)--*--*.sql) + +# ============================================================================ +# PostgreSQL Build Targets +# ============================================================================ + +.PHONY: postgres-check postgres-check-migration postgres-generate-files postgres-build postgres-install postgres-package postgres-clean postgres-test \ + postgres-docker-build postgres-docker-build-asan postgres-docker-run postgres-docker-run-asan postgres-docker-stop postgres-docker-rebuild \ + postgres-docker-debug-build postgres-docker-debug-run postgres-docker-debug-rebuild \ + postgres-docker-shell postgres-dev-rebuild postgres-help unittest-pg \ + postgres-supabase-build postgres-supabase-rebuild postgres-supabase-run-smoke-test \ + postgres-docker-run-smoke-test + +# Verify that a cloudsync----.sql upgrade script exists for the +# current CLOUDSYNC_VERSION in src/cloudsync.h. Release-blocking: a missing +# upgrade script silently breaks ALTER EXTENSION cloudsync UPDATE for every +# existing deployment. Runs in <1s; safe to call on every PR. +postgres-check-migration: + @scripts/check-postgres-migration.sh + +# Check if PostgreSQL is available +postgres-check: + @echo "Checking PostgreSQL installation..." + @which $(PG_CONFIG) > /dev/null || (echo "Error: pg_config not found. Install postgresql-server-dev." && exit 1) + @[ -n "$(CLOUDSYNC_VERSION_FULL)" ] || (echo "Error: could not read CLOUDSYNC_VERSION from src/cloudsync.h" && exit 1) + @[ -n "$(EXTVERSION)" ] || (echo "Error: could not derive MAJOR.MINOR EXTVERSION from CLOUDSYNC_VERSION '$(CLOUDSYNC_VERSION_FULL)'" && exit 1) + @echo "PostgreSQL version: $$($(PG_CONFIG) --version)" + @echo "CloudSync version : $(CLOUDSYNC_VERSION_FULL) (extension version $(EXTVERSION))" + @echo "Extension directory: $(PG_PKGLIBDIR)" + @echo "Share directory: $(PG_SHAREDIR)" + @echo "Include directory: $(PG_INCLUDEDIR)" + +# Render the versioned .sql install script and .control file from their .in +# templates. This is a phony target (rather than file rules keyed on +# $(PG_EXTENSION_SQL) / $(PG_EXTENSION_CONTROL)) so that $(EXTVERSION) never +# appears in a rule's target or prerequisite position — those expansions +# happen at parse time and would force sed/cut to run on every `make` +# invocation, including for non-PG goals. Here, the references live inside +# the recipe body and are only evaluated when this target actually fires. +# Writes via a .tmp + atomic mv so a failed sed can't leave a half-rendered +# output file in the extension share dir. +postgres-generate-files: postgres-check + @echo "Rendering extension files at version $(EXTVERSION) (binary $(CLOUDSYNC_VERSION_FULL))" + @sed 's/@EXTVERSION@/$(EXTVERSION)/g' $(PG_EXTENSION_SQL_IN) > $(PG_EXTENSION_SQL).tmp && mv $(PG_EXTENSION_SQL).tmp $(PG_EXTENSION_SQL) + @sed 's/@EXTVERSION@/$(EXTVERSION)/g' $(PG_EXTENSION_CONTROL_IN) > $(PG_EXTENSION_CONTROL).tmp && mv $(PG_EXTENSION_CONTROL).tmp $(PG_EXTENSION_CONTROL) + +# Build PostgreSQL extension +postgres-build: postgres-generate-files + @echo "Building PostgreSQL extension (version $(EXTVERSION))..." + @echo "Compiling source files..." + @for src in $(PG_ALL_SRC); do \ + echo " CC $$src"; \ + $(CC) $(PG_CPPFLAGS) $(PG_CFLAGS) -c $$src -o $${src%.c}.o || exit 1; \ + done + @echo "Linking $(PG_EXTENSION_LIB)..." + $(CC) $(PG_LDFLAGS) $(PG_EXTRA_CFLAGS) -o $(PG_EXTENSION_LIB) $(PG_OBJS) + @echo "Build complete: $(PG_EXTENSION_LIB)" + +# Install extension to PostgreSQL +postgres-install: postgres-build + @echo "Installing CloudSync extension to PostgreSQL..." + @echo "Installing shared library to $(PG_PKGLIBDIR)/" + install -d $(PG_PKGLIBDIR) + install -m 755 $(PG_EXTENSION_LIB) $(PG_PKGLIBDIR)/ + @echo "Installing SQL script to $(PG_SHAREDIR)/extension/" + install -d $(PG_SHAREDIR)/extension + install -m 644 $(PG_EXTENSION_SQL) $(PG_SHAREDIR)/extension/ + @echo "Installing control file to $(PG_SHAREDIR)/extension/" + install -m 644 $(PG_EXTENSION_CONTROL) $(PG_SHAREDIR)/extension/ + @if [ -n "$(PG_MIGRATION_SQLS)" ]; then \ + echo "Installing $(words $(PG_MIGRATION_SQLS)) migration script(s) to $(PG_SHAREDIR)/extension/"; \ + install -m 644 $(PG_MIGRATION_SQLS) $(PG_SHAREDIR)/extension/; \ + fi + @echo "" + @echo "Installation complete!" + @echo "To use the extension, run in psql:" + @echo " CREATE EXTENSION $(EXTENSION);" + @echo "To upgrade an existing installation, run in psql:" + @echo " ALTER EXTENSION $(EXTENSION) UPDATE;" + +# Package extension files for distribution +PG_DIST_DIR = dist/postgresql + +postgres-package: postgres-build + @echo "Packaging PostgreSQL extension (version $(EXTVERSION))..." + @mkdir -p $(PG_DIST_DIR) + cp $(PG_EXTENSION_LIB) $(PG_DIST_DIR)/ + cp $(PG_EXTENSION_SQL) $(PG_DIST_DIR)/ + cp $(PG_EXTENSION_CONTROL) $(PG_DIST_DIR)/ + @if [ -n "$(PG_MIGRATION_SQLS)" ]; then \ + echo "Including $(words $(PG_MIGRATION_SQLS)) migration script(s)"; \ + cp $(PG_MIGRATION_SQLS) $(PG_DIST_DIR)/; \ + fi + @echo "Package ready in $(PG_DIST_DIR)/" + +# Clean PostgreSQL build artifacts +postgres-clean: + @echo "Cleaning PostgreSQL build artifacts..." + rm -f $(PG_OBJS) $(PG_EXTENSION_LIB) + rm -f $(PG_EXTENSION_SQL) $(PG_EXTENSION_CONTROL) + @echo "Clean complete" + +# Test extension (requires running PostgreSQL) +postgres-test: postgres-install + @echo "Testing CloudSync extension..." + @echo "Dropping existing extension (if any)..." + -psql -U postgres -d postgres -c "DROP EXTENSION IF EXISTS $(EXTENSION) CASCADE;" 2>/dev/null + @echo "Creating extension..." + psql -U postgres -d postgres -c "CREATE EXTENSION $(EXTENSION);" + @echo "Testing version function..." + psql -U postgres -d postgres -c "SELECT $(EXTENSION)_version();" + @echo "Listing extension functions..." + psql -U postgres -d postgres -c "\\df $(EXTENSION)_*" + +# ============================================================================ +# Docker Targets +# ============================================================================ + +DOCKER_IMAGE = sqliteai/sqlite-sync-pg +DOCKER_TAG ?= latest +DOCKER_BUILD_ARGS ?= +SUPABASE_CLI_IMAGE ?= $(shell docker ps --format '{{.Image}} {{.Names}}' | awk '/supabase_db/ {print $$1; exit}') +SUPABASE_CLI_DOCKERFILE ?= docker/postgresql/Dockerfile.supabase +SUPABASE_POSTGRES_TAG ?= 17.6.1.071 +SUPABASE_WORKDIR ?= +SUPABASE_WORKDIR_ARG = $(if $(SUPABASE_WORKDIR),--workdir $(SUPABASE_WORKDIR),) +SUPABASE_DB_HOST ?= 127.0.0.1 +SUPABASE_DB_PORT ?= 54322 +SUPABASE_DB_PASSWORD ?= postgres +PG_DOCKER_DB_HOST ?= localhost +PG_DOCKER_DB_PORT ?= 5432 +PG_DOCKER_DB_NAME ?= postgres +PG_DOCKER_DB_USER ?= postgres +PG_DOCKER_DB_PASSWORD ?= postgres + +# Build Docker image with pre-installed extension +postgres-docker-build: + @echo "Building Docker image via docker compose (rebuilt when sources change)..." + # To force plaintext BuildKit logs, run: make postgres-docker-build DOCKER_BUILD_ARGS="--progress=plain" + cd docker/postgresql && docker compose build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "Docker image built successfully!" + +# Build Docker image with AddressSanitizer enabled (override compose file) +postgres-docker-build-asan: + @echo "Building Docker image with ASAN via docker compose..." + # To force plaintext BuildKit logs, run: make postgres-docker-build-asan DOCKER_BUILD_ARGS=\"--progress=plain\" + cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "ASAN Docker image built successfully!" + +# Build Docker image using docker-compose.debug.yml +postgres-docker-debug-build: + @echo "Building debug Docker image via docker compose..." + # To force plaintext BuildKit logs, run: make postgres-docker-debug-build DOCKER_BUILD_ARGS=\"--progress=plain\" + cd docker/postgresql && docker compose -f docker-compose.debug.yml build $(DOCKER_BUILD_ARGS) + @echo "" + @echo "Debug Docker image built successfully!" + +# Run PostgreSQL container with CloudSync +postgres-docker-run: + @echo "Starting PostgreSQL with CloudSync..." + cd docker/postgresql && docker compose up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Run PostgreSQL container with CloudSync and AddressSanitizer enabled +postgres-docker-run-asan: + @echo "Starting PostgreSQL with CloudSync (ASAN enabled)..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Run PostgreSQL container using docker-compose.debug.yml +postgres-docker-debug-run: + @echo "Starting PostgreSQL with CloudSync (debug compose)..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build + @echo "" + @echo "Container started successfully!" + @echo "" + @echo "Connect with psql:" + @echo " docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test" + @echo "" + @echo "Or from host:" + @echo " psql postgresql://postgres:postgres@localhost:5432/cloudsync_test" + @echo "" + @echo "Enable extension:" + @echo " CREATE EXTENSION cloudsync;" + @echo " SELECT cloudsync_version();" + +# Stop PostgreSQL container +postgres-docker-stop: + @echo "Stopping PostgreSQL container..." + cd docker/postgresql && docker compose down + @echo "Container stopped" + +# Rebuild and restart container +postgres-docker-rebuild: postgres-docker-build + @echo "Rebuilding and restarting container..." + cd docker/postgresql && docker compose down + cd docker/postgresql && docker compose up -d --build + @echo "Container restarted with new image" + +# Rebuild and restart container using docker-compose.debug.yml +postgres-docker-debug-rebuild: postgres-docker-debug-build + @echo "Rebuilding and restarting debug container..." + cd docker/postgresql && docker compose -f docker-compose.debug.yml down + cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build + @echo "Debug container restarted with new image" + +# Interactive shell in container +postgres-docker-shell: + @echo "Opening shell in PostgreSQL container..." + docker exec -it cloudsync-postgres bash + +# Build CloudSync into the Supabase CLI postgres image tag +postgres-supabase-build: + @echo "Building CloudSync image for Supabase CLI..." + @tmp_dockerfile="$$(mktemp ./cloudsync-supabase-cli.XXXXXX)"; \ + src_dockerfile="$(SUPABASE_CLI_DOCKERFILE)"; \ + supabase_cli_image="$(SUPABASE_CLI_IMAGE)"; \ + if [ -z "$$supabase_cli_image" ]; then \ + supabase_cli_image="public.ecr.aws/supabase/postgres:$(SUPABASE_POSTGRES_TAG)"; \ + fi; \ + if [ -z "$$supabase_cli_image" ]; then \ + echo "Error: Supabase CLI postgres image not found."; \ + echo "Run 'supabase start' first, or set SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:."; \ + exit 1; \ + fi; \ + if [ ! -f "$$src_dockerfile" ]; then \ + if [ -f "docker/postgresql/Dockerfile.supabase" ]; then \ + src_dockerfile="docker/postgresql/Dockerfile.supabase"; \ + else \ + echo "Error: Supabase Dockerfile not found (expected $$src_dockerfile)."; \ + rm -f "$$tmp_dockerfile"; \ + exit 1; \ + fi; \ + fi; \ + sed -e "s|^FROM supabase/postgres:[^ ]*|FROM $$supabase_cli_image|" \ + -e "s|^FROM public.ecr.aws/supabase/postgres:[^ ]*|FROM $$supabase_cli_image|" \ + "$$src_dockerfile" > "$$tmp_dockerfile"; \ + if [ ! -s "$$tmp_dockerfile" ]; then \ + echo "Error: Generated Dockerfile is empty."; \ + rm -f "$$tmp_dockerfile"; \ + exit 1; \ + fi; \ + echo "Using base image: $$supabase_cli_image"; \ + echo "Pulling fresh base image to avoid layer accumulation..."; \ + docker pull "$$supabase_cli_image" 2>/dev/null || true; \ + docker build --build-arg SUPABASE_POSTGRES_TAG="$(SUPABASE_POSTGRES_TAG)" --build-arg CLOUDSYNC_VERSION="$(CLOUDSYNC_VERSION_FULL)" -f "$$tmp_dockerfile" -t "$$supabase_cli_image" .; \ + rm -f "$$tmp_dockerfile"; \ + echo "Build complete: $$supabase_cli_image" + +# Rebuild CloudSync image and restart Supabase CLI stack +postgres-supabase-rebuild: postgres-supabase-build + @echo "Restarting Supabase CLI stack..." + @command -v supabase >/dev/null 2>&1 || (echo "Error: supabase CLI not found in PATH." && exit 1) + @supabase stop $(SUPABASE_WORKDIR_ARG) + @supabase start $(SUPABASE_WORKDIR_ARG) + @echo "Supabase CLI stack restarted." + +# Run smoke test against Supabase CLI local database +postgres-supabase-run-test: + @echo "Running Supabase CLI test..." + @PGPASSWORD="$(SUPABASE_DB_PASSWORD)" psql postgresql://supabase_admin@$(SUPABASE_DB_HOST):$(SUPABASE_DB_PORT)/postgres -f test/postgresql/full_test.sql + @echo "Test completed." + +# Run smoke test against Docker standalone database +postgres-docker-run-test: + @echo "Running Docker test..." + @PGPASSWORD="$(PG_DOCKER_DB_PASSWORD)" psql postgresql://$(PG_DOCKER_DB_USER)@$(PG_DOCKER_DB_HOST):$(PG_DOCKER_DB_PORT)/$(PG_DOCKER_DB_NAME) -f test/postgresql/full_test.sql + @echo "Test completed." + +# ============================================================================ +# Development Workflow Targets +# ============================================================================ + +# Quick rebuild inside running container +postgres-dev-rebuild: + @echo "Rebuilding extension inside running container..." + @echo "This is faster than rebuilding the entire Docker image" + docker exec -it cloudsync-postgres bash -c "cd /tmp/cloudsync && make postgres-clean && make postgres-build && make postgres-install" + @echo "" + @echo "Extension rebuilt successfully!" + @echo "" + @echo "To reload the extension in psql, run:" + @echo " DROP EXTENSION cloudsync CASCADE;" + @echo " CREATE EXTENSION cloudsync;" + +# Help target +postgres-help: + @echo "PostgreSQL Extension Build Targets" + @echo "===================================" + @echo "" + @echo "Build & Install:" + @echo " postgres-check - Verify PostgreSQL installation" + @echo " postgres-check-migration - Verify a migration script exists for the current version" + @echo " postgres-generate-files - Render cloudsync.control and cloudsync--.sql from templates" + @echo " postgres-build - Build extension (.so file)" + @echo " postgres-install - Install extension to PostgreSQL" + @echo " postgres-clean - Clean build artifacts" + @echo " postgres-test - Test extension (requires running PostgreSQL)" + @echo "" + @echo "Docker Targets:" + @echo " postgres-docker-build - Build Docker image with pre-installed extension" + @echo " postgres-docker-build-asan - Build Docker image with ASAN enabled" + @echo " postgres-docker-run-asan - Run container with ASAN enabled" + @echo " postgres-docker-debug-build - Build image via docker-compose.debug.yml" + @echo " postgres-docker-debug-run - Run container via docker-compose.debug.yml" + @echo " postgres-docker-debug-rebuild - Rebuild and run docker-compose.debug.yml" + @echo " postgres-docker-run - Start PostgreSQL container" + @echo " postgres-docker-stop - Stop PostgreSQL container" + @echo " postgres-docker-rebuild - Rebuild image and restart container" + @echo " postgres-docker-shell - Open bash shell in running container" + @echo " postgres-supabase-build - Build CloudSync into Supabase CLI postgres image" + @echo " postgres-supabase-rebuild - Build CloudSync image and restart Supabase CLI stack" + @echo " postgres-supabase-run-smoke-test - Run smoke test against Supabase CLI database" + @echo " postgres-docker-run-smoke-test - Run smoke test against Docker database" + @echo "" + @echo "Development:" + @echo " postgres-dev-rebuild - Rebuild extension in running container (fast)" + @echo " unittest-pg - Rebuild container and run smoke test (create extension + version)" + @echo "" + @echo "Examples:" + @echo " make postgres-docker-build # Build image" + @echo " make postgres-docker-build-asan # Build image with ASAN" + @echo " make postgres-docker-run-asan # Run container with ASAN" + @echo " make postgres-docker-debug-build # Build debug compose image" + @echo " make postgres-docker-debug-run # Run debug compose container" + @echo " make postgres-docker-debug-rebuild # Rebuild debug compose container" + @echo " make postgres-docker-run # Start container" + @echo " make postgres-docker-shell # Open shell" + @echo " make postgres-dev-rebuild # Rebuild after code changes" + +# Simple smoke test: rebuild image/container, create extension, and query version +unittest-pg: postgres-docker-rebuild + @echo "Running PostgreSQL extension smoke test..." + cd docker/postgresql && docker compose exec -T postgres psql -U postgres -d cloudsync_test -f /tmp/cloudsync/docker/postgresql/smoke_test.sql + @echo "Smoke test completed." diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..2aaff4b --- /dev/null +++ b/docker/README.md @@ -0,0 +1,352 @@ +# CloudSync Docker Setup + +This directory contains Docker configurations for developing and testing CloudSync with PostgreSQL. + +## Directory Structure + +``` +docker/ +├── postgresql/ # Standalone PostgreSQL with CloudSync +│ ├── Dockerfile # Custom PostgreSQL image +│ ├── docker-compose.yml +│ ├── init.sql # CloudSync metadata tables +│ └── cloudsync.control.in # template; cloudsync.control is generated at build time +``` + +## Option 1: Standalone PostgreSQL + +Use this for simple PostgreSQL development and testing. + +### Quick Start + +```bash +# Build Docker image with CloudSync extension +make postgres-docker-build + +# Start PostgreSQL container +make postgres-docker-run + +# Test the extension +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test -c "CREATE EXTENSION cloudsync; SELECT cloudsync_version();" +``` + +This starts: +- PostgreSQL 16 on `localhost:5432` +- CloudSync extension pre-installed +- pgAdmin on `localhost:5050` (optional, use `--profile admin`) + +### Configuration + +- **Database**: `cloudsync_test` +- **Username**: `postgres` +- **Password**: `postgres` + +### Development Workflow + +After making changes to the source code: + +```bash +# Quick rebuild inside running container (fast!) +make postgres-dev-rebuild + +# Then reload the extension in psql +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test +``` + +```sql +DROP EXTENSION cloudsync CASCADE; +CREATE EXTENSION cloudsync; +SELECT cloudsync_version(); +``` + +### Using pgAdmin (Optional) + +Start with the admin profile: + +```bash +docker-compose --profile admin up -d +``` + +Access pgAdmin at http://localhost:5050: +- Email: `admin@cloudsync.local` +- Password: `admin` + +### VS Code Dev Container Debugging (PostgreSQL) + +Use this when you want breakpoints in the extension code. +The dev container uses `docker/postgresql/Dockerfile.debug` and `docker/postgresql/docker-compose.debug.yml`, which build the extension with debug symbols. +Required VS Code extensions: +- `ms-vscode-remote.remote-containers` (Dev Containers) +- `ms-vscode.cpptools` (C/C++ debugging) + +1) Open the dev container +VS Code -> Command Palette -> `Dev Containers: Reopen in Container` + +2) Connect with `psql` (inside the dev container) +```bash +psql -U postgres -d cloudsync_test +``` + +3) Enable the extension if needed +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +4) Get the backend PID (inside `psql`) +```sql +SELECT pg_backend_pid(); +``` + +5) Attach the debugger (VS Code dev container window) +Run and Debug -> `Attach to Postgres (gdb)` -> pick the PID from step 4 -> Continue + +6) Trigger your breakpoint +Run the SQL that exercises the code path. If `psql` blocks, the backend is paused at a breakpoint; continue in the debugger. + +## Option 2: Supabase Integration (cli) + +Use this when you're running `supabase start` and want CloudSync inside the local stack. +The Supabase CLI uses a bundled PostgreSQL image (for example, +`public.ecr.aws/supabase/postgres:17.6.1.071`). Build a matching image that +includes CloudSync, then tag it with the same name so the CLI reuses it. This +keeps your local Supabase stack intact (auth, realtime, storage, etc.) while +enabling the extension in the CLI-managed Postgres container. + +### Prerequisites + +- Supabase CLI installed (`supabase start` works) +- Docker running + +### Setup + +1. Initialize a Supabase project (use a separate workdir to keep generated files + out of the repo): + ```bash + mkdir -p ~/supabase-local + supabase init --workdir ~/supabase-local + ``` + +2. Start Supabase once so the CLI pulls the Postgres image: + ```bash + supabase start --workdir ~/supabase-local + ``` + +3. Build and tag a CloudSync image using the same tag as the running CLI stack: + ```bash + make postgres-supabase-build + ``` + This auto-detects the running `supabase_db` image tag and rebuilds it with + CloudSync installed. If you need to override the full image tag, set + `SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:`. + Example: + ```bash + SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:17.6.1.071 make postgres-supabase-build + ``` + You can also set the Supabase base image tag explicitly (defaults to + `17.6.1.071`). This only affects the base image used in the Dockerfile: + ```bash + SUPABASE_POSTGRES_TAG=17.6.1.071 make postgres-supabase-build + ``` + +4. Restart the stack: + ```bash + supabase stop --workdir ~/supabase-local + supabase start --workdir ~/supabase-local + ``` + +### Using the CloudSync Extension + +You can load the extension automatically from a migration, or enable it +manually. + +Migration-based (notes for CLI): Supabase CLI migrations run as the `postgres` +role, which cannot create C extensions by default. Use manual enable or grant +`USAGE` on language `c` once, then migrations will work. Note: `c` is an +untrusted language, so `GRANT USAGE ON LANGUAGE c` is only allowed for +superusers. On the CLI/local stack, the simplest approach is to enable the +extension manually as `supabase_admin` after `supabase db reset`. + +If you still want a migration file, add: +```bash +~/supabase-local/supabase/migrations/00000000000000_cloudsync.sql +``` +Contents: +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +Then either: +- run `GRANT USAGE ON LANGUAGE c TO postgres;` once as `supabase_admin`, or +- skip the migration and enable the extension manually after `supabase db reset`. + +Manual enable (no reset required): + +Connect as the Supabase superuser (C extensions require superuser or language +privileges), then enable the extension: + +```bash +psql postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres +``` + +```sql +CREATE EXTENSION cloudsync; +SELECT cloudsync_version(); +``` + +If you want to use the `postgres` role instead: + +```sql +GRANT USAGE ON LANGUAGE c TO postgres; +``` + +### Rebuilding After Changes + +If you modify the CloudSync source code, rebuild the CLI image and restart: + +```bash +make postgres-supabase-rebuild SUPABASE_WORKDIR=~/supabase-local +``` + +### Supabase Realtime Migration Error (app_schema_version) + +If Supabase Realtime fails to start with: + +``` +ERROR 42P01 (undefined_table) relation "app_schema_version" does not exist +``` + +it's caused by CloudSync's `app_schema_change` event trigger firing during +migrations while Realtime uses a restricted `search_path`. Fix it by +fully qualifying the table in the trigger function: + +```sql +CREATE TABLE IF NOT EXISTS public.app_schema_version ( + version BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY +); + +CREATE OR REPLACE FUNCTION bump_app_schema_version() +RETURNS event_trigger AS $$ +BEGIN + INSERT INTO public.app_schema_version DEFAULT VALUES; +END; +$$ LANGUAGE plpgsql; + +DROP EVENT TRIGGER IF EXISTS app_schema_change; +CREATE EVENT TRIGGER app_schema_change +ON ddl_command_end +EXECUTE FUNCTION bump_app_schema_version(); +``` + +## Development Workflow + +### 1. Make Changes + +Edit source files in `src/postgresql/` or `src/` (shared code). + +### 2. Rebuild Extension + +**Fast method** (rebuild in running container): +```bash +make postgres-dev-rebuild +``` + +**Or manually**: +```bash +docker exec -it cloudsync-postgres bash +cd /tmp/cloudsync +make postgres-clean && make postgres-build && make postgres-install +``` + +### 3. Reload Extension in PostgreSQL + +```bash +docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test +``` + +```sql +-- Reload extension +DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION cloudsync; + +-- Test your changes +SELECT cloudsync_version(); +SELECT cloudsync_init('test_table'); +``` + +## Troubleshooting + +### Extension Not Found + +If you get "could not open extension control file", the extension wasn't installed correctly: + +```bash +# Check installation paths +pg_config --sharedir # Should contain cloudsync.control +pg_config --pkglibdir # Should contain cloudsync.so + +# Reinstall +cd /tmp/cloudsync +make install POSTGRES=1 +``` + +### Build Errors + +If you encounter build errors: + +```bash +# Install missing dependencies +apt-get update +apt-get install -y build-essential postgresql-server-dev-16 + +# Clean and rebuild +make clean +make POSTGRES=1 +``` + +### Database Connection Issues + +If you can't connect to PostgreSQL: + +```bash +# Check if PostgreSQL is running +docker ps | grep postgres + +# Check logs +docker logs cloudsync-postgres + +# Restart container +docker-compose restart +``` + +## Environment Variables + +You can customize the setup using environment variables: + +```bash +# PostgreSQL +export POSTGRES_PASSWORD=mypassword +export POSTGRES_DB=mydb + +# Ports +export POSTGRES_PORT=5432 +export PGADMIN_PORT=5050 + +docker-compose up -d +``` + +## Cleaning Up + +```bash +# Stop containers +docker-compose down + +# Remove volumes (deletes all data!) +docker-compose down -v + +# Remove images +docker rmi sqliteai/sqlite-sync-pg:latest +``` + +## Next Steps + +- See [API.md](../API.md) for CloudSync API documentation diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile new file mode 100644 index 0000000..b86e6dc --- /dev/null +++ b/docker/postgresql/Dockerfile @@ -0,0 +1,49 @@ +# PostgreSQL Docker image with CloudSync extension pre-installed +ARG POSTGRES_TAG=17 +FROM postgres:${POSTGRES_TAG} + +# Derive the major version from PG_MAJOR (set by the official postgres image) +# and install the matching server-dev package +RUN apt-get update && apt-get install -y \ + build-essential \ + postgresql-server-dev-${PG_MAJOR} \ + git \ + make \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension +RUN make postgres-build && \ + make postgres-install && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" diff --git a/docker/postgresql/Dockerfile.debug b/docker/postgresql/Dockerfile.debug new file mode 100644 index 0000000..3f77c04 --- /dev/null +++ b/docker/postgresql/Dockerfile.debug @@ -0,0 +1,97 @@ +# PostgreSQL Docker image with CloudSync extension (debug build) +FROM postgres:17 + +# Enable ASAN build flags when requested (used by docker-compose.asan.yml). +ARG ENABLE_ASAN=0 + +# Install build dependencies and debug symbols +RUN apt-get update && apt-get install -y \ + ca-certificates \ + gnupg \ + wget \ + && . /etc/os-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && echo "deb-src http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg-src.list \ + && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && echo "deb http://deb.debian.org/debian-debug ${VERSION_CODENAME}-debug main" > /etc/apt/sources.list.d/debian-debug.list \ + && echo "deb-src http://deb.debian.org/debian ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/debian-src.list \ + && apt-get update && apt-get install -y \ + build-essential \ + bison \ + dpkg-dev \ + flex \ + gdb \ + libicu-dev \ + libreadline-dev \ + libasan8 \ + libssl-dev \ + postgresql-server-dev-17 \ + postgresql-17-dbgsym \ + git \ + make \ + zlib1g-dev \ + && apt-get source postgresql-17 \ + && mkdir -p /usr/src/postgresql-17 \ + && srcdir="$(find . -maxdepth 1 -type d -name 'postgresql-17*' | head -n 1)" \ + && if [ -n "$srcdir" ]; then cp -a "$srcdir"/. /usr/src/postgresql-17/; fi \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Build PostgreSQL from source without optimizations for better gdb visibility +RUN set -eux; \ + cd /usr/src/postgresql-17; \ + ./configure --enable-debug --enable-cassert --without-icu CFLAGS="-O0 -g3 -fno-omit-frame-pointer"; \ + make -j"$(nproc)"; \ + make install + +ENV PATH="/usr/local/pgsql/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib:${LD_LIBRARY_PATH}" + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension with debug flags +RUN set -eux; \ + ASAN_CFLAGS=""; \ + ASAN_LDFLAGS=""; \ + if [ "${ENABLE_ASAN}" = "1" ]; then \ + ASAN_CFLAGS="-fsanitize=address"; \ + ASAN_LDFLAGS="-fsanitize=address"; \ + fi; \ + make postgres-build PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-install PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension (debug)" diff --git a/docker/postgresql/Dockerfile.debug-no-optimization b/docker/postgresql/Dockerfile.debug-no-optimization new file mode 100644 index 0000000..3f77c04 --- /dev/null +++ b/docker/postgresql/Dockerfile.debug-no-optimization @@ -0,0 +1,97 @@ +# PostgreSQL Docker image with CloudSync extension (debug build) +FROM postgres:17 + +# Enable ASAN build flags when requested (used by docker-compose.asan.yml). +ARG ENABLE_ASAN=0 + +# Install build dependencies and debug symbols +RUN apt-get update && apt-get install -y \ + ca-certificates \ + gnupg \ + wget \ + && . /etc/os-release \ + && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && echo "deb-src http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg-src.list \ + && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && echo "deb http://deb.debian.org/debian-debug ${VERSION_CODENAME}-debug main" > /etc/apt/sources.list.d/debian-debug.list \ + && echo "deb-src http://deb.debian.org/debian ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/debian-src.list \ + && apt-get update && apt-get install -y \ + build-essential \ + bison \ + dpkg-dev \ + flex \ + gdb \ + libicu-dev \ + libreadline-dev \ + libasan8 \ + libssl-dev \ + postgresql-server-dev-17 \ + postgresql-17-dbgsym \ + git \ + make \ + zlib1g-dev \ + && apt-get source postgresql-17 \ + && mkdir -p /usr/src/postgresql-17 \ + && srcdir="$(find . -maxdepth 1 -type d -name 'postgresql-17*' | head -n 1)" \ + && if [ -n "$srcdir" ]; then cp -a "$srcdir"/. /usr/src/postgresql-17/; fi \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Build PostgreSQL from source without optimizations for better gdb visibility +RUN set -eux; \ + cd /usr/src/postgresql-17; \ + ./configure --enable-debug --enable-cassert --without-icu CFLAGS="-O0 -g3 -fno-omit-frame-pointer"; \ + make -j"$(nproc)"; \ + make install + +ENV PATH="/usr/local/pgsql/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib:${LD_LIBRARY_PATH}" + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build and install the CloudSync extension with debug flags +RUN set -eux; \ + ASAN_CFLAGS=""; \ + ASAN_LDFLAGS=""; \ + if [ "${ENABLE_ASAN}" = "1" ]; then \ + ASAN_CFLAGS="-fsanitize=address"; \ + ASAN_LDFLAGS="-fsanitize=address"; \ + fi; \ + make postgres-build PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-install PG_DEBUG=1 \ + PG_CFLAGS="-fPIC -Wall -Wextra -Wno-unused-parameter -std=c11 -g -O0 -fno-omit-frame-pointer ${ASAN_CFLAGS}" \ + PG_LDFLAGS="-shared ${ASAN_LDFLAGS}" \ + PG_CPPFLAGS="-I$(pg_config --includedir-server) -Isrc -Isrc/postgresql -Imodules/fractional-indexing -DCLOUDSYNC_POSTGRESQL_BUILD -D_POSIX_C_SOURCE=200809L -D_GNU_SOURCE" && \ + make postgres-clean + +# Verify installation +RUN echo "Verifying CloudSync extension installation..." && \ + ls -la $(pg_config --pkglibdir)/cloudsync.so && \ + ls -la $(pg_config --sharedir)/extension/cloudsync* && \ + echo "CloudSync extension installed successfully" + +# Set default PostgreSQL credentials +ENV POSTGRES_PASSWORD=postgres +ENV POSTGRES_DB=cloudsync_test + +# Expose PostgreSQL port +EXPOSE 5432 + +# Copy initialization script (creates CloudSync metadata tables) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="1.0" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension (debug)" diff --git a/docker/postgresql/Dockerfile.release b/docker/postgresql/Dockerfile.release new file mode 100644 index 0000000..4d9e7aa --- /dev/null +++ b/docker/postgresql/Dockerfile.release @@ -0,0 +1,50 @@ +# PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension +# +# Usage: +# docker build \ +# --build-arg POSTGRES_TAG=17 \ +# --build-arg CLOUDSYNC_VERSION=1.0.6 \ +# -t sqlite-sync-postgres:17 \ +# -f docker/postgresql/Dockerfile.release . +# +# Or pull the pre-built image from Docker Hub: +# docker pull sqlitecloud/sqlite-sync-postgres:17 +# + +ARG POSTGRES_TAG=17 +FROM postgres:${POSTGRES_TAG} + +ARG CLOUDSYNC_VERSION +ARG TARGETARCH + +# Map Docker platform arch to artifact arch +RUN case "${TARGETARCH}" in \ + amd64) ARCH="x86_64" ;; \ + arm64) ARCH="arm64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \ + ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz" && \ + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}" && \ + echo "Downloading ${URL}" && \ + curl -fSL "${URL}" -o /tmp/cloudsync.tar.gz && \ + mkdir -p /tmp/cloudsync && \ + tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync && \ + install -m 755 /tmp/cloudsync/cloudsync.so "$(pg_config --pkglibdir)/" && \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql "$(pg_config --sharedir)/extension/" && \ + install -m 644 /tmp/cloudsync/cloudsync.control "$(pg_config --sharedir)/extension/" && \ + rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz && \ + apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* + +# Verify installation +RUN ls -la "$(pg_config --pkglibdir)/cloudsync.so" && \ + ls -la "$(pg_config --sharedir)/extension/cloudsync"* && \ + echo "CloudSync extension installed successfully" + +# Copy initialization script (auto-creates the extension on first start) +COPY docker/postgresql/init.sql /docker-entrypoint-initdb.d/ + +EXPOSE 5432 + +LABEL org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" \ + org.opencontainers.image.source="https://github.com/sqliteai/sqlite-sync" diff --git a/docker/postgresql/Dockerfile.supabase b/docker/postgresql/Dockerfile.supabase new file mode 100644 index 0000000..983a74a --- /dev/null +++ b/docker/postgresql/Dockerfile.supabase @@ -0,0 +1,97 @@ +# Build stage for CloudSync extension (match Supabase runtime) +ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} AS cloudsync-builder + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + ca-certificates \ + git \ + make \ + && rm -rf /var/lib/apt/lists/* + +# Create directory for extension source +WORKDIR /tmp/cloudsync + +# Copy entire source tree (needed for includes and makefiles) +COPY src/ ./src/ +COPY modules/ ./modules/ +COPY docker/ ./docker/ +COPY Makefile . + +# Build the CloudSync extension using Supabase's pg_config +ENV CLOUDSYNC_PG_CONFIG=/root/.nix-profile/bin/pg_config +RUN if [ ! -x "$CLOUDSYNC_PG_CONFIG" ]; then \ + echo "Error: pg_config not found at $CLOUDSYNC_PG_CONFIG."; \ + exit 1; \ + fi; \ + make postgres-build PG_CONFIG="$CLOUDSYNC_PG_CONFIG" + +# Collect build artifacts (avoid installing into the Nix store) +RUN mkdir -p /tmp/cloudsync-artifacts/lib /tmp/cloudsync-artifacts/extension && \ + cp /tmp/cloudsync/cloudsync.so /tmp/cloudsync-artifacts/lib/ && \ + cp /tmp/cloudsync/src/postgresql/cloudsync--*.sql /tmp/cloudsync-artifacts/extension/ && \ + cp /tmp/cloudsync/docker/postgresql/cloudsync.control /tmp/cloudsync-artifacts/extension/ && \ + # Include per-release upgrade scripts so ALTER EXTENSION ... UPDATE works + if ls /tmp/cloudsync/src/postgresql/migrations/cloudsync--*--*.sql 1>/dev/null 2>&1; then \ + cp /tmp/cloudsync/src/postgresql/migrations/cloudsync--*--*.sql /tmp/cloudsync-artifacts/extension/; \ + fi + +# Runtime image based on Supabase Postgres +ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} + +# Extension version (derived from src/cloudsync.h by the Makefile and passed in +# as a build arg); used only for the image label. +ARG CLOUDSYNC_VERSION=unknown + +# Match builder pg_config path +ENV CLOUDSYNC_PG_CONFIG=/root/.nix-profile/bin/pg_config + +# Install CloudSync extension artifacts +COPY --from=cloudsync-builder /tmp/cloudsync-artifacts/ /tmp/cloudsync-artifacts/ +RUN if [ ! -x "$CLOUDSYNC_PG_CONFIG" ]; then \ + echo "Error: pg_config not found at $CLOUDSYNC_PG_CONFIG."; \ + exit 1; \ + fi; \ + PKGLIBDIR="`$CLOUDSYNC_PG_CONFIG --pkglibdir`"; \ + # Supabase wraps postgres and overrides libdir via NIX_PGLIBDIR. + NIX_PGLIBDIR="`grep -E \"^export NIX_PGLIBDIR\" /usr/bin/postgres | sed -E \"s/.*'([^']+)'.*/\\1/\"`"; \ + if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi; \ + SHAREDIR_PGCONFIG="`$CLOUDSYNC_PG_CONFIG --sharedir`"; \ + SHAREDIR_STD="/usr/share/postgresql"; \ + install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension" && \ + install -m 755 /tmp/cloudsync-artifacts/lib/cloudsync.so "$PKGLIBDIR/" && \ + install -m 644 /tmp/cloudsync-artifacts/extension/cloudsync* "$SHAREDIR_PGCONFIG/extension/"; \ + if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ]; then \ + install -d "$SHAREDIR_STD/extension" && \ + install -m 644 /tmp/cloudsync-artifacts/extension/cloudsync* "$SHAREDIR_STD/extension/"; \ + fi + +# Verify installation +RUN if [ ! -x "$CLOUDSYNC_PG_CONFIG" ]; then \ + echo "Error: pg_config not found at $CLOUDSYNC_PG_CONFIG."; \ + exit 1; \ + fi; \ + NIX_PGLIBDIR="`grep -E \"^export NIX_PGLIBDIR\" /usr/bin/postgres | sed -E \"s/.*'([^']+)'.*/\\1/\"`"; \ + echo "Verifying CloudSync extension installation..." && \ + if [ -n "$NIX_PGLIBDIR" ]; then \ + ls -la "$NIX_PGLIBDIR/cloudsync.so"; \ + else \ + ls -la "`$CLOUDSYNC_PG_CONFIG --pkglibdir`/cloudsync.so"; \ + fi && \ + ls -la "`$CLOUDSYNC_PG_CONFIG --sharedir`/extension/cloudsync"* && \ + if [ -d "/usr/share/postgresql/extension" ]; then \ + ls -la /usr/share/postgresql/extension/cloudsync*; \ + fi && \ + echo "CloudSync extension installed successfully" + +# Expose PostgreSQL port +EXPOSE 5432 + +# Return to root directory +WORKDIR / + +# Add label with extension version +LABEL org.sqliteai.cloudsync.version="${CLOUDSYNC_VERSION}" \ + org.sqliteai.cloudsync.description="PostgreSQL with CloudSync CRDT extension" diff --git a/docker/postgresql/Dockerfile.supabase.release b/docker/postgresql/Dockerfile.supabase.release new file mode 100644 index 0000000..5ef83f3 --- /dev/null +++ b/docker/postgresql/Dockerfile.supabase.release @@ -0,0 +1,72 @@ +# Supabase PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension +# +# Usage: +# docker build \ +# --build-arg SUPABASE_POSTGRES_TAG=15.8.1.085 \ +# --build-arg CLOUDSYNC_VERSION=1.0.6 \ +# -f docker/postgresql/Dockerfile.supabase.release \ +# -t my-cloudsync-supabase-postgres . +# +# Or pull the pre-built image from Docker Hub: +# docker pull sqlitecloud/sqlite-sync-supabase:15 +# + +ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} + +ARG CLOUDSYNC_VERSION +ARG TARGETARCH + +ENV CLOUDSYNC_PG_CONFIG=/root/.nix-profile/bin/pg_config + +# Download pre-compiled extension and install into Supabase's Nix layout +RUN case "${TARGETARCH}" in \ + amd64) ARCH="x86_64" ;; \ + arm64) ARCH="arm64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + # Derive PG major version from pg_config + PG_MAJOR=$(${CLOUDSYNC_PG_CONFIG} --version | sed 's/[^0-9]*//' | cut -d. -f1) && \ + apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && \ + ASSET="cloudsync-postgresql${PG_MAJOR}-linux-${ARCH}-${CLOUDSYNC_VERSION}.tar.gz" && \ + URL="https://github.com/sqliteai/sqlite-sync/releases/download/${CLOUDSYNC_VERSION}/${ASSET}" && \ + echo "Downloading ${URL}" && \ + curl -fSL "${URL}" -o /tmp/cloudsync.tar.gz && \ + mkdir -p /tmp/cloudsync && \ + tar -xzf /tmp/cloudsync.tar.gz -C /tmp/cloudsync && \ + # Resolve Supabase's Nix library path + PKGLIBDIR="$(${CLOUDSYNC_PG_CONFIG} --pkglibdir)" && \ + NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)" && \ + if [ -n "$NIX_PGLIBDIR" ]; then PKGLIBDIR="$NIX_PGLIBDIR"; fi && \ + SHAREDIR_PGCONFIG="$(${CLOUDSYNC_PG_CONFIG} --sharedir)" && \ + SHAREDIR_STD="/usr/share/postgresql" && \ + install -d "$PKGLIBDIR" "$SHAREDIR_PGCONFIG/extension" && \ + install -m 755 /tmp/cloudsync/cloudsync.so "$PKGLIBDIR/" && \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_PGCONFIG/extension/" && \ + if [ "$SHAREDIR_STD" != "$SHAREDIR_PGCONFIG" ]; then \ + install -d "$SHAREDIR_STD/extension" && \ + install -m 644 /tmp/cloudsync/cloudsync--*.sql /tmp/cloudsync/cloudsync.control "$SHAREDIR_STD/extension/"; \ + fi && \ + rm -rf /tmp/cloudsync /tmp/cloudsync.tar.gz && \ + apt-get purge -y curl && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* + +# Verify installation +RUN NIX_PGLIBDIR="$(grep -E '^export NIX_PGLIBDIR' /usr/bin/postgres | sed -E "s/.*'([^']+)'.*/\1/" || true)" && \ + echo "Verifying CloudSync extension installation..." && \ + if [ -n "$NIX_PGLIBDIR" ]; then \ + ls -la "$NIX_PGLIBDIR/cloudsync.so"; \ + else \ + ls -la "$(${CLOUDSYNC_PG_CONFIG} --pkglibdir)/cloudsync.so"; \ + fi && \ + ls -la "$(${CLOUDSYNC_PG_CONFIG} --sharedir)/extension/cloudsync"* && \ + if [ -d "/usr/share/postgresql/extension" ]; then \ + ls -la /usr/share/postgresql/extension/cloudsync*; \ + fi && \ + echo "CloudSync extension installed successfully" + +EXPOSE 5432 + +WORKDIR / + +LABEL org.sqliteai.cloudsync.description="Supabase PostgreSQL with CloudSync CRDT extension" \ + org.opencontainers.image.source="https://github.com/sqliteai/sqlite-sync" diff --git a/docker/postgresql/cloudsync.control.in b/docker/postgresql/cloudsync.control.in new file mode 100644 index 0000000..704af37 --- /dev/null +++ b/docker/postgresql/cloudsync.control.in @@ -0,0 +1,13 @@ +# CloudSync PostgreSQL Extension Control File +# +# Generated from cloudsync.control.in by docker/Makefile.postgresql. +# Do not edit the generated file; edit the .in template instead. +# The version below is read from CLOUDSYNC_VERSION in src/cloudsync.h. + +comment = 'CloudSync - CRDT-based multi-master database synchronization' +default_version = '@EXTVERSION@' +relocatable = true +requires = '' +superuser = false +module_pathname = '$libdir/cloudsync' +trusted = true diff --git a/docker/postgresql/docker-compose.asan.yml b/docker/postgresql/docker-compose.asan.yml new file mode 100644 index 0000000..b4f2f84 --- /dev/null +++ b/docker/postgresql/docker-compose.asan.yml @@ -0,0 +1,8 @@ +services: + postgres: + build: + args: + ENABLE_ASAN: "1" + environment: + LD_PRELOAD: /usr/lib/aarch64-linux-gnu/libasan.so.8 + ASAN_OPTIONS: detect_leaks=0,abort_on_error=1,allocator_may_return_null=1 diff --git a/docker/postgresql/docker-compose.debug.yml b/docker/postgresql/docker-compose.debug.yml new file mode 100644 index 0000000..d445670 --- /dev/null +++ b/docker/postgresql/docker-compose.debug.yml @@ -0,0 +1,58 @@ +services: + postgres: + build: + context: ../.. + dockerfile: docker/postgresql/Dockerfile.debug-no-optimization + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cloudsync_test + ports: + - "5432:5432" + ulimits: + core: -1 + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + volumes: + # Mount source code for development (allows quick rebuilds) + - ../../src:/tmp/cloudsync/src:ro + - ../../docker:/tmp/cloudsync/docker:ro + - ../../Makefile:/tmp/cloudsync/Makefile:ro + - ../../.vscode:/tmp/cloudsync/.vscode:ro + # Persist database data + - postgres_data:/var/lib/postgresql/data + # Mount init script + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Optional: pgAdmin for database management + pgadmin: + image: dpage/pgadmin4:latest + container_name: cloudsync-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@cloudsync.local + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_CONFIG_SERVER_MODE: 'False' + ports: + - "5050:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + - postgres + profiles: + - admin + +volumes: + postgres_data: + pgadmin_data: + +networks: + default: + name: cloudsync-network diff --git a/docker/postgresql/docker-compose.yml b/docker/postgresql/docker-compose.yml new file mode 100644 index 0000000..5a3257d --- /dev/null +++ b/docker/postgresql/docker-compose.yml @@ -0,0 +1,53 @@ +services: + postgres: + build: + context: ../.. + dockerfile: docker/postgresql/Dockerfile + args: + POSTGRES_TAG: ${POSTGRES_TAG:-17} + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cloudsync_test + ports: + - "5432:5432" + volumes: + # Mount source code for development (allows quick rebuilds) + - ../../src:/tmp/cloudsync/src:ro + - ../../docker:/tmp/cloudsync/docker:ro + - ../../Makefile:/tmp/cloudsync/Makefile:ro + # Persist database data + - postgres_data:/var/lib/postgresql/data + # Mount init script + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Optional: pgAdmin for database management + pgadmin: + image: dpage/pgadmin4:latest + container_name: cloudsync-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: admin@cloudsync.local + PGADMIN_DEFAULT_PASSWORD: admin + PGADMIN_CONFIG_SERVER_MODE: 'False' + ports: + - "5050:80" + volumes: + - pgadmin_data:/var/lib/pgadmin + depends_on: + - postgres + profiles: + - admin + +volumes: + postgres_data: + pgadmin_data: + +networks: + default: + name: cloudsync-network diff --git a/docker/postgresql/init.sql b/docker/postgresql/init.sql new file mode 100644 index 0000000..b892371 --- /dev/null +++ b/docker/postgresql/init.sql @@ -0,0 +1,10 @@ +-- CloudSync PostgreSQL Initialization Script +-- This script loads the CloudSync extension during database init + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Log initialization +DO $$ +BEGIN + RAISE NOTICE 'CloudSync tables initialized successfully'; +END $$; diff --git a/docs/BLOCK-LWW.md b/docs/BLOCK-LWW.md new file mode 100644 index 0000000..6765b9e --- /dev/null +++ b/docs/BLOCK-LWW.md @@ -0,0 +1,124 @@ +# Block-Level LWW + +Standard CRDT sync resolves conflicts at the **cell level**: if two devices edit the same column of the same row, one value wins entirely. This works for short values like names or statuses, but for longer text content (markdown documents, notes, agent memory files) the entire text is replaced even if edits were in different parts. + +**Block-Level LWW** (Last-Writer-Wins) solves this by splitting text columns into **blocks** (lines by default) and tracking each block independently. When two devices edit different lines of the same text, **both edits are preserved** after sync. Only when two devices edit the *same* line does LWW apply. + +This feature was specifically designed to keep **markdown files** in sync across devices and AI agents. Agents that independently edit different sections of a shared document (adding notes, updating status, appending logs) can do so without overwriting each other's work. + +## How It Works + +1. **Enable block tracking** on a text column using `cloudsync_set_column()`. +2. On INSERT or UPDATE, text is automatically split into blocks using a delimiter (default: newline `\n`). +3. Each block gets a unique fractional index position, enabling insertions between blocks without reindexing. +4. During sync, changes are merged block-by-block rather than replacing the whole cell. +5. The base column always contains the current full text, your queries work unchanged. + +## Setup + +```sql +-- Create a table with a text column for long-form content +CREATE TABLE notes ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '' +); + +-- Initialize sync on the table +SELECT cloudsync_init('notes'); + +-- Enable block-level LWW on the "body" column +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block'); +``` + +## Example: Two-Device Merge + +```sql +-- Device A: create a note +INSERT INTO notes (id, title, body) VALUES ( + 'note-001', + 'Meeting Notes', + 'Line 1: Welcome +Line 2: Agenda +Line 3: Action items' +); + +-- Sync Device A -> Cloud -> Device B +-- Both devices now have the same 3-line note. +``` + +```sql +-- Device A (offline): edit line 1 +UPDATE notes SET body = 'Line 1: Welcome everyone +Line 2: Agenda +Line 3: Action items' WHERE id = 'note-001'; + +-- Device B (offline): edit line 3 +UPDATE notes SET body = 'Line 1: Welcome +Line 2: Agenda +Line 3: Action items - DONE' WHERE id = 'note-001'; +``` + +```sql +-- After both devices sync, the merged result: +-- 'Line 1: Welcome everyone +-- Line 2: Agenda +-- Line 3: Action items - DONE' +-- +-- Both edits are preserved because they affected different lines. +``` + +## Custom Delimiter + +For paragraph-level tracking (useful for long-form markdown documents), set a custom delimiter: + +```sql +-- Use double newline as delimiter (paragraph separator) +SELECT cloudsync_set_column('notes', 'body', 'delimiter', ' + +'); +``` + +## Materializing Text + +After a merge, the `body` column is updated automatically. You can also manually trigger materialization: + +```sql +-- Reconstruct body from blocks for a specific row +SELECT cloudsync_text_materialize('notes', 'body', 'note-001'); + +-- With a composite primary key (e.g., PRIMARY KEY (tenant_id, doc_id)) +SELECT cloudsync_text_materialize('docs', 'body', 'tenant-1', 'doc-001'); + +-- Then read normally +SELECT body FROM notes WHERE id = 'note-001'; +``` + +## Mixed Columns + +Block-level LWW can be enabled on specific columns while other columns use standard cell-level LWW: + +```sql +CREATE TABLE docs ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT NOT NULL DEFAULT '', -- standard LWW (cell-level) + body TEXT NOT NULL DEFAULT '', -- block LWW (line-level) + status TEXT NOT NULL DEFAULT '' -- standard LWW (cell-level) +); + +SELECT cloudsync_init('docs'); +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block'); + +-- Concurrent edits to "title" or "status" use normal LWW. +-- Concurrent edits to "body" merge at the line level. +``` + +## Key Properties + +- **Non-conflicting edits are preserved**: Two users editing different lines both see their changes after sync. +- **Same-line conflicts use LWW**: If two users edit the same line, the last writer wins. +- **Custom delimiters**: Use paragraph separators (`\n\n`), sentence boundaries, or any string. +- **Mixed columns**: A table can have both regular and block-level LWW columns. +- **Transparent reads**: The base column always contains the current full text. + +For API details, see the [API Reference](../API.md). diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md new file mode 100644 index 0000000..3b3943d --- /dev/null +++ b/docs/INSTALLATION.md @@ -0,0 +1,143 @@ +# Installation + +Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: + +- Linux: x86 and ARM +- macOS: x86 and ARM +- Windows: x86 +- Android +- iOS + +## SQLite CLI / C + +```sql +-- In SQLite CLI +.load ./cloudsync + +-- In SQL +SELECT load_extension('./cloudsync'); +``` + +## Swift Package + +[Add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). + +```swift +import CloudSync + +var db: OpaquePointer? +sqlite3_open(":memory:", &db) +sqlite3_enable_load_extension(db, 1) +var errMsg: UnsafeMutablePointer? = nil +sqlite3_load_extension(db, CloudSync.path, nil, &errMsg) +var stmt: OpaquePointer? +sqlite3_prepare_v2(db, "SELECT cloudsync_version()", -1, &stmt, nil) +defer { sqlite3_finalize(stmt) } +sqlite3_step(stmt) +log("cloudsync_version(): \(String(cString: sqlite3_column_text(stmt, 0)))") +sqlite3_close(db) +``` + +## Android + +Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: + +```gradle +implementation 'ai.sqlite:sync:1.0.0' +``` + +```java +SQLiteCustomExtension cloudsyncExtension = new SQLiteCustomExtension( + getApplicationInfo().nativeLibraryDir + "/cloudsync", null); +SQLiteDatabaseConfiguration config = new SQLiteDatabaseConfiguration( + getCacheDir().getPath() + "/cloudsync_test.db", + SQLiteDatabase.CREATE_IF_NECESSARY | SQLiteDatabase.OPEN_READWRITE, + Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(cloudsyncExtension) +); +SQLiteDatabase db = SQLiteDatabase.openDatabase(config, null, null); +``` + +For full implementation details, see the [complete Android example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/android/README.md). + +## Expo + +Install the Expo package: + +```bash +npm install @sqliteai/sqlite-sync-expo +``` + +Add to your `app.json`: + +```json +{ + "expo": { + "plugins": ["@sqliteai/sqlite-sync-expo"] + } +} +``` + +Run prebuild: + +```bash +npx expo prebuild --clean +``` + +Load the extension: + +```typescript +import { Platform } from 'react-native'; +import { getDylibPath, open } from '@op-engineering/op-sqlite'; + +const db = open({ name: 'mydb.db' }); + +// Load SQLite Sync extension +if (Platform.OS === 'ios') { + const path = getDylibPath('ai.sqlite.cloudsync', 'CloudSync'); + db.loadExtension(path); +} else { + db.loadExtension('cloudsync'); +} +``` + +## React Native + +Install the React Native library: + +```bash +npm install @sqliteai/sqlite-sync-react-native +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native). + +## Flutter + +Add the [sqlite_sync](https://pub.dev/packages/sqlite_sync) package to your project: + +```bash +flutter pub add sqlite_sync # Flutter projects +dart pub add sqlite_sync # Dart projects +``` + +```dart +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_sync/sqlite_sync.dart'; + +sqlite3.loadSqliteSyncExtension(); +final db = sqlite3.openInMemory(); +print(db.select('SELECT cloudsync_version()')); +``` + +For a complete example, see the [Flutter example](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/examples/flutter/README.md). + +## WASM + +The WebAssembly version of SQLite with the SQLite Sync extension is available on npm: + +```bash +npm install @sqliteai/sqlite-wasm +``` + +See the [npm package](https://www.npmjs.com/package/@sqliteai/sqlite-wasm) for usage details. diff --git a/docs/ROW-LEVEL-SECURITY.md b/docs/ROW-LEVEL-SECURITY.md new file mode 100644 index 0000000..0d81de0 --- /dev/null +++ b/docs/ROW-LEVEL-SECURITY.md @@ -0,0 +1,49 @@ +# Row-Level Security + +SQLite Sync supports **Row-Level Security (RLS)** through the underlying [SQLite Cloud](https://sqlitecloud.io/) infrastructure. RLS allows you to use a **single shared cloud database** while each client only sees and modifies its own data. Policies are enforced on the server, so the security boundary is at the database level, not in application code. + +## How It Works + +- Control not just who can read or write a table, but **which specific rows** they can access. +- Each device syncs only the rows it is authorized to see: no full dataset download, no client-side filtering. + +For example: + +- User A can only see and edit their own data. +- User B can access a different set of rows, even within the same shared table. + +## Benefits + +- **Single database, multiple tenants**: one cloud database serves all users. RLS policies partition data per user or role, eliminating the need to provision separate databases. +- **Efficient sync**: each client downloads only its authorized rows, reducing bandwidth and local storage. +- **Server-enforced security**: policies are evaluated on the server during sync. A compromised or modified client cannot bypass access controls. +- **Simplified development**: no need to implement permission logic in your application. Define policies once in the database and they apply everywhere. + +## Authentication + +RLS requires token-based authentication. Use `cloudsync_network_set_token()` instead of `cloudsync_network_set_apikey()`: + +```sql +SELECT cloudsync_network_init('your-managed-database-id'); +SELECT cloudsync_network_set_token('your_auth_token'); +SELECT cloudsync_network_sync(); +``` + +For more information on access tokens, see the [Access Tokens documentation](https://docs.sqlitecloud.io/docs/access-tokens). + +## Schema Considerations + +When using RLS with multi-tenant schemas, UNIQUE constraints must be globally unique across all tenants in the cloud database. For columns that should only be unique within a tenant, use composite UNIQUE constraints: + +```sql +CREATE TABLE users ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL DEFAULT '', + UNIQUE(tenant_id, email) +); +``` + +For more schema guidelines, see [Database Schema Recommendations](./schema.md). + +For full RLS documentation, see the [SQLite Cloud RLS documentation](https://docs.sqlitecloud.io/docs/rls). diff --git a/docs/SCHEMA.md b/docs/SCHEMA.md new file mode 100644 index 0000000..d9f0527 --- /dev/null +++ b/docs/SCHEMA.md @@ -0,0 +1,93 @@ +# Database Schema Recommendations + +When designing your database schema for SQLite Sync, follow these guidelines to ensure correct CRDT behavior and conflict resolution. + +## Schema Consistency Across Devices + +All databases participating in the same sync (every client and the cloud database) **must have the same set of synced tables with identical structure**: + +- The same tables must be created on every participant. +- Each table must be initialized with `cloudsync_init()` on every participant. +- Column names, types, and constraints must match across participants. + +sqlite-sync computes a **schema hash** from the synced tables and includes it in every sync payload. The server rejects payloads whose schema hash it does not recognize, failing with an error like: + +``` +cloudsync operation failed: Cannot apply the received payload because the schema hash is unknown +``` + +If you need different clients to see different subsets of data (for example, per-tenant or per-workspace isolation), do **not** give each client a different table. Instead, use a single shared schema and scope the data with a column such as `tenant_id` or `workspace_id`, then enforce isolation server-side with [Row-Level Security](./row-level-security.md). + +## Primary Key Requirements + +- **Use globally unique identifiers**: Always use TEXT primary keys with UUIDs or ULIDs. +- **Avoid auto-incrementing integers**: Integer primary keys cause conflicts across multiple devices. +- **Use `cloudsync_uuid()`**: Generates UUIDv7 identifiers optimized for distributed systems. +- **Note:** Any write operation with a NULL primary key value will be rejected with an error. + +```sql +-- Recommended: Globally unique TEXT primary key +CREATE TABLE users ( + id TEXT PRIMARY KEY, -- Use cloudsync_uuid() + name TEXT NOT NULL DEFAULT '', + email TEXT UNIQUE NOT NULL DEFAULT '' +); + +-- Avoid: Auto-incrementing integer primary key +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- Causes conflicts across devices + name TEXT NOT NULL DEFAULT '', + email TEXT UNIQUE NOT NULL DEFAULT '' +); +``` + +## Column Constraint Guidelines + +- All `NOT NULL` columns (except primary keys) **must** have `DEFAULT` values. +- For optional data, use nullable columns instead of empty strings. + +```sql +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending', + priority INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + assigned_to TEXT -- Nullable for optional data +); +``` + +## UNIQUE Constraint Considerations + +In multi-tenant scenarios with Row-Level Security, UNIQUE constraints must be globally unique across all tenants in the cloud database. Use composite UNIQUE constraints for per-tenant uniqueness: + +```sql +-- Multi-tenant: Composite unique constraint +CREATE TABLE users ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL DEFAULT '', + UNIQUE(tenant_id, email) -- Unique email per tenant +); +``` + +## Foreign Key Compatibility + +Foreign key constraints may conflict with the CRDT merge algorithm: + +- CRDT changes are applied column-by-column during synchronization. Columns may be temporarily assigned DEFAULT values, so foreign key defaults must reference existing rows. +- RLS policies may block CASCADE DELETE/UPDATE operations on related rows. + +**Recommendations:** +- Prefer application-level cascade logic over database-level CASCADE actions. +- Use nullable foreign keys to avoid DEFAULT value issues. +- Test synchronization scenarios with foreign key constraints enabled. + +## Trigger Compatibility + +Triggers can cause issues during synchronization: + +- **Duplicate operations**: Triggers that modify synchronized tables may apply changes twice during merge. +- **Column-by-column processing**: UPDATE triggers may fire multiple times per row as each column is processed. + +Avoid triggers that write to synchronized tables. Use application-level logic instead. diff --git a/docs/internal/grafana-dashboard.json b/docs/internal/grafana-dashboard.json new file mode 100644 index 0000000..ace2326 --- /dev/null +++ b/docs/internal/grafana-dashboard.json @@ -0,0 +1,590 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "N. of distinct devices which made at least one request", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n COUNT(DISTINCT project_id || '/' || database || '/' || site_id) AS devices\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Activity (distinct devices)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "Total number of requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n COUNT(*) AS requests\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "description": "Total amount of traffic in MB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n SUM(bytes_in + bytes_out) / 1024 / 1024 AS megabytes\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic MB (In+Out)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "dfaqrid9yhvk0b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "megabytes apply" + }, + "properties": [ + { + "id": "displayName", + "value": "Out (MB)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "megabytes check" + }, + "properties": [ + { + "id": "displayName", + "value": "In (MB)" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "sum", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "postgres", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__unixEpochGroup(created_at, '1h') AS time,\n action,\n SUM(bytes_in + bytes_out) / 1024 / 1024 AS megabytes\nFROM cloudsync_metrics\nWHERE $__unixEpochFilter(created_at)\n AND action ~ '${action:regex}'\n AND ('${ProjectID}' = '' OR project_id ~ '${ProjectID}')\n AND ('${SiteID}' = '' OR site_id ~ '${SiteID}')\nGROUP BY 1, 2\nORDER BY 1;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Traffic MB", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": [ + "check", + "apply" + ], + "value": [ + "check", + "apply" + ] + }, + "definition": "SELECT DISTINCT action\nFROM cloudsync_metrics\nORDER BY action;", + "description": "", + "includeAll": false, + "label": "Action", + "multi": true, + "name": "action", + "options": [], + "query": "SELECT DISTINCT action\nFROM cloudsync_metrics\nORDER BY action;", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "", + "value": "" + }, + "label": "ProjectID", + "name": "ProjectID", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "current": { + "text": "", + "value": "" + }, + "name": "SiteID", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "CloudSync", + "uid": "advqx5s", + "version": 40 +} \ No newline at end of file diff --git a/docs/Network.md b/docs/internal/network.md similarity index 84% rename from docs/Network.md rename to docs/internal/network.md index 7120231..9af6e15 100644 --- a/docs/Network.md +++ b/docs/internal/network.md @@ -34,14 +34,6 @@ This is useful when: You must provide implementations for the following C functions: - ```c - bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string); - - // Parses `conn_string` and fills the `network_data` structure with connection information (e.g. base URL, endpoints, credentials). - // Returns `true` on success, `false` on error (you can use `sqlite3_result_error` to report errors to SQLite). - - ``` - ```c bool network_send_buffer (network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); @@ -50,11 +42,12 @@ This is useful when: ``` ```c - NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header); + NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers); // Performs a network request (GET or POST depending on `is_post_request`) to the specified `endpoint`, using the given `authentication` token or header. // If `json_payload` is provided, it will be sent as the POST body (for `is_post_request == true`). // If `zero_terminated == true`, ensure that the returned buffer is null-terminated. + // `extra_headers` is an array of `nextra_headers` request-header lines (each formatted as `"Name: value"`) appended to the standard headers (`Authorization`, `X-CloudSync-Org`, `Content-Type`). Pass `NULL, 0` for none. Used by call sites to send `X-CloudSync-Version` on every cloudsync API call and `X-CloudSync-Capabilities: check-status-response` on calls to the `/check` endpoint. // Returns a `NETWORK_RESULT` enum value indicating success, error, or timeout. ``` diff --git a/docs/internal/postgres-flyio.md b/docs/internal/postgres-flyio.md new file mode 100644 index 0000000..3fd3399 --- /dev/null +++ b/docs/internal/postgres-flyio.md @@ -0,0 +1,861 @@ +# Self-Hosting PostgreSQL on Fly.io with CloudSync Extension + +This guide deploys a standalone PostgreSQL instance with CloudSync on Fly.io, plus a minimal JWT auth server for token generation. No Supabase required. + +By the end you will have: + +- A Fly.io VM running PostgreSQL with the CloudSync CRDT extension +- A JWT auth server (Node.js) — choose between: + - **HS256 (shared secret)** — simplest setup, signs tokens with a base64-encoded secret + - **RS256 (JWKS)** — production-ready, signs with a private key and exposes a public JWKS endpoint +- A custom Postgres image published to Docker Hub + +## Prerequisites + +| Tool | Purpose | Install | +|------|---------|---------| +| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Build the custom Postgres image | `brew install --cask docker` | +| [Fly CLI (`flyctl`)](https://fly.io/docs/flyctl/install/) | Provision and manage Fly.io machines | `brew install flyctl` | +| [Git](https://git-scm.com/) | Clone repositories | `brew install git` | +| [Docker Hub](https://hub.docker.com/) account | Host your custom Postgres image | Free signup | + +You also need a [Fly.io account](https://fly.io/app/sign-up). + +### Fly.io VM requirements + +Since this is just PostgreSQL + a small auth server (not a full Supabase stack), resource requirements are much lower: + +| Resource | Minimum | Recommended | +|----------|---------|-------------| +| RAM | 1 GB | 2 GB | +| CPU | 1 core | 2 cores | +| Disk | 4 GB SSD | 10 GB+ | + +--- + +## Step 1: Initialize git submodules + +```bash +cd /path/to/sqlite-sync-dev +git submodule update --init --recursive +``` + +Without this, the build fails with `fractional_indexing.h: No such file or directory`. + +--- + +## Step 2: Build the custom Postgres image + +From the sqlite-sync-dev repo root: + +```bash +make postgres-docker-build +``` + +This builds `postgres:17` with CloudSync pre-installed using `docker/postgresql/Dockerfile`. + +Verify: + +```bash +docker images | grep sqlite-sync-pg +``` + +--- + +## Step 3: Build for amd64 and push to Docker Hub + +If you're on Apple Silicon, you must cross-build for Fly.io's x86 VMs: + +```bash +docker build --platform linux/amd64 \ + -f docker/postgresql/Dockerfile \ + -t /postgres-cloudsync:17 \ + . + +docker push /postgres-cloudsync:17 +``` + +> On Intel Mac or Linux x86, the default build is already amd64. + +--- + +## Step 4: Provision a Fly.io VM + +### 4a. Log in and create the app + +```bash +fly auth login +fly apps create +``` + +### 4b. Create a persistent volume + +```bash +fly volumes create pg_data --app --region --size 4 +``` + +### 4c. Create a Fly Machine + +```bash +fly machine run ubuntu:24.04 \ + --app \ + --region \ + --vm-size shared-cpu-2x \ + --vm-memory 2048 \ + --volume pg_data:/data \ + --name postgres-vm \ + -- sleep inf +``` + +### 4d. Allocate a public IP + +```bash +fly ips allocate-v4 --shared --app +fly ips allocate-v6 --app +``` + +--- + +## Step 5: Set up Docker on the VM + +### 5a. SSH into the machine + +```bash +fly ssh console --app +``` + +### 5b. Install Docker Engine + +```bash +apt-get update +apt-get install -y ca-certificates curl gnupg + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +``` + +### 5c. Configure Docker storage driver + +```bash +apt-get install -y fuse-overlayfs +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json +``` + +### 5d. Start Docker + +```bash +dockerd & +# Wait for "API listen on /var/run/docker.sock" +``` + +--- + +## Step 6: Create the Docker Compose stack + +Create the project directory: + +```bash +mkdir -p /data/cloudsync-postgres +cd /data/cloudsync-postgres +``` + +### 6a. Generate a JWT secret + +```bash +JWT_SECRET=$(openssl rand -base64 32) +echo "JWT_SECRET=$JWT_SECRET" > .env +echo "POSTGRES_PASSWORD=$(openssl rand -base64 16)" >> .env +echo "Your JWT secret: $JWT_SECRET" +echo "Save this secret — you'll need it for CloudSync server configuration." +``` + +### 6b. Create docker-compose.yml + +```bash +cat > docker-compose.yml << 'EOF' +services: + db: + image: /postgres-cloudsync:17 + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: postgres + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + auth: + image: node:22-alpine + container_name: cloudsync-auth + environment: + JWT_SECRET: ${JWT_SECRET} + PORT: 3001 + ports: + - "3001:3001" + volumes: + - ./auth-server:/app + working_dir: /app + command: ["node", "server.js"] + restart: unless-stopped + +volumes: + pg_data: +EOF +``` + +### 6c. Create the Postgres init script + +```bash +cat > init.sql << 'EOF' +CREATE EXTENSION IF NOT EXISTS cloudsync; +EOF +``` + +### 6d. Create the JWT auth server + +```bash +mkdir -p auth-server +``` + +Create the package file: + +```bash +cat > auth-server/package.json << 'EOF' +{ + "name": "cloudsync-auth", + "version": "1.0.0", + "private": true, + "dependencies": { + "jsonwebtoken": "^9.0.0" + } +} +EOF +``` + +Create the auth server: + +```bash +cat > auth-server/server.js << 'AUTHEOF' +const http = require("http"); +const jwt = require("jsonwebtoken"); + +const PORT = process.env.PORT || 3001; +const JWT_SECRET = process.env.JWT_SECRET; + +if (!JWT_SECRET) { + console.error("JWT_SECRET environment variable is required"); + process.exit(1); +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => { + try { resolve(JSON.parse(data)); } + catch { reject(new Error("Invalid JSON")); } + }); + req.on("error", reject); + }); +} + +function respond(res, status, body) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(body)); +} + +const server = http.createServer(async (req, res) => { + // Health check + if (req.method === "GET" && req.url === "/healthz") { + return respond(res, 200, { status: "ok" }); + } + + // Generate token + // POST /token { "sub": "user-id", "role": "rls_role", "expiresIn": "24h" } + if (req.method === "POST" && req.url === "/token") { + try { + const body = await parseBody(req); + const sub = body.sub || "anonymous"; + const role = body.role; + const expiresIn = body.expiresIn || "24h"; + const claims = body.claims || {}; + + if (!role) { + return respond(res, 400, { error: "role is required" }); + } + + const token = jwt.sign( + { sub, role, ...claims }, + JWT_SECRET, + { expiresIn, algorithm: "HS256" } + ); + + return respond(res, 200, { token, expiresIn }); + } catch (err) { + return respond(res, 400, { error: err.message }); + } + } + + respond(res, 404, { error: "Not found" }); +}); + +server.listen(PORT, () => { + console.log("Auth server listening on port " + PORT); +}); +AUTHEOF +``` + +Install dependencies: + +```bash +docker run --rm -v $(pwd)/auth-server:/app -w /app node:22-alpine npm install +``` + +### 6e. (Optional) Create the JWKS auth server + +If you need RS256/JWKS-based authentication instead of (or in addition to) the shared secret approach, create a second auth server that generates an RSA key pair on startup and exposes a JWKS endpoint. + +```bash +mkdir -p auth-server-jwks + +cat > auth-server-jwks/package.json << 'EOF' +{ + "name": "cloudsync-auth-jwks", + "version": "1.0.0", + "private": true, + "dependencies": { + "jsonwebtoken": "^9.0.0", + "jose": "^5.0.0" + } +} +EOF + +cat > auth-server-jwks/server.js << 'EOF' +const http = require("http"); +const jwt = require("jsonwebtoken"); +const crypto = require("crypto"); +const { exportJWK } = require("jose"); + +const PORT = process.env.PORT || 3002; +const ISSUER = process.env.ISSUER || "cloudsync-auth-jwks"; +const KID = "cloudsync-key-1"; + +let privateKey, publicKey, jwksResponse; + +async function init() { + const pair = crypto.generateKeyPairSync("rsa", { + modulusLength: 2048, + publicKeyEncoding: { type: "spki", format: "pem" }, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, + }); + privateKey = pair.privateKey; + publicKey = pair.publicKey; + + const publicKeyObject = crypto.createPublicKey(publicKey); + const jwk = await exportJWK(publicKeyObject); + jwk.kid = KID; + jwk.alg = "RS256"; + jwk.use = "sig"; + jwksResponse = JSON.stringify({ keys: [jwk] }); + + console.log("RSA key pair generated (kid: " + KID + ")"); +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let data = ""; + req.on("data", (chunk) => (data += chunk)); + req.on("end", () => { + try { resolve(JSON.parse(data)); } + catch { reject(new Error("Invalid JSON")); } + }); + req.on("error", reject); + }); +} + +function respond(res, status, body) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(typeof body === "string" ? body : JSON.stringify(body)); +} + +const server = http.createServer(async (req, res) => { + if (req.method === "GET" && req.url === "/healthz") { + return respond(res, 200, { status: "ok" }); + } + + // JWKS endpoint — CloudSync server fetches this to verify tokens + if (req.method === "GET" && req.url === "/.well-known/jwks.json") { + res.writeHead(200, { "Content-Type": "application/json" }); + return res.end(jwksResponse); + } + + // POST /token { "sub": "user-id", "role": "rls_role", "expiresIn": "24h" } + if (req.method === "POST" && req.url === "/token") { + try { + const body = await parseBody(req); + const sub = body.sub || "anonymous"; + const role = body.role; + const expiresIn = body.expiresIn || "24h"; + const claims = body.claims || {}; + + if (!role) { + return respond(res, 400, { error: "role is required" }); + } + + const token = jwt.sign( + { sub, role, iss: ISSUER, ...claims }, + privateKey, + { expiresIn, algorithm: "RS256", keyid: KID } + ); + + return respond(res, 200, { token, expiresIn }); + } catch (err) { + return respond(res, 400, { error: err.message }); + } + } + + respond(res, 404, { error: "Not found" }); +}); + +init().then(() => { + server.listen(PORT, () => { + console.log("JWKS Auth server listening on port " + PORT); + }); +}); +EOF + +docker run --rm -v $(pwd)/auth-server-jwks:/app -w /app node:22-alpine npm install +``` + +Add the JWKS auth service to `docker-compose.yml`: + +```yaml + auth-jwks: + image: node:22-alpine + container_name: cloudsync-auth-jwks + environment: + PORT: 3002 + ISSUER: cloudsync-auth-jwks + ports: + - "3002:3002" + volumes: + - ./auth-server-jwks:/app + working_dir: /app + command: ["node", "server.js"] + restart: unless-stopped +``` + +> **Note:** The JWKS server generates a new RSA key pair each time it starts. For production, persist the key pair to a volume so tokens remain valid across restarts. + +--- + +## Step 7: Start the stack + +```bash +cd /data/cloudsync-postgres +docker compose up -d +``` + +Verify: + +```bash +docker compose ps + +# Test Postgres +docker compose exec db psql -U postgres -c "SELECT cloudsync_version();" + +# Test HS256 auth server +curl http://localhost:3001/healthz + +# Test JWKS auth server (if enabled) +curl http://localhost:3002/healthz +curl http://localhost:3002/.well-known/jwks.json +``` + +--- + +## Step 8: Generate a JWT token + +Before generating JWTs for PostgreSQL, create the database role referenced by the token's `role` claim and grant it the permissions CloudSync needs. + +### 8a. Create and grant the JWT role + +Create the role: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "CREATE ROLE rls_role NOLOGIN;" +``` + +Grant schema and table permissions on current and future tables: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT USAGE ON SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO rls_role;" +docker compose exec db psql -U postgres -d test_database_1 -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO rls_role;" +``` + +Allow the connection-string user to switch into that role: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "GRANT rls_role TO postgres;" +``` + +Verify: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d postgres -c "SELECT rolname, rolsuper, rolcanlogin, rolbypassrls FROM pg_roles WHERE rolname = 'rls_role';" +docker compose exec db psql -U postgres -d test_database_1 -c "\\ddp" +``` + +If you want to test the exact session shape CloudSync uses: + +```bash +cd /data/cloudsync-postgres +docker compose exec db psql -U postgres -d test_database_1 -c "BEGIN; SELECT set_config('request.jwt.claims', '{\"sub\":\"test-user-1\",\"role\":\"rls_role\"}', true); SET LOCAL ROLE rls_role; SELECT current_role, current_setting('request.jwt.claims', true); ROLLBACK;" +``` + +**HS256 (shared secret):** + +```bash +curl -X POST http://localhost:3001/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1", "role": "rls_role"}' +``` + +**RS256 (JWKS):** + +```bash +curl -X POST http://localhost:3002/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1", "role": "rls_role"}' +``` + +Response (both): + +```json +{"token":"eyJhbG...","expiresIn":"24h"} +``` + +--- + +## Step 9: Register with CloudSync server + +```bash +export CLOUDSYNC_URL="https://your-cloudsync-server.fly.dev" +export ORG_API_KEY="" + +# Get the Postgres password from .env +source /data/cloudsync-postgres/.env + +# Connection string (same Fly org — use .internal network) +export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@.internal:5432/postgres" + +# Or via fly proxy from local machine: +# fly proxy 5432:5432 -a +# export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" +``` + +Register the database: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{ + "label": "Fly.io Postgres", + "connectionString": "'"$CONNECTION_STRING"'", + "provider": "postgres", + "projectId": "cloudsync-postgres-flyio", + "databaseName": "postgres" + }' +``` + +Save the returned `managedDatabaseId`: + +```bash +export MANAGED_DATABASE_ID="" +``` + +--- + +## Step 10: Test CloudSync sync + +### 10a. Create and enable a test table + +```bash +# On the Fly VM +docker compose exec db psql -U postgres -c " +CREATE TABLE IF NOT EXISTS todos ( + id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); +" + +# Enable sync via CloudSync API +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"tables":["todos"]}' +``` + +### 10b. Generate a token and sync from SQLite + +```bash +# Get a JWT token from the auth server +TOKEN=$(curl -s -X POST http://localhost:3001/token \ + -H "Content-Type: application/json" \ + -d '{"sub": "user-1"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") +``` + +In a SQLite client: + +```sql +.load path/to/cloudsync + +CREATE TABLE todos ( + id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); + +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_token(''); + +INSERT INTO todos (title) VALUES ('Test from SQLite'); +SELECT cloudsync_network_sync(500, 5); +``` + +Verify on Postgres: + +```bash +docker compose exec db psql -U postgres -c "SELECT * FROM todos;" +``` + +--- + +## Step 11: CloudSync server JWT configuration + +The CloudSync server needs to validate tokens from your auth server. Configuration depends on which auth method you chose. + +### Option A: HS256 (shared secret) + +In the CloudSync dashboard, go to your PostgreSQL project → **Configuration** → **Edit connection settings**: +- Under **JWT secret**, enter your `JWT_SECRET` value from `.env` +- Click **Save** + +Both the auth server and CloudSync must use the same raw secret string (not base64-decoded). + +### Option B: RS256 (JWKS) + +Configure the JWKS auth server and CloudSync to use asymmetric key verification. + +**1. Update docker-compose.yml - JWKS auth server ISSUER:** + +```yaml + auth-jwks: + environment: + ISSUER: http://.internal:3002 +``` + +The issuer is the **base URL** (CloudSync automatically appends `/.well-known/jwks.json`). + +**2. Configure CloudSync to accept this issuer:** + +In the CloudSync dashboard for this PostgreSQL project: +- Go to **Configuration** tab → **Edit connection settings** +- Under **JWT allowed issuers**, enter: + ``` + http://.internal:3002 + ``` + +CloudSync will: +1. Receive JWT tokens with `iss: http://.internal:3002` +2. Validate the issuer matches the allowed list +3. Fetch the public key from `http://.internal:3002/.well-known/jwks.json` +4. Verify the token signature + +This is how production auth systems (Auth0, Supabase, Firebase) work — no shared secrets needed. + +--- + +## Access your services + +| Service | URL | +|---------|-----| +| **PostgreSQL** | `postgres://postgres:@.internal:5432/postgres` | +| **Auth Server (HS256)** | `http://.internal:3001` | +| **Auth Server (JWKS)** | `http://.internal:3002` | +| **JWKS Endpoint** | `http://.internal:3002/.well-known/jwks.json` | + +From your local machine, use `fly proxy`: + +```bash +fly proxy 5432:5432 -a # Postgres +fly proxy 3001:3001 -a # Auth server (HS256) +fly proxy 3002:3002 -a # Auth server (JWKS) +``` + +--- + +## Reference: CloudSync Configuration + +After deployment, use these values to configure CloudSync dashboard: + +### Database Connection + +``` +postgresql://postgres:@.internal:5432/postgres +``` + +Replace: +- ``: from `.env` file +- ``: your Fly.io app name + +### JWT Secret (HS256) + +For simple/development setups using shared secrets: + +```env +JWT_SECRET= +``` + +Enter this in CloudSync dashboard → **Configuration** → **JWT secret** + +### JWT Issuer (RS256 with JWKS) + +For production setups using asymmetric keys: + +``` +http://.internal:3002 +``` + +Enter this in CloudSync dashboard → **Configuration** → **JWT allowed issuers** + +CloudSync will automatically fetch the public key from: +``` +http://.internal:3002/.well-known/jwks.json +``` + +--- + +## Maintenance + +### Startup script (survives VM restarts) + +Fly VM root filesystem resets on stop/start — only `/data` persists. Create a startup script: + +```bash +cat > /data/startup.sh << 'SCRIPT' +#!/bin/bash +set -e + +echo "=== Installing Docker ===" +apt-get update && apt-get install -y ca-certificates curl gnupg fuse-overlayfs + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + +echo "=== Configuring Docker ===" +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json + +echo "=== Starting Docker ===" +dockerd > /data/dockerd.log 2>&1 & +until docker info > /dev/null 2>&1; do sleep 1; done +echo "Docker is ready!" + +echo "=== Starting CloudSync Postgres ===" +cd /data/cloudsync-postgres +docker compose up -d + +echo "=== Done! ===" +SCRIPT +chmod +x /data/startup.sh +``` + +After any VM restart: + +```bash +fly ssh console --app +/data/startup.sh +``` + +### Update CloudSync extension + +On your local machine: + +```bash +cd /path/to/sqlite-sync-dev +git pull && git submodule update --init --recursive +docker build --platform linux/amd64 \ + -f docker/postgresql/Dockerfile \ + -t /postgres-cloudsync:17 \ + . +docker push /postgres-cloudsync:17 +``` + +On the Fly VM: + +```bash +cd /data/cloudsync-postgres +docker compose pull db +docker compose up -d db +``` + +### View logs + +```bash +docker compose logs -f # All services +docker compose logs -f db # Postgres only +docker compose logs -f auth # Auth server only +``` + +--- + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| `fractional_indexing.h: No such file or directory` | Run `git submodule update --init --recursive` before building | +| `cloudsync_version()` not found | Init scripts only run on first start. Run `CREATE EXTENSION IF NOT EXISTS cloudsync;` manually | +| Auth server won't start | Check `docker compose logs auth`. Ensure `npm install` was run in `auth-server/` | +| Token verification fails (HS256) | Ensure `JWT_SECRET` matches exactly — CloudSync uses the raw string, not base64-decoded | +| Token verification fails (JWKS) | Ensure CloudSync can reach the JWKS endpoint and `JWT_ISSUER` matches the `ISSUER` env var | +| JWKS keys lost after restart | The JWKS server generates new keys on each start. For production, persist keys to a volume | +| Docker commands not found after VM restart | Run `/data/startup.sh` — Fly VM root filesystem resets on stop/start | +| `fuse-overlayfs` not working | Install it: `apt-get install -y fuse-overlayfs` | +| Can't connect to Postgres from outside Fly | Use `fly proxy 5432:5432 -a ` | diff --git a/docs/PriKey.md b/docs/internal/pri-key.md similarity index 100% rename from docs/PriKey.md rename to docs/internal/pri-key.md diff --git a/docs/RowID.md b/docs/internal/row-id.md similarity index 100% rename from docs/RowID.md rename to docs/internal/row-id.md diff --git a/docs/internal/supabase-flyio.md b/docs/internal/supabase-flyio.md new file mode 100644 index 0000000..762d0ab --- /dev/null +++ b/docs/internal/supabase-flyio.md @@ -0,0 +1,1001 @@ +# Self-Hosting Supabase on Fly.io with CloudSync Extension + +This guide walks you through deploying a full self-hosted Supabase stack on a Fly.io VM, with the CloudSync PostgreSQL extension pre-installed. By the end you will have: + +- A Fly.io VM running all 13 Supabase services via Docker Compose +- PostgreSQL with the CloudSync CRDT extension baked in +- Supabase Studio dashboard accessible over HTTPS +- A custom Postgres image published to Docker Hub + +## Prerequisites + +Install these on your **local machine** before starting: + +| Tool | Purpose | Install | +|------|---------|---------| +| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Build the custom Postgres image | See [Installing Docker Desktop](#installing-docker-desktop) below | +| [Fly CLI (`flyctl`)](https://fly.io/docs/flyctl/install/) | Provision and manage Fly.io machines | `brew install flyctl` (macOS) or `curl -L https://fly.io/install.sh \| sh` | +| [Git](https://git-scm.com/) | Clone repositories | `brew install git` (macOS) | +| [Docker Hub](https://hub.docker.com/) account | Host your custom Postgres image | Free signup at hub.docker.com | + +You also need a [Fly.io account](https://fly.io/app/sign-up). A credit card is required even for free tier. + +### Installing Docker Desktop + +Docker Desktop is the application that lets you build and run container images on your Mac. + +1. **Download** from https://www.docker.com/products/docker-desktop/ — pick **Apple chip** (M1/M2/M3/M4) or **Intel chip** depending on your Mac. + + > Not sure which you have? Click the Apple menu () → **About This Mac**. It will say either "Apple M1/M2/M3/M4" or "Intel". + +2. **Install**: Open the downloaded `.dmg` file and drag Docker into your Applications folder. + +3. **Launch**: Open Docker from Applications (or Spotlight: Cmd+Space → type "Docker"). It will ask for your password to install system components — that's normal. + +4. **Wait**: A whale icon appears in your menu bar. Wait until it says "Docker Desktop is running" (the whale stops animating). + +5. **Verify** in Terminal: + ```bash + docker --version + # Should output: Docker version 27.x.x or similar + ``` + +### Setting up Docker Hub + +Docker Hub is a free cloud registry where you'll upload your custom Postgres image so the Fly.io server can download it. + +1. **Sign up** at https://hub.docker.com/signup — pick a username. This becomes your image prefix (e.g., `myusername/supabase-postgres-cloudsync`). + +2. **Log in from Terminal**: + ```bash + docker login + ``` + Enter your Docker Hub username and password. You should see "Login Succeeded". + +### Fly.io VM requirements + +Supabase runs 13 services simultaneously (Postgres, Auth, PostgREST, Realtime, Studio, Kong, Storage, etc.), which is why it needs more resources than a typical single app. + +| Resource | Minimum | Recommended | +|----------|---------|-------------| +| RAM | 4 GB | 8 GB+ | +| CPU | 2 cores | 4 cores | +| Disk | 50 GB SSD | 80 GB+ | + +--- + +## Step 1: Initialize git submodules + +The CloudSync extension depends on the [fractional-indexing](https://github.com/sqliteai/fractional-indexing) library, which is included as a git submodule. If you haven't done this already, initialize it: + +```bash +cd /path/to/sqlite-sync-dev +git submodule update --init --recursive +``` + +Without this, the build will fail with `fractional_indexing.h: No such file or directory`. + +--- + +## Step 2: Build the custom Supabase Postgres image + +> **Important — match the Postgres version!** Check which Postgres version the Supabase docker-compose uses by looking at the `db` service `image` tag in `docker-compose.yml` (e.g., `supabase/postgres:15.8.1.085` means PG 15). You must build your custom image with the **same tag**. Using the wrong version will cause init script failures. + +The `make postgres-supabase-build` command does the following: + +1. **Pulls the official Supabase Postgres base image** (e.g., `public.ecr.aws/supabase/postgres:15.8.1.085`) — this is Supabase's standard PostgreSQL image that ships with ~30 extensions pre-installed (PostGIS, pgvector, etc.) +2. **Runs a multi-stage Docker build** using `docker/postgresql/Dockerfile.supabase`: + - **Stage 1 (builder)**: Installs C build tools (`gcc`, `make`), copies the CloudSync source code (`src/`, `modules/`), and compiles `cloudsync.so` against Supabase's `pg_config` + - **Stage 2 (runtime)**: Starts from a clean Supabase Postgres image and copies in just three kinds of file: + - `cloudsync.so` — the compiled extension binary + - `cloudsync.control` — tells PostgreSQL the extension's name and default version (generated at build time from `cloudsync.control.in`, with the version read from `src/cloudsync.h`) + - `cloudsync--.sql` — the SQL that defines all CloudSync functions for the current release (e.g. `cloudsync--1.0.16.sql`), plus any `cloudsync----.sql` upgrade scripts shipped under `src/postgresql/migrations/` +3. **Tags the result** with the same name as the base image, so it's a drop-in replacement + +To find the correct tag, clone the Supabase repo and check: + +```bash +grep 'image: supabase/postgres:' supabase/docker/docker-compose.yml +# Example output: image: supabase/postgres:15.8.1.085 +# Use the version after the colon as your SUPABASE_POSTGRES_TAG +``` + +Run from the sqlite-sync-dev repo root: + +```bash +make postgres-supabase-build SUPABASE_POSTGRES_TAG=15.8.1.085 +``` + +Verify the image was built: + +```bash +docker images | grep supabase-postgres-cloudsync +# Should show: /supabase-postgres-cloudsync 15.8.1.085 ... +``` + +Verify CloudSync is installed inside the image: + +```bash +docker run --rm /supabase-postgres-cloudsync:15.8.1.085 \ + find / -name "cloudsync*" -type f 2>/dev/null +# Should list cloudsync.so, cloudsync.control, and cloudsync--.sql +# (plus any cloudsync----.sql upgrade scripts) +# in /nix/store/...-postgresql-and-plugins-15.8/ paths +``` + +--- + +## Step 3: Build for the correct architecture and push to Docker Hub + +The Fly.io VM needs to pull your custom image from a container registry. We use Docker Hub (free, no extra auth needed on the VM). + +> **Important — architecture mismatch**: If you're building on an Apple Silicon Mac (M1/M2/M3/M4), `make postgres-supabase-build` produces an ARM image. Fly.io VMs run x86 (amd64) by default, so the ARM image won't work. You must build for the target architecture explicitly. + +First, pull the base image for amd64 (this ensures Docker has the correct platform variant cached): + +```bash +docker pull --platform linux/amd64 public.ecr.aws/supabase/postgres:15.8.1.085 +``` + +Then build for `linux/amd64` (x86, which is what Fly.io uses): + +```bash +docker build --platform linux/amd64 \ + --build-arg SUPABASE_POSTGRES_TAG=15.8.1.085 \ + -f docker/postgresql/Dockerfile.supabase \ + -t /supabase-postgres-cloudsync:15.8.1.085 \ + . +``` + +Push the image (you must be logged in: `docker login`): + +```bash +docker push /supabase-postgres-cloudsync:15.8.1.085 +``` + +> **Note**: `docker buildx build ... --push` may fail with ECR registry resolution errors. The two-step approach above (build then push) is more reliable. + +> If you're building on an Intel Mac or a Linux x86 machine, `make postgres-supabase-build` already produces an amd64 image, so you can simply tag and push: +> ```bash +> docker tag public.ecr.aws/supabase/postgres:15.8.1.085 \ +> /supabase-postgres-cloudsync:15.8.1.085 +> docker push /supabase-postgres-cloudsync:15.8.1.085 +> ``` + +--- + +## Step 4: Provision a Fly.io VM + +We use a Fly Machine as a plain Linux VM running Docker Compose — not Fly's container orchestration. + +### 4a. Log in to Fly + +```bash +fly auth login +``` + +This opens your browser to authenticate with your Fly.io account. + +### 4b. Create a Fly app + +```bash +fly apps create +``` + +### 4c. Create a persistent volume for data + +```bash +fly volumes create supabase_data --app --region --size 50 +``` + +Pick a [region](https://fly.io/docs/reference/regions/) close to you. You can see all available regions with `fly platform regions`. Common choices: + +| Code | Location | +|------|----------| +| `fra` | Frankfurt, Germany | +| `ams` | Amsterdam, Netherlands | +| `lhr` | London, UK | +| `ord` | Chicago, US | +| `iad` | Virginia, US | +| `sin` | Singapore | + +When prompted "Do you still want to use the volumes feature?", type `y` — the warning about multiple volumes is for high-availability production setups; a single volume is fine for testing. + +### 4d. Create a Fly Machine + +```bash +fly machine run ubuntu:24.04 \ + --app \ + --region \ + --vm-size shared-cpu-4x \ + --vm-memory 4096 \ + --volume supabase_data:/data \ + --name supabase-vm \ + -- sleep inf +``` + +The `-- sleep inf` at the end is important — it tells the VM to run an infinite sleep process so it stays alive. Without it, the Ubuntu container exits immediately and the machine stops. + +This creates an Ubuntu 24.04 VM with 4 CPU cores, 4 GB RAM, and your 50 GB volume mounted at `/data`. + +The VM size (`shared-cpu-4x` + 4096 MB) meets Supabase's minimum requirements. For a test/dev deployment this is fine. You can resize later with `fly machine update` if needed. + +### 4e. Allocate a public IP + +```bash +fly ips allocate-v4 --shared --app +fly ips allocate-v6 --app +``` + +Note the IPv4 address — you'll need it for `SUPABASE_PUBLIC_URL`. + +--- + +## Step 5: Set up Docker and Supabase on the VM + +### 5a. SSH into the machine + +```bash +fly ssh console --app +``` + +### 5b. Install Docker Engine + +```bash +apt-get update +apt-get install -y ca-certificates curl gnupg + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + +# Verify +docker --version +docker compose version +``` + +### 5c. Configure Docker storage driver + +Docker's default `overlayfs` storage driver doesn't work inside a Fly VM (you'll get "failed to convert whiteout file: operation not permitted" errors). Use `fuse-overlayfs` instead — it works in unprivileged environments like Fly VMs and is much faster and more space-efficient than the `vfs` fallback. + +Install fuse-overlayfs: + +```bash +apt-get install -y fuse-overlayfs +``` + +Configure Docker to use it: + +```bash +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json +``` + +> **Why not `vfs`?** The `vfs` driver copies every image layer in full instead of sharing them. This means 13 Supabase images can use 35GB+ of disk (vs ~5-8GB with fuse-overlayfs), and image pulls/extraction are extremely slow (30-60 minutes vs a few minutes). Avoid `vfs` unless fuse-overlayfs doesn't work. + +### 5d. Start the Docker daemon + +The Fly VM doesn't auto-start Docker. You need to start it manually: + +```bash +dockerd & +``` + +Wait for the `API listen on /var/run/docker.sock` message before running any Docker commands. This message means Docker is ready. + +> **Note**: If Docker is already running, you'll see a "process is still running" error. That's fine — it means Docker is already available. + +### 5e. Clone and set up Supabase + +```bash +cd /data +git clone --depth 1 https://github.com/supabase/supabase +mkdir -p supabase-docker +cp -rf supabase/docker/* supabase-docker/ +cp supabase/docker/.env.example supabase-docker/.env +cd supabase-docker +``` + +--- + +## Step 6: Configure secrets + +### 6a. Generate keys automatically + +```bash +sh ./utils/generate-keys.sh +``` + +Review the output. The script updates `.env` with generated `JWT_SECRET`, `ANON_KEY`, and `SERVICE_ROLE_KEY`. + +### 6a.1. Get the JWT secret later + +If you need the Supabase Auth JWT secret after setup, read the `JWT_SECRET` value from the same `.env` file used by Docker Compose: + +```bash +cd /data/supabase-docker +grep '^JWT_SECRET=' .env +``` + +That value is the secret GoTrue (Supabase Auth) uses to sign and verify access tokens. + +If you want to confirm what the running auth container sees, check the container environment: + +```bash +docker compose exec auth printenv GOTRUE_JWT_SECRET +``` + +Both commands should return the same value. If they do not, restart the stack after updating `.env`: + +```bash +docker compose up -d +``` + +### 6b. Edit `.env` manually for remaining values + +```bash +# Install a text editor if needed +apt-get install -y nano +nano .env +``` + +Set these values: + +```env +############ +# Required # +############ + +# Database — letters and numbers only, no special characters +POSTGRES_PASSWORD= + +# URLs — replace with your Fly app's public IP or domain +SUPABASE_PUBLIC_URL=http://:8000 +API_EXTERNAL_URL=http://:8000 +SITE_URL=http://localhost:3000 + +# Dashboard login credentials +DASHBOARD_USERNAME=supabase +DASHBOARD_PASSWORD= + +############ +# Secrets # +############ +# These should already be set by generate-keys.sh, but verify they exist: +# JWT_SECRET= +# ANON_KEY= +# SERVICE_ROLE_KEY= + +# Generate the rest if not already set: +# openssl rand -base64 48 → SECRET_KEY_BASE +# openssl rand -hex 16 → VAULT_ENC_KEY (must be exactly 32 chars) +# openssl rand -base64 24 → PG_META_CRYPTO_KEY +# openssl rand -base64 24 → LOGFLARE_PUBLIC_ACCESS_TOKEN +# openssl rand -base64 24 → LOGFLARE_PRIVATE_ACCESS_TOKEN +# openssl rand -hex 16 → S3_PROTOCOL_ACCESS_KEY_ID +# openssl rand -hex 32 → S3_PROTOCOL_ACCESS_KEY_SECRET +# openssl rand -hex 16 → MINIO_ROOT_PASSWORD +``` + +--- + +## Step 7: Swap in the CloudSync Postgres image + +Edit `docker-compose.yml` and find the `db` service (near the top). Replace the `image` line: + +```yaml +services: + db: + # BEFORE: image: supabase/postgres:${POSTGRES_VERSION} + # AFTER: + image: sqlitecloud/sqlite-sync-supabase:15.8.1.085 +``` + +Use `sqlitecloud/sqlite-sync-supabase:15.8.1.085` if you want the published image. If you built and pushed your own image in Step 3 for internal testing, use that exact image path instead. + +### Add the CloudSync init script + +Create the init SQL: + +```bash +cat > volumes/db/cloudsync.sql << 'EOF' +CREATE EXTENSION IF NOT EXISTS cloudsync; +EOF +``` + +Add a volume mount to the `db` service in `docker-compose.yml`: + +```yaml +services: + db: + volumes: + # ... existing volume mounts ... + - ./volumes/db/cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z +``` + +The `100-` prefix ensures CloudSync loads after Supabase's own init scripts (numbered 97-99). + +> **Important**: Init scripts only run when the data directory is empty (first start). If you've already started Postgres once and need to add the extension, connect and run `CREATE EXTENSION cloudsync;` manually. + +--- + +## Step 8: Start Supabase + +```bash +cd /data/supabase-docker +docker compose pull +docker compose up -d +``` + +Wait ~1 minute for all services to start, then verify: + +```bash +docker compose ps +``` + +All services should show `Up (healthy)`. If any service is unhealthy: + +```bash +docker compose logs +``` + +### Fix: services fail with "password authentication failed" + +If you see `FATAL: password authentication failed for user "authenticator"` (or `supabase_auth_admin`, `supabase_storage_admin`, `supabase_admin`) in the logs, the database users were created with a different password than what's in `.env`. This happens when `POSTGRES_PASSWORD` was changed after the first start, or when the DB data persists across reinstalls. + +Fix by updating all user passwords to match your `.env`: + +```bash +# Replace YOUR_PASSWORD with the value of POSTGRES_PASSWORD from your .env file +docker compose exec db psql -U postgres -c "ALTER USER authenticator WITH PASSWORD 'YOUR_PASSWORD';" +docker compose exec db psql -U postgres -c "ALTER USER supabase_auth_admin WITH PASSWORD 'YOUR_PASSWORD';" +docker compose exec db psql -U postgres -c "ALTER USER supabase_storage_admin WITH PASSWORD 'YOUR_PASSWORD';" +docker compose exec db psql -U postgres -c "ALTER USER supabase_admin WITH PASSWORD 'YOUR_PASSWORD';" +``` + +Then restart: + +```bash +docker compose restart +``` + +### Fix: analytics (Logflare) keeps crashing + +The analytics service (Logflare) often fails in self-hosted setups due to migration issues. Since it's **optional** (only used for log analytics, not required for CloudSync or core Supabase features), the simplest fix is to disable it. + +In `docker-compose.yml`: + +1. **Remove the `analytics` dependency** from the `studio` service's `depends_on` block. Delete these lines: + ```yaml + analytics: + condition: service_healthy + ``` + If `depends_on:` becomes empty after removing, delete the `depends_on:` line too. + +2. **Comment out the `LOGFLARE_URL`** environment variable in the `studio` service: + ```yaml + # LOGFLARE_URL: http://analytics:4000 + ``` + +3. **Restart**: + ```bash + docker compose stop analytics + docker compose up -d + ``` + +### Fix: CloudSync extension not found + +If `SELECT cloudsync_version();` returns "function does not exist", the init script didn't run (it only runs on first boot when the data directory is empty). Create the extension manually: + +```bash +docker compose exec db psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS cloudsync;" +``` + +### Fix: Auth service crashes with "must be owner of function uid" + +When the auth (GoTrue) service fails to start and logs show errors like: + +``` +error executing migrations: must be owner of function uid (SQLSTATE 42501) +``` + +This happens because the `auth.uid()`, `auth.role()`, and `auth.email()` functions were created by `postgres` (via the init scripts or CloudSync extension), but the auth service runs migrations as `supabase_auth_admin` and expects to own those functions. + +**Symptoms:** +- Auth service keeps restarting +- Supabase Studio shows "Failed to retrieve users" or "column users.banned_until does not exist" (because auth migrations didn't complete) +- `docker compose logs auth` shows the `SQLSTATE 42501` ownership error + +**Fix:** Transfer ownership of the functions to `supabase_auth_admin`: + +```bash +docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.uid() OWNER TO supabase_auth_admin;" +docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.role() OWNER TO supabase_auth_admin;" +docker compose exec db psql -U postgres -c "ALTER FUNCTION auth.email() OWNER TO supabase_auth_admin;" +docker compose restart auth +``` + +After restarting, wait ~30 seconds and check that auth is healthy: + +```bash +docker compose ps auth +docker compose logs --tail=20 auth +``` + +You should see auth in a `healthy` state and the migrations completing successfully. + +### Fix: Supavisor (connection pooler) keeps crashing + +If `docker compose logs supavisor` shows: + +``` +Setting RLIMIT_NOFILE to 100000 +/app/limits.sh: line 6: ulimit: open files: cannot modify limit: Operation not permitted +``` + +This happens because Supavisor's startup script (`/app/limits.sh`) tries to increase the open-file limit to 100,000, but the Fly VM's kernel has a lower cap (typically 10,240) and doesn't allow it. The script failure crashes the container. + +**Fix:** Override the entrypoint in `docker-compose.yml` to skip the limits script. Add this line right after `container_name: supabase-pooler`: + +```yaml + entrypoint: ["/usr/bin/tini", "-s", "-g", "--"] +``` + +Or apply with sed: + +```bash +sed -i '/container_name: supabase-pooler/a\ entrypoint: ["/usr/bin/tini", "-s", "-g", "--"]' /data/supabase-docker/docker-compose.yml +``` + +This keeps the same `tini` init process but skips `/app/limits.sh`. The VM's default open-file limit (10,240) is sufficient for testing. + +Then restart: + +```bash +docker compose up -d supavisor +``` + +--- + +## Step 9: Verify CloudSync + +Connect to the database: + +```bash +docker compose exec db psql -U postgres +``` + +```sql +-- Check the extension is installed +SELECT * FROM pg_extension WHERE extname = 'cloudsync'; + +-- Check version +SELECT cloudsync_version(); +``` + +If `cloudsync_version()` returns "function does not exist", create the extension manually: + +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +--- + +## Step 10: Test CloudSync Sync + +This section walks through testing the full sync flow: registering the database with the CloudSync server, creating tables, enabling sync, and running a roundtrip test. + +### Prerequisites + +You need a running **CloudSync server**. This can be the staging server or a local instance. + +```bash +export CLOUDSYNC_URL="https://cloudsync-staging-testing.fly.dev" # CloudSync server URL +export ORG_API_KEY="" # Organization API key +``` + +#### Connection string + +The CloudSync server needs a PostgreSQL connection string to reach your database. There are two options depending on where your CloudSync server runs: + +**Option A: CloudSync on the same Fly org (`.internal` network)** + +If both the CloudSync server and the Supabase VM are in the same Fly org, they can communicate over Fly's **private internal network** — no public port exposure needed. Connect directly to the `db` container's mapped port (5432 is exposed on the host by default in docker-compose): + +```bash +# Direct connection (no Supavisor) — recommended for CloudSync server-to-server +export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@.internal:5432/postgres" +``` + +**Option B: CloudSync running outside Fly (e.g., local machine, another cloud)** + +Use `fly proxy` to tunnel the Postgres port to your local machine: + +```bash +# In a separate terminal — keep this running +fly proxy 5432:5432 -a +``` + +This makes the remote Postgres available at `localhost:5432`. Then use: + +```bash +export CONNECTION_STRING="postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" +``` + +> **Note:** The proxy must stay running in a separate terminal for the duration of your session. If the proxy disconnects, just re-run the command. + +To verify the connection works: + +```bash +# Option A: SSH into the VM and test locally +fly ssh console --app +docker compose exec db psql -U postgres -c "SELECT 1;" + +# Option B: With fly proxy running, test from your local machine +psql "postgres://postgres:$POSTGRES_PASSWORD@localhost:5432/postgres" -c "SELECT 1;" +``` + +### 10a. Verify CloudSync server is reachable + +```bash +curl "$CLOUDSYNC_URL/healthz" +# Expected: {"status":"ok"} +``` + +### 10b. Register the Supabase database with CloudSync + +This tells CloudSync where to find your PostgreSQL database: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{ + "label": "Supabase Fly.io Test", + "connectionString": "'"$CONNECTION_STRING"'", + "provider": "postgres", + "flavor": "supabase", + "projectId": "cloudsync-supabase-test", + "databaseName": "postgres" + }' +``` + +Save the returned `managedDatabaseId` — you'll need it for all subsequent operations: + +```bash +export MANAGED_DATABASE_ID="" +``` + +### 10c. Verify database connectivity + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/verify" \ + --header "Authorization: Bearer $ORG_API_KEY" +``` + +Expected: status should show the database is reachable. + +### 10d. Create a test table on the Supabase database + +SSH into the Fly VM and create a table: + +```bash +docker compose exec db psql -U postgres -c " +CREATE TABLE IF NOT EXISTS todos ( + id TEXT PRIMARY KEY DEFAULT cloudsync_uuid(), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); +" +``` + +### 10e. Enable CloudSync on the table + +From your local machine, enable sync via the management API: + +```bash +curl --request POST "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/enable" \ + --header "Authorization: Bearer $ORG_API_KEY" \ + --header "Content-Type: application/json" \ + --data '{"tables":["todos"]}' +``` + +Verify: + +```bash +curl --request GET "$CLOUDSYNC_URL/v1/databases/$MANAGED_DATABASE_ID/cloudsync/tables" \ + --header "Authorization: Bearer $ORG_API_KEY" +``` + +The `todos` table should show `"enabled": true`. + +### 10f. Test sync roundtrip from a SQLite client + +On your local machine, create a SQLite database and sync: + +```sql +-- Load the sqlite-sync extension +.load path/to/cloudsync + +-- Create the same table schema +CREATE TABLE todos ( + id TEXT PRIMARY KEY DEFAULT (cloudsync_uuid()), + title TEXT NOT NULL DEFAULT '', + done BOOLEAN DEFAULT false +); +SELECT cloudsync_init('todos'); + +-- Configure network +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_token(''); + +-- Insert a row locally +INSERT INTO todos (title) VALUES ('Test from SQLite'); + +-- Sync: send local changes, check for remote changes +SELECT cloudsync_network_sync(500, 5); +``` + +Then verify the row arrived on Supabase: + +```bash +docker compose exec db psql -U postgres -c "SELECT * FROM todos;" +``` + +### 10g. Test reverse sync (Supabase → SQLite) + +Insert a row directly on Supabase: + +```bash +docker compose exec db psql -U postgres -c " +INSERT INTO todos (id, title, done) VALUES (cloudsync_uuid(), 'Test from Supabase', false); +" +``` + +Then sync from the SQLite client: + +```sql +SELECT cloudsync_network_check_changes(); +SELECT * FROM todos; +``` + +The row from Supabase should appear in SQLite. + +--- + +## Step 11: Access your services + +| Service | URL | +|---------|-----| +| **Supabase Studio** | `http://:8000` | +| REST API | `http://:8000/rest/v1/` | +| Auth API | `http://:8000/auth/v1/` | +| Storage API | `http://:8000/storage/v1/` | +| Realtime | `http://:8000/realtime/v1/` | + +Studio dashboard requires a username and password. To find them, check your `.env` file on the VM: + +```bash +grep DASHBOARD /data/supabase-docker/.env +``` + +The values are `DASHBOARD_USERNAME` (default: `supabase`) and `DASHBOARD_PASSWORD` (default: `this_password_is_insecure_and_should_be_updated`). + +> **Note:** The Fly VM doesn't expose ports publicly by default. Use `fly proxy` to access services from your local machine: +> ```bash +> fly proxy 8000:8000 -a +> ``` +> Then open `http://localhost:8000` in your browser. + +### Connect to Postgres directly + +Use `fly proxy` to tunnel the Postgres port to your local machine: + +```bash +# In a separate terminal — keep this running +fly proxy 5432:5432 -a +``` + +Then connect from your local machine: + +```bash +psql 'postgres://postgres:@localhost:5432/postgres' +``` + +> **Tip:** You can proxy multiple ports at once by running multiple `fly proxy` commands in separate terminals (e.g., `8000` for Studio and `5432` for Postgres). + +--- + +## Step 11: Set up HTTPS (production) + +For production use, put a reverse proxy in front of Kong. The simplest option is [Caddy](https://caddyserver.com/) which handles TLS automatically. + +On the Fly VM: + +```bash +apt-get install -y debian-keyring debian-archive-keyring apt-transport-https +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg +curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list +apt-get update +apt-get install -y caddy +``` + +Create `/etc/caddy/Caddyfile`: + +``` +your-domain.com { + reverse_proxy localhost:8000 +} +``` + +```bash +systemctl enable caddy +systemctl start caddy +``` + +Then update `.env`: + +```env +SUPABASE_PUBLIC_URL=https://your-domain.com +API_EXTERNAL_URL=https://your-domain.com +``` + +Restart Supabase: + +```bash +cd /data/supabase-docker +docker compose down && docker compose up -d +``` + +--- + +## Maintenance + +### Update Supabase services + +```bash +cd /data/supabase-docker +# Update image tags in docker-compose.yml, then: +docker compose pull +docker compose down && docker compose up -d +``` + +### Update CloudSync extension + +On your local machine, rebuild and push the image: + +```bash +cd /path/to/sqlite-sync-dev +git pull # get latest CloudSync code +git submodule update --init --recursive # ensure submodules are up to date +make postgres-supabase-build SUPABASE_POSTGRES_TAG=15.8.1.085 +docker tag public.ecr.aws/supabase/postgres:15.8.1.085 \ + /supabase-postgres-cloudsync:15.8.1.085 +docker push /supabase-postgres-cloudsync:15.8.1.085 +``` + +On the Fly VM: + +```bash +cd /data/supabase-docker +docker compose pull db +docker compose up -d db +``` + +### View logs + +```bash +# All services +docker compose logs -f + +# Specific service +docker compose logs -f db +``` + +### Change database password + +```bash +cd /data/supabase-docker +sh ./utils/db-passwd.sh +docker compose up -d --force-recreate +``` + +### Stop and restart the Fly machine + +The Fly machine's root filesystem resets on every stop/start — only the `/data` volume persists. This means Docker must be reinstalled each time. To automate this, create a startup script (run once): + +```bash +cat > /data/startup.sh << 'SCRIPT' +#!/bin/bash +set -e + +echo "=== Installing Docker ===" +apt-get update && apt-get install -y ca-certificates curl gnupg fuse-overlayfs + +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +chmod a+r /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin + +echo "=== Configuring Docker ===" +mkdir -p /etc/docker +echo '{"storage-driver":"fuse-overlayfs","data-root":"/data/docker"}' > /etc/docker/daemon.json + +echo "=== Starting Docker ===" +dockerd > /data/dockerd.log 2>&1 & + +echo "Waiting for Docker to be ready..." +until docker info > /dev/null 2>&1; do sleep 1; done +echo "Docker is ready!" + +echo "=== Starting Supabase ===" +cd /data/supabase-docker +docker compose up -d + +echo "=== Done! ===" +echo "Run 'docker compose ps' to check service status" +SCRIPT +chmod +x /data/startup.sh +``` + +From then on, every time you restart the machine: + +```bash +# From your local machine: +fly machine stop 287920ea023108 -a # stop +fly machine start 287920ea023108 -a # start +fly ssh console --app + +# On the VM — one command does everything: +/data/startup.sh +``` + +Docker images and Supabase data are on `/data`, so they survive restarts. Only Docker itself needs reinstalling (~1-2 minutes). + +### Stop Supabase (without stopping the machine) + +```bash +docker compose down +``` + +### Destroy everything (irreversible) + +```bash +docker compose down -v +rm -rf volumes/db/data volumes/storage +``` + +--- + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| `fractional_indexing.h: No such file or directory` | Run `git submodule update --init --recursive` before building. The fractional-indexing library is a git submodule that must be initialized. | +| `cloudsync.so: cannot open shared object file` | The custom image wasn't used. Verify `docker compose ps` shows your image, not the default one. | +| Init script didn't run / `cloudsync_version()` not found | Init scripts only run on first start. Run `CREATE EXTENSION IF NOT EXISTS cloudsync;` manually via `docker compose exec db psql -U postgres`. | +| `password authentication failed for user "authenticator"` | Database users have a different password than `.env`. See "Fix: services fail with password authentication failed" above. | +| Analytics (Logflare) keeps crashing | Disable it — see "Fix: analytics keeps crashing" above. It's optional and not needed for CloudSync. | +| `cannot stop container ... did not receive an exit event` | Zombie container. Kill Docker: `kill -9 $(pidof dockerd) $(pidof containerd)`, remove container files: `rm -rf /data/docker/containers/*`, restart dockerd. | +| Services unhealthy after start | Wait 2 minutes. Check `docker compose logs `. Most common: password mismatch (see fix above). | +| Can't pull custom image from VM | Make sure the image is public on Docker Hub, or run `docker login` on the VM. | +| ARM build errors | Fly defaults to x86. If using ARM machines, rebuild with `--platform linux/arm64`. | +| Version mismatch | Run `SHOW server_version;` in psql and ensure your `SUPABASE_POSTGRES_TAG` matches the major version. | +| `no space left on device` during pull | The `fuse-overlayfs` driver is efficient but if disk fills up, extend the volume: `fly volumes extend --size 80 -a `. | +| Docker commands not found after machine restart | The Fly VM root filesystem resets on stop/start. Run `/data/startup.sh` to reinstall Docker. See "Stop and restart the Fly machine" section. | +| Auth crashes with `must be owner of function uid` | Transfer function ownership: `ALTER FUNCTION auth.uid() OWNER TO supabase_auth_admin;` and same for `auth.role()` and `auth.email()`. See "Fix: Auth service crashes" above. | +| Studio shows "column users.banned_until does not exist" | Auth migrations didn't complete. Fix the auth ownership issue above and restart auth. | +| Supavisor crashes with `ulimit: cannot modify limit` | Override entrypoint to skip `/app/limits.sh`. See "Fix: Supavisor keeps crashing" above. | diff --git a/docs/postgresql/CLIENT.md b/docs/postgresql/CLIENT.md new file mode 100644 index 0000000..7aca207 --- /dev/null +++ b/docs/postgresql/CLIENT.md @@ -0,0 +1,218 @@ +# SQLite Sync + +**SQLite Sync** is a multi-platform extension that brings a true **local-first experience** to your applications with minimal effort. It extends standard SQLite tables with built-in support for offline work and automatic synchronization, allowing multiple devices to operate independently — even without a network connection — while seamlessly staying in sync. + +With SQLite Sync, developers can build **distributed, collaborative applications** while continuing to rely on the **simplicity, reliability, and performance of SQLite**. + +Under the hood, SQLite Sync uses advanced **CRDT (Conflict-free Replicated Data Type)** algorithms and data structures designed specifically for **collaborative, distributed systems**: + +- Devices can update data independently, even without a network connection. +- When they reconnect, all changes are **merged automatically and without conflicts**. +- **No data loss. No overwrites. No manual conflict resolution.** + +## Conversion Between SQLite and PostgreSQL Tables + +In this version, make sure to **manually create** the same tables in the PostgreSQL database as used in the SQLite client. + +This guide shows how to manually convert a SQLite table definition to PostgreSQL +so CloudSync can sync between a PostgreSQL server and SQLite clients. + +### 1) Primary Keys + +- Use **TEXT** primary keys in SQLite. +- PostgreSQL primary keys can be **TEXT** or **UUID**. If the PK type + isn't explicitly mapped to a DBTYPE (like UUID), it will be converted to TEXT + in the payload so it remains compatible with the SQLite extension. +- Generate IDs with `cloudsync_uuid()` on both sides. +- Avoid INTEGER auto-increment PKs. + +SQLite: +```sql +id TEXT PRIMARY KEY +``` + +PostgreSQL: +```sql +id TEXT PRIMARY KEY +``` + +PostgreSQL (UUID): +```sql +id UUID PRIMARY KEY +``` + +### 2) NOT NULL Columns Must Have DEFAULTs + +CloudSync merges column-by-column. Any NOT NULL (non-PK) column needs a DEFAULT +to avoid constraint failures during merges. + +Example: +```sql +title TEXT NOT NULL DEFAULT '' +count INTEGER NOT NULL DEFAULT 0 +``` + +### 3) Safe Type Mapping + +Use types that map cleanly to CloudSync's DBTYPEs: + +- INTEGER → `INTEGER` (SQLite) / `INTEGER` (Postgres) +- FLOAT → `REAL` / `DOUBLE` (SQLite) / `DOUBLE PRECISION` (Postgres) +- TEXT → `TEXT` (both) +- BLOB → `BLOB` (SQLite) / `BYTEA` (Postgres) + +Avoid: JSON/JSONB, UUID, INET, CIDR, RANGE, ARRAY unless you accept text-cast +behavior. + +### 4) Defaults That Match Semantics + +Use defaults that serialize the same on both sides: + +- TEXT: `DEFAULT ''` +- INTEGER: `DEFAULT 0` +- FLOAT: `DEFAULT 0.0` +- BLOB: `DEFAULT X'00'` (SQLite) vs `DEFAULT E'\\x00'` (Postgres) + +### 5) Foreign Keys and Triggers + +- Foreign keys can cause merge conflicts; test carefully. +- Application triggers will fire during merge; keep them idempotent or disable + in synced tables. + +### 6) Example Conversion + +SQLite: +```sql +CREATE TABLE notes ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + body TEXT DEFAULT '', + views INTEGER NOT NULL DEFAULT 0, + rating REAL DEFAULT 0.0, + data BLOB +); +``` + +PostgreSQL: +```sql +CREATE TABLE notes ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + body TEXT DEFAULT '', + views INTEGER NOT NULL DEFAULT 0, + rating DOUBLE PRECISION DEFAULT 0.0, + data BYTEA +); +``` + +### 7) Enable CloudSync + +SQLite: +```sql +.load dist/cloudsync.dylib +SELECT cloudsync_init('notes'); +``` + +PostgreSQL: +```sql +CREATE EXTENSION cloudsync; +SELECT cloudsync_init('notes'); +``` + +### Checklist + +- [ ] PKs are TEXT (or UUID in PostgreSQL) +- [ ] All NOT NULL columns have DEFAULT +- [ ] Only INTEGER/FLOAT/TEXT/BLOB-compatible types +- [ ] Same column names and order +- [ ] Same defaults (semantic match) + +Please follow [these Database Schema Recommendations](https://github.com/sqliteai/sqlite-sync?tab=readme-ov-file#database-schema-recommendations) + +## Pre-built Binaries + +Download the appropriate pre-built binary for your platform from the official [Releases](https://github.com/sqliteai/sqlite-sync/releases) page: + +- Linux: x86 and ARM +- macOS: x86 and ARM +- Windows: x86 +- Android +- iOS + +## Loading the Extension + +``` +-- In SQLite CLI +.load ./cloudsync + +-- In SQL +SELECT load_extension('./cloudsync'); +``` + +## WASM Version -> React client-side + +``` +npm i @sqliteai/sqlite-wasm +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-wasm) + +## Swift Package + +You can [add this repository as a package dependency to your Swift project](https://developer.apple.com/documentation/xcode/adding-package-dependencies-to-your-app#Add-a-package-dependency). After adding the package, you'll need to set up SQLite with extension loading by following steps 4 and 5 of [this guide](https://github.com/sqliteai/sqlite-extensions-guide/blob/main/platforms/ios.md#4-set-up-sqlite-with-extension-loading). + +## Android Package + +Add the [following](https://central.sonatype.com/artifact/ai.sqlite/sync) to your Gradle dependencies: + +``` +implementation 'ai.sqlite:sync:1.0.0' +``` + +## Expo + +Install the Expo package: + +``` +npm install @sqliteai/sqlite-sync-expo +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo) + +## React Native + +Install the React Native library: + +``` +npm install @sqliteai/sqlite-sync-react-native +``` + +Then follow the instructions from the [README](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native) + +## Node -> React server-side + +```js +npm i better-sqlite3 +npm i @sqliteai/sqlite-sync + +echo "import { getExtensionPath } from '@sqliteai/sqlite-sync'; +import Database from 'better-sqlite3'; + +const db = new Database(':memory:'); +db.loadExtension(getExtensionPath()); + +// Ready to use +const version = db.prepare('SELECT cloudsync_version()').pluck().get(); +console.log('Sync extension version:', version);" >> index.js + +node index.js +``` + +## Naming Clarification + +- **sqlite-sync** → Client-side SQLite extension +- **cloudsync** → Synchronization server microservice +- **postgres-sync** → PostgreSQL extension + +The sqlite-sync extension is loaded in SQLite under the extension name: +`cloudsync` diff --git a/docs/postgresql/README.md b/docs/postgresql/README.md new file mode 100644 index 0000000..e457b3a --- /dev/null +++ b/docs/postgresql/README.md @@ -0,0 +1,102 @@ +# Architecture Overview + +The **SQLite AI offline-sync solution** consists of three main components: +* **SQLite Sync**: Native client-side SQLite extension +* **CloudSync**: Synchronization microservice +* **Postgres Sync**: Native PostgreSQL extension + +Together, these components provide a complete, production-grade **offline-first synchronization stack** for SQLite and PostgreSQL. + + +# SQLite Sync + +**SQLite Sync** is a native SQLite extension that must be installed and loaded on all client devices. +We provide prebuilt binaries for: +* Desktop and mobile platforms +* WebAssembly (WASM) +* Popular package managers and frameworks including React Native, Expo, Node, Swift PM and Android AAR + +### Architecture Refactoring +The extension has been refactored to support both **SQLite** and **PostgreSQL** backends. +* All database-specific native calls have been isolated in [database.h](../../src/database.h) +* Each database engine implements its own engine-dependent layer +* The core **CRDT logic** is fully shared across engines + +This modular design improves **portability**, **maintainability**, and **cross-database consistency**. +### Testing & Reliability +* Shared CRDT and SQLite components include extensive unit tests +* Code coverage exceeds **90%** +* PostgreSQL-specific code has its own dedicated test suite + +### Key Features +* Deep integration with SQLite — the default database for Edge applications +* Built-in network layer exposed as ordinary SQLite functions +* Cross-platform, language-agnostic payload format +* Works seamlessly in any framework or programming language + +Unlike other offline-sync solutions, **SQLite Sync embeds networking directly inside SQLite**, eliminating external sync SDKs. + +### Supported CRDTs +Currently implemented CRDT algorithms: +* **Last-Write-Wins (LWW)** +* **Grow-Only Set (G-Set)** + +Additional CRDTs can be implemented if needed, though LWW covers most real-world use cases. + + +# CloudSync + +**CloudSync** is a lightweight, stateless microservice responsible for synchronizing clients with central servers. +### Responsibilities +* Synchronizes clients with: + * **SQLite Cloud servers** + * **PostgreSQL servers** +* Manages upload and download of CRDT payloads +* Stores payloads via **AWS S3** +* Collects operational metrics (connected devices, sync volume, traffic, etc.) +* Exposes a complete **REST API** + +### Technology Stack + +* Written in **Go** +* Built on the high-performance **Gin Web Framework** +* Fully **multitenant** +* Connects to multiple DBMS backends +* Stateless architecture enables horizontal scaling simply by adding nodes +* Serialized job queue ensures **no job loss**, even after restarts + +### Observability + +* Metrics dashboard available in [grafana-dashboard.json](../internal/grafana-dashboard.json) + + +# Postgres Sync + +**Postgres Sync** is a native PostgreSQL extension derived from SQLite Sync. +### Features +* Implements the same CRDT algorithms available in SQLite Sync +* Applies CRDT logic to: + * Changes coming from synchronized clients + * Changes made directly in PostgreSQL (CLI, Drizzle, dashboards, etc.) + +This ensures **full bidirectional consistency**, regardless of where changes originate. + +### Schema Handling +SQLite does not support schemas, while PostgreSQL does. To bridge this difference, Postgres Sync introduces a mechanism to: + +* Associate each synchronized table with a specific PostgreSQL schema +* Allow different schemas per table + +This preserves PostgreSQL-native organization while maintaining SQLite compatibility. + +# Current Limitations + +The PostgreSQL integration is actively evolving. Current limitations include: + +* **Table Creation**: Tables must currently be created manually in PostgreSQL before synchronization. We are implementing automatic translation of SQLite CREATE TABLE statements to PostgreSQL syntax. +* **Beta Status**: While extensively tested, the PostgreSQL sync stack should currently be considered **beta software**. Please report any issues; we are committed to resolving them quickly. + +# Next +* [CLIENT](client.md) installation and setup +* [SUPABASE](integrations/supabase.md) configuration and setup +* [SPORT-TRACKER APP](examples/sport-app-supabase.md) demo web app based on SQLite Sync WASM \ No newline at end of file diff --git a/docs/postgresql/examples/sport-tracker-app-supabase.md b/docs/postgresql/examples/sport-tracker-app-supabase.md new file mode 100644 index 0000000..e0ffb9e --- /dev/null +++ b/docs/postgresql/examples/sport-tracker-app-supabase.md @@ -0,0 +1,43 @@ +# Sport Tracker app with SQLite Sync 🚵 + +A Vite/React demonstration app showcasing [**SQLite Sync**](https://github.com/sqliteai/sqlite-sync) implementation for **offline-first** data synchronization across multiple devices. This example illustrates how to integrate SQLite AI's sync capabilities into modern web applications with proper authentication via [Access Token](https://docs.sqlitecloud.io/docs/access-tokens) and [Row-Level Security (RLS)](https://docs.sqlitecloud.io/docs/rls). + +> This app uses the packed WASM version of SQLite with the [SQLite Sync extension enabled](https://www.npmjs.com/package/@sqliteai/sqlite-wasm). + +**The source code is located in [examples/sport-tracker-app](../../examples/sport-tracker-app/)** + +## Setup Instructions + +### 1. Prerequisites +- Node.js 20.x or \>=22.12.0 + +### 2. Database Setup +1. Create database +2. Execute the schema with [sport-tracker-schema-postgres.sql](../../examples/sport-tracker-app/sport-tracker-schema-postgres.sql). +3. Enable CloudSync for all tables on the remote database with: + ```sql + CREATE EXTENSION IF NOT EXISTS cloudsync; + SELECT cloudsync_init('users_sport'); + SELECT cloudsync_init('workouts'); + SELECT cloudsync_init('activities'); + ``` + +### 3. Environment Configuration + +Rename the `.env.example` into `.env` and fill with your values. + +- `VITE_SQLITECLOUD_CONNECTION_STRING`: the url to the CloudSync server: https://cloudsync-staging.fly.dev/ +- `VITE_SQLITECLOUD_DATABASE`: remote database name. +- `VITE_SQLITECLOUD_API_KEY`: a valid user's JWT token. Refresh it when it expires. +- `VITE_SQLITECLOUD_API_URL`: Supabase project API URL. + +### 4. Installation & Run + +```bash +npm install +npm run dev +``` + +### Demo + +Continue reading on the official [README](https://github.com/sqliteai/sqlite-sync/blob/main/examples/sport-tracker-app/README.md#demo-use-case-multi-user-sync-scenario). \ No newline at end of file diff --git a/docs/postgresql/examples/todo-app-postgres.md b/docs/postgresql/examples/todo-app-postgres.md new file mode 100644 index 0000000..827866a --- /dev/null +++ b/docs/postgresql/examples/todo-app-postgres.md @@ -0,0 +1,64 @@ +# Expo CloudSync Example + +A simple Expo example demonstrating SQLite synchronization with CloudSync and PostgreSQL. Build cross-platform apps that sync data seamlessly across devices. + +https://github.com/user-attachments/assets/21a0332a-7f8f-468b-bd5c-004049e70763 + +## 🚀 Quick Start + +### 1. Clone the template + +Create a new project using this template: +```bash +npx create-expo-app MyApp --template @sqliteai/todoapp@dev +cd MyApp +``` + +### 2. Setup + +1. Execute the exact schema from [`to-do-app.sql`](../../examples/to-do-app/to-do-app.sql). +2. Enable CloudSync for all tables on the remote database with: + ```sql + CREATE EXTENSION IF NOT EXISTS cloudsync; + SELECT cloudsync_init('tasks'); + SELECT cloudsync_init('tags'); + SELECT cloudsync_init('tasks_tags'); + ``` +3. Rename the `.env.example` into `.env` and fill with your values. +4. If you're testing with a local server define also the `ANDROID_CONNECTION_STRING` variable and use a different connection string for it, replace localhost with `10.0.2.2`. + +``` +CONNECTION_STRING="http://localhost:8091/postgres" +ANDROID_CONNECTION_STRING="http://10.0.2.2:8091/postgres" +API_TOKEN="token" +``` + +5. Fill the `API_TOKEN` variable with the access token. + +> **⚠️ SECURITY WARNING**: This example puts database connection strings directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** +> +> **Why this is unsafe:** +> - Connection strings contain sensitive credentials +> - Client-side apps expose all environment variables to users +> - Anyone can inspect your app and extract database credentials +> +> **For production apps:** +> - Use the secure [sport-tracker-app](https://github.com/sqliteai/sqlite-sync/tree/main/examples/sport-tracker-app) pattern with authentication tokens and row-level security +> - Never embed database credentials in client applications + +### 4. Build and run the App + +```bash +npx expo prebuild # run once +npm run ios # or android +``` + +## ✨ Features + +- **Add Tasks** - Create new tasks with titles and optional tags. +- **Edit Task Status** - Update task status when completed. +- **Delete Tasks** - Remove tasks from your list. +- **Dropdown Menu** - Select categories for tasks from a predefined list. +- **Cross-Platform** - Works on iOS and Android +- **Offline Support** - Works offline, syncs when connection returns + diff --git a/docs/postgresql/quickstarts/postgres.md b/docs/postgresql/quickstarts/postgres.md new file mode 100644 index 0000000..fcfefe8 --- /dev/null +++ b/docs/postgresql/quickstarts/postgres.md @@ -0,0 +1,164 @@ +# CloudSync Quick Start: Self-Hosted PostgreSQL + +This guide helps you enable CloudSync on a **self-hosted PostgreSQL database**. CloudSync adds offline-first synchronization capabilities to your PostgreSQL database. + +## Step 1: Deploy PostgreSQL with CloudSync + +You can enable CloudSync in one of two ways: +- Use the published Docker image if you run PostgreSQL in Docker +- Install the released extension files into an existing native PostgreSQL installation + +### Option A: Docker + +Use the published PostgreSQL image that already includes the CloudSync extension: +- `sqlitecloud/sqlite-sync-postgres:15` +- `sqlitecloud/sqlite-sync-postgres:17` + +Example using Docker Compose: + +```yaml +services: + db: + image: sqlitecloud/sqlite-sync-postgres:17 + container_name: cloudsync-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: your-secure-password + POSTGRES_DB: postgres + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro + +volumes: + pg_data: +``` + +Create `init.sql`: +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +Run: +```bash +docker compose up -d +``` + +### Option B: Existing PostgreSQL Without Docker + +If you already run PostgreSQL directly on a VM or bare metal, download the release tarball that matches your operating system, CPU architecture, and PostgreSQL major version. + +Extract the archive, then copy the extension files into PostgreSQL's extension directories. The tarball ships `cloudsync.control`, a `cloudsync--.sql` install script for the current release, and — from release 1.0.17 onward — any `cloudsync----.sql` upgrade scripts needed so existing installations can run `ALTER EXTENSION cloudsync UPDATE`. + +```bash +cp cloudsync.so "$(pg_config --pkglibdir)/" +cp cloudsync.control cloudsync--*.sql "$(pg_config --sharedir)/extension/" +``` + +Then connect to PostgreSQL and enable the extension: + +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +--- + +## Step 2: Verify the Extension + +If you are using Docker: + +```bash +docker compose exec db psql -U postgres -d postgres -c "SELECT cloudsync_version();" +``` + +If you are using an existing PostgreSQL installation without Docker: + +```bash +psql -U postgres -d postgres -c "SELECT cloudsync_version();" +``` + +If the extension is installed correctly, PostgreSQL returns the CloudSync version string. + +### Upgrading a later release + +CloudSync uses the first two components of its semver as the PostgreSQL extension version (for example, `1.0.17` installs as extension version `1.0`). How you upgrade depends on which component changed: + +- **PATCH release** (e.g. `1.0.17 → 1.0.18`): pull the new Docker image or replace the extension files on disk and restart PostgreSQL. No SQL-level upgrade is needed — `installed_version` stays at `1.0` and the new binary takes over on reconnect. `SELECT cloudsync_version();` confirms the new semver. +- **MINOR or MAJOR release** (e.g. `1.0.x → 1.1.0`): pull the new artifacts as above, then run once per database: + + ```sql + ALTER EXTENSION cloudsync UPDATE; + ``` + + PostgreSQL applies any `cloudsync----.sql` upgrade scripts shipped with the release and moves `installed_version` to the new value. + +You can check the current state at any time: + +```sql +SELECT name, default_version, installed_version +FROM pg_available_extensions +WHERE name = 'cloudsync'; +``` + +If `installed_version` is behind `default_version` after a release, run `ALTER EXTENSION cloudsync UPDATE;` to catch up. + +--- + +## Step 3: Register Your Database in the CloudSync Dashboard + +In the [CloudSync dashboard](https://dashboard.sqlitecloud.io/), create a new workspace with the **PostgreSQL** provider, then add a project with your PostgreSQL connection string: + +``` +postgresql://user:password@host:5432/database +``` + +--- + +## Step 4: Enable CloudSync on Tables + +In the dashboard, go to the **Database Setup** tab, select the tables you want to sync, and click **Deploy Changes**. + +--- + +## Step 5: Set Up Authentication + +On the **Client Integration** tab you'll find your **Database ID** and authentication settings. + +### Quick Test with API Key (Recommended for Testing) + +The fastest way to test CloudSync without per-user access control — no JWT setup needed. + +With API key authentication, CloudSync uses the database role resolved from the API-key-authenticated connection when available; otherwise it falls back to the role from the connection string. + +```sql +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_apikey(':'); +SELECT cloudsync_network_sync(); +``` + +### Using JWT Tokens (For RLS and Production) + +1. Set **Row Level Security** to **Yes, enforce RLS** +2. Under **Authentication (JWT)**, click **Configure authentication** and choose: + - **HMAC Secret (HS256):** + - Enter your JWT secret (or generate one: `openssl rand -base64 32`) + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. + - **JWKS Issuer Validation:** + - Enter the issuer base URL from your token's `iss` claim (for example `https://your-auth-domain`) + - By default, CloudSync uses OIDC discovery: it requests `/.well-known/openid-configuration` and reads the returned `jwks_uri` + - Optionally set an **Explicit JWKS URI** to bypass OIDC discovery and use a specific JWKS endpoint directly. This must be a full HTTPS URI. + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. +3. CloudSync validates JWTs as follows: + - **HS256:** uses the configured JWT secret + - **JWKS:** uses the explicit `jwksUri` when provided; otherwise CloudSync requests `/.well-known/openid-configuration` and reads `jwks_uri` + - CloudSync does not fall back directly to `/.well-known/jwks.json` when discovery is used +4. For claim details and RLS examples, see: + - [JWT Claims Reference](../reference/jwt-claims.md) + - [RLS Reference](../reference/rls.md) +5. In your client code: + ```sql + SELECT cloudsync_network_init(''); + SELECT cloudsync_network_set_token(''); + SELECT cloudsync_network_sync(); + ``` diff --git a/docs/postgresql/quickstarts/supabase-self-hosted.md b/docs/postgresql/quickstarts/supabase-self-hosted.md new file mode 100644 index 0000000..8665191 --- /dev/null +++ b/docs/postgresql/quickstarts/supabase-self-hosted.md @@ -0,0 +1,165 @@ +# CloudSync Quick Start: Self-Hosted Supabase + +This guide helps you enable CloudSync on a **fresh or existing** self-hosted Supabase instance. CloudSync adds offline-first synchronization capabilities to your PostgreSQL database. + +## Step 1: Use the CloudSync Supabase Image + +When deploying or updating your Supabase instance, use the published CloudSync Supabase image instead of the standard Supabase Postgres image. + +### For New Deployments + +Follow [Supabase's Installing Supabase](https://supabase.com/docs/guides/self-hosting/docker#installing-supabase) guide to set up the initial files and `.env` configuration. Then, before the first `docker compose up -d`, update your `docker-compose.yml` to use the CloudSync-enabled Postgres image: + +```yaml +db: + # Supabase on PostgreSQL 15 + image: sqlitecloud/sqlite-sync-supabase:15 + # instead of: public.ecr.aws/supabase/postgres:15.8.1.085 + + # OR Supabase on PostgreSQL 17 + image: sqlitecloud/sqlite-sync-supabase:17 + # instead of: public.ecr.aws/supabase/postgres:17.6.1.071 +``` + +Use the CloudSync image tag that matches your Supabase PostgreSQL major version. The published major tags `sqlitecloud/sqlite-sync-supabase:15` and `sqlitecloud/sqlite-sync-supabase:17` are the standard choice. Exact Supabase base-image tags may also be published for some releases, but they are optional and not required for normal setup. + +### Add the CloudSync Init Script + +Create the init SQL: + +```bash +mkdir -p volumes/db +cat > volumes/db/cloudsync.sql << 'EOF' +CREATE EXTENSION IF NOT EXISTS cloudsync; +EOF +``` + +Add a volume mount to the `db` service in `docker-compose.yml`: + +```yaml +services: + db: + volumes: + # ... existing volume mounts ... + - ./volumes/db/cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z +``` + +The `100-` prefix ensures CloudSync loads after Supabase's own init scripts, which are typically numbered `98-99` in the self-hosted Docker Compose setup. + +Then start Supabase: + +```bash +docker compose pull +docker compose up -d +``` + +### For Existing Deployments + +Follow [Supabase's Updating](https://supabase.com/docs/guides/self-hosting/docker#updating) guide. When updating the Postgres image, replace the default image with the matching CloudSync image: + +```bash +# Update docker-compose.yml to use: +# sqlitecloud/sqlite-sync-supabase:15 +# or sqlitecloud/sqlite-sync-supabase:17 +docker compose pull +docker compose down && docker compose up -d +``` + +If Postgres has already been initialized and you are adding CloudSync afterward, the init script will not run automatically. Connect to the database and run: + +```sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +--- + +## Step 2: Verify the Extension + +```bash +docker compose exec db psql -U supabase_admin -d postgres -c "SELECT cloudsync_version();" +``` + +If the extension is installed correctly, PostgreSQL returns the CloudSync version string. + +### Upgrading a later release + +CloudSync uses the first two components of its semver as the PostgreSQL extension version (for example, `1.0.17` installs as extension version `1.0`). How you upgrade depends on which component changed: + +- **PATCH release** (e.g. `1.0.17 → 1.0.18`): pull the matching `sqlitecloud/sqlite-sync-supabase:` image and restart the `db` service. No SQL-level upgrade is needed — `installed_version` stays at `1.0` and the new binary takes over on reconnect. `SELECT cloudsync_version();` confirms the new semver. +- **MINOR or MAJOR release** (e.g. `1.0.x → 1.1.0`): pull the new image and restart as above, then run once per database: + + ```sql + ALTER EXTENSION cloudsync UPDATE; + ``` + + PostgreSQL applies any `cloudsync----.sql` upgrade scripts shipped with the release and moves `installed_version` to the new value. + +You can check the current state at any time: + +```sql +SELECT name, default_version, installed_version +FROM pg_available_extensions +WHERE name = 'cloudsync'; +``` + +If `installed_version` is behind `default_version` after a release, run `ALTER EXTENSION cloudsync UPDATE;` to catch up. + +--- + +## Step 3: Register Your Database in the CloudSync Dashboard + +In the [CloudSync dashboard](https://dashboard.sqlitecloud.io/), create a new workspace with the **Supabase (Self-hosted)** provider, then add a project with your PostgreSQL connection string: + +``` +postgresql://user:password@host:5432/database +``` + +--- + +## Step 4: Enable CloudSync on Tables + +In the dashboard, go to the **Database Setup** tab, select the tables you want to sync, and click **Deploy Changes**. + +--- + +## Step 5: Set Up Authentication + +On the **Client Integration** tab you'll find your **Database ID** and authentication settings. + +### Quick Test with API Key (Recommended for Testing) + +The fastest way to test CloudSync without per-user access control — no JWT setup needed. + +With API key authentication, CloudSync uses the database role resolved from the API-key-authenticated connection when available; otherwise it falls back to the role from the connection string. + +```sql +SELECT cloudsync_network_init(''); +SELECT cloudsync_network_set_apikey(':'); +SELECT cloudsync_network_sync(); +``` + +### Using JWT Tokens (For RLS and Production) + +1. Set **Row Level Security** to **Yes, enforce RLS** +2. Under **Authentication (JWT)**, click **Configure authentication** and choose: + - **HMAC Secret (HS256):** + - Enter your `JWT_SECRET` from Supabase's `.env` + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. + - **JWKS Issuer Validation:** + - Enter the issuer base URL from your token's `iss` claim (for example `https://your-auth-domain`) + - By default, CloudSync uses OIDC discovery: it requests `/.well-known/openid-configuration` and reads the returned `jwks_uri` + - Optionally set an **Explicit JWKS URI** to bypass OIDC discovery and use a specific JWKS endpoint directly. This must be a full HTTPS URI. + - Optionally add **Expected audiences**. When configured, a token's `aud` claim must contain at least one of the configured audience values. +3. CloudSync validates JWTs as follows: + - **HS256:** uses the configured JWT secret + - **JWKS:** uses the explicit `jwksUri` when provided; otherwise CloudSync requests `/.well-known/openid-configuration` and reads `jwks_uri` + - CloudSync does not fall back directly to `/.well-known/jwks.json` when discovery is used +4. For claim details and RLS examples, see: + - [JWT Claims Reference](../reference/jwt-claims.md) + - [RLS Reference](../reference/rls.md) +5. In your client code: + ```sql + SELECT cloudsync_network_init(''); + SELECT cloudsync_network_set_token(''); + SELECT cloudsync_network_sync(); + ``` diff --git a/docs/postgresql/reference/jwt-claims.md b/docs/postgresql/reference/jwt-claims.md new file mode 100644 index 0000000..09635fd --- /dev/null +++ b/docs/postgresql/reference/jwt-claims.md @@ -0,0 +1,246 @@ +# JWT Claims Reference + +## HS256 Claims + +Use this mode when CloudSync validates JWTs with `jwtSecret`. + +| Claim | Required? | Notes | +| ------- | ---------- | ------------------------------------------------------------------------------------------------- | +| `sub` | ⚠️ Depends | Not required by CloudSync itself, but commonly used by application-specific RLS policies | +| `email` | ❌ No | Optional app-specific claim; not validated by CloudSync | +| `role` | ✅ Yes | Required for PostgreSQL JWT-authenticated requests because CloudSync uses it for `SET LOCAL ROLE` | +| `iss` | ❌ No | Optional in HS256 mode | +| `aud` | ⚠️ Depends | Required only when `jwtExpectedAudiences` is configured | +| `iat` | ❌ No | Optional issued-at timestamp; not validated by CloudSync | +| `exp` | ✅ Yes | Required and validated by CloudSync | + +## JWKS Claims + +Use this mode when CloudSync validates JWTs with `jwtAllowedIssuers` and optional `jwksUri`. + +| Claim | Required? | Notes | +| -------- | ---------- | ------------------------------------------------------------------------------------------------- | +| `sub` | ⚠️ Depends | Not required by CloudSync itself, but commonly used by application-specific RLS policies | +| `email` | ❌ No | Optional app-specific claim; not validated by CloudSync | +| `role` | ✅ Yes | Required for PostgreSQL JWT-authenticated requests because CloudSync uses it for `SET LOCAL ROLE` | +| `iss` | ✅ Yes | Required for JWKS / issuer-based validation | +| `aud` | ⚠️ Depends | Required only when `jwtExpectedAudiences` is configured | +| `iat` | ❌ No | Optional issued-at timestamp; not validated by CloudSync | +| `exp` | ✅ Yes | Required and validated by CloudSync | +| Header `kid` | ✅ Yes | Required in the JWT header so CloudSync can select the verification key from the JWKS | + +## Custom Claims Examples + +| Claim | Use Case | +| --------------- | -------------------------- | +| `org_id` | Multi-tenant apps | +| `team_id` | Team-based access | +| `permissions` | Fine-grained access | +| `scope` | OAuth scopes | +| `department_id` | Department-based filtering | +| `is_admin` | Admin flag | + +--- + +## How RLS Works with JWT Claims + +**Flow:** + +``` +1. Client sends JWT token to CloudSync +2. CloudSync validates JWT and extracts claims +3. CloudSync passes claims to PostgreSQL as session variables +4. PostgreSQL policies can read session variables via current_setting() +5. Policies filter data based on claims +6. Only authorized rows returned to client +``` + +## PostgreSQL Role Requirement + +For PostgreSQL JWT authentication, the `role` claim must name a real database role that CloudSync can switch into with `SET LOCAL ROLE`. + +That role should: + +- already exist in PostgreSQL +- have the schema, table, and sequence privileges your sync operations need (see [Required Grants](#required-grants)) +- be grantable by the connection-string user + +If the JWT contains a `role` that does not exist, or the connection user cannot switch into it, PostgreSQL sync operations will fail even if the JWT itself is otherwise valid. + +### Creating the Role + +A typical setup uses a `NOLOGIN` role that your connection user enters via `SET LOCAL ROLE` after JWT verification: + +```sql +CREATE ROLE rls_role NOLOGIN; + +-- Allow the connection-string user (e.g. `postgres`) to switch into it +GRANT rls_role TO postgres; +``` + +### Required Grants + +`cloudsync_payload_apply` running as a non-superuser touches several internal CloudSync objects during apply — not just your user table. If any grant is missing on an internal object, the per-PK savepoint silently rolls back the write and the caller sees a non-zero column-change count with no rows landing (see [RLS Troubleshooting](./rls.md#apply-reports-a-count-but-rows-are-missing)). + +There are two equivalent ways to configure this: the **recommended default-privileges pattern** (future-proof) or the **explicit minimum grant set** (tighter, for audited deployments). + +#### Recommended: default-privileges pattern + +Run this **before** `CREATE EXTENSION cloudsync`, as the role that will install the extension (typically `postgres`). Objects created afterwards — including all CloudSync internal tables and future `cloudsync_init` shadows — inherit the grants automatically: + +```sql +GRANT USAGE ON SCHEMA public TO rls_role; +GRANT USAGE ON SCHEMA auth TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER + ON TABLES TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO rls_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT EXECUTE ON FUNCTIONS TO rls_role; + +CREATE EXTENSION IF NOT EXISTS cloudsync; +``` + +**If the extension is already installed**, `ALTER DEFAULT PRIVILEGES` doesn't apply retroactively — backfill existing objects with a one-time broad grant, then still set defaults for future creations: + +```sql +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO rls_role; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO rls_role; +-- (plus the ALTER DEFAULT PRIVILEGES block above) +``` + +#### Explicit minimum grant set + +For audited deployments that need an explicit allowlist, the tightest set that allows `cloudsync_payload_apply` to work under a non-superuser: + +```sql +GRANT USAGE ON SCHEMA public TO rls_role; +GRANT USAGE ON SCHEMA auth TO rls_role; + +-- User table (RLS policies filter rows within these grants) +GRANT SELECT, INSERT, UPDATE, DELETE ON your_table TO rls_role; + +-- Per-table CRDT shadow (created by cloudsync_init) +GRANT SELECT, INSERT, UPDATE, DELETE ON your_table_cloudsync TO rls_role; + +-- CloudSync metadata tables +GRANT SELECT, INSERT, UPDATE, DELETE ON + cloudsync_settings, + cloudsync_table_settings, + cloudsync_site_id, + cloudsync_schema_versions, + app_schema_version +TO rls_role; + +-- cloudsync_changes view: SELECT for apply-path readback, INSERT for the +-- INSTEAD OF trigger that feeds column changes into the flush buffer +GRANT SELECT, INSERT ON cloudsync_changes TO rls_role; + +-- BIGSERIAL-backed sequence on cloudsync_site_id.id (nextval needs USAGE) +GRANT USAGE ON SEQUENCE cloudsync_site_id_id_seq TO rls_role; + +-- Your user table's sequence, if it uses SERIAL / IDENTITY +-- GRANT USAGE, SELECT ON SEQUENCE your_table_id_seq TO rls_role; +``` + +Notes on the minimum set: + +- **No `EXECUTE` grants on `cloudsync_*` functions or `auth.uid()` are required**, because PostgreSQL defaults `CREATE FUNCTION` to `EXECUTE TO PUBLIC`. If your cluster has revoked PUBLIC execute, grant `EXECUTE` explicitly on `cloudsync_payload_apply`, `cloudsync_payload_encode`, `cloudsync_changes_select`, `cloudsync_changes_insert_trigger`, `cloudsync_siteid`, `cloudsync_pk_encode`, and `cloudsync_encode_value`. +- **`app_schema_version` is not `cloudsync_*`-prefixed** — easy to miss in `cloudsync_%`-pattern grants. +- **Per-table shadows follow the `
_cloudsync` convention** — repeat the DML grant for every table passed to `cloudsync_init`. +- **Administrative functions** such as `cloudsync_init`, `cloudsync_enable`, `cloudsync_set*`, `cloudsync_terminate`, `cloudsync_cleanup`, `cloudsync_begin_alter`, and `cloudsync_commit_alter` should be run by the database owner during setup, not by client JWT roles. +- The minimum set will need widening if a future CloudSync version adds new internal objects. The default-privileges pattern above is future-proof. + +### Service Role (RLS Bypass) + +For server-side workers that need to apply payloads without RLS enforcement (admin restores, cross-user sync, maintenance jobs), create a dedicated role with `BYPASSRLS`: + +```sql +CREATE ROLE service_role NOLOGIN BYPASSRLS; +GRANT service_role TO postgres; +``` + +Apply the same grants as for `rls_role`. Use this role only from trusted server code, never from JWT-gated request paths. + +--- + +## How CloudSync Passes JWT Claims to PostgreSQL + +**For PostgreSQL JWT-authenticated requests, CloudSync validates the JWT and passes all claims to PostgreSQL as a session variable:** + +```go +// CloudSync (internal implementation) +userData := token.Claims // map[string]any with all JWT claims +claimJSON, _ := json.Marshal(userData) + +// Pass all claims as JSON to PostgreSQL session +db.Exec( + `SELECT set_config('request.jwt.claims', $1, true)`, + string(claimJSON) +) +``` + +**Result:** All JWT claims are available in PostgreSQL as JSON in `request.jwt.claims`, and CloudSync also sets `SET LOCAL ROLE` from the JWT `role` claim. + +**Example:** If JWT contains: + +```json +{ + "sub": "550e8400-e29b-41d4-a716-446655440000", + "email": "user@example.com", + "role": "authenticated", + "org_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" +} +``` + +Then in PostgreSQL: + +```sql +-- Returns: {"sub":"550e8400...","email":"user@example.com","role":"authenticated","org_id":"aaaaaaaa..."} +current_setting('request.jwt.claims') + +-- Access any claim from the JSON +user_id = (current_setting('request.jwt.claims')::jsonb->>'sub')::uuid +email = (current_setting('request.jwt.claims')::jsonb->>'email') +role = (current_setting('request.jwt.claims')::jsonb->>'role') +org_id = (current_setting('request.jwt.claims')::jsonb->>'org_id')::uuid +``` + +--- + +## Optional: Helper Functions for JWT Claims + +CloudSync validates JWTs and passes all claims to PostgreSQL via `request.jwt.claims` — no PostgreSQL extension is required for JWT verification. The validation happens entirely in the CloudSync microservice. + +However, writing `(current_setting('request.jwt.claims')::jsonb->>'sub')::uuid` in every RLS policy is verbose. Following the pattern used by Supabase and Neon, you can optionally create a small set of helper functions in a dedicated schema: + +```sql +-- Create a schema for auth helpers (optional, but keeps things clean) +CREATE SCHEMA IF NOT EXISTS auth; + +-- Returns all JWT claims as JSONB +CREATE OR REPLACE FUNCTION auth.session() + RETURNS jsonb AS $$ + SELECT current_setting('request.jwt.claims', true)::jsonb; +$$ LANGUAGE SQL STABLE; + +-- Returns the user ID (sub claim) +CREATE OR REPLACE FUNCTION auth.user_id() + RETURNS text AS $$ + SELECT auth.session()->>'sub'; +$$ LANGUAGE SQL STABLE; + +-- Returns the user's role claim +CREATE OR REPLACE FUNCTION auth.role() + RETURNS text AS $$ + SELECT auth.session()->>'role'; +$$ LANGUAGE SQL STABLE; +``` + +> **Note:** These are just convenience wrappers — they read from the same `request.jwt.claims` session variable that CloudSync sets. + +--- diff --git a/docs/postgresql/reference/rls.md b/docs/postgresql/reference/rls.md new file mode 100644 index 0000000..26e4bd5 --- /dev/null +++ b/docs/postgresql/reference/rls.md @@ -0,0 +1,202 @@ +# Row Level Security (RLS) with CloudSync + +CloudSync is fully compatible with PostgreSQL Row Level Security. Standard RLS policies work out of the box. + +## How It Works + +### Column-batch merge + +CloudSync resolves CRDT conflicts at the column level — a sync payload may contain individual column changes arriving one at a time. Before writing to the target table, CloudSync buffers all winning column values for the same primary key and flushes them as a single SQL statement. This ensures the database sees a complete row with all columns present. + +### UPDATE vs INSERT selection + +When flushing a batch, CloudSync chooses the statement type based on whether the row already exists locally: + +- **New row**: `INSERT ... ON CONFLICT DO UPDATE` — all columns are present (including the ownership column), so the INSERT `WITH CHECK` policy can evaluate correctly. +- **Existing row**: `UPDATE ... SET ... WHERE pk = ...` — only the changed columns are set. The UPDATE `USING` policy checks the existing row, which already has the correct ownership column value. + +### Per-PK savepoint isolation + +Each primary key's flush is wrapped in its own savepoint. When RLS denies a write: + +1. The database raises an error inside the savepoint +2. CloudSync rolls back that savepoint, releasing all resources acquired during the failed statement +3. Processing continues with the next primary key + +This means a single payload can contain a mix of allowed and denied rows — allowed rows commit normally, denied rows are silently skipped. The caller receives the total number of column changes processed (including denied ones) rather than an error. + +## Quick Setup + +Given a table with an ownership column (`user_id`): + +```sql +CREATE TABLE documents ( + id TEXT PRIMARY KEY, + user_id UUID, + title TEXT, + content TEXT +); + +SELECT cloudsync_init('documents'); +``` + +Enable RLS and create standard policies: + +```sql +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON documents FOR SELECT + USING (auth.uid() = user_id); + +CREATE POLICY "insert_own" ON documents FOR INSERT + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "update_own" ON documents FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "delete_own" ON documents FOR DELETE + USING (auth.uid() = user_id); +``` + +When you authenticate PostgreSQL requests with JWTs, CloudSync also executes `SET LOCAL ROLE` using the JWT `role` claim. That means the role named in the token must already exist in PostgreSQL and must have the permissions needed to read and write the synced tables. See [PostgreSQL Role Requirement](./jwt-claims.md#postgresql-role-requirement). + +## Example: Two-User Sync with RLS + +This example shows the complete flow of syncing data between two databases where the target enforces RLS. + +### Setup + +```sql +-- Source database (DB A) — no RLS, represents the sync server +CREATE TABLE documents ( + id TEXT PRIMARY KEY, user_id UUID, title TEXT, content TEXT +); +SELECT cloudsync_init('documents'); + +-- Target database (DB B) — RLS enforced +CREATE TABLE documents ( + id TEXT PRIMARY KEY, user_id UUID, title TEXT, content TEXT +); +SELECT cloudsync_init('documents'); +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +-- (policies as above) +``` + +### Insert sync + +User 1 creates a document on DB A: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc1', 'user1-uuid', 'Hello', 'World'); +``` + +Apply the payload on DB B as the authenticated user: + +```sql +-- On DB B (running as user1) +SET app.current_user_id = 'user1-uuid'; +SET ROLE authenticated; +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +``` + +The insert succeeds because `user_id` matches `auth.uid()`. + +### Insert denial + +User 1 tries to sync a document owned by user 2: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc2', 'user2-uuid', 'Secret', 'Data'); +``` + +```sql +-- On DB B (running as user1) +SET app.current_user_id = 'user1-uuid'; +SET ROLE authenticated; +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +``` + +The insert is denied by RLS. The row does not appear in DB B. No error is raised to the caller — CloudSync isolates the failure via a per-PK savepoint and continues processing the remaining payload. + +### Partial update sync + +User 1 updates only the title of their own document: + +```sql +-- On DB A +UPDATE documents SET title = 'Hello Updated' WHERE id = 'doc1'; +``` + +The sync payload contains only the changed column (`title`). CloudSync detects that the row already exists on DB B and uses a plain `UPDATE` statement: + +```sql +UPDATE documents SET title = $2 WHERE id = $1; +``` + +The UPDATE policy checks the existing row (which has the correct `user_id`), so it succeeds. + +### Mixed payload + +When a single payload contains rows for multiple users, CloudSync handles each primary key independently: + +```sql +-- On DB A +INSERT INTO documents VALUES ('doc3', 'user1-uuid', 'Mine', '...'); +INSERT INTO documents VALUES ('doc4', 'user2-uuid', 'Theirs', '...'); +``` + +```sql +-- On DB B (running as user1) +SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); +-- doc3 is inserted (allowed), doc4 is silently skipped (denied) +``` + +## Supabase Notes + +When using Supabase: + +1. **auth.uid()**: Returns the authenticated user's UUID from the JWT claims. +2. **JWT propagation**: Ensure the JWT token is set before sync operations: + ```sql + SELECT set_config('request.jwt.claims', '{"sub": "user-uuid", ...}', true); + ``` +3. **Service role bypass**: The Supabase service role bypasses RLS entirely. Use the `authenticated` role for user-context operations where RLS enforcement is desired. + +## Troubleshooting + +### "new row violates row-level security policy" + +**Symptom**: Insert operations fail during sync. + +**Cause**: The ownership column value doesn't match the authenticated user. + +**Solution**: Verify that: +- The JWT / session variable is set correctly before calling `cloudsync_payload_apply` +- The `user_id` column in the synced data matches `auth.uid()` +- RLS policies reference the correct ownership column + +### Apply reports a count, but rows are missing + +**Symptom**: `cloudsync_payload_apply` returns a non-zero column-change count, but `SELECT` on the target table shows no new rows. No error is raised to the caller. + +**Cause**: The calling role is missing a grant on one of CloudSync's internal objects — the per-table shadow (`
_cloudsync`), a metadata table (`cloudsync_settings`, `cloudsync_site_id`, `cloudsync_table_settings`, `cloudsync_schema_versions`, `app_schema_version`), the `cloudsync_changes` view, or the `cloudsync_site_id_id_seq` sequence. The per-PK savepoint rolls the write back, but `cloudsync_payload_apply` still returns the number of column changes it processed. + +**Solution**: Apply the full grant set from [JWT Claims → Required Grants](./jwt-claims.md#required-grants). To pinpoint which object is missing, re-run the apply as a superuser or raise log verbosity and inspect the server log for `permission denied` entries preceded by the `cloudsync_payload_apply` call. + +### Debugging + +```sql +-- Check current auth context +SELECT auth.uid(); + +-- Inspect a specific row's ownership +SELECT id, user_id FROM documents WHERE id = 'problematic-pk'; + +-- Temporarily disable RLS to inspect all data +ALTER TABLE documents DISABLE ROW LEVEL SECURITY; +-- ... inspect ... +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +``` diff --git a/examples/simple-todo-db/README.md b/examples/simple-todo-db/README.md index c9967a5..56f4d8f 100644 --- a/examples/simple-todo-db/README.md +++ b/examples/simple-todo-db/README.md @@ -20,11 +20,15 @@ Before using the local CLI, you need to set up your cloud database: 2. Name your database (e.g., "todo_app.sqlite") 3. Click **"Create"** -### 1.3 Get Connection Details -1. Copy the **Connection String** (format: `sqlitecloud://projectid.sqlite.cloud/database.sqlite`) +### 1.3 Enable OffSync +1. Click the **OffSync** button next to your database, then **Enable OffSync** and confirm with the **Enable** button +2. In the **Configuration** tab copy the **Database ID** (format: `db_*`) + +### 1.4 Get Auth Details +1. In your project dashboard, click **Settings**, then **API Keys** 2. Copy an **API Key** -### 1.4 Configure Row-Level Security (Optional) +### 1.5 Configure Row-Level Security (Optional) 1. In your database dashboard, go to **"Security"** → **"Row-Level Security"** 2. Enable RLS for tables you want to secure 3. Create policies to control user access (e.g., users can only see their own tasks) @@ -59,7 +63,7 @@ Tables must be created on both the local database and SQLite Cloud with identica -- Create the main tasks table -- Note: Primary key MUST be TEXT (not INTEGER) for global uniqueness CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -84,7 +88,7 @@ SELECT cloudsync_is_enabled('tasks'); - Execute the same CREATE TABLE statement: ```sql CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -104,11 +108,11 @@ SELECT cloudsync_is_enabled('tasks'); ```sql -- Configure connection to SQLite Cloud --- Replace with your actual connection string from Step 1.3 -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/todo_app.sqlite'); +-- Replace with your managedDatabaseId from the OffSync page on the SQLiteCloud dashboard from Step 1.3 +SELECT cloudsync_network_init('your-managed-database-id'); -- Configure authentication: --- Set your API key from Step 1.3 +-- Set your API key from Step 1.4 SELECT cloudsync_network_set_apikey('your-api-key-here'); -- Or use token authentication (required for Row-Level Security) -- SELECT cloudsync_network_set_token('your_auth_token'); @@ -149,7 +153,7 @@ sqlite3 todo_device_b.db ```sql -- Create identical table structure CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY NOT NULL, + id TEXT PRIMARY KEY, userid TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', description TEXT DEFAULT '', @@ -163,12 +167,12 @@ CREATE TABLE IF NOT EXISTS tasks ( SELECT cloudsync_init('tasks'); -- Connect to the same cloud database -SELECT cloudsync_network_init('sqlitecloud://your-project-id.sqlite.cloud/todo_app.sqlite'); +SELECT cloudsync_network_init('your-managed-database-id'); SELECT cloudsync_network_set_apikey('your-api-key-here'); -- Pull data from Device A - repeat until data is received SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received) +-- Check "receive.rows" in the JSON result to see if data was received SELECT cloudsync_network_sync(); -- Verify data was synced @@ -199,7 +203,7 @@ SELECT cloudsync_network_sync(); ```sql -- Get updates from Device B - repeat until data is received SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received) +-- Check "receive.rows" in the JSON result to see if data was received SELECT cloudsync_network_sync(); -- View all tasks (should now include Device B's additions) @@ -232,7 +236,7 @@ SELECT cloudsync_network_has_unsent_changes(); -- When network returns, sync automatically resolves conflicts -- Repeat until all changes are synchronized SELECT cloudsync_network_sync(); --- Keep calling until the function returns > 0 (indicating data was received/sent) +-- Check "receive.rows" and "send.status" in the JSON result SELECT cloudsync_network_sync(); ``` diff --git a/examples/sport-tracker-app/.env.example b/examples/sport-tracker-app/.env.example index ce5c7cc..c534674 100644 --- a/examples/sport-tracker-app/.env.example +++ b/examples/sport-tracker-app/.env.example @@ -1,6 +1,5 @@ -# Copy from from the SQLite Cloud Dashboard -# eg: sqlitecloud://myhost.cloud:8860/my-remote-database.sqlite -VITE_SQLITECLOUD_CONNECTION_STRING= +# Copy the managedDatabaseId from the OffSync page on the SQLiteCloud Dashboard +VITE_SQLITECLOUD_MANAGED_DATABASE_ID= # The database name # eg: my-remote-database.sqlite VITE_SQLITECLOUD_DATABASE= diff --git a/examples/sport-tracker-app/package.json b/examples/sport-tracker-app/package.json index f1a6da7..3615f8c 100644 --- a/examples/sport-tracker-app/package.json +++ b/examples/sport-tracker-app/package.json @@ -17,6 +17,7 @@ "@types/react": "^19.1.8", "@types/react-dom": "^19.1.6", "@vitejs/plugin-react": "^4.6.0", + "pg": "^8.17.2", "react": "^19.1.0", "react-dom": "^19.1.0" } diff --git a/examples/sport-tracker-app/sport-tracker-schema-postgres.sql b/examples/sport-tracker-app/sport-tracker-schema-postgres.sql new file mode 100644 index 0000000..1288f16 --- /dev/null +++ b/examples/sport-tracker-app/sport-tracker-schema-postgres.sql @@ -0,0 +1,29 @@ +-- PostgreSQL schema +-- Use this schema to create the remote database on PostgreSQL/PostgREST + +CREATE TABLE IF NOT EXISTS users_sport ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + name TEXT UNIQUE NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users_sport (id) +); + +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); diff --git a/examples/sport-tracker-app/sport-tracker-schema.sql b/examples/sport-tracker-app/sport-tracker-schema.sql index 9e8ce1a..8877730 100644 --- a/examples/sport-tracker-app/sport-tracker-schema.sql +++ b/examples/sport-tracker-app/sport-tracker-schema.sql @@ -1,7 +1,7 @@ -- SQL schema -- Use this exact schema to create the remote database on the on SQLite Cloud -CREATE TABLE IF NOT EXISTS users ( +CREATE TABLE IF NOT EXISTS users_sport ( id TEXT PRIMARY KEY NOT NULL, -- UUID's HIGHLY RECOMMENDED for global uniqueness name TEXT UNIQUE NOT NULL DEFAULT '' ); @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS activities ( date TEXT, notes TEXT, user_id TEXT, - FOREIGN KEY (user_id) REFERENCES users (id) + FOREIGN KEY (user_id) REFERENCES users_sport (id) ); CREATE TABLE IF NOT EXISTS workouts ( diff --git a/examples/sport-tracker-app/src/SQLiteSync.ts b/examples/sport-tracker-app/src/SQLiteSync.ts index 3140b25..0b639d6 100644 --- a/examples/sport-tracker-app/src/SQLiteSync.ts +++ b/examples/sport-tracker-app/src/SQLiteSync.ts @@ -111,21 +111,23 @@ export class SQLiteSync { const now = new Date(); if (!token) { - console.log("SQLite Sync: No token available, requesting new one from API"); - const tokenData = await this.fetchNewToken(userId, name); - localStorage.setItem( - SQLiteSync.TOKEN_KEY_PREFIX, - JSON.stringify(tokenData) + console.log( + "SQLite Sync: No token available, requesting new one from API", ); + const tokenData = await this.fetchNewToken(userId, name); + // localStorage.setItem( + // SQLiteSync.TOKEN_KEY_PREFIX, + // JSON.stringify(tokenData), + // ); token = tokenData.token; console.log("SQLite Sync: New token obtained and stored in localStorage"); } else if (tokenExpiry && tokenExpiry <= now) { console.warn("SQLite Sync: Token expired, requesting new one from API"); const tokenData = await this.fetchNewToken(userId, name); - localStorage.setItem( - SQLiteSync.TOKEN_KEY_PREFIX, - JSON.stringify(tokenData) - ); + // localStorage.setItem( + // SQLiteSync.TOKEN_KEY_PREFIX, + // JSON.stringify(tokenData), + // ); token = tokenData.token; console.log("SQLite Sync: New token obtained and stored in localStorage"); } else { @@ -143,69 +145,18 @@ export class SQLiteSync { */ private async fetchNewToken( userId: string, - name: string + name: string, ): Promise> { - const response = await fetch( - `${import.meta.env.VITE_SQLITECLOUD_API_URL}/v2/tokens`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY}`, - }, - body: JSON.stringify({ - userId, - name, - expiresAt: new Date( - Date.now() + SQLiteSync.TOKEN_EXPIRY_MINUTES * 60 * 1000 - ).toISOString(), - }), - } - ); - - if (!response.ok) { - throw new Error(`Failed to get token: ${response.status}`); - } - - const result = await response.json(); - return result.data; + const jwt = await Promise.resolve(import.meta.env.VITE_SQLITECLOUD_API_KEY); + return { + token: jwt + }; } /** * Checks if a valid token exists in localStorage */ static hasValidToken(): boolean { - const storedTokenData = localStorage.getItem(SQLiteSync.TOKEN_KEY_PREFIX); - - if (!storedTokenData) { - console.log("SQLite Sync: No token data found in localStorage"); - return false; - } - - try { - const parsed: TokenData = JSON.parse(storedTokenData); - - // Check if token exists - if (!parsed.token) { - console.log("SQLite Sync: Token data exists but no token found"); - return false; - } - - // Check if token is expired - if (parsed.expiresAt) { - const tokenExpiry = new Date(parsed.expiresAt); - const now = new Date(); - if (tokenExpiry <= now) { - console.log("SQLite Sync: Token found but expired"); - return false; - } - } - - console.log("SQLite Sync: Valid token found in localStorage"); - return true; - } catch (e) { - console.error("SQLite Sync: Failed to parse stored token:", e); - return false; - } + return false; } } diff --git a/examples/sport-tracker-app/src/components/UserCreation.tsx b/examples/sport-tracker-app/src/components/UserCreation.tsx index 05defb1..fd90b15 100644 --- a/examples/sport-tracker-app/src/components/UserCreation.tsx +++ b/examples/sport-tracker-app/src/components/UserCreation.tsx @@ -15,26 +15,22 @@ interface UserCreationProps { */ const fetchRemoteUsers = async (): Promise => { const response = await fetch( - `${import.meta.env.VITE_SQLITECLOUD_API_URL}/v2/weblite/sql`, + `${import.meta.env.VITE_SQLITECLOUD_API_URL}/rest/v1/users_sport?select=id,name`, { - method: "POST", + method: "GET", headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY}`, + Accept: "application/json", + Authorization: `Bearer ${import.meta.env.VITE_SQLITECLOUD_API_KEY || ""}`, }, - body: JSON.stringify({ - sql: "SELECT id, name FROM users;", - database: import.meta.env.VITE_SQLITECLOUD_DATABASE || "", - }), } ); - + if (!response.ok) { throw new Error(`Failed to fetch users: ${response.status}`); } const result = await response.json(); - return result.data; + return result as User[]; }; const UserCreation: React.FC = ({ @@ -58,6 +54,7 @@ const UserCreation: React.FC = ({ setRemoteUsers(users); } catch (error) { console.error("Failed to load remote users:", error); + alert("Failed to load remote users. Error: " + error); } finally { setIsLoadingRemoteUsers(false); } diff --git a/examples/sport-tracker-app/src/db/databaseOperations.ts b/examples/sport-tracker-app/src/db/databaseOperations.ts index 5bae19a..2413d27 100644 --- a/examples/sport-tracker-app/src/db/databaseOperations.ts +++ b/examples/sport-tracker-app/src/db/databaseOperations.ts @@ -216,7 +216,7 @@ export const getDatabaseOperations = (db: any) => ({ try { db.exec({ - sql: "INSERT INTO users (id, name) VALUES (?, ?)", + sql: "INSERT INTO users_sport (id, name) VALUES (?, ?)", bind: [userId, name], }); return { id: userId, name }; @@ -228,7 +228,7 @@ export const getDatabaseOperations = (db: any) => ({ getUsers() { const users: any[] = []; db.exec({ - sql: "SELECT id, name FROM users ORDER BY name", + sql: "SELECT id, name FROM users_sport ORDER BY name", callback: (row: any) => { users.push({ id: row[0], @@ -243,7 +243,7 @@ export const getDatabaseOperations = (db: any) => ({ const { id } = data; let user = null; db.exec({ - sql: "SELECT id, name FROM users WHERE id = ?", + sql: "SELECT id, name FROM users_sport WHERE id = ?", bind: [id], callback: (row: any) => { user = { @@ -268,7 +268,7 @@ export const getDatabaseOperations = (db: any) => ({ // Get total counts (always show total for comparison) db.exec({ - sql: "SELECT COUNT(*) FROM users WHERE name != ?", + sql: "SELECT COUNT(*) FROM users_sport WHERE name != ?", bind: ["coach"], callback: (row: any) => (counts.totalUsers = row[0]), }); @@ -287,7 +287,7 @@ export const getDatabaseOperations = (db: any) => ({ if (user_id && !is_coach) { // Regular user - count only their data db.exec({ - sql: "SELECT COUNT(*) FROM users WHERE name != ?", + sql: "SELECT COUNT(*) FROM users_sport WHERE name != ?", bind: ["coach"], callback: (row: any) => (counts.users = row[0]), }); diff --git a/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts b/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts index 97329db..79fe440 100644 --- a/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts +++ b/examples/sport-tracker-app/src/db/sqliteSyncOperations.ts @@ -26,7 +26,7 @@ export const getSqliteSyncOperations = (db: any) => ({ * This will push changes to the cloud and check for changes from the cloud. * The first attempt may not find anything to apply, but subsequent attempts * will find changes if they exist. - */ + */ sqliteSyncNetworkSync() { console.log("SQLite Sync - Starting sync..."); db.exec("SELECT cloudsync_network_sync(1000, 2);"); @@ -38,7 +38,7 @@ export const getSqliteSyncOperations = (db: any) => ({ */ sqliteSyncSendChanges() { console.log( - "SQLite Sync - Sending changes to your the SQLite Cloud node..." + "SQLite Sync - Sending changes to your the SQLite Cloud node...", ); db.exec("SELECT cloudsync_network_send_changes();"); console.log("SQLite Sync - Changes sent"); @@ -84,18 +84,18 @@ export const initSQLiteSync = (db: any) => { } // Initialize SQLite Sync - db.exec(`SELECT cloudsync_init('users');`); + db.exec(`SELECT cloudsync_init('users_sport');`); db.exec(`SELECT cloudsync_init('activities');`); db.exec(`SELECT cloudsync_init('workouts');`); // ...or initialize all tables at once // db.exec('SELECT cloudsync_init("*");'); - // Initialize SQLite Sync with the SQLite Cloud Connection String. - // On the SQLite Cloud Dashboard, enable OffSync (SQLite Sync) - // on the remote database and copy the Connection String. + // Initialize SQLite Sync with the managedDatabaseId. + // On the SQLite Cloud Dashboard, enable OffSync (SQLite Sync) + // on the remote database and copy the managedDatabaseId. db.exec( `SELECT cloudsync_network_init('${ - import.meta.env.VITE_SQLITECLOUD_CONNECTION_STRING + import.meta.env.VITE_SQLITECLOUD_MANAGED_DATABASE_ID }')` ); }; diff --git a/examples/to-do-app/.env.example b/examples/to-do-app/.env.example index 4b2816d..ba99068 100644 --- a/examples/to-do-app/.env.example +++ b/examples/to-do-app/.env.example @@ -1,3 +1,3 @@ -# Copy from the SQLite Cloud Dashboard -# eg: sqlitecloud://myhost.cloud:8860/my-remote-database.sqlite?apikey=myapikey -CONNECTION_STRING = "" \ No newline at end of file +# Copy from the OffSync page on the SQLiteCloud Dashboard +MANAGED_DATABASE_ID = "" +API_TOKEN = diff --git a/examples/to-do-app/README.md b/examples/to-do-app/README.md index bc77123..0e3d0c0 100644 --- a/examples/to-do-app/README.md +++ b/examples/to-do-app/README.md @@ -24,10 +24,10 @@ cd MyApp Rename the `.env.example` into `.env` and fill with your values. -> **⚠️ SECURITY WARNING**: This example puts database connection strings directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** +> **⚠️ SECURITY WARNING**: This example puts database API Keys directly in `.env` files for demonstration purposes only. **Do not use this pattern in production.** > > **Why this is unsafe:** -> - Connection strings contain sensitive credentials +> - API Keys allow access to sensitive credentials > - Client-side apps expose all environment variables to users > - Anyone can inspect your app and extract database credentials > diff --git a/examples/to-do-app/components/SyncContext.js b/examples/to-do-app/components/SyncContext.js index e964f4a..7b076ef 100644 --- a/examples/to-do-app/components/SyncContext.js +++ b/examples/to-do-app/components/SyncContext.js @@ -58,10 +58,14 @@ export const SyncProvider = ({ children }) => { const result = await Promise.race([queryPromise, timeoutPromise]); - if (result.rows && result.rows.length > 0 && result.rows[0]['cloudsync_network_check_changes()'] > 0) { - console.log(`${result.rows[0]['cloudsync_network_check_changes()']} changes detected, triggering refresh`); - // Defer refresh to next tick to avoid blocking current interaction - setTimeout(() => triggerRefresh(), 0); + const raw = result.rows?.[0]?.['cloudsync_network_check_changes()']; + if (raw) { + const { receive } = JSON.parse(raw); + if (receive.rows > 0) { + console.log(`${receive.rows} changes detected in [${receive.tables}], triggering refresh`); + // Defer refresh to next tick to avoid blocking current interaction + setTimeout(() => triggerRefresh(), 0); + } } } catch (error) { console.error('Error checking for changes:', error); diff --git a/examples/to-do-app/components/TaskRow.js b/examples/to-do-app/components/TaskRow.js index ff8b3cf..654993d 100644 --- a/examples/to-do-app/components/TaskRow.js +++ b/examples/to-do-app/components/TaskRow.js @@ -1,6 +1,6 @@ import { useState, useRef, useEffect } from "react"; import { View, Text, StyleSheet, TouchableOpacity } from "react-native"; -import Icon from "react-native-vector-icons/FontAwesome"; +import Icon from "@react-native-vector-icons/fontawesome"; import { Swipeable } from "react-native-gesture-handler"; export default TaskRow = ({ task, updateTask, handleDelete }) => { diff --git a/examples/to-do-app/hooks/useCategories.js b/examples/to-do-app/hooks/useCategories.js index f46215c..519f7ec 100644 --- a/examples/to-do-app/hooks/useCategories.js +++ b/examples/to-do-app/hooks/useCategories.js @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react' import { Platform } from 'react-native'; import { db } from "../db/dbConnection"; -import { CONNECTION_STRING } from "@env"; +import { MANAGED_DATABASE_ID, API_TOKEN } from "@env"; import { getDylibPath } from "@op-engineering/op-sqlite"; import { randomUUID } from 'expo-crypto'; import { useSyncContext } from '../components/SyncContext'; @@ -65,14 +65,18 @@ const useCategories = () => { await db.execute('CREATE TABLE IF NOT EXISTS tags (uuid TEXT NOT NULL PRIMARY KEY, name TEXT, UNIQUE(name));') await db.execute('CREATE TABLE IF NOT EXISTS tasks_tags (uuid TEXT NOT NULL PRIMARY KEY, task_uuid TEXT, tag_uuid TEXT, FOREIGN KEY (task_uuid) REFERENCES tasks(uuid), FOREIGN KEY (tag_uuid) REFERENCES tags(uuid));') - await db.execute(`SELECT cloudsync_init('*');`); - await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', [randomUUID(), 'Work']) - await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', [randomUUID(), 'Personal']) + await db.execute(`SELECT cloudsync_init('tasks');`); + await db.execute(`SELECT cloudsync_init('tags');`); + await db.execute(`SELECT cloudsync_init('tasks_tags');`); + + await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', ['work', 'Work']) + await db.execute('INSERT OR IGNORE INTO tags (uuid, name) VALUES (?, ?)', ['personal', 'Personal']) - if (CONNECTION_STRING && CONNECTION_STRING.startsWith('sqlitecloud://')) { - await db.execute(`SELECT cloudsync_network_init('${CONNECTION_STRING}');`); + if (MANAGED_DATABASE_ID && API_TOKEN) { + await db.execute(`SELECT cloudsync_network_init('${MANAGED_DATABASE_ID}');`); + await db.execute(`SELECT cloudsync_network_set_token('${API_TOKEN}');`) } else { - throw new Error('No valid CONNECTION_STRING provided, cloudsync_network_init will not be called'); + throw new Error('No valid MANAGED_DATABASE_ID or API_TOKEN provided, cloudsync_network_init will not be called'); } db.execute('SELECT cloudsync_network_sync(100, 10);') diff --git a/examples/to-do-app/package.json b/examples/to-do-app/package.json index 82534ff..cb05893 100644 --- a/examples/to-do-app/package.json +++ b/examples/to-do-app/package.json @@ -1,13 +1,22 @@ { "name": "@sqliteai/todoapp", - "version": "1.0.3", + "version": "1.0.7", "description": "An Expo template for building apps with the SQLite CloudSync extension", "repository": { "type": "git", "url": "git+https://github.com/sqliteai/sqlite-sync.git" }, "author": "SQLiteAI", - "keywords": ["expo-template", "sqlite", "cloudsync", "todo", "sync", "react-native", "expo", "template"], + "keywords": [ + "expo-template", + "sqlite", + "cloudsync", + "todo", + "sync", + "react-native", + "expo", + "template" + ], "main": "expo/AppEntry.js", "scripts": { "start": "expo start", @@ -21,19 +30,23 @@ "dependencies": { "@op-engineering/op-sqlite": "14.1.4", "@react-native-picker/picker": "2.11.1", + "@react-native-vector-icons/fontawesome": "^12.4.0", + "@react-native-vector-icons/material-design-icons": "^12.4.0", "@react-navigation/native": "^7.1.17", "@react-navigation/stack": "^7.4.7", "expo": "^53.0.22", + "expo-asset": "^12.0.12", + "expo-constants": "^18.0.13", "expo-crypto": "~14.1.5", "expo-status-bar": "~2.2.3", + "prop-types": "^15.8.1", "react": "19.0.0", - "react-native": "0.79.5", + "react-native": "0.79.6", "react-native-gesture-handler": "~2.24.0", "react-native-paper": "5.14.5", "react-native-picker-select": "^9.3.1", "react-native-safe-area-context": "5.4.0", - "react-native-screens": "~4.11.1", - "react-native-vector-icons": "^10.3.0" + "react-native-screens": "~4.11.1" }, "devDependencies": { "@babel/core": "7.28.3", diff --git a/examples/to-do-app/plugins/CloudSyncSetup.js b/examples/to-do-app/plugins/CloudSyncSetup.js index f483901..3cba189 100644 --- a/examples/to-do-app/plugins/CloudSyncSetup.js +++ b/examples/to-do-app/plugins/CloudSyncSetup.js @@ -1,4 +1,4 @@ -const { withDangerousMod, withXcodeProject } = require('@expo/config-plugins'); +const { withDangerousMod, withXcodeProject, withInfoPlist } = require('@expo/config-plugins'); const fs = require('fs'); const path = require('path'); const https = require('https'); @@ -277,8 +277,21 @@ const withCloudSync = (config) => { // iOS setup - add to Xcode project config = withCloudSyncFramework(config); - + + // iOS setup - register icon fonts in Info.plist + config = withInfoPlist(config, (config) => { + const fonts = config.modResults.UIAppFonts || []; + if (!fonts.includes('FontAwesome.ttf')) { + fonts.push('FontAwesome.ttf'); + } + if (!fonts.includes('MaterialDesignIcons.ttf')) { + fonts.push('MaterialDesignIcons.ttf'); + } + config.modResults.UIAppFonts = fonts; + return config; + }); + return config; }; -module.exports = withCloudSync; \ No newline at end of file +module.exports = withCloudSync; diff --git a/examples/to-do-app/screens/Home.js b/examples/to-do-app/screens/Home.js index ae9c2bf..7e0321a 100644 --- a/examples/to-do-app/screens/Home.js +++ b/examples/to-do-app/screens/Home.js @@ -1,7 +1,7 @@ import { useState } from "react"; import { View, Text, StyleSheet, FlatList, Alert } from "react-native"; import { Button } from "react-native-paper"; -import Icon from "react-native-vector-icons/FontAwesome"; +import Icon from "@react-native-vector-icons/fontawesome"; import TaskRow from "../components/TaskRow"; import AddTaskModal from "../components/AddTaskModal"; import useTasks from "../hooks/useTasks" diff --git a/modules/fractional-indexing b/modules/fractional-indexing new file mode 160000 index 0000000..b9af0ec --- /dev/null +++ b/modules/fractional-indexing @@ -0,0 +1 @@ +Subproject commit b9af0ec5b818bca29919e1a8d42b142feb71f269 diff --git a/packages/expo/README.md b/packages/expo/README.md index ee4c321..0eac7b3 100644 --- a/packages/expo/README.md +++ b/packages/expo/README.md @@ -21,7 +21,7 @@ node generate-expo-package.js Example: ```bash -node generate-expo-package.js 0.8.57 ../../artifacts ./expo-package +node generate-expo-package.js 1.0.0 ../../artifacts ./expo-package cd expo-package && npm publish --provenance --access public ``` @@ -54,7 +54,7 @@ To test the generator locally, you need to set up mock artifacts that simulate w **Option A: Download from latest release** ```bash -VERSION="0.8.57" # or latest version +VERSION="1.0.0" # or latest version mkdir -p artifacts/cloudsync-apple-xcframework mkdir -p artifacts/cloudsync-android-arm64-v8a @@ -100,7 +100,7 @@ cp -r dist/CloudSync.xcframework artifacts/cloudsync-apple-xcframework/ ```bash cd packages/expo -node generate-expo-package.js 0.8.57 ../../artifacts ./expo-package +node generate-expo-package.js 1.0.0 ../../artifacts ./expo-package ``` ### Step 3: Test in a Expo app diff --git a/packages/expo/generate-expo-package.js b/packages/expo/generate-expo-package.js index de89071..4a28f1f 100644 --- a/packages/expo/generate-expo-package.js +++ b/packages/expo/generate-expo-package.js @@ -10,7 +10,7 @@ * node generate-expo-package.js * * Example: - * node generate-expo-package.js 0.8.53 ./artifacts ./expo-package + * node generate-expo-package.js 1.0.0 ./artifacts ./expo-package */ const fs = require('fs'); @@ -396,7 +396,7 @@ function main() { if (args.length < 3) { console.error('Usage: node generate-expo-package.js '); - console.error('Example: node generate-expo-package.js 0.8.53 ./artifacts ./expo-package'); + console.error('Example: node generate-expo-package.js 1.0.0 ./artifacts ./expo-package'); process.exit(1); } @@ -405,7 +405,7 @@ function main() { // Validate version format if (!/^\d+\.\d+\.\d+$/.test(version)) { console.error(`Error: Invalid version format: ${version}`); - console.error('Version must be in semver format (e.g., 0.8.53)'); + console.error('Version must be in semver format (e.g., 1.0.0)'); process.exit(1); } diff --git a/packages/node/README.md b/packages/node/README.md index 91cc894..234e935 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -136,6 +136,8 @@ Error thrown when the SQLite Sync extension cannot be found for the current plat ## Related Projects +- **[@sqliteai/sqlite-sync-react-native](https://www.npmjs.com/package/@sqliteai/sqlite-sync-react-native)** - SQLite Sync for React Native +- **[@sqliteai/sqlite-sync-expo](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo)** - SQLite Sync for Expo - **[@sqliteai/sqlite-vector](https://www.npmjs.com/package/@sqliteai/sqlite-vector)** - Vector search and similarity matching - **[@sqliteai/sqlite-ai](https://www.npmjs.com/package/@sqliteai/sqlite-ai)** - On-device AI inference and embedding generation - **[@sqliteai/sqlite-js](https://www.npmjs.com/package/@sqliteai/sqlite-js)** - Define SQLite functions in JavaScript diff --git a/packages/node/generate-platform-packages.js b/packages/node/generate-platform-packages.js index b725f6d..32bef4a 100644 --- a/packages/node/generate-platform-packages.js +++ b/packages/node/generate-platform-packages.js @@ -10,7 +10,7 @@ * node generate-platform-packages.js * * Example: - * node generate-platform-packages.js 0.8.53 ./artifacts ./platform-packages + * node generate-platform-packages.js 1.0.0 ./artifacts ./platform-packages */ const fs = require('fs'); @@ -165,7 +165,7 @@ function main() { if (args.length < 3) { console.error('Usage: node generate-platform-packages.js '); - console.error('Example: node generate-platform-packages.js 0.8.53 ./artifacts ./platform-packages'); + console.error('Example: node generate-platform-packages.js 1.0.0 ./artifacts ./platform-packages'); process.exit(1); } @@ -181,7 +181,7 @@ function main() { // Validate version format if (!/^\d+\.\d+\.\d+$/.test(version)) { console.error(`Error: Invalid version format: ${version}`); - console.error('Version must be in semver format (e.g., 0.8.53)'); + console.error('Version must be in semver format (e.g., 1.0.0)'); process.exit(1); } diff --git a/packages/node/package.json b/packages/node/package.json index f8ecd1d..1536b4e 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@sqliteai/sqlite-sync", - "version": "0.8.53", + "version": "1.0.0", "description": "SQLite Sync extension for Node.js - Sync on-device databases with the cloud", "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -53,13 +53,13 @@ "node": ">=16.0.0" }, "optionalDependencies": { - "@sqliteai/sqlite-sync-darwin-arm64": "0.8.53", - "@sqliteai/sqlite-sync-darwin-x86_64": "0.8.53", - "@sqliteai/sqlite-sync-linux-arm64": "0.8.53", - "@sqliteai/sqlite-sync-linux-arm64-musl": "0.8.53", - "@sqliteai/sqlite-sync-linux-x86_64": "0.8.53", - "@sqliteai/sqlite-sync-linux-x86_64-musl": "0.8.53", - "@sqliteai/sqlite-sync-win32-x86_64": "0.8.53" + "@sqliteai/sqlite-sync-darwin-arm64": "1.0.0", + "@sqliteai/sqlite-sync-darwin-x86_64": "1.0.0", + "@sqliteai/sqlite-sync-linux-arm64": "1.0.0", + "@sqliteai/sqlite-sync-linux-arm64-musl": "1.0.0", + "@sqliteai/sqlite-sync-linux-x86_64": "1.0.0", + "@sqliteai/sqlite-sync-linux-x86_64-musl": "1.0.0", + "@sqliteai/sqlite-sync-win32-x86_64": "1.0.0" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/packages/swift/CloudSync.swift b/packages/swift/CloudSync.swift new file mode 100644 index 0000000..73a0618 --- /dev/null +++ b/packages/swift/CloudSync.swift @@ -0,0 +1,15 @@ +// CloudSync.swift +// Provides the path to the CloudSync SQLite extension for use with sqlite3_load_extension. + +import Foundation + +public struct CloudSync { + /// Returns the absolute path to the CloudSync dylib for use with sqlite3_load_extension. + public static var path: String { + #if os(macOS) + return Bundle.main.bundlePath + "/Contents/Frameworks/CloudSync.framework/CloudSync" + #else + return Bundle.main.bundlePath + "/Frameworks/CloudSync.framework/CloudSync" + #endif + } +} diff --git a/packages/swift/extension/CloudSync.swift b/packages/swift/extension/CloudSync.swift deleted file mode 100644 index b7da66e..0000000 --- a/packages/swift/extension/CloudSync.swift +++ /dev/null @@ -1,19 +0,0 @@ -// CloudSync.swift -// This file serves as a placeholder for the CloudSync target. -// The actual SQLite extension is built using the Makefile through the build plugin. - -import Foundation - -/// Placeholder structure for CloudSync -public struct CloudSync { - /// Returns the path to the built CloudSync dylib inside the XCFramework - public static var path: String { - #if os(macOS) - return "CloudSync.xcframework/macos-arm64_x86_64/CloudSync.framework/CloudSync" - #elseif targetEnvironment(simulator) - return "CloudSync.xcframework/ios-arm64_x86_64-simulator/CloudSync.framework/CloudSync" - #else - return "CloudSync.xcframework/ios-arm64/CloudSync.framework/CloudSync" - #endif - } -} \ No newline at end of file diff --git a/packages/swift/plugin/CloudSync.swift b/packages/swift/plugin/CloudSync.swift deleted file mode 100644 index ccb95cc..0000000 --- a/packages/swift/plugin/CloudSync.swift +++ /dev/null @@ -1,60 +0,0 @@ -import PackagePlugin -import Foundation - -@main -struct CloudSync: BuildToolPlugin { - /// Entry point for creating build commands for targets in Swift packages. - func createBuildCommands(context: PluginContext, target: Target) async throws -> [Command] { - let packageDirectory = context.package.directoryURL - let outputDirectory = context.pluginWorkDirectoryURL - return createCloudSyncBuildCommands(packageDirectory: packageDirectory, outputDirectory: outputDirectory) - } -} - -#if canImport(XcodeProjectPlugin) -import XcodeProjectPlugin - -extension CloudSync: XcodeBuildToolPlugin { - // Entry point for creating build commands for targets in Xcode projects. - func createBuildCommands(context: XcodePluginContext, target: XcodeTarget) throws -> [Command] { - let outputDirectory = context.pluginWorkDirectoryURL - return createCloudSyncBuildCommands(packageDirectory: nil, outputDirectory: outputDirectory) - } -} - -#endif - -/// Shared function to create CloudSync build commands -func createCloudSyncBuildCommands(packageDirectory: URL?, outputDirectory: URL) -> [Command] { - - // For Xcode projects, use current directory; for Swift packages, use provided packageDirectory - let workingDirectory = packageDirectory?.path ?? "$(pwd)" - let packageDirInfo = packageDirectory != nil ? "Package directory: \(packageDirectory!.path)" : "Working directory: $(pwd)" - - return [ - .prebuildCommand( - displayName: "Building CloudSync XCFramework", - executable: URL(fileURLWithPath: "/bin/bash"), - arguments: [ - "-c", - """ - set -e - echo "Starting CloudSync XCFramework prebuild..." - echo "\(packageDirInfo)" - - # Clean and create output directory - rm -rf "\(outputDirectory.path)" - mkdir -p "\(outputDirectory.path)" - - # Build directly from source directory with custom output paths - cd "\(workingDirectory)" && \ - echo "Building XCFramework with native network..." && \ - make xcframework NATIVE_NETWORK=ON DIST_DIR="\(outputDirectory.path)" BUILD_RELEASE="\(outputDirectory.path)/build/release" BUILD_TEST="\(outputDirectory.path)/build/test" && \ - rm -rf "\(outputDirectory.path)/build" && \ - echo "XCFramework build completed successfully!" - """ - ], - outputFilesDirectory: outputDirectory - ) - ] -} \ No newline at end of file diff --git a/scripts/check-postgres-migration.sh b/scripts/check-postgres-migration.sh new file mode 100755 index 0000000..f14acd9 --- /dev/null +++ b/scripts/check-postgres-migration.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# Enforce the PostgreSQL extension versioning contract on every PR/push. +# +# The extension version (default_version in cloudsync.control, and the +# cloudsync--.sql filename) is MAJOR.MINOR only — it's derived from the +# first two components of CLOUDSYNC_VERSION in src/cloudsync.h. The full +# semver of the binary is reported by the cloudsync_version() SQL function. +# +# Contract: +# - PATCH bumps (e.g. 1.0.16 -> 1.0.17) keep EXTVERSION the same ('1.0'). +# Binary-only release; no SQL surface changes allowed; no user action +# needed after swapping the .so. +# - MINOR / MAJOR bumps (e.g. 1.0.x -> 1.1.0 or 1.x -> 2.0) move EXTVERSION. +# Must ship a matching cloudsync----.sql upgrade script so +# existing deployments can ALTER EXTENSION cloudsync UPDATE. +# +# This script runs in one of two modes depending on whether EXTVERSION moved +# since the most recent ancestor semver tag: +# +# (a) EXTVERSION unchanged (patch release): +# diff the current cloudsync.sql.in against the previous tag's install +# script. If they differ, fail: SQL surface changed without a MINOR +# bump, which would silently break users whose pg_extension.extversion +# stays at the old value. +# +# (b) EXTVERSION changed (minor/major release): +# require src/postgresql/migrations/cloudsync----.sql. +# +# Exit codes: +# 0 - contract satisfied (no bump, or bump with migration present) +# 1 - contract violated (SQL drift without bump, or missing migration) +# 2 - misconfigured environment (no git, missing header, unresolvable tag) + +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel 2>/dev/null || true) +if [ -z "$repo_root" ]; then + echo "Error: not inside a git working tree; cannot determine previous release tag." >&2 + exit 2 +fi +cd "$repo_root" + +header="src/cloudsync.h" +sql_template="src/postgresql/cloudsync.sql.in" + +[ -f "$header" ] || { echo "Error: $header not found." >&2; exit 2; } +[ -f "$sql_template" ] || { echo "Error: $sql_template not found." >&2; exit 2; } + +# Read full semver from the header and derive MAJOR.MINOR. +current_full=$(sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' "$header") +if [ -z "$current_full" ]; then + echo "Error: could not read CLOUDSYNC_VERSION from $header." >&2 + exit 2 +fi +current_ext=$(printf '%s\n' "$current_full" | cut -d. -f1-2) +if [ -z "$current_ext" ]; then + echo "Error: could not derive MAJOR.MINOR from CLOUDSYNC_VERSION '$current_full'." >&2 + exit 2 +fi + +# Find the latest ancestor semver tag. +prev_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*' 2>/dev/null || true) +if [ -z "$prev_tag" ]; then + echo "No prior semver tag reachable from HEAD; skipping migration check." + echo "(This is expected on an initial release or a shallow clone without tags.)" + exit 0 +fi + +# Resolve what EXTVERSION the previous release shipped with. +# Pre-new-scheme tags: tracked cloudsync.control held default_version. +# New-scheme tags: control is generated; read CLOUDSYNC_VERSION from header +# at that tag and truncate. +prev_full=$(git show "${prev_tag}:${header}" 2>/dev/null \ + | sed -n 's/^#define CLOUDSYNC_VERSION[[:space:]]*"\([^"]*\)".*/\1/p' || true) + +prev_ext=$(git show "${prev_tag}:docker/postgresql/cloudsync.control" 2>/dev/null \ + | sed -n "s/^default_version = '\\([^']*\\)'.*/\\1/p" \ + | head -1 || true) + +if [ -z "$prev_ext" ] && [ -n "$prev_full" ]; then + prev_ext=$(printf '%s\n' "$prev_full" | cut -d. -f1-2) +fi + +if [ -z "$prev_ext" ]; then + echo "Error: could not determine EXTVERSION at tag ${prev_tag}." >&2 + exit 2 +fi + +# --------------------------------------------------------------------------- +# Mode (a): EXTVERSION unchanged — verify SQL surface didn't drift. +# --------------------------------------------------------------------------- +if [ "$prev_ext" = "$current_ext" ]; then + # Produce a normalized view of the previous release's install script. + if git cat-file -e "${prev_tag}:${sql_template}" 2>/dev/null; then + # New-scheme tag: substitute @EXTVERSION@ with that tag's EXTVERSION. + prev_sql=$(git show "${prev_tag}:${sql_template}" \ + | sed "s/@EXTVERSION@/${prev_ext}/g") + else + # Old-scheme tag: the literal cloudsync--.sql was tracked. + prev_install_path="src/postgresql/cloudsync--${prev_ext}.sql" + if ! git cat-file -e "${prev_tag}:${prev_install_path}" 2>/dev/null; then + echo "Error: could not find previous install script at ${prev_tag}:${prev_install_path}." >&2 + exit 2 + fi + prev_sql=$(git show "${prev_tag}:${prev_install_path}") + fi + + # Current install script, rendered (substitute @EXTVERSION@ -> current_ext). + curr_sql=$(sed "s/@EXTVERSION@/${current_ext}/g" "$sql_template") + + if [ "$prev_sql" = "$curr_sql" ]; then + echo "OK: patch-only release candidate." + echo " EXTVERSION unchanged at '${current_ext}' since ${prev_tag}." + echo " Binary semver: ${prev_full:-unknown} -> ${current_full}." + echo " SQL surface identical; no migration script required." + exit 0 + fi + + cat >&2 < ${current_full} + +The install script (src/postgresql/cloudsync.sql.in) differs from what the +previous release shipped, but EXTVERSION is still '${current_ext}'. Existing +deployments have pg_extension.extversion = '${current_ext}' and will NOT run +any upgrade script when they swap in the new .so, so the new SQL bindings +won't be applied to their catalog. This silently breaks users. + +Pick one: + + 1. Bump MINOR in src/cloudsync.h (e.g. 1.0.x -> 1.1.0), and add + src/postgresql/migrations/cloudsync--${current_ext}--.sql + with the DDL deltas (CREATE OR REPLACE FUNCTION ..., etc.). This is the + correct choice for any intentional SQL-surface change. + + 2. Revert the SQL-level change in cloudsync.sql.in if it was accidental + (e.g. a refactor that went further than intended). + +Diff (previous -> current, normalized): + +EOF + # Show a readable diff; fall back to a terse message if diff is unavailable. + if command -v diff >/dev/null 2>&1; then + diff -u <(printf '%s\n' "$prev_sql") <(printf '%s\n' "$curr_sql") >&2 || true + else + echo "(install 'diff' to see line-level changes)" >&2 + fi + exit 1 +fi + +# --------------------------------------------------------------------------- +# Mode (b): EXTVERSION changed — require a migration file. +# --------------------------------------------------------------------------- +expected="src/postgresql/migrations/cloudsync--${prev_ext}--${current_ext}.sql" +if [ ! -f "$expected" ]; then + cat >&2 < ${current_ext} covered)." +echo " Binary semver: ${prev_full:-unknown} -> ${current_full}." diff --git a/src/block.c b/src/block.c new file mode 100644 index 0000000..dd99b26 --- /dev/null +++ b/src/block.c @@ -0,0 +1,303 @@ +// +// block.c +// cloudsync +// +// Block-level LWW CRDT support for text/blob fields. +// + +#include +#include +#include +#include "block.h" +#include "utils.h" +#include "fractional_indexing.h" + +// MARK: - Col name helpers - + +bool block_is_block_colname(const char *col_name) { + if (!col_name) return false; + return strchr(col_name, BLOCK_SEPARATOR) != NULL; +} + +char *block_extract_base_colname(const char *col_name) { + if (!col_name) return NULL; + const char *sep = strchr(col_name, BLOCK_SEPARATOR); + if (!sep) return cloudsync_string_dup(col_name); + return cloudsync_string_ndup(col_name, (size_t)(sep - col_name)); +} + +const char *block_extract_position_id(const char *col_name) { + if (!col_name) return NULL; + const char *sep = strchr(col_name, BLOCK_SEPARATOR); + if (!sep) return NULL; + return sep + 1; +} + +char *block_build_colname(const char *base_col, const char *position_id) { + if (!base_col || !position_id) return NULL; + size_t blen = strlen(base_col); + size_t plen = strlen(position_id); + char *result = (char *)cloudsync_memory_alloc(blen + 1 + plen + 1); + if (!result) return NULL; + memcpy(result, base_col, blen); + result[blen] = BLOCK_SEPARATOR; + memcpy(result + blen + 1, position_id, plen); + result[blen + 1 + plen] = '\0'; + return result; +} + +// MARK: - Text splitting - + +static block_list_t *block_list_create(void) { + block_list_t *list = (block_list_t *)cloudsync_memory_zeroalloc(sizeof(block_list_t)); + return list; +} + +static bool block_list_append(block_list_t *list, const char *content, size_t content_len, const char *position_id) { + if (list->count >= list->capacity) { + int new_cap = list->capacity ? list->capacity * 2 : 16; + block_entry_t *new_entries = (block_entry_t *)cloudsync_memory_realloc( + list->entries, (uint64_t)(new_cap * sizeof(block_entry_t))); + if (!new_entries) return false; + list->entries = new_entries; + list->capacity = new_cap; + } + block_entry_t *e = &list->entries[list->count]; + e->content = cloudsync_string_ndup(content, content_len); + e->position_id = position_id ? cloudsync_string_dup(position_id) : NULL; + if (!e->content) return false; + list->count++; + return true; +} + +void block_list_free(block_list_t *list) { + if (!list) return; + for (int i = 0; i < list->count; i++) { + if (list->entries[i].content) cloudsync_memory_free(list->entries[i].content); + if (list->entries[i].position_id) cloudsync_memory_free(list->entries[i].position_id); + } + if (list->entries) cloudsync_memory_free(list->entries); + cloudsync_memory_free(list); +} + +block_list_t *block_list_create_empty(void) { + return block_list_create(); +} + +bool block_list_add(block_list_t *list, const char *content, const char *position_id) { + if (!list) return false; + return block_list_append(list, content, strlen(content), position_id); +} + +block_list_t *block_split(const char *text, const char *delimiter) { + block_list_t *list = block_list_create(); + if (!list) return NULL; + + if (!text || !*text) { + // Empty text produces a single empty block + block_list_append(list, "", 0, NULL); + return list; + } + + size_t dlen = strlen(delimiter); + if (dlen == 0) { + // No delimiter: entire text is one block + block_list_append(list, text, strlen(text), NULL); + return list; + } + + const char *start = text; + const char *found; + while ((found = strstr(start, delimiter)) != NULL) { + size_t seg_len = (size_t)(found - start); + if (!block_list_append(list, start, seg_len, NULL)) { + block_list_free(list); + return NULL; + } + start = found + dlen; + } + // Last segment (after last delimiter or entire string if no delimiter found) + if (!block_list_append(list, start, strlen(start), NULL)) { + block_list_free(list); + return NULL; + } + + return list; +} + +// MARK: - Fractional indexing (via fractional-indexing submodule) - + +// Wrapper for malloc: fractional_indexing expects size_t (32-bit on WASM) but cloudsync_memory_alloc takes uint64_t. +// A direct cast passes strict call_indirect type checking in WASM and triggers a RuntimeError. +static void *fi_malloc_wrapper(size_t size) { + return cloudsync_memory_alloc((uint64_t)size); +} + +// Wrapper for calloc: fractional_indexing expects (count, size) but cloudsync_memory_zeroalloc takes a single size. +static void *fi_calloc_wrapper(size_t count, size_t size) { + return cloudsync_memory_zeroalloc((uint64_t)(count * size)); +} + +void block_init_allocator(void) { + fractional_indexing_allocator alloc = { + .malloc = fi_malloc_wrapper, + .calloc = fi_calloc_wrapper, + .free = cloudsync_memory_free + }; + fractional_indexing_set_allocator(&alloc); +} + +char *block_position_between(const char *before, const char *after) { + return generate_key_between(before, after); +} + +char **block_initial_positions(int count) { + if (count <= 0) return NULL; + return generate_n_keys_between(NULL, NULL, count); +} + +// MARK: - Block diff - + +static block_diff_t *block_diff_create(void) { + block_diff_t *diff = (block_diff_t *)cloudsync_memory_zeroalloc(sizeof(block_diff_t)); + return diff; +} + +static bool block_diff_append(block_diff_t *diff, block_diff_type type, const char *position_id, const char *content) { + if (diff->count >= diff->capacity) { + int new_cap = diff->capacity ? diff->capacity * 2 : 16; + block_diff_entry_t *new_entries = (block_diff_entry_t *)cloudsync_memory_realloc( + diff->entries, (uint64_t)(new_cap * sizeof(block_diff_entry_t))); + if (!new_entries) return false; + diff->entries = new_entries; + diff->capacity = new_cap; + } + block_diff_entry_t *e = &diff->entries[diff->count]; + e->type = type; + e->position_id = cloudsync_string_dup(position_id); + e->content = content ? cloudsync_string_dup(content) : NULL; + diff->count++; + return true; +} + +void block_diff_free(block_diff_t *diff) { + if (!diff) return; + for (int i = 0; i < diff->count; i++) { + if (diff->entries[i].position_id) cloudsync_memory_free(diff->entries[i].position_id); + if (diff->entries[i].content) cloudsync_memory_free(diff->entries[i].content); + } + if (diff->entries) cloudsync_memory_free(diff->entries); + cloudsync_memory_free(diff); +} + +// Content-based matching diff algorithm: +// 1. Build a consumed-set from old blocks +// 2. For each new block, find the first unconsumed old block with matching content +// 3. Matched blocks keep their position_id (UNCHANGED) +// 4. Unmatched new blocks get new position_ids (ADDED) +// 5. Unconsumed old blocks are REMOVED +// Modified blocks are detected when content changed but position stayed (handled as MODIFIED) +block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, + const char **new_parts, int new_count) { + block_diff_t *diff = block_diff_create(); + if (!diff) return NULL; + + // Track which old blocks have been consumed + bool *old_consumed = NULL; + if (old_count > 0) { + old_consumed = (bool *)cloudsync_memory_zeroalloc((uint64_t)(old_count * sizeof(bool))); + if (!old_consumed) { + block_diff_free(diff); + return NULL; + } + } + + // For each new block, try to find a matching unconsumed old block + // Use a simple forward scan to preserve ordering + int old_scan = 0; + char *last_position = NULL; + + for (int ni = 0; ni < new_count; ni++) { + bool found = false; + + // Scan forward in old blocks for a content match + for (int oi = old_scan; oi < old_count; oi++) { + if (old_consumed[oi]) continue; + + if (strcmp(old_blocks[oi].content, new_parts[ni]) == 0) { + // Exact match — mark any skipped old blocks as REMOVED + for (int si = old_scan; si < oi; si++) { + if (!old_consumed[si]) { + block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[si].position_id, NULL); + old_consumed[si] = true; + } + } + old_consumed[oi] = true; + old_scan = oi + 1; + last_position = old_blocks[oi].position_id; + found = true; + break; + } + } + + if (!found) { + // New block — needs a new position_id + const char *next_pos = NULL; + // Find the next unconsumed old block's position for the upper bound + for (int oi = old_scan; oi < old_count; oi++) { + if (!old_consumed[oi]) { + next_pos = old_blocks[oi].position_id; + break; + } + } + + char *new_pos = block_position_between(last_position, next_pos); + if (new_pos) { + block_diff_append(diff, BLOCK_DIFF_ADDED, new_pos, new_parts[ni]); + last_position = diff->entries[diff->count - 1].position_id; + cloudsync_memory_free(new_pos); + } + } + } + + // Mark remaining unconsumed old blocks as REMOVED + for (int oi = old_scan; oi < old_count; oi++) { + if (!old_consumed[oi]) { + block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[oi].position_id, NULL); + } + } + + if (old_consumed) cloudsync_memory_free(old_consumed); + return diff; +} + +// MARK: - Materialization - + +char *block_materialize_text(const char **blocks, int count, const char *delimiter) { + if (count == 0) return cloudsync_string_dup(""); + if (!delimiter) delimiter = BLOCK_DEFAULT_DELIMITER; + + size_t dlen = strlen(delimiter); + size_t total = 0; + for (int i = 0; i < count; i++) { + total += strlen(blocks[i]); + if (i < count - 1) total += dlen; + } + + char *result = (char *)cloudsync_memory_alloc(total + 1); + if (!result) return NULL; + + size_t offset = 0; + for (int i = 0; i < count; i++) { + size_t blen = strlen(blocks[i]); + memcpy(result + offset, blocks[i], blen); + offset += blen; + if (i < count - 1) { + memcpy(result + offset, delimiter, dlen); + offset += dlen; + } + } + result[offset] = '\0'; + + return result; +} diff --git a/src/block.h b/src/block.h new file mode 100644 index 0000000..fa43369 --- /dev/null +++ b/src/block.h @@ -0,0 +1,120 @@ +// +// block.h +// cloudsync +// +// Block-level LWW CRDT support for text/blob fields. +// Instead of replacing an entire text column on conflict, +// the text is split into blocks (lines/paragraphs) that are +// independently version-tracked and merged. +// + +#ifndef __CLOUDSYNC_BLOCK__ +#define __CLOUDSYNC_BLOCK__ + +#include +#include +#include + +// The separator character used in col_name to distinguish block entries +// from regular column entries. Format: "col_name\x1Fposition_id" +#define BLOCK_SEPARATOR '\x1F' +#define BLOCK_SEPARATOR_STR "\x1F" +#define BLOCK_DEFAULT_DELIMITER "\n" + +// Column-level algorithm for block tracking +typedef enum { + col_algo_normal = 0, + col_algo_block = 1 +} col_algo_t; + +// A single block from splitting text +typedef struct { + char *content; // block text (owned, must be freed) + char *position_id; // fractional index position (owned, must be freed) +} block_entry_t; + +// Array of blocks +typedef struct { + block_entry_t *entries; + int count; + int capacity; +} block_list_t; + +// Diff result for comparing old and new block lists +typedef enum { + BLOCK_DIFF_UNCHANGED = 0, + BLOCK_DIFF_ADDED = 1, + BLOCK_DIFF_MODIFIED = 2, + BLOCK_DIFF_REMOVED = 3 +} block_diff_type; + +typedef struct { + block_diff_type type; + char *position_id; // the position_id (owned, must be freed) + char *content; // new content (owned, must be freed; NULL for REMOVED) +} block_diff_entry_t; + +typedef struct { + block_diff_entry_t *entries; + int count; + int capacity; +} block_diff_t; + +// Initialize the fractional-indexing library to use cloudsync's allocator. +// Must be called once before any block_position_between / block_initial_positions calls. +void block_init_allocator(void); + +// Check if a col_name is a block entry (contains BLOCK_SEPARATOR) +bool block_is_block_colname(const char *col_name); + +// Extract the base column name from a block col_name (caller must free) +// e.g., "body\x1F0.5" -> "body" +char *block_extract_base_colname(const char *col_name); + +// Extract the position_id from a block col_name +// e.g., "body\x1F0.5" -> "0.5" +const char *block_extract_position_id(const char *col_name); + +// Build a block col_name from base + position_id (caller must free) +// e.g., ("body", "0.5") -> "body\x1F0.5" +char *block_build_colname(const char *base_col, const char *position_id); + +// Split text into blocks using the given delimiter +block_list_t *block_split(const char *text, const char *delimiter); + +// Free a block list +void block_list_free(block_list_t *list); + +// Generate fractional index position IDs for N initial blocks +// Returns array of N strings (caller must free each + the array) +char **block_initial_positions(int count); + +// Generate a position ID that sorts between 'before' and 'after' +// Either can be NULL (meaning beginning/end of sequence) +// Caller must free the result +char *block_position_between(const char *before, const char *after); + +// Compute diff between old blocks (with position IDs) and new content blocks +// old_blocks: existing blocks from metadata (with position_ids) +// new_parts: new text split by delimiter (no position_ids yet) +// new_count: number of new parts +block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, + const char **new_parts, int new_count); + +// Free a diff result +void block_diff_free(block_diff_t *diff); + +// Create an empty block list (for accumulating existing blocks) +block_list_t *block_list_create_empty(void); + +// Add a block entry to a list (content and position_id are copied) +bool block_list_add(block_list_t *list, const char *content, const char *position_id); + +// Concatenate block values with delimiter +// blocks: array of content strings (in position order) +// count: number of blocks +// delimiter: separator between blocks +// Returns allocated string (caller must free) +char *block_materialize_text(const char **blocks, int count, const char *delimiter); + +#endif diff --git a/src/cloudsync.c b/src/cloudsync.c index cb955c4..908e9c1 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -17,16 +17,12 @@ #include #include "cloudsync.h" -#include "cloudsync_private.h" #include "lz4.h" #include "pk.h" -#include "vtab.h" +#include "sql.h" #include "utils.h" #include "dbutils.h" - -#ifndef CLOUDSYNC_OMIT_NETWORK -#include "network.h" -#endif +#include "block.h" #ifdef _WIN32 #include @@ -51,35 +47,22 @@ #endif #endif -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT1 -#endif - -#ifndef UNUSED_PARAMETER -#define UNUSED_PARAMETER(X) (void)(X) -#endif - -#ifdef _WIN32 -#define APIEXPORT __declspec(dllexport) -#else -#define APIEXPORT -#endif - -#define CLOUDSYNC_DEFAULT_ALGO "cls" -#define CLOUDSYNC_INIT_NTABLES 128 -#define CLOUDSYNC_VALUE_NOTSET -1 +#define CLOUDSYNC_INIT_NTABLES 64 #define CLOUDSYNC_MIN_DB_VERSION 0 -#define CLOUDSYNC_PAYLOAD_MINBUF_SIZE 512*1024 -#define CLOUDSYNC_PAYLOAD_VERSION 1 -#define CLOUDSYNC_PAYLOAD_SIGNATURE 'CLSY' -#define CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY "cloudsync_payload_apply_callback" +#define CLOUDSYNC_PAYLOAD_MINBUF_SIZE (512*1024) +#define CLOUDSYNC_PAYLOAD_SIGNATURE 0x434C5359 /* 'C','L','S','Y' */ +#define CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL 1 +#define CLOUDSYNC_PAYLOAD_VERSION_1 CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL +#define CLOUDSYNC_PAYLOAD_VERSION_2 2 +#define CLOUDSYNC_PAYLOAD_VERSION_LATEST CLOUDSYNC_PAYLOAD_VERSION_2 +#define CLOUDSYNC_PAYLOAD_MIN_VERSION_WITH_CHECKSUM CLOUDSYNC_PAYLOAD_VERSION_2 #ifndef MAX #define MAX(a, b) (((a)>(b))?(a):(b)) #endif -#define DEBUG_SQLITE_ERROR(_rc, _fn, _db) do {if (_rc != SQLITE_OK) printf("Error in %s: %s\n", _fn, sqlite3_errmsg(_db));} while (0) +#define DEBUG_DBERROR(_rc, _fn, _data) do {if (_rc != DBRES_OK) printf("Error in %s: %s\n", _fn, database_errmsg(_data));} while (0) typedef enum { CLOUDSYNC_PK_INDEX_TBL = 0, @@ -94,121 +77,167 @@ typedef enum { } CLOUDSYNC_PK_INDEX; typedef enum { - CLOUDSYNC_STMT_VALUE_ERROR = -1, - CLOUDSYNC_STMT_VALUE_UNCHANGED = 0, - CLOUDSYNC_STMT_VALUE_CHANGED = 1, -} CLOUDSYNC_STMT_VALUE; - -typedef struct { - sqlite3_context *context; - int index; -} cloudsync_pk_decode_context; + DBVM_VALUE_ERROR = -1, + DBVM_VALUE_UNCHANGED = 0, + DBVM_VALUE_CHANGED = 1, +} DBVM_VALUE; #define SYNCBIT_SET(_data) _data->insync = 1 #define SYNCBIT_RESET(_data) _data->insync = 0 -#define BUMP_SEQ(_data) ((_data)->seq += 1, (_data)->seq - 1) -// MARK: - +// MARK: - Deferred column-batch merge - typedef struct { - table_algo algo; // CRDT algoritm associated to the table - char *name; // table name - char **col_name; // array of column names - sqlite3_stmt **col_merge_stmt; // array of merge insert stmt (indexed by col_name) - sqlite3_stmt **col_value_stmt; // array of column value stmt (indexed by col_name) - int *col_id; // array of column id - int ncols; // number of non primary key cols - int npks; // number of primary key cols - bool enabled; // flag to check if a table is enabled or disabled - #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - bool rowid_only; // a table with no primary keys other than the implicit rowid - #endif - - char **pk_name; // array of primary key names - - // precompiled statements - sqlite3_stmt *meta_pkexists_stmt; // check if a primary key already exist in the augmented table - sqlite3_stmt *meta_sentinel_update_stmt; // update a local sentinel row - sqlite3_stmt *meta_sentinel_insert_stmt; // insert a local sentinel row - sqlite3_stmt *meta_row_insert_update_stmt; // insert/update a local row - sqlite3_stmt *meta_row_drop_stmt; // delete rows from meta - sqlite3_stmt *meta_update_move_stmt; // update rows in meta when pk changes - sqlite3_stmt *meta_local_cl_stmt; // compute local cl value - sqlite3_stmt *meta_winner_clock_stmt; // get the rowid of the last inserted/updated row in the meta table - sqlite3_stmt *meta_merge_delete_drop; - sqlite3_stmt *meta_zero_clock_stmt; - sqlite3_stmt *meta_col_version_stmt; - sqlite3_stmt *meta_site_id_stmt; - - sqlite3_stmt *real_col_values_stmt; // retrieve all column values based on pk - sqlite3_stmt *real_merge_delete_stmt; - sqlite3_stmt *real_merge_sentinel_stmt; - - bool is_altering; - -} cloudsync_table_context; + const char *col_name; // pointer into table_context->col_name[idx] (stable) + dbvalue_t *col_value; // duplicated via database_value_dup (owned) + int64_t col_version; + int64_t db_version; + uint8_t site_id[UUID_LEN]; + int site_id_len; + int64_t seq; +} merge_pending_entry; + +typedef struct { + cloudsync_table_context *table; + char *pk; // malloc'd copy, freed on flush + int pk_len; + int64_t cl; + bool sentinel_pending; + bool row_exists; // true when the PK already exists locally + int count; + int capacity; + merge_pending_entry *entries; + + // Statement cache — reuse the prepared statement when the column + // combination and row_exists flag match between consecutive PK flushes. + dbvm_t *cached_vm; + bool cached_row_exists; + int cached_col_count; + const char **cached_col_names; // array of pointers into table_context (not owned) +} merge_pending_batch; + +// MARK: - struct cloudsync_pk_decode_bind_context { - sqlite3_stmt *vm; - char *tbl; - int64_t tbl_len; - const void *pk; - int64_t pk_len; - char *col_name; - int64_t col_name_len; - int64_t col_version; - int64_t db_version; - const void *site_id; - int64_t site_id_len; - int64_t cl; - int64_t seq; + dbvm_t *vm; + char *tbl; + int64_t tbl_len; + const void *pk; + int64_t pk_len; + char *col_name; + int64_t col_name_len; + int64_t col_version; + int64_t db_version; + const void *site_id; + int64_t site_id_len; + int64_t cl; + int64_t seq; }; struct cloudsync_context { - sqlite3_context *sqlite_ctx; + void *db; + char errmsg[1024]; + int errcode; - char *libversion; - uint8_t site_id[UUID_LEN]; - int insync; - int debug; - bool merge_equal_values; - bool temp_bool; // temporary value used in callback - void *aux_data; + char *libversion; + uint8_t site_id[UUID_LEN]; + int insync; + int debug; + bool merge_equal_values; + void *aux_data; // stmts and context values - bool pragma_checked; // we need to check PRAGMAs only once per transaction - sqlite3_stmt *schema_version_stmt; - sqlite3_stmt *data_version_stmt; - sqlite3_stmt *db_version_stmt; - sqlite3_stmt *getset_siteid_stmt; - int data_version; - int schema_version; - uint64_t schema_hash; - - // set at the start of each transaction on the first invocation and - // re-set on transaction commit or rollback - sqlite3_int64 db_version; - // the version that the db will be set to at the end of the transaction - // if that transaction were to commit at the time this value is checked - sqlite3_int64 pending_db_version; + dbvm_t *schema_version_stmt; + dbvm_t *data_version_stmt; + dbvm_t *db_version_stmt; + dbvm_t *getset_siteid_stmt; + int data_version; + int schema_version; + uint64_t schema_hash; + + // set at transaction start and reset on commit/rollback + int64_t db_version; + // version the DB would have if the transaction committed now + int64_t pending_db_version; // used to set an order inside each transaction - int seq; - - // augmented tables are stored in-memory so we do not need to retrieve information about col names and cid - // from the disk each time a write statement is performed - // we do also not need to use an hash map here because for few tables the direct in-memory comparison with table name is faster - cloudsync_table_context **tables; - int tables_count; - int tables_alloc; + int seq; + + // optional schema_name to be set in the cloudsync_table_context + char *current_schema; + + // augmented tables are stored in-memory so we do not need to retrieve information about + // col_names and cid from the disk each time a write statement is performed + // we do also not need to use an hash map here because for few tables the direct + // in-memory comparison with table name is faster + cloudsync_table_context **tables; // dense vector: [0..tables_count-1] are valid + int tables_count; // size + int tables_cap; // capacity + + int skip_decode_idx; // -1 in sqlite, col_value index in postgresql + + // deferred column-batch merge (active during payload_apply) + merge_pending_batch *pending_batch; }; -typedef struct { +struct cloudsync_table_context { + table_algo algo; // CRDT algoritm associated to the table + char *name; // table name + char *schema; // table schema + char *meta_ref; // schema-qualified meta table name (e.g. "schema"."name_cloudsync") + char *base_ref; // schema-qualified base table name (e.g. "schema"."name") + char **col_name; // array of column names + dbvm_t **col_merge_stmt; // array of merge insert stmt (indexed by col_name) + dbvm_t **col_value_stmt; // array of column value stmt (indexed by col_name) + int *col_id; // array of column id + col_algo_t *col_algo; // per-column algorithm (normal or block) + char **col_delimiter; // per-column delimiter for block splitting (NULL = default "\n") + bool has_block_cols; // quick check: does this table have any block columns? + dbvm_t *block_value_read_stmt; // SELECT col_value FROM blocks table + dbvm_t *block_value_write_stmt; // INSERT OR REPLACE into blocks table + dbvm_t *block_value_delete_stmt; // DELETE from blocks table + dbvm_t *block_list_stmt; // SELECT block entries for materialization + char *blocks_ref; // schema-qualified blocks table name + int ncols; // number of non primary key cols + int npks; // number of primary key cols + bool enabled; // flag to check if a table is enabled or disabled + #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + bool rowid_only; // a table with no primary keys other than the implicit rowid + #endif + + char **pk_name; // array of primary key names + + // precompiled statements + dbvm_t *meta_pkexists_stmt; // check if a primary key already exist in the augmented table + dbvm_t *meta_sentinel_update_stmt; // update a local sentinel row + dbvm_t *meta_sentinel_insert_stmt; // insert a local sentinel row + dbvm_t *meta_row_insert_update_stmt; // insert/update a local row + dbvm_t *meta_row_drop_stmt; // delete rows from meta + dbvm_t *meta_update_move_stmt; // update rows in meta when pk changes + dbvm_t *meta_local_cl_stmt; // compute local cl value + dbvm_t *meta_winner_clock_stmt; // get the rowid of the last inserted/updated row in the meta table + dbvm_t *meta_merge_delete_drop; + dbvm_t *meta_zero_clock_stmt; + dbvm_t *meta_col_version_stmt; + dbvm_t *meta_site_id_stmt; + + dbvm_t *real_col_values_stmt; // retrieve all column values based on pk + dbvm_t *real_merge_delete_stmt; + dbvm_t *real_merge_sentinel_stmt; + + bool is_altering; // flag to track if a table alteration is in progress + + // context + cloudsync_context *context; +}; + +struct cloudsync_payload_context { char *buffer; + size_t bsize; size_t balloc; size_t bused; uint64_t nrows; uint16_t ncols; -} cloudsync_data_payload; +}; #ifdef _MSC_VER #pragma pack(push, 1) // For MSVC: pack struct with 1-byte alignment @@ -218,24 +247,16 @@ typedef struct { #endif typedef struct PACKED { - uint32_t signature; // 'CLSY' - uint8_t version; // protocol version - uint8_t libversion[3]; // major.minor.patch + uint32_t signature; // 'CLSY' + uint8_t version; // protocol version + uint8_t libversion[3]; // major.minor.patch uint32_t expanded_size; uint16_t ncols; uint32_t nrows; uint64_t schema_hash; - uint8_t unused[6]; // padding to ensure the struct is exactly 32 bytes + uint8_t checksum[6]; // 48 bits checksum (to ensure struct is 32 bytes) } cloudsync_payload_header; -typedef struct { - sqlite3_value *table_name; - sqlite3_value **new_values; - sqlite3_value **old_values; - int count; - int capacity; -} cloudsync_update_payload; - #ifdef _MSC_VER #pragma pack(pop) #endif @@ -247,114 +268,106 @@ bool force_uncompressed_blob = false; #define CHECK_FORCE_UNCOMPRESSED_BUFFER() #endif -int db_version_rebuild_stmt (sqlite3 *db, cloudsync_context *data); -int cloudsync_load_siteid (sqlite3 *db, cloudsync_context *data); -int local_mark_insert_or_update_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, sqlite3_int64 db_version, int seq); +// Internal prototypes +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq); -// MARK: - STMT Utils - +// MARK: - CRDT algos - -CLOUDSYNC_STMT_VALUE stmt_execute (sqlite3_stmt *stmt, cloudsync_context *data) { - int rc = sqlite3_step(stmt); - if (rc != SQLITE_ROW && rc != SQLITE_DONE) { - if (data) DEBUG_SQLITE_ERROR(rc, "stmt_execute", sqlite3_db_handle(stmt)); - sqlite3_reset(stmt); - return CLOUDSYNC_STMT_VALUE_ERROR; - } +table_algo cloudsync_algo_from_name (const char *algo_name) { + if (algo_name == NULL) return table_algo_none; - CLOUDSYNC_STMT_VALUE result = CLOUDSYNC_STMT_VALUE_CHANGED; + if ((strcasecmp(algo_name, "CausalLengthSet") == 0) || (strcasecmp(algo_name, "cls") == 0)) return table_algo_crdt_cls; + if ((strcasecmp(algo_name, "GrowOnlySet") == 0) || (strcasecmp(algo_name, "gos") == 0)) return table_algo_crdt_gos; + if ((strcasecmp(algo_name, "DeleteWinsSet") == 0) || (strcasecmp(algo_name, "dws") == 0)) return table_algo_crdt_dws; + if ((strcasecmp(algo_name, "AddWinsSet") == 0) || (strcasecmp(algo_name, "aws") == 0)) return table_algo_crdt_aws; + + // if nothing is found + return table_algo_none; +} + +const char *cloudsync_algo_name (table_algo algo) { + switch (algo) { + case table_algo_crdt_cls: return "cls"; + case table_algo_crdt_gos: return "gos"; + case table_algo_crdt_dws: return "dws"; + case table_algo_crdt_aws: return "aws"; + case table_algo_none: return NULL; + } + return NULL; +} + +// MARK: - DBVM Utils - + +DBVM_VALUE dbvm_execute (dbvm_t *stmt, cloudsync_context *data) { + if (!stmt) return DBVM_VALUE_ERROR; + + int rc = databasevm_step(stmt); + if (rc != DBRES_ROW && rc != DBRES_DONE) { + if (data) DEBUG_DBERROR(rc, "stmt_execute", data); + databasevm_reset(stmt); + return DBVM_VALUE_ERROR; + } + + DBVM_VALUE result = DBVM_VALUE_CHANGED; if (stmt == data->data_version_stmt) { - int version = sqlite3_column_int(stmt, 0); + int version = (int)database_column_int(stmt, 0); if (version != data->data_version) { data->data_version = version; } else { - result = CLOUDSYNC_STMT_VALUE_UNCHANGED; + result = DBVM_VALUE_UNCHANGED; } } else if (stmt == data->schema_version_stmt) { - int version = sqlite3_column_int(stmt, 0); + int version = (int)database_column_int(stmt, 0); if (version > data->schema_version) { data->schema_version = version; } else { - result = CLOUDSYNC_STMT_VALUE_UNCHANGED; + result = DBVM_VALUE_UNCHANGED; } } else if (stmt == data->db_version_stmt) { - data->db_version = (rc == SQLITE_DONE) ? CLOUDSYNC_MIN_DB_VERSION : sqlite3_column_int64(stmt, 0); + data->db_version = (rc == DBRES_DONE) ? CLOUDSYNC_MIN_DB_VERSION : database_column_int(stmt, 0); } - sqlite3_reset(stmt); + databasevm_reset(stmt); return result; } -int stmt_count (sqlite3_stmt *stmt, const char *value, size_t len, int type) { +int dbvm_count (dbvm_t *stmt, const char *value, size_t len, int type) { int result = -1; - int rc = SQLITE_OK; + int rc = DBRES_OK; if (value) { - rc = (type == SQLITE_TEXT) ? sqlite3_bind_text(stmt, 1, value, (int)len, SQLITE_STATIC) : sqlite3_bind_blob(stmt, 1, value, (int)len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = (type == DBTYPE_TEXT) ? databasevm_bind_text(stmt, 1, value, (int)len) : databasevm_bind_blob(stmt, 1, value, len); + if (rc != DBRES_OK) goto cleanup; } - rc = sqlite3_step(stmt); - if (rc == SQLITE_DONE) { + rc = databasevm_step(stmt); + if (rc == DBRES_DONE) { result = 0; - rc = SQLITE_OK; - } else if (rc == SQLITE_ROW) { - result = sqlite3_column_int(stmt, 0); - rc = SQLITE_OK; + rc = DBRES_OK; + } else if (rc == DBRES_ROW) { + result = (int)database_column_int(stmt, 0); + rc = DBRES_OK; } cleanup: - DEBUG_SQLITE_ERROR(rc, "stmt_count", sqlite3_db_handle(stmt)); - sqlite3_reset(stmt); + databasevm_reset(stmt); return result; } -sqlite3_stmt *stmt_reset (sqlite3_stmt *stmt) { - sqlite3_clear_bindings(stmt); - sqlite3_reset(stmt); - return NULL; -} - -int stmts_add_tocontext (sqlite3 *db, cloudsync_context *data) { - DEBUG_DBFUNCTION("cloudsync_add_stmts"); - - if (data->data_version_stmt == NULL) { - const char *sql = "PRAGMA data_version;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->data_version_stmt, NULL); - DEBUG_STMT("data_version_stmt %p", data->data_version_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("data_version_stmt: %s", sql); - } - - if (data->schema_version_stmt == NULL) { - const char *sql = "PRAGMA schema_version;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->schema_version_stmt, NULL); - DEBUG_STMT("schema_version_stmt %p", data->schema_version_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("schema_version_stmt: %s", sql); - } - - if (data->getset_siteid_stmt == NULL) { - // get and set index of the site_id - // in SQLite, we can’t directly combine an INSERT and a SELECT to both insert a row and return an identifier (rowid) in a single statement, - // however, we can use a workaround by leveraging the INSERT statement with ON CONFLICT DO UPDATE and then combining it with RETURNING rowid - const char *sql = "INSERT INTO cloudsync_site_id (site_id) VALUES (?) ON CONFLICT(site_id) DO UPDATE SET site_id = site_id RETURNING rowid;"; - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->getset_siteid_stmt, NULL); - DEBUG_STMT("getset_siteid_stmt %p", data->getset_siteid_stmt); - if (rc != SQLITE_OK) return rc; - DEBUG_SQL("getset_siteid_stmt: %s", sql); - } - - return db_version_rebuild_stmt(db, data); +void dbvm_reset (dbvm_t *stmt) { + if (!stmt) return; + databasevm_clear_bindings(stmt); + databasevm_reset(stmt); } // MARK: - Database Version - -char *db_version_build_query (sqlite3 *db) { +int cloudsync_dbversion_build_query (cloudsync_context *data, char **sql_out) { // this function must be manually called each time tables changes // because the query plan changes too and it must be re-prepared // unfortunately there is no other way - + // we need to execute a query like: /* SELECT max(version) as version FROM ( @@ -367,77 +380,78 @@ char *db_version_build_query (sqlite3 *db) { SELECT value as version FROM cloudsync_settings WHERE key = 'pre_alter_dbversion' ) */ - + // the good news is that the query can be computed in SQLite without the need to do any extra computation from the host language - const char *sql = "WITH table_names AS (" - "SELECT format('%w', name) as tbl_name " - "FROM sqlite_master " - "WHERE type='table' " - "AND name LIKE '%_cloudsync'" - "), " - "query_parts AS (" - "SELECT 'SELECT max(db_version) as version FROM \"' || tbl_name || '\"' as part FROM table_names" - "), " - "combined_query AS (" - "SELECT GROUP_CONCAT(part, ' UNION ALL ') || ' UNION SELECT value as version FROM cloudsync_settings WHERE key = ''pre_alter_dbversion''' as full_query FROM query_parts" - ") " - "SELECT 'SELECT max(version) as version FROM (' || full_query || ');' FROM combined_query;"; - return dbutils_text_select(db, sql); -} - -int db_version_rebuild_stmt (sqlite3 *db, cloudsync_context *data) { + + *sql_out = NULL; + return database_select_text(data, SQL_DBVERSION_BUILD_QUERY, sql_out); +} + +int cloudsync_dbversion_rebuild (cloudsync_context *data) { if (data->db_version_stmt) { - sqlite3_finalize(data->db_version_stmt); + databasevm_finalize(data->db_version_stmt); data->db_version_stmt = NULL; } - - sqlite3_int64 count = dbutils_table_settings_count_tables(db); - if (count == 0) return SQLITE_OK; - else if (count == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - return SQLITE_ERROR; - } - - char *sql = db_version_build_query(db); - if (!sql) return SQLITE_NOMEM; + + int64_t count = dbutils_table_settings_count_tables(data); + if (count == 0) return DBRES_OK; + else if (count == -1) return cloudsync_set_dberror(data); + + char *sql = NULL; + int rc = cloudsync_dbversion_build_query(data, &sql); + if (rc != DBRES_OK) return cloudsync_set_dberror(data); + + // A NULL SQL with rc == OK means the generator produced a NULL row: + // sqlite_master has no *_cloudsync meta-tables (for example, the user + // dropped the base table and its meta-table without calling + // cloudsync_cleanup, leaving stale cloudsync_table_settings rows). + // Treat this the same as count == 0: no prepared statement, db_version + // stays at the minimum and will be rebuilt on the next cloudsync_init. + // Genuine errors from database_select_text are handled above. + if (!sql) return DBRES_OK; DEBUG_SQL("db_version_stmt: %s", sql); - - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &data->db_version_stmt, NULL); + + rc = databasevm_prepare(data, sql, (void **)&data->db_version_stmt, DBFLAG_PERSISTENT); DEBUG_STMT("db_version_stmt %p", data->db_version_stmt); cloudsync_memory_free(sql); return rc; } -int db_version_rerun (sqlite3 *db, cloudsync_context *data) { - CLOUDSYNC_STMT_VALUE schema_changed = stmt_execute(data->schema_version_stmt, data); - if (schema_changed == CLOUDSYNC_STMT_VALUE_ERROR) return -1; - - if (schema_changed == CLOUDSYNC_STMT_VALUE_CHANGED) { - int rc = db_version_rebuild_stmt(db, data); - if (rc != SQLITE_OK) return -1; +int cloudsync_dbversion_rerun (cloudsync_context *data) { + DBVM_VALUE schema_changed = dbvm_execute(data->schema_version_stmt, data); + if (schema_changed == DBVM_VALUE_ERROR) return -1; + + if (schema_changed == DBVM_VALUE_CHANGED) { + int rc = cloudsync_dbversion_rebuild(data); + if (rc != DBRES_OK) return -1; } - - CLOUDSYNC_STMT_VALUE rc = stmt_execute(data->db_version_stmt, data); - if (rc == CLOUDSYNC_STMT_VALUE_ERROR) return -1; + + if (!data->db_version_stmt) { + data->db_version = CLOUDSYNC_MIN_DB_VERSION; + return 0; + } + + DBVM_VALUE rc = dbvm_execute(data->db_version_stmt, data); + if (rc == DBVM_VALUE_ERROR) return -1; return 0; } -int db_version_check_uptodate (sqlite3 *db, cloudsync_context *data) { +int cloudsync_dbversion_check_uptodate (cloudsync_context *data) { // perform a PRAGMA data_version to check if some other process write any data - CLOUDSYNC_STMT_VALUE rc = stmt_execute(data->data_version_stmt, data); - if (rc == CLOUDSYNC_STMT_VALUE_ERROR) return -1; + DBVM_VALUE rc = dbvm_execute(data->data_version_stmt, data); + if (rc == DBVM_VALUE_ERROR) return -1; // db_version is already set and there is no need to update it - if (data->db_version != CLOUDSYNC_VALUE_NOTSET && rc == CLOUDSYNC_STMT_VALUE_UNCHANGED) return 0; + if (data->db_version != CLOUDSYNC_VALUE_NOTSET && rc == DBVM_VALUE_UNCHANGED) return 0; - return db_version_rerun(db, data); + return cloudsync_dbversion_rerun(data); } -sqlite3_int64 db_version_next (sqlite3 *db, cloudsync_context *data, sqlite3_int64 merging_version) { - int rc = db_version_check_uptodate(db, data); - if (rc != SQLITE_OK) return -1; +int64_t cloudsync_dbversion_next (cloudsync_context *data, int64_t merging_version) { + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != DBRES_OK) return -1; - sqlite3_int64 result = data->db_version + 1; + int64_t result = data->db_version + 1; if (result < data->pending_db_version) result = data->pending_db_version; if (merging_version != CLOUDSYNC_VALUE_NOTSET && result < merging_version) result = merging_version; data->pending_db_version = result; @@ -445,18 +459,6 @@ sqlite3_int64 db_version_next (sqlite3 *db, cloudsync_context *data, sqlite3_int return result; } -// MARK: - - -void *cloudsync_get_auxdata (sqlite3_context *context) { - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - return (data) ? data->aux_data : NULL; -} - -void cloudsync_set_auxdata (sqlite3_context *context, void *xdata) { - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - if (data) data->aux_data = xdata; -} - // MARK: - PK Context - char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len) { @@ -482,152 +484,242 @@ int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx) { return ctx->db_version; } -// MARK: - Table Utils - +// MARK: - CloudSync Context - -char *table_build_values_sql (sqlite3 *db, cloudsync_table_context *table) { - char *sql = NULL; - - /* - This SQL statement dynamically generates a SELECT query for a specified table. - It uses Common Table Expressions (CTEs) to construct the column names and - primary key conditions based on the table schema, which is obtained through - the `pragma_table_info` function. +int cloudsync_insync (cloudsync_context *data) { + return data->insync; +} + +void *cloudsync_siteid (cloudsync_context *data) { + return (void *)data->site_id; +} - 1. `col_names` CTE: - - Retrieves a comma-separated list of non-primary key column names from - the specified table's schema. +void cloudsync_reset_siteid (cloudsync_context *data) { + memset(data->site_id, 0, sizeof(uint8_t) * UUID_LEN); +} - 2. `pk_where` CTE: - - Retrieves a condition string representing the primary key columns in the - format: "column1=? AND column2=? AND ...", used to create the WHERE clause - for selecting rows based on primary key values. +int cloudsync_load_siteid (cloudsync_context *data) { + // check if site_id was already loaded + if (data->site_id[0] != 0) return DBRES_OK; + + // load site_id + char *buffer = NULL; + int64_t size = 0; + int rc = database_select_blob(data, SQL_SITEID_SELECT_ROWID0, &buffer, &size); + if (rc != DBRES_OK) return rc; + if (!buffer || size != UUID_LEN) { + if (buffer) cloudsync_memory_free(buffer); + return cloudsync_set_error(data, "Unable to retrieve siteid", DBRES_MISUSE); + } + + memcpy(data->site_id, buffer, UUID_LEN); + cloudsync_memory_free(buffer); + + return DBRES_OK; +} - 3. Final SELECT: - - Constructs the complete SELECT statement as a string, combining: - - Column names from `col_names`. - - The target table name. - - The WHERE clause conditions from `pk_where`. +int64_t cloudsync_dbversion (cloudsync_context *data) { + return data->db_version; +} - The resulting query can be used to select rows from the table based on primary - key values, and can be executed within the application to retrieve data dynamically. - */ +int cloudsync_bumpseq (cloudsync_context *data) { + int value = data->seq; + data->seq += 1; + return value; +} - // Unfortunately in SQLite column names (or table names) cannot be bound parameters in a SELECT statement - // otherwise we should have used something like SELECT 'SELECT ? FROM %w WHERE rowid=?'; +void cloudsync_update_schema_hash (cloudsync_context *data) { + database_update_schema_hash(data, &data->schema_hash); +} - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); +void *cloudsync_db (cloudsync_context *data) { + return data->db; +} - #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - if (table->rowid_only) { - sql = memory_mprintf("WITH col_names AS (SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid) SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE rowid=?;'", table->name, table->name); - goto process_process; +int cloudsync_add_dbvms (cloudsync_context *data) { + DEBUG_DBFUNCTION("cloudsync_add_stmts"); + + if (data->data_version_stmt == NULL) { + int rc = databasevm_prepare(data, SQL_DATA_VERSION, (void **)&data->data_version_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("data_version_stmt %p", data->data_version_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("data_version_stmt: %s", SQL_DATA_VERSION); + } + + if (data->schema_version_stmt == NULL) { + int rc = databasevm_prepare(data, SQL_SCHEMA_VERSION, (void **)&data->schema_version_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("schema_version_stmt %p", data->schema_version_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("schema_version_stmt: %s", SQL_SCHEMA_VERSION); + } + + if (data->getset_siteid_stmt == NULL) { + // get and set index of the site_id + // in SQLite, we can’t directly combine an INSERT and a SELECT to both insert a row and return an identifier (rowid) in a single statement, + // however, we can use a workaround by leveraging the INSERT statement with ON CONFLICT DO UPDATE and then combining it with RETURNING rowid + int rc = databasevm_prepare(data, SQL_SITEID_GETSET_ROWID_BY_SITEID, (void **)&data->getset_siteid_stmt, DBFLAG_PERSISTENT); + DEBUG_STMT("getset_siteid_stmt %p", data->getset_siteid_stmt); + if (rc != DBRES_OK) return rc; + DEBUG_SQL("getset_siteid_stmt: %s", SQL_SITEID_GETSET_ROWID_BY_SITEID); } - #endif - sql = cloudsync_memory_mprintf("WITH col_names AS (SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid), pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, table->name, singlequote_escaped_table_name); + return cloudsync_dbversion_rebuild(data); +} + +int cloudsync_set_error (cloudsync_context *data, const char *err_user, int err_code) { + // force err_code to be something different than OK + if (err_code == DBRES_OK) err_code = database_errcode(data); + if (err_code == DBRES_OK) err_code = DBRES_ERROR; -#if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES -process_process: -#endif - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); + // compute a meaningful error message + if (err_user == NULL) { + snprintf(data->errmsg, sizeof(data->errmsg), "%s", database_errmsg(data)); + } else { + const char *db_error = database_errmsg(data); + char db_error_copy[sizeof(data->errmsg)]; + int rc = database_errcode(data); + if (rc == DBRES_OK) { + snprintf(data->errmsg, sizeof(data->errmsg), "%s", err_user); + } else { + if (db_error == data->errmsg) { + snprintf(db_error_copy, sizeof(db_error_copy), "%s", db_error); + db_error = db_error_copy; + } + snprintf(data->errmsg, sizeof(data->errmsg), "%s (%s)", err_user, db_error); + } + } - return query; + data->errcode = err_code; + return err_code; +} + +int cloudsync_set_dberror (cloudsync_context *data) { + return cloudsync_set_error(data, NULL, DBRES_OK); +} + +const char *cloudsync_errmsg (cloudsync_context *data) { + return data->errmsg; +} + +int cloudsync_errcode (cloudsync_context *data) { + return data->errcode; +} + +void cloudsync_reset_error (cloudsync_context *data) { + data->errmsg[0] = 0; + data->errcode = DBRES_OK; +} + +void *cloudsync_auxdata (cloudsync_context *data) { + return data->aux_data; +} + +void cloudsync_set_auxdata (cloudsync_context *data, void *xdata) { + data->aux_data = xdata; +} + +void cloudsync_set_schema (cloudsync_context *data, const char *schema) { + if (data->current_schema && schema && strcmp(data->current_schema, schema) == 0) return; + if (data->current_schema) cloudsync_memory_free(data->current_schema); + data->current_schema = NULL; + if (schema) data->current_schema = cloudsync_string_dup_lowercase(schema); } -char *table_build_mergedelete_sql (sqlite3 *db, cloudsync_table_context *table) { +const char *cloudsync_schema (cloudsync_context *data) { + return data->current_schema; +} + +const char *cloudsync_table_schema (cloudsync_context *data, const char *table_name) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return NULL; + + return table->schema; +} + +// MARK: - Table Utils - + +void table_pknames_free (char **names, int nrows) { + if (!names) return; + for (int i = 0; i < nrows; ++i) {cloudsync_memory_free(names[i]);} + cloudsync_memory_free(names); +} + +char *table_build_mergedelete_sql (cloudsync_table_context *table) { #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { - char *sql = memory_mprintf("DELETE FROM \"%w\" WHERE rowid=?;", table->name); + char *sql = memory_mprintf(SQL_DELETE_ROW_BY_ROWID, table->name); return sql; } #endif - - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - char *sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'DELETE FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + + return sql_build_delete_by_pk(table->context, table->name, table->schema); } -char *table_build_mergeinsert_sql (sqlite3 *db, cloudsync_table_context *table, const char *colname) { +char *table_build_mergeinsert_sql (cloudsync_table_context *table, const char *colname) { char *sql = NULL; #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { if (colname == NULL) { // INSERT OR IGNORE INTO customers (first_name,last_name) VALUES (?,?); - sql = memory_mprintf("INSERT OR IGNORE INTO \"%w\" (rowid) VALUES (?);", table->name); + sql = memory_mprintf(SQL_INSERT_ROWID_IGNORE, table->name); } else { // INSERT INTO customers (first_name,last_name,age) VALUES (?,?,?) ON CONFLICT DO UPDATE SET age=?; - sql = memory_mprintf("INSERT INTO \"%w\" (rowid, \"%w\") VALUES (?, ?) ON CONFLICT DO UPDATE SET \"%w\"=?;", table->name, colname, colname); + sql = memory_mprintf(SQL_UPSERT_ROWID_AND_COL_BY_ROWID, table->name, colname, colname); } return sql; } #endif - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - if (colname == NULL) { // is sentinel insert - sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk), pk_bind AS (SELECT group_concat('?') AS pk_binding FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'INSERT OR IGNORE INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ') VALUES (' || (SELECT pk_binding FROM pk_bind) || ');'", table->name, table->name, singlequote_escaped_table_name); + sql = sql_build_insert_pk_ignore(table->context, table->name, table->schema); } else { - char *singlequote_escaped_col_name = cloudsync_memory_mprintf("%q", colname); - sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk), pk_bind AS (SELECT group_concat('?') AS pk_binding FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'INSERT INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ',\"%w\") VALUES (' || (SELECT pk_binding FROM pk_bind) || ',?) ON CONFLICT DO UPDATE SET \"%w\"=?;'", table->name, table->name, singlequote_escaped_table_name, singlequote_escaped_col_name, singlequote_escaped_col_name); - cloudsync_memory_free(singlequote_escaped_col_name); - + sql = sql_build_upsert_pk_and_col(table->context, table->name, colname, table->schema); } - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + return sql; } -char *table_build_value_sql (sqlite3 *db, cloudsync_table_context *table, const char *colname) { - char *colnamequote = dbutils_is_star_table(colname) ? "" : "\""; - +char *table_build_value_sql (cloudsync_table_context *table, const char *colname) { #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES if (table->rowid_only) { - char *sql = memory_mprintf("SELECT %s%w%s FROM \"%w\" WHERE rowid=?;", colnamequote, colname, colnamequote, table->name); + char *colnamequote = "\""; + char *sql = memory_mprintf(SQL_SELECT_COLS_BY_ROWID_FMT, colnamequote, colname, colnamequote, table->name); return sql; } #endif // SELECT age FROM customers WHERE first_name=? AND last_name=?; - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - char *singlequote_escaped_col_name = cloudsync_memory_mprintf("%q", colname); - char *sql = cloudsync_memory_mprintf("WITH pk_where AS (SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk) SELECT 'SELECT %s%w%s FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'", table->name, colnamequote, singlequote_escaped_col_name, colnamequote, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_col_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) return NULL; - - char *query = dbutils_text_select(db, sql); - cloudsync_memory_free(sql); - - return query; + return sql_build_select_cols_by_pk(table->context, table->name, colname, table->schema); } -cloudsync_table_context *table_create (const char *name, table_algo algo) { +cloudsync_table_context *table_create (cloudsync_context *data, const char *name, table_algo algo) { DEBUG_DBFUNCTION("table_create %s", name); cloudsync_table_context *table = (cloudsync_table_context *)cloudsync_memory_zeroalloc(sizeof(cloudsync_table_context)); if (!table) return NULL; + table->context = data; table->algo = algo; - table->name = cloudsync_string_dup(name, true); + table->name = cloudsync_string_dup_lowercase(name); + + // Detect schema from metadata table location. If metadata table doesn't + // exist yet (during initialization), fall back to cloudsync_schema() which + // returns the explicitly set schema or current_schema(). + table->schema = database_table_schema(name); + if (!table->schema) { + const char *fallback_schema = cloudsync_schema(data); + if (fallback_schema) { + table->schema = cloudsync_string_dup(fallback_schema); + } + } + if (!table->name) { cloudsync_memory_free(table); return NULL; } + table->meta_ref = database_build_meta_ref(table->schema, table->name); + table->base_ref = database_build_base_ref(table->schema, table->name); table->enabled = true; return table; @@ -637,55 +729,72 @@ void table_free (cloudsync_table_context *table) { DEBUG_DBFUNCTION("table_free %s", (table) ? (table->name) : "NULL"); if (!table) return; - if (table->ncols > 0) { - if (table->col_name) { - for (int i=0; incols; ++i) { - cloudsync_memory_free(table->col_name[i]); - } - cloudsync_memory_free(table->col_name); + if (table->col_name) { + for (int i=0; incols; ++i) { + cloudsync_memory_free(table->col_name[i]); } - if (table->col_merge_stmt) { - for (int i=0; incols; ++i) { - sqlite3_finalize(table->col_merge_stmt[i]); - } - cloudsync_memory_free(table->col_merge_stmt); + cloudsync_memory_free(table->col_name); + } + if (table->col_merge_stmt) { + for (int i=0; incols; ++i) { + databasevm_finalize(table->col_merge_stmt[i]); } - if (table->col_value_stmt) { - for (int i=0; incols; ++i) { - sqlite3_finalize(table->col_value_stmt[i]); - } - cloudsync_memory_free(table->col_value_stmt); + cloudsync_memory_free(table->col_merge_stmt); + } + if (table->col_value_stmt) { + for (int i=0; incols; ++i) { + databasevm_finalize(table->col_value_stmt[i]); } - if (table->col_id) { - cloudsync_memory_free(table->col_id); + cloudsync_memory_free(table->col_value_stmt); + } + if (table->col_id) { + cloudsync_memory_free(table->col_id); + } + if (table->col_algo) { + cloudsync_memory_free(table->col_algo); + } + if (table->col_delimiter) { + for (int i=0; incols; ++i) { + if (table->col_delimiter[i]) cloudsync_memory_free(table->col_delimiter[i]); } + cloudsync_memory_free(table->col_delimiter); } - - if (table->pk_name) sqlite3_free_table(table->pk_name); + + if (table->block_value_read_stmt) databasevm_finalize(table->block_value_read_stmt); + if (table->block_value_write_stmt) databasevm_finalize(table->block_value_write_stmt); + if (table->block_value_delete_stmt) databasevm_finalize(table->block_value_delete_stmt); + if (table->block_list_stmt) databasevm_finalize(table->block_list_stmt); + if (table->blocks_ref) cloudsync_memory_free(table->blocks_ref); + if (table->name) cloudsync_memory_free(table->name); - if (table->meta_pkexists_stmt) sqlite3_finalize(table->meta_pkexists_stmt); - if (table->meta_sentinel_update_stmt) sqlite3_finalize(table->meta_sentinel_update_stmt); - if (table->meta_sentinel_insert_stmt) sqlite3_finalize(table->meta_sentinel_insert_stmt); - if (table->meta_row_insert_update_stmt) sqlite3_finalize(table->meta_row_insert_update_stmt); - if (table->meta_row_drop_stmt) sqlite3_finalize(table->meta_row_drop_stmt); - if (table->meta_update_move_stmt) sqlite3_finalize(table->meta_update_move_stmt); - if (table->meta_local_cl_stmt) sqlite3_finalize(table->meta_local_cl_stmt); - if (table->meta_winner_clock_stmt) sqlite3_finalize(table->meta_winner_clock_stmt); - if (table->meta_merge_delete_drop) sqlite3_finalize(table->meta_merge_delete_drop); - if (table->meta_zero_clock_stmt) sqlite3_finalize(table->meta_zero_clock_stmt); - if (table->meta_col_version_stmt) sqlite3_finalize(table->meta_col_version_stmt); - if (table->meta_site_id_stmt) sqlite3_finalize(table->meta_site_id_stmt); - - if (table->real_col_values_stmt) sqlite3_finalize(table->real_col_values_stmt); - if (table->real_merge_delete_stmt) sqlite3_finalize(table->real_merge_delete_stmt); - if (table->real_merge_sentinel_stmt) sqlite3_finalize(table->real_merge_sentinel_stmt); + if (table->schema) cloudsync_memory_free(table->schema); + if (table->meta_ref) cloudsync_memory_free(table->meta_ref); + if (table->base_ref) cloudsync_memory_free(table->base_ref); + if (table->pk_name) table_pknames_free(table->pk_name, table->npks); + if (table->meta_pkexists_stmt) databasevm_finalize(table->meta_pkexists_stmt); + if (table->meta_sentinel_update_stmt) databasevm_finalize(table->meta_sentinel_update_stmt); + if (table->meta_sentinel_insert_stmt) databasevm_finalize(table->meta_sentinel_insert_stmt); + if (table->meta_row_insert_update_stmt) databasevm_finalize(table->meta_row_insert_update_stmt); + if (table->meta_row_drop_stmt) databasevm_finalize(table->meta_row_drop_stmt); + if (table->meta_update_move_stmt) databasevm_finalize(table->meta_update_move_stmt); + if (table->meta_local_cl_stmt) databasevm_finalize(table->meta_local_cl_stmt); + if (table->meta_winner_clock_stmt) databasevm_finalize(table->meta_winner_clock_stmt); + if (table->meta_merge_delete_drop) databasevm_finalize(table->meta_merge_delete_drop); + if (table->meta_zero_clock_stmt) databasevm_finalize(table->meta_zero_clock_stmt); + if (table->meta_col_version_stmt) databasevm_finalize(table->meta_col_version_stmt); + if (table->meta_site_id_stmt) databasevm_finalize(table->meta_site_id_stmt); + + if (table->real_col_values_stmt) databasevm_finalize(table->real_col_values_stmt); + if (table->real_merge_delete_stmt) databasevm_finalize(table->real_merge_delete_stmt); + if (table->real_merge_sentinel_stmt) databasevm_finalize(table->real_merge_sentinel_stmt); cloudsync_memory_free(table); } -int table_add_stmts (sqlite3 *db, cloudsync_table_context *table, int ncols) { - int rc = SQLITE_OK; +int table_add_stmts (cloudsync_table_context *table, int ncols) { + int rc = DBRES_OK; char *sql = NULL; + cloudsync_context *data = table->context; // META TABLE statements @@ -694,162 +803,160 @@ int table_add_stmts (sqlite3 *db, cloudsync_table_context *table, int ncols) { // precompile the pk exists statement // we do not need an index on the pk column because it is already covered by the fact that it is part of the prikeys // EXPLAIN QUERY PLAN reports: SEARCH table_name USING PRIMARY KEY (pk=?) - sql = cloudsync_memory_mprintf("SELECT EXISTS(SELECT 1 FROM \"%w_cloudsync\" WHERE pk = ? LIMIT 1);", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_ROW_EXISTS_BY_PK, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_pkexists_stmt: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_pkexists_stmt, NULL); - + + rc = databasevm_prepare(data, sql, (void **)&table->meta_pkexists_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; - + if (rc != DBRES_OK) goto cleanup; + // precompile the update local sentinel statement - sql = cloudsync_memory_mprintf("UPDATE \"%w_cloudsync\" SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, db_version = ?, seq = ?, site_id = 0 WHERE pk = ? AND col_name = '%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_sentinel_update_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_sentinel_update_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_sentinel_update_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the insert local sentinel statement - sql = cloudsync_memory_mprintf("INSERT INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id) SELECT ?, '%s', 1, ?, ?, 0 WHERE 1 ON CONFLICT DO UPDATE SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, db_version = ?, seq = ?, site_id = 0;", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, table->meta_ref, table->meta_ref, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_sentinel_insert_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_sentinel_insert_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_sentinel_insert_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the insert/update local row statement - sql = cloudsync_memory_mprintf("INSERT INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id ) SELECT ?, ?, ?, ?, ?, 0 WHERE 1 ON CONFLICT DO UPDATE SET col_version = col_version + 1, db_version = ?, seq = ?, site_id = 0;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION, table->meta_ref, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_row_insert_update_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_row_insert_update_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_row_insert_update_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the delete rows from meta - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_row_drop_stmt: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_row_drop_stmt, NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->meta_row_drop_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // precompile the update rows from meta when pk changes // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details - sql = cloudsync_memory_mprintf("UPDATE OR REPLACE \"%w_cloudsync\" SET pk=?, db_version=?, col_version=1, seq=cloudsync_seq(), site_id=0 WHERE (pk=? AND col_name!='%s');", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = sql_build_rekey_pk_and_reset_version_except_col(data, table->name, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_update_move_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_update_move_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_update_move_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // local cl - sql = cloudsync_memory_mprintf("SELECT COALESCE((SELECT col_version FROM \"%w_cloudsync\" WHERE pk=? AND col_name='%s'), (SELECT 1 FROM \"%w_cloudsync\" WHERE pk=?));", table->name, CLOUDSYNC_TOMBSTONE_VALUE, table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_local_cl_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_local_cl_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_local_cl_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // rowid of the last inserted/updated row in the meta table - sql = cloudsync_memory_mprintf("INSERT OR REPLACE INTO \"%w_cloudsync\" (pk, col_name, col_version, db_version, seq, site_id) VALUES (?, ?, ?, cloudsync_db_version_next(?), ?, ?) RETURNING ((db_version << 30) | seq);", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_winner_clock_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_winner_clock_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_winner_clock_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_merge_delete_drop: %s", sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_merge_delete_drop, NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->meta_merge_delete_drop, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // zero clock - sql = cloudsync_memory_mprintf("UPDATE \"%w_cloudsync\" SET col_version = 0, db_version = cloudsync_db_version_next(?) WHERE pk=? AND col_name!='%s';", table->name, CLOUDSYNC_TOMBSTONE_VALUE); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_zero_clock_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_zero_clock_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_zero_clock_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // col_version - sql = cloudsync_memory_mprintf("SELECT col_version FROM \"%w_cloudsync\" WHERE pk=? AND col_name=?;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_col_version_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_col_version_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_col_version_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // site_id - sql = cloudsync_memory_mprintf("SELECT site_id FROM \"%w_cloudsync\" WHERE pk=? AND col_name=?;", table->name); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL, table->meta_ref); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("meta_site_id_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->meta_site_id_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->meta_site_id_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; // REAL TABLE statements - + // precompile the get column value statement if (ncols > 0) { - sql = table_build_values_sql(db, table); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = sql_build_select_nonpk_by_pk(data, table->name, table->schema); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_col_values_stmt: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_col_values_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_col_values_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; } - sql = table_build_mergedelete_sql(db, table); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = table_build_mergedelete_sql(table); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_merge_delete: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_merge_delete_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_merge_delete_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; - sql = table_build_mergeinsert_sql(db, table, NULL); - if (!sql) {rc = SQLITE_NOMEM; goto cleanup;} + sql = table_build_mergeinsert_sql(table, NULL); + if (!sql) {rc = DBRES_NOMEM; goto cleanup;} DEBUG_SQL("real_merge_sentinel: %s", sql); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->real_merge_sentinel_stmt, NULL); + rc = databasevm_prepare(data, sql, (void **)&table->real_merge_sentinel_stmt, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto cleanup; + if (rc != DBRES_OK) goto cleanup; cleanup: - if (rc != SQLITE_OK) printf("table_add_stmts error: %s\n", sqlite3_errmsg(db)); + if (rc != DBRES_OK) DEBUG_ALWAYS("table_add_stmts error: %d %s\n", rc, database_errmsg(data)); return rc; } cloudsync_table_context *table_lookup (cloudsync_context *data, const char *table_name) { DEBUG_DBFUNCTION("table_lookup %s", table_name); - for (int i=0; itables_count; ++i) { - const char *name = (data->tables[i]) ? data->tables[i]->name : NULL; - if ((name) && (strcasecmp(name, table_name) == 0)) { - return data->tables[i]; + if (table_name) { + for (int i=0; itables_count; ++i) { + if ((strcasecmp(data->tables[i]->name, table_name) == 0)) return data->tables[i]; } } return NULL; } -sqlite3_stmt *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index) { +void *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index) { DEBUG_DBFUNCTION("table_column_lookup %s", col_name); for (int i=0; incols; ++i) { @@ -863,145 +970,152 @@ sqlite3_stmt *table_column_lookup (cloudsync_table_context *table, const char *c return NULL; } -int table_remove (cloudsync_context *data, const char *table_name) { +int table_remove (cloudsync_context *data, cloudsync_table_context *table) { + const char *table_name = table->name; DEBUG_DBFUNCTION("table_remove %s", table_name); - for (int i=0; itables_count; ++i) { - const char *name = (data->tables[i]) ? data->tables[i]->name : NULL; - if ((name) && (strcasecmp(name, table_name) == 0)) { - data->tables[i] = NULL; - return i; + for (int i = 0; i < data->tables_count; ++i) { + cloudsync_table_context *t = data->tables[i]; + + // pointer compare is fastest but fallback to strcasecmp if not same pointer + if ((t == table) || ((strcasecmp(t->name, table_name) == 0))) { + int last = data->tables_count - 1; + data->tables[i] = data->tables[last]; // move last into the hole (keeps array dense) + data->tables[last] = NULL; // NULLify tail (as an extra security measure) + data->tables_count--; + return data->tables_count; } } + return -1; } int table_add_to_context_cb (void *xdata, int ncols, char **values, char **names) { cloudsync_table_context *table = (cloudsync_table_context *)xdata; - - sqlite3 *db = sqlite3_db_handle(table->meta_pkexists_stmt); - if (!db) return SQLITE_ERROR; - + cloudsync_context *data = table->context; + int index = table->ncols; for (int i=0; icol_id[index] = cid; - table->col_name[index] = cloudsync_string_dup(name, true); - if (!table->col_name[index]) return 1; - - char *sql = table_build_mergeinsert_sql(db, table, name); - if (!sql) return SQLITE_NOMEM; + table->col_name[index] = cloudsync_string_dup_lowercase(name); + if (!table->col_name[index]) goto error; + + char *sql = table_build_mergeinsert_sql(table, name); + if (!sql) goto error; DEBUG_SQL("col_merge_stmt[%d]: %s", index, sql); - - int rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->col_merge_stmt[index], NULL); + + int rc = databasevm_prepare(data, sql, (void **)&table->col_merge_stmt[index], DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) return rc; - if (!table->col_merge_stmt[index]) return SQLITE_MISUSE; - - sql = table_build_value_sql(db, table, name); - if (!sql) return SQLITE_NOMEM; + if (rc != DBRES_OK) goto error; + if (!table->col_merge_stmt[index]) goto error; + + sql = table_build_value_sql(table, name); + if (!sql) goto error; DEBUG_SQL("col_value_stmt[%d]: %s", index, sql); - - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &table->col_value_stmt[index], NULL); + + rc = databasevm_prepare(data, sql, (void **)&table->col_value_stmt[index], DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) return rc; - if (!table->col_value_stmt[index]) return SQLITE_MISUSE; + if (rc != DBRES_OK) goto error; + if (!table->col_value_stmt[index]) goto error; } table->ncols += 1; - + return 0; + +error: + // clean up partially-initialized entry at index + if (table->col_name[index]) {cloudsync_memory_free(table->col_name[index]); table->col_name[index] = NULL;} + if (table->col_merge_stmt[index]) {databasevm_finalize(table->col_merge_stmt[index]); table->col_merge_stmt[index] = NULL;} + if (table->col_value_stmt[index]) {databasevm_finalize(table->col_value_stmt[index]); table->col_value_stmt[index] = NULL;} + return 1; } -bool table_add_to_context (sqlite3 *db, cloudsync_context *data, table_algo algo, const char *table_name) { - DEBUG_DBFUNCTION("cloudsync_context_add_table %s", table_name); +bool table_ensure_capacity (cloudsync_context *data) { + if (data->tables_count < data->tables_cap) return true; - // check if table is already in the global context and in that case just return + int new_cap = data->tables_cap ? data->tables_cap * 2 : CLOUDSYNC_INIT_NTABLES; + size_t bytes = (size_t)new_cap * sizeof(*data->tables); + void *p = cloudsync_memory_realloc(data->tables, bytes); + if (!p) return false; + + data->tables = (cloudsync_table_context **)p; + data->tables_cap = new_cap; + return true; +} + +bool table_add_to_context (cloudsync_context *data, table_algo algo, const char *table_name) { + DEBUG_DBFUNCTION("cloudsync_context_add_table %s", table_name); + + // Check if table already initialized in this connection's context. + // Note: This prevents same-connection duplicate initialization. + // SQLite clients cannot distinguish schemas, so having 'public.users' + // and 'auth.users' would cause sync ambiguity. Users should avoid + // initializing tables with the same name in different schemas. + // If two concurrent connections initialize tables with the same name + // in different schemas, the behavior is undefined. cloudsync_table_context *table = table_lookup(data, table_name); if (table) return true; - // is there any space available? - if (data->tables_alloc <= data->tables_count + 1) { - // realloc tables - cloudsync_table_context **clone = (cloudsync_table_context **)cloudsync_memory_realloc(data->tables, sizeof(cloudsync_table_context) * data->tables_alloc + CLOUDSYNC_INIT_NTABLES); - if (!clone) goto abort_add_table; - - // reset new entries - for (int i=data->tables_alloc; itables_alloc + CLOUDSYNC_INIT_NTABLES; ++i) { - clone[i] = NULL; - } - - // replace old ptr - data->tables = clone; - data->tables_alloc += CLOUDSYNC_INIT_NTABLES; - } + // check for space availability + if (!table_ensure_capacity(data)) return false; - // setup a new table context - table = table_create(table_name, algo); + // setup a new table + table = table_create(data, table_name, algo); if (!table) return false; // fill remaining metadata in the table - char *sql = cloudsync_memory_mprintf("SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", table_name); - if (!sql) goto abort_add_table; - table->npks = (int)dbutils_int_select(db, sql); - cloudsync_memory_free(sql); - if (table->npks == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - goto abort_add_table; - } - + int count = database_count_pk(data, table_name, false, table->schema); + if (count < 0) {cloudsync_set_dberror(data); goto abort_add_table;} + table->npks = count; if (table->npks == 0) { #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - return false; + goto abort_add_table; #else table->rowid_only = true; table->npks = 1; // rowid #endif } - sql = cloudsync_memory_mprintf("SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0;", table_name); - if (!sql) goto abort_add_table; - int64_t ncols = (int64_t)dbutils_int_select(db, sql); - cloudsync_memory_free(sql); - if (ncols == -1) { - dbutils_context_result_error(data->sqlite_ctx, "%s", sqlite3_errmsg(db)); - goto abort_add_table; - } - - int rc = table_add_stmts(db, table, (int)ncols); - if (rc != SQLITE_OK) goto abort_add_table; + int ncols = database_count_nonpk(data, table_name, table->schema); + if (ncols < 0) {cloudsync_set_dberror(data); goto abort_add_table;} + int rc = table_add_stmts(table, ncols); + if (rc != DBRES_OK) goto abort_add_table; // a table with only pk(s) is totally legal if (ncols > 0) { - table->col_name = (char **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(char *) * ncols)); + table->col_name = (char **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(char *) * ncols)); if (!table->col_name) goto abort_add_table; - - table->col_id = (int *)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(int) * ncols)); + + table->col_id = (int *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(int) * ncols)); if (!table->col_id) goto abort_add_table; - - table->col_merge_stmt = (sqlite3_stmt **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(sqlite3_stmt *) * ncols)); + + table->col_merge_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_merge_stmt) goto abort_add_table; - - table->col_value_stmt = (sqlite3_stmt **)cloudsync_memory_alloc((sqlite3_uint64)(sizeof(sqlite3_stmt *) * ncols)); + + table->col_value_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_value_stmt) goto abort_add_table; - - sql = cloudsync_memory_mprintf("SELECT name, cid FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", table_name); - if (!sql) goto abort_add_table; - int rc = sqlite3_exec(db, sql, table_add_to_context_cb, (void *)table, NULL); - cloudsync_memory_free(sql); - if (rc == SQLITE_ABORT) goto abort_add_table; - } - - // lookup the first free slot - for (int i=0; itables_alloc; ++i) { - if (data->tables[i] == NULL) { - data->tables[i] = table; - if (i > data->tables_count - 1) ++data->tables_count; - break; - } + + table->col_algo = (col_algo_t *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(col_algo_t) * ncols)); + if (!table->col_algo) goto abort_add_table; + + table->col_delimiter = (char **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(char *) * ncols)); + if (!table->col_delimiter) goto abort_add_table; + + // Pass empty string when schema is NULL; SQL will fall back to current_schema() + const char *schema = table->schema ? table->schema : ""; + char *sql = cloudsync_memory_mprintf(SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID, + table_name, schema, table_name, schema); + if (!sql) goto abort_add_table; + rc = database_exec_callback(data, sql, table_add_to_context_cb, (void *)table); + cloudsync_memory_free(sql); + if (rc == DBRES_ABORT) goto abort_add_table; } + // append newly created table + data->tables[data->tables_count++] = table; return true; abort_add_table: @@ -1009,13 +1123,10 @@ bool table_add_to_context (sqlite3 *db, cloudsync_context *data, table_algo algo return false; } -bool table_remove_from_context (cloudsync_context *data, cloudsync_table_context *table) { - return (table_remove(data, table->name) != -1); -} +dbvm_t *cloudsync_colvalue_stmt (cloudsync_context *data, const char *tbl_name, bool *persistent) { + dbvm_t *vm = NULL; + *persistent = false; -sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, const char *tbl_name, bool *persistent) { - sqlite3_stmt *vm = NULL; - cloudsync_table_context *table = table_lookup(data, tbl_name); if (table) { char *col_name = NULL; @@ -1025,8 +1136,8 @@ sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, con vm = table_column_lookup(table, col_name, false, NULL); *persistent = true; } else { - char *sql = table_build_value_sql(db, table, "*"); - sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + char *sql = table_build_value_sql(table, "*"); + databasevm_prepare(data, sql, (void **)&vm, 0); cloudsync_memory_free(sql); *persistent = false; } @@ -1035,124 +1146,400 @@ sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, con return vm; } +bool table_enabled (cloudsync_table_context *table) { + return table->enabled; +} + +void table_set_enabled (cloudsync_table_context *table, bool value) { + table->enabled = value; +} + +int table_count_cols (cloudsync_table_context *table) { + return table->ncols; +} + +int table_count_pks (cloudsync_table_context *table) { + return table->npks; +} + +const char *table_colname (cloudsync_table_context *table, int index) { + return table->col_name[index]; +} + +bool table_pk_exists (cloudsync_table_context *table, const char *value, size_t len) { + // check if a row with the same primary key already exists + // if so, this means the row might have been previously deleted (sentinel) + return (dbvm_count(table->meta_pkexists_stmt, value, len, DBTYPE_BLOB) > 0); +} + +char **table_pknames (cloudsync_table_context *table) { + return table->pk_name; +} + +void table_set_pknames (cloudsync_table_context *table, char **pknames) { + table_pknames_free(table->pk_name, table->npks); + table->pk_name = pknames; +} + +bool table_algo_isgos (cloudsync_table_context *table) { + return (table->algo == table_algo_crdt_gos); +} + +const char *table_schema (cloudsync_table_context *table) { + return table->schema; +} + // MARK: - Merge Insert - -sqlite3_int64 merge_get_local_cl (cloudsync_table_context *table, const char *pk, int pklen, const char **err) { - sqlite3_stmt *vm = table->meta_local_cl_stmt; - sqlite3_int64 result = -1; +int64_t merge_get_local_cl (cloudsync_table_context *table, const char *pk, int pklen) { + dbvm_t *vm = table->meta_local_cl_stmt; + int64_t result = -1; - int rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 2, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 2, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) result = sqlite3_column_int64(vm, 0); - else if (rc == SQLITE_DONE) result = 0; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) result = database_column_int(vm, 0); + else if (rc == DBRES_DONE) result = 0; cleanup: - if (result == -1) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (result == -1) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return result; } -int merge_get_col_version (cloudsync_table_context *table, const char *col_name, const char *pk, int pklen, sqlite3_int64 *version, const char **err) { - sqlite3_stmt *vm = table->meta_col_version_stmt; +int merge_get_col_version (cloudsync_table_context *table, const char *col_name, const char *pk, int pklen, int64_t *version) { + dbvm_t *vm = table->meta_col_version_stmt; - int rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_text(vm, 2, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - *version = sqlite3_column_int64(vm, 0); - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + *version = database_column_int(vm, 0); + rc = DBRES_OK; } cleanup: - if ((rc != SQLITE_OK) && (rc != SQLITE_DONE)) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if ((rc != DBRES_OK) && (rc != DBRES_DONE)) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return rc; } -int merge_set_winner_clock (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pk_len, const char *colname, sqlite3_int64 col_version, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { +int merge_set_winner_clock (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pk_len, const char *colname, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { // get/set site_id - sqlite3_stmt *vm = data->getset_siteid_stmt; - int rc = sqlite3_bind_blob(vm, 1, (const void *)site_id, site_len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + dbvm_t *vm = data->getset_siteid_stmt; + int rc = databasevm_bind_blob(vm, 1, (const void *)site_id, site_len); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_step(vm); - if (rc != SQLITE_ROW) goto cleanup_merge; + rc = databasevm_step(vm); + if (rc != DBRES_ROW) goto cleanup_merge; - int64_t ord = sqlite3_column_int64(vm, 0); - stmt_reset(vm); + int64_t ord = database_column_int(vm, 0); + dbvm_reset(vm); vm = table->meta_winner_clock_stmt; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pk_len, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pk_len); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_text(vm, 2, (colname) ? colname : CLOUDSYNC_TOMBSTONE_VALUE, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_text(vm, 2, (colname) ? colname : CLOUDSYNC_TOMBSTONE_VALUE, -1); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 3, col_version); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 3, col_version); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_bind_int64(vm, 6, ord); - if (rc != SQLITE_OK) goto cleanup_merge; + rc = databasevm_bind_int(vm, 6, ord); + if (rc != DBRES_OK) goto cleanup_merge; - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - *rowid = sqlite3_column_int64(vm, 0); - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + *rowid = database_column_int(vm, 0); + rc = DBRES_OK; } cleanup_merge: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } -int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *col_name, sqlite3_value *col_value, sqlite3_int64 col_version, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - int index; - sqlite3_stmt *vm = table_column_lookup(table, col_name, true, &index); - if (vm == NULL) { - *err = "Unable to retrieve column merge precompiled statement in merge_insert_col."; - return SQLITE_MISUSE; +// MARK: - Deferred column-batch merge functions - + +static int merge_pending_add (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq) { + merge_pending_batch *batch = data->pending_batch; + + // Store table and PK on first entry + if (batch->table == NULL) { + batch->table = table; + batch->pk = (char *)cloudsync_memory_alloc(pklen); + if (!batch->pk) return cloudsync_set_error(data, "merge_pending_add: out of memory for pk", DBRES_NOMEM); + memcpy(batch->pk, pk, pklen); + batch->pk_len = pklen; + } + + // Ensure capacity + if (batch->count >= batch->capacity) { + int new_cap = batch->capacity ? batch->capacity * 2 : 8; + merge_pending_entry *new_entries = (merge_pending_entry *)cloudsync_memory_realloc(batch->entries, new_cap * sizeof(merge_pending_entry)); + if (!new_entries) return cloudsync_set_error(data, "merge_pending_add: out of memory for entries", DBRES_NOMEM); + batch->entries = new_entries; + batch->capacity = new_cap; + } + + // Resolve col_name to a stable pointer from the table context + // (the incoming col_name may point to VM-owned memory that gets freed on reset) + int col_idx = -1; + table_column_lookup(table, col_name, true, &col_idx); + const char *stable_col_name = (col_idx >= 0) ? table_colname(table, col_idx) : NULL; + if (!stable_col_name) return cloudsync_set_error(data, "merge_pending_add: column not found in table context", DBRES_ERROR); + + merge_pending_entry *e = &batch->entries[batch->count]; + e->col_name = stable_col_name; + e->col_value = col_value ? (dbvalue_t *)database_value_dup(col_value) : NULL; + e->col_version = col_version; + e->db_version = db_version; + e->site_id_len = (site_len <= (int)sizeof(e->site_id)) ? site_len : (int)sizeof(e->site_id); + memcpy(e->site_id, site_id, e->site_id_len); + e->seq = seq; + + batch->count++; + return DBRES_OK; +} + +static void merge_pending_free_entries (merge_pending_batch *batch) { + if (batch->entries) { + for (int i = 0; i < batch->count; i++) { + if (batch->entries[i].col_value) { + database_value_free(batch->entries[i].col_value); + batch->entries[i].col_value = NULL; + } + } + } + if (batch->pk) { + cloudsync_memory_free(batch->pk); + batch->pk = NULL; + } + batch->table = NULL; + batch->pk_len = 0; + batch->cl = 0; + batch->sentinel_pending = false; + batch->row_exists = false; + batch->count = 0; +} + +static int merge_flush_pending (cloudsync_context *data) { + merge_pending_batch *batch = data->pending_batch; + if (!batch) return DBRES_OK; + + int rc = DBRES_OK; + bool flush_savepoint = false; + + // Nothing to write — handle sentinel-only case or skip + if (batch->count == 0 && !(batch->sentinel_pending && batch->table)) { + goto cleanup; + } + + // Wrap database operations in a savepoint so that on failure (e.g. RLS + // denial) the rollback properly releases all executor resources (open + // relations, snapshots, plan cache) acquired during the failed statement. + flush_savepoint = (database_begin_savepoint(data, "merge_flush") == DBRES_OK); + + if (batch->count == 0) { + // Sentinel with no winning columns (PK-only row) + dbvm_t *vm = batch->table->real_merge_sentinel_stmt; + rc = pk_decode_prikey(batch->pk, (size_t)batch->pk_len, pk_decode_bind_callback, vm); + if (rc < 0) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + goto cleanup; + } + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + goto cleanup; + } + goto cleanup; + } + + // Check if cached prepared statement can be reused + cloudsync_table_context *table = batch->table; + dbvm_t *vm = NULL; + bool cache_hit = false; + + if (batch->cached_vm && + batch->cached_row_exists == batch->row_exists && + batch->cached_col_count == batch->count) { + cache_hit = true; + for (int i = 0; i < batch->count; i++) { + if (batch->cached_col_names[i] != batch->entries[i].col_name) { + cache_hit = false; + break; + } + } + } + + if (cache_hit) { + vm = batch->cached_vm; + dbvm_reset(vm); + } else { + // Invalidate old cache + if (batch->cached_vm) { + databasevm_finalize(batch->cached_vm); + batch->cached_vm = NULL; + } + + // Build multi-column SQL + const char **colnames = (const char **)cloudsync_memory_alloc(batch->count * sizeof(const char *)); + if (!colnames) { + rc = cloudsync_set_error(data, "merge_flush_pending: out of memory", DBRES_NOMEM); + goto cleanup; + } + for (int i = 0; i < batch->count; i++) { + colnames[i] = batch->entries[i].col_name; + } + + char *sql = batch->row_exists + ? sql_build_update_pk_and_multi_cols(data, table->name, colnames, batch->count, table->schema) + : sql_build_upsert_pk_and_multi_cols(data, table->name, colnames, batch->count, table->schema); + cloudsync_memory_free(colnames); + + if (!sql) { + rc = cloudsync_set_error(data, "merge_flush_pending: unable to build multi-column upsert SQL", DBRES_ERROR); + goto cleanup; + } + + rc = databasevm_prepare(data, sql, &vm, 0); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) { + rc = cloudsync_set_error(data, "merge_flush_pending: unable to prepare statement", rc); + goto cleanup; + } + + // Update cache + batch->cached_vm = vm; + batch->cached_row_exists = batch->row_exists; + batch->cached_col_count = batch->count; + // Reallocate cached_col_names if needed + if (batch->cached_col_count > 0) { + const char **new_names = (const char **)cloudsync_memory_realloc( + batch->cached_col_names, batch->count * sizeof(const char *)); + if (new_names) { + for (int i = 0; i < batch->count; i++) { + new_names[i] = batch->entries[i].col_name; + } + batch->cached_col_names = new_names; + } + } + } + + // Bind PKs (positions 1..npks) + int npks = pk_decode_prikey(batch->pk, (size_t)batch->pk_len, pk_decode_bind_callback, vm); + if (npks < 0) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + rc = DBRES_ERROR; + goto cleanup; + } + + // Bind column values (positions npks+1..npks+count) + for (int i = 0; i < batch->count; i++) { + merge_pending_entry *e = &batch->entries[i]; + int bind_idx = npks + 1 + i; + if (e->col_value) { + rc = databasevm_bind_value(vm, bind_idx, e->col_value); + } else { + rc = databasevm_bind_null(vm, bind_idx); + } + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + goto cleanup; + } + } + + // Execute with SYNCBIT and GOS handling + if (table->algo == table_algo_crdt_gos) table->enabled = 0; + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (table->algo == table_algo_crdt_gos) table->enabled = 1; + + if (rc != DBRES_DONE) { + cloudsync_set_dberror(data); + goto cleanup; + } + rc = DBRES_OK; + + // Call merge_set_winner_clock for each buffered entry + int64_t rowid = 0; + for (int i = 0; i < batch->count; i++) { + merge_pending_entry *e = &batch->entries[i]; + int clock_rc = merge_set_winner_clock(data, table, batch->pk, batch->pk_len, + e->col_name, e->col_version, e->db_version, + (const char *)e->site_id, e->site_id_len, + e->seq, &rowid); + if (clock_rc != DBRES_OK) { + rc = clock_rc; + goto cleanup; + } + } + +cleanup: + merge_pending_free_entries(batch); + if (flush_savepoint) { + if (rc == DBRES_OK) database_commit_savepoint(data, "merge_flush"); + else database_rollback_savepoint(data, "merge_flush"); } + return rc; +} + +int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + int index; + dbvm_t *vm = table_column_lookup(table, col_name, true, &index); + if (vm == NULL) return cloudsync_set_error(data, "Unable to retrieve column merge precompiled statement in merge_insert_col", DBRES_MISUSE); // INSERT INTO table (pk1, pk2, col_name) VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET col_name=?;" // bind primary key(s) int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); + cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } - // bind value + // bind value (always bind all expected parameters for correct prepared statement handling) if (col_value) { - rc = sqlite3_bind_value(vm, table->npks+1, col_value); - if (rc == SQLITE_OK) rc = sqlite3_bind_value(vm, table->npks+2, col_value); - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; - } - + rc = databasevm_bind_value(vm, table->npks+1, col_value); + if (rc == DBRES_OK) rc = databasevm_bind_value(vm, table->npks+2, col_value); + } else { + rc = databasevm_bind_null(vm, table->npks+1); + if (rc == DBRES_OK) rc = databasevm_bind_null(vm, table->npks+2); } - + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } + // perform real operation and disable triggers // in case of GOS we reused the table->col_merge_stmt statement @@ -1161,133 +1548,138 @@ int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, c // the trick is to disable that trigger before executing the statement if (table->algo == table_algo_crdt_gos) table->enabled = 0; SYNCBIT_SET(data); - rc = sqlite3_step(vm); - DEBUG_MERGE("merge_insert(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], sqlite3_expanded_sql(vm), rc); - stmt_reset(vm); + rc = databasevm_step(vm); + DEBUG_MERGE("merge_insert(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], databasevm_sql(vm), rc); + dbvm_reset(vm); SYNCBIT_RESET(data); if (table->algo == table_algo_crdt_gos) table->enabled = 1; - if (rc != SQLITE_DONE) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); + if (rc != DBRES_DONE) { + cloudsync_set_dberror(data); return rc; } - return merge_set_winner_clock(data, table, pk, pklen, col_name, col_version, db_version, site_id, site_len, seq, rowid, err); + return merge_set_winner_clock(data, table, pk, pklen, col_name, col_version, db_version, site_id, site_len, seq, rowid); } -int merge_delete (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *colname, sqlite3_int64 cl, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - int rc = SQLITE_OK; +int merge_delete (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, const char *colname, int64_t cl, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + int rc = DBRES_OK; // reset return value *rowid = 0; // bind pk - sqlite3_stmt *vm = table->real_merge_delete_stmt; + dbvm_t *vm = table->real_merge_delete_stmt; rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } // perform real operation and disable triggers SYNCBIT_SET(data); - rc = sqlite3_step(vm); - DEBUG_MERGE("merge_delete(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], sqlite3_expanded_sql(vm), rc); - stmt_reset(vm); + rc = databasevm_step(vm); + DEBUG_MERGE("merge_delete(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], databasevm_sql(vm), rc); + dbvm_reset(vm); SYNCBIT_RESET(data); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); return rc; } - rc = merge_set_winner_clock(data, table, pk, pklen, colname, cl, db_version, site_id, site_len, seq, rowid, err); - if (rc != SQLITE_OK) return rc; + rc = merge_set_winner_clock(data, table, pk, pklen, colname, cl, db_version, site_id, site_len, seq, rowid); + if (rc != DBRES_OK) return rc; // drop clocks _after_ setting the winner clock so we don't lose track of the max db_version!! // this must never come before `set_winner_clock` vm = table->meta_merge_delete_drop; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc == SQLITE_OK) rc = sqlite3_step(vm); - stmt_reset(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - } + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc == DBRES_OK) rc = databasevm_step(vm); + dbvm_reset(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) cloudsync_set_dberror(data); return rc; } -int merge_zeroclock_on_resurrect(cloudsync_table_context *table, sqlite3_int64 db_version, const char *pk, int pklen, const char **err) { - sqlite3_stmt *vm = table->meta_zero_clock_stmt; +int merge_zeroclock_on_resurrect(cloudsync_table_context *table, int64_t db_version, const char *pk, int pklen) { + dbvm_t *vm = table->meta_zero_clock_stmt; - int rc = sqlite3_bind_int64(vm, 1, db_version); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_int(vm, 1, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 2, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 2, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(table->context); + dbvm_reset(vm); return rc; } // executed only if insert_cl == local_cl -int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, sqlite3_value *insert_value, const char *site_id, int site_len, const char *col_name, sqlite3_int64 col_version, bool *didwin_flag, const char **err) { +int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, dbvalue_t *insert_value, const char *site_id, int site_len, const char *col_name, int64_t col_version, bool *didwin_flag) { if (col_name == NULL) col_name = CLOUDSYNC_TOMBSTONE_VALUE; - sqlite3_int64 local_version; - int rc = merge_get_col_version(table, col_name, pk, pklen, &local_version, err); - if (rc == SQLITE_DONE) { + int64_t local_version; + int rc = merge_get_col_version(table, col_name, pk, pklen, &local_version); + if (rc == DBRES_DONE) { // no rows returned, the incoming change wins if there's nothing there locally *didwin_flag = true; - return SQLITE_OK; + return DBRES_OK; } - if (rc != SQLITE_OK) return rc; + if (rc != DBRES_OK) return rc; - // rc == SQLITE_OK, means that a row with a version exists + // rc == DBRES_OK, means that a row with a version exists if (local_version != col_version) { - if (col_version > local_version) {*didwin_flag = true; return SQLITE_OK;} - if (col_version < local_version) {*didwin_flag = false; return SQLITE_OK;} + if (col_version > local_version) {*didwin_flag = true; return DBRES_OK;} + if (col_version < local_version) {*didwin_flag = false; return DBRES_OK;} } - // rc == SQLITE_ROW and col_version == local_version, need to compare values - + // rc == DBRES_ROW and col_version == local_version, need to compare values + // retrieve col_value precompiled statement - sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); - if (!vm) { - *err = "Unable to retrieve column value precompiled statement in merge_did_cid_win."; - return SQLITE_ERROR; - } - - // bind primary key values - rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, (void *)vm); - if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; + bool is_block_col = block_is_block_colname(col_name) && table_has_block_cols(table); + dbvm_t *vm; + if (is_block_col) { + // Block column: read value from blocks table (pk + col_name bindings) + vm = table_block_value_read_stmt(table); + if (!vm) return cloudsync_set_error(data, "Unable to retrieve block value read statement in merge_did_cid_win", DBRES_ERROR); + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) { dbvm_reset(vm); return cloudsync_set_dberror(data); } + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) { dbvm_reset(vm); return cloudsync_set_dberror(data); } + } else { + vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) return cloudsync_set_error(data, "Unable to retrieve column value precompiled statement in merge_did_cid_win", DBRES_ERROR); + + // bind primary key values + rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, (void *)vm); + if (rc < 0) { + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } } // execute vm - sqlite3_value *local_value; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) { + dbvalue_t *local_value; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) { // meta entry exists but the actual value is missing // we should allow the value_compare function to make a decision // value_compare has been modified to handle the case where lvalue is NULL local_value = NULL; - rc = SQLITE_OK; - } else if (rc == SQLITE_ROW) { - local_value = sqlite3_column_value(vm, 0); - rc = SQLITE_OK; + rc = DBRES_OK; + } else if (rc == DBRES_ROW) { + local_value = database_column_value(vm, 0); + rc = DBRES_OK; } else { goto cleanup; } @@ -1295,7 +1687,8 @@ int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, // compare values int ret = dbutils_value_compare(insert_value, local_value); // reset after compare, otherwise local value would be deallocated - vm = stmt_reset(vm); + dbvm_reset(vm); + vm = NULL; bool compare_site_id = (ret == 0 && data->merge_equal_values == true); if (!compare_site_id) { @@ -1305,125 +1698,269 @@ int merge_did_cid_win (cloudsync_context *data, cloudsync_table_context *table, // values are the same and merge_equal_values is true vm = table->meta_site_id_stmt; - rc = sqlite3_bind_blob(vm, 1, (const void *)pk, pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_bind_text(vm, 2, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - const void *local_site_id = sqlite3_column_blob(vm, 0); + rc = databasevm_bind_blob(vm, 1, (const void *)pk, pklen); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_bind_text(vm, 2, col_name, -1); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + const void *local_site_id = database_column_blob(vm, 0, NULL); + if (!local_site_id) { + dbvm_reset(vm); + return cloudsync_set_error(data, "NULL site_id in cloudsync table, table is probably corrupted", DBRES_ERROR); + } ret = memcmp(site_id, local_site_id, site_len); *didwin_flag = (ret > 0); - stmt_reset(vm); - return SQLITE_OK; + dbvm_reset(vm); + return DBRES_OK; } // handle error condition here - stmt_reset(vm); - *err = "Unable to find site_id for previous change. The cloudsync table is probably corrupted."; - return SQLITE_ERROR; + dbvm_reset(vm); + return cloudsync_set_error(data, "Unable to find site_id for previous change, cloudsync table is probably corrupted", DBRES_ERROR); cleanup: - if (rc != SQLITE_OK) *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - if (vm) stmt_reset(vm); + if (rc != DBRES_OK) cloudsync_set_dberror(data); + dbvm_reset(vm); return rc; } -int merge_sentinel_only_insert (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, sqlite3_int64 cl, sqlite3_int64 db_version, const char *site_id, int site_len, sqlite3_int64 seq, sqlite3_int64 *rowid, const char **err) { - +int merge_sentinel_only_insert (cloudsync_context *data, cloudsync_table_context *table, const char *pk, int pklen, int64_t cl, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { + // reset return value *rowid = 0; - - // bind pk - sqlite3_stmt *vm = table->real_merge_sentinel_stmt; - int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); - if (rc < 0) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - rc = sqlite3_errcode(sqlite3_db_handle(vm)); - stmt_reset(vm); - return rc; - } - - // perform real operation and disable triggers - SYNCBIT_SET(data); - rc = sqlite3_step(vm); - stmt_reset(vm); - SYNCBIT_RESET(data); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc != SQLITE_OK) { - *err = sqlite3_errmsg(sqlite3_db_handle(vm)); - return rc; + + if (data->pending_batch == NULL) { + // Immediate mode: execute base table INSERT + dbvm_t *vm = table->real_merge_sentinel_stmt; + int rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm); + if (rc < 0) { + rc = cloudsync_set_dberror(data); + dbvm_reset(vm); + return rc; + } + + SYNCBIT_SET(data); + rc = databasevm_step(vm); + dbvm_reset(vm); + SYNCBIT_RESET(data); + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) { + cloudsync_set_dberror(data); + return rc; + } + } else { + // Batch mode: skip base table INSERT, the batch flush will create the row + merge_pending_batch *batch = data->pending_batch; + batch->sentinel_pending = true; + if (batch->table == NULL) { + batch->table = table; + batch->pk = (char *)cloudsync_memory_alloc(pklen); + if (!batch->pk) return cloudsync_set_error(data, "merge_sentinel_only_insert: out of memory for pk", DBRES_NOMEM); + memcpy(batch->pk, pk, pklen); + batch->pk_len = pklen; + } } - - rc = merge_zeroclock_on_resurrect(table, db_version, pk, pklen, err); - if (rc != SQLITE_OK) return rc; - - return merge_set_winner_clock(data, table, pk, pklen, NULL, cl, db_version, site_id, site_len, seq, rowid, err); + + // Metadata operations always execute regardless of batch mode + int rc = merge_zeroclock_on_resurrect(table, db_version, pk, pklen); + if (rc != DBRES_OK) return rc; + + return merge_set_winner_clock(data, table, pk, pklen, NULL, cl, db_version, site_id, site_len, seq, rowid); } -int cloudsync_merge_insert_gos (sqlite3_vtab *vtab, cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, const char *insert_name, sqlite3_value *insert_value, sqlite3_int64 insert_col_version, sqlite3_int64 insert_db_version, const char *insert_site_id, int insert_site_id_len, sqlite3_int64 insert_seq, sqlite3_int64 *rowid) { - // Grow-Only Set (GOS) Algorithm: Only insertions are allowed, deletions and updates are prevented from a trigger. - - const char *err = NULL; - int rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, - insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform GOS merge_insert_col: %s", err); +// MARK: - Block-level merge helpers - + +// Store a block value in the blocks table +static int block_store_value (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *block_colname, dbvalue_t *col_value) { + dbvm_t *vm = table->block_value_write_stmt; + if (!vm) return cloudsync_set_error(data, "block_store_value: blocks table not initialized", DBRES_MISUSE); + + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, block_colname, -1); + if (rc != DBRES_OK) goto cleanup; + if (col_value) { + rc = databasevm_bind_value(vm, 3, col_value); + } else { + rc = databasevm_bind_null(vm, 3); } - + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + +cleanup: + if (rc != DBRES_OK) cloudsync_set_dberror(data); + databasevm_reset(vm); return rc; } -int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { - // this function performs the merging logic for an insert in a cloud-synchronized table. It handles - // different scenarios including conflicts, causal lengths, delete operations, and resurrecting rows - // based on the incoming data (from remote nodes or clients) and the local database state +// Delete a block value from the blocks table +static int block_delete_value (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *block_colname) { + dbvm_t *vm = table->block_value_delete_stmt; + if (!vm) return cloudsync_set_error(data, "block_delete_value: blocks table not initialized", DBRES_MISUSE); - // this function handles different CRDT algorithms (GOS, DWS, AWS, and CLS). - // the merging strategy is determined based on the table->algo value. - - // meta table declaration: - // tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL," - // "col_value ANY, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL," - // "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL - - // meta information to retrieve from arguments: - // argv[0] -> table name (TEXT) - // argv[1] -> primary key (BLOB) - // argv[2] -> column name (TEXT or NULL if sentinel) - // argv[3] -> column value (ANY) - // argv[4] -> column version (INTEGER) - // argv[5] -> database version (INTEGER) - // argv[6] -> site ID (BLOB, identifies the origin of the update) - // argv[7] -> causal length (INTEGER, tracks the order of operations) - // argv[8] -> sequence number (INTEGER, unique per operation) - - // extract table name - const char *insert_tbl = (const char *)sqlite3_value_text(argv[0]); - - // lookup table - cloudsync_context *data = cloudsync_vtab_get_context(vtab); - cloudsync_table_context *table = table_lookup(data, insert_tbl); - if (!table) return cloudsync_vtab_set_error(vtab, "Unable to find table %s,", insert_tbl); - - // extract the remaining fields from the input values - const char *insert_pk = (const char *)sqlite3_value_blob(argv[1]); - int insert_pk_len = sqlite3_value_bytes(argv[1]); - const char *insert_name = (sqlite3_value_type(argv[2]) == SQLITE_NULL) ? CLOUDSYNC_TOMBSTONE_VALUE : (const char *)sqlite3_value_text(argv[2]); - sqlite3_value *insert_value = argv[3]; - sqlite3_int64 insert_col_version = sqlite3_value_int64(argv[4]); - sqlite3_int64 insert_db_version = sqlite3_value_int64(argv[5]); - const char *insert_site_id = (const char *)sqlite3_value_blob(argv[6]); - int insert_site_id_len = sqlite3_value_bytes(argv[6]); - sqlite3_int64 insert_cl = sqlite3_value_int64(argv[7]); - sqlite3_int64 insert_seq = sqlite3_value_int64(argv[8]); - const char *err = NULL; - - // perform different logic for each different table algorithm - if (table->algo == table_algo_crdt_gos) return cloudsync_merge_insert_gos(vtab, data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); - + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, block_colname, -1); + if (rc != DBRES_OK) goto cleanup; + + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + +cleanup: + if (rc != DBRES_OK) cloudsync_set_dberror(data); + databasevm_reset(vm); + return rc; +} + +// Materialize all alive blocks for a base column into the base table +int block_materialize_column (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name) { + if (!table->block_list_stmt) return cloudsync_set_error(data, "block_materialize_column: blocks table not initialized", DBRES_MISUSE); + + // Find column index and delimiter + int col_idx = -1; + for (int i = 0; i < table->ncols; i++) { + if (strcasecmp(table->col_name[i], base_col_name) == 0) { + col_idx = i; + break; + } + } + if (col_idx < 0) return cloudsync_set_error(data, "block_materialize_column: column not found", DBRES_ERROR); + const char *delimiter = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; + + // Build the LIKE pattern for block col_names: "base_col\x1F%" + char *like_pattern = block_build_colname(base_col_name, "%"); + if (!like_pattern) return DBRES_NOMEM; + + // Query alive blocks from blocks table joined with metadata + // block_list_stmt: SELECT b.col_value FROM blocks b JOIN meta m + // ON b.pk = m.pk AND b.col_name = m.col_name + // WHERE b.pk = ? AND b.col_name LIKE ? AND m.col_version % 2 = 1 + // ORDER BY b.col_name + dbvm_t *vm = table->block_list_stmt; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + rc = databasevm_bind_text(vm, 2, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + // Bind pk again for the join condition (parameter 3) + rc = databasevm_bind_blob(vm, 3, pk, pklen); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + rc = databasevm_bind_text(vm, 4, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + + // Collect block values + const char **block_values = NULL; + int block_count = 0; + int block_cap = 0; + + while ((rc = databasevm_step(vm)) == DBRES_ROW) { + const char *value = database_column_text(vm, 0); + if (block_count >= block_cap) { + int new_cap = block_cap ? block_cap * 2 : 16; + const char **new_arr = (const char **)cloudsync_memory_realloc((void *)block_values, (uint64_t)(new_cap * sizeof(char *))); + if (!new_arr) { rc = DBRES_NOMEM; break; } + block_values = new_arr; + block_cap = new_cap; + } + block_values[block_count] = value ? cloudsync_string_dup(value) : cloudsync_string_dup(""); + block_count++; + } + databasevm_reset(vm); + cloudsync_memory_free(like_pattern); + + if (rc != DBRES_DONE && rc != DBRES_OK && rc != DBRES_ROW) { + // Free collected values + for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); + if (block_values) cloudsync_memory_free((void *)block_values); + return cloudsync_set_dberror(data); + } + + // Materialize text (NULL when no alive blocks) + char *text = (block_count > 0) ? block_materialize_text(block_values, block_count, delimiter) : NULL; + for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); + if (block_values) cloudsync_memory_free((void *)block_values); + if (block_count > 0 && !text) return DBRES_NOMEM; + + // Update the base table column via the col_merge_stmt (with triggers disabled) + dbvm_t *merge_vm = table->col_merge_stmt[col_idx]; + if (!merge_vm) { cloudsync_memory_free(text); return DBRES_ERROR; } + + // Bind PKs + rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, merge_vm); + if (rc < 0) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return DBRES_ERROR; } + + // Bind the text value twice (INSERT value + ON CONFLICT UPDATE value) + int npks = table->npks; + if (text) { + rc = databasevm_bind_text(merge_vm, npks + 1, text, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } + rc = databasevm_bind_text(merge_vm, npks + 2, text, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } + } else { + rc = databasevm_bind_null(merge_vm, npks + 1); + if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } + rc = databasevm_bind_null(merge_vm, npks + 2); + if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } + } + + // Execute with triggers disabled + table->enabled = 0; + SYNCBIT_SET(data); + rc = databasevm_step(merge_vm); + databasevm_reset(merge_vm); + SYNCBIT_RESET(data); + table->enabled = 1; + + cloudsync_memory_free(text); + + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc != DBRES_OK) return cloudsync_set_dberror(data); + return DBRES_OK; +} + +// Accessor for has_block_cols flag +bool table_has_block_cols (cloudsync_table_context *table) { + return table && table->has_block_cols; +} + +// Get block column algo for a given column index +col_algo_t table_col_algo (cloudsync_table_context *table, int index) { + if (!table || !table->col_algo || index < 0 || index >= table->ncols) return col_algo_normal; + return table->col_algo[index]; +} + +// Get block delimiter for a given column index +const char *table_col_delimiter (cloudsync_table_context *table, int index) { + if (!table || !table->col_delimiter || index < 0 || index >= table->ncols) return BLOCK_DEFAULT_DELIMITER; + return table->col_delimiter[index] ? table->col_delimiter[index] : BLOCK_DEFAULT_DELIMITER; +} + +// Block column struct accessors (for use outside cloudsync.c where struct is opaque) +dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table) { return table ? table->block_value_read_stmt : NULL; } +dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table) { return table ? table->block_value_write_stmt : NULL; } +dbvm_t *table_block_list_stmt (cloudsync_table_context *table) { return table ? table->block_list_stmt : NULL; } +const char *table_blocks_ref (cloudsync_table_context *table) { return table ? table->blocks_ref : NULL; } + +void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter) { + if (!table || !table->col_delimiter || col_idx < 0 || col_idx >= table->ncols) return; + if (table->col_delimiter[col_idx]) cloudsync_memory_free(table->col_delimiter[col_idx]); + table->col_delimiter[col_idx] = delimiter ? cloudsync_string_dup(delimiter) : NULL; +} + +// Find column index by name +int table_col_index (cloudsync_table_context *table, const char *col_name) { + if (!table || !col_name) return -1; + for (int i = 0; i < table->ncols; i++) { + if (strcasecmp(table->col_name[i], col_name) == 0) return i; + } + return -1; +} + +int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, int64_t insert_cl, const char *insert_name, dbvalue_t *insert_value, int64_t insert_col_version, int64_t insert_db_version, const char *insert_site_id, int insert_site_id_len, int64_t insert_seq, int64_t *rowid) { // Handle DWS and AWS algorithms here // Delete-Wins Set (DWS): table_algo_crdt_dws // Add-Wins Set (AWS): table_algo_crdt_aws @@ -1432,14 +1969,12 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, // compute the local causal length for the row based on the primary key // the causal length is used to determine the order of operations and resolve conflicts. - sqlite3_int64 local_cl = merge_get_local_cl(table, insert_pk, insert_pk_len, &err); - if (local_cl < 0) { - return cloudsync_vtab_set_error(vtab, "Unable to compute local causal length: %s", err); - } + int64_t local_cl = merge_get_local_cl(table, insert_pk, insert_pk_len); + if (local_cl < 0) return cloudsync_set_error(data, "Unable to compute local causal length", DBRES_ERROR); // if the incoming causal length is older than the local causal length, we can safely ignore it // because the local changes are more recent - if (insert_cl < local_cl) return SQLITE_OK; + if (insert_cl < local_cl) return DBRES_OK; // check if the operation is a delete by examining the causal length // even causal lengths typically signify delete operations @@ -1447,24 +1982,24 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, if (is_delete) { // if it's a delete, check if the local state is at the same causal length // if it is, no further action is needed - if (local_cl == insert_cl) return SQLITE_OK; + if (local_cl == insert_cl) return DBRES_OK; // perform a delete merge if the causal length is newer than the local one int rc = merge_delete(data, table, insert_pk, insert_pk_len, insert_name, insert_col_version, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_delete: %s", err); + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_delete", rc); return rc; } // if the operation is a sentinel-only insert (indicating a new row or resurrected row with no column update), handle it separately. bool is_sentinel_only = (strcmp(insert_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0); if (is_sentinel_only) { - if (local_cl == insert_cl) return SQLITE_OK; + if (local_cl == insert_cl) return DBRES_OK; // perform a sentinel-only insert to track the existence of the row int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_col_version, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_sentinel_only_insert: %s", err); + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); return rc; } @@ -1479,89 +2014,402 @@ int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, // this handles out-of-order deliveries where the row was deleted and is now being re-inserted if (needs_resurrect && (row_exists_locally || (!row_exists_locally && insert_cl > 1))) { int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_cl, - insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform merge_sentinel_only_insert: %s", err); - return rc; - } + insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); } // at this point, we determine whether the incoming change wins based on causal length // this can be due to a resurrection, a non-existent local row, or a conflict resolution bool flag = false; - int rc = merge_did_cid_win(data, table, insert_pk, insert_pk_len, insert_value, insert_site_id, insert_site_id_len, insert_name, insert_col_version, &flag, &err); - if (rc != SQLITE_OK) { - cloudsync_vtab_set_error(vtab, "Unable to perform merge_did_cid_win: %s", err); - return rc; - } + int rc = merge_did_cid_win(data, table, insert_pk, insert_pk_len, insert_value, insert_site_id, insert_site_id_len, insert_name, insert_col_version, &flag); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to perform merge_did_cid_win", rc); // check if the incoming change wins and should be applied bool does_cid_win = ((needs_resurrect) || (!row_exists_locally) || (flag)); - if (!does_cid_win) return SQLITE_OK; - + if (!does_cid_win) return DBRES_OK; + + // Block-level LWW: if the incoming col_name is a block entry (contains \x1F), + // bypass the normal base-table write and instead store the value in the blocks table. + // The base table column will be materialized from all alive blocks. + if (block_is_block_colname(insert_name) && table->has_block_cols) { + // Store or delete block value in blocks table depending on tombstone status + if (insert_col_version % 2 == 0) { + // Tombstone: remove from blocks table + rc = block_delete_value(data, table, insert_pk, insert_pk_len, insert_name); + } else { + rc = block_store_value(data, table, insert_pk, insert_pk_len, insert_name, insert_value); + } + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to store/delete block value", rc); + + // Set winner clock in metadata + rc = merge_set_winner_clock(data, table, insert_pk, insert_pk_len, insert_name, + insert_col_version, insert_db_version, + insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to set winner clock for block", rc); + + // Materialize the full column from blocks into the base table + char *base_col = block_extract_base_colname(insert_name); + if (base_col) { + rc = block_materialize_column(data, table, insert_pk, insert_pk_len, base_col); + cloudsync_memory_free(base_col); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to materialize block column", rc); + } + + return DBRES_OK; + } + // perform the final column insert or update if the incoming change wins - rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid, &err); - if (rc != SQLITE_OK) cloudsync_vtab_set_error(vtab, "Unable to perform merge_insert_col: %s", err); + if (data->pending_batch) { + // Propagate row_exists_locally to the batch on the first winning column. + // This lets merge_flush_pending choose UPDATE vs INSERT ON CONFLICT, + // which matters when RLS policies reference columns not in the payload. + if (data->pending_batch->table == NULL) { + data->pending_batch->row_exists = row_exists_locally; + } + rc = merge_pending_add(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_pending_add", rc); + } else { + rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_insert_col", rc); + } + return rc; } -// MARK: - Private - +// MARK: - Block column setup - + +// Migrate existing tracked rows to block format when block-level LWW is first enabled on a column. +// Scans the metadata table for alive rows with the plain col_name entry (not yet block entries), +// reads each row's current value from the base table, splits it into blocks, and inserts +// the block entries into both the blocks table and the metadata table. +// Uses INSERT OR IGNORE semantics so the operation is safe to call multiple times. +static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table_context *table, int col_idx) { + const char *col_name = table->col_name[col_idx]; + if (!col_name || !table->meta_ref || !table->blocks_ref) return DBRES_OK; + + const char *delim = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Phase 1: collect all existing PKs that have an alive regular col_name entry + // AND do not yet have any entries in the blocks table for this column. + // The NOT IN filter makes this idempotent: rows that were already migrated + // (or had their blocks created via INSERT) are skipped on subsequent calls. + // We collect PKs before writing so that writes to the metadata table (Phase 2) + // do not perturb the read cursor on the same table. + char *like_pattern = block_build_colname(col_name, "%"); + if (!like_pattern) return DBRES_NOMEM; + + char *scan_sql = cloudsync_memory_mprintf(SQL_META_SCAN_COL_FOR_MIGRATION, table->meta_ref, table->blocks_ref); + if (!scan_sql) { cloudsync_memory_free(like_pattern); return DBRES_NOMEM; } + dbvm_t *scan_vm = NULL; + int rc = databasevm_prepare(data, scan_sql, &scan_vm, 0); + cloudsync_memory_free(scan_sql); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); return rc; } + + rc = databasevm_bind_text(scan_vm, 1, col_name, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_finalize(scan_vm); return rc; } + // Bind like_pattern as ?2 and keep it alive until after all scan steps complete, + // because databasevm_bind_text uses SQLITE_STATIC (no copy). + rc = databasevm_bind_text(scan_vm, 2, like_pattern, -1); + if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_finalize(scan_vm); return rc; } + + // Collect pk blobs into a dynamically-grown array of owned copies + void **pks = NULL; + size_t *pklens = NULL; + int pk_count = 0; + int pk_cap = 0; + + while ((rc = databasevm_step(scan_vm)) == DBRES_ROW) { + size_t pklen = 0; + const void *pk = database_column_blob(scan_vm, 0, &pklen); + if (!pk || pklen == 0) continue; + + if (pk_count >= pk_cap) { + int new_cap = pk_cap ? pk_cap * 2 : 8; + void **new_pks = (void **)cloudsync_memory_realloc(pks, (uint64_t)(new_cap * sizeof(void *))); + size_t *new_pklens = (size_t *)cloudsync_memory_realloc(pklens, (uint64_t)(new_cap * sizeof(size_t))); + if (!new_pks || !new_pklens) { + cloudsync_memory_free(new_pks ? new_pks : pks); + cloudsync_memory_free(new_pklens ? new_pklens : pklens); + databasevm_finalize(scan_vm); + return DBRES_NOMEM; + } + pks = new_pks; + pklens = new_pklens; + pk_cap = new_cap; + } -bool cloudsync_config_exists (sqlite3 *db) { - return dbutils_table_exists(db, CLOUDSYNC_SITEID_NAME) == true; -} + pks[pk_count] = cloudsync_memory_alloc((uint64_t)pklen); + if (!pks[pk_count]) { rc = DBRES_NOMEM; break; } + memcpy(pks[pk_count], pk, pklen); + pklens[pk_count] = pklen; + pk_count++; + } -void *cloudsync_context_create (void) { - cloudsync_context *data = (cloudsync_context *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(cloudsync_context))); - DEBUG_SETTINGS("cloudsync_context_create %p", data); - - data->libversion = CLOUDSYNC_VERSION; - data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; - #if CLOUDSYNC_DEBUG - data->debug = 1; - #endif - - // allocate space for 128 tables (it can grow if needed) - data->tables = (cloudsync_table_context **)cloudsync_memory_zeroalloc((uint64_t)(CLOUDSYNC_INIT_NTABLES * sizeof(cloudsync_table_context *))); - if (!data->tables) { - cloudsync_memory_free(data); - return NULL; + databasevm_finalize(scan_vm); + cloudsync_memory_free(like_pattern); // safe to free after scan_vm is finalized + if (rc != DBRES_DONE && rc != DBRES_OK) { + for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return rc; } - data->tables_alloc = CLOUDSYNC_INIT_NTABLES; - data->tables_count = 0; - - return data; -} -void cloudsync_context_free (void *ptr) { - DEBUG_SETTINGS("cloudsync_context_free %p", ptr); - if (!ptr) return; - - cloudsync_context *data = (cloudsync_context*)ptr; - cloudsync_memory_free(data->tables); - cloudsync_memory_free(data); -} + if (pk_count == 0) { + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return DBRES_OK; + } + + // Phase 2: for each collected PK, read the column value, split into blocks, + // and insert into the blocks table + metadata using INSERT OR IGNORE. + + char *meta_sql = cloudsync_memory_mprintf(SQL_META_INSERT_BLOCK_IGNORE, table->meta_ref); + if (!meta_sql) { rc = DBRES_NOMEM; goto cleanup_pks; } + dbvm_t *meta_vm = NULL; + rc = databasevm_prepare(data, meta_sql, &meta_vm, 0); + cloudsync_memory_free(meta_sql); + if (rc != DBRES_OK) goto cleanup_pks; + + char *blocks_sql = cloudsync_memory_mprintf(SQL_BLOCKS_INSERT_IGNORE, table->blocks_ref); + if (!blocks_sql) { databasevm_finalize(meta_vm); rc = DBRES_NOMEM; goto cleanup_pks; } + dbvm_t *blocks_vm = NULL; + rc = databasevm_prepare(data, blocks_sql, &blocks_vm, 0); + cloudsync_memory_free(blocks_sql); + if (rc != DBRES_OK) { databasevm_finalize(meta_vm); goto cleanup_pks; } + + dbvm_t *val_vm = (dbvm_t *)table_column_lookup(table, col_name, false, NULL); + + for (int p = 0; p < pk_count; p++) { + const void *pk = pks[p]; + size_t pklen = pklens[p]; + + if (!val_vm) continue; + + // Read current column value from the base table + int bind_rc = pk_decode_prikey((char *)pk, pklen, pk_decode_bind_callback, (void *)val_vm); + if (bind_rc < 0) { databasevm_reset(val_vm); continue; } + + int step_rc = databasevm_step(val_vm); + const char *text = (step_rc == DBRES_ROW) ? database_column_text(val_vm, 0) : NULL; + // Make a copy of text before resetting val_vm, as the pointer is only valid until reset + char *text_copy = text ? cloudsync_string_dup(text) : NULL; + databasevm_reset(val_vm); + + if (!text_copy) continue; // NULL column value: nothing to migrate + + // Split text into blocks and store each one + block_list_t *blocks = block_split(text_copy, delim); + cloudsync_memory_free(text_copy); + if (!blocks) continue; + + char **positions = block_initial_positions(blocks->count); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col_name, positions[b]); + if (block_cn) { + // Metadata entry (skip if this block position already exists) + databasevm_bind_blob(meta_vm, 1, pk, (int)pklen); + databasevm_bind_text(meta_vm, 2, block_cn, -1); + databasevm_bind_int(meta_vm, 3, 1); // col_version = 1 (alive) + databasevm_bind_int(meta_vm, 4, db_version); + databasevm_bind_int(meta_vm, 5, cloudsync_bumpseq(data)); + databasevm_step(meta_vm); + databasevm_reset(meta_vm); + + // Block value (skip if this block position already exists) + databasevm_bind_blob(blocks_vm, 1, pk, (int)pklen); + databasevm_bind_text(blocks_vm, 2, block_cn, -1); + databasevm_bind_text(blocks_vm, 3, blocks->entries[b].content, -1); + databasevm_step(blocks_vm); + databasevm_reset(blocks_vm); + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } -const char *cloudsync_context_init (sqlite3 *db, cloudsync_context *data, sqlite3_context *context) { - if (!data && context) data = (cloudsync_context *)sqlite3_user_data(context); + databasevm_finalize(meta_vm); + databasevm_finalize(blocks_vm); + rc = DBRES_OK; - // perform init just the first time, if the site_id field is not set. - // The data->site_id value could exists while settings tables don't exists if the - // cloudsync_context_init was previously called in init transaction that was rolled back - // because of an error during the init process. - if (data->site_id[0] == 0 || !dbutils_table_exists(db, CLOUDSYNC_SITEID_NAME)) { - if (dbutils_settings_init(db, data, context) != SQLITE_OK) return NULL; - if (stmts_add_tocontext(db, data) != SQLITE_OK) return NULL; - if (cloudsync_load_siteid(db, data) != SQLITE_OK) return NULL; - - data->sqlite_ctx = context; - data->schema_hash = dbutils_schema_hash(db); - } - - return (const char *)data->site_id; +cleanup_pks: + for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); + cloudsync_memory_free(pks); + cloudsync_memory_free(pklens); + return rc; } -void cloudsync_sync_key(cloudsync_context *data, const char *key, const char *value) { +int cloudsync_setup_block_column (cloudsync_context *data, const char *table_name, const char *col_name, const char *delimiter, bool persist) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return cloudsync_set_error(data, "cloudsync_setup_block_column: table not found", DBRES_ERROR); + + // Find column index + int col_idx = table_col_index(table, col_name); + if (col_idx < 0) { + char buf[1024]; + snprintf(buf, sizeof(buf), "cloudsync_setup_block_column: column '%s' not found in table '%s'", col_name, table_name); + return cloudsync_set_error(data, buf, DBRES_ERROR); + } + + // Set column algo + table->col_algo[col_idx] = col_algo_block; + table->has_block_cols = true; + + // Set delimiter (can be NULL for default) + if (table->col_delimiter[col_idx]) { + cloudsync_memory_free(table->col_delimiter[col_idx]); + table->col_delimiter[col_idx] = NULL; + } + if (delimiter) { + table->col_delimiter[col_idx] = cloudsync_string_dup(delimiter); + } + + // Create blocks table if not already done + if (!table->blocks_ref) { + table->blocks_ref = database_build_blocks_ref(table->schema, table->name); + if (!table->blocks_ref) return DBRES_NOMEM; + + // CREATE TABLE IF NOT EXISTS + char *sql = cloudsync_memory_mprintf(SQL_BLOCKS_CREATE_TABLE, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to create blocks table", rc); + + // Prepare block statements + // Write: upsert into blocks (pk, col_name, col_value) + sql = cloudsync_memory_mprintf(SQL_BLOCKS_UPSERT, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_write_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // Read: SELECT col_value FROM blocks WHERE pk = ? AND col_name = ? + sql = cloudsync_memory_mprintf(SQL_BLOCKS_SELECT, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_read_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // Delete: DELETE FROM blocks WHERE pk = ? AND col_name = ? + sql = cloudsync_memory_mprintf(SQL_BLOCKS_DELETE, table->blocks_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_value_delete_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + // List alive blocks for materialization + sql = cloudsync_memory_mprintf(SQL_BLOCKS_LIST_ALIVE, table->blocks_ref, table->meta_ref); + if (!sql) return DBRES_NOMEM; + rc = databasevm_prepare(data, sql, (void **)&table->block_list_stmt, DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + } + + // Persist settings (skipped when called from the settings loader, since + // writing to cloudsync_table_settings while sqlite3_exec is iterating it + // re-feeds the rewritten row to the cursor and causes an infinite loop). + if (persist) { + int rc = dbutils_table_settings_set_key_value(data, table_name, col_name, "algo", "block"); + if (rc != DBRES_OK) return rc; + + if (delimiter) { + rc = dbutils_table_settings_set_key_value(data, table_name, col_name, "delimiter", delimiter); + if (rc != DBRES_OK) return rc; + } + + // Migrate any existing tracked rows: populate the blocks table and metadata with + // block entries derived from the current column value, so that subsequent UPDATE + // operations can diff against the real existing state instead of treating everything + // as new, and so this node participates correctly in LWW conflict resolution. + rc = block_migrate_existing_rows(data, table, col_idx); + if (rc != DBRES_OK) return rc; + } + + return DBRES_OK; +} + +// MARK: - Private - + +bool cloudsync_config_exists (cloudsync_context *data) { + return database_internal_table_exists(data, CLOUDSYNC_SITEID_NAME) == true; +} + +bool cloudsync_context_is_initialized (cloudsync_context *data) { + // A fully initialized context has its persistent "is the DB stale" probe + // prepared. cloudsync_context_init prepares data_version_stmt (via + // cloudsync_add_dbvms) only after the cloudsync_site_id table exists, so + // a non-NULL pointer means cloudsync_init has been called at least once + // on this connection. Used to produce actionable error messages when + // callers hit a function before calling cloudsync_init. + return data != NULL && data->data_version_stmt != NULL; +} + +cloudsync_context *cloudsync_context_create (void *db) { + cloudsync_context *data = (cloudsync_context *)cloudsync_memory_zeroalloc((uint64_t)(sizeof(cloudsync_context))); + if (!data) return NULL; + DEBUG_SETTINGS("cloudsync_context_create %p", data); + + data->libversion = CLOUDSYNC_VERSION; + data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; + #if CLOUDSYNC_DEBUG + data->debug = 1; + #endif + + // allocate space for 64 tables (it can grow if needed) + uint64_t mem_needed = (uint64_t)(CLOUDSYNC_INIT_NTABLES * sizeof(cloudsync_table_context *)); + data->tables = (cloudsync_table_context **)cloudsync_memory_zeroalloc(mem_needed); + if (!data->tables) {cloudsync_memory_free(data); return NULL;} + + data->tables_cap = CLOUDSYNC_INIT_NTABLES; + data->tables_count = 0; + data->db = db; + + // SQLite exposes col_value as ANY, but other databases require a concrete type. + // In PostgreSQL we expose col_value as bytea, which holds the pk-encoded value bytes (type + data). + // Because col_value is already encoded, we skip decoding this field and pass it through as bytea. + // It is decoded to the target column type just before applying changes to the base table. + data->skip_decode_idx = (db == NULL) ? CLOUDSYNC_PK_INDEX_COLVALUE : -1; + + return data; +} + +void cloudsync_context_free (void *ctx) { + cloudsync_context *data = (cloudsync_context *)ctx; + DEBUG_SETTINGS("cloudsync_context_free %p", data); + if (!data) return; + + // free all table contexts and prepared statements + cloudsync_terminate(data); + + cloudsync_memory_free(data->tables); + cloudsync_memory_free(data); +} + +const char *cloudsync_context_init (cloudsync_context *data) { + if (!data) return NULL; + + // perform init just the first time, if the site_id field is not set. + // The data->site_id value could exists while settings tables don't exists if the + // cloudsync_context_init was previously called in init transaction that was rolled back + // because of an error during the init process. + if (data->site_id[0] == 0 || !database_internal_table_exists(data, CLOUDSYNC_SITEID_NAME)) { + if (dbutils_settings_init(data) != DBRES_OK) return NULL; + if (cloudsync_add_dbvms(data) != DBRES_OK) return NULL; + if (cloudsync_load_siteid(data) != DBRES_OK) return NULL; + data->schema_hash = database_schema_hash(data); + } + + return (const char *)data->site_id; +} + +void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value) { DEBUG_SETTINGS("cloudsync_sync_key key: %s value: %s", key, value); // sync data @@ -1575,6 +2423,11 @@ void cloudsync_sync_key(cloudsync_context *data, const char *key, const char *va if (value && (value[0] != 0) && (value[0] != '0')) data->debug = 1; return; } + + if (strcmp(key, CLOUDSYNC_KEY_SCHEMA) == 0) { + cloudsync_set_schema(data, value); + return; + } } #if 0 @@ -1592,7 +2445,7 @@ int cloudsync_commit_hook (void *ctx) { data->pending_db_version = CLOUDSYNC_VALUE_NOTSET; data->seq = 0; - return SQLITE_OK; + return DBRES_OK; } void cloudsync_rollback_hook (void *ctx) { @@ -1602,38 +2455,82 @@ void cloudsync_rollback_hook (void *ctx) { data->seq = 0; } -int cloudsync_finalize_alter (sqlite3_context *context, cloudsync_context *data, cloudsync_table_context *table) { - int rc = SQLITE_OK; - sqlite3 *db = sqlite3_context_db_handle(context); +int cloudsync_begin_alter (cloudsync_context *data, const char *table_name) { + // init cloudsync_settings + if (cloudsync_context_init(data) == NULL) { + return DBRES_MISUSE; + } + + // lookup table + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to find table %s", table_name); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); + } + + // idempotent: if already altering, return OK + if (table->is_altering) return DBRES_OK; + + // retrieve primary key(s) + char **names = NULL; + int nrows = 0; + int rc = database_pk_names(data, table_name, &names, &nrows); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to get primary keys for table %s", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_begin_alter; + } + + // sanity check the number of primary keys + if (nrows != table_count_pks(table)) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Number of primary keys for table %s changed before ALTER", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_begin_alter; + } + + // drop original triggers + rc = database_delete_triggers(data, table_name); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to delete triggers for table %s in cloudsync_begin_alter.", table_name); + cloudsync_set_error(data, buffer, DBRES_ERROR); + goto rollback_begin_alter; + } + + table_set_pknames(table, names); + table->is_altering = true; + return DBRES_OK; + +rollback_begin_alter: + if (names) table_pknames_free(names, nrows); + return rc; +} - db_version_check_uptodate(db, data); +int cloudsync_finalize_alter (cloudsync_context *data, cloudsync_table_context *table) { + // check if dbversion needed to be updated + cloudsync_dbversion_check_uptodate(data); - // If primary key columns change (in the schema) - // We need to drop, re-create and backfill - // the clock table. - // A change in pk columns means a change in all identities - // of all rows. - // We can determine this by comparing unique index on lookaside table vs - // pks on source table - char *errmsg = NULL; + // if primary-key columns change, all row identities change. + // In that case, the clock table must be dropped, recreated, + // and backfilled. We detect this by comparing the unique index + // in the lookaside table with the source table's PKs. + + // retrieve primary keys (to check is they changed) char **result = NULL; - int nrows, ncols; - char *sql = cloudsync_memory_mprintf("SELECT name FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table->name); - rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); - goto finalize; - } else if (errmsg || ncols != 1) { - rc = SQLITE_MISUSE; + int nrows = 0; + int rc = database_pk_names (data, table->name, &result, &nrows); + if (rc != DBRES_OK || nrows == 0) { + if (nrows == 0) rc = DBRES_MISUSE; goto finalize; } - bool pk_diff = false; - if (nrows != table->npks) { - pk_diff = true; - } else { - for (int i=0; inpks); + if (!pk_diff) { + for (int i = 0; i < nrows; ++i) { if (strcmp(table->pk_name[i], result[i]) != 0) { pk_diff = true; break; @@ -1643,236 +2540,396 @@ int cloudsync_finalize_alter (sqlite3_context *context, cloudsync_context *data, if (pk_diff) { // drop meta-table, it will be recreated - char *sql = cloudsync_memory_mprintf("DROP TABLE IF EXISTS \"%w_cloudsync\";", table->name); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + char *sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->meta_ref); + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } } else { // compact meta-table // delete entries for removed columns - char *sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE \"col_name\" NOT IN (" - "SELECT name FROM pragma_table_info('%q') UNION SELECT '%s'" - ")", table->name, table->name, CLOUDSYNC_TOMBSTONE_VALUE); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + const char *schema = table->schema ? table->schema : ""; + char *sql = sql_build_delete_cols_not_in_schema_query(schema, table->name, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE); + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } - char *singlequote_escaped_table_name = cloudsync_memory_mprintf("%q", table->name); - sql = cloudsync_memory_mprintf("SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;", singlequote_escaped_table_name, singlequote_escaped_table_name); - cloudsync_memory_free(singlequote_escaped_table_name); - if (!sql) { - rc = SQLITE_NOMEM; - goto finalize; - } - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "rowid"; + sql = sql_build_pk_qualified_collist_query(schema, table->name); + if (!sql) {rc = DBRES_NOMEM; goto finalize;} + + char *pkclause = NULL; + rc = database_select_text(data, sql, &pkclause); cloudsync_memory_free(sql); + if (rc != DBRES_OK) goto finalize; + char *pkvalues = (pkclause) ? pkclause : "rowid"; // delete entries related to rows that no longer exist in the original table, but preserve tombstone - sql = cloudsync_memory_mprintf("DELETE FROM \"%w_cloudsync\" WHERE (\"col_name\" != '%s' OR (\"col_name\" = '%s' AND col_version %% 2 != 0)) AND NOT EXISTS (SELECT 1 FROM \"%w\" WHERE \"%w_cloudsync\".pk = cloudsync_pk_encode(%s) LIMIT 1);", table->name, CLOUDSYNC_TOMBSTONE_VALUE, CLOUDSYNC_TOMBSTONE_VALUE, table->name, table->name, pkvalues); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK, table->meta_ref, CLOUDSYNC_TOMBSTONE_VALUE, CLOUDSYNC_TOMBSTONE_VALUE, table->base_ref, table->meta_ref, pkvalues); + rc = database_exec(data, sql); if (pkclause) cloudsync_memory_free(pkclause); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - DEBUG_SQLITE_ERROR(rc, "cloudsync_finalize_alter", db); + if (rc != DBRES_OK) { + DEBUG_DBERROR(rc, "cloudsync_finalize_alter", data); goto finalize; } } + // update key to be later used in cloudsync_dbversion_rebuild char buf[256]; - snprintf(buf, sizeof(buf), "%lld", data->db_version); - dbutils_settings_set_key_value(db, context, "pre_alter_dbversion", buf); + snprintf(buf, sizeof(buf), "%" PRId64, data->db_version); + dbutils_settings_set_key_value(data, "pre_alter_dbversion", buf); finalize: - sqlite3_free_table(result); - sqlite3_free(errmsg); - + table_pknames_free(result, nrows); + return rc; +} + +int cloudsync_commit_alter (cloudsync_context *data, const char *table_name) { + int rc = DBRES_MISUSE; + cloudsync_table_context *table = NULL; + + // init cloudsync_settings + if (cloudsync_context_init(data) == NULL) { + cloudsync_set_error(data, "Unable to initialize cloudsync context", DBRES_MISUSE); + goto rollback_finalize_alter; + } + + // lookup table + table = table_lookup(data, table_name); + if (!table) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to find table %s", table_name); + cloudsync_set_error(data, buffer, DBRES_MISUSE); + goto rollback_finalize_alter; + } + + // idempotent: if not altering, return OK + if (!table->is_altering) return DBRES_OK; + + rc = cloudsync_finalize_alter(data, table); + if (rc != DBRES_OK) goto rollback_finalize_alter; + + // the table is outdated, delete it and it will be reloaded in the cloudsync_init_internal + // is_altering is reset implicitly because table_free + cloudsync_init_table + // will reallocate the table context with zero-initialized memory + table_remove(data, table); + table_free(table); + table = NULL; + + // init again cloudsync for the table + table_algo algo_current = dbutils_table_settings_get_algo(data, table_name); + if (algo_current == table_algo_none) algo_current = dbutils_table_settings_get_algo(data, "*"); + rc = cloudsync_init_table(data, table_name, cloudsync_algo_name(algo_current), CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK); + if (rc != DBRES_OK) goto rollback_finalize_alter; + + return DBRES_OK; + +rollback_finalize_alter: + if (table) { + table_set_pknames(table, NULL); + table->is_altering = false; + } return rc; } -int cloudsync_refill_metatable (sqlite3 *db, cloudsync_context *data, const char *table_name) { +// MARK: - Filter Rewrite - + +// Replace bare column names in a filter expression with prefix-qualified names. +// E.g., filter="user_id = 42", prefix="NEW", columns=["user_id","id"] → "NEW.\"user_id\" = 42" +// Columns must be sorted by length descending by the caller to avoid partial matches. +// Skips content inside single-quoted string literals. +// Returns a newly allocated string (caller must free with cloudsync_memory_free), or NULL on error. +// Helper: check if an identifier token matches a column name. +static bool filter_is_column (const char *token, size_t token_len, char **columns, int ncols) { + for (int i = 0; i < ncols; ++i) { + if (strlen(columns[i]) == token_len && strncmp(token, columns[i], token_len) == 0) + return true; + } + return false; +} + +// Helper: check if character is part of a SQL identifier. +static bool filter_is_ident_char (char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_'; +} + +char *cloudsync_filter_add_row_prefix (const char *filter, const char *prefix, char **columns, int ncols) { + if (!filter || !prefix || !columns || ncols <= 0) return NULL; + + size_t filter_len = strlen(filter); + size_t prefix_len = strlen(prefix); + + // Each identifier match grows by at most (prefix_len + 3) bytes. + // Worst case: the entire filter is one repeated column reference separated by + // single characters, so up to (filter_len / 2) matches. Use a safe upper bound. + size_t max_growth = (filter_len / 2 + 1) * (prefix_len + 3); + size_t cap = filter_len + max_growth + 64; + char *result = (char *)cloudsync_memory_alloc(cap); + if (!result) return NULL; + size_t out = 0; + + // Single pass: tokenize into identifiers, quoted strings, and everything else. + size_t i = 0; + while (i < filter_len) { + // Skip single-quoted string literals verbatim (handle '' escape) + if (filter[i] == '\'') { + result[out++] = filter[i++]; + while (i < filter_len) { + if (filter[i] == '\'') { + result[out++] = filter[i++]; + // '' is an escaped quote — keep going + if (i < filter_len && filter[i] == '\'') { + result[out++] = filter[i++]; + continue; + } + break; // single ' ends the literal + } + result[out++] = filter[i++]; + } + continue; + } + + // Extract identifier token + if (filter_is_ident_char(filter[i])) { + size_t start = i; + while (i < filter_len && filter_is_ident_char(filter[i])) ++i; + size_t token_len = i - start; + + if (filter_is_column(&filter[start], token_len, columns, ncols)) { + // Emit PREFIX."column_name" + memcpy(&result[out], prefix, prefix_len); out += prefix_len; + result[out++] = '.'; + result[out++] = '"'; + memcpy(&result[out], &filter[start], token_len); out += token_len; + result[out++] = '"'; + } else { + // Not a column — copy as-is + memcpy(&result[out], &filter[start], token_len); out += token_len; + } + continue; + } + + // Any other character — copy as-is + result[out++] = filter[i++]; + } + + result[out] = '\0'; + return result; +} + +int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name) { cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return SQLITE_INTERNAL; - - sqlite3_stmt *vm = NULL; - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - - char *sql = cloudsync_memory_mprintf("SELECT group_concat('\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - char *pkclause_identifiers = dbutils_text_select(db, sql); - char *pkvalues_identifiers = (pkclause_identifiers) ? pkclause_identifiers : "rowid"; + if (!table) return DBRES_ERROR; + + char *sql = cloudsync_memory_mprintf(SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE, table->meta_ref); + int rc = database_exec(data, sql); cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("SELECT group_concat('cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - char *pkdecode = dbutils_text_select(db, sql); - char *pkdecodeval = (pkdecode) ? pkdecode : "cloudsync_pk_decode(pk, 1) AS rowid"; + if (rc != DBRES_OK) return rc; + + return cloudsync_refill_metatable(data, table_name); +} + +int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name) { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return DBRES_ERROR; + + dbvm_t *vm = NULL; + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Read row-level filter from settings (if any) + char filter_buf[2048]; + int frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", filter_buf, sizeof(filter_buf)); + const char *filter = (frc == DBRES_OK && filter_buf[0]) ? filter_buf : NULL; + + const char *schema = table->schema ? table->schema : ""; + char *sql = sql_build_pk_collist_query(schema, table_name); + char *pkclause_identifiers = NULL; + int rc = database_select_text(data, sql, &pkclause_identifiers); cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("SELECT cloudsync_insert('%q', %s) FROM (SELECT %s FROM \"%w\" EXCEPT SELECT %s FROM \"%w_cloudsync\");", table_name, pkvalues_identifiers, pkvalues_identifiers, table_name, pkdecodeval, table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != DBRES_OK) goto finalize; + char *pkvalues_identifiers = (pkclause_identifiers) ? pkclause_identifiers : "rowid"; + + // Use database-specific query builder to handle type differences in composite PKs + sql = sql_build_insert_missing_pks_query(schema, table_name, pkvalues_identifiers, table->base_ref, table->meta_ref, filter); + if (!sql) {rc = DBRES_NOMEM; goto finalize;} + rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - + if (rc != DBRES_OK) goto finalize; + // fill missing colums // for each non-pk column: // The new query does 1 encode per source row and one indexed NOT-EXISTS probe. - // The old plan does many decodes per candidate and can’t use an index to rule out matches quickly—so it burns CPU and I/O. - - sql = cloudsync_memory_mprintf("WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\") SELECT _cstemp1.pk FROM _cstemp1 WHERE NOT EXISTS (SELECT 1 FROM \"%w_cloudsync\" _cstemp2 WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?);", pkvalues_identifiers, table_name, table_name); - rc = sqlite3_prepare_v3(db, sql, -1, SQLITE_PREPARE_PERSISTENT, &vm, NULL); + // The old plan does many decodes per candidate and can't use an index to rule out matches quickly—so it burns CPU and I/O. + + if (filter) { + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED, pkvalues_identifiers, table->base_ref, filter, table->meta_ref); + } else { + sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL, pkvalues_identifiers, table->base_ref, table->meta_ref); + } + rc = databasevm_prepare(data, sql, (void **)&vm, DBFLAG_PERSISTENT); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + if (rc != DBRES_OK) goto finalize; for (int i=0; incols; ++i) { char *col_name = table->col_name[i]; - rc = sqlite3_bind_text(vm, 1, col_name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize; - + rc = databasevm_bind_text(vm, 1, col_name, -1); + if (rc != DBRES_OK) goto finalize; + while (1) { - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - const char *pk = (const char *)sqlite3_column_text(vm, 0); - size_t pklen = strlen(pk); - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, col_name, db_version, BUMP_SEQ(data)); - } else if (rc == SQLITE_DONE) { - rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_ROW) { + size_t pklen = 0; + const void *pk = (const char *)database_column_blob(vm, 0, &pklen); + if (!pk) { rc = DBRES_ERROR; break; } + rc = local_mark_insert_or_update_meta(table, pk, pklen, col_name, db_version, cloudsync_bumpseq(data)); + } else if (rc == DBRES_DONE) { + rc = DBRES_OK; break; } else { break; } } - if (rc != SQLITE_OK) goto finalize; + if (rc != DBRES_OK) goto finalize; - sqlite3_reset(vm); + databasevm_reset(vm); } finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("cloudsync_refill_metatable error: %s", sqlite3_errmsg(db)); + if (rc != DBRES_OK) {DEBUG_ALWAYS("cloudsync_refill_metatable error: %s", database_errmsg(data));} if (pkclause_identifiers) cloudsync_memory_free(pkclause_identifiers); - if (pkdecode) cloudsync_memory_free(pkdecode); - if (vm) sqlite3_finalize(vm); + if (vm) databasevm_finalize(vm); return rc; } // MARK: - Local - -int local_update_sentinel (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - sqlite3_stmt *vm = table->meta_sentinel_update_stmt; +int local_update_sentinel (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + dbvm_t *vm = table->meta_sentinel_update_stmt; if (!vm) return -1; - int rc = sqlite3_bind_int64(vm, 1, db_version); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_int(vm, 1, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 2, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_blob(vm, 3, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 3, pk, (int)pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_update_sentinel", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_update_sentinel", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_sentinel_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - sqlite3_stmt *vm = table->meta_sentinel_insert_stmt; +int local_mark_insert_sentinel_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + dbvm_t *vm = table->meta_sentinel_insert_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, (int)pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 2, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 3, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 3, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_insert_sentinel", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_insert_sentinel", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_or_update_meta_impl (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, int col_version, sqlite3_int64 db_version, int seq) { +int local_mark_insert_or_update_meta_impl (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int col_version, int64_t db_version, int seq) { - sqlite3_stmt *vm = table->meta_row_insert_update_stmt; + dbvm_t *vm = table->meta_row_insert_update_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_text(vm, 2, (col_name) ? col_name : CLOUDSYNC_TOMBSTONE_VALUE, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_text(vm, 2, (col_name) ? col_name : CLOUDSYNC_TOMBSTONE_VALUE, -1); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 3, col_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 3, col_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 4, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 4, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 5, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 5, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int64(vm, 6, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 6, db_version); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_bind_int(vm, 7, seq); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 7, seq); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_insert_or_update", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_insert_or_update", table->context); + databasevm_reset(vm); return rc; } -int local_mark_insert_or_update_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *col_name, sqlite3_int64 db_version, int seq) { - return local_mark_insert_or_update_meta_impl(db, table, pk, pklen, col_name, 1, db_version, seq); +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq) { + return local_mark_insert_or_update_meta_impl(table, pk, pklen, col_name, 1, db_version, seq); +} + +int local_mark_delete_block_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname, int64_t db_version, int seq) { + // Mark a block as deleted by setting col_version = 2 (even = deleted) + return local_mark_insert_or_update_meta_impl(table, pk, pklen, block_colname, 2, db_version, seq); +} + +int block_delete_value_external (cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname) { + return block_delete_value(data, table, pk, (int)pklen, block_colname); } -int local_mark_delete_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, sqlite3_int64 db_version, int seq) { - return local_mark_insert_or_update_meta_impl(db, table, pk, pklen, NULL, 2, db_version, seq); +int local_mark_delete_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq) { + return local_mark_insert_or_update_meta_impl(table, pk, pklen, NULL, 2, db_version, seq); } -int local_drop_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen) { - sqlite3_stmt *vm = table->meta_row_drop_stmt; +int local_drop_meta (cloudsync_table_context *table, const void *pk, size_t pklen) { + dbvm_t *vm = table->meta_row_drop_stmt; if (!vm) return -1; - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_drop_meta", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_drop_meta", table->context); + databasevm_reset(vm); return rc; } -int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const char *pk, size_t pklen, const char *pk2, size_t pklen2, sqlite3_int64 db_version) { +int local_update_move_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const void *pk2, size_t pklen2, int64_t db_version) { /* * This function moves non-sentinel metadata entries from an old primary key (OLD.pk) * to a new primary key (NEW.pk) when a primary key change occurs. @@ -1885,7 +2942,7 @@ int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const c * may be applied incorrectly, leading to data inconsistency. * * When performing the update, a unique `seq` must be assigned to each metadata row. This can be achieved - * by either incrementing the maximum sequence value in the table or using a function (e.g., `bump_seq(data)`) + * by either incrementing the maximum sequence value in the table or using a function (e.g., cloudsync_bumpseq(data)) * that generates a unique sequence for each row. The update query should ensure that each row moved * from OLD.pk to NEW.pk gets a distinct `seq` to maintain proper versioning and ordering of changes. */ @@ -1893,42 +2950,58 @@ int local_update_move_meta (sqlite3 *db, cloudsync_table_context *table, const c // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details // pk2 is the old pk - sqlite3_stmt *vm = table->meta_update_move_stmt; + dbvm_t *vm = table->meta_update_move_stmt; if (!vm) return -1; // new primary key - int rc = sqlite3_bind_blob(vm, 1, pk, (int)pklen, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + int rc = databasevm_bind_blob(vm, 1, pk, pklen); + if (rc != DBRES_OK) goto cleanup; // new db_version - rc = sqlite3_bind_int64(vm, 2, db_version); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_int(vm, 2, db_version); + if (rc != DBRES_OK) goto cleanup; // old primary key - rc = sqlite3_bind_blob(vm, 3, pk2, (int)pklen2, SQLITE_STATIC); - if (rc != SQLITE_OK) goto cleanup; + rc = databasevm_bind_blob(vm, 3, pk2, pklen2); + if (rc != DBRES_OK) goto cleanup; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; cleanup: - DEBUG_SQLITE_ERROR(rc, "local_update_move_meta", db); - sqlite3_reset(vm); + DEBUG_DBERROR(rc, "local_update_move_meta", table->context); + databasevm_reset(vm); return rc; } // MARK: - Payload Encode / Decode - -bool cloudsync_buffer_free (cloudsync_data_payload *payload) { - if (payload) { - if (payload->buffer) cloudsync_memory_free(payload->buffer); - memset(payload, 0, sizeof(cloudsync_data_payload)); - } - - return false; +static void cloudsync_payload_checksum_store (cloudsync_payload_header *header, uint64_t checksum) { + uint64_t h = checksum & 0xFFFFFFFFFFFFULL; // keep 48 bits + header->checksum[0] = (uint8_t)(h >> 40); + header->checksum[1] = (uint8_t)(h >> 32); + header->checksum[2] = (uint8_t)(h >> 24); + header->checksum[3] = (uint8_t)(h >> 16); + header->checksum[4] = (uint8_t)(h >> 8); + header->checksum[5] = (uint8_t)(h >> 0); +} + +static uint64_t cloudsync_payload_checksum_load (cloudsync_payload_header *header) { + return ((uint64_t)header->checksum[0] << 40) | + ((uint64_t)header->checksum[1] << 32) | + ((uint64_t)header->checksum[2] << 24) | + ((uint64_t)header->checksum[3] << 16) | + ((uint64_t)header->checksum[4] << 8) | + ((uint64_t)header->checksum[5] << 0); +} + +static bool cloudsync_payload_checksum_verify (cloudsync_payload_header *header, uint64_t checksum) { + uint64_t checksum1 = cloudsync_payload_checksum_load(header); + uint64_t checksum2 = checksum & 0xFFFFFFFFFFFFULL; + return (checksum1 == checksum2); } -bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { +static bool cloudsync_payload_encode_check (cloudsync_payload_context *payload, size_t needed) { if (payload->nrows == 0) needed += sizeof(cloudsync_payload_header); // alloc/resize buffer @@ -1937,7 +3010,11 @@ bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { size_t balloc = payload->balloc + needed; char *buffer = cloudsync_memory_realloc(payload->buffer, balloc); - if (!buffer) return cloudsync_buffer_free(payload); + if (!buffer) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + memset(payload, 0, sizeof(cloudsync_payload_context)); + return false; + } payload->buffer = buffer; payload->balloc = balloc; @@ -1947,6 +3024,11 @@ bool cloudsync_buffer_check (cloudsync_data_payload *payload, size_t needed) { return true; } +size_t cloudsync_payload_context_size (size_t *header_size) { + if (header_size) *header_size = sizeof(cloudsync_payload_header); + return sizeof(cloudsync_payload_context); +} + void cloudsync_payload_header_init (cloudsync_payload_header *header, uint32_t expanded_size, uint16_t ncols, uint32_t nrows, uint64_t hash) { memset(header, 0, sizeof(cloudsync_payload_header)); assert(sizeof(cloudsync_payload_header)==32); @@ -1955,146 +3037,170 @@ void cloudsync_payload_header_init (cloudsync_payload_header *header, uint32_t e sscanf(CLOUDSYNC_VERSION, "%d.%d.%d", &major, &minor, &patch); header->signature = htonl(CLOUDSYNC_PAYLOAD_SIGNATURE); - header->version = CLOUDSYNC_PAYLOAD_VERSION; - header->libversion[0] = major; - header->libversion[1] = minor; - header->libversion[2] = patch; + header->version = CLOUDSYNC_PAYLOAD_VERSION_2; + header->libversion[0] = (uint8_t)major; + header->libversion[1] = (uint8_t)minor; + header->libversion[2] = (uint8_t)patch; header->expanded_size = htonl(expanded_size); header->ncols = htons(ncols); header->nrows = htonl(nrows); header->schema_hash = htonll(hash); } -void cloudsync_payload_encode_step (sqlite3_context *context, int argc, sqlite3_value **argv) { +int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv) { DEBUG_FUNCTION("cloudsync_payload_encode_step"); // debug_values(argc, argv); - // allocate/get the session context - cloudsync_data_payload *payload = (cloudsync_data_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_data_payload)); - if (!payload) return; - // check if the step function is called for the first time - if (payload->nrows == 0) payload->ncols = argc; + if (payload->nrows == 0) payload->ncols = (uint16_t)argc; - size_t breq = pk_encode_size(argv, argc, 0); - if (cloudsync_buffer_check(payload, breq) == false) return; + size_t breq = pk_encode_size((dbvalue_t **)argv, argc, 0, data->skip_decode_idx); + if (cloudsync_payload_encode_check(payload, breq) == false) { + return cloudsync_set_error(data, "Not enough memory to resize payload internal buffer", DBRES_NOMEM); + } char *buffer = payload->buffer + payload->bused; - char *ptr = pk_encode(argv, argc, buffer, false, NULL); - assert(buffer == ptr); + size_t bsize = payload->balloc - payload->bused; + char *p = pk_encode((dbvalue_t **)argv, argc, buffer, false, &bsize, data->skip_decode_idx); + if (!p) return cloudsync_set_error(data, "An error occurred while encoding payload", DBRES_ERROR); // update buffer payload->bused += breq; // increment row counter ++payload->nrows; + + return DBRES_OK; } -void cloudsync_payload_encode_final (sqlite3_context *context) { +int cloudsync_payload_encode_final (cloudsync_payload_context *payload, cloudsync_context *data) { DEBUG_FUNCTION("cloudsync_payload_encode_final"); - - // get the session context - cloudsync_data_payload *payload = (cloudsync_data_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_data_payload)); - if (!payload) return; if (payload->nrows == 0) { - sqlite3_result_null(context); - return; + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + return DBRES_OK; } - // encode payload + if (payload->nrows > UINT32_MAX) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, "Maximum number of payload rows reached", DBRES_ERROR); + return DBRES_ERROR; + } + + // sanity check about buffer size int header_size = (int)sizeof(cloudsync_payload_header); - int real_buffer_size = (int)(payload->bused - header_size); - int zbound = LZ4_compressBound(real_buffer_size); - char *buffer = cloudsync_memory_alloc(zbound + header_size); - if (!buffer) { - cloudsync_buffer_free(payload); - sqlite3_result_error_code(context, SQLITE_NOMEM); - return; + int64_t buffer_size = (int64_t)payload->bused - (int64_t)header_size; + if (buffer_size < 0) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, "cloudsync_encode: internal size underflow", DBRES_ERROR); + return DBRES_ERROR; + } + if (buffer_size > INT_MAX) { + if (payload->buffer) cloudsync_memory_free(payload->buffer); + payload->buffer = NULL; + payload->bsize = 0; + cloudsync_set_error(data, "cloudsync_encode: payload too large to compress (INT_MAX limit)", DBRES_ERROR); + return DBRES_ERROR; } + // try to allocate buffer used for compressed data + int real_buffer_size = (int)buffer_size; + int zbound = LZ4_compressBound(real_buffer_size); + char *zbuffer = cloudsync_memory_alloc(zbound + header_size); // if for some reasons allocation fails then just skip compression - // adjust buffer to compress to skip the reserved header + // skip the reserved header from the buffer to compress char *src_buffer = payload->buffer + sizeof(cloudsync_payload_header); - int zused = LZ4_compress_default(src_buffer, buffer+header_size, real_buffer_size, zbound); + int zused = (zbuffer) ? LZ4_compress_default(src_buffer, zbuffer+header_size, real_buffer_size, zbound) : 0; bool use_uncompressed_buffer = (!zused || zused > real_buffer_size); CHECK_FORCE_UNCOMPRESSED_BUFFER(); // setup payload header - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - cloudsync_payload_header header; - cloudsync_payload_header_init(&header, (use_uncompressed_buffer) ? 0 : real_buffer_size, payload->ncols, (uint32_t)payload->nrows, data->schema_hash); + cloudsync_payload_header header = {0}; + uint32_t expanded_size = (use_uncompressed_buffer) ? 0 : real_buffer_size; + cloudsync_payload_header_init(&header, expanded_size, payload->ncols, (uint32_t)payload->nrows, data->schema_hash); // if compression fails or if compressed size is bigger than original buffer, then use the uncompressed buffer if (use_uncompressed_buffer) { - cloudsync_memory_free(buffer); - buffer = payload->buffer; + if (zbuffer) cloudsync_memory_free(zbuffer); + zbuffer = payload->buffer; zused = real_buffer_size; } + // compute checksum of the buffer + uint64_t checksum = pk_checksum(zbuffer + header_size, zused); + cloudsync_payload_checksum_store(&header, checksum); + // copy header and data to SQLite BLOB - memcpy(buffer, &header, sizeof(cloudsync_payload_header)); - int blob_size = zused+sizeof(cloudsync_payload_header); - sqlite3_result_blob(context, buffer, blob_size, SQLITE_TRANSIENT); + memcpy(zbuffer, &header, sizeof(cloudsync_payload_header)); + int blob_size = zused + sizeof(cloudsync_payload_header); + payload->bsize = blob_size; // cleanup memory - cloudsync_buffer_free(payload); - if (!use_uncompressed_buffer) cloudsync_memory_free(buffer); -} - -cloudsync_payload_apply_callback_t cloudsync_get_payload_apply_callback(sqlite3 *db) { - return (sqlite3_libversion_number() >= 3044000) ? sqlite3_get_clientdata(db, CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY) : NULL; + if (zbuffer != payload->buffer) { + cloudsync_memory_free (payload->buffer); + payload->buffer = zbuffer; + } + + return DBRES_OK; } -void cloudsync_set_payload_apply_callback(sqlite3 *db, cloudsync_payload_apply_callback_t callback) { - if (sqlite3_libversion_number() >= 3044000) { - sqlite3_set_clientdata(db, CLOUDSYNC_PAYLOAD_APPLY_CALLBACK_KEY, (void*)callback, NULL); - } +char *cloudsync_payload_blob (cloudsync_payload_context *payload, int64_t *blob_size, int64_t *nrows) { + DEBUG_FUNCTION("cloudsync_payload_blob"); + + if (blob_size) *blob_size = (int64_t)payload->bsize; + if (nrows) *nrows = (int64_t)payload->nrows; + return payload->buffer; } -int cloudsync_pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { +static int cloudsync_payload_decode_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { cloudsync_pk_decode_bind_context *decode_context = (cloudsync_pk_decode_bind_context*)xdata; int rc = pk_decode_bind_callback(decode_context->vm, index, type, ival, dval, pval); - if (rc == SQLITE_OK) { + if (rc == DBRES_OK) { // the dbversion index is smaller than seq index, so it is processed first // when processing the dbversion column: save the value to the tmp_dbversion field // when processing the seq column: update the dbversion and seq fields only if the current dbversion is greater than the last max value switch (index) { case CLOUDSYNC_PK_INDEX_TBL: - if (type == SQLITE_TEXT) { + if (type == DBTYPE_TEXT) { decode_context->tbl = pval; decode_context->tbl_len = ival; } break; case CLOUDSYNC_PK_INDEX_PK: - if (type == SQLITE_BLOB) { + if (type == DBTYPE_BLOB) { decode_context->pk = pval; decode_context->pk_len = ival; } break; case CLOUDSYNC_PK_INDEX_COLNAME: - if (type == SQLITE_TEXT) { + if (type == DBTYPE_TEXT) { decode_context->col_name = pval; decode_context->col_name_len = ival; } break; case CLOUDSYNC_PK_INDEX_COLVERSION: - if (type == SQLITE_INTEGER) decode_context->col_version = ival; + if (type == DBTYPE_INTEGER) decode_context->col_version = ival; break; case CLOUDSYNC_PK_INDEX_DBVERSION: - if (type == SQLITE_INTEGER) decode_context->db_version = ival; + if (type == DBTYPE_INTEGER) decode_context->db_version = ival; break; case CLOUDSYNC_PK_INDEX_SITEID: - if (type == SQLITE_BLOB) { + if (type == DBTYPE_BLOB) { decode_context->site_id = pval; decode_context->site_id_len = ival; } break; case CLOUDSYNC_PK_INDEX_CL: - if (type == SQLITE_INTEGER) decode_context->cl = ival; + if (type == DBTYPE_INTEGER) decode_context->cl = ival; break; case CLOUDSYNC_PK_INDEX_SEQ: - if (type == SQLITE_INTEGER) decode_context->seq = ival; + if (type == DBTYPE_INTEGER) decode_context->seq = ival; break; } } @@ -2104,1024 +3210,506 @@ int cloudsync_pk_decode_bind_callback (void *xdata, int index, int type, int64_t // #ifndef CLOUDSYNC_OMIT_RLS_VALIDATION -int cloudsync_payload_apply (sqlite3_context *context, const char *payload, int blen) { +int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *pnrows) { + // Guard against calling payload_apply before cloudsync_init: without this, + // the settings lookups at the top of this function would each emit a + // "no such table: cloudsync_settings" debug line, control would fall + // through to the meta-table insert, and the function would ultimately + // return an error with an empty errmsg — SQLite then surfaces that as + // the confusing "Runtime error: not an error". + if (!cloudsync_context_is_initialized(data)) { + return cloudsync_set_error(data, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_payload_apply().", + DBRES_MISUSE); + } + + // sanity check + if (blen < (int)sizeof(cloudsync_payload_header)) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid payload length", DBRES_MISUSE); + // decode header cloudsync_payload_header header; memcpy(&header, payload, sizeof(cloudsync_payload_header)); - + header.signature = ntohl(header.signature); header.expanded_size = ntohl(header.expanded_size); header.ncols = ntohs(header.ncols); header.nrows = ntohl(header.nrows); header.schema_hash = ntohll(header.schema_hash); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + // compare schema_hash only if not disabled and if the received payload was created with the current header version + // to avoid schema hash mismatch when processed by a peer with a different extension version during software updates. + if (dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SKIP_SCHEMA_HASH_CHECK) == 0 && header.version == CLOUDSYNC_PAYLOAD_VERSION_LATEST ) { + if (header.schema_hash != data->schema_hash) { + if (!database_check_schema_hash(data, header.schema_hash)) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Cannot apply the received payload because the schema hash is unknown %llu.", header.schema_hash); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); + } + } + } // sanity check header if ((header.signature != CLOUDSYNC_PAYLOAD_SIGNATURE) || (header.ncols == 0)) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: invalid signature or column size."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return -1; + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid signature or column size", DBRES_MISUSE); } const char *buffer = payload + sizeof(cloudsync_payload_header); - blen -= sizeof(cloudsync_payload_header); - + size_t buf_len = (size_t)blen - sizeof(cloudsync_payload_header); + + // sanity check checksum (only if version is >= 2) + if (header.version >= CLOUDSYNC_PAYLOAD_MIN_VERSION_WITH_CHECKSUM) { + uint64_t checksum = pk_checksum(buffer, buf_len); + if (cloudsync_payload_checksum_verify(&header, checksum) == false) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid checksum", DBRES_MISUSE); + } + } + // check if payload is compressed char *clone = NULL; if (header.expanded_size != 0) { clone = (char *)cloudsync_memory_alloc(header.expanded_size); - if (!clone) {sqlite3_result_error_code(context, SQLITE_NOMEM); return -1;} - - uint32_t rc = LZ4_decompress_safe(buffer, clone, blen, header.expanded_size); - if (rc <= 0 || rc != header.expanded_size) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to decompress BLOB (%d).", rc); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return -1; + if (!clone) return cloudsync_set_error(data, "Unable to allocate memory to uncompress payload", DBRES_NOMEM); + + int lz4_rc = LZ4_decompress_safe(buffer, clone, (int)buf_len, (int)header.expanded_size); + if (lz4_rc <= 0 || (uint32_t)lz4_rc != header.expanded_size) { + if (clone) cloudsync_memory_free(clone); + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to decompress BLOB", DBRES_MISUSE); } - + buffer = (const char *)clone; + buf_len = (size_t)header.expanded_size; } - sqlite3 *db = sqlite3_context_db_handle(context); - // precompile the insert statement - sqlite3_stmt *vm = NULL; - const char *sql = "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) VALUES (?,?,?,?,?,?,?,?,?);"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: error while compiling SQL statement (%s).", sqlite3_errmsg(db)); + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_CHANGES_INSERT_ROW, &vm, 0); + if (rc != DBRES_OK) { if (clone) cloudsync_memory_free(clone); - return -1; + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: error while compiling SQL statement", rc); } // process buffer, one row at a time uint16_t ncols = header.ncols; uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; - bool in_savepoint = false; - int dbversion = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_DBVERSION); - int seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_SEQ); + int dbversion = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION); + int seq = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_CHECK_SEQ); cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; - void *payload_apply_xdata = NULL; - cloudsync_payload_apply_callback_t payload_apply_callback = cloudsync_get_payload_apply_callback(db); - + + // Initialize deferred column-batch merge + merge_pending_batch batch = {0}; + data->pending_batch = &batch; + bool in_savepoint = false; + const void *last_pk = NULL; + int64_t last_pk_len = 0; + const char *last_tbl = NULL; + int64_t last_tbl_len = 0; + for (uint32_t i=0; iskip_decode_idx, cloudsync_payload_decode_callback, &decoded_context); + if (res == -1) { + merge_flush_pending(data); + data->pending_batch = NULL; + if (batch.cached_vm) { databasevm_finalize(batch.cached_vm); batch.cached_vm = NULL; } + if (batch.cached_col_names) { cloudsync_memory_free(batch.cached_col_names); batch.cached_col_names = NULL; } + if (batch.entries) { cloudsync_memory_free(batch.entries); batch.entries = NULL; } + if (in_savepoint) database_rollback_savepoint(data, "cloudsync_payload_apply"); + rc = DBRES_ERROR; + goto cleanup; + } + // Detect PK/table/db_version boundary to flush pending batch + bool pk_changed = (last_pk != NULL && + (last_pk_len != decoded_context.pk_len || + memcmp(last_pk, decoded_context.pk, last_pk_len) != 0)); + bool tbl_changed = (last_tbl != NULL && + (last_tbl_len != decoded_context.tbl_len || + memcmp(last_tbl, decoded_context.tbl, last_tbl_len) != 0)); bool db_version_changed = (last_payload_db_version != decoded_context.db_version); - // Release existing savepoint if db_version changed + // Flush pending batch before any boundary change + if (pk_changed || tbl_changed || db_version_changed) { + int flush_rc = merge_flush_pending(data); + if (flush_rc != DBRES_OK) { + rc = flush_rc; + // continue processing remaining rows + } + } + + // Per-db_version savepoints group rows with the same source db_version + // into one transaction. In SQLite autocommit mode, the RELEASE triggers + // the commit hook which bumps data->db_version and resets seq, ensuring + // unique (db_version, seq) tuples across groups. In PostgreSQL SPI, + // database_in_transaction() is always true so this block is inactive — + // the inner per-PK savepoint in merge_flush_pending handles RLS instead. if (in_savepoint && db_version_changed) { - rc = sqlite3_exec(db, "RELEASE cloudsync_payload_apply;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to release a savepoint (%s).", sqlite3_errmsg(db)); - if (clone) cloudsync_memory_free(clone); - return -1; + rc = database_commit_savepoint(data, "cloudsync_payload_apply"); + if (rc != DBRES_OK) { + merge_pending_free_entries(&batch); + data->pending_batch = NULL; + cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to release a savepoint", rc); + goto cleanup; } in_savepoint = false; } - // Start new savepoint if needed - bool in_transaction = sqlite3_get_autocommit(db) != true; - if (!in_transaction && db_version_changed) { - rc = sqlite3_exec(db, "SAVEPOINT cloudsync_payload_apply;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Error on cloudsync_payload_apply: unable to start a transaction (%s).", sqlite3_errmsg(db)); - if (clone) cloudsync_memory_free(clone); - return -1; + if (!in_savepoint && db_version_changed && !database_in_transaction(data)) { + rc = database_begin_savepoint(data, "cloudsync_payload_apply"); + if (rc != DBRES_OK) { + merge_pending_free_entries(&batch); + data->pending_batch = NULL; + cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to start a transaction", rc); + goto cleanup; } - last_payload_db_version = decoded_context.db_version; in_savepoint = true; } - - if (approved) { - rc = sqlite3_step(vm); - if (rc != SQLITE_DONE) { - // don't "break;", the error can be due to a RLS policy. - // in case of error we try to apply the following changes - printf("cloudsync_payload_apply error on db_version %lld/%lld: (%d) %s\n", decoded_context.db_version, decoded_context.seq, rc, sqlite3_errmsg(db)); - } + + // Track db_version for batch-flush boundary detection + if (db_version_changed) { + last_payload_db_version = decoded_context.db_version; } - - if (payload_apply_callback) payload_apply_callback(&payload_apply_xdata, &decoded_context, db, data, CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY, rc); - + + // Update PK/table tracking + last_pk = decoded_context.pk; + last_pk_len = decoded_context.pk_len; + last_tbl = decoded_context.tbl; + last_tbl_len = decoded_context.tbl_len; + + rc = databasevm_step(vm); + if (rc != DBRES_DONE) { + // don't "break;", the error can be due to a RLS policy. + // in case of error we try to apply the following changes + } + buffer += seek; - blen -= seek; - stmt_reset(vm); + buf_len -= seek; + dbvm_reset(vm); } - + + // Final flush after loop + { + int flush_rc = merge_flush_pending(data); + if (flush_rc != DBRES_OK && rc == DBRES_OK) rc = flush_rc; + } + data->pending_batch = NULL; + if (in_savepoint) { - sql = "RELEASE cloudsync_payload_apply;"; - int rc1 = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc1 != SQLITE_OK) rc = rc1; + int rc1 = database_commit_savepoint(data, "cloudsync_payload_apply"); + if (rc1 != DBRES_OK) rc = rc1; } - char *lasterr = (rc != SQLITE_OK && rc != SQLITE_DONE) ? cloudsync_string_dup(sqlite3_errmsg(db), false) : NULL; - - if (payload_apply_callback) { - payload_apply_callback(&payload_apply_xdata, &decoded_context, db, data, CLOUDSYNC_PAYLOAD_APPLY_CLEANUP, rc); + // save last error (unused if function returns OK) + if (rc != DBRES_OK && rc != DBRES_DONE) { + cloudsync_set_dberror(data); } - if (rc == SQLITE_DONE) rc = SQLITE_OK; - if (rc == SQLITE_OK) { + if (rc == DBRES_DONE) rc = DBRES_OK; + if (rc == DBRES_OK) { char buf[256]; if (decoded_context.db_version >= dbversion) { - snprintf(buf, sizeof(buf), "%lld", decoded_context.db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); + snprintf(buf, sizeof(buf), "%" PRId64, decoded_context.db_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); if (decoded_context.seq != seq) { - snprintf(buf, sizeof(buf), "%lld", decoded_context.seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_SEQ, buf); + snprintf(buf, sizeof(buf), "%" PRId64, decoded_context.seq); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); } } } +cleanup: + // cleanup merge_pending_batch + if (batch.cached_vm) { databasevm_finalize(batch.cached_vm); batch.cached_vm = NULL; } + if (batch.cached_col_names) { cloudsync_memory_free(batch.cached_col_names); batch.cached_col_names = NULL; } + if (batch.entries) { cloudsync_memory_free(batch.entries); batch.entries = NULL; } + // cleanup vm - if (vm) sqlite3_finalize(vm); - + if (vm) databasevm_finalize(vm); + // cleanup memory if (clone) cloudsync_memory_free(clone); - - if (rc != SQLITE_OK) { - sqlite3_result_error(context, lasterr, -1); - sqlite3_result_error_code(context, SQLITE_MISUSE); - cloudsync_memory_free(lasterr); - return -1; - } - - // return the number of processed rows - sqlite3_result_int(context, nrows); - return nrows; -} -void cloudsync_payload_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_payload_decode"); - //debug_values(argc, argv); - - // sanity check payload type - if (sqlite3_value_type(argv[0]) != SQLITE_BLOB) { - dbutils_context_result_error(context, "Error on cloudsync_payload_decode: value must be a BLOB."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - // sanity check payload size - int blen = sqlite3_value_bytes(argv[0]); - if (blen < (int)sizeof(cloudsync_payload_header)) { - dbutils_context_result_error(context, "Error on cloudsync_payload_decode: invalid input size."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - // obtain payload - const char *payload = (const char *)sqlite3_value_blob(argv[0]); - - // apply changes - cloudsync_payload_apply(context, payload, blen); + // error already saved in (save last error) + if (rc != DBRES_OK) return rc; + + // return the number of processed rows + if (pnrows) *pnrows = nrows; + return DBRES_OK; } // MARK: - Payload load/store - -int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq) { - sqlite3 *db = sqlite3_context_db_handle(context); - - *db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION); - if (*db_version < 0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return SQLITE_ERROR;} - - *seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_SEQ); - if (*seq < 0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return SQLITE_ERROR;} +int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version) { + // retrieve current db_version and seq + *db_version = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_SEND_DBVERSION); + if (*db_version < 0) return DBRES_ERROR; // retrieve BLOB char sql[1024]; - snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes) " - "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, NULL)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))", *db_version, *db_version, *seq); + snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes WHERE site_id=cloudsync_siteid()) " + "SELECT * FROM (SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload, max_db_version AS max_db_version FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND db_version>%d) WHERE payload IS NOT NULL", *db_version); - int rc = dbutils_blob_int_int_select(db, sql, blob, blob_size, new_db_version, new_seq); - if (rc != SQLITE_OK) { - sqlite3_result_error(context, "cloudsync_network_send_changes unable to get changes", -1); - sqlite3_result_error_code(context, rc); - return rc; - } + int64_t len = 0; + int rc = database_select_blob_int(data, sql, blob, &len, new_db_version); + *blob_size = (int)len; + if (rc != DBRES_OK) return rc; // exit if there is no data to send - if (blob == NULL || blob_size == 0) return SQLITE_OK; + if (*blob == NULL || *blob_size == 0) return DBRES_OK; return rc; } #ifdef CLOUDSYNC_DESKTOP_OS - -void cloudsync_payload_save (sqlite3_context *context, int argc, sqlite3_value **argv) { +int cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *size) { DEBUG_FUNCTION("cloudsync_payload_save"); - // sanity check argument - if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "Unable to retrieve file path.", -1); - return; - } - - // retrieve full path to file - const char *path = (const char *)sqlite3_value_text(argv[0]); - cloudsync_file_delete(path); + // silently delete any other payload with the same name + cloudsync_file_delete(payload_path); // retrieve payload char *blob = NULL; - int blob_size = 0, db_version = 0, seq = 0; - sqlite3_int64 new_db_version = 0, new_seq = 0; - int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq); - if (rc != SQLITE_OK) return; + int blob_size = 0, db_version = 0; + int64_t new_db_version = 0; + int rc = cloudsync_payload_get(data, &blob, &blob_size, &db_version, &new_db_version); + if (rc != DBRES_OK) { + if (db_version < 0) return cloudsync_set_error(data, "Unable to retrieve db_version", rc); + return cloudsync_set_error(data, "Unable to retrieve changes in cloudsync_payload_save", rc); + } - // exit if there is no data to send - if (blob == NULL || blob_size == 0) return; + // exit if there is no data to save + if (blob == NULL || blob_size == 0) { + if (size) *size = 0; + return DBRES_OK; + } // write payload to file - bool res = cloudsync_file_write(path, blob, (size_t)blob_size); - sqlite3_free(blob); - + bool res = cloudsync_file_write(payload_path, blob, (size_t)blob_size); + cloudsync_memory_free(blob); if (res == false) { - sqlite3_result_error(context, "Unable to write payload to file path.", -1); - return; - } - - // update db_version and seq - char buf[256]; - sqlite3 *db = sqlite3_context_db_handle(context); - if (new_db_version != db_version) { - snprintf(buf, sizeof(buf), "%lld", new_db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - } - if (new_seq != seq) { - snprintf(buf, sizeof(buf), "%lld", new_seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); + return cloudsync_set_error(data, "Unable to write payload to file path", DBRES_IOERR); } // returns blob size - sqlite3_result_int64(context, (sqlite3_int64)blob_size); -} - -void cloudsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_payload_load"); - - // sanity check argument - if (sqlite3_value_type(argv[0]) != SQLITE_TEXT) { - sqlite3_result_error(context, "Unable to retrieve file path.", -1); - return; - } - - // retrieve full path to file - const char *path = (const char *)sqlite3_value_text(argv[0]); - - sqlite3_int64 payload_size = 0; - char *payload = cloudsync_file_read(path, &payload_size); - if (!payload) { - if (payload_size == -1) sqlite3_result_error(context, "Unable to read payload from file path.", -1); - if (payload) cloudsync_memory_free(payload); - return; - } - - int nrows = (payload_size) ? cloudsync_payload_apply (context, payload, (int)payload_size) : 0; - if (payload) cloudsync_memory_free(payload); - - // returns number of applied rows - if (nrows != -1) sqlite3_result_int(context, nrows); + if (size) *size = blob_size; + return DBRES_OK; } - #endif -// MARK: - Public - - -void cloudsync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_version"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, CLOUDSYNC_VERSION, -1, SQLITE_STATIC); -} - -void cloudsync_siteid (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_siteid"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3_result_blob(context, data->site_id, UUID_LEN, SQLITE_STATIC); -} - -void cloudsync_db_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_db_version"); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - int rc = db_version_check_uptodate(db, data); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to retrieve db_version (%s).", sqlite3_errmsg(db)); - return; - } - - sqlite3_result_int64(context, data->db_version); -} - -void cloudsync_db_version_next (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_db_version_next"); - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - sqlite3_int64 merging_version = (argc == 1) ? sqlite3_value_int64(argv[0]) : CLOUDSYNC_VALUE_NOTSET; - sqlite3_int64 value = db_version_next(db, data, merging_version); - if (value == -1) { - dbutils_context_result_error(context, "Unable to retrieve next_db_version (%s).", sqlite3_errmsg(db)); - return; - } - - sqlite3_result_int64(context, value); -} - -void cloudsync_seq (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_seq"); - - // retrieve context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3_result_int(context, BUMP_SEQ(data)); -} - -void cloudsync_uuid (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_uuid"); - - char value[UUID_STR_MAXLEN]; - char *uuid = cloudsync_uuid_v7_string(value, true); - sqlite3_result_text(context, uuid, -1, SQLITE_TRANSIENT); -} - -// MARK: - - -void cloudsync_set (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set"); - - // sanity check parameters - const char *key = (const char *)sqlite3_value_text(argv[0]); - const char *value = (const char *)sqlite3_value_text(argv[1]); - - // silently fails - if (key == NULL) return; - - sqlite3 *db = sqlite3_context_db_handle(context); - dbutils_settings_set_key_value(db, context, key, value); -} - -void cloudsync_set_column (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set_column"); - - const char *tbl = (const char *)sqlite3_value_text(argv[0]); - const char *col = (const char *)sqlite3_value_text(argv[1]); - const char *key = (const char *)sqlite3_value_text(argv[2]); - const char *value = (const char *)sqlite3_value_text(argv[3]); - dbutils_table_settings_set_key_value(NULL, context, tbl, col, key, value); -} - -void cloudsync_set_table (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_set_table"); - - const char *tbl = (const char *)sqlite3_value_text(argv[0]); - const char *key = (const char *)sqlite3_value_text(argv[1]); - const char *value = (const char *)sqlite3_value_text(argv[2]); - dbutils_table_settings_set_key_value(NULL, context, tbl, "*", key, value); -} - -void cloudsync_is_sync (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_is_sync"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - if (data->insync) { - sqlite3_result_int(context, 1); - return; - } - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - sqlite3_result_int(context, (table) ? (table->enabled == 0) : 0); -} - -void cloudsync_col_value (sqlite3_context *context, int argc, sqlite3_value **argv) { - // DEBUG_FUNCTION("cloudsync_col_value"); - - // argv[0] -> table name - // argv[1] -> column name - // argv[2] -> encoded pk - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in clousdsync_colvalue.", table_name); - return; - } - - // retrieve column name - const char *col_name = (const char *)sqlite3_value_text(argv[1]); - - // check for special tombstone value - if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { - sqlite3_result_null(context); - return; - } - - // extract the right col_value vm associated to the column name - sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); - if (!vm) { - sqlite3_result_error(context, "Unable to retrieve column value precompiled statement in clousdsync_colvalue.", -1); - return; - } - - // bind primary key values - int rc = pk_decode_prikey((char *)sqlite3_value_blob(argv[2]), (size_t)sqlite3_value_bytes(argv[2]), pk_decode_bind_callback, (void *)vm); - if (rc < 0) goto cleanup; - - // execute vm - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) { - rc = SQLITE_OK; - sqlite3_result_text(context, CLOUDSYNC_RLS_RESTRICTED_VALUE, -1, SQLITE_STATIC); - } else if (rc == SQLITE_ROW) { - // store value result - rc = SQLITE_OK; - sqlite3_result_value(context, sqlite3_column_value(vm, 0)); - } - -cleanup: - if (rc != SQLITE_OK) { - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_result_error(context, sqlite3_errmsg(db), -1); - } - sqlite3_reset(vm); -} - -void cloudsync_pk_encode (sqlite3_context *context, int argc, sqlite3_value **argv) { - size_t bsize = 0; - char *buffer = pk_encode_prikey(argv, argc, NULL, &bsize); - if (!buffer) { - sqlite3_result_null(context); - return; - } - sqlite3_result_blob(context, (const void *)buffer, (int)bsize, SQLITE_TRANSIENT); - cloudsync_memory_free(buffer); -} - -int cloudsync_pk_decode_set_result_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { - cloudsync_pk_decode_context *decode_context = (cloudsync_pk_decode_context *)xdata; - // decode_context->index is 1 based - // index is 0 based - if (decode_context->index != index+1) return SQLITE_OK; - - int rc = 0; - sqlite3_context *context = decode_context->context; - switch (type) { - case SQLITE_INTEGER: - sqlite3_result_int64(context, ival); - break; - - case SQLITE_FLOAT: - sqlite3_result_double(context, dval); - break; +// MARK: - Core - - case SQLITE_NULL: - sqlite3_result_null(context); - break; - - case SQLITE_TEXT: - sqlite3_result_text(context, pval, (int)ival, SQLITE_TRANSIENT); - break; - - case SQLITE_BLOB: - sqlite3_result_blob(context, pval, (int)ival, SQLITE_TRANSIENT); - break; - } - - return rc; -} - - -void cloudsync_pk_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { - const char *pk = (const char *)sqlite3_value_text(argv[0]); - int i = sqlite3_value_int(argv[1]); - - cloudsync_pk_decode_context xdata = {.context = context, .index = i}; - pk_decode_prikey((char *)pk, strlen(pk), cloudsync_pk_decode_set_result_callback, &xdata); -} - -// MARK: - - -void cloudsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_insert %s", sqlite3_value_text(argv[0])); - // debug_values(argc-1, &argv[1]); - - // argv[0] is table name - // argv[1]..[N] is primary key(s) - - // table_cloudsync - // pk -> encode(argc-1, &argv[1]) - // col_name -> name - // col_version -> 0/1 +1 - // db_version -> check - // site_id 0 - // seq -> sqlite_master - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_insert.", table_name); - return; - } +int cloudsync_table_sanity_check (cloudsync_context *data, const char *name, CLOUDSYNC_INIT_FLAG init_flags) { + DEBUG_DBFUNCTION("cloudsync_table_sanity_check %s", name); + char buffer[2048]; - // encode the primary key values into a buffer - char buffer[1024]; - size_t pklen = sizeof(buffer); - char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; + // sanity check table name + if (name == NULL) { + return cloudsync_set_error(data, "cloudsync_init requires a non-null table parameter", DBRES_ERROR); } - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - - // check if a row with the same primary key already exists - // if so, this means the row might have been previously deleted (sentinel) - bool pk_exists = (bool)stmt_count(table->meta_pkexists_stmt, pk, pklen, SQLITE_BLOB); - int rc = SQLITE_OK; - - if (table->ncols == 0) { - // if there are no columns other than primary keys, insert a sentinel record - rc = local_mark_insert_sentinel_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - } else if (pk_exists){ - // if a row with the same primary key already exists, update the sentinel record - rc = local_update_sentinel(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - } - - // process each non-primary key column for insert or update - for (int i=0; incols; ++i) { - // mark the column as inserted or updated in the metadata - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, table->col_name[i], db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; + // avoid allocating heap memory for SQL statements by setting a maximum length of 512 characters + // for table names. This limit is reasonable and helps prevent memory management issues. + const size_t maxlen = CLOUDSYNC_MAX_TABLENAME_LEN; + if (strlen(name) > maxlen) { + snprintf(buffer, sizeof(buffer), "Table name cannot be longer than %d characters", (int)maxlen); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - // free memory if the primary key was dynamically allocated - if (pk != buffer) cloudsync_memory_free(pk); -} - -void cloudsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_delete %s", sqlite3_value_text(argv[0])); - // debug_values(argc-1, &argv[1]); + // check if already initialized + cloudsync_table_context *table = table_lookup(data, name); + if (table) return DBRES_OK; - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name); - return; + // check if table exists + if (database_table_exists(data, name, cloudsync_schema(data)) == false) { + snprintf(buffer, sizeof(buffer), "Table %s does not exist", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - int rc = SQLITE_OK; + // no more than 128 columns can be used as a composite primary key (SQLite hard limit) + int npri_keys = database_count_pk(data, name, false, cloudsync_schema(data)); + if (npri_keys < 0) return cloudsync_set_dberror(data); + if (npri_keys > 128) return cloudsync_set_error(data, "No more than 128 columns can be used to form a composite primary key", DBRES_ERROR); - // encode the primary key values into a buffer - char buffer[1024]; - size_t pklen = sizeof(buffer); - char *pk = pk_encode_prikey(&argv[1], table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; + #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + // if count == 0 means that rowid will be used as primary key (BTW: very bad choice for the user) + if (npri_keys == 0) { + snprintf(buffer, sizeof(buffer), "Rowid only tables are not supported, all primary keys must be explicitly set and declared as NOT NULL (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } + #endif - // mark the row as deleted by inserting a delete sentinel into the metadata - rc = local_mark_delete_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // remove any metadata related to the old rows associated with this primary key - rc = local_drop_meta(db, table, pk, pklen); - if (rc != SQLITE_OK) goto cleanup; - -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - // free memory if the primary key was dynamically allocated - if (pk != buffer) cloudsync_memory_free(pk); -} - -// MARK: - - -void cloudsync_update_payload_free (cloudsync_update_payload *payload) { - for (int i=0; icount; i++) { - sqlite3_value_free(payload->new_values[i]); - sqlite3_value_free(payload->old_values[i]); - } - cloudsync_memory_free(payload->new_values); - cloudsync_memory_free(payload->old_values); - sqlite3_value_free(payload->table_name); - payload->new_values = NULL; - payload->old_values = NULL; - payload->table_name = NULL; - payload->count = 0; - payload->capacity = 0; -} - -int cloudsync_update_payload_append (cloudsync_update_payload *payload, sqlite3_value *v1, sqlite3_value *v2, sqlite3_value *v3) { - if (payload->count >= payload->capacity) { - int newcap = payload->capacity ? payload->capacity * 2 : 128; - - sqlite3_value **new_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->new_values, newcap * sizeof(*new_values_2)); - if (!new_values_2) return SQLITE_NOMEM; - payload->new_values = new_values_2; - - sqlite3_value **old_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->old_values, newcap * sizeof(*old_values_2)); - if (!old_values_2) return SQLITE_NOMEM; - payload->old_values = old_values_2; - - payload->capacity = newcap; - } - - int index = payload->count; - if (payload->table_name == NULL) payload->table_name = sqlite3_value_dup(v1); - else if (dbutils_value_compare(payload->table_name, v1) != 0) return SQLITE_NOMEM; - payload->new_values[index] = sqlite3_value_dup(v2); - payload->old_values[index] = sqlite3_value_dup(v3); - payload->count++; - - // sanity check memory allocations - bool v1_can_be_null = (sqlite3_value_type(v1) == SQLITE_NULL); - bool v2_can_be_null = (sqlite3_value_type(v2) == SQLITE_NULL); - bool v3_can_be_null = (sqlite3_value_type(v3) == SQLITE_NULL); - - if ((payload->table_name == NULL) && (!v1_can_be_null)) return SQLITE_NOMEM; - if ((payload->old_values[index] == NULL) && (!v2_can_be_null)) return SQLITE_NOMEM; - if ((payload->new_values[index] == NULL) && (!v3_can_be_null)) return SQLITE_NOMEM; - - return SQLITE_OK; -} - -void cloudsync_update_step (sqlite3_context *context, int argc, sqlite3_value **argv) { - // argv[0] => table_name - // argv[1] => new_column_value - // argv[2] => old_column_value - - // allocate/get the update payload - cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); - if (!payload) {sqlite3_result_error_nomem(context); return;} - - if (cloudsync_update_payload_append(payload, argv[0], argv[1], argv[2]) != SQLITE_OK) { - sqlite3_result_error_nomem(context); - } -} - -void cloudsync_update_final (sqlite3_context *context) { - cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); - if (!payload || payload->count == 0) return; - - // retrieve context - sqlite3 *db = sqlite3_context_db_handle(context); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // lookup table - const char *table_name = (const char *)sqlite3_value_text(payload->table_name); - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name); - return; - } - - // compute the next database version for tracking changes - sqlite3_int64 db_version = db_version_next(db, data, CLOUDSYNC_VALUE_NOTSET); - int rc = SQLITE_OK; - - // Check if the primary key(s) have changed - bool prikey_changed = false; - for (int i=0; inpks; ++i) { - if (dbutils_value_compare(payload->old_values[i], payload->new_values[i]) != 0) { - prikey_changed = true; - break; + bool skip_int_pk_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK) != 0; + if (!skip_int_pk_check) { + if (npri_keys == 1) { + // the affinity of a column is determined by the declared type of the column, + // according to the following rules in the order shown: + // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. + int npri_keys_int = database_count_int_pk(data, name, cloudsync_schema(data)); + if (npri_keys_int < 0) return cloudsync_set_dberror(data); + if (npri_keys == npri_keys_int) { + snprintf(buffer, sizeof(buffer), "Table %s uses a single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } + } } - - // encode the NEW primary key values into a buffer (used later for indexing) - char buffer[1024]; - char buffer2[1024]; - size_t pklen = sizeof(buffer); - size_t oldpklen = sizeof(buffer2); - char *oldpk = NULL; - - char *pk = pk_encode_prikey(payload->new_values, table->npks, buffer, &pklen); - if (!pk) { - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; - } - - if (prikey_changed) { - // if the primary key has changed, we need to handle the row differently: - // 1. mark the old row (OLD primary key) as deleted - // 2. create a new row (NEW primary key) - // encode the OLD primary key into a buffer - oldpk = pk_encode_prikey(payload->old_values, table->npks, buffer2, &oldpklen); - if (!oldpk) { - if (pk != buffer) cloudsync_memory_free(pk); - sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); - return; + // if user declared explicit primary key(s) then make sure they are all declared as NOT NULL + #if CLOUDSYNC_CHECK_NOTNULL_PRIKEYS + bool skip_notnull_prikeys_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK) != 0; + if (!skip_notnull_prikeys_check) { + if (npri_keys > 0) { + int npri_keys_notnull = database_count_pk(data, name, true, cloudsync_schema(data)); + if (npri_keys_notnull < 0) return cloudsync_set_dberror(data); + if (npri_keys != npri_keys_notnull) { + snprintf(buffer, sizeof(buffer), "All primary keys must be explicitly declared as NOT NULL (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } } - - // mark the rows with the old primary key as deleted in the metadata (old row handling) - rc = local_mark_delete_meta(db, table, oldpk, oldpklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // move non-sentinel metadata entries from OLD primary key to NEW primary key - // handles the case where some metadata is retained across primary key change - // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details - rc = local_update_move_meta(db, table, pk, pklen, oldpk, oldpklen, db_version); - if (rc != SQLITE_OK) goto cleanup; - - // mark a new sentinel row with the new primary key in the metadata - rc = local_mark_insert_sentinel_meta(db, table, pk, pklen, db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; - - // free memory if the OLD primary key was dynamically allocated - if (oldpk != buffer2) cloudsync_memory_free(oldpk); - oldpk = NULL; } + #endif - // compare NEW and OLD values (excluding primary keys) to handle column updates - for (int i=0; incols; i++) { - int col_index = table->npks + i; // Regular columns start after primary keys - - if (dbutils_value_compare(payload->old_values[col_index], payload->new_values[col_index]) != 0) { - // if a column value has changed, mark it as updated in the metadata - // columns are in cid order - rc = local_mark_insert_or_update_meta(db, table, pk, pklen, table->col_name[i], db_version, BUMP_SEQ(data)); - if (rc != SQLITE_OK) goto cleanup; + // check for columns declared as NOT NULL without a DEFAULT value. + // Otherwise, col_merge_stmt would fail if changes to other columns are inserted first. + bool skip_notnull_default_check = (init_flags & CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK) != 0; + if (!skip_notnull_default_check) { + int n_notnull_nodefault = database_count_notnull_without_default(data, name, cloudsync_schema(data)); + if (n_notnull_nodefault < 0) return cloudsync_set_dberror(data); + if (n_notnull_nodefault > 0) { + snprintf(buffer, sizeof(buffer), "All non-primary key columns declared as NOT NULL must have a DEFAULT value. (table %s)", name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } } -cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, sqlite3_errmsg(db), -1); - if (pk != buffer) cloudsync_memory_free(pk); - if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); - - cloudsync_update_payload_free(payload); + return DBRES_OK; } -// MARK: - +int cloudsync_cleanup_internal (cloudsync_context *data, cloudsync_table_context *table) { + if (cloudsync_context_init(data) == NULL) return DBRES_MISUSE; -int cloudsync_cleanup_internal (sqlite3_context *context, const char *table_name) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) return SQLITE_MISUSE; - - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return SQLITE_OK; - - table_remove_from_context(data, table); - table_free(table); - // drop meta-table - char *sql = cloudsync_memory_mprintf("DROP TABLE IF EXISTS \"%w_cloudsync\";", table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + const char *table_name = table->name; + char *sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->meta_ref); + int rc = database_exec(data, sql); cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup.", table_name); - sqlite3_result_error_code(context, rc); - return rc; + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup", table_name); + return cloudsync_set_error(data, buffer, rc); } - - // drop original triggers - dbutils_delete_triggers(db, table_name); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to drop cloudsync table %s_cloudsync in cloudsync_cleanup.", table_name); - sqlite3_result_error_code(context, rc); - return rc; - } - - // remove all table related settings - dbutils_table_settings_set_key_value(db, context, table_name, NULL, NULL, NULL); - - return SQLITE_OK; -} -void cloudsync_cleanup_all (sqlite3_context *context) { - char *sql = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'cloudsync_%' AND name NOT LIKE '%_cloudsync';"; - - sqlite3 *db = sqlite3_context_db_handle(context); - char **result = NULL; - int nrows, ncols; - char *errmsg; - int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); - if (errmsg || ncols != 1) { - printf("cloudsync_cleanup_all error: %s\n", errmsg ? errmsg : "invalid table"); - goto cleanup; - } - - rc = SQLITE_OK; - for (int i = ncols; i < nrows+ncols; i+=ncols) { - int rc2 = cloudsync_cleanup_internal(context, result[i]); - if (rc2 != SQLITE_OK) rc = rc2; - } - - if (rc == SQLITE_OK) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->site_id[0] = 0; - dbutils_settings_cleanup(db); + // drop blocks table if this table has block LWW columns + if (table->blocks_ref) { + sql = cloudsync_memory_mprintf(SQL_DROP_CLOUDSYNC_TABLE, table->blocks_ref); + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to drop blocks table %s_cloudsync_blocks in cloudsync_cleanup", table_name); + return cloudsync_set_error(data, buffer, rc); + } } - -cleanup: - sqlite3_free_table(result); - sqlite3_free(errmsg); -} -void cloudsync_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_cleanup"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - sqlite3 *db = sqlite3_context_db_handle(context); - - if (dbutils_is_star_table(table)) cloudsync_cleanup_all(context); - else cloudsync_cleanup_internal(context, table); + // drop original triggers + rc = database_delete_triggers(data, table_name); + if (rc != DBRES_OK) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unable to delete triggers for table %s", table_name); + return cloudsync_set_error(data, buffer, rc); + } - if (dbutils_table_exists(db, CLOUDSYNC_TABLE_SETTINGS_NAME) == true) dbutils_update_schema_hash(db, &data->schema_hash); + // remove all table related settings + dbutils_table_settings_set_key_value(data, table_name, NULL, NULL, NULL); + return DBRES_OK; } -void cloudsync_enable_disable (sqlite3_context *context, const char *table_name, bool value) { - DEBUG_FUNCTION("cloudsync_enable_disable"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); +int cloudsync_cleanup (cloudsync_context *data, const char *table_name) { cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) return; - - table->enabled = value; -} - -int cloudsync_enable_disable_all_callback (void *xdata, int ncols, char **values, char **names) { - sqlite3_context *context = (sqlite3_context *)xdata; - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - bool value = data->temp_bool; - - for (int i=0; ienabled = value; - } + if (!table) return DBRES_OK; - return SQLITE_OK; -} - -void cloudsync_enable_disable_all (sqlite3_context *context, bool value) { - DEBUG_FUNCTION("cloudsync_enable_disable_all"); + // TODO: check what happen if cloudsync_cleanup_internal failes (not eveything dropped) and the table is still in memory? - char *sql = "SELECT name FROM sqlite_master WHERE type='table';"; + int rc = cloudsync_cleanup_internal(data, table); + if (rc != DBRES_OK) return rc; - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->temp_bool = value; - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_exec(db, sql, cloudsync_enable_disable_all_callback, context, NULL); -} - -void cloudsync_enable (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_enable"); + int counter = table_remove(data, table); + table_free(table); - const char *table = (const char *)sqlite3_value_text(argv[0]); - if (dbutils_is_star_table(table)) cloudsync_enable_disable_all(context, true); - else cloudsync_enable_disable(context, table, true); -} - -void cloudsync_disable (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_disable"); + if (counter == 0) { + // cleanup database on last table + cloudsync_reset_siteid(data); + dbutils_settings_cleanup(data); + } else { + if (database_internal_table_exists(data, CLOUDSYNC_TABLE_SETTINGS_NAME) == true) { + cloudsync_update_schema_hash(data); + } + } - const char *table = (const char *)sqlite3_value_text(argv[0]); - if (dbutils_is_star_table(table)) cloudsync_enable_disable_all(context, false); - else cloudsync_enable_disable(context, table, false); + return DBRES_OK; } -void cloudsync_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_is_enabled"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = table_lookup(data, table_name); - - int result = (table && table->enabled) ? 1 : 0; - sqlite3_result_int(context, result); +int cloudsync_cleanup_all (cloudsync_context *data) { + return database_cleanup(data); } -void cloudsync_terminate (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_terminate"); - - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - for (int i=0; itables_count; ++i) { - if (data->tables[i]) table_free(data->tables[i]); - data->tables[i] = NULL; +int cloudsync_terminate (cloudsync_context *data) { + // can't use for/loop here because data->tables_count is changed by table_remove + while (data->tables_count > 0) { + cloudsync_table_context *t = data->tables[data->tables_count - 1]; + table_remove(data, t); + table_free(t); } - if (data->schema_version_stmt) sqlite3_finalize(data->schema_version_stmt); - if (data->data_version_stmt) sqlite3_finalize(data->data_version_stmt); - if (data->db_version_stmt) sqlite3_finalize(data->db_version_stmt); - if (data->getset_siteid_stmt) sqlite3_finalize(data->getset_siteid_stmt); + if (data->schema_version_stmt) databasevm_finalize(data->schema_version_stmt); + if (data->data_version_stmt) databasevm_finalize(data->data_version_stmt); + if (data->db_version_stmt) databasevm_finalize(data->db_version_stmt); + if (data->getset_siteid_stmt) databasevm_finalize(data->getset_siteid_stmt); + if (data->current_schema) cloudsync_memory_free(data->current_schema); data->schema_version_stmt = NULL; data->data_version_stmt = NULL; data->db_version_stmt = NULL; data->getset_siteid_stmt = NULL; + data->current_schema = NULL; // reset the site_id so the cloudsync_context_init will be executed again // if any other cloudsync function is called after terminate data->site_id[0] = 0; - sqlite3_result_int(context, 1); -} - -// MARK: - - -int cloudsync_load_siteid (sqlite3 *db, cloudsync_context *data) { - // check if site_id was already loaded - if (data->site_id[0] != 0) return SQLITE_OK; - - // load site_id - int size, rc; - char *buffer = dbutils_blob_select(db, "SELECT site_id FROM cloudsync_site_id WHERE rowid=0;", &size, data->sqlite_ctx, &rc); - if (!buffer) return rc; - if (size != UUID_LEN) return SQLITE_MISUSE; - - memcpy(data->site_id, buffer, UUID_LEN); - cloudsync_memory_free(buffer); - - return SQLITE_OK; + return 1; } -int cloudsync_init_internal (sqlite3_context *context, const char *table_name, const char *algo_name, bool skip_int_pk_check) { - DEBUG_FUNCTION("cloudsync_init_internal"); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - +int cloudsync_init_table (cloudsync_context *data, const char *table_name, const char *algo_name, CLOUDSYNC_INIT_FLAG init_flags) { // sanity check table and its primary key(s) - if (dbutils_table_sanity_check(db, context, table_name, skip_int_pk_check) == false) { - return SQLITE_MISUSE; - } + int rc = cloudsync_table_sanity_check(data, table_name, init_flags); + if (rc != DBRES_OK) return rc; // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) return SQLITE_MISUSE; + if (cloudsync_context_init(data) == NULL) { + return cloudsync_set_error(data, "Unable to initialize cloudsync context", DBRES_MISUSE); + } // sanity check algo name (if exists) table_algo algo_new = table_algo_none; - if (!algo_name) { - algo_name = CLOUDSYNC_DEFAULT_ALGO; - } + if (!algo_name) algo_name = CLOUDSYNC_DEFAULT_ALGO; - algo_new = crdt_algo_from_name(algo_name); + algo_new = cloudsync_algo_from_name(algo_name); if (algo_new == table_algo_none) { - dbutils_context_result_error(context, "algo name %s does not exist", crdt_algo_name); - return SQLITE_MISUSE; + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "Unknown CRDT algorithm name %s", algo_name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); + } + + // DWS and AWS algorithms are not yet implemented in the merge logic + if (algo_new == table_algo_crdt_dws || algo_new == table_algo_crdt_aws) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "CRDT algorithm %s is not yet supported", algo_name); + return cloudsync_set_error(data, buffer, DBRES_ERROR); } // check if table name was already augmented - table_algo algo_current = dbutils_table_settings_get_algo(db, table_name); + table_algo algo_current = dbutils_table_settings_get_algo(data, table_name); // sanity check algorithm if ((algo_new == algo_current) && (algo_current != table_algo_none)) { @@ -3134,11 +3722,10 @@ int cloudsync_init_internal (sqlite3_context *context, const char *table_name, c algo_new = algo_current; } else if ((algo_new != table_algo_none) && (algo_current == table_algo_none)) { // write table algo name in settings - dbutils_table_settings_set_key_value(NULL, context, table_name, "*", "algo", algo_name); + dbutils_table_settings_set_key_value(data, table_name, "*", "algo", algo_name); } else { // error condition - dbutils_context_result_error(context, "%s", "Before changing a table algorithm you must call cloudsync_cleanup(table_name)"); - return SQLITE_MISUSE; + return cloudsync_set_error(data, "The function cloudsync_cleanup(table) must be called before changing a table algorithm", DBRES_MISUSE); } // Run the following function even if table was already augmented. @@ -3148,424 +3735,34 @@ int cloudsync_init_internal (sqlite3_context *context, const char *table_name, c // sync algo with table (unused in this version) // cloudsync_sync_table_key(data, table_name, "*", CLOUDSYNC_KEY_ALGO, crdt_algo_name(algo_new)); + // read row-level filter from settings (if any) + char init_filter_buf[2048]; + int init_frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", init_filter_buf, sizeof(init_filter_buf)); + const char *init_filter = (init_frc == DBRES_OK && init_filter_buf[0]) ? init_filter_buf : NULL; + // check triggers - int rc = dbutils_check_triggers(db, table_name, algo_new); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "An error occurred while creating triggers: %s (%d)", sqlite3_errmsg(db), rc); - return SQLITE_MISUSE; - } + rc = database_create_triggers(data, table_name, algo_new, init_filter); + if (rc != DBRES_OK) return cloudsync_set_error(data, "An error occurred while creating triggers", DBRES_MISUSE); // check meta-table - rc = dbutils_check_metatable(db, table_name, algo_new); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "An error occurred while creating metatable: %s (%d)", sqlite3_errmsg(db), rc); - return SQLITE_MISUSE; - } + rc = database_create_metatable(data, table_name); + if (rc != DBRES_OK) return cloudsync_set_error(data, "An error occurred while creating metatable", DBRES_MISUSE); // add prepared statements - if (stmts_add_tocontext(db, data) != SQLITE_OK) { - dbutils_context_result_error(context, "%s", "An error occurred while trying to compile prepared SQL statements."); - return SQLITE_MISUSE; + if (cloudsync_add_dbvms(data) != DBRES_OK) { + return cloudsync_set_error(data, "An error occurred while trying to compile prepared SQL statements", DBRES_MISUSE); } // add table to in-memory data context - if (table_add_to_context(db, data, algo_new, table_name) == false) { - dbutils_context_result_error(context, "An error occurred while adding %s table information to global context", table_name); - return SQLITE_MISUSE; + if (table_add_to_context(data, algo_new, table_name) == false) { + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "An error occurred while adding %s table information to global context", table_name); + return cloudsync_set_error(data, buffer, DBRES_MISUSE); } - if (cloudsync_refill_metatable(db, data, table_name) != SQLITE_OK) { - dbutils_context_result_error(context, "%s", "An error occurred while trying to fill the augmented table."); - return SQLITE_MISUSE; + if (cloudsync_refill_metatable(data, table_name) != DBRES_OK) { + return cloudsync_set_error(data, "An error occurred while trying to fill the augmented table", DBRES_MISUSE); } - return SQLITE_OK; -} - -int cloudsync_init_all (sqlite3_context *context, const char *algo_name, bool skip_int_pk_check) { - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT name, '%s' FROM sqlite_master WHERE type='table' and name NOT LIKE 'sqlite_%%' AND name NOT LIKE 'cloudsync_%%' AND name NOT LIKE '%%_cloudsync';", (algo_name) ? algo_name : CLOUDSYNC_DEFAULT_ALGO); - - sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3_stmt *vm = NULL; - int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto abort_init_all; - - while (1) { - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) break; - else if (rc != SQLITE_ROW) goto abort_init_all; - - const char *table = (const char *)sqlite3_column_text(vm, 0); - const char *algo = (const char *)sqlite3_column_text(vm, 1); - rc = cloudsync_init_internal(context, table, algo, skip_int_pk_check); - if (rc != SQLITE_OK) {cloudsync_cleanup_internal(context, table); goto abort_init_all;} - } - rc = SQLITE_OK; - -abort_init_all: - if (vm) sqlite3_finalize(vm); - return rc; -} - -void cloudsync_init (sqlite3_context *context, const char *table, const char *algo, bool skip_int_pk_check) { - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - data->sqlite_ctx = context; - - sqlite3 *db = sqlite3_context_db_handle(context); - int rc = sqlite3_exec(db, "SAVEPOINT cloudsync_init;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to create cloudsync_init savepoint. %s", sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - return; - } - - if (dbutils_is_star_table(table)) rc = cloudsync_init_all(context, algo, skip_int_pk_check); - else rc = cloudsync_init_internal(context, table, algo, skip_int_pk_check); - - if (rc == SQLITE_OK) { - rc = sqlite3_exec(db, "RELEASE cloudsync_init", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to release cloudsync_init savepoint. %s", sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - } - } - - // in case of error, rollback transaction - if (rc != SQLITE_OK) { - sqlite3_exec(db, "ROLLBACK TO cloudsync_init; RELEASE cloudsync_init", NULL, NULL, NULL); - return; - } - - dbutils_update_schema_hash(db, &data->schema_hash); - - // returns site_id as TEXT - char buffer[UUID_STR_MAXLEN]; - cloudsync_uuid_v7_stringify(data->site_id, buffer, false); - sqlite3_result_text(context, buffer, -1, NULL); -} - -void cloudsync_init3 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init2"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - const char *algo = (const char *)sqlite3_value_text(argv[1]); - bool skip_int_pk_check = (bool)sqlite3_value_int(argv[2]); - - cloudsync_init(context, table, algo, skip_int_pk_check); -} - -void cloudsync_init2 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init2"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - const char *algo = (const char *)sqlite3_value_text(argv[1]); - - cloudsync_init(context, table, algo, false); -} - -void cloudsync_init1 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_init1"); - - const char *table = (const char *)sqlite3_value_text(argv[0]); - - cloudsync_init(context, table, NULL, false); -} - -// MARK: - - -void cloudsync_begin_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_begin_alter"); - char *errmsg = NULL; - char **result = NULL; - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) { - sqlite3_result_error(context, "Unable to init the cloudsync context.", -1); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - cloudsync_table_context *table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to find table %s", table_name); - sqlite3_result_error_code(context, SQLITE_MISUSE); - return; - } - - if (table->is_altering) return; - - // create a savepoint to manage the alter operations as a transaction - int rc = sqlite3_exec(db, "SAVEPOINT cloudsync_alter", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - sqlite3_result_error(context, "Unable to create cloudsync_alter savepoint.", -1); - sqlite3_result_error_code(context, rc); - goto rollback_begin_alter; - } - - int nrows, ncols; - char *sql = cloudsync_memory_mprintf("SELECT name FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); - rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); - cloudsync_memory_free(sql); - if (errmsg || ncols != 1 || nrows != table->npks) { - dbutils_context_result_error(context, "Unable to get primary keys for table %s (%s)", table_name, errmsg); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_begin_alter; - } - - // drop original triggers - dbutils_delete_triggers(db, table_name); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, "Unable to delete triggers for table %s in cloudsync_begin_alter.", table_name); - sqlite3_result_error_code(context, rc); - goto rollback_begin_alter; - } - - if (table->pk_name) sqlite3_free_table(table->pk_name); - table->pk_name = result; - table->is_altering = true; - return; - -rollback_begin_alter: - sqlite3_exec(db, "ROLLBACK TO cloudsync_alter; RELEASE cloudsync_alter;", NULL, NULL, NULL); - -cleanup_begin_alter: - sqlite3_free_table(result); - sqlite3_free(errmsg); -} - -void cloudsync_commit_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_commit_alter"); - - const char *table_name = (const char *)sqlite3_value_text(argv[0]); - cloudsync_table_context *table = NULL; - - // get database reference - sqlite3 *db = sqlite3_context_db_handle(context); - - // retrieve global context - cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); - - // init cloudsync_settings - if (cloudsync_context_init(db, data, context) == NULL) { - dbutils_context_result_error(context, "Unable to init the cloudsync context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; - } - - table = table_lookup(data, table_name); - if (!table) { - dbutils_context_result_error(context, "Unable to find table context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; - } - - if (!table->is_altering) return; - - if (!table->pk_name) { - dbutils_context_result_error(context, "Unable to find table context."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto rollback_finalize_alter; - } - - int rc = cloudsync_finalize_alter(context, data, table); - if (rc != SQLITE_OK) goto rollback_finalize_alter; - - // the table is outdated, delete it and it will be reloaded in the cloudsync_init_internal - table_remove(data, table_name); - table_free(table); - table = NULL; - - // init again cloudsync for the table - table_algo algo_current = dbutils_table_settings_get_algo(db, table_name); - if (algo_current == table_algo_none) algo_current = dbutils_table_settings_get_algo(db, "*"); - rc = cloudsync_init_internal(context, table_name, crdt_algo_name(algo_current), true); - if (rc != SQLITE_OK) goto rollback_finalize_alter; - - // release savepoint - rc = sqlite3_exec(db, "RELEASE cloudsync_alter", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - dbutils_context_result_error(context, sqlite3_errmsg(db)); - sqlite3_result_error_code(context, rc); - goto rollback_finalize_alter; - } - - dbutils_update_schema_hash(db, &data->schema_hash); - - // flag is reset here; note that table_remove + table_free destroyed the old table, - // and cloudsync_init_internal created a fresh one with is_altering = false (zero-alloc). - - return; - -rollback_finalize_alter: - sqlite3_exec(db, "ROLLBACK TO cloudsync_alter; RELEASE cloudsync_alter;", NULL, NULL, NULL); - if (table) { - sqlite3_free_table(table->pk_name); - table->pk_name = NULL; - table->is_altering = false; - } -} - -// MARK: - Main Entrypoint - - -int cloudsync_register (sqlite3 *db, char **pzErrMsg) { - int rc = SQLITE_OK; - - // there's no built-in way to verify if sqlite3_cloudsync_init has already been called - // for this specific database connection, we use a workaround: we attempt to retrieve the - // cloudsync_version and check for an error, an error indicates that initialization has not been performed - if (sqlite3_exec(db, "SELECT cloudsync_version();", NULL, NULL, NULL) == SQLITE_OK) return SQLITE_OK; - - // init memory debugger (NOOP in production) - cloudsync_memory_init(1); - - // init context - void *ctx = cloudsync_context_create(); - if (!ctx) { - if (pzErrMsg) *pzErrMsg = "Not enought memory to create a database context"; - return SQLITE_NOMEM; - } - - // register functions - - // PUBLIC functions - rc = dbutils_register_function(db, "cloudsync_version", cloudsync_version, 0, pzErrMsg, ctx, cloudsync_context_free); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init1, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init2, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_init", cloudsync_init3, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - - rc = dbutils_register_function(db, "cloudsync_enable", cloudsync_enable, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_disable", cloudsync_disable, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_is_enabled", cloudsync_is_enabled, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_cleanup", cloudsync_cleanup, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_terminate", cloudsync_terminate, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set", cloudsync_set, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set_table", cloudsync_set_table, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_set_column", cloudsync_set_column, 4, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_siteid", cloudsync_siteid, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version", cloudsync_db_version, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version_next", cloudsync_db_version_next, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_db_version_next", cloudsync_db_version_next, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_begin_alter", cloudsync_begin_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_commit_alter", cloudsync_commit_alter, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_uuid", cloudsync_uuid, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - // PAYLOAD - rc = dbutils_register_aggregate(db, "cloudsync_payload_encode", cloudsync_payload_encode_step, cloudsync_payload_encode_final, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_payload_decode", cloudsync_payload_decode, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - #ifdef CLOUDSYNC_DESKTOP_OS - rc = dbutils_register_function(db, "cloudsync_payload_save", cloudsync_payload_save, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_payload_load", cloudsync_payload_load, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - #endif - - // PRIVATE functions - rc = dbutils_register_function(db, "cloudsync_is_sync", cloudsync_is_sync, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_insert", cloudsync_insert, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_aggregate(db, "cloudsync_update", cloudsync_update_step, cloudsync_update_final, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_delete", cloudsync_delete, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_col_value", cloudsync_col_value, 3, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_pk_encode", cloudsync_pk_encode, -1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_pk_decode", cloudsync_pk_decode, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_seq", cloudsync_seq, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - // NETWORK LAYER - #ifndef CLOUDSYNC_OMIT_NETWORK - rc = cloudsync_network_register(db, pzErrMsg, ctx); - if (rc != SQLITE_OK) return rc; - #endif - - cloudsync_context *data = (cloudsync_context *)ctx; - sqlite3_commit_hook(db, cloudsync_commit_hook, ctx); - sqlite3_rollback_hook(db, cloudsync_rollback_hook, ctx); - - // register eponymous only changes virtual table - rc = cloudsync_vtab_register_changes (db, data); - if (rc != SQLITE_OK) return rc; - - // load config, if exists - if (cloudsync_config_exists(db)) { - cloudsync_context_init(db, ctx, NULL); - - // make sure to update internal version to current version - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); - } - - return SQLITE_OK; -} - -APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { - DEBUG_FUNCTION("sqlite3_cloudsync_init"); - - #ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT2(pApi); - #endif - - return cloudsync_register(db, pzErrMsg); + return DBRES_OK; } diff --git a/src/cloudsync.h b/src/cloudsync.h index 4fb84b7..56c4d2b 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -9,20 +9,149 @@ #define __CLOUDSYNC__ #include +#include #include -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif +#include "database.h" +#include "block.h" #ifdef __cplusplus extern "C" { #endif -#define CLOUDSYNC_VERSION "0.8.68" +#define CLOUDSYNC_VERSION "1.0.20" +#define CLOUDSYNC_MAX_TABLENAME_LEN 512 + +#define CLOUDSYNC_VALUE_NOTSET -1 +#define CLOUDSYNC_TOMBSTONE_VALUE "__[RIP]__" +#define CLOUDSYNC_RLS_RESTRICTED_VALUE "__[RLS]__" +#define CLOUDSYNC_DISABLE_ROWIDONLY_TABLES 1 +#define CLOUDSYNC_DEFAULT_ALGO "cls" + +#define CLOUDSYNC_CHANGES_NCOLS 9 + +typedef enum { + CLOUDSYNC_INIT_FLAG_NONE = 0, + CLOUDSYNC_INIT_FLAG_SKIP_INT_PK_CHECK = 1 << 0, // 1 + CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_DEFAULT_CHECK = 1 << 1, // 2 + CLOUDSYNC_INIT_FLAG_SKIP_NOT_NULL_PRIKEYS_CHECK = 1 << 2 // 4 +} CLOUDSYNC_INIT_FLAG; + +// CRDT Algos +table_algo cloudsync_algo_from_name (const char *algo_name); +const char *cloudsync_algo_name (table_algo algo); + +// Opaque structures +typedef struct cloudsync_payload_context cloudsync_payload_context; +typedef struct cloudsync_table_context cloudsync_table_context; + +// CloudSync context +cloudsync_context *cloudsync_context_create (void *db); +const char *cloudsync_context_init (cloudsync_context *data); +void cloudsync_context_free (void *ctx); + +// CloudSync global +int cloudsync_init_table (cloudsync_context *data, const char *table_name, const char *algo_name, CLOUDSYNC_INIT_FLAG init_flags); +int cloudsync_cleanup (cloudsync_context *data, const char *table_name); +int cloudsync_cleanup_all (cloudsync_context *data); +int cloudsync_terminate (cloudsync_context *data); +int cloudsync_insync (cloudsync_context *data); +int cloudsync_bumpseq (cloudsync_context *data); +void *cloudsync_siteid (cloudsync_context *data); +void cloudsync_reset_siteid (cloudsync_context *data); +void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value); +int64_t cloudsync_dbversion_next (cloudsync_context *data, int64_t merging_version); +int64_t cloudsync_dbversion (cloudsync_context *data); +void cloudsync_update_schema_hash (cloudsync_context *data); +int cloudsync_dbversion_check_uptodate (cloudsync_context *data); +bool cloudsync_config_exists (cloudsync_context *data); +bool cloudsync_context_is_initialized (cloudsync_context *data); +dbvm_t *cloudsync_colvalue_stmt (cloudsync_context *data, const char *tbl_name, bool *persistent); + +// CloudSync alter table +int cloudsync_begin_alter (cloudsync_context *data, const char *table_name); +int cloudsync_commit_alter (cloudsync_context *data, const char *table_name); + +// CloudSync getter/setter +void *cloudsync_db (cloudsync_context *data); +void *cloudsync_auxdata (cloudsync_context *data); +void cloudsync_set_auxdata (cloudsync_context *data, void *xdata); +int cloudsync_set_error (cloudsync_context *data, const char *err_user, int err_code); +int cloudsync_set_dberror (cloudsync_context *data); +const char *cloudsync_errmsg (cloudsync_context *data); +int cloudsync_errcode (cloudsync_context *data); +void cloudsync_reset_error (cloudsync_context *data); +int cloudsync_commit_hook (void *ctx); +void cloudsync_rollback_hook (void *ctx); +void cloudsync_set_schema (cloudsync_context *data, const char *schema); +const char *cloudsync_schema (cloudsync_context *data); +const char *cloudsync_table_schema (cloudsync_context *data, const char *table_name); + +// Payload +int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *nrows); +int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv); +int cloudsync_payload_encode_final (cloudsync_payload_context *payload, cloudsync_context *data); +char *cloudsync_payload_blob (cloudsync_payload_context *payload, int64_t *blob_size, int64_t *nrows); +size_t cloudsync_payload_context_size (size_t *header_size); +int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version); +int cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *blob_size); // available only on Desktop OS (no WASM, no mobile) + +// CloudSync table context +int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name); +int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name); +cloudsync_table_context *table_lookup (cloudsync_context *data, const char *table_name); +void *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index); +bool table_enabled (cloudsync_table_context *table); +void table_set_enabled (cloudsync_table_context *table, bool value); +bool table_add_to_context (cloudsync_context *data, table_algo algo, const char *table_name); +bool table_pk_exists (cloudsync_table_context *table, const char *value, size_t len); +int table_count_cols (cloudsync_table_context *table); +int table_count_pks (cloudsync_table_context *table); +const char *table_colname (cloudsync_table_context *table, int index); +char **table_pknames (cloudsync_table_context *table); +void table_set_pknames (cloudsync_table_context *table, char **pknames); +bool table_algo_isgos (cloudsync_table_context *table); +const char *table_schema (cloudsync_table_context *table); +int table_remove (cloudsync_context *data, cloudsync_table_context *table); +void table_free (cloudsync_table_context *table); + +// Block-level LWW support +bool table_has_block_cols (cloudsync_table_context *table); +col_algo_t table_col_algo (cloudsync_table_context *table, int index); +const char *table_col_delimiter (cloudsync_table_context *table, int index); +int table_col_index (cloudsync_table_context *table, const char *col_name); +int block_materialize_column (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name); +int cloudsync_setup_block_column (cloudsync_context *data, const char *table_name, const char *col_name, const char *delimiter, bool persist); + +// Block column accessors (avoids accessing opaque struct from outside cloudsync.c) +dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table); +dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table); +dbvm_t *table_block_list_stmt (cloudsync_table_context *table); +const char *table_blocks_ref (cloudsync_table_context *table); +void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter); + +// local merge/apply +int local_mark_insert_sentinel_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_update_sentinel (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *col_name, int64_t db_version, int seq); +int local_mark_delete_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); +int local_mark_delete_block_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname, int64_t db_version, int seq); +int block_delete_value_external (cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname); +int local_drop_meta (cloudsync_table_context *table, const void *pk, size_t pklen); +int local_update_move_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const void *pk2, size_t pklen2, int64_t db_version); + +// used by changes virtual table +int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid); +int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, int64_t insert_cl, const char *insert_name, dbvalue_t *insert_value, int64_t insert_col_version, int64_t insert_db_version, const char *insert_site_id, int insert_site_id_len, int64_t insert_seq, int64_t *rowid); + +// filter rewrite +char *cloudsync_filter_add_row_prefix(const char *filter, const char *prefix, char **columns, int ncols); -int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); +// decode bind context +char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len); +void *cloudsync_pk_context_pk (cloudsync_pk_decode_bind_context *ctx, int64_t *pk_len); +char *cloudsync_pk_context_colname (cloudsync_pk_decode_bind_context *ctx, int64_t *colname_len); +int64_t cloudsync_pk_context_cl (cloudsync_pk_decode_bind_context *ctx); +int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx); #ifdef __cplusplus } diff --git a/src/cloudsync_endian.h b/src/cloudsync_endian.h new file mode 100644 index 0000000..4109ea7 --- /dev/null +++ b/src/cloudsync_endian.h @@ -0,0 +1,99 @@ +// +// cloudsync_endian.h +// cloudsync +// +// Created by Marco Bambini on 17/01/26. +// + +#ifndef __CLOUDSYNC_ENDIAN__ +#define __CLOUDSYNC_ENDIAN__ + +#include + +#if defined(_MSC_VER) + #include // _byteswap_uint64 +#endif + +// ======================================================= +// bswap64 - portable +// ======================================================= + +static inline uint64_t bswap64_u64(uint64_t v) { +#if defined(_MSC_VER) + return _byteswap_uint64(v); + +#elif defined(__has_builtin) + #if __has_builtin(__builtin_bswap64) + return __builtin_bswap64(v); + #else + return ((v & 0x00000000000000FFull) << 56) | + ((v & 0x000000000000FF00ull) << 40) | + ((v & 0x0000000000FF0000ull) << 24) | + ((v & 0x00000000FF000000ull) << 8) | + ((v & 0x000000FF00000000ull) >> 8) | + ((v & 0x0000FF0000000000ull) >> 24) | + ((v & 0x00FF000000000000ull) >> 40) | + ((v & 0xFF00000000000000ull) >> 56); + #endif + +#elif defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(v); + +#else + return ((v & 0x00000000000000FFull) << 56) | + ((v & 0x000000000000FF00ull) << 40) | + ((v & 0x0000000000FF0000ull) << 24) | + ((v & 0x00000000FF000000ull) << 8) | + ((v & 0x000000FF00000000ull) >> 8) | + ((v & 0x0000FF0000000000ull) >> 24) | + ((v & 0x00FF000000000000ull) >> 40) | + ((v & 0xFF00000000000000ull) >> 56); +#endif +} + +// ======================================================= +// Compile-time endianness detection +// ======================================================= + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__) + #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define HOST_IS_LITTLE_ENDIAN 1 + #elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define HOST_IS_LITTLE_ENDIAN 0 + #endif +#endif + +// WebAssembly is currently defined as little-endian in all major toolchains +#if !defined(HOST_IS_LITTLE_ENDIAN) && (defined(__wasm__) || defined(__EMSCRIPTEN__)) + #define HOST_IS_LITTLE_ENDIAN 1 +#endif + +// Runtime fallback if unknown at compile-time +static inline int host_is_little_endian_runtime (void) { + const uint16_t x = 1; + return *((const uint8_t*)&x) == 1; +} + +// ======================================================= +// Public API +// ======================================================= + +static inline uint64_t host_to_be64 (uint64_t v) { +#if defined(HOST_IS_LITTLE_ENDIAN) + #if HOST_IS_LITTLE_ENDIAN + return bswap64_u64(v); + #else + return v; + #endif +#else + return host_is_little_endian_runtime() ? bswap64_u64(v) : v; +#endif +} + +static inline uint64_t be64_to_host (uint64_t v) { + // same operation (bswap if little-endian) + return host_to_be64(v); +} + +#endif + diff --git a/src/cloudsync_private.h b/src/cloudsync_private.h deleted file mode 100644 index ae5575f..0000000 --- a/src/cloudsync_private.h +++ /dev/null @@ -1,55 +0,0 @@ -// -// cloudsync_private.h -// cloudsync -// -// Created by Marco Bambini on 30/05/25. -// - -#ifndef __CLOUDSYNC_PRIVATE__ -#define __CLOUDSYNC_PRIVATE__ - -#include -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif - - -#define CLOUDSYNC_TOMBSTONE_VALUE "__[RIP]__" -#define CLOUDSYNC_RLS_RESTRICTED_VALUE "__[RLS]__" -#define CLOUDSYNC_DISABLE_ROWIDONLY_TABLES 1 - -typedef enum { - CLOUDSYNC_PAYLOAD_APPLY_WILL_APPLY = 1, - CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY = 2, - CLOUDSYNC_PAYLOAD_APPLY_CLEANUP = 3 -} CLOUDSYNC_PAYLOAD_APPLY_STEPS; - -typedef struct cloudsync_context cloudsync_context; -typedef struct cloudsync_pk_decode_bind_context cloudsync_pk_decode_bind_context; - -int cloudsync_merge_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid); -void cloudsync_sync_key (cloudsync_context *data, const char *key, const char *value); - -// used by network layer -const char *cloudsync_context_init (sqlite3 *db, cloudsync_context *data, sqlite3_context *context); -void *cloudsync_get_auxdata (sqlite3_context *context); -void cloudsync_set_auxdata (sqlite3_context *context, void *xdata); -int cloudsync_payload_apply (sqlite3_context *context, const char *payload, int blen); -int cloudsync_payload_get (sqlite3_context *context, char **blob, int *blob_size, int *db_version, int *seq, sqlite3_int64 *new_db_version, sqlite3_int64 *new_seq); - -// used by core -typedef bool (*cloudsync_payload_apply_callback_t)(void **xdata, cloudsync_pk_decode_bind_context *decoded_change, sqlite3 *db, cloudsync_context *data, int step, int rc); -void cloudsync_set_payload_apply_callback(sqlite3 *db, cloudsync_payload_apply_callback_t callback); - -bool cloudsync_config_exists (sqlite3 *db); -sqlite3_stmt *cloudsync_colvalue_stmt (sqlite3 *db, cloudsync_context *data, const char *tbl_name, bool *persistent); -char *cloudsync_pk_context_tbl (cloudsync_pk_decode_bind_context *ctx, int64_t *tbl_len); -void *cloudsync_pk_context_pk (cloudsync_pk_decode_bind_context *ctx, int64_t *pk_len); -char *cloudsync_pk_context_colname (cloudsync_pk_decode_bind_context *ctx, int64_t *colname_len); -int64_t cloudsync_pk_context_cl (cloudsync_pk_decode_bind_context *ctx); -int64_t cloudsync_pk_context_dbversion (cloudsync_pk_decode_bind_context *ctx); - - -#endif diff --git a/src/database.h b/src/database.h new file mode 100644 index 0000000..56bb2d6 --- /dev/null +++ b/src/database.h @@ -0,0 +1,163 @@ +// +// database.h +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +#ifndef __CLOUDSYNC_DATABASE__ +#define __CLOUDSYNC_DATABASE__ + +#include +#include +#include +#include + +typedef void dbvm_t; +typedef void dbvalue_t; + +typedef enum { + DBRES_OK = 0, + DBRES_ERROR = 1, + DBRES_ABORT = 4, + DBRES_NOMEM = 7, + DBRES_IOERR = 10, + DBRES_CONSTRAINT = 19, + DBRES_MISUSE = 21, + DBRES_ROW = 100, + DBRES_DONE = 101 +} DBRES; + +typedef enum { + DBTYPE_INTEGER = 1, + DBTYPE_FLOAT = 2, + DBTYPE_TEXT = 3, + DBTYPE_BLOB = 4, + DBTYPE_NULL = 5 +} DBTYPE; + +typedef enum { + DBFLAG_PERSISTENT = 0x01 +} DBFLAG; + +// The type of CRDT chosen for a table controls what rows are included or excluded when merging tables together from different databases +typedef enum { + table_algo_none = 0, + table_algo_crdt_cls = 100, // CausalLengthSet + table_algo_crdt_gos, // GrowOnlySet + table_algo_crdt_dws, // DeleteWinsSet + table_algo_crdt_aws // AddWinsSet +} table_algo; + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +// OPAQUE STRUCT +typedef struct cloudsync_context cloudsync_context; + +// CALLBACK +typedef int (*database_exec_cb) (void *xdata, int argc, char **values, char **names); + +int database_exec (cloudsync_context *data, const char *sql); +int database_exec_callback (cloudsync_context *data, const char *sql, database_exec_cb, void *xdata); +int database_select_int (cloudsync_context *data, const char *sql, int64_t *value); +int database_select_text (cloudsync_context *data, const char *sql, char **value); +int database_select_blob (cloudsync_context *data, const char *sql, char **value, int64_t *value_len); +int database_select_blob_int (cloudsync_context *data, const char *sql, char **value, int64_t *value_len, int64_t *value2); +int database_write (cloudsync_context *data, const char *sql, const char **values, DBTYPE types[], int lens[], int count); +bool database_table_exists (cloudsync_context *data, const char *table_name, const char *schema); +bool database_internal_table_exists (cloudsync_context *data, const char *name); +bool database_trigger_exists (cloudsync_context *data, const char *table_name); +int database_create_metatable (cloudsync_context *data, const char *table_name); +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter); +int database_delete_triggers (cloudsync_context *data, const char *table_name); +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count); +int database_cleanup (cloudsync_context *data); + +int database_count_pk (cloudsync_context *data, const char *table_name, bool not_null, const char *schema); +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema); +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema); +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema); + +int64_t database_schema_version (cloudsync_context *data); +uint64_t database_schema_hash (cloudsync_context *data); +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash); +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash); + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name); +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name); +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name); +bool database_in_transaction (cloudsync_context *data); +int database_errcode (cloudsync_context *data); +const char *database_errmsg (cloudsync_context *data); + +// VM +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags); +int databasevm_step (dbvm_t *vm); +void databasevm_finalize (dbvm_t *vm); +void databasevm_reset (dbvm_t *vm); +void databasevm_clear_bindings (dbvm_t *vm); +const char *databasevm_sql (dbvm_t *vm); + +// BINDING +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size); +int databasevm_bind_double (dbvm_t *vm, int index, double value); +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value); +int databasevm_bind_null (dbvm_t *vm, int index); +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size); +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value); + +// VALUE +const void *database_value_blob (dbvalue_t *value); +double database_value_double (dbvalue_t *value); +int64_t database_value_int (dbvalue_t *value); +const char *database_value_text (dbvalue_t *value); +int database_value_bytes (dbvalue_t *value); +int database_value_type (dbvalue_t *value); +void database_value_free (dbvalue_t *value); +void *database_value_dup (dbvalue_t *value); + +// COLUMN +const void *database_column_blob (dbvm_t *vm, int index, size_t *len); +double database_column_double (dbvm_t *vm, int index); +int64_t database_column_int (dbvm_t *vm, int index); +const char *database_column_text (dbvm_t *vm, int index); +dbvalue_t *database_column_value (dbvm_t *vm, int index); +int database_column_bytes (dbvm_t *vm, int index); +int database_column_type (dbvm_t *vm, int index); + +// MEMORY +void *dbmem_alloc (uint64_t size); +void *dbmem_zeroalloc (uint64_t size); +void *dbmem_realloc (void *ptr, uint64_t new_size); +char *dbmem_mprintf(const char *format, ...); +char *dbmem_vmprintf (const char *format, va_list list); +void dbmem_free (void *ptr); +uint64_t dbmem_size (void *ptr); + +// SQL +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta); +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema); +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema); +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema); +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema); +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema); +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col); +char *sql_build_delete_cols_not_in_schema_query(const char *schema, const char *table_name, const char *meta_ref, const char *pkcol); +char *sql_build_pk_collist_query(const char *schema, const char *table_name); +char *sql_build_pk_decode_selectlist_query(const char *schema, const char *table_name); +char *sql_build_pk_qualified_collist_query(const char *schema, const char *table_name); +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, const char *pkvalues_identifiers, const char *base_ref, const char *meta_ref, const char *filter); + +char *database_table_schema(const char *table_name); +char *database_build_meta_ref(const char *schema, const char *table_name); +char *database_build_base_ref(const char *schema, const char *table_name); +char *database_build_blocks_ref(const char *schema, const char *table_name); + +// OPAQUE STRUCT used by pk_context functions +typedef struct cloudsync_pk_decode_bind_context cloudsync_pk_decode_bind_context; + +#endif diff --git a/src/dbutils.c b/src/dbutils.c index 6278d0d..4e565fe 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -6,14 +6,13 @@ // #include +#include + +#include "sql.h" #include "utils.h" #include "dbutils.h" #include "cloudsync.h" -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - #if CLOUDSYNC_UNITTEST char *OUT_OF_MEMORY_BUFFER = "OUT_OF_MEMORY_BUFFER"; #ifndef SQLITE_MAX_ALLOCATION_SIZE @@ -21,235 +20,54 @@ char *OUT_OF_MEMORY_BUFFER = "OUT_OF_MEMORY_BUFFER"; #endif #endif -typedef struct { - int type; - int len; - int rc; - union { - sqlite3_int64 intValue; - double doubleValue; - char *stringValue; - } value; -} DATABASE_RESULT; - -typedef struct { - sqlite3 *db; - cloudsync_context *data; -} dbutils_settings_table_context; - -int dbutils_settings_check_version (sqlite3 *db, const char *version); - -// MARK: - General - - -DATABASE_RESULT dbutils_exec (sqlite3_context *context, sqlite3 *db, const char *sql, const char **values, int types[], int lens[], int count, DATABASE_RESULT results[], int expected_types[], int result_count) { - DEBUG_DBFUNCTION("dbutils_exec %s", sql); - - sqlite3_stmt *pstmt = NULL; - bool is_write = (result_count == 0); - #ifdef CLOUDSYNC_UNITTEST - bool is_test = (result_count == 1 && expected_types[0] == SQLITE_NOMEM); - #endif - int type = 0; - - // compile sql - int rc = sqlite3_prepare_v2(db, sql, -1, &pstmt, NULL); - if (rc != SQLITE_OK) goto dbutils_exec_finalize; - - // check bindings - for (int i=0; i r_int); } break; - case SQLITE_FLOAT: { - double l_double = sqlite3_value_double(lvalue); - double r_double = sqlite3_value_double(rvalue); + case DBTYPE_FLOAT: { + double l_double = database_value_double(lvalue); + double r_double = database_value_double(rvalue); return (l_double < r_double) ? -1 : (l_double > r_double); } break; - case SQLITE_NULL: + case DBTYPE_NULL: break; - case SQLITE_TEXT: { - const unsigned char *l_text = sqlite3_value_text(lvalue); - const unsigned char *r_text = sqlite3_value_text(rvalue); + case DBTYPE_TEXT: { + const char *l_text = database_value_text(lvalue); + const char *r_text = database_value_text(rvalue); + if (l_text == NULL && r_text == NULL) return 0; + if (l_text == NULL && r_text != NULL) return -1; + if (l_text != NULL && r_text == NULL) return 1; return strcmp((const char *)l_text, (const char *)r_text); } break; - case SQLITE_BLOB: { - const void *l_blob = sqlite3_value_blob(lvalue); - const void *r_blob = sqlite3_value_blob(rvalue); - int l_size = sqlite3_value_bytes(lvalue); - int r_size = sqlite3_value_bytes(rvalue); + case DBTYPE_BLOB: { + const void *l_blob = database_value_blob(lvalue); + const void *r_blob = database_value_blob(rvalue); + if (l_blob == NULL && r_blob == NULL) return 0; + if (l_blob == NULL && r_blob != NULL) return -1; + if (l_blob != NULL && r_blob == NULL) return 1; + int l_size = database_value_bytes(lvalue); + int r_size = database_value_bytes(rvalue); int cmp = memcmp(l_blob, r_blob, (l_size < r_size) ? l_size : r_size); return (cmp != 0) ? cmp : (l_size - r_size); } break; @@ -258,553 +76,142 @@ int dbutils_value_compare (sqlite3_value *lvalue, sqlite3_value *rvalue) { return 0; } -void dbutils_context_result_error (sqlite3_context *context, const char *format, ...) { - char buffer[4096]; - - va_list arg; - va_start (arg, format); - vsnprintf(buffer, sizeof(buffer), format, arg); - va_end (arg); - - if (context) sqlite3_result_error(context, buffer, -1); -} - -// MARK: - - -void dbutils_debug_value (sqlite3_value *value) { - switch (sqlite3_value_type(value)) { - case SQLITE_INTEGER: - printf("\t\tINTEGER: %lld\n", sqlite3_value_int64(value)); +void dbutils_debug_value (dbvalue_t *value) { + switch (database_value_type(value)) { + case DBTYPE_INTEGER: + printf("\t\tINTEGER: %" PRId64 "\n", database_value_int(value)); break; - case SQLITE_FLOAT: - printf("\t\tFLOAT: %f\n", sqlite3_value_double(value)); + case DBTYPE_FLOAT: + printf("\t\tFLOAT: %f\n", database_value_double(value)); break; - case SQLITE_TEXT: - printf("\t\tTEXT: %s (%d)\n", sqlite3_value_text(value), sqlite3_value_bytes(value)); + case DBTYPE_TEXT: + printf("\t\tTEXT: %s (%d)\n", database_value_text(value), database_value_bytes(value)); break; - case SQLITE_BLOB: - printf("\t\tBLOB: %p (%d)\n", (char *)sqlite3_value_blob(value), sqlite3_value_bytes(value)); + case DBTYPE_BLOB: + printf("\t\tBLOB: %p (%d)\n", (char *)database_value_blob(value), database_value_bytes(value)); break; - case SQLITE_NULL: + case DBTYPE_NULL: printf("\t\tNULL\n"); break; } } -void dbutils_debug_values (int argc, sqlite3_value **argv) { +void dbutils_debug_values (dbvalue_t **argv, int argc) { for (int i = 0; i < argc; i++) { dbutils_debug_value(argv[i]); } } -int dbutils_debug_stmt (sqlite3 *db, bool print_result) { - sqlite3_stmt *stmt = NULL; - int counter = 0; - while ((stmt = sqlite3_next_stmt(db, stmt))) { - ++counter; - if (print_result) printf("Unfinalized stmt statement: %p\n", stmt); - } - return counter; -} - -// MARK: - - -int dbutils_register_function (sqlite3 *db, const char *name, void (*ptr)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { - DEBUG_DBFUNCTION("dbutils_register_function %s", name); - - const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; - int rc = sqlite3_create_function_v2(db, name, nargs, DEFAULT_FLAGS, ctx, ptr, NULL, NULL, ctx_free); - - if (rc != SQLITE_OK) { - if (pzErrMsg) *pzErrMsg = cloudsync_memory_mprintf("Error creating function %s: %s", name, sqlite3_errmsg(db)); - return rc; - } - - return SQLITE_OK; -} - -int dbutils_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { - DEBUG_DBFUNCTION("dbutils_register_aggregate %s", name); - - const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; - int rc = sqlite3_create_function_v2(db, name, nargs, DEFAULT_FLAGS, ctx, NULL, xstep, xfinal, ctx_free); - - if (rc != SQLITE_OK) { - if (pzErrMsg) *pzErrMsg = cloudsync_memory_mprintf("Error creating aggregate function %s: %s", name, sqlite3_errmsg(db)); - return rc; - } - - return SQLITE_OK; -} - -bool dbutils_system_exists (sqlite3 *db, const char *name, const char *type) { - DEBUG_DBFUNCTION("dbutils_system_exists %s: %s", type, name); - - sqlite3_stmt *vm = NULL; - bool result = false; - - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type='%s' AND name=?1 COLLATE NOCASE);", type); - int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_bind_text(vm, 1, name, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_step(vm); - if (rc == SQLITE_ROW) { - result = (bool)sqlite3_column_int(vm, 0); - rc = SQLITE_OK; - } - -finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("Error executing %s in dbutils_system_exists for type %s name %s (%s).", sql, type, name, sqlite3_errmsg(db)); - if (vm) sqlite3_finalize(vm); - return result; -} - -bool dbutils_table_exists (sqlite3 *db, const char *name) { - return dbutils_system_exists(db, name, "table"); -} +// MARK: - Settings - -bool dbutils_trigger_exists (sqlite3 *db, const char *name) { - return dbutils_system_exists(db, name, "trigger"); +int dbutils_binary_comparison (int x, int y) { + return (x == y) ? 0 : (x > y ? 1 : -1); } -bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const char *name, bool skip_int_pk_check) { - DEBUG_DBFUNCTION("dbutils_table_sanity_check %s", name); - - char buffer[2048]; - size_t blen = sizeof(buffer); +int dbutils_settings_get_value (cloudsync_context *data, const char *key, char *buffer, size_t *blen, int64_t *intvalue) { + DEBUG_SETTINGS("dbutils_settings_get_value key: %s", key); - // sanity check table name - if (name == NULL) { - dbutils_context_result_error(context, "%s", "cloudsync_init requires a non-null table parameter"); - return false; + // if intvalue requested: buffer/blen optional + size_t buffer_len = 0; + if (intvalue) { + *intvalue = 0; + } else { + if (!buffer || !blen || *blen == 0) return DBRES_MISUSE; + buffer[0] = 0; + buffer_len = *blen; + *blen = 0; } - // avoid allocating heap memory for SQL statements by setting a maximum length of 1900 characters - // for table names. This limit is reasonable and helps prevent memory management issues. - const size_t maxlen = blen - 148; - if (strlen(name) > maxlen) { - dbutils_context_result_error(context, "Table name cannot be longer than %d characters", maxlen); - return false; - } + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_SETTINGS_GET_VALUE, (void **)&vm, 0); + if (rc != DBRES_OK) goto finalize_get_value; - // check if table exists - if (dbutils_table_exists(db, name) == false) { - dbutils_context_result_error(context, "Table %s does not exist", name); - return false; - } + rc = databasevm_bind_text(vm, 1, key, -1); + if (rc != DBRES_OK) goto finalize_get_value; - // no more than 128 columns can be used as a composite primary key (SQLite hard limit) - char *sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", name); - sqlite3_int64 count = dbutils_int_select(db, sql); - if (count > 128) { - dbutils_context_result_error(context, "No more than 128 columns can be used to form a composite primary key"); - return false; - } else if (count == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else if (rc != DBRES_ROW) goto finalize_get_value; - #if CLOUDSYNC_DISABLE_ROWIDONLY_TABLES - // if count == 0 means that rowid will be used as primary key (BTW: very bad choice for the user) - if (count == 0) { - dbutils_context_result_error(context, "Rowid only tables are not supported, all primary keys must be explicitly set and declared as NOT NULL (table %s)", name); - return false; - } - #endif - - if (!skip_int_pk_check) { - if (count == 1) { - // the affinity of a column is determined by the declared type of the column, - // according to the following rules in the order shown: - // 1. If the declared type contains the string "INT" then it is assigned INTEGER affinity. - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", name); - sqlite3_int64 count2 = dbutils_int_select(db, sql); - if (count == count2) { - dbutils_context_result_error(context, "Table %s uses an single-column INTEGER primary key. For CRDT replication, primary keys must be globally unique. Consider using a TEXT primary key with UUIDs or ULID to avoid conflicts across nodes. If you understand the risk and still want to use this INTEGER primary key, set the third argument of the cloudsync_init function to 1 to skip this check.", name); - return false; - } - if (count2 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - } - } + // SQLITE_ROW case + if (rc == DBRES_ROW) { + rc = DBRES_OK; - // if user declared explicit primary key(s) then make sure they are all declared as NOT NULL - if (count > 0) { - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0 AND \"notnull\"=1;", name); - sqlite3_int64 count2 = dbutils_int_select(db, sql); - if (count2 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - if (count != count2) { - dbutils_context_result_error(context, "All primary keys must be explicitly declared as NOT NULL (table %s)", name); - return false; + // NULL case + if (database_column_type(vm, 0) == DBTYPE_NULL) { + goto finalize_get_value; } - } - - // check for columns declared as NOT NULL without a DEFAULT value. - // Otherwise, col_merge_stmt would fail if changes to other columns are inserted first. - sql = sqlite3_snprintf((int)blen, buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0 AND \"notnull\"=1 AND \"dflt_value\" IS NULL;", name); - sqlite3_int64 count3 = dbutils_int_select(db, sql); - if (count3 == -1) { - dbutils_context_result_error(context, "%s", sqlite3_errmsg(db)); - return false; - } - if (count3 > 0) { - dbutils_context_result_error(context, "All non-primary key columns declared as NOT NULL must have a DEFAULT value. (table %s)", name); - return false; - } - - return true; -} - -int dbutils_delete_triggers (sqlite3 *db, const char *table) { - DEBUG_DBFUNCTION("dbutils_delete_triggers %s", table); - - // from dbutils_table_sanity_check we already know that 2048 is OK - char buffer[2048]; - size_t blen = sizeof(buffer); - int rc = SQLITE_ERROR; - - char *sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%w\";", table); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - -finalize: - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_delete_triggers error %s (%s)", sqlite3_errmsg(db), sql); - return rc; -} - -int dbutils_check_triggers (sqlite3 *db, const char *table, table_algo algo) { - DEBUG_DBFUNCTION("dbutils_check_triggers %s", table); - - if (dbutils_settings_check_version(db, "0.8.25") <= 0) { - dbutils_delete_triggers(db, table); - } - - char *trigger_name = NULL; - int rc = SQLITE_NOMEM; - - // common part - char *trigger_when = cloudsync_memory_mprintf("FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table); - if (!trigger_when) goto finalize; - - // INSERT TRIGGER - // NEW.prikey1, NEW.prikey2... - trigger_name = cloudsync_memory_mprintf("cloudsync_after_insert_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - rc = SQLITE_NOMEM; - char *sql = cloudsync_memory_mprintf("SELECT group_concat('NEW.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table); - if (!sql) goto finalize; - - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "NEW.rowid"; - cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER INSERT ON \"%w\" %s BEGIN SELECT cloudsync_insert('%q', %s); END", trigger_name, table, trigger_when, table, pkvalues); - if (pkclause) cloudsync_memory_free(pkclause); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - rc = SQLITE_NOMEM; - - if (algo != table_algo_crdt_gos) { - rc = SQLITE_NOMEM; - - // UPDATE TRIGGER - // NEW.prikey1, NEW.prikey2, OLD.prikey1, OLD.prikey2, NEW.col1, OLD.col1, NEW.col2, OLD.col2... - trigger_name = cloudsync_memory_mprintf("cloudsync_after_update_%s", table); - if (!trigger_name) goto finalize; - if (!dbutils_trigger_exists(db, trigger_name)) { - // Generate VALUES clause for all columns using a CTE to avoid compound SELECT limits - // First, get all primary key columns in order - char *pk_values_sql = cloudsync_memory_mprintf( - "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') " - "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", - table, table); - if (!pk_values_sql) goto finalize; - - char *pk_values_list = dbutils_text_select(db, pk_values_sql); - cloudsync_memory_free(pk_values_sql); - - // Then get all regular columns in order - char *col_values_sql = cloudsync_memory_mprintf( - "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') " - "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", - table, table); - if (!col_values_sql) goto finalize; - - char *col_values_list = dbutils_text_select(db, col_values_sql); - cloudsync_memory_free(col_values_sql); - - // Build the complete VALUES query - char *values_query; - if (col_values_list && strlen(col_values_list) > 0) { - // Table has both primary keys and regular columns - values_query = cloudsync_memory_mprintf( - "WITH column_data(table_name, new_value, old_value) AS (VALUES %s, %s) " - "SELECT table_name, new_value, old_value FROM column_data", - pk_values_list, col_values_list); - cloudsync_memory_free(col_values_list); - } else { - // Table has only primary keys - values_query = cloudsync_memory_mprintf( - "WITH column_data(table_name, new_value, old_value) AS (VALUES %s) " - "SELECT table_name, new_value, old_value FROM column_data", - pk_values_list); - } - - if (pk_values_list) cloudsync_memory_free(pk_values_list); - if (!values_query) goto finalize; - - // Create the trigger with aggregate function - char *sql = cloudsync_memory_mprintf( - "CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN " - "SELECT cloudsync_update(table_name, new_value, old_value) FROM (%s); " - "END", - trigger_name, table, trigger_when, values_query); - - cloudsync_memory_free(values_query); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + // INT case + if (intvalue) { + *intvalue = database_column_int(vm, 0); + goto finalize_get_value; } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - } else { - // Grow Only Set - // In a grow-only set, the update operation is not allowed. - // A grow-only set is a type of CRDT (Conflict-free Replicated Data Type) where the only permissible operation is to add elements to the set, - // without ever removing or modifying them. - // Once an element is added to the set, it remains there permanently, which guarantees that the set only grows over time. - trigger_name = cloudsync_memory_mprintf("cloudsync_before_update_%s", table); - if (!trigger_name) goto finalize; - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" BEFORE UPDATE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: UPDATE operation is not allowed on table %w.'); END", trigger_name, table, table, table); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; + // buffer case + const char *value = database_column_text(vm, 0); + size_t size = (size_t)database_column_bytes(vm, 0); + if (!value || size == 0) goto finalize_get_value; + if (size + 1 > buffer_len) { + rc = DBRES_NOMEM; + } else { + memcpy(buffer, value, size); + buffer[size] = '\0'; + *blen = size; } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; } - // DELETE TRIGGER - // OLD.prikey1, OLD.prikey2... - if (algo != table_algo_crdt_gos) { - trigger_name = cloudsync_memory_mprintf("cloudsync_after_delete_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("SELECT group_concat('OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table); - if (!sql) goto finalize; - - char *pkclause = dbutils_text_select(db, sql); - char *pkvalues = (pkclause) ? pkclause : "OLD.rowid"; - cloudsync_memory_free(sql); - - sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER DELETE ON \"%w\" %s BEGIN SELECT cloudsync_delete('%q',%s); END", trigger_name, table, trigger_when, table, pkvalues); - if (pkclause) cloudsync_memory_free(pkclause); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - - cloudsync_memory_free(trigger_name); - trigger_name = NULL; - } else { - // Grow Only Set - // In a grow-only set, the delete operation is not allowed. - trigger_name = cloudsync_memory_mprintf("cloudsync_before_delete_%s", table); - if (!trigger_name) goto finalize; - - if (!dbutils_trigger_exists(db, trigger_name)) { - char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" BEFORE DELETE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: DELETE operation is not allowed on table %w.'); END", trigger_name, table, table, table); - if (!sql) goto finalize; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) goto finalize; - } - cloudsync_memory_free(trigger_name); - trigger_name = NULL; +finalize_get_value: + if (rc != DBRES_OK) { + DEBUG_ALWAYS("dbutils_settings_get_value error %s", database_errmsg(data)); } - rc = SQLITE_OK; - -finalize: - if (trigger_name) cloudsync_memory_free(trigger_name); - if (trigger_when) cloudsync_memory_free(trigger_when); - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_create_triggers error %s (%d)", sqlite3_errmsg(db), rc); - return rc; -} - -int dbutils_check_metatable (sqlite3 *db, const char *table, table_algo algo) { - DEBUG_DBFUNCTION("dbutils_check_metatable %s", table); - - // WITHOUT ROWID is available starting from SQLite version 3.8.2 (2013-12-06) and later - char *sql = cloudsync_memory_mprintf("CREATE TABLE IF NOT EXISTS \"%w_cloudsync\" (pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER, db_version INTEGER, site_id INTEGER DEFAULT 0, seq INTEGER, PRIMARY KEY (pk, col_name)) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS \"%w_cloudsync_db_idx\" ON \"%w_cloudsync\" (db_version);", table, table, table); - if (!sql) return SQLITE_NOMEM; - - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - DEBUG_SQL("\n%s", sql); - cloudsync_memory_free(sql); - + if (vm) databasevm_finalize(vm); return rc; } - -sqlite3_int64 dbutils_schema_version (sqlite3 *db) { - DEBUG_DBFUNCTION("dbutils_schema_version"); - - return dbutils_int_select(db, "PRAGMA schema_version;"); -} - -bool dbutils_is_star_table (const char *table_name) { - return (table_name && (strlen(table_name) == 1) && table_name[0] == '*'); -} - -// MARK: - Settings - - -int binary_comparison (int x, int y) { - if (x == y) return 0; - if (x > y) return 1; - return -1; -} - -char *dbutils_settings_get_value (sqlite3 *db, const char *key, char *buffer, size_t blen) { - DEBUG_SETTINGS("dbutils_settings_get_value key: %s", key); - - // check if heap allocation must be forced - if (!buffer || blen == 0) blen = 0; - size_t size = 0; - - sqlite3_stmt *vm = NULL; - char *sql = "SELECT value FROM cloudsync_settings WHERE key=?1;"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize_get_value; - - rc = sqlite3_bind_text(vm, 1, key, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; - - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - else if (rc != SQLITE_ROW) goto finalize_get_value; - - // SQLITE_ROW case - if (sqlite3_column_type(vm, 0) == SQLITE_NULL) { - rc = SQLITE_OK; - goto finalize_get_value; - } - - const unsigned char *value = sqlite3_column_text(vm, 0); - #if CLOUDSYNC_UNITTEST - size = (buffer == OUT_OF_MEMORY_BUFFER) ? (SQLITE_MAX_ALLOCATION_SIZE + 1) :(size_t)sqlite3_column_bytes(vm, 0); - #else - size = (size_t)sqlite3_column_bytes(vm, 0); - #endif - if (size + 1 > blen) { - buffer = cloudsync_memory_alloc((sqlite3_uint64)(size + 1)); - if (!buffer) { - rc = SQLITE_NOMEM; - goto finalize_get_value; - } - } - - memcpy(buffer, value, size+1); - rc = SQLITE_OK; - -finalize_get_value: - #if CLOUDSYNC_UNITTEST - if ((rc == SQLITE_NOMEM) && (size == SQLITE_MAX_ALLOCATION_SIZE + 1)) rc = SQLITE_OK; - #endif - if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_settings_get_value error %s", sqlite3_errmsg(db)); - if (vm) sqlite3_finalize(vm); - - return buffer; -} - -int dbutils_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *key, const char *value) { +int dbutils_settings_set_key_value (cloudsync_context *data, const char *key, const char *value) { + if (!key) return DBRES_MISUSE; DEBUG_SETTINGS("dbutils_settings_set_key_value key: %s value: %s", key, value); - - int rc = SQLITE_OK; - if (db == NULL) db = sqlite3_context_db_handle(context); - - if (key && value) { - char *sql = "REPLACE INTO cloudsync_settings (key, value) VALUES (?1, ?2);"; + + int rc = DBRES_OK; + if (value) { const char *values[] = {key, value}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 2); - } - - if (value == NULL) { - char *sql = "DELETE FROM cloudsync_settings WHERE key = ?1;"; + rc = database_write(data, SQL_SETTINGS_SET_KEY_VALUE_REPLACE, values, types, lens, 2); + } else { const char *values[] = {key}; - int types[] = {SQLITE_TEXT}; + DBTYPE types[] = {DBTYPE_TEXT}; int lens[] = {-1}; - rc = dbutils_write(db, context, sql, values, types, lens, 1); + rc = database_write(data, SQL_SETTINGS_SET_KEY_VALUE_DELETE, values, types, lens, 1); } - - cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - if (rc == SQLITE_OK && data) cloudsync_sync_key(data, key, value); + + if (rc == DBRES_OK && data) cloudsync_sync_key(data, key, value); return rc; } -int dbutils_settings_get_int_value (sqlite3 *db, const char *key) { +int dbutils_settings_get_int_value (cloudsync_context *data, const char *key) { DEBUG_SETTINGS("dbutils_settings_get_int_value key: %s", key); - char buffer[256] = {0}; - if (dbutils_settings_get_value(db, key, buffer, sizeof(buffer)) == NULL) return -1; + int64_t value = 0; + if (dbutils_settings_get_value(data, key, NULL, NULL, &value) != DBRES_OK) return -1; - return (int)strtol(buffer, NULL, 0); + return (int)value; } -int dbutils_settings_check_version (sqlite3 *db, const char *version) { +int64_t dbutils_settings_get_int64_value (cloudsync_context *data, const char *key) { + DEBUG_SETTINGS("dbutils_settings_get_int_value key: %s", key); + int64_t value = 0; + if (dbutils_settings_get_value(data, key, NULL, NULL, &value) != DBRES_OK) return -1; + + return value; +} + +int dbutils_settings_check_version (cloudsync_context *data, const char *version) { DEBUG_SETTINGS("dbutils_settings_check_version"); char buffer[256]; - if (dbutils_settings_get_value(db, CLOUDSYNC_KEY_LIBVERSION, buffer, sizeof(buffer)) == NULL) return -666; + size_t len = sizeof(buffer); + if (dbutils_settings_get_value(data, CLOUDSYNC_KEY_LIBVERSION, buffer, &len, NULL) != DBRES_OK) return -666; int major1, minor1, patch1; int major2, minor2, patch2; @@ -814,9 +221,9 @@ int dbutils_settings_check_version (sqlite3 *db, const char *version) { if (count1 != 3 || count2 != 3) return -666; int res = 0; - if ((res = binary_comparison(major1, major2)) == 0) { - if ((res = binary_comparison(minor1, minor2)) == 0) { - return binary_comparison(patch1, patch2); + if ((res = dbutils_binary_comparison(major1, major2)) == 0) { + if ((res = dbutils_binary_comparison(minor1, minor2)) == 0) { + return dbutils_binary_comparison(patch1, patch2); } } @@ -824,318 +231,271 @@ int dbutils_settings_check_version (sqlite3 *db, const char *version) { return res; } -char *dbutils_table_settings_get_value (sqlite3 *db, const char *table, const char *column, const char *key, char *buffer, size_t blen) { - DEBUG_SETTINGS("dbutils_table_settings_get_value table: %s column: %s key: %s", table, column, key); - - // check if heap allocation must be forced - if (!buffer || blen == 0) blen = 0; - size_t size = 0; +int dbutils_table_settings_get_value (cloudsync_context *data, const char *table, const char *column_name, const char *key, char *buffer, size_t blen) { + DEBUG_SETTINGS("dbutils_table_settings_get_value table: %s column: %s key: %s", table, column_name, key); + + if (!buffer || blen == 0) return DBRES_MISUSE; + buffer[0] = 0; - sqlite3_stmt *vm = NULL; - char *sql = "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; - int rc = sqlite3_prepare(db, sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize_get_value; + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_TABLE_SETTINGS_GET_VALUE, (void **)&vm, 0); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 1, table, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 1, table, -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 2, (column) ? column : "*", -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 2, (column_name) ? column_name : "*", -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_bind_text(vm, 3, key, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) goto finalize_get_value; + rc = databasevm_bind_text(vm, 3, key, -1); + if (rc != DBRES_OK) goto finalize_get_value; - rc = sqlite3_step(vm); - if (rc == SQLITE_DONE) rc = SQLITE_OK; - else if (rc != SQLITE_ROW) goto finalize_get_value; + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else if (rc != DBRES_ROW) goto finalize_get_value; // SQLITE_ROW case - if (sqlite3_column_type(vm, 0) == SQLITE_NULL) { - rc = SQLITE_OK; - goto finalize_get_value; - } - - const unsigned char *value = sqlite3_column_text(vm, 0); - #if CLOUDSYNC_UNITTEST - size = (buffer == OUT_OF_MEMORY_BUFFER) ? (SQLITE_MAX_ALLOCATION_SIZE + 1) :(size_t)sqlite3_column_bytes(vm, 0); - #else - size = (size_t)sqlite3_column_bytes(vm, 0); - #endif - if (size + 1 > blen) { - buffer = cloudsync_memory_alloc((sqlite3_uint64)(size + 1)); - if (!buffer) { - rc = SQLITE_NOMEM; + if (rc == DBRES_ROW) { + rc = DBRES_OK; + + // NULL case + if (database_column_type(vm, 0) == DBTYPE_NULL) { goto finalize_get_value; } - } - memcpy(buffer, value, size+1); - rc = SQLITE_OK; - -finalize_get_value: - #if CLOUDSYNC_UNITTEST - if ((rc == SQLITE_NOMEM) && (size == SQLITE_MAX_ALLOCATION_SIZE + 1)) rc = SQLITE_OK; - #endif - if (rc != SQLITE_OK) { - DEBUG_ALWAYS("cloudsync_table_settings error %s", sqlite3_errmsg(db)); + const char *value = database_column_text(vm, 0); + size_t size = (size_t)database_column_bytes(vm, 0); + if (size + 1 > blen) { + rc = DBRES_NOMEM; + } else { + memcpy(buffer, value, size); + buffer[size] = '\0'; + } } - if (vm) sqlite3_finalize(vm); - return buffer; +finalize_get_value: + if (rc != DBRES_OK) { + DEBUG_ALWAYS("cloudsync_table_settings error %s", database_errmsg(data)); + } + if (vm) databasevm_finalize(vm); + return rc; } -int dbutils_table_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *table, const char *column, const char *key, const char *value) { - DEBUG_SETTINGS("dbutils_table_settings_set_key_value table: %s column: %s key: %s", table, column, key); +int dbutils_table_settings_set_key_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, const char *value) { + DEBUG_SETTINGS("dbutils_table_settings_set_key_value table: %s column: %s key: %s", table_name, column_name, key); - int rc = SQLITE_OK; - if (db == NULL) db = sqlite3_context_db_handle(context); + int rc = DBRES_OK; // sanity check tbl_name - if (table == NULL) { - if (context) sqlite3_result_error(context, "cloudsync_set_table/set_column requires a non-null table parameter", -1); - return SQLITE_ERROR; + if (table_name == NULL) { + return cloudsync_set_error(data, "cloudsync_set_table/set_column requires a non-null table parameter", DBRES_ERROR); } // sanity check column name - if (column == NULL) column = "*"; + if (column_name == NULL) column_name = "*"; // remove all table_name entries if (key == NULL) { - char *sql = "DELETE FROM cloudsync_table_settings WHERE tbl_name=?1;"; - const char *values[] = {table}; - int types[] = {SQLITE_TEXT}; + const char *values[] = {table_name}; + DBTYPE types[] = {DBTYPE_TEXT}; int lens[] = {-1}; - rc = dbutils_write(db, context, sql, values, types, lens, 1); + rc = database_write(data, SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE, values, types, lens, 1); return rc; } if (key && value) { - char *sql = "REPLACE INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES (?1, ?2, ?3, ?4);"; - const char *values[] = {table, column, key, value}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT}; + const char *values[] = {table_name, column_name, key, value}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1, -1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 4); + rc = database_write(data, SQL_TABLE_SETTINGS_REPLACE, values, types, lens, 4); } if (value == NULL) { - char *sql = "DELETE FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; - const char *values[] = {table, column, key}; - int types[] = {SQLITE_TEXT, SQLITE_TEXT, SQLITE_TEXT}; + const char *values[] = {table_name, column_name, key}; + DBTYPE types[] = {DBTYPE_TEXT, DBTYPE_TEXT, DBTYPE_TEXT}; int lens[] = {-1, -1, -1}; - rc = dbutils_write(db, context, sql, values, types, lens, 3); + rc = database_write(data, SQL_TABLE_SETTINGS_DELETE_ONE, values, types, lens, 3); } // unused in this version // cloudsync_context *data = (context) ? (cloudsync_context *)sqlite3_user_data(context) : NULL; - // if (rc == SQLITE_OK && data) cloudsync_sync_table_key(data, table, column, key, value); + // if (rc == DBRES_OK && data) cloudsync_sync_table_key(data, table, column, key, value); return rc; } -sqlite3_int64 dbutils_table_settings_count_tables (sqlite3 *db) { +int64_t dbutils_table_settings_count_tables (cloudsync_context *data) { DEBUG_SETTINGS("dbutils_table_settings_count_tables"); - return dbutils_int_select(db, "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"); + + int64_t count = 0; + int rc = database_select_int(data, SQL_TABLE_SETTINGS_COUNT_TABLES, &count); + return (rc == DBRES_OK) ? count : 0; } -table_algo dbutils_table_settings_get_algo (sqlite3 *db, const char *table_name) { +table_algo dbutils_table_settings_get_algo (cloudsync_context *data, const char *table_name) { DEBUG_SETTINGS("dbutils_table_settings_get_algo %s", table_name); char buffer[512]; - char *value = dbutils_table_settings_get_value(db, table_name, "*", "algo", buffer, sizeof(buffer)); - return (value) ? crdt_algo_from_name(value) : table_algo_none; + int rc = dbutils_table_settings_get_value(data, table_name, "*", "algo", buffer, sizeof(buffer)); + return (rc == DBRES_OK) ? cloudsync_algo_from_name(buffer) : table_algo_none; } int dbutils_settings_load_callback (void *xdata, int ncols, char **values, char **names) { cloudsync_context *data = (cloudsync_context *)xdata; - - for (int i=0; idata; - sqlite3 *db = context->db; + cloudsync_context *data = (cloudsync_context *)xdata; - for (int i=0; ischema_version != dbutils_schema_version(db))) { + if ((settings_exists == true) && (data->schema_version != database_schema_version(data))) { // SOMEONE CHANGED SCHEMAs SO WE NEED TO RECHECK AUGMENTED TABLES and RELATED TRIGGERS assert(0); } */ - return SQLITE_OK; -} - -int dbutils_update_schema_hash(sqlite3 *db, uint64_t *hash) { - char *schemasql = "SELECT group_concat(LOWER(sql)) FROM sqlite_master " - "WHERE type = 'table' AND name IN (SELECT tbl_name FROM cloudsync_table_settings ORDER BY tbl_name) " - "ORDER BY name;"; - char *schema = dbutils_text_select(db, schemasql); - if (!schema) return SQLITE_ERROR; - - sqlite3_uint64 h = fnv1a_hash(schema, strlen(schema)); - cloudsync_memory_free(schema); - if (hash && *hash == h) return SQLITE_CONSTRAINT; - - char sql[1024]; - snprintf(sql, sizeof(sql), "INSERT INTO cloudsync_schema_versions (hash, seq) " - "VALUES (%lld, COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " - "ON CONFLICT(hash) DO UPDATE SET " - " seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", (sqlite3_int64)h); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if (rc == SQLITE_OK && hash) *hash = h; - return rc; + return DBRES_OK; } -sqlite3_uint64 dbutils_schema_hash (sqlite3 *db) { - DEBUG_DBFUNCTION("dbutils_schema_version"); - - return (sqlite3_uint64)dbutils_int_select(db, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC limit 1;"); -} - -bool dbutils_check_schema_hash (sqlite3 *db, sqlite3_uint64 hash) { - DEBUG_DBFUNCTION("dbutils_check_schema_hash"); - - // a change from the current version of the schema or from previous known schema can be applied - // a change from a newer schema version not yet applied to this peer cannot be applied - // so a schema hash is valid if it exists in the cloudsync_schema_versions table - - // the idea is to allow changes on stale peers and to be able to apply these changes on peers with newer schema, - // but it requires alter table operation on augmented tables only add new columns and never drop columns for backward compatibility - char sql[1024]; - snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%lld)", hash); - - return (dbutils_int_select(db, sql) == 1); -} - - -int dbutils_settings_cleanup (sqlite3 *db) { - const char *sql = "DROP TABLE IF EXISTS cloudsync_settings; DROP TABLE IF EXISTS cloudsync_site_id; DROP TABLE IF EXISTS cloudsync_table_settings; DROP TABLE IF EXISTS cloudsync_schema_versions; "; - return sqlite3_exec(db, sql, NULL, NULL, NULL); +int dbutils_settings_cleanup (cloudsync_context *data) { + return database_exec(data, SQL_SETTINGS_CLEANUP_DROP_ALL); } diff --git a/src/dbutils.h b/src/dbutils.h index b245f6a..472469a 100644 --- a/src/dbutils.h +++ b/src/dbutils.h @@ -10,7 +10,6 @@ #include #include "utils.h" -#include "cloudsync_private.h" #define CLOUDSYNC_SETTINGS_NAME "cloudsync_settings" #define CLOUDSYNC_SITEID_NAME "cloudsync_site_id" @@ -23,50 +22,28 @@ #define CLOUDSYNC_KEY_CHECK_SEQ "check_seq" #define CLOUDSYNC_KEY_SEND_DBVERSION "send_dbversion" #define CLOUDSYNC_KEY_SEND_SEQ "send_seq" +#define CLOUDSYNC_KEY_SCHEMA "schema" #define CLOUDSYNC_KEY_DEBUG "debug" #define CLOUDSYNC_KEY_ALGO "algo" - -// general -int dbutils_write_simple (sqlite3 *db, const char *sql); -int dbutils_write (sqlite3 *db, sqlite3_context *context, const char *sql, const char **values, int types[], int len[], int count); -sqlite3_int64 dbutils_int_select (sqlite3 *db, const char *sql); -char *dbutils_text_select (sqlite3 *db, const char *sql); -char *dbutils_blob_select (sqlite3 *db, const char *sql, int *size, sqlite3_context *context, int *rc); -int dbutils_blob_int_int_select (sqlite3 *db, const char *sql, char **blob, int *size, sqlite3_int64 *int1, sqlite3_int64 *int2); - -int dbutils_register_function (sqlite3 *db, const char *name, void (*ptr)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)); -int dbutils_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)); - -int dbutils_debug_stmt (sqlite3 *db, bool print_result); -void dbutils_debug_values (int argc, sqlite3_value **argv); -void dbutils_debug_value (sqlite3_value *value); - -int dbutils_value_compare (sqlite3_value *v1, sqlite3_value *v2); -void dbutils_context_result_error (sqlite3_context *context, const char *format, ...); - -bool dbutils_system_exists (sqlite3 *db, const char *name, const char *type); -bool dbutils_table_exists (sqlite3 *db, const char *name); -bool dbutils_trigger_exists (sqlite3 *db, const char *name); -bool dbutils_table_sanity_check (sqlite3 *db, sqlite3_context *context, const char *name, bool skip_int_pk_check); -bool dbutils_is_star_table (const char *table_name); - -int dbutils_delete_triggers (sqlite3 *db, const char *table); -int dbutils_check_triggers (sqlite3 *db, const char *table, table_algo algo); -int dbutils_check_metatable (sqlite3 *db, const char *table, table_algo algo); -sqlite3_int64 dbutils_schema_version (sqlite3 *db); +#define CLOUDSYNC_KEY_SKIP_SCHEMA_HASH_CHECK "skip_schema_hash_check" // settings -int dbutils_settings_cleanup (sqlite3 *db); -int dbutils_settings_init (sqlite3 *db, void *cloudsync_data, sqlite3_context *context); -int dbutils_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *key, const char *value); -int dbutils_settings_get_int_value (sqlite3 *db, const char *key); -char *dbutils_settings_get_value (sqlite3 *db, const char *key, char *buffer, size_t blen); -int dbutils_table_settings_set_key_value (sqlite3 *db, sqlite3_context *context, const char *table, const char *column, const char *key, const char *value); -sqlite3_int64 dbutils_table_settings_count_tables (sqlite3 *db); -char *dbutils_table_settings_get_value (sqlite3 *db, const char *table, const char *column, const char *key, char *buffer, size_t blen); -table_algo dbutils_table_settings_get_algo (sqlite3 *db, const char *table_name); -int dbutils_update_schema_hash(sqlite3 *db, uint64_t *hash); -sqlite3_uint64 dbutils_schema_hash (sqlite3 *db); -bool dbutils_check_schema_hash (sqlite3 *db, sqlite3_uint64 hash); +int dbutils_settings_init (cloudsync_context *data); +int dbutils_settings_cleanup (cloudsync_context *data); +int dbutils_settings_check_version (cloudsync_context *data, const char *version); +int dbutils_settings_set_key_value (cloudsync_context *data, const char *key, const char *value); +int dbutils_settings_get_int_value (cloudsync_context *data, const char *key); +int64_t dbutils_settings_get_int64_value (cloudsync_context *data, const char *key); + +// table settings +int dbutils_table_settings_set_key_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, const char *value); +int64_t dbutils_table_settings_count_tables (cloudsync_context *data); +int dbutils_table_settings_get_value (cloudsync_context *data, const char *table_name, const char *column_name, const char *key, char *buffer, size_t blen); +table_algo dbutils_table_settings_get_algo (cloudsync_context *data, const char *table_name); + +// others +void dbutils_debug_values (dbvalue_t **argv, int argc); +void dbutils_debug_value (dbvalue_t *value); +int dbutils_value_compare (dbvalue_t *v1, dbvalue_t *v2); #endif diff --git a/src/jsmn.h b/src/jsmn.h new file mode 100644 index 0000000..dca2bb5 --- /dev/null +++ b/src/jsmn.h @@ -0,0 +1,471 @@ +/* + * MIT License + * + * Copyright (c) 2010 Serge Zaitsev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef JSMN_H +#define JSMN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_UNDEFINED = 0, + JSMN_OBJECT = 1 << 0, + JSMN_ARRAY = 1 << 1, + JSMN_STRING = 1 << 2, + JSMN_PRIMITIVE = 1 << 3 +} jsmntype_t; + +enum jsmnerr { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3 +}; + +/** + * JSON token description. + * type type (object, array, string etc.) + * start start position in JSON data string + * end end position in JSON data string + */ +typedef struct jsmntok { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string. + */ +typedef struct jsmn_parser { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g. parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each + * describing + * a single JSON object. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** + * Allocates a fresh unused token from the token pool. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fills token type and boundaries. + */ +static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, + const int start, const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Fills next available token with JSON primitive. + */ +static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t': + case '\r': + case '\n': + case ' ': + case ',': + case ']': + case '}': + goto found; + default: + /* to quiet a warning from gcc*/ + break; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + +found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Fills next token with JSON string. + */ +static int jsmn_parse_string(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + /* Skip starting quote */ + parser->pos++; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\' && parser->pos + 1 < len) { + int i; + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': + case '/': + case '\\': + case 'b': + case 'f': + case 'r': + case 'n': + case 't': + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; + i++) { + /* If it isn't a hex character we have an error */ + if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * Parse JSON string and fill tokens. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens) { + int r; + int i; + jsmntok_t *token; + int count = parser->toknext; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': + case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + return JSMN_ERROR_NOMEM; + } + if (parser->toksuper != -1) { + jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT + /* In strict mode an object or array can't become a key */ + if (t->type == JSMN_OBJECT) { + return JSMN_ERROR_INVAL; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': + case ']': + if (tokens == NULL) { + break; + } + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + if (token->type != type || parser->toksuper == -1) { + return JSMN_ERROR_INVAL; + } + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) { + return JSMN_ERROR_INVAL; + } + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + case '\t': + case '\r': + case '\n': + case ' ': + break; + case ':': + parser->toksuper = parser->toknext - 1; + break; + case ',': + if (tokens != NULL && parser->toksuper != -1 && + tokens[parser->toksuper].type != JSMN_ARRAY && + tokens[parser->toksuper].type != JSMN_OBJECT) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else + for (i = parser->toknext - 1; i >= 0; i--) { + if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { + if (tokens[i].start != -1 && tokens[i].end == -1) { + parser->toksuper = i; + break; + } + } + } +#endif + } + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 't': + case 'f': + case 'n': + /* And they must not be keys of the object */ + if (tokens != NULL && parser->toksuper != -1) { + const jsmntok_t *t = &tokens[parser->toksuper]; + if (t->type == JSMN_OBJECT || + (t->type == JSMN_STRING && t->size != 0)) { + return JSMN_ERROR_INVAL; + } + } +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens != NULL) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * Creates a new parser based over a given buffer with an array of tokens + * available. + */ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */ diff --git a/src/network.c b/src/network.c deleted file mode 100644 index 37fe1ff..0000000 --- a/src/network.c +++ /dev/null @@ -1,959 +0,0 @@ -// -// network.c -// cloudsync -// -// Created by Marco Bambini on 12/12/24. -// - -#ifndef CLOUDSYNC_OMIT_NETWORK - -#include -#include "network.h" -#include "dbutils.h" -#include "utils.h" -#include "cloudsync_private.h" -#include "network_private.h" - -#ifndef SQLITE_WASM_EXTRA_INIT -#ifndef CLOUDSYNC_OMIT_CURL -#include "curl/curl.h" -#endif -#else -#define curl_free(x) free(x) -char *substr(const char *start, const char *end); -#endif - -#ifdef __ANDROID__ -#include "cacert.h" -static size_t cacert_len = sizeof(cacert_pem) - 1; -#endif - -#define CLOUDSYNC_NETWORK_MINBUF_SIZE 512 -#define CLOUDSYNC_SESSION_TOKEN_MAXSIZE 4096 - -#define DEFAULT_SYNC_WAIT_MS 100 -#define DEFAULT_SYNC_MAX_RETRIES 1 - -#define MAX_QUERY_VALUE_LEN 256 - -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - -// MARK: - - -struct network_data { - char site_id[UUID_STR_MAXLEN]; - char *authentication; // apikey or token - char *check_endpoint; - char *upload_endpoint; -}; - -typedef struct { - char *buffer; - size_t balloc; - size_t bused; - int zero_term; -} network_buffer; - - -typedef struct { - const char *data; - size_t size; - size_t read_pos; -} network_read_data; - -// MARK: - - -void network_result_cleanup (NETWORK_RESULT *res) { - if (res->xfree) { - res->xfree(res->xdata); - } else if (res->buffer) { - cloudsync_memory_free(res->buffer); - } -} - -char *network_data_get_siteid (network_data *data) { - return data->site_id; -} - -bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, bool duplicate) { - if (duplicate) { - // auth is optional - char *s1 = (auth) ? cloudsync_string_dup(auth, false) : NULL; - if (auth && !s1) return false; - char *s2 = cloudsync_string_dup(check, false); - if (!s2) {if (auth && s1) sqlite3_free(s1); return false;} - char *s3 = cloudsync_string_dup(upload, false); - if (!s3) {if (auth && s1) sqlite3_free(s1); sqlite3_free(s2); return false;} - - auth = s1; - check = s2; - upload = s3; - } - - data->authentication = auth; - data->check_endpoint = check; - data->upload_endpoint = upload; - return true; -} - -void network_data_free (network_data *data) { - if (!data) return; - - if (data->authentication) cloudsync_memory_free(data->authentication); - if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); - if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); - cloudsync_memory_free(data); -} - -// MARK: - Utils - - -#ifndef CLOUDSYNC_OMIT_CURL -static bool network_buffer_check (network_buffer *data, size_t needed) { - // alloc/resize buffer - if (data->bused + needed > data->balloc) { - if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; - size_t balloc = data->balloc + needed; - - char *buffer = cloudsync_memory_realloc(data->buffer, balloc); - if (!buffer) return false; - - data->buffer = buffer; - data->balloc = balloc; - } - - return true; -} - -static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { - network_buffer *data = (network_buffer *)xdata; - - size_t ptr_size = (size*nmemb); - if (data->zero_term) ptr_size += 1; - - if (network_buffer_check(data, ptr_size) == false) return -1; - memcpy(data->buffer+data->bused, ptr, size*nmemb); - data->bused += size*nmemb; - if (data->zero_term) data->buffer[data->bused] = 0; - - return (size * nmemb); -} - -NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { - char *buffer = NULL; - size_t blen = 0; - struct curl_slist* headers = NULL; - char errbuf[CURL_ERROR_SIZE] = {0}; - long response_code = 0; - - CURL *curl = curl_easy_init(); - if (!curl) return (NETWORK_RESULT){CLOUDSYNC_NETWORK_ERROR, NULL, 0, NULL, NULL}; - - // a buffer to store errors in - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); - - CURLcode rc = curl_easy_setopt(curl, CURLOPT_URL, endpoint); - if (rc != CURLE_OK) goto cleanup; - - // set PEM - #ifdef __ANDROID__ - struct curl_blob pem_blob = { - .data = (void *)cacert_pem, - .len = cacert_len, - .flags = CURL_BLOB_NOCOPY - }; - curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); - #endif - - if (custom_header) headers = curl_slist_append(headers, custom_header); - - if (authentication) { - char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; - snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); - headers = curl_slist_append(headers, auth_header); - - if (json_payload) headers = curl_slist_append(headers, "Content-Type: application/json"); - } - - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - network_buffer netdata = {NULL, 0, 0, (zero_terminated) ? 1 : 0}; - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &netdata); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, network_receive_callback); - - // add optional JSON payload (implies setting CURLOPT_POST to 1) - // or set the CURLOPT_POST option - if (json_payload) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload); - } else if (is_post_request) { - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0L); - } - - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - rc = curl_easy_perform(curl); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - if (rc == CURLE_OK) { - buffer = netdata.buffer; - blen = netdata.bused; - } else if (netdata.buffer) { - cloudsync_memory_free(netdata.buffer); - netdata.buffer = NULL; - } - -cleanup: - if (curl) curl_easy_cleanup(curl); - if (headers) curl_slist_free_all(headers); - - // build result - NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; - if (rc == CURLE_OK && response_code < 400) { - result.code = (buffer && blen) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; - result.buffer = buffer; - result.blen = blen; - } else { - result.code = CLOUDSYNC_NETWORK_ERROR; - result.buffer = buffer ? buffer : (errbuf[0]) ? cloudsync_string_dup(errbuf, false) : NULL; - result.blen = buffer ? blen : rc; - } - - return result; -} - -static size_t network_read_callback(char *buffer, size_t size, size_t nitems, void *userdata) { - network_read_data *rd = (network_read_data *)userdata; - size_t max_read = size * nitems; - size_t bytes_left = rd->size - rd->read_pos; - size_t to_copy = bytes_left < max_read ? bytes_left : max_read; - - if (to_copy > 0) { - memcpy(buffer, rd->data + rd->read_pos, to_copy); - rd->read_pos += to_copy; - } - - return to_copy; -} - -bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { - struct curl_slist *headers = NULL; - curl_mime *mime = NULL; - bool result = false; - char errbuf[CURL_ERROR_SIZE] = {0}; - - // init curl - CURL *curl = curl_easy_init(); - if (!curl) return false; - - // set the URL - if (curl_easy_setopt(curl, CURLOPT_URL, endpoint) != CURLE_OK) goto cleanup; - - // set PEM - #ifdef __ANDROID__ - struct curl_blob pem_blob = { - .data = (void *)cacert_pem, - .len = cacert_len, - .flags = CURL_BLOB_NOCOPY - }; - curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); - #endif - - // a buffer to store errors in - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); - - // type header - headers = curl_slist_append(headers, "Accept: text/plain"); - - if (authentication) { - // init authorization header - char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; - snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", data->authentication); - headers = curl_slist_append(headers, auth_header); - } - - // Set headers if needed (S3 pre-signed URLs usually do not require additional headers) - headers = curl_slist_append(headers, "Content-Type: application/octet-stream"); - - if (!headers) goto cleanup; - if (curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers) != CURLE_OK) goto cleanup; - - // Set HTTP PUT method - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - - // Set the size of the blob - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)blob_size); - - // Provide the data using a custom read callback - network_read_data rdata = { - .data = (const char *)blob, - .size = blob_size, - .read_pos = 0 - }; - - curl_easy_setopt(curl, CURLOPT_READFUNCTION, network_read_callback); - curl_easy_setopt(curl, CURLOPT_READDATA, &rdata); - - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - - // perform the upload - CURLcode rc = curl_easy_perform(curl); - if (rc == CURLE_OK) result = true; - -cleanup: - if (mime) curl_mime_free(mime); - if (curl) curl_easy_cleanup(curl); - if (headers) curl_slist_free_all(headers); - return result; -} -#endif - -int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) { - int rc = 0; - switch (result->code) { - case CLOUDSYNC_NETWORK_OK: - sqlite3_result_error_code(context, SQLITE_OK); - sqlite3_result_int(context, 0); - rc = 0; - break; - - case CLOUDSYNC_NETWORK_ERROR: - sqlite3_result_error(context, (result->buffer) ? result->buffer : "Memory error.", -1); - sqlite3_result_error_code(context, SQLITE_ERROR); - rc = -1; - break; - - case CLOUDSYNC_NETWORK_BUFFER: - sqlite3_result_error_code(context, SQLITE_OK); - sqlite3_result_text(context, result->buffer, (int)result->blen, SQLITE_TRANSIENT); - rc = (int)result->blen; - break; - } - - return rc; -} - -int network_download_changes (sqlite3_context *context, const char *download_url) { - DEBUG_FUNCTION("network_download_changes"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) { - sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); - return -1; - } - - NETWORK_RESULT result = network_receive_buffer(data, download_url, NULL, false, false, NULL, NULL); - - int rc = SQLITE_OK; - if (result.code == CLOUDSYNC_NETWORK_BUFFER) { - rc = cloudsync_payload_apply(context, result.buffer, (int)result.blen); - } else { - rc = network_set_sqlite_result(context, &result); - } - network_result_cleanup(&result); - - return rc; -} - -char *network_authentication_token(const char *key, const char *value) { - size_t len = strlen(key) + strlen(value) + 64; - char *buffer = cloudsync_memory_zeroalloc(len); - if (!buffer) return NULL; - - // build new token - // we don't need a prefix because the token alreay include a prefix "sqa_" - snprintf(buffer, len, "%s", value); - - return buffer; -} - -int network_extract_query_param(const char *query, const char *key, char *output, size_t output_size) { - if (!query || !key || !output || output_size == 0) { - return -1; // Invalid input - } - - size_t key_len = strlen(key); - const char *p = query; - #ifdef SQLITE_WASM_EXTRA_INIT - if (*p == '?') p++; - #endif - - while (p && *p) { - // Find the start of a key=value pair - const char *key_start = p; - const char *eq = strchr(key_start, '='); - if (!eq) break; // No '=' found, malformed query string - - size_t current_key_len = eq - key_start; - - // Check if the key matches (ensuring it's the full key) - if (current_key_len == key_len && strncmp(key_start, key, key_len) == 0) { - // Extract the value - const char *value_start = eq + 1; - const char *end = strchr(value_start, '&'); // Find end of value - - size_t value_len = (end) ? (size_t)(end - value_start) : strlen(value_start); - if (value_len >= output_size) { - return -2; // Output buffer too small - } - - strncpy(output, value_start, value_len); - output[value_len] = '\0'; // Null-terminate - return 0; // Success - } - - // Move to the next parameter - p = strchr(p, '&'); - if (p) p++; // Skip '&' - } - - return -3; // Key not found -} - -#if !defined(CLOUDSYNC_OMIT_CURL) || defined(SQLITE_WASM_EXTRA_INIT) -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string) { - // compute endpoints - bool result = false; - - char *scheme = NULL; - char *host = NULL; - char *port = NULL; - char *database = NULL; - char *query = NULL; - - char *authentication = NULL; - char *check_endpoint = NULL; - char *upload_endpoint = NULL; - - char *conn_string_https = NULL; - - #ifndef SQLITE_WASM_EXTRA_INIT - CURLUcode rc = CURLUE_OUT_OF_MEMORY; - CURLU *url = curl_url(); - if (!url) goto finalize; - #endif - - conn_string_https = cloudsync_string_replace_prefix(conn_string, "sqlitecloud://", "https://"); - - #ifndef SQLITE_WASM_EXTRA_INIT - // set URL: https://UUID.g5.sqlite.cloud:443/chinook.sqlite?apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo - rc = curl_url_set(url, CURLUPART_URL, conn_string_https, 0); - if (rc != CURLUE_OK) goto finalize; - - // https (MANDATORY) - rc = curl_url_get(url, CURLUPART_SCHEME, &scheme, 0); - if (rc != CURLUE_OK) goto finalize; - - // UUID.g5.sqlite.cloud (MANDATORY) - rc = curl_url_get(url, CURLUPART_HOST, &host, 0); - if (rc != CURLUE_OK) goto finalize; - - // 443 (OPTIONAL) - rc = curl_url_get(url, CURLUPART_PORT, &port, 0); - if (rc != CURLUE_OK && rc != CURLUE_NO_PORT) goto finalize; - char *port_or_default = port && strcmp(port, "8860") != 0 ? port : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - - // /chinook.sqlite (MANDATORY) - rc = curl_url_get(url, CURLUPART_PATH, &database, 0); - if (rc != CURLUE_OK) goto finalize; - - // apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo (OPTIONAL) - rc = curl_url_get(url, CURLUPART_QUERY, &query, 0); - if (rc != CURLUE_OK && rc != CURLUE_NO_QUERY) goto finalize; - #else - // Parse: scheme://host[:port]/path?query - const char *p = strstr(conn_string_https, "://"); - if (!p) goto finalize; - scheme = substr(conn_string_https, p); - p += 3; - const char *host_start = p; - const char *host_end = strpbrk(host_start, ":/?"); - if (!host_end) goto finalize; - host = substr(host_start, host_end); - p = host_end; - if (*p == ':') { - ++p; - const char *port_end = strpbrk(p, "/?"); - if (!port_end) goto finalize; - port = substr(p, port_end); - p = port_end; - } - if (*p == '/') { - const char *path_start = p; - const char *path_end = strchr(path_start, '?'); - if (!path_end) path_end = path_start + strlen(path_start); - database = substr(path_start, path_end); - p = path_end; - } - if (*p == '?') { - query = strdup(p); - } - if (!scheme || !host || !database) goto finalize; - char *port_or_default = port && strcmp(port, "8860") != 0 ? port : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - #endif - - if (query != NULL) { - char value[MAX_QUERY_VALUE_LEN]; - if (!authentication && network_extract_query_param(query, "apikey", value, sizeof(value)) == 0) { - authentication = network_authentication_token("apikey", value); - } - if (!authentication && network_extract_query_param(query, "token", value, sizeof(value)) == 0) { - authentication = network_authentication_token("token", value); - } - } - - size_t requested = strlen(scheme) + strlen(host) + strlen(port_or_default) + strlen(CLOUDSYNC_ENDPOINT_PREFIX) + strlen(database) + 64; - check_endpoint = (char *)cloudsync_memory_zeroalloc(requested); - upload_endpoint = (char *)cloudsync_memory_zeroalloc(requested); - if ((!upload_endpoint) || (!check_endpoint)) goto finalize; - - snprintf(check_endpoint, requested, "%s://%s:%s/%s%s/%s", scheme, host, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database, data->site_id); - snprintf(upload_endpoint, requested, "%s://%s:%s/%s%s/%s/%s", scheme, host, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database, data->site_id, CLOUDSYNC_ENDPOINT_UPLOAD); - - result = true; - -finalize: - if (result == false) { - // store proper result code/message - #ifndef SQLITE_WASM_EXTRA_INIT - if (rc != CURLUE_OK) sqlite3_result_error(context, curl_url_strerror(rc), -1); - sqlite3_result_error_code(context, (rc != CURLUE_OK) ? SQLITE_ERROR : SQLITE_NOMEM); - #else - sqlite3_result_error(context, "URL parse error", -1); - sqlite3_result_error_code(context, SQLITE_ERROR); - #endif - - // cleanup memory managed by the extension - if (authentication) cloudsync_memory_free(authentication); - if (check_endpoint) cloudsync_memory_free(check_endpoint); - if (upload_endpoint) cloudsync_memory_free(upload_endpoint); - } - - if (result) { - if (authentication) { - if (data->authentication) cloudsync_memory_free(data->authentication); - data->authentication = authentication; - } - - if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); - data->check_endpoint = check_endpoint; - - if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); - data->upload_endpoint = upload_endpoint; - } - - // cleanup memory - #ifndef SQLITE_WASM_EXTRA_INIT - if (url) curl_url_cleanup(url); - #endif - if (scheme) curl_free(scheme); - if (host) curl_free(host); - if (port) curl_free(port); - if (database) curl_free(database); - if (query) curl_free(query); - if (conn_string_https && conn_string_https != conn_string) cloudsync_memory_free(conn_string_https); - - return result; -} -#endif - -void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { - sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); - sqlite3_result_error_code(context, SQLITE_ERROR); -} - -// MARK: - Init / Cleanup - - -network_data *cloudsync_network_data(sqlite3_context *context) { - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (data) return data; - - data = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); - if (data) cloudsync_set_auxdata(context, data); - return data; -} - -void cloudsync_network_init (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_init"); - - #ifndef CLOUDSYNC_OMIT_CURL - curl_global_init(CURL_GLOBAL_ALL); - #endif - - // no real network operations here - // just setup the network_data struct - network_data *data = cloudsync_network_data(context); - if (!data) goto abort_memory; - - // init context - uint8_t *site_id = (uint8_t *)cloudsync_context_init(sqlite3_context_db_handle(context), NULL, context); - if (!site_id) goto abort_siteid; - - // save site_id string representation: 01957493c6c07e14803727e969f1d2cc - cloudsync_uuid_v7_stringify(site_id, data->site_id, false); - - // connection string is something like: - // https://UUID.g5.sqlite.cloud:443/chinook.sqlite?apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo - // or https://UUID.g5.sqlite.cloud:443/chinook.sqlite - // apikey part is optional and can be replaced by a session token once client is authenticated - - const char *connection_param = (const char *)sqlite3_value_text(argv[0]); - - // compute endpoints - if (network_compute_endpoints(context, data, connection_param) == false) { - // error message/code already set inside network_compute_endpoints - goto abort_cleanup; - } - - cloudsync_set_auxdata(context, data); - sqlite3_result_int(context, SQLITE_OK); - return; - -abort_memory: - dbutils_context_result_error(context, "Unable to allocate memory in cloudsync_network_init."); - sqlite3_result_error_code(context, SQLITE_NOMEM); - goto abort_cleanup; - -abort_siteid: - dbutils_context_result_error(context, "Unable to compute/retrieve site_id."); - sqlite3_result_error_code(context, SQLITE_MISUSE); - goto abort_cleanup; - -abort_cleanup: - cloudsync_set_auxdata(context, NULL); - network_data_free(data); -} - -void cloudsync_network_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_cleanup"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - cloudsync_set_auxdata(context, NULL); - network_data_free(data); - sqlite3_result_int(context, SQLITE_OK); - - #ifndef CLOUDSYNC_OMIT_CURL - curl_global_cleanup(); - #endif -} - -// MARK: - Public - - -bool cloudsync_network_set_authentication_token (sqlite3_context *context, const char *value, bool is_token) { - network_data *data = cloudsync_network_data(context); - if (!data) return false; - - const char *key = (is_token) ? "token" : "apikey"; - char *new_auth_token = network_authentication_token(key, value); - if (!new_auth_token) return false; - - if (data->authentication) cloudsync_memory_free(data->authentication); - data->authentication = new_auth_token; - - return true; -} - -void cloudsync_network_set_token (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_set_token"); - - const char *value = (const char *)sqlite3_value_text(argv[0]); - bool result = cloudsync_network_set_authentication_token(context, value, true); - (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); -} - -void cloudsync_network_set_apikey (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_set_apikey"); - - const char *value = (const char *)sqlite3_value_text(argv[0]); - bool result = cloudsync_network_set_authentication_token(context, value, false); - (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); -} - -// MARK: - - -void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - sqlite3 *db = sqlite3_context_db_handle(context); - - char *sql = "SELECT max(db_version), hex(site_id) FROM cloudsync_changes WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; - int last_local_change = (int)dbutils_int_select(db, sql); - if (last_local_change == 0) { - sqlite3_result_int(context, 0); - return; - } - - int sent_db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_SEND_DBVERSION); - sqlite3_result_int(context, (sent_db_version < last_local_change)); -} - -int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_send_changes"); - - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) {sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); return SQLITE_ERROR;} - - // retrieve payload - char *blob = NULL; - int blob_size = 0, db_version = 0, seq = 0; - sqlite3_int64 new_db_version = 0, new_seq = 0; - int rc = cloudsync_payload_get(context, &blob, &blob_size, &db_version, &seq, &new_db_version, &new_seq); - if (rc != SQLITE_OK) return rc; - - // exit if there is no data to send - if (blob == NULL || blob_size == 0) return SQLITE_OK; - - NETWORK_RESULT res = network_receive_buffer(data, data->upload_endpoint, data->authentication, true, false, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); - if (res.code != CLOUDSYNC_NETWORK_BUFFER) { - cloudsync_memory_free(blob); - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to receive upload URL"); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - const char *s3_url = res.buffer; - bool sent = network_send_buffer(data, s3_url, NULL, blob, blob_size); - cloudsync_memory_free(blob); - if (sent == false) { - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to upload BLOB changes to remote host."); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - char json_payload[2024]; - snprintf(json_payload, sizeof(json_payload), "{\"url\":\"%s\"}", s3_url); - - // free res - network_result_cleanup(&res); - - // notify remote host that we succesfully uploaded changes - res = network_receive_buffer(data, data->upload_endpoint, data->authentication, true, true, json_payload, CLOUDSYNC_HEADER_SQLITECLOUD); - if (res.code != CLOUDSYNC_NETWORK_OK) { - network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to notify BLOB upload to remote host."); - network_result_cleanup(&res); - return SQLITE_ERROR; - } - - // update db_version and seq - char buf[256]; - sqlite3 *db = sqlite3_context_db_handle(context); - if (new_db_version != db_version) { - snprintf(buf, sizeof(buf), "%lld", new_db_version); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - } - if (new_seq != seq) { - snprintf(buf, sizeof(buf), "%lld", new_seq); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); - } - - network_result_cleanup(&res); - return SQLITE_OK; -} - -void cloudsync_network_send_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_send_changes"); - - cloudsync_network_send_changes_internal(context, argc, argv); -} - -int cloudsync_network_check_internal(sqlite3_context *context) { - network_data *data = (network_data *)cloudsync_get_auxdata(context); - if (!data) {sqlite3_result_error(context, "Unable to retrieve CloudSync context.", -1); return -1;} - - sqlite3 *db = sqlite3_context_db_handle(context); - - int db_version = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_DBVERSION); - if (db_version<0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return -1;} - - int seq = dbutils_settings_get_int_value(db, CLOUDSYNC_KEY_CHECK_SEQ); - if (seq<0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return -1;} - - // http://uuid.g5.sqlite.cloud/v1/cloudsync/{dbname}/{site_id}/{db_version}/{seq}/check - // the data->check_endpoint stops after {site_id}, just need to append /{db_version}/{seq}/check - char endpoint[2024]; - snprintf(endpoint, sizeof(endpoint), "%s/%lld/%d/%s", data->check_endpoint, (long long)db_version, seq, CLOUDSYNC_ENDPOINT_CHECK); - - NETWORK_RESULT result = network_receive_buffer(data, endpoint, data->authentication, true, true, NULL, CLOUDSYNC_HEADER_SQLITECLOUD); - int rc = SQLITE_OK; - if (result.code == CLOUDSYNC_NETWORK_BUFFER) { - rc = network_download_changes(context, result.buffer); - } else { - rc = network_set_sqlite_result(context, &result); - } - - network_result_cleanup(&result); - return rc; -} - -void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retries) { - int rc = cloudsync_network_send_changes_internal(context, 0, NULL); - if (rc != SQLITE_OK) return; - - int ntries = 0; - int nrows = 0; - while (ntries < max_retries) { - if (ntries > 0) sqlite3_sleep(wait_ms); - nrows = cloudsync_network_check_internal(context); - if (nrows > 0) break; - ntries++; - } - - sqlite3_result_error_code(context, (nrows == -1) ? SQLITE_ERROR : SQLITE_OK); - if (nrows >= 0) sqlite3_result_int(context, nrows); -} - -void cloudsync_network_sync0 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_sync2"); - - cloudsync_network_sync(context, DEFAULT_SYNC_WAIT_MS, DEFAULT_SYNC_MAX_RETRIES); -} - - -void cloudsync_network_sync2 (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_sync2"); - - int wait_ms = sqlite3_value_int(argv[0]); - int max_retries = sqlite3_value_int(argv[1]); - - cloudsync_network_sync(context, wait_ms, max_retries); -} - - -void cloudsync_network_check_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_check_changes"); - - cloudsync_network_check_internal(context); -} - -void cloudsync_network_reset_sync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { - DEBUG_FUNCTION("cloudsync_network_reset_sync_version"); - - sqlite3 *db = sqlite3_context_db_handle(context); - char *buf = "0"; - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_CHECK_SEQ, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_DBVERSION, buf); - dbutils_settings_set_key_value(db, context, CLOUDSYNC_KEY_SEND_SEQ, buf); -} - -/** - * Cleanup all local data from cloudsync-enabled tables, so the database can be safely reused - * by another user without exposing any data from the previous session. - * - * Warning: this function deletes all data from the tables. Use with caution. - */ -void cloudsync_network_logout (sqlite3_context *context, int argc, sqlite3_value **argv) { - bool completed = false; - char *errmsg = NULL; - sqlite3 *db = sqlite3_context_db_handle(context); - - // if the network layer is enabled, remove the token or apikey - sqlite3_exec(db, "SELECT cloudsync_network_set_token('');", NULL, NULL, NULL); - - // get the list of cloudsync-enabled tables - char *sql = "SELECT tbl_name, key, value FROM cloudsync_table_settings;"; - char **result = NULL; - int nrows, ncols; - int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to get current cloudsync configuration. %s", sqlite3_errmsg(db)); - goto finalize; - } - - // run everything in a savepoint - rc = sqlite3_exec(db, "SAVEPOINT cloudsync_logout_sp;", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to create cloudsync_logout savepoint. %s", sqlite3_errmsg(db)); - return; - } - - // disable cloudsync for all the previously enabled tables: cloudsync_cleanup('*') - rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('*')", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to cleanup current cloudsync configuration. %s", sqlite3_errmsg(db)); - goto finalize; - } - - // delete all the local data for each previously enabled table - // re-enable cloudsync on previously enabled tables - for (int i = 1; i <= nrows; i++) { - char *tbl_name = result[i * ncols + 0]; - char *key = result[i * ncols + 1]; - char *value = result[i * ncols + 2]; - - if (strcmp(key, "algo") != 0) continue; - - sql = cloudsync_memory_mprintf("DELETE FROM \"%w\";", tbl_name); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to delete data from table %s. %s", tbl_name, sqlite3_errmsg(db)); - goto finalize; - } - - sql = cloudsync_memory_mprintf("SELECT cloudsync_init('%q', '%q', 1);", tbl_name, value); - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - cloudsync_memory_free(sql); - if (rc != SQLITE_OK) { - errmsg = cloudsync_memory_mprintf("Unable to enable cloudsync on table %s. %s", tbl_name, sqlite3_errmsg(db)); - goto finalize; - } - } - - completed = true; - -finalize: - if (completed) { - sqlite3_exec(db, "RELEASE cloudsync_logout_sp;", NULL, NULL, NULL); - } else { - // cleanup: - // ROLLBACK TO command reverts the state of the database back to what it was just after the corresponding SAVEPOINT - // then RELEASE to remove the SAVEPOINT from the transaction stack - sqlite3_exec(db, "ROLLBACK TO cloudsync_logout_sp;", NULL, NULL, NULL); - sqlite3_exec(db, "RELEASE cloudsync_logout_sp;", NULL, NULL, NULL); - sqlite3_result_error(context, errmsg, -1); - sqlite3_result_error_code(context, rc); - } - sqlite3_free_table(result); - cloudsync_memory_free(errmsg); -} - -// MARK: - - -int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx) { - int rc = SQLITE_OK; - - rc = dbutils_register_function(db, "cloudsync_network_init", cloudsync_network_init, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_cleanup", cloudsync_network_cleanup, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_set_token", cloudsync_network_set_token, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_set_apikey", cloudsync_network_set_apikey, 1, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_has_unsent_changes", cloudsync_network_has_unsent_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_send_changes", cloudsync_network_send_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_check_changes", cloudsync_network_check_changes, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_sync", cloudsync_network_sync0, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_sync", cloudsync_network_sync2, 2, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_reset_sync_version", cloudsync_network_reset_sync_version, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - rc = dbutils_register_function(db, "cloudsync_network_logout", cloudsync_network_logout, 0, pzErrMsg, ctx, NULL); - if (rc != SQLITE_OK) return rc; - - return rc; -} -#endif diff --git a/src/cacert.h b/src/network/cacert.h similarity index 100% rename from src/cacert.h rename to src/network/cacert.h diff --git a/src/network/jsmn.h b/src/network/jsmn.h new file mode 100644 index 0000000..dca2bb5 --- /dev/null +++ b/src/network/jsmn.h @@ -0,0 +1,471 @@ +/* + * MIT License + * + * Copyright (c) 2010 Serge Zaitsev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef JSMN_H +#define JSMN_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** + * JSON type identifier. Basic types are: + * o Object + * o Array + * o String + * o Other primitive: number, boolean (true/false) or null + */ +typedef enum { + JSMN_UNDEFINED = 0, + JSMN_OBJECT = 1 << 0, + JSMN_ARRAY = 1 << 1, + JSMN_STRING = 1 << 2, + JSMN_PRIMITIVE = 1 << 3 +} jsmntype_t; + +enum jsmnerr { + /* Not enough tokens were provided */ + JSMN_ERROR_NOMEM = -1, + /* Invalid character inside JSON string */ + JSMN_ERROR_INVAL = -2, + /* The string is not a full JSON packet, more bytes expected */ + JSMN_ERROR_PART = -3 +}; + +/** + * JSON token description. + * type type (object, array, string etc.) + * start start position in JSON data string + * end end position in JSON data string + */ +typedef struct jsmntok { + jsmntype_t type; + int start; + int end; + int size; +#ifdef JSMN_PARENT_LINKS + int parent; +#endif +} jsmntok_t; + +/** + * JSON parser. Contains an array of token blocks available. Also stores + * the string being parsed now and current position in that string. + */ +typedef struct jsmn_parser { + unsigned int pos; /* offset in the JSON string */ + unsigned int toknext; /* next token to allocate */ + int toksuper; /* superior token node, e.g. parent object or array */ +} jsmn_parser; + +/** + * Create JSON parser over an array of tokens + */ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** + * Run JSON parser. It parses a JSON data string into and array of tokens, each + * describing + * a single JSON object. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** + * Allocates a fresh unused token from the token pool. + */ +static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *tok; + if (parser->toknext >= num_tokens) { + return NULL; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end = -1; + tok->size = 0; +#ifdef JSMN_PARENT_LINKS + tok->parent = -1; +#endif + return tok; +} + +/** + * Fills token type and boundaries. + */ +static void jsmn_fill_token(jsmntok_t *token, const jsmntype_t type, + const int start, const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size = 0; +} + +/** + * Fills next available token with JSON primitive. + */ +static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + int start; + + start = parser->pos; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + switch (js[parser->pos]) { +#ifndef JSMN_STRICT + /* In strict mode primitive must be followed by "," or "}" or "]" */ + case ':': +#endif + case '\t': + case '\r': + case '\n': + case ' ': + case ',': + case ']': + case '}': + goto found; + default: + /* to quiet a warning from gcc*/ + break; + } + if (js[parser->pos] < 32 || js[parser->pos] >= 127) { + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } +#ifdef JSMN_STRICT + /* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; + return JSMN_ERROR_PART; +#endif + +found: + if (tokens == NULL) { + parser->pos--; + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; + return 0; +} + +/** + * Fills next token with JSON string. + */ +static int jsmn_parse_string(jsmn_parser *parser, const char *js, + const size_t len, jsmntok_t *tokens, + const size_t num_tokens) { + jsmntok_t *token; + + int start = parser->pos; + + /* Skip starting quote */ + parser->pos++; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c = js[parser->pos]; + + /* Quote: end of string */ + if (c == '\"') { + if (tokens == NULL) { + return 0; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + parser->pos = start; + return JSMN_ERROR_NOMEM; + } + jsmn_fill_token(token, JSMN_STRING, start + 1, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + return 0; + } + + /* Backslash: Quoted symbol expected */ + if (c == '\\' && parser->pos + 1 < len) { + int i; + parser->pos++; + switch (js[parser->pos]) { + /* Allowed escaped symbols */ + case '\"': + case '/': + case '\\': + case 'b': + case 'f': + case 'r': + case 'n': + case 't': + break; + /* Allows escaped symbol \uXXXX */ + case 'u': + parser->pos++; + for (i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; + i++) { + /* If it isn't a hex character we have an error */ + if (!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */ + (js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */ + (js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */ + parser->pos = start; + return JSMN_ERROR_INVAL; + } + parser->pos++; + } + parser->pos--; + break; + /* Unexpected symbol */ + default: + parser->pos = start; + return JSMN_ERROR_INVAL; + } + } + } + parser->pos = start; + return JSMN_ERROR_PART; +} + +/** + * Parse JSON string and fill tokens. + */ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, + jsmntok_t *tokens, const unsigned int num_tokens) { + int r; + int i; + jsmntok_t *token; + int count = parser->toknext; + + for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) { + char c; + jsmntype_t type; + + c = js[parser->pos]; + switch (c) { + case '{': + case '[': + count++; + if (tokens == NULL) { + break; + } + token = jsmn_alloc_token(parser, tokens, num_tokens); + if (token == NULL) { + return JSMN_ERROR_NOMEM; + } + if (parser->toksuper != -1) { + jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT + /* In strict mode an object or array can't become a key */ + if (t->type == JSMN_OBJECT) { + return JSMN_ERROR_INVAL; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY); + token->start = parser->pos; + parser->toksuper = parser->toknext - 1; + break; + case '}': + case ']': + if (tokens == NULL) { + break; + } + type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY); +#ifdef JSMN_PARENT_LINKS + if (parser->toknext < 1) { + return JSMN_ERROR_INVAL; + } + token = &tokens[parser->toknext - 1]; + for (;;) { + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + token->end = parser->pos + 1; + parser->toksuper = token->parent; + break; + } + if (token->parent == -1) { + if (token->type != type || parser->toksuper == -1) { + return JSMN_ERROR_INVAL; + } + break; + } + token = &tokens[token->parent]; + } +#else + for (i = parser->toknext - 1; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + if (token->type != type) { + return JSMN_ERROR_INVAL; + } + parser->toksuper = -1; + token->end = parser->pos + 1; + break; + } + } + /* Error if unmatched closing bracket */ + if (i == -1) { + return JSMN_ERROR_INVAL; + } + for (; i >= 0; i--) { + token = &tokens[i]; + if (token->start != -1 && token->end == -1) { + parser->toksuper = i; + break; + } + } +#endif + break; + case '\"': + r = jsmn_parse_string(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + case '\t': + case '\r': + case '\n': + case ' ': + break; + case ':': + parser->toksuper = parser->toknext - 1; + break; + case ',': + if (tokens != NULL && parser->toksuper != -1 && + tokens[parser->toksuper].type != JSMN_ARRAY && + tokens[parser->toksuper].type != JSMN_OBJECT) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else + for (i = parser->toknext - 1; i >= 0; i--) { + if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) { + if (tokens[i].start != -1 && tokens[i].end == -1) { + parser->toksuper = i; + break; + } + } + } +#endif + } + break; +#ifdef JSMN_STRICT + /* In strict mode primitives are: numbers and booleans */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 't': + case 'f': + case 'n': + /* And they must not be keys of the object */ + if (tokens != NULL && parser->toksuper != -1) { + const jsmntok_t *t = &tokens[parser->toksuper]; + if (t->type == JSMN_OBJECT || + (t->type == JSMN_STRING && t->size != 0)) { + return JSMN_ERROR_INVAL; + } + } +#else + /* In non-strict mode every unquoted value is a primitive */ + default: +#endif + r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens); + if (r < 0) { + return r; + } + count++; + if (parser->toksuper != -1 && tokens != NULL) { + tokens[parser->toksuper].size++; + } + break; + +#ifdef JSMN_STRICT + /* Unexpected char in strict mode */ + default: + return JSMN_ERROR_INVAL; +#endif + } + } + + if (tokens != NULL) { + for (i = parser->toknext - 1; i >= 0; i--) { + /* Unmatched opened object or array */ + if (tokens[i].start != -1 && tokens[i].end == -1) { + return JSMN_ERROR_PART; + } + } + } + + return count; +} + +/** + * Creates a new parser based over a given buffer with an array of tokens + * available. + */ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos = 0; + parser->toknext = 0; + parser->toksuper = -1; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */ diff --git a/src/network/network.c b/src/network/network.c new file mode 100644 index 0000000..652f96c --- /dev/null +++ b/src/network/network.c @@ -0,0 +1,1880 @@ +// +// network.c +// cloudsync +// +// Created by Marco Bambini on 12/12/24. +// + +#ifndef CLOUDSYNC_OMIT_NETWORK + +#include +#include +#include +#include +#include + +#ifdef CLOUDSYNC_NETWORK_TRACE +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + +#include "network.h" +#include "../utils.h" +#include "../dbutils.h" +#include "../cloudsync.h" +#include "network_private.h" + +#define JSMN_STATIC +#include "jsmn.h" + +#ifndef SQLITE_WASM_EXTRA_INIT +#ifndef CLOUDSYNC_OMIT_CURL +#include "curl/curl.h" +#endif +#else +#define curl_free(x) free(x) +char *substr(const char *start, const char *end); +#endif + +#ifdef __ANDROID__ +#include "cacert.h" +static size_t cacert_len = sizeof(cacert_pem) - 1; +#endif + +#define CLOUDSYNC_NETWORK_MINBUF_SIZE 512 +#define CLOUDSYNC_SESSION_TOKEN_MAXSIZE 4096 + +#ifndef CLOUDSYNC_CURL_MAXCONNECTS +#define CLOUDSYNC_CURL_MAXCONNECTS 2L +#endif +#ifndef CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS +#define CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS 15L +#endif +#ifndef CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS +#define CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS 60L +#endif +#ifndef CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE +#define CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE (128 * 1024) +#endif + +#define DEFAULT_SYNC_WAIT_MS 100 +#define DEFAULT_SYNC_MAX_RETRIES 1 + +#define MAX_QUERY_VALUE_LEN 256 + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + +// MARK: - + +struct network_data { + char site_id[UUID_STR_MAXLEN]; + char *authentication; // apikey or token + char *org_id; // organization ID for X-CloudSync-Org header + char *ticket; // optional short-lived sync runtime ticket + char *ticket_expires_at; + char *check_endpoint; + char *upload_endpoint; + char *apply_endpoint; + char *status_endpoint; + int ticket_enabled; +#ifndef CLOUDSYNC_OMIT_CURL + CURL *api_curl; + CURL *artifact_curl; + int curl_pool_enabled; +#endif +}; + +#ifdef CLOUDSYNC_NETWORK_TRACE +double network_trace_now_ms(void) { +#ifdef _WIN32 + LARGE_INTEGER freq; + LARGE_INTEGER counter; + QueryPerformanceFrequency(&freq); + QueryPerformanceCounter(&counter); + return ((double)counter.QuadPart * 1000.0) / (double)freq.QuadPart; +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((double)ts.tv_sec * 1000.0) + ((double)ts.tv_nsec / 1000000.0); +#endif +} + +const char *network_trace_endpoint_name(network_data *data, const char *endpoint) { + if (!data || !endpoint) return "unknown"; + if (data->check_endpoint && strcmp(endpoint, data->check_endpoint) == 0) return "check"; + if (data->upload_endpoint && strcmp(endpoint, data->upload_endpoint) == 0) return "upload-url"; + if (data->apply_endpoint && strcmp(endpoint, data->apply_endpoint) == 0) return "apply"; + if (data->status_endpoint && strcmp(endpoint, data->status_endpoint) == 0) return "status"; + return "artifact"; +} + +const char *network_trace_result_name(int code) { + switch (code) { + case CLOUDSYNC_NETWORK_OK: return "ok"; + case CLOUDSYNC_NETWORK_ERROR: return "error"; + case CLOUDSYNC_NETWORK_BUFFER: return "buffer"; + default: return "unknown"; + } +} + +void network_trace_log(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, double elapsed_ms) { + fprintf(stderr, + "[cloudsync-network] endpoint=%s method=%s http_status=%ld result=%s request_bytes=%zu bytes=%zu elapsed_ms=%.2f\n", + network_trace_endpoint_name(data, endpoint), method, http_status, + network_trace_result_name(result_code), request_bytes, bytes, elapsed_ms); +} + +#ifndef CLOUDSYNC_OMIT_CURL +void network_trace_log_curl(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, CURL *curl, bool pooled, double elapsed_ms) { + double namelookup = 0.0; + double connect = 0.0; + double appconnect = 0.0; + double starttransfer = 0.0; + double total = 0.0; + long num_connects = 0; + if (curl) { + curl_easy_getinfo(curl, CURLINFO_NAMELOOKUP_TIME, &namelookup); + curl_easy_getinfo(curl, CURLINFO_CONNECT_TIME, &connect); + curl_easy_getinfo(curl, CURLINFO_APPCONNECT_TIME, &appconnect); + curl_easy_getinfo(curl, CURLINFO_STARTTRANSFER_TIME, &starttransfer); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total); + curl_easy_getinfo(curl, CURLINFO_NUM_CONNECTS, &num_connects); + } + fprintf(stderr, + "[cloudsync-network] endpoint=%s method=%s pool=%s http_status=%ld result=%s request_bytes=%zu bytes=%zu elapsed_ms=%.2f curl_total_ms=%.2f dns_ms=%.2f connect_ms=%.2f tls_ms=%.2f starttransfer_ms=%.2f num_connects=%ld\n", + network_trace_endpoint_name(data, endpoint), method, + pooled ? "on" : "off", http_status, + network_trace_result_name(result_code), request_bytes, bytes, elapsed_ms, + total * 1000.0, namelookup * 1000.0, connect * 1000.0, + appconnect * 1000.0, starttransfer * 1000.0, num_connects); +} +#endif +#endif + +typedef struct { + char *buffer; + size_t balloc; + size_t bused; + int zero_term; +} network_buffer; + + +typedef struct { + const char *data; + size_t size; + size_t read_pos; +} network_read_data; + +typedef struct { + char *ticket; + char *expires_at; +} network_ticket_headers; + +static const char *cloudsync_default_headers[] = { + CLOUDSYNC_HEADER_VERSION_LINE, +}; + +static const char *cloudsync_check_headers[] = { + CLOUDSYNC_HEADER_VERSION_LINE, + CLOUDSYNC_HEADER_CHECK_CAPABILITIES, +}; + +#define ARRAY_LEN(a) ((int)(sizeof(a) / sizeof((a)[0]))) + +// MARK: - + +void network_result_cleanup (NETWORK_RESULT *res) { + if (res->xfree) { + res->xfree(res->xdata); + } else if (res->buffer) { + cloudsync_memory_free(res->buffer); + } +} + +char *network_data_get_siteid (network_data *data) { + return data->site_id; +} + +char *network_data_get_orgid (network_data *data) { + return data->org_id; +} + +char *network_data_get_ticket (network_data *data) { + return data->ticket; +} + +static void network_data_clear_ticket (network_data *data) { + if (!data) return; + if (data->ticket) cloudsync_memory_free(data->ticket); + if (data->ticket_expires_at) cloudsync_memory_free(data->ticket_expires_at); + data->ticket = NULL; + data->ticket_expires_at = NULL; +} + +bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, char *apply, char *status) { + // sanity check + if (!check || !upload) return false; + + // always free previous owned pointers + if (data->authentication) cloudsync_memory_free(data->authentication); + network_data_clear_ticket(data); + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + + // clear pointers + data->authentication = NULL; + data->check_endpoint = NULL; + data->upload_endpoint = NULL; + data->apply_endpoint = NULL; + data->status_endpoint = NULL; + + // make a copy of the new endpoints + char *auth_copy = NULL; + char *check_copy = NULL; + char *upload_copy = NULL; + char *apply_copy = NULL; + char *status_copy = NULL; + + // auth is optional + if (auth) { + auth_copy = cloudsync_string_dup(auth); + if (!auth_copy) goto abort_endpoints; + } + check_copy = cloudsync_string_dup(check); + if (!check_copy) goto abort_endpoints; + + upload_copy = cloudsync_string_dup(upload); + if (!upload_copy) goto abort_endpoints; + + apply_copy = cloudsync_string_dup(apply); + if (!apply_copy) goto abort_endpoints; + + status_copy = cloudsync_string_dup(status); + if (!status_copy) goto abort_endpoints; + + data->authentication = auth_copy; + data->check_endpoint = check_copy; + data->upload_endpoint = upload_copy; + data->apply_endpoint = apply_copy; + data->status_endpoint = status_copy; + return true; + +abort_endpoints: + if (auth_copy) cloudsync_memory_free(auth_copy); + if (check_copy) cloudsync_memory_free(check_copy); + if (upload_copy) cloudsync_memory_free(upload_copy); + if (apply_copy) cloudsync_memory_free(apply_copy); + if (status_copy) cloudsync_memory_free(status_copy); + return false; +} + +void network_data_free (network_data *data) { + if (!data) return; + +#ifndef CLOUDSYNC_OMIT_CURL + if (data->api_curl) curl_easy_cleanup(data->api_curl); + if (data->artifact_curl) curl_easy_cleanup(data->artifact_curl); +#endif + if (data->authentication) cloudsync_memory_free(data->authentication); + if (data->org_id) cloudsync_memory_free(data->org_id); + network_data_clear_ticket(data); + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + cloudsync_memory_free(data); +} + +// MARK: - Utils - + +static bool network_endpoint_is_api(network_data *data, const char *endpoint) { + if (!data || !endpoint) return false; + return (data->check_endpoint && strcmp(endpoint, data->check_endpoint) == 0) || + (data->upload_endpoint && strcmp(endpoint, data->upload_endpoint) == 0) || + (data->apply_endpoint && strcmp(endpoint, data->apply_endpoint) == 0) || + (data->status_endpoint && strcmp(endpoint, data->status_endpoint) == 0); +} + +static bool network_env_disabled(const char *value) { + return value && (strcmp(value, "0") == 0 || strcmp(value, "false") == 0 || strcmp(value, "off") == 0 || strcmp(value, "no") == 0); +} + +static bool network_ticket_enabled(network_data *data) { + if (!data) return false; + if (data->ticket_enabled == 0) { + const char *value = getenv("CLOUDSYNC_NETWORK_TICKET"); + data->ticket_enabled = network_env_disabled(value) ? -1 : 1; + } + return data->ticket_enabled > 0; +} + +bool network_data_should_use_ticket (network_data *data, const char *endpoint, const char *authentication) { + return data && authentication && authentication[0] != '\0' && data->ticket && data->ticket[0] != '\0' && + network_ticket_enabled(data) && network_endpoint_is_api(data, endpoint); +} + +void network_data_update_ticket (network_data *data, const char *ticket, const char *expires_at) { + if (!data || !ticket || ticket[0] == '\0') return; + + char *ticket_copy = cloudsync_string_dup(ticket); + if (!ticket_copy) return; + + char *expires_copy = NULL; + if (expires_at && expires_at[0] != '\0') { + expires_copy = cloudsync_string_dup(expires_at); + if (!expires_copy) { + cloudsync_memory_free(ticket_copy); + return; + } + } + + network_data_clear_ticket(data); + data->ticket = ticket_copy; + data->ticket_expires_at = expires_copy; + +#ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] received_ticket=%s expires_at=%s\n", + data->ticket ? "true" : "false", data->ticket_expires_at ? data->ticket_expires_at : ""); +#endif +} + +#ifndef CLOUDSYNC_OMIT_CURL +static bool network_curl_pool_enabled(network_data *data) { + if (!data) return false; + if (data->curl_pool_enabled == 0) { + const char *value = getenv("CLOUDSYNC_CURL_POOL"); + data->curl_pool_enabled = network_env_disabled(value) ? -1 : 1; + } + return data->curl_pool_enabled > 0; +} + +static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, bool *pooled) { + if (pooled) *pooled = false; + if (!network_curl_pool_enabled(data)) { + return curl_easy_init(); + } + + CURL **slot = network_endpoint_is_api(data, endpoint) ? &data->api_curl : &data->artifact_curl; + if (!*slot) { + *slot = curl_easy_init(); + } else { + curl_easy_reset(*slot); + } + if (!*slot) return NULL; + + curl_easy_setopt(*slot, CURLOPT_MAXCONNECTS, CLOUDSYNC_CURL_MAXCONNECTS); + curl_easy_setopt(*slot, CURLOPT_MAXAGE_CONN, CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS); + curl_easy_setopt(*slot, CURLOPT_MAXLIFETIME_CONN, CLOUDSYNC_CURL_MAXLIFETIME_CONN_SECONDS); + if (pooled) *pooled = true; + return *slot; +} + +static bool network_buffer_check (network_buffer *data, size_t needed) { + // alloc/resize buffer + if (data->bused + needed > data->balloc) { + if (needed < CLOUDSYNC_NETWORK_MINBUF_SIZE) needed = CLOUDSYNC_NETWORK_MINBUF_SIZE; + size_t balloc = data->balloc + needed; + + char *buffer = cloudsync_memory_realloc(data->buffer, balloc); + if (!buffer) return false; + + data->buffer = buffer; + data->balloc = balloc; + } + + return true; +} + +static size_t network_receive_callback (void *ptr, size_t size, size_t nmemb, void *xdata) { + network_buffer *data = (network_buffer *)xdata; + + size_t ptr_size = (size*nmemb); + if (data->zero_term) ptr_size += 1; + + if (network_buffer_check(data, ptr_size) == false) return CURL_WRITEFUNC_ERROR; + memcpy(data->buffer+data->bused, ptr, size*nmemb); + data->bused += size*nmemb; + if (data->zero_term) data->buffer[data->bused] = 0; + + return (size * nmemb); +} + +static bool network_header_eq(const char *line, size_t len, const char *name) { + size_t name_len = strlen(name); + if (len <= name_len || line[name_len] != ':') return false; + for (size_t i = 0; i < name_len; i++) { + if (tolower((unsigned char)line[i]) != tolower((unsigned char)name[i])) return false; + } + return true; +} + +static char *network_header_value_dup(const char *line, size_t len, const char *name) { + size_t name_len = strlen(name); + const char *start = line + name_len + 1; + const char *end = line + len; + + while (start < end && (*start == ' ' || *start == '\t')) start++; + while (end > start && (end[-1] == '\r' || end[-1] == '\n' || end[-1] == ' ' || end[-1] == '\t')) end--; + + size_t value_len = (size_t)(end - start); + char *value = cloudsync_memory_zeroalloc(value_len + 1); + if (!value) return NULL; + memcpy(value, start, value_len); + value[value_len] = '\0'; + return value; +} + +static size_t network_header_callback(char *buffer, size_t size, size_t nitems, void *userdata) { + network_ticket_headers *ticket_headers = (network_ticket_headers *)userdata; + size_t len = size * nitems; + + if (network_header_eq(buffer, len, CLOUDSYNC_HEADER_TICKET)) { + char *ticket = network_header_value_dup(buffer, len, CLOUDSYNC_HEADER_TICKET); + if (ticket) { + if (ticket_headers->ticket) cloudsync_memory_free(ticket_headers->ticket); + ticket_headers->ticket = ticket; + } + } else if (network_header_eq(buffer, len, CLOUDSYNC_HEADER_TICKET_EXPIRES_AT)) { + char *expires_at = network_header_value_dup(buffer, len, CLOUDSYNC_HEADER_TICKET_EXPIRES_AT); + if (expires_at) { + if (ticket_headers->expires_at) cloudsync_memory_free(ticket_headers->expires_at); + ticket_headers->expires_at = expires_at; + } + } + + return len; +} + +NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers) { + char *buffer = NULL; + size_t blen = 0; + struct curl_slist* headers = NULL; + network_ticket_headers ticket_headers = {NULL, NULL}; + char errbuf[CURL_ERROR_SIZE] = {0}; + long response_code = 0; + bool pooled = false; + bool using_ticket = network_data_should_use_ticket(data, endpoint, authentication); + const char *method = (json_payload || is_post_request) ? "POST" : "GET"; +#ifndef CLOUDSYNC_NETWORK_TRACE + (void)method; +#endif +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); + size_t request_bytes = json_payload ? strlen(json_payload) : 0; +#endif + + CURL *curl = network_curl_for_endpoint(data, endpoint, &pooled); + if (!curl) return (NETWORK_RESULT){CLOUDSYNC_NETWORK_ERROR, NULL, 0, NULL, NULL}; + + // a buffer to store errors in + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + + CURLcode rc = curl_easy_setopt(curl, CURLOPT_URL, endpoint); + if (rc != CURLE_OK) goto cleanup; + + // set PEM + #ifdef __ANDROID__ + struct curl_blob pem_blob = { + .data = (void *)cacert_pem, + .len = cacert_len, + .flags = CURL_BLOB_NOCOPY + }; + curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); + #endif + + for (int i = 0; i < nextra_headers; i++) { + struct curl_slist *tmp = curl_slist_append(headers, extra_headers[i]); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (data->org_id) { + char org_header[512]; + snprintf(org_header, sizeof(org_header), "%s: %s", CLOUDSYNC_HEADER_ORG, data->org_id); + struct curl_slist *tmp = curl_slist_append(headers, org_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (json_payload) { + struct curl_slist *tmp = curl_slist_append(headers, "Content-Type: application/json"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + if (authentication) { + char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); + struct curl_slist *tmp = curl_slist_append(headers, auth_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + if (using_ticket) { + char ticket_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(ticket_header, sizeof(ticket_header), "%s: %s", CLOUDSYNC_HEADER_TICKET, data->ticket); + struct curl_slist *tmp = curl_slist_append(headers, ticket_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (headers) curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + network_buffer netdata = {NULL, 0, 0, (zero_terminated) ? 1 : 0}; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &netdata); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, network_receive_callback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &ticket_headers); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, network_header_callback); + + // add optional JSON payload (implies setting CURLOPT_POST to 1) + // or set the CURLOPT_POST option + if (json_payload) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_payload); + } else if (is_post_request) { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0L); + } + + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + rc = curl_easy_perform(curl); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (rc == CURLE_OK) { + buffer = netdata.buffer; + blen = netdata.bused; + if (response_code < 400 && ticket_headers.ticket) { + network_data_update_ticket(data, ticket_headers.ticket, ticket_headers.expires_at); + } + } else if (netdata.buffer) { + cloudsync_memory_free(netdata.buffer); + netdata.buffer = NULL; + } + +cleanup: + if (headers) curl_slist_free_all(headers); + if (ticket_headers.ticket) cloudsync_memory_free(ticket_headers.ticket); + if (ticket_headers.expires_at) cloudsync_memory_free(ticket_headers.expires_at); + + // build result + NETWORK_RESULT result = {0, NULL, 0, NULL, NULL}; + if (rc == CURLE_OK && response_code < 400) { + result.code = (buffer && blen) ? CLOUDSYNC_NETWORK_BUFFER : CLOUDSYNC_NETWORK_OK; + result.buffer = buffer; + result.blen = blen; + } else { + result.code = CLOUDSYNC_NETWORK_ERROR; + result.buffer = buffer ? buffer : (errbuf[0]) ? cloudsync_string_dup(errbuf) : NULL; + result.blen = buffer ? blen : rc; + } + + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log_curl(data, method, endpoint, response_code, result.code, request_bytes, result.blen, curl, pooled, network_trace_now_ms() - trace_start_ms); + #endif + if (curl && !pooled) curl_easy_cleanup(curl); + return result; +} + +static size_t network_read_callback (char *buffer, size_t size, size_t nitems, void *userdata) { + network_read_data *rd = (network_read_data *)userdata; + size_t max_read = size * nitems; + size_t bytes_left = rd->size - rd->read_pos; + size_t to_copy = bytes_left < max_read ? bytes_left : max_read; + + if (to_copy > 0) { + memcpy(buffer, rd->data + rd->read_pos, to_copy); + rd->read_pos += to_copy; + } + + return to_copy; +} + +bool network_send_buffer (network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { + struct curl_slist *headers = NULL; + bool result = false; + char errbuf[CURL_ERROR_SIZE] = {0}; + CURLcode rc = CURLE_OK; + long response_code = 0; + bool pooled = false; +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); +#endif + + // init/reuse curl + CURL *curl = network_curl_for_endpoint(data, endpoint, &pooled); + if (!curl) return false; + + // set the URL + if (curl_easy_setopt(curl, CURLOPT_URL, endpoint) != CURLE_OK) goto cleanup; + + // set PEM + #ifdef __ANDROID__ + struct curl_blob pem_blob = { + .data = (void *)cacert_pem, + .len = cacert_len, + .flags = CURL_BLOB_NOCOPY + }; + curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &pem_blob); + #endif + + // a buffer to store errors in + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + + // type header + struct curl_slist *tmp = curl_slist_append(headers, "Accept: text/plain"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + + if (authentication) { + // init authorization header + char auth_header[CLOUDSYNC_SESSION_TOKEN_MAXSIZE]; + snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", authentication); + struct curl_slist *tmp = curl_slist_append(headers, auth_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + if (data->org_id) { + char org_header[512]; + snprintf(org_header, sizeof(org_header), "%s: %s", CLOUDSYNC_HEADER_ORG, data->org_id); + struct curl_slist *tmp = curl_slist_append(headers, org_header); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + } + + // Set headers if needed (S3 pre-signed URLs usually do not require additional headers) + tmp = curl_slist_append(headers, "Content-Type: application/octet-stream"); + if (!tmp) {rc = CURLE_OUT_OF_MEMORY; goto cleanup;} + headers = tmp; + + if (!headers) goto cleanup; + if (curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers) != CURLE_OK) goto cleanup; + + // Set HTTP PUT method + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + // Set the size of the blob + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)blob_size); + + // Provide the data using a custom read callback + network_read_data rdata = { + .data = (const char *)blob, + .size = blob_size, + .read_pos = 0 + }; + + curl_easy_setopt(curl, CURLOPT_READFUNCTION, network_read_callback); + curl_easy_setopt(curl, CURLOPT_READDATA, &rdata); + + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + // perform the upload + rc = curl_easy_perform(curl); + if (curl) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (rc == CURLE_OK) result = true; + +cleanup: + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log_curl(data, "PUT", endpoint, response_code, + result ? CLOUDSYNC_NETWORK_OK : CLOUDSYNC_NETWORK_ERROR, + (size_t)blob_size, + result ? (size_t)blob_size : 0, + curl, pooled, network_trace_now_ms() - trace_start_ms); + #endif + if (curl && !pooled) curl_easy_cleanup(curl); + if (headers) curl_slist_free_all(headers); + return result; +} +#endif + +int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) { + int rc = 0; + switch (result->code) { + case CLOUDSYNC_NETWORK_OK: + sqlite3_result_error_code(context, SQLITE_OK); + sqlite3_result_int(context, 0); + rc = 0; + break; + + case CLOUDSYNC_NETWORK_ERROR: + sqlite3_result_error(context, (result->buffer) ? result->buffer : "Memory error.", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + rc = -1; + break; + + case CLOUDSYNC_NETWORK_BUFFER: + sqlite3_result_error_code(context, SQLITE_OK); + sqlite3_result_text(context, result->buffer, (int)result->blen, SQLITE_TRANSIENT); + rc = (int)result->blen; + break; + } + return rc; +} + +// If err_out is non-NULL, cloudsync_payload_apply failures are returned via +// *err_out (malloc'd, caller must cloudsync_memory_free) instead of being raised +// on the sqlite3_context. This lets composite callers (cloudsync_network_sync) +// surface apply errors as structured JSON. Endpoint/network errors always raise +// a SQL error regardless of err_out. +int network_download_changes (sqlite3_context *context, const char *download_url, int *pnrows, char **err_out) { + DEBUG_FUNCTION("network_download_changes"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) { + sqlite3_result_error(context, "Unable to retrieve network CloudSync context.", -1); + return -1; + } + + NETWORK_RESULT result = network_receive_buffer(netdata, download_url, NULL, false, false, NULL, NULL, 0); + + int rc = SQLITE_OK; + if (result.code == CLOUDSYNC_NETWORK_BUFFER) { + rc = cloudsync_payload_apply(data, result.buffer, (int)result.blen, pnrows); + if (rc != DBRES_OK) { + const char *msg = cloudsync_errmsg(data); + if (!msg || !msg[0]) msg = "cloudsync_payload_apply failed"; + if (err_out) *err_out = cloudsync_string_dup(msg); + else sqlite3_result_error(context, msg, -1); + if (pnrows) *pnrows = 0; + } + } else if (result.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &result); + rc = -1; + if (pnrows) *pnrows = 0; + } else { + // CLOUDSYNC_NETWORK_OK — no data, not an error + if (pnrows) *pnrows = 0; + } + network_result_cleanup(&result); + + return rc; +} + +char *network_authentication_token (const char *key, const char *value) { + size_t len = strlen(key) + strlen(value) + 64; + char *buffer = cloudsync_memory_zeroalloc(len); + if (!buffer) return NULL; + + // build new token + // we don't need a prefix because the token alreay include a prefix "sqa_" + snprintf(buffer, len, "%s", value); + return buffer; +} + +// MARK: - JSON helpers (jsmn) - + +#define JSMN_MAX_TOKENS 64 + +static bool jsmn_token_eq(const char *json, const jsmntok_t *tok, const char *s) { + return (tok->type == JSMN_STRING && + (int)strlen(s) == tok->end - tok->start && + strncmp(json + tok->start, s, tok->end - tok->start) == 0); +} + +static int jsmn_find_key(const char *json, const jsmntok_t *tokens, int ntokens, const char *key) { + for (int i = 1; i + 1 < ntokens; i++) { + if (jsmn_token_eq(json, &tokens[i], key)) return i; + } + return -1; +} + +static char *json_unescape_string(const char *src, int len) { + char *out = cloudsync_memory_zeroalloc(len + 1); + if (!out) return NULL; + + int j = 0; + for (int i = 0; i < len; ) { + if (src[i] == '\\' && i + 1 < len) { + char c = src[i + 1]; + if (c == '"' || c == '\\' || c == '/') { out[j++] = c; i += 2; } + else if (c == 'n') { out[j++] = '\n'; i += 2; } + else if (c == 'r') { out[j++] = '\r'; i += 2; } + else if (c == 't') { out[j++] = '\t'; i += 2; } + else if (c == 'b') { out[j++] = '\b'; i += 2; } + else if (c == 'f') { out[j++] = '\f'; i += 2; } + else if (c == 'u' && i + 5 < len) { + unsigned int cp = 0; + for (int k = 0; k < 4; k++) { + char h = src[i + 2 + k]; + cp <<= 4; + if (h >= '0' && h <= '9') cp |= h - '0'; + else if (h >= 'a' && h <= 'f') cp |= 10 + h - 'a'; + else if (h >= 'A' && h <= 'F') cp |= 10 + h - 'A'; + } + if (cp < 0x80) { out[j++] = (char)cp; } + else { out[j++] = '?'; } // non-ASCII: replace + i += 6; + } + else { out[j++] = src[i]; i++; } + } else { + out[j++] = src[i]; i++; + } + } + out[j] = '\0'; + return out; +} + +static char *json_extract_string(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return NULL; + + jsmn_parser parser; + jsmntok_t tokens[JSMN_MAX_TOKENS]; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + if (ntokens < 1) return NULL; + + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i < 0 || i + 1 >= ntokens) return NULL; + + jsmntok_t *val = &tokens[i + 1]; + if (val->type != JSMN_STRING) return NULL; + + return json_unescape_string(json + val->start, val->end - val->start); +} + +static int64_t json_extract_int(const char *json, size_t json_len, const char *key, int64_t default_value) { + if (!json || json_len == 0 || !key) return default_value; + + jsmn_parser parser; + jsmntok_t tokens[JSMN_MAX_TOKENS]; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return default_value; + + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i < 0 || i + 1 >= ntokens) return default_value; + + jsmntok_t *val = &tokens[i + 1]; + if (val->type != JSMN_PRIMITIVE) return default_value; + + return strtoll(json + val->start, NULL, 10); +} + +static int json_extract_array_size(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return -1; + + jsmn_parser parser; + jsmntok_t tokens[JSMN_MAX_TOKENS]; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + if (ntokens < 1 || tokens[0].type != JSMN_OBJECT) return -1; + + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i < 0 || i + 1 >= ntokens) return -1; + + jsmntok_t *val = &tokens[i + 1]; + if (val->type != JSMN_ARRAY) return -1; + + return val->size; +} + +// Escape a string for safe embedding as a JSON string value (without surrounding quotes). +// Caller must free with cloudsync_memory_free. +static char *json_escape_string(const char *src) { + if (!src) return NULL; + size_t len = strlen(src); + // worst case: every char becomes \uXXXX (6 bytes) + char *out = cloudsync_memory_zeroalloc(len * 6 + 1); + if (!out) return NULL; + size_t j = 0; + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char)src[i]; + switch (c) { + case '"': out[j++] = '\\'; out[j++] = '"'; break; + case '\\': out[j++] = '\\'; out[j++] = '\\'; break; + case '\b': out[j++] = '\\'; out[j++] = 'b'; break; + case '\f': out[j++] = '\\'; out[j++] = 'f'; break; + case '\n': out[j++] = '\\'; out[j++] = 'n'; break; + case '\r': out[j++] = '\\'; out[j++] = 'r'; break; + case '\t': out[j++] = '\\'; out[j++] = 't'; break; + default: + if (c < 0x20) { + static const char hex[] = "0123456789abcdef"; + out[j++] = '\\'; out[j++] = 'u'; + out[j++] = '0'; out[j++] = '0'; + out[j++] = hex[(c >> 4) & 0xf]; + out[j++] = hex[c & 0xf]; + } else { + out[j++] = (char)c; + } + } + } + out[j] = '\0'; + return out; +} + +// Returns a malloc'd copy of the raw JSON substring for an object-valued key +// (found at any depth). Caller must free with cloudsync_memory_free. +static char *json_extract_object_raw(const char *json, size_t json_len, const char *key) { + if (!json || json_len == 0 || !key) return NULL; + + jsmn_parser parser; + jsmntok_t tokens[JSMN_MAX_TOKENS]; + jsmn_init(&parser); + int ntokens = jsmn_parse(&parser, json, json_len, tokens, JSMN_MAX_TOKENS); + if (ntokens < 1) return NULL; + + int i = jsmn_find_key(json, tokens, ntokens, key); + if (i < 0 || i + 1 >= ntokens) return NULL; + + jsmntok_t *val = &tokens[i + 1]; + if (val->type != JSMN_OBJECT) return NULL; + + int len = val->end - val->start; + if (len <= 0) return NULL; + + char *out = cloudsync_memory_zeroalloc(len + 1); + if (!out) return NULL; + memcpy(out, json + val->start, len); + out[len] = '\0'; + return out; +} + +int network_extract_query_param (const char *query, const char *key, char *output, size_t output_size) { + if (!query || !key || !output || output_size == 0) { + return -1; // Invalid input + } + + size_t key_len = strlen(key); + const char *p = query; + #ifdef SQLITE_WASM_EXTRA_INIT + if (*p == '?') p++; + #endif + + while (p && *p) { + // Find the start of a key=value pair + const char *key_start = p; + const char *eq = strchr(key_start, '='); + if (!eq) break; // No '=' found, malformed query string + + size_t current_key_len = eq - key_start; + + // Check if the key matches (ensuring it's the full key) + if (current_key_len == key_len && strncmp(key_start, key, key_len) == 0) { + // Extract the value + const char *value_start = eq + 1; + const char *end = strchr(value_start, '&'); // Find end of value + + size_t value_len = (end) ? (size_t)(end - value_start) : strlen(value_start); + if (value_len >= output_size) { + return -2; // Output buffer too small + } + + strncpy(output, value_start, value_len); + output[value_len] = '\0'; // Null-terminate + return 0; // Success + } + + // Move to the next parameter + p = strchr(p, '&'); + if (p) p++; // Skip '&' + } + + return -3; // Key not found +} + +static bool network_compute_endpoints_with_address (sqlite3_context *context, network_data *data, const char *address, const char *managedDatabaseId) { + if (!managedDatabaseId || managedDatabaseId[0] == '\0') { + sqlite3_result_error(context, "managedDatabaseId cannot be empty", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + return false; + } + + if (!address || address[0] == '\0') { + sqlite3_result_error(context, "address cannot be empty", -1); + sqlite3_result_error_code(context, SQLITE_ERROR); + return false; + } + + // build endpoints: {address}/v2/cloudsync/databases/{managedDatabaseId}/{siteId}/{action} + size_t requested = strlen(address) + 1 + + strlen(CLOUDSYNC_ENDPOINT_PREFIX) + 1 + strlen(managedDatabaseId) + 1 + + UUID_STR_MAXLEN + 1 + 16; + char *check_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *upload_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *apply_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + char *status_endpoint = (char *)cloudsync_memory_zeroalloc(requested); + + if (!check_endpoint || !upload_endpoint || !apply_endpoint || !status_endpoint) { + sqlite3_result_error_code(context, SQLITE_NOMEM); + if (check_endpoint) cloudsync_memory_free(check_endpoint); + if (upload_endpoint) cloudsync_memory_free(upload_endpoint); + if (apply_endpoint) cloudsync_memory_free(apply_endpoint); + if (status_endpoint) cloudsync_memory_free(status_endpoint); + return false; + } + + // format: {address}/v2/cloudsync/databases/{managedDatabaseID}/{siteId}/{action} + snprintf(check_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_CHECK); + snprintf(upload_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_UPLOAD); + snprintf(apply_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_APPLY); + snprintf(status_endpoint, requested, "%s/%s/%s/%s/%s", + address, CLOUDSYNC_ENDPOINT_PREFIX, managedDatabaseId, data->site_id, CLOUDSYNC_ENDPOINT_STATUS); + + network_data_clear_ticket(data); + + if (data->check_endpoint) cloudsync_memory_free(data->check_endpoint); + data->check_endpoint = check_endpoint; + + if (data->upload_endpoint) cloudsync_memory_free(data->upload_endpoint); + data->upload_endpoint = upload_endpoint; + + if (data->apply_endpoint) cloudsync_memory_free(data->apply_endpoint); + data->apply_endpoint = apply_endpoint; + + if (data->status_endpoint) cloudsync_memory_free(data->status_endpoint); + data->status_endpoint = status_endpoint; + + return true; +} + +void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { + sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); + sqlite3_result_error_code(context, SQLITE_ERROR); +} + +// MARK: - Init / Cleanup - + +network_data *cloudsync_network_data (sqlite3_context *context) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (netdata) return netdata; + + netdata = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); + if (netdata) cloudsync_set_auxdata(data, netdata); + return netdata; +} + +static void cloudsync_network_init_internal (sqlite3_context *context, const char *address, const char *managedDatabaseId) { + #ifndef CLOUDSYNC_OMIT_CURL + curl_global_init(CURL_GLOBAL_ALL); + #endif + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + if (!netdata) goto abort_memory; + + // init context + uint8_t *site_id = (uint8_t *)cloudsync_context_init(data); + if (!site_id) goto abort_siteid; + + // save site_id string representation: 01957493c6c07e14803727e969f1d2cc + cloudsync_uuid_v7_stringify(site_id, netdata->site_id, false); + + // compute endpoints + // authentication can be set later via cloudsync_network_set_token/cloudsync_network_set_apikey + if (network_compute_endpoints_with_address(context, netdata, address, managedDatabaseId) == false) { + goto abort_cleanup; + } + + cloudsync_set_auxdata(data, netdata); + sqlite3_result_int(context, SQLITE_OK); + return; + +abort_memory: + sqlite3_result_error(context, "Unable to allocate memory in cloudsync_network_init.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + goto abort_cleanup; + +abort_siteid: + sqlite3_result_error(context, "Unable to compute/retrieve site_id.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + goto abort_cleanup; + +abort_cleanup: + cloudsync_set_auxdata(data, NULL); + network_data_free(netdata); +} + +void cloudsync_network_init (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_init"); + const char *managedDatabaseId = (const char *)sqlite3_value_text(argv[0]); + cloudsync_network_init_internal(context, CLOUDSYNC_DEFAULT_ADDRESS, managedDatabaseId); +} + +void cloudsync_network_init_custom (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_init_custom"); + const char *address = (const char *)sqlite3_value_text(argv[0]); + const char *managedDatabaseId = (const char *)sqlite3_value_text(argv[1]); + cloudsync_network_init_internal(context, address, managedDatabaseId); +} + +void cloudsync_network_cleanup_internal (sqlite3_context *context) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = cloudsync_network_data(context); + cloudsync_set_auxdata(data, NULL); + network_data_free(netdata); + + #ifndef CLOUDSYNC_OMIT_CURL + curl_global_cleanup(); + #endif +} + +void cloudsync_network_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_cleanup"); + + cloudsync_network_cleanup_internal(context); + sqlite3_result_int(context, SQLITE_OK); +} + +// MARK: - Public - + +bool cloudsync_network_set_authentication_token (sqlite3_context *context, const char *value, bool is_token) { + network_data *data = cloudsync_network_data(context); + if (!data) return false; + + const char *key = (is_token) ? "token" : "apikey"; + char *new_auth_token = network_authentication_token(key, value); + if (!new_auth_token) return false; + + if (data->authentication) cloudsync_memory_free(data->authentication); + network_data_clear_ticket(data); + data->authentication = new_auth_token; + + return true; +} + +void cloudsync_network_set_token (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_set_token"); + + const char *value = (const char *)sqlite3_value_text(argv[0]); + bool result = cloudsync_network_set_authentication_token(context, value, true); + (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); +} + +void cloudsync_network_set_apikey (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_set_apikey"); + + const char *value = (const char *)sqlite3_value_text(argv[0]); + bool result = cloudsync_network_set_authentication_token(context, value, false); + (result) ? sqlite3_result_int(context, SQLITE_OK) : sqlite3_result_error_code(context, SQLITE_NOMEM); +} + +// Returns a malloc'd JSON array string like '["tasks","users"]', or NULL on error/no results. +// Caller must free with cloudsync_memory_free. +static char *network_get_affected_tables(sqlite3 *db, int64_t since_db_version) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, + "SELECT json_group_array(DISTINCT tbl) FROM cloudsync_changes WHERE db_version > ?", + -1, &stmt, NULL); + if (rc != SQLITE_OK) return NULL; + sqlite3_bind_int64(stmt, 1, since_db_version); + + char *result = NULL; + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *json = (const char *)sqlite3_column_text(stmt, 0); + if (json) result = cloudsync_string_dup(json); + } + sqlite3_finalize(stmt); + return result; +} + +// MARK: - Sync result +// +// Error-handling contract for send/check/sync functions: +// - Endpoint/network errors (server unreachable, auth failure, bad URL) +// always raise a SQL error via sqlite3_result_error. +// - cloudsync_payload_apply failures (unknown schema hash, invalid checksum, +// decompression error) are returned as structured JSON via receive.error. +// - Server-reported failures from the SyncStatusResponse failures object are +// forwarded as send.lastFailure (failures.apply) and receive.lastFailure +// (failures.check). Per-function scoping: send_changes emits send.lastFailure +// only; check_changes emits receive.lastFailure only; sync emits both. +// +// Callers that receive JSON can trust that the server was reachable. +// A SQL error means connectivity or configuration is broken. + +typedef struct { + int64_t server_version; // lastOptimisticVersion + int64_t local_version; // new_db_version (max local) + const char *status; // computed status string + int rows_received; // rows from check + char *tables_json; // JSON array of affected table names, caller must cloudsync_memory_free + char *apply_failure_json; // raw JSON object for server-reported failures.apply, caller must cloudsync_memory_free + char *check_failure_json; // raw JSON object for server-reported failures.check, caller must cloudsync_memory_free +} sync_result; + +// Returns a malloc'd raw JSON copy of failures. ("apply" or "check"), +// or NULL when the field is missing or is JSON null. Caller frees with cloudsync_memory_free. +static char *json_extract_failure_stage(const char *json, size_t json_len, const char *stage_key) { + if (!json || json_len == 0 || !stage_key) return NULL; + + char *failures = json_extract_object_raw(json, json_len, "failures"); + if (!failures) return NULL; + + char *stage = json_extract_object_raw(failures, strlen(failures), stage_key); + cloudsync_memory_free(failures); + return stage; +} + +static char *network_base64_encode(const unsigned char *src, size_t len) { + static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + if (!src && len > 0) return NULL; + if (len > (SIZE_MAX - 1) / 4 * 3) return NULL; + + size_t out_len = 4 * ((len + 2) / 3); + char *out = cloudsync_memory_alloc((uint64_t)out_len + 1); + if (!out) return NULL; + + size_t i = 0; + size_t j = 0; + while (i < len) { + uint32_t octet_a = i < len ? src[i++] : 0; + uint32_t octet_b = i < len ? src[i++] : 0; + uint32_t octet_c = i < len ? src[i++] : 0; + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + out[j++] = table[(triple >> 18) & 0x3f]; + out[j++] = table[(triple >> 12) & 0x3f]; + out[j++] = table[(triple >> 6) & 0x3f]; + out[j++] = table[triple & 0x3f]; + } + + if (len % 3 == 1) { + out[out_len - 1] = '='; + out[out_len - 2] = '='; + } else if (len % 3 == 2) { + out[out_len - 1] = '='; + } + + out[out_len] = '\0'; + return out; +} + +static char *network_apply_json_payload(const char *transport_key, const char *transport_value, + int db_version_min, int db_version_max) { + if (!transport_key || !transport_value) return NULL; + + char *escaped_value = json_escape_string(transport_value); + if (!escaped_value) return NULL; + + size_t requested = strlen(transport_key) + strlen(escaped_value) + 128; + char *json_payload = cloudsync_memory_alloc((uint64_t)requested); + if (!json_payload) { + cloudsync_memory_free(escaped_value); + return NULL; + } + + snprintf(json_payload, requested, + "{\"%s\":\"%s\", \"dbVersionMin\":%d, \"dbVersionMax\":%d}", + transport_key, escaped_value, db_version_min, db_version_max); + + cloudsync_memory_free(escaped_value); + return json_payload; +} + +static const char *network_compute_status(int64_t last_optimistic, int64_t last_confirmed, + int gaps_size, int64_t local_version) { + if (last_optimistic < 0 || last_confirmed < 0) return "error"; + if (gaps_size > 0 || last_optimistic < local_version) return "out-of-sync"; + if (last_optimistic == last_confirmed) return "synced"; + return "syncing"; +} + +// MARK: - + +void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3 *db = sqlite3_context_db_handle(context); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return;} + + char *sql = "SELECT max(db_version) FROM cloudsync_changes WHERE site_id == (SELECT site_id FROM cloudsync_site_id WHERE rowid=0)"; + int64_t last_local_change = 0; + int rc = database_select_int(data, sql, &last_local_change); + if (rc != DBRES_OK) { + sqlite3_result_error(context, sqlite3_errmsg(db), -1); + sqlite3_result_error_code(context, rc); + return; + } + + if (last_local_change == 0) { + sqlite3_result_int(context, 0); + return; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + + int64_t last_optimistic_version = -1; + + if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); + } else if (res.code != CLOUDSYNC_NETWORK_OK) { + network_result_to_sqlite_error(context, res, "unable to retrieve current status from remote host."); + network_result_cleanup(&res); + return; + } + + network_result_cleanup(&res); + sqlite3_result_int(context, (last_optimistic_version >= 0 && last_optimistic_version < last_local_change)); +} + +int cloudsync_network_send_changes_internal (sqlite3_context *context, int argc, sqlite3_value **argv, sync_result *out) { + DEBUG_FUNCTION("cloudsync_network_send_changes"); + + // retrieve global context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return SQLITE_ERROR;} + + // retrieve payload + char *blob = NULL; + int blob_size = 0, db_version = 0; + int64_t new_db_version = 0; + int rc = cloudsync_payload_get(data, &blob, &blob_size, &db_version, &new_db_version); + if (rc != SQLITE_OK) { + if (db_version < 0) sqlite3_result_error(context, "Unable to retrieve db_version.", -1); + else sqlite3_result_error(context, "Unable to retrieve changes in cloudsync_network_send_changes", -1); + return rc; + } + + // Case 1: empty local db — no payload and no server state, skip network entirely + if ((blob == NULL || blob_size == 0) && db_version == 0) { + if (out) { + out->server_version = 0; + out->local_version = 0; + out->status = network_compute_status(0, 0, 0, 0); + } + return SQLITE_OK; + } + + NETWORK_RESULT res; + if (blob != NULL && blob_size > 0) { + int db_version_min = db_version+1; + int db_version_max = (int)new_db_version; + if (db_version_min > db_version_max) db_version_min = db_version_max; + + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] send_changes blob_size=%d fast-lane:%s\n", + blob_size, + blob_size <= CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE ? "true" : "false"); + #endif + + if (blob_size <= CLOUDSYNC_NETWORK_FAST_LANE_MAX_BLOB_SIZE) { + char *blob_base64 = network_base64_encode((const unsigned char *)blob, (size_t)blob_size); + cloudsync_memory_free(blob); + if (!blob_base64) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to encode BLOB changes.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return SQLITE_NOMEM; + } + + char *json_payload = network_apply_json_payload("blob", blob_base64, db_version_min, db_version_max); + cloudsync_memory_free(blob_base64); + if (!json_payload) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to allocate apply request payload.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return SQLITE_NOMEM; + } + + res = network_receive_buffer(netdata, netdata->apply_endpoint, netdata->authentication, true, true, json_payload, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + cloudsync_memory_free(json_payload); + } else { + // bulk lane: stage the payload through the upload endpoint and apply by URL + res = network_receive_buffer(netdata, netdata->upload_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + if (res.code != CLOUDSYNC_NETWORK_BUFFER) { + cloudsync_memory_free(blob); + network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to receive upload URL"); + network_result_cleanup(&res); + return SQLITE_ERROR; + } + + char *s3_url = json_extract_string(res.buffer, res.blen, "url"); + if (!s3_url) { + cloudsync_memory_free(blob); + sqlite3_result_error(context, "cloudsync_network_send_changes: missing 'url' in upload response.", -1); + network_result_cleanup(&res); + return SQLITE_ERROR; + } + bool sent = network_send_buffer(netdata, s3_url, NULL, blob, blob_size); + cloudsync_memory_free(blob); + if (sent == false) { + cloudsync_memory_free(s3_url); + network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to upload BLOB changes to remote host."); + network_result_cleanup(&res); + return SQLITE_ERROR; + } + + char *json_payload = network_apply_json_payload("url", s3_url, db_version_min, db_version_max); + cloudsync_memory_free(s3_url); + if (!json_payload) { + sqlite3_result_error(context, "cloudsync_network_send_changes: unable to allocate apply request payload.", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + network_result_cleanup(&res); + return SQLITE_NOMEM; + } + + // free res + network_result_cleanup(&res); + + // notify remote host that we successfully uploaded changes + res = network_receive_buffer(netdata, netdata->apply_endpoint, netdata->authentication, true, true, json_payload, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + cloudsync_memory_free(json_payload); + } + } else { + // there is no data to send, just check the status to update the db_version value in settings and to reply the status + new_db_version = db_version; + res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + } + + int64_t last_optimistic_version = -1; + int64_t last_confirmed_version = -1; + int gaps_size = -1; + char *apply_failure_json = NULL; + char *check_failure_json = NULL; + + if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { + last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); + last_confirmed_version = json_extract_int(res.buffer, res.blen, "lastConfirmedVersion", -1); + gaps_size = json_extract_array_size(res.buffer, res.blen, "gaps"); + if (gaps_size < 0) gaps_size = 0; + apply_failure_json = json_extract_failure_stage(res.buffer, res.blen, "apply"); + check_failure_json = json_extract_failure_stage(res.buffer, res.blen, "check"); + } else if (res.code != CLOUDSYNC_NETWORK_OK) { + network_result_to_sqlite_error(context, res, "cloudsync_network_send_changes unable to apply changes to remote host."); + network_result_cleanup(&res); + return SQLITE_ERROR; + } + + // update db_version in settings + char buf[256]; + if (last_optimistic_version >= 0) { + if (last_optimistic_version != db_version) { + snprintf(buf, sizeof(buf), "%" PRId64, last_optimistic_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + } + } else if (new_db_version != db_version) { + snprintf(buf, sizeof(buf), "%" PRId64, new_db_version); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + } + + // populate sync result + if (out) { + out->server_version = last_optimistic_version; + out->local_version = new_db_version; + out->status = network_compute_status(last_optimistic_version, last_confirmed_version, gaps_size, new_db_version); + out->apply_failure_json = apply_failure_json; + out->check_failure_json = check_failure_json; + apply_failure_json = NULL; + check_failure_json = NULL; + } + if (apply_failure_json) cloudsync_memory_free(apply_failure_json); + if (check_failure_json) cloudsync_memory_free(check_failure_json); + + network_result_cleanup(&res); + return SQLITE_OK; +} + +void cloudsync_network_send_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_send_changes"); + + // send-scoped: emits send.lastFailure (from failures.apply) only. + // failures.check arriving in the same response is parsed but discarded here. + sync_result sr = {.server_version = -1}; + int rc = cloudsync_network_send_changes_internal(context, argc, argv, &sr); + if (rc != SQLITE_OK) { + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + char *buf; + if (sr.apply_failure_json) { + buf = cloudsync_memory_mprintf( + "{\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"lastFailure\":%s}}", + sr.status ? sr.status : "error", + (long long)sr.local_version, (long long)sr.server_version, + sr.apply_failure_json); + } else { + buf = cloudsync_memory_mprintf( + "{\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld}}", + sr.status ? sr.status : "error", + (long long)sr.local_version, (long long)sr.server_version); + } + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync_result *out, char **err_out) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) {sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); return -1;} + + int64_t db_version = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION); + if (db_version<0) {sqlite3_result_error(context, "Unable to retrieve db_version.", -1); return -1;} + + int seq = dbutils_settings_get_int_value(data, CLOUDSYNC_KEY_CHECK_SEQ); + if (seq<0) {sqlite3_result_error(context, "Unable to retrieve seq.", -1); return -1;} + + // Capture local db_version before download so we can query cloudsync_changes afterwards + int64_t prev_dbv = cloudsync_dbversion(data); + + char json_payload[2024]; + snprintf(json_payload, sizeof(json_payload), "{\"dbVersion\":%lld, \"seq\":%d}", (long long)db_version, seq); + + NETWORK_RESULT result = network_receive_buffer(netdata, netdata->check_endpoint, netdata->authentication, true, true, json_payload, cloudsync_check_headers, ARRAY_LEN(cloudsync_check_headers)); + int rc = SQLITE_OK; + if (result.code == CLOUDSYNC_NETWORK_BUFFER) { + // The /check endpoint returns one of two shapes: + // HTTP 200 → {"url": "..."} (artifact ready for download) + // HTTP 202 → SyncStatusResponse (no artifact yet — status snapshot, + // may include failures.check) + // Branch on the presence of "url" rather than HTTP status; both shapes arrive as BUFFER. + char *download_url = json_extract_string(result.buffer, result.blen, "url"); + if (download_url) { + rc = network_download_changes(context, download_url, pnrows, err_out); + cloudsync_memory_free(download_url); + } + // failures.check may appear in either shape; extract opportunistically. + if (out) { + char *check_failure = json_extract_failure_stage(result.buffer, result.blen, "check"); + if (check_failure) { + if (out->check_failure_json) cloudsync_memory_free(out->check_failure_json); + out->check_failure_json = check_failure; + } + } + } else if (result.code == CLOUDSYNC_NETWORK_ERROR) { + network_set_sqlite_result(context, &result); + rc = -1; + } else { + // CLOUDSYNC_NETWORK_OK — no body (older server) — not an error + rc = 0; + } + + if (out && pnrows) out->rows_received = *pnrows; + + // Query cloudsync_changes for affected tables after successful download + if (out && rc == SQLITE_OK && pnrows && *pnrows > 0) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + out->tables_json = network_get_affected_tables(db, prev_dbv); + } + + network_result_cleanup(&result); + return rc; +} + +void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retries) { + sync_result sr = {.server_version = -1}; + int rc = cloudsync_network_send_changes_internal(context, 0, NULL, &sr); + if (rc != SQLITE_OK) { + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + int ntries = 0; + int nrows = 0; + char *receive_err = NULL; + while (ntries < max_retries) { + if (ntries > 0) sqlite3_sleep(wait_ms); + if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } + if (receive_err) { cloudsync_memory_free(receive_err); receive_err = NULL; } + rc = cloudsync_network_check_internal(context, &nrows, &sr, &receive_err); + // a receive error (network or apply) won't fix itself across retries + if (rc != SQLITE_OK) break; + if (nrows > 0) break; + ntries++; + } + + // If the receive phase failed, still emit structured JSON so the caller + // sees that the send phase completed and understands why receive did not. + if (rc != SQLITE_OK && !receive_err) { + receive_err = cloudsync_string_dup("receive failed"); + } + if (receive_err) { + rc = SQLITE_OK; + nrows = 0; + if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } + } + + const char *tables = sr.tables_json ? sr.tables_json : "[]"; + const char *status = sr.status ? sr.status : "error"; + char *escaped_err = receive_err ? json_escape_string(receive_err) : NULL; + + // Build send and receive blocks separately to avoid combinatorial explosion + // across optional fields (send.lastFailure, receive.error, receive.lastFailure). + char *send_part = sr.apply_failure_json + ? cloudsync_memory_mprintf( + "\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"lastFailure\":%s}", + status, (long long)sr.local_version, (long long)sr.server_version, sr.apply_failure_json) + : cloudsync_memory_mprintf( + "\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld}", + status, (long long)sr.local_version, (long long)sr.server_version); + + char *recv_part; + if (escaped_err && sr.check_failure_json) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"error\":\"%s\",\"lastFailure\":%s}", + nrows, tables, escaped_err, sr.check_failure_json); + } else if (escaped_err) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"error\":\"%s\"}", + nrows, tables, escaped_err); + } else if (sr.check_failure_json) { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s,\"lastFailure\":%s}", + nrows, tables, sr.check_failure_json); + } else { + recv_part = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"tables\":%s}", + nrows, tables); + } + + char *buf = cloudsync_memory_mprintf("{%s,%s}", send_part, recv_part); + cloudsync_memory_free(send_part); + cloudsync_memory_free(recv_part); + + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (escaped_err) cloudsync_memory_free(escaped_err); + if (receive_err) cloudsync_memory_free(receive_err); + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +void cloudsync_network_sync0 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_sync2"); + + cloudsync_network_sync(context, DEFAULT_SYNC_WAIT_MS, DEFAULT_SYNC_MAX_RETRIES); +} + + +void cloudsync_network_sync2 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_sync2"); + + int wait_ms = sqlite3_value_int(argv[0]); + int max_retries = sqlite3_value_int(argv[1]); + + cloudsync_network_sync(context, wait_ms, max_retries); +} + + +void cloudsync_network_check_changes (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_check_changes"); + + // check-scoped: emits receive.error (client-side apply) and/or + // receive.lastFailure (server-side failures.check) only — never a send block. + sync_result sr = {.server_version = -1}; + char *receive_err = NULL; + int nrows = 0; + int rc = cloudsync_network_check_internal(context, &nrows, &sr, &receive_err); + + // Endpoint/network errors already raised a SQL error on the context + if (rc != SQLITE_OK && !receive_err) { + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + return; + } + + // Apply errors → structured JSON with receive.error + if (receive_err) { + nrows = 0; + if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } + } + + const char *tables = sr.tables_json ? sr.tables_json : "[]"; + char *escaped = receive_err ? json_escape_string(receive_err) : NULL; + char *buf; + if (escaped && sr.check_failure_json) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"error\":\"%s\",\"lastFailure\":%s}}", + nrows, tables, escaped, sr.check_failure_json); + } else if (escaped) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"error\":\"%s\"}}", + nrows, tables, escaped); + } else if (sr.check_failure_json) { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"lastFailure\":%s}}", + nrows, tables, sr.check_failure_json); + } else { + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s}}", nrows, tables); + } + sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + if (escaped) cloudsync_memory_free(escaped); + if (receive_err) cloudsync_memory_free(receive_err); + if (sr.tables_json) cloudsync_memory_free(sr.tables_json); + if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); +} + +void cloudsync_network_reset_sync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_reset_sync_version"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + char *buf = "0"; + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_DBVERSION, buf); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_SEND_SEQ, buf); +} + +/** + * Cleanup all local data from cloudsync-enabled tables, so the database can be safely reused + * by another user without exposing any data from the previous session. + * + * Warning: this function deletes all data from the tables. Use with caution. + */ +void cloudsync_network_logout (sqlite3_context *context, int argc, sqlite3_value **argv) { + bool savepoint_created = false; + bool completed = false; + char *errmsg = NULL; + int rc = SQLITE_ERROR; + sqlite3 *db = sqlite3_context_db_handle(context); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // if the network layer is enabled, remove the token or apikey + sqlite3_exec(db, "SELECT cloudsync_network_set_token('');", NULL, NULL, NULL); + + // get the list of cloudsync-enabled tables + char *sql = "SELECT tbl_name, key, value FROM cloudsync_table_settings;"; + char **result = NULL; + int nrows, ncols; + rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, NULL); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to get current cloudsync configuration %s", sqlite3_errmsg(db)); + goto finalize; + } + + // run everything in a savepoint + rc = database_begin_savepoint(data, "cloudsync_logout_savepoint"); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to create cloudsync_logout savepoint %s", cloudsync_errmsg(data)); + goto finalize; + } + savepoint_created = true; + + rc = cloudsync_cleanup_all(data); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to cleanup current database %s", cloudsync_errmsg(data)); + goto finalize; + } + + // delete all the local data for each previously enabled table + // re-enable cloudsync on previously enabled tables + for (int i = 1; i <= nrows; i++) { + char *tbl_name = result[i * ncols + 0]; + char *key = result[i * ncols + 1]; + char *value = result[i * ncols + 2]; + + if (strcmp(key, "algo") != 0) continue; + + sql = cloudsync_memory_mprintf("DELETE FROM \"%w\";", tbl_name); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + cloudsync_memory_free(sql); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to delete data from table %s. %s", tbl_name, sqlite3_errmsg(db)); + goto finalize; + } + + sql = cloudsync_memory_mprintf("SELECT cloudsync_init('%q', '%q', 1);", tbl_name, value); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + cloudsync_memory_free(sql); + if (rc != SQLITE_OK) { + errmsg = cloudsync_memory_mprintf("Unable to enable cloudsync on table %s. %s", tbl_name, sqlite3_errmsg(db)); + goto finalize; + } + } + + completed = true; + +finalize: + if (completed) { + database_commit_savepoint(data, "cloudsync_logout_savepoint"); + cloudsync_network_cleanup_internal(context); + sqlite3_result_int(context, SQLITE_OK); + } else { + // cleanup: + // ROLLBACK TO command reverts the state of the database back to what it was just after the corresponding SAVEPOINT + // then RELEASE to remove the SAVEPOINT from the transaction stack + if (savepoint_created) database_rollback_savepoint(data, "cloudsync_logout_savepoint"); + sqlite3_result_error(context, errmsg, -1); + sqlite3_result_error_code(context, rc); + } + sqlite3_free_table(result); + cloudsync_memory_free(errmsg); +} + +void cloudsync_network_status (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_network_status"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + network_data *netdata = (network_data *)cloudsync_auxdata(data); + if (!netdata) { + sqlite3_result_error(context, "Unable to retrieve CloudSync network context.", -1); + return; + } + + NETWORK_RESULT res = network_receive_buffer(netdata, netdata->status_endpoint, netdata->authentication, true, false, NULL, cloudsync_default_headers, ARRAY_LEN(cloudsync_default_headers)); + network_set_sqlite_result(context, &res); + network_result_cleanup(&res); +} + +// MARK: - + +int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx) { + const int DEFAULT_FLAGS = SQLITE_UTF8 | SQLITE_INNOCUOUS; + int rc = SQLITE_OK; + + rc = sqlite3_create_function(db, "cloudsync_network_init", 1, DEFAULT_FLAGS, ctx, cloudsync_network_init, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_create_function(db, "cloudsync_network_init_custom", 2, DEFAULT_FLAGS, ctx, cloudsync_network_init_custom, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_cleanup", 0, DEFAULT_FLAGS, ctx, cloudsync_network_cleanup, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_set_token", 1, DEFAULT_FLAGS, ctx, cloudsync_network_set_token, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_set_apikey", 1, DEFAULT_FLAGS, ctx, cloudsync_network_set_apikey, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_has_unsent_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_has_unsent_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_send_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_send_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_check_changes", 0, DEFAULT_FLAGS, ctx, cloudsync_network_check_changes, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_sync", 0, DEFAULT_FLAGS, ctx, cloudsync_network_sync0, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_sync", 2, DEFAULT_FLAGS, ctx, cloudsync_network_sync2, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_reset_sync_version", 0, DEFAULT_FLAGS, ctx, cloudsync_network_reset_sync_version, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_logout", 0, DEFAULT_FLAGS, ctx, cloudsync_network_logout, NULL, NULL); + if (rc != SQLITE_OK) return rc; + + rc = sqlite3_create_function(db, "cloudsync_network_status", 0, DEFAULT_FLAGS, ctx, cloudsync_network_status, NULL, NULL); + if (rc != SQLITE_OK) return rc; + +cleanup: + if ((rc != SQLITE_OK) && (pzErrMsg)) { + *pzErrMsg = sqlite3_mprintf("Error creating function in cloudsync_network_register: %s", sqlite3_errmsg(db)); + } + + return rc; +} + +#endif diff --git a/src/network.h b/src/network/network.h similarity index 68% rename from src/network.h rename to src/network/network.h index 73b7c79..0c7e7de 100644 --- a/src/network.h +++ b/src/network/network.h @@ -8,7 +8,13 @@ #ifndef __CLOUDSYNC_NETWORK__ #define __CLOUDSYNC_NETWORK__ -#include "cloudsync.h" +#include "../cloudsync.h" + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif int cloudsync_network_register (sqlite3 *db, char **pzErrMsg, void *ctx); diff --git a/src/network.m b/src/network/network.m similarity index 50% rename from src/network.m rename to src/network/network.m index 6f4d2c1..89c00f9 100644 --- a/src/network.m +++ b/src/network/network.m @@ -13,63 +13,18 @@ void network_buffer_cleanup (void *xdata) { if (xdata) CFRelease(xdata); } -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string) { - NSString *conn = [NSString stringWithUTF8String:conn_string]; - NSString *conn_string_https = nil; - - if ([conn hasPrefix:@"sqlitecloud://"]) { - conn_string_https = [conn stringByReplacingCharactersInRange:NSMakeRange(0, [@"sqlitecloud://" length]) withString:@"https://"]; - } else { - conn_string_https = conn; - } - - NSURL *url = [NSURL URLWithString:conn_string_https]; - if (!url) return false; - - NSString *scheme = url.scheme; // "https" - if (!scheme) return false; - NSString *host = url.host; // "cn5xiooanz.global3.ryujaz.sqlite.cloud" - if (!host) return false; - - NSString *port = url.port.stringValue; - NSString *database = url.path; // "/chinook-cloudsync.sqlite" - if (!database) return false; - - NSString *query = url.query; // "apikey=hWDanFolRT9WDK0p54lufNrIyfgLZgtMw6tb6fbPmpo" (OPTIONAL) - NSString *authentication = nil; - - if (query) { - NSURLComponents *components = [NSURLComponents componentsWithString:[@"http://dummy?" stringByAppendingString:query]]; - NSArray *items = components.queryItems; - for (NSURLQueryItem *item in items) { - // build new token - // apikey: just write the key for retrocompatibility - // other keys, like token: add a prefix, i.e. token= - - if ([item.name isEqualToString:@"apikey"]) { - authentication = item.value; - break; - } - if ([item.name isEqualToString:@"token"]) { - authentication = [NSString stringWithFormat:@"%@=%@", item.name, item.value]; - break; - } - } - } - - char *site_id = network_data_get_siteid(data); - char *port_or_default = (port && strcmp(port.UTF8String, "8860") != 0) ? (char *)port.UTF8String : CLOUDSYNC_DEFAULT_ENDPOINT_PORT; - - NSString *check_endpoint = [NSString stringWithFormat:@"%s://%s:%s/%s%s/%s", scheme.UTF8String, host.UTF8String, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database.UTF8String, site_id]; - NSString *upload_endpoint = [NSString stringWithFormat: @"%s://%s:%s/%s%s/%s/%s", scheme.UTF8String, host.UTF8String, port_or_default, CLOUDSYNC_ENDPOINT_PREFIX, database.UTF8String, site_id, CLOUDSYNC_ENDPOINT_UPLOAD]; - - return network_data_set_endpoints(data, (char *)authentication.UTF8String, (char *)check_endpoint.UTF8String, (char *)upload_endpoint.UTF8String, true); -} - bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size) { +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); +#endif NSString *urlString = [NSString stringWithUTF8String:endpoint]; NSURL *url = [NSURL URLWithString:urlString]; - if (!url) return false; + if (!url) { + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, "PUT", endpoint, 0, CLOUDSYNC_NETWORK_ERROR, (size_t)blob_size, 0, network_trace_now_ms() - trace_start_ms); + #endif + return false; + } NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; [request setHTTPMethod:@"PUT"]; @@ -81,21 +36,27 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a [request setValue:authString forHTTPHeaderField:@"Authorization"]; } + char *org_id = network_data_get_orgid(data); + if (org_id) { + [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + } + NSData *bodyData = [NSData dataWithBytes:blob length:blob_size]; [request setHTTPBody:bodyData]; __block bool success = false; + __block NSInteger statusCode = 0; dispatch_semaphore_t sema = dispatch_semaphore_create(0); NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; NSURLSessionDataTask *task = [session dataTaskWithRequest:request - completionHandler:^(NSData * _Nullable data, + completionHandler:^(NSData * _Nullable responseBody, NSURLResponse * _Nullable response, NSError * _Nullable error) { if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { - NSInteger statusCode = [(NSHTTPURLResponse *)response statusCode]; + statusCode = [(NSHTTPURLResponse *)response statusCode]; success = (statusCode >= 200 && statusCode < 300); } dispatch_semaphore_signal(sema); @@ -103,12 +64,30 @@ bool network_send_buffer(network_data *data, const char *endpoint, const char *a [task resume]; dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; + + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, "PUT", endpoint, (long)statusCode, + success ? CLOUDSYNC_NETWORK_OK : CLOUDSYNC_NETWORK_ERROR, + (size_t)blob_size, + success ? (size_t)blob_size : 0, + network_trace_now_ms() - trace_start_ms); + #endif return success; } -NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header) { +NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers) { +#ifdef CLOUDSYNC_NETWORK_TRACE + double trace_start_ms = network_trace_now_ms(); + size_t request_bytes = json_payload ? strlen(json_payload) : 0; +#endif + const char *method = (json_payload || is_post_request) ? "POST" : "GET"; + bool using_ticket = network_data_should_use_ticket(data, endpoint, authentication); +#ifndef CLOUDSYNC_NETWORK_TRACE + (void)method; +#endif NSString *urlString = [NSString stringWithUTF8String:endpoint]; NSURL *url = [NSURL URLWithString:urlString]; @@ -119,24 +98,38 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.buffer = (char *)msg.UTF8String; result.xdata = (void *)CFBridgingRetain(msg); result.xfree = network_buffer_cleanup; + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, method, endpoint, 0, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif return result; } NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url]; request.HTTPMethod = (json_payload || is_post_request) ? @"POST" : @"GET"; - if (custom_header) { - NSString *header = [NSString stringWithUTF8String:custom_header]; - NSArray *parts = [header componentsSeparatedByString:@": "]; - if (parts.count == 2) { - [request setValue:parts[1] forHTTPHeaderField:parts[0]]; + for (int i = 0; i < nextra_headers; i++) { + NSString *header = [NSString stringWithUTF8String:extra_headers[i]]; + NSRange sep = [header rangeOfString:@": "]; + if (sep.location != NSNotFound) { + NSString *name = [header substringToIndex:sep.location]; + NSString *value = [header substringFromIndex:sep.location + sep.length]; + [request setValue:value forHTTPHeaderField:name]; } } + char *org_id = network_data_get_orgid(data); + if (org_id) { + [request setValue:[NSString stringWithUTF8String:org_id] forHTTPHeaderField:@CLOUDSYNC_HEADER_ORG]; + } + if (authentication) { NSString *authString = [NSString stringWithFormat:@"Bearer %s", authentication]; [request setValue:authString forHTTPHeaderField:@"Authorization"]; } + if (using_ticket) { + char *ticket = network_data_get_ticket(data); + [request setValue:[NSString stringWithUTF8String:ticket] forHTTPHeaderField:@CLOUDSYNC_HEADER_TICKET]; + } if (json_payload) { [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"]; @@ -150,29 +143,57 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, __block NSString *responseError = nil; __block NSInteger statusCode = 0; __block NSInteger errorCode = 0; + __block NSString *responseTicket = nil; + __block NSString *responseTicketExpiresAt = nil; dispatch_semaphore_t sema = dispatch_semaphore_create(0); - NSURLSession *session = [NSURLSession sharedSession]; - NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) { - responseData = data; + NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; + NSURLSession *session = [NSURLSession sessionWithConfiguration:config]; + NSURLSessionDataTask *task = [session dataTaskWithRequest:request completionHandler:^(NSData *responseBody, NSURLResponse *response, NSError *error) { + responseData = responseBody; if (error) { responseError = [error localizedDescription]; errorCode = [error code]; } if ([response isKindOfClass:[NSHTTPURLResponse class]]) { - statusCode = [(NSHTTPURLResponse *)response statusCode]; + NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response; + statusCode = [httpResponse statusCode]; + NSDictionary *headers = [httpResponse allHeaderFields]; + for (id key in headers) { + NSString *name = [key description]; + NSString *value = [[headers objectForKey:key] description]; + if ([name caseInsensitiveCompare:@CLOUDSYNC_HEADER_TICKET] == NSOrderedSame) { + responseTicket = value; + } else if ([name caseInsensitiveCompare:@CLOUDSYNC_HEADER_TICKET_EXPIRES_AT] == NSOrderedSame) { + responseTicketExpiresAt = value; + } + } } dispatch_semaphore_signal(sema); }]; [task resume]; dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); + [session finishTasksAndInvalidate]; + + if (!responseError && (statusCode >= 200 && statusCode < 300) && responseTicket && [responseTicket length] > 0) { + network_data_update_ticket(data, [responseTicket UTF8String], + responseTicketExpiresAt ? [responseTicketExpiresAt UTF8String] : NULL); + } if (!responseError && (statusCode >= 200 && statusCode < 300)) { // check if OK should be returned if (responseData == nil || [responseData length] == 0) { - return (NETWORK_RESULT){CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; + NETWORK_RESULT result = {CLOUDSYNC_NETWORK_OK, NULL, 0, NULL, NULL}; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif + return result; } // otherwise return a buffer @@ -180,6 +201,14 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.code = CLOUDSYNC_NETWORK_BUFFER; if (zero_terminated) { NSString *utf8String = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding]; + if (!utf8String) { + NSString *msg = @"Response is not valid UTF-8"; + NETWORK_RESULT error_result = {CLOUDSYNC_NETWORK_ERROR, (char *)msg.UTF8String, 0, (void *)CFBridgingRetain(msg), network_buffer_cleanup}; + #ifdef CLOUDSYNC_NETWORK_TRACE + network_trace_log(data, method, endpoint, (long)statusCode, error_result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif + return error_result; + } result.buffer = (char *)utf8String.UTF8String; result.xdata = (void *)CFBridgingRetain(utf8String); } else { @@ -189,6 +218,13 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.blen = [responseData length]; result.xfree = network_buffer_cleanup; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, result.blen, network_trace_now_ms() - trace_start_ms); + #endif return result; } @@ -212,5 +248,12 @@ NETWORK_RESULT network_receive_buffer(network_data *data, const char *endpoint, result.xfree = network_buffer_cleanup; result.blen = responseError ? (size_t)errorCode : (size_t)statusCode; + #ifdef CLOUDSYNC_NETWORK_TRACE + fprintf(stderr, + "[cloudsync-network] endpoint=%s using_ticket=%s\n", + network_trace_endpoint_name(data, endpoint), + using_ticket ? "true" : "false"); + network_trace_log(data, method, endpoint, (long)statusCode, result.code, request_bytes, 0, network_trace_now_ms() - trace_start_ms); + #endif return result; } diff --git a/src/network/network_private.h b/src/network/network_private.h new file mode 100644 index 0000000..dae4774 --- /dev/null +++ b/src/network/network_private.h @@ -0,0 +1,57 @@ +// +// network_private.h +// cloudsync +// +// Created by Marco Bambini on 23/05/25. +// + +#ifndef __CLOUDSYNC_NETWORK_PRIVATE__ +#define __CLOUDSYNC_NETWORK_PRIVATE__ + +#define CLOUDSYNC_DEFAULT_ADDRESS "https://cloudsync.sqlite.ai" +#define CLOUDSYNC_ENDPOINT_PREFIX "v2/cloudsync/databases" +#define CLOUDSYNC_ENDPOINT_UPLOAD "upload" +#define CLOUDSYNC_ENDPOINT_CHECK "check" +#define CLOUDSYNC_ENDPOINT_APPLY "apply" +#define CLOUDSYNC_ENDPOINT_STATUS "status" +#define CLOUDSYNC_HEADER_ORG "X-CloudSync-Org" +#define CLOUDSYNC_HEADER_VERSION "X-CloudSync-Version" +#define CLOUDSYNC_HEADER_TICKET "X-CloudSync-Ticket" +#define CLOUDSYNC_HEADER_TICKET_EXPIRES_AT "X-CloudSync-Ticket-Expires-At" +// CLOUDSYNC_VERSION is defined in cloudsync.h — include it before this header at use sites. +#define CLOUDSYNC_HEADER_VERSION_LINE CLOUDSYNC_HEADER_VERSION ": " CLOUDSYNC_VERSION +#define CLOUDSYNC_HEADER_CHECK_CAPABILITIES "X-CloudSync-Capabilities: check-status-response" + +#define CLOUDSYNC_NETWORK_OK 1 +#define CLOUDSYNC_NETWORK_ERROR 2 +#define CLOUDSYNC_NETWORK_BUFFER 3 + +typedef struct network_data network_data; + +typedef struct { + int code; // network code: OK, ERROR, BUFFER + char *buffer; // network buffer + size_t blen; // blen if code is SQLITE_OK, rc in case of error + void *xdata; // optional custom external data + void (*xfree) (void *); // optional custom free callback +} NETWORK_RESULT; + +char *network_data_get_siteid (network_data *data); +char *network_data_get_orgid (network_data *data); +char *network_data_get_ticket (network_data *data); +bool network_data_should_use_ticket (network_data *data, const char *endpoint, const char *authentication); +void network_data_update_ticket (network_data *data, const char *ticket, const char *expires_at); +bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, char *apply, char *status); + +bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); +NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers); + +#ifdef CLOUDSYNC_NETWORK_TRACE +const char *network_trace_endpoint_name(network_data *data, const char *endpoint); +const char *network_trace_result_name(int code); +void network_trace_log(network_data *data, const char *method, const char *endpoint, long http_status, int result_code, size_t request_bytes, size_t bytes, double elapsed_ms); +double network_trace_now_ms(void); +#endif + + +#endif diff --git a/src/network_private.h b/src/network_private.h deleted file mode 100644 index cd970c5..0000000 --- a/src/network_private.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// network_private.h -// cloudsync -// -// Created by Marco Bambini on 23/05/25. -// - -#ifndef __CLOUDSYNC_NETWORK_PRIVATE__ -#define __CLOUDSYNC_NETWORK_PRIVATE__ - -#define CLOUDSYNC_ENDPOINT_PREFIX "v1/cloudsync" -#define CLOUDSYNC_ENDPOINT_UPLOAD "upload" -#define CLOUDSYNC_ENDPOINT_CHECK "check" -#define CLOUDSYNC_DEFAULT_ENDPOINT_PORT "443" -#define CLOUDSYNC_HEADER_SQLITECLOUD "Accept: sqlc/plain" - -#define CLOUDSYNC_NETWORK_OK 1 -#define CLOUDSYNC_NETWORK_ERROR 2 -#define CLOUDSYNC_NETWORK_BUFFER 3 - -typedef struct network_data network_data; - -typedef struct { - int code; // network code: OK, ERROR, BUFFER - char *buffer; // network buffer - size_t blen; // blen if code is SQLITE_OK, rc in case of error - void *xdata; // optional custom external data - void (*xfree) (void *); // optional custom free callback -} NETWORK_RESULT; - -char *network_data_get_siteid (network_data *data); -bool network_data_set_endpoints (network_data *data, char *auth, char *check, char *upload, bool duplicate); - -bool network_compute_endpoints (sqlite3_context *context, network_data *data, const char *conn_string); -bool network_send_buffer(network_data *data, const char *endpoint, const char *authentication, const void *blob, int blob_size); -NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char *custom_header); - - -#endif diff --git a/src/pk.c b/src/pk.c index ae605d1..97a6639 100644 --- a/src/pk.c +++ b/src/pk.c @@ -7,11 +7,14 @@ #include "pk.h" #include "utils.h" +#include "cloudsync_endian.h" +#include "cloudsync.h" -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - +#include +#include +#include +#include + /* The pk_encode and pk_decode functions are designed to serialize and deserialize an array of values (sqlite_value structures) @@ -67,45 +70,50 @@ SQLITE_EXTENSION_INIT3 * Versatility: The ability to handle multiple data types and variable-length data makes this solution suitable for complex data structures. * Simplicity: The functions are designed to be straightforward to use, with clear memory management responsibilities. + Notes + + * Floating point values are encoded as IEEE754 double, 64-bit, big-endian byte order. + */ // Three bits are reserved for the type field, so only values in the 0..7 range can be used (8 values) // SQLITE already reserved values from 1 to 5 -// #define SQLITE_INTEGER 1 -// #define SQLITE_FLOAT 2 -// #define SQLITE_TEXT 3 -// #define SQLITE_BLOB 4 -// #define SQLITE_NULL 5 -#define SQLITE_NEGATIVE_INTEGER 0 -#define SQLITE_MAX_NEGATIVE_INTEGER 6 -#define SQLITE_NEGATIVE_FLOAT 7 +// #define SQLITE_INTEGER 1 // now DBTYPE_INTEGER +// #define SQLITE_FLOAT 2 // now DBTYPE_FLOAT +// #define SQLITE_TEXT 3 // now DBTYPE_TEXT +// #define SQLITE_BLOB 4 // now DBTYPE_BLOB +// #define SQLITE_NULL 5 // now DBTYPE_NULL +#define DATABASE_TYPE_NEGATIVE_INTEGER 0 // was SQLITE_NEGATIVE_INTEGER +#define DATABASE_TYPE_MAX_NEGATIVE_INTEGER 6 // was SQLITE_MAX_NEGATIVE_INTEGER +#define DATABASE_TYPE_NEGATIVE_FLOAT 7 // was SQLITE_NEGATIVE_FLOAT -// MARK: - Decoding - +char * const PRIKEY_NULL_CONSTRAINT_ERROR = "PRIKEY_NULL_CONSTRAINT_ERROR"; + +// MARK: - Public Callbacks - int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { - // default decode callback used to bind values to a sqlite3_stmt vm + // default decode callback used to bind values to a dbvm_t vm - sqlite3_stmt *vm = (sqlite3_stmt *)xdata; - int rc = SQLITE_OK; + int rc = DBRES_OK; switch (type) { - case SQLITE_INTEGER: - rc = sqlite3_bind_int64(vm, index+1, ival); + case DBTYPE_INTEGER: + rc = databasevm_bind_int(xdata, index+1, ival); break; - case SQLITE_FLOAT: - rc = sqlite3_bind_double(vm, index+1, dval); + case DBTYPE_FLOAT: + rc = databasevm_bind_double(xdata, index+1, dval); break; - case SQLITE_NULL: - rc = sqlite3_bind_null(vm, index+1); + case DBTYPE_NULL: + rc = databasevm_bind_null(xdata, index+1); break; - case SQLITE_TEXT: - rc = sqlite3_bind_text(vm, index+1, pval, (int)ival, SQLITE_STATIC); + case DBTYPE_TEXT: + rc = databasevm_bind_text(xdata, index+1, pval, (int)ival); break; - case SQLITE_BLOB: - rc = sqlite3_bind_blob64(vm, index+1, (const void *)pval, (sqlite3_uint64)ival, SQLITE_STATIC); + case DBTYPE_BLOB: + rc = databasevm_bind_blob(xdata, index+1, (const void *)pval, ival); break; } @@ -114,108 +122,198 @@ int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, dou int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { switch (type) { - case SQLITE_INTEGER: - printf("%d\tINTEGER:\t%lld\n", index, (long long)ival); + case DBTYPE_INTEGER: + printf("%d\tINTEGER:\t%" PRId64 "\n", index, ival); break; - case SQLITE_FLOAT: + case DBTYPE_FLOAT: printf("%d\tFLOAT:\t%.5f\n", index, dval); break; - case SQLITE_NULL: + case DBTYPE_NULL: printf("%d\tNULL\n", index); break; - case SQLITE_TEXT: - printf("%d\tTEXT:\t%s\n", index, pval); + case DBTYPE_TEXT: + printf("%d\tTEXT:\t%.*s\n", index, (int)ival, pval); break; - case SQLITE_BLOB: - printf("%d\tBLOB:\t%lld bytes\n", index, (long long)ival); + case DBTYPE_BLOB: + printf("%d\tBLOB:\t%" PRId64 " bytes\n", index, ival); break; } - return SQLITE_OK; + return DBRES_OK; } -uint8_t pk_decode_u8 (char *buffer, size_t *bseek) { - uint8_t value = buffer[*bseek]; +uint64_t pk_checksum (const char *buffer, size_t blen) { + const uint8_t *p = (const uint8_t *)buffer; + uint64_t h = 14695981039346656037ULL; + for (size_t i = 0; i < blen; i++) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; +} + +// MARK: - Decoding - + +static inline int pk_decode_check_bounds (size_t bseek, size_t blen, size_t need) { + // bounds check helper for decoding + if (bseek > blen) return 0; + return need <= (blen - bseek); +} + +int pk_decode_u8 (const uint8_t *buffer, size_t blen, size_t *bseek, uint8_t *out) { + if (!pk_decode_check_bounds(*bseek, blen, 1)) return 0; + *out = buffer[*bseek]; *bseek += 1; - return value; + return 1; } -int64_t pk_decode_int64 (char *buffer, size_t *bseek, size_t nbytes) { - int64_t value = 0; +static int pk_decode_uint64 (const uint8_t *buffer, size_t blen, size_t *bseek, size_t nbytes, uint64_t *out) { + if (nbytes > 8) return 0; + if (!pk_decode_check_bounds(*bseek, blen, nbytes)) return 0; // decode bytes in big-endian order (most significant byte first) + uint64_t v = 0; for (size_t i = 0; i < nbytes; i++) { - value = (value << 8) | (uint8_t)buffer[*bseek]; + v = (v << 8) | (uint64_t)buffer[*bseek]; (*bseek)++; } - return value; + *out = v; + return 1; } -char *pk_decode_data (char *buffer, size_t *bseek, int32_t blen) { - char *value = buffer + *bseek; - *bseek += blen; +static int pk_decode_data (const uint8_t *buffer, size_t blen, size_t *bseek, size_t n, const uint8_t **out) { + if (!pk_decode_check_bounds(*bseek, blen, n)) return 0; + *out = buffer + *bseek; + *bseek += n; - return value; + return 1; } -double pk_decode_double (char *buffer, size_t *bseek) { - double value = 0; - int64_t int64value = pk_decode_int64(buffer, bseek, sizeof(int64_t)); - memcpy(&value, &int64value, sizeof(int64_t)); +int pk_decode_double (const uint8_t *buffer, size_t blen, size_t *bseek, double *out) { + // Doubles are encoded as IEEE754 64-bit, big-endian. + // Convert back to host order before memcpy into double. + + uint64_t bits_be = 0; + if (!pk_decode_uint64(buffer, blen, bseek, sizeof(uint64_t), &bits_be)) return 0; - return value; + uint64_t bits = be64_to_host(bits_be); + double value = 0.0; + memcpy(&value, &bits, sizeof(bits)); + *out = value; + return 1; } -int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata) { +int pk_decode (char *buffer, size_t blen, int count, size_t *seek, int skip_decode_idx, pk_decode_callback cb, void *xdata) { + const uint8_t *ubuf = (const uint8_t *)buffer; size_t bseek = (seek) ? *seek : 0; - if (count == -1) count = pk_decode_u8(buffer, &bseek); - - for (size_t i = 0; i < (size_t)count; i++) { - uint8_t type_byte = (uint8_t)pk_decode_u8(buffer, &bseek); - int type = (int)(type_byte & 0x07); - size_t nbytes = (type_byte >> 3) & 0x1F; + if (count == -1) { + uint8_t c = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &c)) return -1; + count = (int)c; + } - switch (type) { - case SQLITE_MAX_NEGATIVE_INTEGER: { - int64_t value = INT64_MIN; - type = SQLITE_INTEGER; - if (cb) if (cb(xdata, (int)i, type, value, 0.0, NULL) != SQLITE_OK) return -1; + for (size_t i = 0; i < (size_t)count; i++) { + uint8_t type_byte = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &type_byte)) return -1; + int raw_type = (int)(type_byte & 0x07); + size_t nbytes = (size_t)((type_byte >> 3) & 0x1F); + + // skip_decode wants the raw encoded slice (type_byte + optional len/int + payload) + // we still must parse with the *raw* type to know how much to skip + bool skip_decode = ((skip_decode_idx >= 0) && (i == (size_t)skip_decode_idx)); + size_t initial_bseek = bseek - 1; // points to type_byte + + switch (raw_type) { + case DATABASE_TYPE_MAX_NEGATIVE_INTEGER: { + // must not carry length bits + if (nbytes != 0) return -1; + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + int64_t value = INT64_MIN; + if (cb) if (cb(xdata, (int)i, DBTYPE_INTEGER, value, 0.0, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_NEGATIVE_INTEGER: - case SQLITE_INTEGER: { - int64_t value = pk_decode_int64(buffer, &bseek, nbytes); - if (type == SQLITE_NEGATIVE_INTEGER) {value = -value; type = SQLITE_INTEGER;} - if (cb) if (cb(xdata, (int)i, type, value, 0.0, NULL) != SQLITE_OK) return -1; + case DATABASE_TYPE_NEGATIVE_INTEGER: + case DBTYPE_INTEGER: { + // validate nbytes to avoid UB/overreads + if (nbytes < 1 || nbytes > 8) return -1; + uint64_t u = 0; + if (!pk_decode_uint64(ubuf, blen, &bseek, nbytes, &u)) return -1; + + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + int64_t value = (int64_t)u; + if (raw_type == DATABASE_TYPE_NEGATIVE_INTEGER) value = -value; + if (cb) if (cb(xdata, (int)i, DBTYPE_INTEGER, value, 0.0, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_NEGATIVE_FLOAT: - case SQLITE_FLOAT: { - double value = pk_decode_double(buffer, &bseek); - if (type == SQLITE_NEGATIVE_FLOAT) {value = -value; type = SQLITE_FLOAT;} - if (cb) if (cb(xdata, (int)i, type, 0, value, NULL) != SQLITE_OK) return -1; + case DATABASE_TYPE_NEGATIVE_FLOAT: + case DBTYPE_FLOAT: { + // encoder stores float type with no length bits, so enforce nbytes==0 + if (nbytes != 0) return -1; + double value = 0.0; + if (!pk_decode_double(ubuf, blen, &bseek, &value)) return -1; + + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (raw_type == DATABASE_TYPE_NEGATIVE_FLOAT) value = -value; + if (cb) if (cb(xdata, (int)i, DBTYPE_FLOAT, 0, value, NULL) != DBRES_OK) return -1; + } } break; - case SQLITE_TEXT: - case SQLITE_BLOB: { - int64_t length = pk_decode_int64(buffer, &bseek, nbytes); - char *value = pk_decode_data(buffer, &bseek, (int32_t)length); - if (cb) if (cb(xdata, (int)i, type, length, 0.0, value) != SQLITE_OK) return -1; + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + // validate nbytes for length field + if (nbytes < 1 || nbytes > 8) return -1; + uint64_t ulen = 0; + if (!pk_decode_uint64(ubuf, blen, &bseek, nbytes, &ulen)) return -1; + + // ensure ulen fits in size_t on this platform + if (ulen > (uint64_t)SIZE_MAX) return -1; + size_t len = (size_t)ulen; + const uint8_t *p = NULL; + if (!pk_decode_data(ubuf, blen, &bseek, len, &p)) return -1; + + if (skip_decode) { + // return the full encoded slice (type_byte + len bytes + payload) + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (cb) if (cb(xdata, (int)i, raw_type, (int64_t)len, 0.0, (char *)p) != DBRES_OK) return -1; + } } break; - case SQLITE_NULL: { - if (cb) if (cb(xdata, (int)i, type, 0, 0.0, NULL) != SQLITE_OK) return -1; + case DBTYPE_NULL: { + if (nbytes != 0) return -1; + if (skip_decode) { + size_t slice_len = bseek - initial_bseek; + if (cb) if (cb(xdata, (int)i, DBTYPE_BLOB, (int64_t)slice_len, 0.0, (char *)(buffer + initial_bseek)) != DBRES_OK) return -1; + } else { + if (cb) if (cb(xdata, (int)i, DBTYPE_NULL, 0, 0.0, NULL) != DBRES_OK) return -1; + } } break; + + default: + // should never reach this point + return -1; } } @@ -223,55 +321,86 @@ int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (voi return count; } -int pk_decode_prikey (char *buffer, size_t blen, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata) { +int pk_decode_prikey (char *buffer, size_t blen, pk_decode_callback cb, void *xdata) { + const uint8_t *ubuf = (const uint8_t *)buffer; size_t bseek = 0; - uint8_t count = pk_decode_u8(buffer, &bseek); - return pk_decode(buffer, blen, count, &bseek, cb, xdata); + uint8_t count = 0; + if (!pk_decode_u8(ubuf, blen, &bseek, &count)) return -1; + return pk_decode(buffer, blen, count, &bseek, -1, cb, xdata); } // MARK: - Encoding - size_t pk_encode_nbytes_needed (int64_t value) { - if (value <= 0x7F) return 1; // 7 bits - if (value <= 0x7FFF) return 2; // 15 bits - if (value <= 0x7FFFFF) return 3; // 23 bits - if (value <= 0x7FFFFFFF) return 4; // 31 bits - if (value <= 0x7FFFFFFFFF) return 5; // 39 bits - if (value <= 0x7FFFFFFFFFFF) return 6; // 47 bits - if (value <= 0x7FFFFFFFFFFFFF) return 7; // 55 bits - return 8; // Larger than 7-byte range, needs 8 bytes + uint64_t v = (uint64_t)value; + if (v <= 0xFFULL) return 1; + if (v <= 0xFFFFULL) return 2; + if (v <= 0xFFFFFFULL) return 3; + if (v <= 0xFFFFFFFFULL) return 4; + if (v <= 0xFFFFFFFFFFULL) return 5; + if (v <= 0xFFFFFFFFFFFFULL) return 6; + if (v <= 0xFFFFFFFFFFFFFFULL) return 7; + return 8; +} + +static inline int pk_encode_add_overflow_size (size_t a, size_t b, size_t *out) { + // safe size_t addition helper (prevents overflow) + if (b > (SIZE_MAX - a)) return 1; + *out = a + b; + return 0; } -size_t pk_encode_size (sqlite3_value **argv, int argc, int reserved) { +size_t pk_encode_size (dbvalue_t **argv, int argc, int reserved, int skip_idx) { // estimate the required buffer size size_t required = reserved; size_t nbytes; - int64_t val, len; + int64_t val; for (int i = 0; i < argc; i++) { - switch (sqlite3_value_type(argv[i])) { - case SQLITE_INTEGER: - val = sqlite3_value_int64(argv[i]); + switch (database_value_type(argv[i])) { + case DBTYPE_INTEGER: { + val = database_value_int(argv[i]); if (val == INT64_MIN) { - required += 1; + if (pk_encode_add_overflow_size(required, 1, &required)) return SIZE_MAX; break; } if (val < 0) val = -val; nbytes = pk_encode_nbytes_needed(val); - required += 1 + nbytes; - break; - case SQLITE_FLOAT: - required += 1 + sizeof(int64_t); - break; - case SQLITE_TEXT: - case SQLITE_BLOB: - len = (int32_t)sqlite3_value_bytes(argv[i]); - nbytes = pk_encode_nbytes_needed(len); - required += 1 + len + nbytes; - break; - case SQLITE_NULL: - required += 1; - break; + + size_t tmp = 0; + if (pk_encode_add_overflow_size(1, nbytes, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_FLOAT: { + size_t tmp = 0; + if (pk_encode_add_overflow_size(1, sizeof(uint64_t), &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + size_t len_sz = (size_t)database_value_bytes(argv[i]); + if (i == skip_idx) { + if (pk_encode_add_overflow_size(required, len_sz, &required)) return SIZE_MAX; + break; + } + + // Ensure length can be represented by encoder (we encode length with up to 8 bytes) + // pk_encode_nbytes_needed expects int64-ish values; clamp-check here. + if (len_sz > (size_t)INT64_MAX) return SIZE_MAX; + nbytes = pk_encode_nbytes_needed((int64_t)len_sz); + + size_t tmp = 0; + // 1(type) + nbytes(len) + len_sz(payload) + if (pk_encode_add_overflow_size(1, nbytes, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(tmp, len_sz, &tmp)) return SIZE_MAX; + if (pk_encode_add_overflow_size(required, tmp, &required)) return SIZE_MAX; + } break; + + case DBTYPE_NULL: { + if (pk_encode_add_overflow_size(required, 1, &required)) return SIZE_MAX; + } break; } } @@ -283,9 +412,9 @@ size_t pk_encode_u8 (char *buffer, size_t bseek, uint8_t value) { return bseek; } -size_t pk_encode_int64 (char *buffer, size_t bseek, int64_t value, size_t nbytes) { +static size_t pk_encode_uint64 (char *buffer, size_t bseek, uint64_t value, size_t nbytes) { for (size_t i = 0; i < nbytes; i++) { - buffer[bseek++] = (uint8_t)((value >> (8 * (nbytes - 1 - i))) & 0xFF); + buffer[bseek++] = (uint8_t)((value >> (8 * (nbytes - 1 - i))) & 0xFFu); } return bseek; } @@ -295,69 +424,111 @@ size_t pk_encode_data (char *buffer, size_t bseek, char *data, size_t datalen) { return bseek + datalen; } -char *pk_encode (sqlite3_value **argv, int argc, char *b, bool is_prikey, size_t *bsize) { +char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bsize, int skip_idx) { size_t bseek = 0; - size_t blen = 0; char *buffer = b; + // always compute blen (even if it is not a primary key) + size_t blen = pk_encode_size(argv, argc, (is_prikey) ? 1 : 0, skip_idx); + if (blen == SIZE_MAX) return NULL; + if (argc < 0) return NULL; + // in primary-key encoding the number of items must be explicitly added to the encoded buffer if (is_prikey) { - // 1 is the number of items in the serialization (always 1 byte so max 255 primary keys, even if there is an hard SQLite limit of 128) - blen = pk_encode_size(argv, argc, 1); + if (!bsize) return NULL; + // must fit in a single byte + if (argc > 255) return NULL; + + // if schema does not enforce NOT NULL on primary keys, check at runtime + #ifndef CLOUDSYNC_CHECK_NOTNULL_PRIKEYS + for (int i = 0; i < argc; i++) { + if (database_value_type(argv[i]) == DBTYPE_NULL) return PRIKEY_NULL_CONSTRAINT_ERROR; + } + #endif + + // 1 is the number of items in the serialization + // always 1 byte so max 255 primary keys, even if there is an hard SQLite limit of 128 size_t blen_curr = *bsize; - buffer = (blen > blen_curr || b == NULL) ? cloudsync_memory_alloc((sqlite3_uint64)blen) : b; + buffer = (blen > blen_curr || b == NULL) ? cloudsync_memory_alloc((uint64_t)blen) : b; if (!buffer) return NULL; // the first u8 value is the total number of items in the primary key(s) - bseek = pk_encode_u8(buffer, 0, argc); + bseek = pk_encode_u8(buffer, 0, (uint8_t)argc); + } else { + // ensure buffer exists and is large enough also in non-prikey mode + size_t curr = (bsize) ? *bsize : 0; + if (buffer == NULL || curr < blen) return NULL; } for (int i = 0; i < argc; i++) { - int type = sqlite3_value_type(argv[i]); + int type = database_value_type(argv[i]); switch (type) { - case SQLITE_INTEGER: { - int64_t value = sqlite3_value_int64(argv[i]); + case DBTYPE_INTEGER: { + int64_t value = database_value_int(argv[i]); if (value == INT64_MIN) { - bseek = pk_encode_u8(buffer, bseek, SQLITE_MAX_NEGATIVE_INTEGER); + bseek = pk_encode_u8(buffer, bseek, DATABASE_TYPE_MAX_NEGATIVE_INTEGER); break; } - if (value < 0) {value = -value; type = SQLITE_NEGATIVE_INTEGER;} + if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_INTEGER;} size_t nbytes = pk_encode_nbytes_needed(value); - uint8_t type_byte = (nbytes << 3) | type; + uint8_t type_byte = (uint8_t)((nbytes << 3) | type); bseek = pk_encode_u8(buffer, bseek, type_byte); - bseek = pk_encode_int64(buffer, bseek, value, nbytes); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)value, nbytes); } break; - case SQLITE_FLOAT: { - double value = sqlite3_value_double(argv[i]); - if (value < 0) {value = -value; type = SQLITE_NEGATIVE_FLOAT;} - int64_t net_double; - memcpy(&net_double, &value, sizeof(int64_t)); - bseek = pk_encode_u8(buffer, bseek, type); - bseek = pk_encode_int64(buffer, bseek, net_double, sizeof(int64_t)); + case DBTYPE_FLOAT: { + // Encode doubles as IEEE754 64-bit, big-endian + double value = database_value_double(argv[i]); + if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_FLOAT;} + uint64_t bits; + memcpy(&bits, &value, sizeof(bits)); + bits = host_to_be64(bits); + bseek = pk_encode_u8(buffer, bseek, (uint8_t)type); + bseek = pk_encode_uint64(buffer, bseek, bits, sizeof(bits)); } break; - case SQLITE_TEXT: - case SQLITE_BLOB: { - int32_t len = (int32_t)sqlite3_value_bytes(argv[i]); - size_t nbytes = pk_encode_nbytes_needed(len); - uint8_t type_byte = (nbytes << 3) | sqlite3_value_type(argv[i]); + case DBTYPE_TEXT: + case DBTYPE_BLOB: { + size_t len = (size_t)database_value_bytes(argv[i]); + if (i == skip_idx) { + memcpy(buffer + bseek, (char *)database_value_blob(argv[i]), len); + bseek += len; + break; + } + + if (len > (size_t)INT64_MAX) return NULL; + size_t nbytes = pk_encode_nbytes_needed((int64_t)len); + uint8_t type_byte = (uint8_t)((nbytes << 3) | database_value_type(argv[i])); bseek = pk_encode_u8(buffer, bseek, type_byte); - bseek = pk_encode_int64(buffer, bseek, len, nbytes); - bseek = pk_encode_data(buffer, bseek, (char *)sqlite3_value_blob(argv[i]), len); + bseek = pk_encode_uint64(buffer, bseek, (uint64_t)len, nbytes); + bseek = pk_encode_data(buffer, bseek, (char *)database_value_blob(argv[i]), len); } break; - case SQLITE_NULL: { - bseek = pk_encode_u8(buffer, bseek, SQLITE_NULL); + case DBTYPE_NULL: { + bseek = pk_encode_u8(buffer, bseek, DBTYPE_NULL); } break; } } - if (bsize) *bsize = blen; + // return actual bytes written; for prikey it's equal to blen, but safer to report bseek + if (bsize) *bsize = bseek; return buffer; } -char *pk_encode_prikey (sqlite3_value **argv, int argc, char *b, size_t *bsize) { - return pk_encode(argv, argc, b, true, bsize); +char *pk_encode_prikey (dbvalue_t **argv, int argc, char *b, size_t *bsize) { + return pk_encode(argv, argc, b, true, bsize, -1); +} + +char *pk_encode_value (dbvalue_t *value, size_t *bsize) { + dbvalue_t *argv[1] = {value}; + + size_t blen = pk_encode_size(argv, 1, 0, -1); + if (blen == SIZE_MAX) return NULL; + + char *buffer = cloudsync_memory_alloc((uint64_t)blen); + if (!buffer) return NULL; + + *bsize = blen; + return pk_encode(argv, 1, buffer, false, bsize, -1); } diff --git a/src/pk.h b/src/pk.h index ebcc074..ea9a390 100644 --- a/src/pk.h +++ b/src/pk.h @@ -8,23 +8,23 @@ #ifndef __CLOUDSYNC_PK__ #define __CLOUDSYNC_PK__ -#include #include -#include +#include #include +#include "database.h" -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif +typedef int (*pk_decode_callback) (void *xdata, int index, int type, int64_t ival, double dval, char *pval); + +extern char * const PRIKEY_NULL_CONSTRAINT_ERROR; -char *pk_encode_prikey (sqlite3_value **argv, int argc, char *b, size_t *bsize); -char *pk_encode (sqlite3_value **argv, int argc, char *b, bool is_prikey, size_t *bsize); -int pk_decode_prikey (char *buffer, size_t blen, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata); -int pk_decode(char *buffer, size_t blen, int count, size_t *seek, int (*cb) (void *xdata, int index, int type, int64_t ival, double dval, char *pval), void *xdata); -int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); -int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); -size_t pk_encode_size (sqlite3_value **argv, int argc, int reserved); +char *pk_encode_prikey (dbvalue_t **argv, int argc, char *b, size_t *bsize); +char *pk_encode_value (dbvalue_t *value, size_t *bsize); +char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bsize, int skip_idx); +int pk_decode_prikey (char *buffer, size_t blen, pk_decode_callback cb, void *xdata); +int pk_decode (char *buffer, size_t blen, int count, size_t *seek, int skip_decode_idx, pk_decode_callback cb, void *xdata); +int pk_decode_bind_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); +int pk_decode_print_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval); +size_t pk_encode_size (dbvalue_t **argv, int argc, int reserved, int skip_idx); +uint64_t pk_checksum (const char *buffer, size_t blen); #endif diff --git a/src/postgresql/cloudsync.sql.in b/src/postgresql/cloudsync.sql.in new file mode 100644 index 0000000..edfa4d3 --- /dev/null +++ b/src/postgresql/cloudsync.sql.in @@ -0,0 +1,318 @@ +-- CloudSync Extension for PostgreSQL +-- Version @EXTVERSION@ + +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cloudsync" to load this file. \quit + +-- ============================================================================ +-- Public Functions +-- ============================================================================ + +-- Get extension version +CREATE OR REPLACE FUNCTION cloudsync_version() +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_version' +LANGUAGE C IMMUTABLE STRICT; + +-- Get site identifier (UUID) +CREATE OR REPLACE FUNCTION cloudsync_siteid() +RETURNS bytea +AS 'MODULE_PATHNAME', 'pg_cloudsync_siteid' +LANGUAGE C STABLE; + +-- Generate a new UUID +CREATE OR REPLACE FUNCTION cloudsync_uuid() +RETURNS uuid +AS 'MODULE_PATHNAME', 'cloudsync_uuid' +LANGUAGE C VOLATILE; + +-- Get current database version +CREATE OR REPLACE FUNCTION cloudsync_db_version() +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version' +LANGUAGE C STABLE; + +-- Get next database version (with optional merging version) +CREATE OR REPLACE FUNCTION cloudsync_db_version_next() +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version_next' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_db_version_next(merging_version bigint) +RETURNS bigint +AS 'MODULE_PATHNAME', 'cloudsync_db_version_next' +LANGUAGE C VOLATILE; + +-- Initialize CloudSync for a table (3 variants for 1-3 arguments) +-- Returns site_id as bytea +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text, algo text) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +CREATE OR REPLACE FUNCTION cloudsync_init(table_name text, algo text, init_flags integer) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_init' +LANGUAGE C VOLATILE; + +-- Enable sync for a table +CREATE OR REPLACE FUNCTION cloudsync_enable(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_enable' +LANGUAGE C VOLATILE; + +-- Disable sync for a table +CREATE OR REPLACE FUNCTION cloudsync_disable(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_disable' +LANGUAGE C VOLATILE; + +-- Check if table is sync-enabled +CREATE OR REPLACE FUNCTION cloudsync_is_enabled(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_is_enabled' +LANGUAGE C STABLE; + +-- Cleanup orphaned metadata for a table +CREATE OR REPLACE FUNCTION cloudsync_cleanup(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_cleanup' +LANGUAGE C VOLATILE; + +-- Terminate CloudSync +CREATE OR REPLACE FUNCTION cloudsync_terminate() +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_terminate' +LANGUAGE C VOLATILE; + +-- Set global configuration +CREATE OR REPLACE FUNCTION cloudsync_set(key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set' +LANGUAGE C VOLATILE; + +-- Set table-level configuration +CREATE OR REPLACE FUNCTION cloudsync_set_table(table_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_table' +LANGUAGE C VOLATILE; + +-- Set row-level filter for conditional sync +CREATE OR REPLACE FUNCTION cloudsync_set_filter(table_name text, filter_expr text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_filter' +LANGUAGE C VOLATILE; + +-- Clear row-level filter +CREATE OR REPLACE FUNCTION cloudsync_clear_filter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_clear_filter' +LANGUAGE C VOLATILE; + +-- Set column-level configuration +CREATE OR REPLACE FUNCTION cloudsync_set_column(table_name text, column_name text, key text, value text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_set_column' +LANGUAGE C VOLATILE; + +-- Begin schema alteration +CREATE OR REPLACE FUNCTION cloudsync_begin_alter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_begin_alter' +LANGUAGE C VOLATILE; + +-- Commit schema alteration +CREATE OR REPLACE FUNCTION cloudsync_commit_alter(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_commit_alter' +LANGUAGE C VOLATILE; + +-- Payload encoding (aggregate function) +CREATE OR REPLACE FUNCTION cloudsync_payload_encode_transfn(state internal, tbl text, pk bytea, col_name text, col_value bytea, col_version bigint, db_version bigint, site_id bytea, cl bigint, seq bigint) +RETURNS internal +AS 'MODULE_PATHNAME', 'cloudsync_payload_encode_transfn' +LANGUAGE C; + +CREATE OR REPLACE FUNCTION cloudsync_payload_encode_finalfn(state internal) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_payload_encode_finalfn' +LANGUAGE C; + +CREATE OR REPLACE AGGREGATE cloudsync_payload_encode(text, bytea, text, bytea, bigint, bigint, bytea, bigint, bigint) ( + SFUNC = cloudsync_payload_encode_transfn, + STYPE = internal, + FINALFUNC = cloudsync_payload_encode_finalfn +); + +-- Payload decoding and application +CREATE OR REPLACE FUNCTION cloudsync_payload_decode(payload bytea) +RETURNS integer +AS 'MODULE_PATHNAME', 'cloudsync_payload_decode' +LANGUAGE C VOLATILE; + +-- Alias for payload_decode +CREATE OR REPLACE FUNCTION cloudsync_payload_apply(payload bytea) +RETURNS integer +AS 'MODULE_PATHNAME', 'pg_cloudsync_payload_apply' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Private/Internal Functions +-- ============================================================================ + +-- Check if table has sync metadata +CREATE OR REPLACE FUNCTION cloudsync_is_sync(table_name text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_is_sync' +LANGUAGE C STABLE; + +-- Internal insert handler (variadic for multiple PK columns) +CREATE OR REPLACE FUNCTION cloudsync_insert(table_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_insert' +LANGUAGE C VOLATILE; + +-- Internal delete handler (variadic for multiple PK columns) +CREATE OR REPLACE FUNCTION cloudsync_delete(table_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_delete' +LANGUAGE C VOLATILE; + +-- Internal update tracking (aggregate function) +CREATE OR REPLACE FUNCTION cloudsync_update_transfn(state internal, table_name text, new_value anyelement, old_value anyelement) +RETURNS internal +AS 'MODULE_PATHNAME', 'cloudsync_update_transfn' +LANGUAGE C; + +CREATE OR REPLACE FUNCTION cloudsync_update_finalfn(state internal) +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_update_finalfn' +LANGUAGE C; + +CREATE AGGREGATE cloudsync_update(text, anyelement, anyelement) ( + SFUNC = cloudsync_update_transfn, + STYPE = internal, + FINALFUNC = cloudsync_update_finalfn +); + +-- Get sequence number +CREATE OR REPLACE FUNCTION cloudsync_seq() +RETURNS integer +AS 'MODULE_PATHNAME', 'cloudsync_seq' +LANGUAGE C VOLATILE; + +-- Encode primary key (variadic for multiple columns) +CREATE OR REPLACE FUNCTION cloudsync_pk_encode(VARIADIC pk_values "any") +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_pk_encode' +LANGUAGE C IMMUTABLE STRICT; + +-- Decode primary key component +CREATE OR REPLACE FUNCTION cloudsync_pk_decode(encoded_pk bytea, index integer) +RETURNS text +AS 'MODULE_PATHNAME', 'cloudsync_pk_decode' +LANGUAGE C IMMUTABLE STRICT; + +-- ============================================================================ +-- Changes Functions +-- ============================================================================ + +CREATE OR REPLACE FUNCTION cloudsync_encode_value(anyelement) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_encode_value' +LANGUAGE C IMMUTABLE; + +-- Encoded column value helper (PG): returns cloudsync-encoded bytea +CREATE OR REPLACE FUNCTION cloudsync_col_value( + table_name text, + col_name text, + pk bytea +) +RETURNS bytea +AS 'MODULE_PATHNAME', 'cloudsync_col_value' +LANGUAGE C STABLE; + +-- SetReturningFunction: To implement SELECT FROM cloudsync_changes +CREATE FUNCTION cloudsync_changes_select( + min_db_version bigint DEFAULT 0, + filter_site_id bytea DEFAULT NULL +) +RETURNS TABLE ( + tbl text, + pk bytea, + col_name text, + col_value bytea, -- pk_encoded value bytes + col_version bigint, + db_version bigint, + site_id bytea, + cl bigint, + seq bigint +) +AS 'MODULE_PATHNAME', 'cloudsync_changes_select' +LANGUAGE C STABLE; + +-- View con lo stesso nome della vtab SQLite +CREATE OR REPLACE VIEW cloudsync_changes AS +SELECT * FROM cloudsync_changes_select(0, NULL); + +-- Trigger function to implement INSERT on the cloudsync_changes view +CREATE FUNCTION cloudsync_changes_insert_trigger() +RETURNS trigger +AS 'MODULE_PATHNAME', 'cloudsync_changes_insert_trigger' +LANGUAGE C; + +CREATE OR REPLACE TRIGGER cloudsync_changes_insert +INSTEAD OF INSERT ON cloudsync_changes +FOR EACH ROW +EXECUTE FUNCTION cloudsync_changes_insert_trigger(); + +-- Set current schema name +CREATE OR REPLACE FUNCTION cloudsync_set_schema(schema text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'pg_cloudsync_set_schema' +LANGUAGE C VOLATILE; + +-- Get current schema name (if any) +CREATE OR REPLACE FUNCTION cloudsync_schema() +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_schema' +LANGUAGE C VOLATILE; + +-- Get current schema name (if any) +CREATE OR REPLACE FUNCTION cloudsync_table_schema(table_name text) +RETURNS text +AS 'MODULE_PATHNAME', 'pg_cloudsync_table_schema' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Block-level LWW Functions +-- ============================================================================ + +-- Materialize block-level column into base table +CREATE OR REPLACE FUNCTION cloudsync_text_materialize(table_name text, col_name text, VARIADIC pk_values "any") +RETURNS boolean +AS 'MODULE_PATHNAME', 'cloudsync_text_materialize' +LANGUAGE C VOLATILE; + +-- ============================================================================ +-- Type Casts +-- ============================================================================ + +-- Cast function: converts bigint to boolean (0 = false, non-zero = true) +-- Required because BOOLEAN values are encoded as INT8 in sync payloads, +-- but PostgreSQL has no built-in cast from bigint to boolean. +CREATE FUNCTION cloudsync_int8_to_bool(bigint) RETURNS boolean AS $$ + SELECT $1 <> 0 +$$ LANGUAGE SQL IMMUTABLE STRICT; + +-- ASSIGNMENT cast: auto-applies in INSERT/UPDATE context only +-- This enables BOOLEAN column sync where values are encoded as INT8. +-- Using ASSIGNMENT (not IMPLICIT) to avoid unintended conversions in WHERE clauses. +CREATE CAST (bigint AS boolean) + WITH FUNCTION cloudsync_int8_to_bool(bigint) + AS ASSIGNMENT; diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c new file mode 100644 index 0000000..4d0ed6a --- /dev/null +++ b/src/postgresql/cloudsync_postgresql.c @@ -0,0 +1,2912 @@ +// +// cloudsync_postgresql.c +// cloudsync +// +// Created by Claude Code on 18/12/25. +// + +// Define POSIX feature test macros before any includes +#define _POSIX_C_SOURCE 200809L + +// PostgreSQL requires postgres.h to be included FIRST +#include "postgres.h" +#include "utils/datum.h" +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "catalog/namespace.h" +#include "executor/spi.h" +#include "utils/lsyscache.h" +#include "utils/array.h" +#include "fmgr.h" +#include "funcapi.h" +#include "pgvalue.h" +#include "storage/ipc.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" +#include "utils/uuid.h" +#include "nodes/nodeFuncs.h" // exprTypmod, exprCollation +#include "nodes/pg_list.h" // linitial +#include "nodes/primnodes.h" // FuncExpr + +// CloudSync headers (after PostgreSQL headers) +#include "../cloudsync.h" +#include "../block.h" +#include "../database.h" +#include "../dbutils.h" +#include "../pk.h" +#include "../utils.h" + +// Note: network.h is not needed for PostgreSQL implementation + +PG_MODULE_MAGIC; + +// Note: PG_FUNCTION_INFO_V1 macros are declared before each function implementation below +// They should NOT be duplicated here to avoid redefinition errors + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +#define CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA "E'\\\\x0b095f5f5b524c535d5f5f'::bytea" +#define CLOUDSYNC_NULL_VALUE_BYTEA "E'\\\\x05'::bytea" + +// External declaration +Datum database_column_datum (dbvm_t *vm, int index); + +// MARK: - Context Management - + +// Global context stored per backend +static cloudsync_context *pg_cloudsync_context = NULL; + +static void cloudsync_pg_context_init (cloudsync_context *data) { + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + + PG_TRY(); + { + if (cloudsync_config_exists(data)) { + if (cloudsync_context_init(data) == NULL) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("An error occurred while trying to initialize context"))); + } + + // update schema hash if upgrading from an older version + if (dbutils_settings_check_version(data, NULL) != 0) { + cloudsync_update_schema_hash(data); + } + + // make sure to update internal version to current version + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + } + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +// Get or create the CloudSync context for this backend +static cloudsync_context *get_cloudsync_context(void) { + if (pg_cloudsync_context == NULL) { + // Create context - db_t is not used in PostgreSQL mode + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + cloudsync_context *data = cloudsync_context_create(NULL); + MemoryContextSwitchTo(old); + if (!data) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to create a database context"))); + } + // Set early to prevent infinite recursion: during init, SQL queries may call + // cloudsync_schema() which calls get_cloudsync_context(). Without early assignment, + // each nested call sees NULL and tries to reinitialize, causing stack overflow. + pg_cloudsync_context = data; + PG_TRY(); + { + cloudsync_pg_context_init(data); + } + PG_CATCH(); + { + pg_cloudsync_context = NULL; + cloudsync_context_free(data); + PG_RE_THROW(); + } + PG_END_TRY(); + } + + return pg_cloudsync_context; +} + +// MARK: - Extension Entry Points - + +void _PG_init (void) { + // Extension initialization + // SPI will be connected per-function call + elog(DEBUG1, "CloudSync extension loading"); + + // Initialize memory debugger (NOOP in production) + cloudsync_memory_init(1); + + // Set fractional-indexing allocator to use cloudsync memory + block_init_allocator(); +} + +void _PG_fini (void) { + // Extension cleanup + elog(DEBUG1, "CloudSync extension unloading"); + + // Free global context if it exists + if (pg_cloudsync_context) { + cloudsync_context_free(pg_cloudsync_context); + pg_cloudsync_context = NULL; + } +} + +// MARK: - Public SQL Functions - + +// cloudsync_version() - Returns extension version +PG_FUNCTION_INFO_V1(cloudsync_version); +Datum cloudsync_version (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + PG_RETURN_TEXT_P(cstring_to_text(CLOUDSYNC_VERSION)); +} + +// cloudsync_siteid() - Get site identifier (UUID) +PG_FUNCTION_INFO_V1(pg_cloudsync_siteid); +Datum pg_cloudsync_siteid (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + const void *siteid = cloudsync_siteid(data); + + if (!siteid) { + PG_RETURN_NULL(); + } + + // Return as bytea (binary UUID) + bytea *result = (bytea *)palloc(VARHDRSZ + UUID_LEN); + SET_VARSIZE(result, VARHDRSZ + UUID_LEN); + memcpy(VARDATA(result), siteid, UUID_LEN); + + PG_RETURN_BYTEA_P(result); +} + +// cloudsync_uuid() - Generate a new UUID +PG_FUNCTION_INFO_V1(cloudsync_uuid); +Datum cloudsync_uuid (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + uint8_t uuid_bytes[UUID_LEN]; + cloudsync_uuid_v7(uuid_bytes); + + // Format as text with dashes (matches SQLite implementation) + char uuid_str[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify(uuid_bytes, uuid_str, true); + + // Parse into PostgreSQL UUID type + Datum uuid_datum = DirectFunctionCall1(uuid_in, CStringGetDatum(uuid_str)); + + PG_RETURN_DATUM(uuid_datum); +} + +// cloudsync_db_version() - Get current database version +PG_FUNCTION_INFO_V1(cloudsync_db_version); +Datum cloudsync_db_version (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int64_t version = 0; + bool spi_connected = false; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != DBRES_OK) { + // When cloudsync_init was never called, data_version_stmt is NULL + // and database_errmsg() is empty, producing the unhelpful "Unable + // to retrieve db_version ()". Detect the uninitialized state and + // return an actionable message instead. The extra check only runs + // on the error branch, so it costs nothing on the sync hot path. + if (!cloudsync_context_is_initialized(data)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version()."))); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to retrieve db_version (%s)", database_errmsg(data)))); + } + + version = cloudsync_dbversion(data); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_INT64(version); +} + +// cloudsync_db_version_next([merging_version]) - Get next database version +PG_FUNCTION_INFO_V1(cloudsync_db_version_next); +Datum cloudsync_db_version_next (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + int64_t next_version = 0; + bool spi_connected = false; + + int64_t merging_version = CLOUDSYNC_VALUE_NOTSET; + if (PG_NARGS() == 1 && !PG_ARGISNULL(0)) { + merging_version = PG_GETARG_INT64(0); + } + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + next_version = cloudsync_dbversion_next(data, merging_version); + if (next_version == -1) { + // Previously this path silently returned -1, which is worse than + // an error because callers cannot distinguish a bogus version + // number from a real one. Emit an actionable message when the + // root cause is that cloudsync_init has never been called. + if (!cloudsync_context_is_initialized(data)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version_next()."))); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to retrieve next_db_version (%s)", database_errmsg(data)))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_INT64(next_version); +} + +// MARK: - Table Initialization - + +// Internal helper for cloudsync_init - replicates dbsync_init logic from SQLite +// Returns site_id as bytea on success, raises error on failure +static bytea *cloudsync_init_internal (cloudsync_context *data, const char *table, const char *algo, CLOUDSYNC_INIT_FLAG init_flags) { + bytea *result = NULL; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + + PG_TRY(); + { + // Begin savepoint for transactional init + int rc = database_begin_savepoint(data, "cloudsync_init"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to create cloudsync_init savepoint: %s", database_errmsg(data)))); + } + + // Initialize table for sync + rc = cloudsync_init_table(data, table, algo, init_flags); + ereport(DEBUG1, (errmsg("cloudsync_init_internal cloudsync_init_table %d", rc))); + + if (rc == DBRES_OK) { + rc = database_commit_savepoint(data, "cloudsync_init"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_init savepoint: %s", database_errmsg(data)))); + } + + // Persist schema to settings now that the settings table exists + const char *cur_schema = cloudsync_schema(data); + if (cur_schema) { + dbutils_settings_set_key_value(data, "schema", cur_schema); + } + } else { + // In case of error, rollback sub-transaction and raise. + // We intentionally avoid database_rollback_savepoint() here + // because it calls database_refresh_snapshot() which pushes a + // new active snapshot. Pushing a snapshot after rollback and + // before ereport(ERROR) leaves portal->portalSnapshot non-NULL + // when PL/pgSQL's exception handler later calls + // EnsurePortalSnapshotExists(), triggering + // Assert(portal->portalSnapshot == NULL) on debug builds. + char err[1024]; + snprintf(err, sizeof(err), "%s", cloudsync_errmsg(data)); + if (GetCurrentTransactionNestLevel() > 1) { + RollbackAndReleaseCurrentSubTransaction(); + } + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", err))); + } + + cloudsync_update_schema_hash(data); + + // Build site_id as bytea to return + // Use SPI_palloc so the allocation survives SPI_finish + result = (bytea *)SPI_palloc(UUID_LEN + VARHDRSZ); + SET_VARSIZE(result, UUID_LEN + VARHDRSZ); + memcpy(VARDATA(result), cloudsync_siteid(data), UUID_LEN); + + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + return result; +} + +// cloudsync_init(table_name, [algo], [init_flags]) - Initialize table for sync +// Supports 1-3 arguments with defaults: algo=NULL, init_flags=CLOUDSYNC_INIT_FLAG_NONE +PG_FUNCTION_INFO_V1(cloudsync_init); +Datum cloudsync_init (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + // Default values + const char *algo = NULL; + int init_flags = CLOUDSYNC_INIT_FLAG_NONE; + + // Handle optional arguments + int nargs = PG_NARGS(); + + if (nargs >= 2 && !PG_ARGISNULL(1)) { + algo = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + + if (nargs >= 3 && !PG_ARGISNULL(2)) { + init_flags = PG_GETARG_INT32(2); + } + + cloudsync_context *data = get_cloudsync_context(); + + // Call internal helper and return site_id as bytea + bytea *result = cloudsync_init_internal(data, table, algo, init_flags); + PG_RETURN_BYTEA_P(result); +} + +// MARK: - Table Enable/Disable Functions - + +// Internal helper for enable/disable +static void cloudsync_enable_disable (const char *table_name, bool value) { + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, table_name); + if (table) table_set_enabled(table, value); +} + +// cloudsync_enable - Enable sync for a table +PG_FUNCTION_INFO_V1(cloudsync_enable); +Datum cloudsync_enable (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_enable_disable(table, true); + PG_RETURN_BOOL(true); +} + +// cloudsync_disable - Disable sync for a table +PG_FUNCTION_INFO_V1(cloudsync_disable); +Datum cloudsync_disable (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_enable_disable(table, false); + PG_RETURN_BOOL(true); +} + +// cloudsync_is_enabled - Check if table is sync-enabled +PG_FUNCTION_INFO_V1(cloudsync_is_enabled); +Datum cloudsync_is_enabled (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + cloudsync_context *data = get_cloudsync_context(); + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_table_context *table = table_lookup(data, table_name); + + bool result = (table && table_enabled(table)); + PG_RETURN_BOOL(result); +} + +// MARK: - Cleanup and Termination - + +// cloudsync_cleanup - Cleanup orphaned metadata for a table +PG_FUNCTION_INFO_V1(pg_cloudsync_cleanup); +Datum pg_cloudsync_cleanup (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_cleanup(data, table); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + if (spi_connected) SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + PG_RETURN_BOOL(true); +} + +// cloudsync_terminate - Terminate CloudSync +PG_FUNCTION_INFO_V1(pg_cloudsync_terminate); +Datum pg_cloudsync_terminate (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_terminate(data); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(rc == DBRES_OK); +} + +// MARK: - Settings Functions - + +// cloudsync_set - Set global configuration +PG_FUNCTION_INFO_V1(cloudsync_set); +Datum cloudsync_set (PG_FUNCTION_ARGS) { + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + + // Silently fail if key is NULL (matches SQLite behavior) + if (key == NULL) { + PG_RETURN_BOOL(true); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + dbutils_settings_set_key_value(data, key, value); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_set_table - Set table-level configuration +PG_FUNCTION_INFO_V1(cloudsync_set_table); +Datum cloudsync_set_table (PG_FUNCTION_ARGS) { + const char *tbl = NULL; + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + if (!PG_ARGISNULL(2)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(2)); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + dbutils_table_settings_set_key_value(data, tbl, "*", key, value); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_set_column - Set column-level configuration +PG_FUNCTION_INFO_V1(cloudsync_set_column); +Datum cloudsync_set_column (PG_FUNCTION_ARGS) { + const char *tbl = NULL; + const char *col = NULL; + const char *key = NULL; + const char *value = NULL; + + if (!PG_ARGISNULL(0)) { + tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + if (!PG_ARGISNULL(1)) { + col = text_to_cstring(PG_GETARG_TEXT_PP(1)); + } + if (!PG_ARGISNULL(2)) { + key = text_to_cstring(PG_GETARG_TEXT_PP(2)); + } + if (!PG_ARGISNULL(3)) { + value = text_to_cstring(PG_GETARG_TEXT_PP(3)); + } + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Handle block column setup: cloudsync_set_column('tbl', 'col', 'algo', 'block') + if (key && value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { + int rc = cloudsync_setup_block_column(data, tbl, col, NULL, true); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + } else { + // Handle delimiter setting: cloudsync_set_column('tbl', 'col', 'delimiter', '\n\n') + if (key && strcmp(key, "delimiter") == 0) { + cloudsync_table_context *table = table_lookup(data, tbl); + if (table) { + int col_idx = table_col_index(table, col); + if (col_idx >= 0 && table_col_algo(table, col_idx) == col_algo_block) { + table_set_col_delimiter(table, col_idx, value); + } + } + } + dbutils_table_settings_set_key_value(data, tbl, col, key, value); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Row Filter - + +// cloudsync_set_filter - Set a row-level filter for conditional sync +PG_FUNCTION_INFO_V1(cloudsync_set_filter); +Datum cloudsync_set_filter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_set_filter: table and filter expression required"))); + } + + const char *tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + const char *filter_expr = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Guard against calling set_filter before the target table has been + // set up for sync: without this, we'd drop and fail to recreate + // triggers, emitting ten+ noisy "does not exist, skipping" NOTICEs + // followed by a generic "error recreating triggers" message that + // does not point at the real cause. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync_set_filter: table '%s' is not configured for sync. Call SELECT cloudsync_init('%s') first.", tbl, tbl))); + } + + // Store filter in table settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", filter_expr); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop triggers + database_delete_triggers(data, tbl); + + // Reconnect SPI so that the catalog changes from DROP are visible + SPI_finish(); + spi_connected = false; + spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + // Recreate triggers with filter + int rc = database_create_triggers(data, tbl, algo, filter_expr); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_set_filter: error recreating triggers"))); + } + + // Clean and refill metatable with the new filter + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_set_filter: error resetting metatable"))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// cloudsync_clear_filter - Remove the row-level filter for a table +PG_FUNCTION_INFO_V1(cloudsync_clear_filter); +Datum cloudsync_clear_filter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_clear_filter: table name required"))); + } + + const char *tbl = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + cloudsync_context *data = get_cloudsync_context(); + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + // Guard against calling clear_filter before the target table has + // been set up for sync — see cloudsync_set_filter for the same + // rationale. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cloudsync_clear_filter: table '%s' is not configured for sync. Call SELECT cloudsync_init('%s') first.", tbl, tbl))); + } + + // Remove filter from settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", NULL); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop triggers + database_delete_triggers(data, tbl); + + // Reconnect SPI so that the catalog changes from DROP are visible + SPI_finish(); + spi_connected = false; + spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + // Recreate triggers without filter + int rc = database_create_triggers(data, tbl, algo, NULL); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_clear_filter: error recreating triggers"))); + } + + // Clean and refill metatable without filter (all rows) + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cloudsync_clear_filter: error resetting metatable"))); + } + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Schema Alteration - + +// cloudsync_begin_alter - Begin schema alteration +PG_FUNCTION_INFO_V1(pg_cloudsync_begin_alter); +Datum pg_cloudsync_begin_alter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); + } + + PG_TRY(); + { + database_begin_savepoint(data, "cloudsync_alter"); + rc = cloudsync_begin_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + } + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s", cloudsync_errmsg(data)))); + } + PG_RETURN_BOOL(true); +} + +// cloudsync_commit_alter - Commit schema alteration +// +// This wrapper manages SPI in two phases to avoid the PostgreSQL warning +// "subtransaction left non-empty SPI stack". The subtransaction was opened +// by a prior cloudsync_begin_alter call, so SPI_connect() here creates a +// connection at the subtransaction level. We must disconnect SPI before +// cloudsync_commit_alter releases that subtransaction, then reconnect +// for post-commit work (cloudsync_update_schema_hash). +// Prepared statements survive SPI_finish via SPI_keepplan/TopMemoryContext. +PG_FUNCTION_INFO_V1(pg_cloudsync_commit_alter); +Datum pg_cloudsync_commit_alter (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + + // Phase 1: SPI work before savepoint release + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed"))); + } + + PG_TRY(); + { + rc = cloudsync_commit_alter(data, table_name); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + // Disconnect SPI before savepoint boundary + SPI_finish(); + + if (rc != DBRES_OK) { + // Rollback savepoint (SPI disconnected, no warning) + database_rollback_savepoint(data, "cloudsync_alter"); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + // Release savepoint (SPI disconnected, no warning) + rc = database_commit_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_alter savepoint: %s", database_errmsg(data)))); + } + + // Phase 2: reconnect SPI for post-commit work + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed after savepoint release"))); + } + + PG_TRY(); + { + cloudsync_update_schema_hash(data); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_BOOL(true); +} + +// MARK: - Payload Functions - + +// Aggregate function: cloudsync_payload_encode transition function +PG_FUNCTION_INFO_V1(cloudsync_payload_encode_transfn); +Datum cloudsync_payload_encode_transfn (PG_FUNCTION_ARGS) { + MemoryContext aggContext; + cloudsync_payload_context *payload = NULL; + + if (!AggCheckCallContext(fcinfo, &aggContext)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_payload_encode_transfn called in non-aggregate context"))); + } + + // Get or allocate aggregate state + if (PG_ARGISNULL(0)) { + MemoryContext oldContext = MemoryContextSwitchTo(aggContext); + payload = (cloudsync_payload_context *)palloc0(cloudsync_payload_context_size(NULL)); + MemoryContextSwitchTo(oldContext); + } else { + payload = (cloudsync_payload_context *)PG_GETARG_POINTER(0); + } + + int argc = 0; + cloudsync_context *data = get_cloudsync_context(); + pgvalue_t **argv = pgvalues_from_args(fcinfo, 1, &argc); + + // Wrap variadic args into pgvalue_t so pk/payload helpers can read types safely. + if (argc > 0) { + int rc = cloudsync_payload_encode_step(payload, data, argc, (dbvalue_t **)argv); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + } + + // payload_encode_step does not retain pgvalue_t*, free transient wrappers now + for (int i = 0; i < argc; i++) { + pgvalue_free(argv[i]); + } + if (argv) cloudsync_memory_free(argv); + + PG_RETURN_POINTER(payload); +} + +// Aggregate function: cloudsync_payload_encode finalize function +PG_FUNCTION_INFO_V1(cloudsync_payload_encode_finalfn); +Datum cloudsync_payload_encode_finalfn (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + PG_RETURN_NULL(); + } + + cloudsync_payload_context *payload = (cloudsync_payload_context *)PG_GETARG_POINTER(0); + cloudsync_context *data = get_cloudsync_context(); + + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + + int64_t blob_size = 0; + char *blob = cloudsync_payload_blob(payload, &blob_size, NULL); + + if (!blob) { + PG_RETURN_NULL(); + } + + bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); + SET_VARSIZE(result, VARHDRSZ + blob_size); + memcpy(VARDATA(result), blob, blob_size); + + cloudsync_memory_free(blob); + + PG_RETURN_BYTEA_P(result); +} + +// Payload decode - Apply changes from payload +PG_FUNCTION_INFO_V1(cloudsync_payload_decode); +Datum cloudsync_payload_decode (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("payload cannot be NULL"))); + } + + bytea *payload_data = PG_GETARG_BYTEA_P(0); + int blen = VARSIZE(payload_data) - VARHDRSZ; + + // Sanity check payload size + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + if (blen < (int)header_size) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Invalid payload size"))); + } + + const char *payload = VARDATA(payload_data); + cloudsync_context *data = get_cloudsync_context(); + int rc = DBRES_OK; + int nrows = 0; + bool spi_connected = false; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + rc = cloudsync_payload_apply(data, payload, blen, &nrows); + } + PG_CATCH(); + { + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (spi_connected) SPI_finish(); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + } + PG_RETURN_INT32(nrows); +} + +// Alias for payload_decode +PG_FUNCTION_INFO_V1(pg_cloudsync_payload_apply); +Datum pg_cloudsync_payload_apply (PG_FUNCTION_ARGS) { + return cloudsync_payload_decode(fcinfo); +} + +// MARK: - Private/Internal Functions - + +typedef struct cloudsync_pg_cleanup_state { + char *pk; + char pk_buffer[1024]; + pgvalue_t **argv; + int argc; + bool spi_connected; +} cloudsync_pg_cleanup_state; + +static void cloudsync_pg_cleanup(int code, Datum arg) { + cloudsync_pg_cleanup_state *state = (cloudsync_pg_cleanup_state *)DatumGetPointer(arg); + if (!state) return; + UNUSED_PARAMETER(code); + + if (state->pk && state->pk != state->pk_buffer) { + cloudsync_memory_free(state->pk); + } + state->pk = NULL; + + for (int i = 0; i < state->argc; i++) { + pgvalue_free(state->argv[i]); + } + if (state->argv) cloudsync_memory_free(state->argv); + state->argv = NULL; + state->argc = 0; + + if (state->spi_connected) { + SPI_finish(); + state->spi_connected = false; + } +} + +// cloudsync_is_sync - Check if table has sync metadata +PG_FUNCTION_INFO_V1(cloudsync_is_sync); +Datum cloudsync_is_sync (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + + if (cloudsync_insync(data)) { + PG_RETURN_BOOL(true); + } + + if (PG_ARGISNULL(0)) { + PG_RETURN_BOOL(false); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_table_context *table = table_lookup(data, table_name); + + bool result = (table && (table_enabled(table) == 0)); + PG_RETURN_BOOL(result); +} + +typedef struct cloudsync_update_payload { + pgvalue_t *table_name; + pgvalue_t **new_values; + pgvalue_t **old_values; + int count; + int capacity; + MemoryContext mcxt; + // Context-owned callback info for early-exit cleanup. + // We null the payload pointer on normal finalization to avoid double-free. + struct cloudsync_mcxt_cb_info *mcxt_cb_info; +} cloudsync_update_payload; + +static void cloudsync_update_payload_free (cloudsync_update_payload *payload); + +typedef struct cloudsync_mcxt_cb_info { + MemoryContext mcxt; + const char *name; + cloudsync_update_payload *payload; +} cloudsync_mcxt_cb_info; + +static void cloudsync_mcxt_reset_cb (void *arg) { + cloudsync_mcxt_cb_info *info = (cloudsync_mcxt_cb_info *)arg; + if (!info) return; + if (!info->payload) return; + + // Context reset means the aggregate state would be lost; clean it here. + cloudsync_update_payload_free(info->payload); + info->payload = NULL; +} + +static void cloudsync_update_payload_free (cloudsync_update_payload *payload) { + if (!payload) return; + + if (payload->mcxt_cb_info) { + // Normal finalize path: prevent the reset callback from double-free. + payload->mcxt_cb_info->payload = NULL; + } + + for (int i = 0; i < payload->count; i++) { + pgvalue_free(payload->new_values[i]); + pgvalue_free(payload->old_values[i]); + } + if (payload->new_values) pfree(payload->new_values); + if (payload->old_values) pfree(payload->old_values); + if (payload->table_name) pgvalue_free(payload->table_name); + + payload->new_values = NULL; + payload->old_values = NULL; + payload->table_name = NULL; + payload->count = 0; + payload->capacity = 0; + payload->mcxt = NULL; + payload->mcxt_cb_info = NULL; +} + +static bool cloudsync_update_payload_append (cloudsync_update_payload *payload, pgvalue_t *table_name, pgvalue_t *new_value, pgvalue_t *old_value) { + if (!payload) return false; + if (!payload->mcxt || !MemoryContextIsValid(payload->mcxt)) { + elog(DEBUG1, "cloudsync_update_payload_append invalid payload context payload=%p mcxt=%p", payload, payload->mcxt); + return false; + } + if (payload->count < 0 || payload->capacity < 0) { + elog(DEBUG1, "cloudsync_update_payload_append invalid counters payload=%p count=%d cap=%d", payload, payload->count, payload->capacity); + return false; + } + + if (payload->count >= payload->capacity) { + int newcap = payload->capacity ? payload->capacity * 2 : 128; + elog(DEBUG1, "cloudsync_update_payload_append newcap=%d", newcap); + MemoryContext old = MemoryContextSwitchTo(payload->mcxt); + if (payload->capacity == 0) { + payload->new_values = (pgvalue_t **)palloc0(newcap * sizeof(*payload->new_values)); + payload->old_values = (pgvalue_t **)palloc0(newcap * sizeof(*payload->old_values)); + } else { + payload->new_values = (pgvalue_t **)repalloc(payload->new_values, newcap * sizeof(*payload->new_values)); + payload->old_values = (pgvalue_t **)repalloc(payload->old_values, newcap * sizeof(*payload->old_values)); + } + payload->capacity = newcap; + MemoryContextSwitchTo(old); + } + + if (payload->count >= payload->capacity) { + elog(DEBUG1, + "cloudsync_update_payload_append count>=capacity payload=%p count=%d " + "cap=%d new_values=%p old_values=%p", + payload, payload->count, payload->capacity, payload->new_values, + payload->old_values); + return false; + } + + int index = payload->count; + if (payload->table_name == NULL) { + payload->table_name = table_name; + } else { + // Compare within the payload context so any lazy text/detoast buffers + // are allocated in a stable context (not ExprContext). + MemoryContext old = MemoryContextSwitchTo(payload->mcxt); + int cmp = dbutils_value_compare((dbvalue_t *)payload->table_name, (dbvalue_t *)table_name); + MemoryContextSwitchTo(old); + if (cmp != 0) { + return false; + } + pgvalue_free(table_name); + } + + payload->new_values[index] = new_value; + payload->old_values[index] = old_value; + payload->count++; + + return true; +} + +// cloudsync_seq - Get sequence number +PG_FUNCTION_INFO_V1(cloudsync_seq); +Datum cloudsync_seq (PG_FUNCTION_ARGS) { + UNUSED_PARAMETER(fcinfo); + + cloudsync_context *data = get_cloudsync_context(); + int seq = cloudsync_bumpseq(data); + + PG_RETURN_INT32(seq); +} + +// cloudsync_pk_encode - Encode primary key from variadic arguments +PG_FUNCTION_INFO_V1(cloudsync_pk_encode); +Datum cloudsync_pk_encode (PG_FUNCTION_ARGS) { + int argc = 0; + pgvalue_t **argv = NULL; + + // Signature is VARIADIC "any", so extract all arguments starting from index 0 + argv = pgvalues_from_args(fcinfo, 0, &argc); + if (!argv || argc == 0) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_pk_encode requires at least one primary key value"))); + } + + // Normalize all values to text for consistent PK encoding + // (PG triggers cast PK values to ::text; SQL callers must match) + pgvalues_normalize_to_text(argv, argc); + + size_t pklen = 0; + char *encoded = pk_encode_prikey((dbvalue_t **)argv, argc, NULL, &pklen); + if (!encoded || encoded == PRIKEY_NULL_CONSTRAINT_ERROR) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_pk_encode failed to encode primary key"))); + } + + bytea *result = (bytea *)palloc(pklen + VARHDRSZ); + SET_VARSIZE(result, pklen + VARHDRSZ); + memcpy(VARDATA(result), encoded, pklen); + cloudsync_memory_free(encoded); + + for (int i = 0; i < argc; i++) { + pgvalue_free(argv[i]); + } + if (argv) cloudsync_memory_free(argv); + + PG_RETURN_BYTEA_P(result); +} + +typedef struct cloudsync_pk_decode_ctx { + int target_index; + text *result; + bool found; +} cloudsync_pk_decode_ctx; + +static int cloudsync_pk_decode_set_result (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_pk_decode_ctx *ctx = (cloudsync_pk_decode_ctx *)xdata; + if (!ctx || ctx->found || (index + 1) != ctx->target_index) return DBRES_OK; + + switch (type) { + case DBTYPE_INTEGER: { + char *cstr = DatumGetCString(DirectFunctionCall1(int8out, Int64GetDatum(ival))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + break; + } + case DBTYPE_FLOAT: { + char *cstr = DatumGetCString(DirectFunctionCall1(float8out, Float8GetDatum(dval))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + break; + } + case DBTYPE_TEXT: { + ctx->result = cstring_to_text_with_len(pval, (int)ival); + break; + } + case DBTYPE_BLOB: { + bytea *ba = (bytea *)palloc(ival + VARHDRSZ); + SET_VARSIZE(ba, ival + VARHDRSZ); + memcpy(VARDATA(ba), pval, (size_t)ival); + char *cstr = DatumGetCString(DirectFunctionCall1(byteaout, PointerGetDatum(ba))); + ctx->result = cstring_to_text(cstr); + pfree(cstr); + pfree(ba); + break; + } + case DBTYPE_NULL: + default: + ctx->result = NULL; + break; + } + + ctx->found = true; + return DBRES_OK; +} + +// cloudsync_pk_decode - Decode primary key component at given index +PG_FUNCTION_INFO_V1(cloudsync_pk_decode); +Datum cloudsync_pk_decode (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + PG_RETURN_NULL(); + } + + bytea *ba = PG_GETARG_BYTEA_P(0); + int index = PG_GETARG_INT32(1); + if (index < 1) PG_RETURN_NULL(); + + cloudsync_pk_decode_ctx ctx = { + .target_index = index, + .result = NULL, + .found = false + }; + + char *buffer = VARDATA(ba); + size_t blen = (size_t)(VARSIZE(ba) - VARHDRSZ); + if (pk_decode_prikey(buffer, blen, cloudsync_pk_decode_set_result, &ctx) < 0) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_pk_decode failed to decode primary key"))); + } + + if (!ctx.found || ctx.result == NULL) PG_RETURN_NULL(); + PG_RETURN_TEXT_P(ctx.result); +} + +// cloudsync_insert - Internal insert handler +// Signature: cloudsync_insert(table_name text, VARIADIC pk_values anyarray) +PG_FUNCTION_INFO_V1(cloudsync_insert); +Datum cloudsync_insert (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + // Connect SPI for database operations + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + // Lookup table (load from settings if needed) + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_insert", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_insert", table_name))); + } + } + + // Extract PK values from VARIADIC "any" (args starting from index 1) + cleanup.argv = pgvalues_from_args(fcinfo, 1, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + // Verify we have the correct number of PK columns + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + + // Encode the primary key values into a buffer + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + + if (!cleanup.pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Insert aborted because primary key in table %s contains NULL values", table_name))); + } + + // Compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // Check if a row with the same primary key already exists + // (if so, this might be a previously deleted sentinel) + bool pk_exists = table_pk_exists(table, cleanup.pk, pklen); + int rc = DBRES_OK; + + if (table_count_cols(table) == 0) { + // If there are no columns other than primary keys, insert a sentinel record + rc = local_mark_insert_sentinel_meta(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + } else if (pk_exists) { + // If a row with the same primary key already exists, update the sentinel record + rc = local_update_sentinel(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + } + + if (rc == DBRES_OK) { + // Process each non-primary key column for insert or update + for (int i = 0; i < table_count_cols(table); i++) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: read value from base table, split into blocks, store each block + dbvm_t *val_vm = table_column_lookup(table, table_colname(table, i), false, NULL); + if (!val_vm) { rc = DBRES_ERROR; break; } + + int bind_rc = pk_decode_prikey(cleanup.pk, pklen, pk_decode_bind_callback, (void *)val_vm); + if (bind_rc < 0) { databasevm_reset(val_vm); rc = DBRES_ERROR; break; } + + int step_rc = databasevm_step(val_vm); + if (step_rc == DBRES_ROW) { + const char *text = database_column_text(val_vm, 0); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + block_list_t *blocks = block_split(text ? text : "", delim); + if (blocks) { + char **positions = block_initial_positions(blocks->count); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col, positions[b]); + if (block_cn) { + rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); + + // Store block value in blocks table + dbvm_t *wvm = table_block_value_write_stmt(table); + if (wvm && rc == DBRES_OK) { + databasevm_bind_blob(wvm, 1, cleanup.pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + if (rc != DBRES_OK) break; + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } + } + databasevm_reset(val_vm); + if (step_rc == DBRES_ROW || step_rc == DBRES_DONE) { if (rc == DBRES_OK) continue; } + if (rc != DBRES_OK) break; + } else { + rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) break; + } + } + } + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +// cloudsync_delete - Internal delete handler +// Signature: cloudsync_delete(table_name text, VARIADIC pk_values anyarray) +PG_FUNCTION_INFO_V1(cloudsync_delete); +Datum cloudsync_delete (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_delete", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_delete", table_name))); + } + } + + // Extract PK values from VARIADIC "any" (args starting from index 1) + cleanup.argv = pgvalues_from_args(fcinfo, 1, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + int rc = DBRES_OK; + + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + if (!cleanup.pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Delete aborted because primary key in table %s contains NULL values", table_name))); + } + + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + rc = local_mark_delete_meta(table, cleanup.pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc == DBRES_OK) { + rc = local_drop_meta(table, cleanup.pk, pklen); + } + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(cloudsync_update_transfn); +Datum cloudsync_update_transfn (PG_FUNCTION_ARGS) { + MemoryContext aggContext; + MemoryContext allocContext = NULL; + cloudsync_update_payload *payload = NULL; + + if (!AggCheckCallContext(fcinfo, &aggContext)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn called in non-aggregate context"))); + } + + allocContext = aggContext; + if (aggContext && aggContext->name && strcmp(aggContext->name, "ExprContext") == 0 && aggContext->parent) { + allocContext = aggContext->parent; + } + + if (PG_ARGISNULL(0)) { + MemoryContext old = MemoryContextSwitchTo(allocContext); + payload = (cloudsync_update_payload *)palloc0(sizeof(cloudsync_update_payload)); + payload->mcxt = allocContext; + MemoryContextSwitchTo(old); + } else { + payload = (cloudsync_update_payload *)PG_GETARG_POINTER(0); + if (payload->mcxt == NULL || payload->mcxt != allocContext) { + elog(DEBUG1, "cloudsync_update_transfn repairing payload context payload=%p old_mcxt=%p new_mcxt=%p", payload, payload->mcxt, allocContext); + payload->mcxt = allocContext; + } + } + + if (!payload) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn payload is null"))); + } + + if (payload->mcxt_cb_info && payload->mcxt_cb_info->mcxt != allocContext) { + payload->mcxt_cb_info->payload = NULL; + payload->mcxt_cb_info = NULL; + } + + if (!payload->mcxt_cb_info) { + MemoryContext old = MemoryContextSwitchTo(allocContext); + // info and cb are automatically freed when that context is reset or deleted + cloudsync_mcxt_cb_info *info = (cloudsync_mcxt_cb_info *)palloc0(sizeof(*info)); + info->mcxt = allocContext; + info->name = allocContext ? allocContext->name : ""; + info->payload = payload; + MemoryContextCallback *cb = (MemoryContextCallback *)palloc0(sizeof(*cb)); + cb->func = cloudsync_mcxt_reset_cb; + cb->arg = info; + MemoryContextRegisterResetCallback(allocContext, cb); + payload->mcxt_cb_info = info; + MemoryContextSwitchTo(old); + } + + if (payload->count < 0 || payload->capacity < 0 ||payload->count > payload->capacity) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn invalid payload state: count=%d cap=%d", payload->count, payload->capacity))); + } + + elog(DEBUG1, + "cloudsync_update_transfn contexts current=%p name=%s agg=%p name=%s " + "alloc=%p name=%s", + CurrentMemoryContext, + CurrentMemoryContext ? CurrentMemoryContext->name : "", aggContext, + aggContext ? aggContext->name : "", allocContext, + allocContext ? allocContext->name : ""); + + Oid table_type = get_fn_expr_argtype(fcinfo->flinfo, 1); + bool table_null = PG_ARGISNULL(1); + Datum table_datum = table_null ? (Datum)0 : PG_GETARG_DATUM(1); + Oid new_type = get_fn_expr_argtype(fcinfo->flinfo, 2); + bool new_null = PG_ARGISNULL(2); + Datum new_datum = new_null ? (Datum)0 : PG_GETARG_DATUM(2); + Oid old_type = get_fn_expr_argtype(fcinfo->flinfo, 3); + bool old_null = PG_ARGISNULL(3); + Datum old_datum = old_null ? (Datum)0 : PG_GETARG_DATUM(3); + + if (!OidIsValid(table_type) || !OidIsValid(new_type) || !OidIsValid(old_type)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn invalid argument types"))); + } + + MemoryContext old_ctx = MemoryContextSwitchTo(allocContext); + // debug code + // MemoryContextStats(allocContext); + pgvalue_t *table_name = pgvalue_create(table_datum, table_type, -1, fcinfo->fncollation, table_null); + pgvalue_t *new_value = pgvalue_create(new_datum, new_type, -1, fcinfo->fncollation, new_null); + pgvalue_t *old_value = pgvalue_create(old_datum, old_type, -1, fcinfo->fncollation, old_null); + if (table_name) pgvalue_ensure_detoast(table_name); + if (new_value) pgvalue_ensure_detoast(new_value); + if (old_value) pgvalue_ensure_detoast(old_value); + MemoryContextSwitchTo(old_ctx); + + if (!table_name || !new_value || !old_value) { + if (table_name) pgvalue_free(table_name); + if (new_value) pgvalue_free(new_value); + if (old_value) pgvalue_free(old_value); + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync_update_transfn failed to allocate values"))); + } + + if (!cloudsync_update_payload_append(payload, table_name, new_value, old_value)) { + if (table_name && payload->table_name != table_name) pgvalue_free(table_name); + if (new_value) pgvalue_free(new_value); + if (old_value) pgvalue_free(old_value); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cloudsync_update_transfn failed to append payload"))); + } + + PG_RETURN_POINTER(payload); +} + +PG_FUNCTION_INFO_V1(cloudsync_update_finalfn); +Datum cloudsync_update_finalfn (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + PG_RETURN_BOOL(true); + } + + cloudsync_update_payload *payload = (cloudsync_update_payload *)PG_GETARG_POINTER(0); + if (!payload || payload->count == 0) { + PG_RETURN_BOOL(true); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = NULL; + int rc = DBRES_OK; + bool spi_connected = false; + char buffer[1024]; + char buffer2[1024]; + size_t pklen = sizeof(buffer); + size_t oldpklen = sizeof(buffer2); + char *pk = NULL; + char *oldpk = NULL; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + spi_connected = true; + + PG_TRY(); + { + const char *table_name = database_value_text((dbvalue_t *)payload->table_name); + table = table_lookup(data, table_name); + if (!table) { + char meta_name[1024]; + snprintf(meta_name, sizeof(meta_name), "%s_cloudsync", table_name); + if (!database_table_exists(data, meta_name, cloudsync_schema(data))) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_update", table_name))); + } + + table_algo algo = dbutils_table_settings_get_algo(data, table_name); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + if (!table_add_to_context(data, algo, table_name)) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to load table context for %s", table_name))); + } + + table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Unable to retrieve table name %s in cloudsync_update", table_name))); + } + } + + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + int pk_count = table_count_pks(table); + if (payload->count < pk_count) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Not enough primary key values in cloudsync_update payload"))); + } + int max_expected = pk_count + table_count_cols(table); + if (payload->count > max_expected) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Too many values in cloudsync_update payload: got " + "%d expected <= %d", + payload->count, max_expected))); + } + + bool prikey_changed = false; + for (int i = 0; i < pk_count; i++) { + if (dbutils_value_compare((dbvalue_t *)payload->old_values[i], (dbvalue_t *)payload->new_values[i]) != 0) { + prikey_changed = true; + break; + } + } + + pk = pk_encode_prikey((dbvalue_t **)payload->new_values, pk_count, buffer, &pklen); + if (!pk) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Not enough memory to encode the primary key(s)"))); + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + pk = NULL; + ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("Update aborted because primary key in table %s contains NULL values", table_name))); + } + if (prikey_changed) { + oldpk = pk_encode_prikey((dbvalue_t **)payload->old_values, pk_count, buffer2, &oldpklen); + if (!oldpk) { + rc = DBRES_NOMEM; + goto cleanup; + } + + rc = local_mark_delete_meta(table, oldpk, oldpklen, db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + + rc = local_update_move_meta(table, pk, pklen, oldpk, oldpklen, db_version); + if (rc != DBRES_OK) goto cleanup; + + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + } + + for (int i = 0; i < table_count_cols(table); i++) { + int col_index = pk_count + i; + if (col_index >= payload->count) break; + + if (dbutils_value_compare((dbvalue_t *)payload->old_values[col_index], (dbvalue_t *)payload->new_values[col_index]) != 0) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: diff old and new text, emit per-block metadata changes + const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + // Read existing blocks from blocks table + block_list_t *old_blocks = block_list_create_empty(); + char *like_pattern = block_build_colname(col, "%"); + if (like_pattern && old_blocks) { + char *list_sql = cloudsync_memory_mprintf( + "SELECT col_name, col_value FROM %s WHERE pk = $1 AND col_name LIKE $2 ORDER BY col_name COLLATE \"C\"", + table_blocks_ref(table)); + if (list_sql) { + dbvm_t *list_vm = NULL; + if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { + databasevm_bind_blob(list_vm, 1, pk, (int)pklen); + databasevm_bind_text(list_vm, 2, like_pattern, -1); + while (databasevm_step(list_vm) == DBRES_ROW) { + const char *bcn = database_column_text(list_vm, 0); + const char *bval = database_column_text(list_vm, 1); + const char *pos = block_extract_position_id(bcn); + if (pos && old_blocks) { + block_list_add(old_blocks, bval ? bval : "", pos); + } + } + databasevm_finalize(list_vm); + } + cloudsync_memory_free(list_sql); + } + } + + // Split new text into parts (NULL text = all blocks removed) + block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); + if (new_blocks && old_blocks) { + // Build array of new content strings (NULL when count is 0) + const char **new_parts = NULL; + if (new_blocks->count > 0) { + new_parts = (const char **)cloudsync_memory_alloc( + (uint64_t)(new_blocks->count * sizeof(char *))); + if (new_parts) { + for (int b = 0; b < new_blocks->count; b++) { + new_parts[b] = new_blocks->entries[b].content; + } + } + } + + if (new_parts || new_blocks->count == 0) { + block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, + new_parts, new_blocks->count); + if (diff) { + for (int d = 0; d < diff->count; d++) { + block_diff_entry_t *de = &diff->entries[d]; + char *block_cn = block_build_colname(col, de->position_id); + if (!block_cn) continue; + + if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Store block value + if (rc == DBRES_OK && table_block_value_write_stmt(table)) { + dbvm_t *wvm = table_block_value_write_stmt(table); + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, de->content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + } else if (de->type == BLOCK_DIFF_REMOVED) { + // Mark block as deleted in metadata (even col_version) + rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Remove from blocks table + if (rc == DBRES_OK) { + block_delete_value_external(data, table, pk, pklen, block_cn); + } + } + cloudsync_memory_free(block_cn); + if (rc != DBRES_OK) break; + } + block_diff_free(diff); + } + if (new_parts) cloudsync_memory_free((void *)new_parts); + } + } + if (new_blocks) block_list_free(new_blocks); + if (old_blocks) block_list_free(old_blocks); + if (like_pattern) cloudsync_memory_free(like_pattern); + if (rc != DBRES_OK) goto cleanup; + } else { + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) goto cleanup; + } + } + } + +cleanup: + if (pk != buffer) cloudsync_memory_free(pk); + if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); + } + PG_CATCH(); + { + if (payload) { + cloudsync_update_payload_free(payload); + } + if (spi_connected) SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (payload) { + cloudsync_update_payload_free(payload); + } + if (spi_connected) SPI_finish(); + + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + } + + PG_RETURN_BOOL(true); +} + +// Placeholder - not implemented yet +PG_FUNCTION_INFO_V1(cloudsync_payload_encode); +Datum cloudsync_payload_encode (PG_FUNCTION_ARGS) { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cloudsync_payload_encode should not be called directly - use aggregate version"))); + PG_RETURN_NULL(); +} + +// MARK: - Schema - + +PG_FUNCTION_INFO_V1(pg_cloudsync_set_schema); +Datum pg_cloudsync_set_schema (PG_FUNCTION_ARGS) { + const char *schema = NULL; + + if (!PG_ARGISNULL(0)) { + schema = text_to_cstring(PG_GETARG_TEXT_PP(0)); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_set_schema(data, schema); + + // Persist schema to settings so it is restored on context re-initialization. + // Only persist if settings table exists (it may not exist before cloudsync_init). + int spi_rc = SPI_connect(); + if (spi_rc == SPI_OK_CONNECT) { + if (database_internal_table_exists(data, CLOUDSYNC_SETTINGS_NAME)) { + dbutils_settings_set_key_value(data, "schema", schema); + } + SPI_finish(); + } + + PG_RETURN_BOOL(true); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_schema); +Datum pg_cloudsync_schema (PG_FUNCTION_ARGS) { + cloudsync_context *data = get_cloudsync_context(); + const char *schema = cloudsync_schema(data); + + if (!schema) { + PG_RETURN_NULL(); + } + + PG_RETURN_TEXT_P(cstring_to_text(schema)); +} + +PG_FUNCTION_INFO_V1(pg_cloudsync_table_schema); +Datum pg_cloudsync_table_schema (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + cloudsync_context *data = get_cloudsync_context(); + const char *schema = cloudsync_table_schema(data, table_name); + + if (!schema) { + PG_RETURN_NULL(); + } + + PG_RETURN_TEXT_P(cstring_to_text(schema)); +} + +// MARK: - Changes - + +// Encode a single value using cloudsync pk encoding +static bytea *cloudsync_encode_value_from_datum (Datum val, Oid typeid, int32 typmod, Oid collation, bool isnull) { + pgvalue_t *v = pgvalue_create(val, typeid, typmod, collation, isnull); + if (!v) { + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync: failed to allocate value"))); + } + if (!isnull) { + pgvalue_ensure_detoast(v); + } + + size_t encoded_len = pk_encode_size((dbvalue_t **)&v, 1, 0, -1); + bytea *out = (bytea *)palloc(VARHDRSZ + encoded_len); + if (!out) { + pgvalue_free(v); + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("cloudsync: failed to allocate encoding buffer"))); + } + + pk_encode((dbvalue_t **)&v, 1, VARDATA(out), false, &encoded_len, -1); + SET_VARSIZE(out, VARHDRSZ + encoded_len); + + pgvalue_free(v); + return out; +} + +// Encode a NULL value using cloudsync pk encoding +static bytea *cloudsync_encode_null_value (void) { + return cloudsync_encode_value_from_datum((Datum)0, TEXTOID, -1, InvalidOid, true); +} + +// Hold a decoded pk-encoded value with its original type +typedef struct { + int dbtype; + int64_t ival; + double dval; + char *pval; + int64_t len; + bool isnull; +} cloudsync_decoded_value; + +// Decode a single pk-encoded value into a typed representation +static int cloudsync_decode_value_cb (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_decoded_value *out = (cloudsync_decoded_value *)xdata; + if (!out || index != 0) return DBRES_ERROR; + + out->dbtype = type; + out->isnull = false; + out->ival = 0; + out->dval = 0.0; + out->pval = NULL; + out->len = 0; + + switch (type) { + case DBTYPE_INTEGER: + out->ival = ival; + break; + case DBTYPE_FLOAT: + out->dval = dval; + break; + case DBTYPE_TEXT: + out->pval = pnstrdup(pval, (int)ival); + out->len = ival; + break; + case DBTYPE_BLOB: + if (ival > 0) { + out->pval = (char *)palloc((size_t)ival); + memcpy(out->pval, pval, (size_t)ival); + } + out->len = ival; + break; + case DBTYPE_NULL: + out->isnull = true; + break; + default: + return DBRES_ERROR; + } + return DBRES_OK; +} + +// Map a column Oid to the decoded type Oid that would be used for non-NULL values. +// This ensures NULL and non-NULL values use consistent types for SPI plan caching. +// The mapping must match pgvalue_dbtype() in pgvalue.c which determines encode/decode types. +// For example, INT4OID columns decode to INT8OID, UUIDOID columns decode to TEXTOID. +static Oid map_column_oid_to_decoded_oid(Oid col_oid) { + switch (col_oid) { + // Integer types → INT8OID (all integers decode to int64) + // Must match DBTYPE_INTEGER cases in pgvalue_dbtype() + case INT2OID: + case INT4OID: + case INT8OID: + case BOOLOID: // BOOLEAN encodes/decodes as INTEGER + case CHAROID: // "char" encodes/decodes as INTEGER + case OIDOID: // OID encodes/decodes as INTEGER + return INT8OID; + // Float types → FLOAT8OID (all floats decode to double) + // Must match DBTYPE_FLOAT cases in pgvalue_dbtype() + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return FLOAT8OID; + // Binary types → BYTEAOID + // Must match DBTYPE_BLOB cases in pgvalue_dbtype() + case BYTEAOID: + return BYTEAOID; + // All other types (text, varchar, uuid, json, date, timestamp, etc.) → TEXTOID + // These all encode/decode as DBTYPE_TEXT + default: + return TEXTOID; + } +} + +// Get the Oid of a column from the system catalog. +// Requires SPI to be connected. Returns InvalidOid if not found. +static Oid get_column_oid(const char *schema, const char *table_name, const char *column_name) { + if (!table_name || !column_name) return InvalidOid; + + const char *query = + "SELECT a.atttypid " + "FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid " + "LEFT JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = $1 " + "AND a.attname = $2 " + "AND a.attnum > 0 " + "AND NOT a.attisdropped " + "AND (n.nspname = $3 OR $3 IS NULL)"; + + Oid argtypes[3] = {TEXTOID, TEXTOID, TEXTOID}; + Datum values[3]; + char nulls[3] = {' ', ' ', schema ? ' ' : 'n'}; + + values[0] = CStringGetTextDatum(table_name); + values[1] = CStringGetTextDatum(column_name); + values[2] = schema ? CStringGetTextDatum(schema) : (Datum)0; + + int ret = SPI_execute_with_args(query, 3, argtypes, values, nulls, true, 1); + + pfree(DatumGetPointer(values[0])); + pfree(DatumGetPointer(values[1])); + if (schema) pfree(DatumGetPointer(values[2])); + + if (ret != SPI_OK_SELECT || SPI_processed == 0) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return InvalidOid; + } + + bool isnull; + Datum col_oid = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + Oid result = isnull ? InvalidOid : DatumGetObjectId(col_oid); + SPI_freetuptable(SPI_tuptable); + return result; +} + +// Decode encoded bytea into a pgvalue_t with the decoded base type. +// Type casting to the target column type is handled by the SQL statement. +static pgvalue_t *cloudsync_decode_bytea_to_pgvalue (bytea *encoded, bool *out_isnull) { + // Decode input guardrails. + if (out_isnull) *out_isnull = true; + if (!encoded) return NULL; + + // Decode bytea into C types with dbtype info. + cloudsync_decoded_value dv = {.isnull = true}; + int blen = (int)VARSIZE_ANY_EXHDR(encoded); + int decoded = pk_decode((char *)VARDATA_ANY(encoded), (size_t)blen, 1, NULL, -1, cloudsync_decode_value_cb, &dv); + if (decoded != 1) ereport(ERROR, (errmsg("cloudsync: failed to decode encoded value"))); + if (out_isnull) *out_isnull = dv.isnull; + if (dv.isnull) return NULL; + + // Map decoded C types into a PostgreSQL Datum with the base type. + // The SQL statement handles casting to the target column type via $n::typename. + Oid typoid = TEXTOID; + Datum datum; + + switch (dv.dbtype) { + case DBTYPE_INTEGER: + typoid = INT8OID; + datum = Int64GetDatum(dv.ival); + break; + case DBTYPE_FLOAT: + typoid = FLOAT8OID; + datum = Float8GetDatum(dv.dval); + break; + case DBTYPE_TEXT: { + typoid = TEXTOID; + Size tlen = dv.pval ? (Size)dv.len : 0; + text *t = (text *)palloc(VARHDRSZ + tlen); + SET_VARSIZE(t, VARHDRSZ + tlen); + if (tlen > 0) memmove(VARDATA(t), dv.pval, tlen); + datum = PointerGetDatum(t); + } break; + case DBTYPE_BLOB: { + typoid = BYTEAOID; + bytea *ba = (bytea *)palloc(VARHDRSZ + dv.len); + SET_VARSIZE(ba, VARHDRSZ + dv.len); + if (dv.len > 0) memcpy(VARDATA(ba), dv.pval, (size_t)dv.len); + datum = PointerGetDatum(ba); + } break; + case DBTYPE_NULL: + if (out_isnull) *out_isnull = true; + if (dv.pval) pfree(dv.pval); + return NULL; + default: + if (dv.pval) pfree(dv.pval); + ereport(ERROR, (errmsg("cloudsync: unsupported decoded type"))); + } + + if (dv.pval) pfree(dv.pval); + + return pgvalue_create(datum, typoid, -1, InvalidOid, false); +} + +PG_FUNCTION_INFO_V1(cloudsync_encode_value); +Datum cloudsync_encode_value(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + Oid typeoid = get_fn_expr_argtype(fcinfo->flinfo, 0); + int32 typmod = -1; + Oid collid = PG_GET_COLLATION(); + + if (!OidIsValid(typeoid) || typeoid == ANYELEMENTOID) { + if (fcinfo->flinfo->fn_expr && IsA(fcinfo->flinfo->fn_expr, FuncExpr)) { + FuncExpr *fexpr = (FuncExpr *) fcinfo->flinfo->fn_expr; + if (fexpr->args && list_length(fexpr->args) >= 1) { + Node *arg = (Node *) linitial(fexpr->args); + typeoid = exprType(arg); + typmod = exprTypmod(arg); + collid = exprCollation(arg); + } + } + } + + if (!OidIsValid(typeoid) || typeoid == ANYELEMENTOID) { + ereport(ERROR, (errmsg("cloudsync_encode_any: unable to resolve argument type"))); + } + + Datum val = PG_GETARG_DATUM(0); + bytea *result = cloudsync_encode_value_from_datum(val, typeoid, typmod, collid, false); + PG_RETURN_BYTEA_P(result); +} + +PG_FUNCTION_INFO_V1(cloudsync_col_value); +Datum cloudsync_col_value(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cloudsync_col_value arguments cannot be NULL"))); + } + + // argv[0] -> table name + // argv[1] -> column name + // argv[2] -> encoded pk + + char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *col_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); + bytea *encoded_pk = PG_GETARG_BYTEA_P(2); + + // check for special tombstone value + if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errmsg("Unable to retrieve table name %s in clousdsync_col_value.", table_name))); + } + + // Block column: if col_name contains \x1F, read from blocks table + if (block_is_block_colname(col_name) && table_has_block_cols(table)) { + dbvm_t *bvm = table_block_value_read_stmt(table); + if (!bvm) { + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + + bytea *encoded_pk_b = PG_GETARG_BYTEA_P(2); + size_t b_pk_len = (size_t)VARSIZE_ANY_EXHDR(encoded_pk_b); + int brc = databasevm_bind_blob(bvm, 1, VARDATA_ANY(encoded_pk_b), (uint64_t)b_pk_len); + if (brc != DBRES_OK) { databasevm_reset(bvm); ereport(ERROR, (errmsg("cloudsync_col_value block bind error"))); } + brc = databasevm_bind_text(bvm, 2, col_name, -1); + if (brc != DBRES_OK) { databasevm_reset(bvm); ereport(ERROR, (errmsg("cloudsync_col_value block bind error"))); } + + brc = databasevm_step(bvm); + if (brc == DBRES_ROW) { + size_t blob_len = 0; + const void *blob = database_column_blob(bvm, 0, &blob_len); + bytea *result = NULL; + if (blob && blob_len > 0) { + result = (bytea *)palloc(VARHDRSZ + blob_len); + SET_VARSIZE(result, VARHDRSZ + blob_len); + memcpy(VARDATA(result), blob, blob_len); + } + databasevm_reset(bvm); + if (result) PG_RETURN_BYTEA_P(result); + PG_RETURN_NULL(); + } else { + databasevm_reset(bvm); + bytea *null_encoded = cloudsync_encode_null_value(); + PG_RETURN_BYTEA_P(null_encoded); + } + } + + // extract the right col_value vm associated to the column name + dbvm_t *vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) { + ereport(ERROR, (errmsg("Unable to retrieve column value precompiled statement in clousdsync_col_value."))); + } + + // bind primary key values + size_t pk_len = (size_t)VARSIZE_ANY_EXHDR(encoded_pk); + int count = pk_decode_prikey((char *)VARDATA_ANY(encoded_pk), pk_len, pk_decode_bind_callback, (void *)vm); + if (count <= 0) { + ereport(ERROR, (errmsg("Unable to decode primary key value in clousdsync_col_value."))); + } + + // execute vm + int rc = databasevm_step(vm); + if (rc == DBRES_DONE) { + databasevm_reset(vm); + // row not found (RLS or genuinely missing) — return the RLS sentinel as bytea + const char *rls = CLOUDSYNC_RLS_RESTRICTED_VALUE; + size_t rls_len = strlen(rls); + bytea *result = (bytea *)palloc(VARHDRSZ + rls_len); + SET_VARSIZE(result, VARHDRSZ + rls_len); + memcpy(VARDATA(result), rls, rls_len); + PG_RETURN_BYTEA_P(result); + } else if (rc == DBRES_ROW) { + // copy value before reset invalidates SPI tuple memory + size_t blob_len = 0; + const void *blob = database_column_blob(vm, 0, &blob_len); + bytea *result = NULL; + if (blob && blob_len > 0) { + result = (bytea *)palloc(VARHDRSZ + blob_len); + SET_VARSIZE(result, VARHDRSZ + blob_len); + memcpy(VARDATA(result), blob, blob_len); + } + databasevm_reset(vm); + if (result) PG_RETURN_BYTEA_P(result); + PG_RETURN_NULL(); + } + + databasevm_reset(vm); + ereport(ERROR, (errmsg("cloudsync_col_value error: %s", cloudsync_errmsg(data)))); + PG_RETURN_NULL(); // unreachable, silences compiler +} + +// MARK: - Block-level LWW - + +PG_FUNCTION_INFO_V1(cloudsync_text_materialize); +Datum cloudsync_text_materialize (PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cloudsync_text_materialize: table_name and col_name cannot be NULL"))); + } + + const char *table_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + const char *col_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + cloudsync_context *data = get_cloudsync_context(); + cloudsync_pg_cleanup_state cleanup = {0}; + + int spi_rc = SPI_connect(); + if (spi_rc != SPI_OK_CONNECT) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("SPI_connect failed: %d", spi_rc))); + } + cleanup.spi_connected = true; + + PG_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + { + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Unable to retrieve table name %s in cloudsync_text_materialize", table_name))); + } + + int col_idx = table_col_index(table, col_name); + if (col_idx < 0 || table_col_algo(table, col_idx) != col_algo_block) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Column %s in table %s is not configured as block-level", col_name, table_name))); + } + + // Extract PK values from VARIADIC "any" (args starting from index 2) + cleanup.argv = pgvalues_from_args(fcinfo, 2, &cleanup.argc); + + // Normalize PK values to text for consistent encoding + pgvalues_normalize_to_text(cleanup.argv, cleanup.argc); + + int expected_pks = table_count_pks(table); + if (cleanup.argc != expected_pks) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Expected %d primary key values, got %d", expected_pks, cleanup.argc))); + } + + size_t pklen = sizeof(cleanup.pk_buffer); + cleanup.pk = pk_encode_prikey((dbvalue_t **)cleanup.argv, cleanup.argc, cleanup.pk_buffer, &pklen); + if (!cleanup.pk || cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + if (cleanup.pk == PRIKEY_NULL_CONSTRAINT_ERROR) cleanup.pk = NULL; + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Failed to encode primary key(s)"))); + } + + int rc = block_materialize_column(data, table, cleanup.pk, (int)pklen, col_name); + if (rc != DBRES_OK) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("%s", cloudsync_errmsg(data)))); + } + } + PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); + + cloudsync_pg_cleanup(0, PointerGetDatum(&cleanup)); + PG_RETURN_BOOL(true); +} + +// Track SRF execution state across calls +typedef struct { + Portal portal; + TupleDesc outdesc; + bool spi_connected; +} SRFState; + +// Build the UNION ALL SQL for cloudsync_changes SRF +static char * build_union_sql (void) { + char *result = NULL; + MemoryContext caller_ctx = CurrentMemoryContext; + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errmsg("cloudsync: SPI_connect failed"))); + } + + PG_TRY(); + { + const char *sql = + "SELECT n.nspname, c.relname " + "FROM pg_class c " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relkind = 'r' " + " AND n.nspname NOT IN ('pg_catalog','information_schema') " + " AND c.relname LIKE '%\\_cloudsync' ESCAPE '\\' " + "ORDER BY n.nspname, c.relname"; + + int rc = SPI_execute(sql, true, 0); + if (rc != SPI_OK_SELECT || !SPI_tuptable) { + ereport(ERROR, (errmsg("cloudsync: SPI_execute failed while listing *_cloudsync"))); + } + + StringInfoData buf; + initStringInfo(&buf); + + uint64 ntables = SPI_processed; + char **nsp_list = NULL; + char **rel_list = NULL; + if (ntables > 0) { + nsp_list = (char **)palloc0(sizeof(char *) * ntables); + rel_list = (char **)palloc0(sizeof(char *) * ntables); + } + for (uint64 i = 0; i < ntables; i++) { + HeapTuple tup = SPI_tuptable->vals[i]; + TupleDesc td = SPI_tuptable->tupdesc; + char *nsp = SPI_getvalue(tup, td, 1); + char *rel = SPI_getvalue(tup, td, 2); + if (!nsp || !rel) { + if (nsp) pfree(nsp); + if (rel) pfree(rel); + continue; + } + nsp_list[i] = pstrdup(nsp); + rel_list[i] = pstrdup(rel); + pfree(nsp); + pfree(rel); + } + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + + bool first = true; + for (uint64 i = 0; i < ntables; i++) { + char *nsp = nsp_list ? nsp_list[i] : NULL; + char *rel = rel_list ? rel_list[i] : NULL; + if (!nsp || !rel) { + if (nsp) pfree(nsp); + if (rel) pfree(rel); + continue; + } + + size_t rlen = strlen(rel); + if (rlen <= 10) {pfree(nsp); pfree(rel); continue;} /* "_cloudsync" */ + + char *base = pstrdup(rel); + base[rlen - 10] = '\0'; + + char *quoted_base = quote_literal_cstr(base); + const char *quoted_nsp = quote_identifier(nsp); + const char *quoted_rel = quote_identifier(rel); + + if (!first) appendStringInfoString(&buf, " UNION ALL "); + first = false; + + + /* + * Build a single SELECT per table that: + * - reads change rows from
_cloudsync (t1) + * - joins the base table (b) using decoded PK components + * - computes col_value in-SQL with a CASE over col_name + * + * This avoids calling cloudsync_col_value() (and therefore avoids + * executing extra SPI queries per row), while still honoring RLS: + * if the base row is not visible, the LEFT JOIN yields NULL and we + * return the restricted sentinel value (then filtered out). + */ + + char *nsp_lit = quote_literal_cstr(nsp); + char *base_lit = quote_literal_cstr(base); + + /* Collect PK columns (name + SQL type) */ + StringInfoData pkq; + initStringInfo(&pkq); + appendStringInfo(&pkq, + "SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS typ " + "FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(i.indkey) " + "WHERE i.indisprimary AND n.nspname = %s AND c.relname = %s " + "ORDER BY array_position(i.indkey, a.attnum)", + nsp_lit, base_lit + ); + int pkrc = SPI_execute(pkq.data, true, 0); + pfree(pkq.data); + if (pkrc != SPI_OK_SELECT || (SPI_processed == 0) || (!SPI_tuptable)) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: unable to resolve primary key for %s.%s", nsp, base))); + } + uint64 npk = SPI_processed; + + StringInfoData joincond; + initStringInfo(&joincond); + for (uint64 k = 0; k < npk; k++) { + HeapTuple pkt = SPI_tuptable->vals[k]; + TupleDesc pkd = SPI_tuptable->tupdesc; + char *pkname = SPI_getvalue(pkt, pkd, 1); + char *pktype = SPI_getvalue(pkt, pkd, 2); + if (!pkname || !pktype) { + if (pkname) pfree(pkname); + if (pktype) pfree(pktype); + pfree(joincond.data); + SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: invalid pk metadata for %s.%s", nsp, base))); + } + + if (k > 0) appendStringInfoString(&joincond, " AND "); + appendStringInfo(&joincond, + "b.%s = cloudsync_pk_decode(t1.pk, %llu)::%s", + quote_identifier(pkname), + (unsigned long long)(k + 1), + pktype + ); + pfree(pkname); + pfree(pktype); + } + SPI_freetuptable(SPI_tuptable); + + // Check if blocks table exists for this table + char blocks_tbl_name[1024]; + snprintf(blocks_tbl_name, sizeof(blocks_tbl_name), "%s_cloudsync_blocks", base); + StringInfoData btq; + initStringInfo(&btq); + appendStringInfo(&btq, + "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relname = %s AND n.nspname = %s AND c.relkind = 'r'", + quote_literal_cstr(blocks_tbl_name), nsp_lit); + int btrc = SPI_execute(btq.data, true, 1); + bool has_blocks_table = (btrc == SPI_OK_SELECT && SPI_processed > 0); + if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; } + pfree(btq.data); + + /* Collect all base-table columns to build CASE over t1.col_name */ + StringInfoData colq; + initStringInfo(&colq); + appendStringInfo(&colq, + "SELECT a.attname " + "FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE a.attnum > 0 AND NOT a.attisdropped " + " AND n.nspname = %s AND c.relname = %s " + "ORDER BY a.attnum", + nsp_lit, base_lit + ); + int colrc = SPI_execute(colq.data, true, 0); + pfree(colq.data); + if (colrc != SPI_OK_SELECT || !SPI_tuptable) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + ereport(ERROR, (errmsg("cloudsync: unable to resolve columns for %s.%s", nsp, base))); + } + uint64 ncols = SPI_processed; + + StringInfoData caseexpr; + initStringInfo(&caseexpr); + appendStringInfoString(&caseexpr, + "CASE " + "WHEN t1.col_name = '" CLOUDSYNC_TOMBSTONE_VALUE "' THEN " CLOUDSYNC_NULL_VALUE_BYTEA " " + "WHEN b.ctid IS NULL THEN " CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA " " + ); + if (has_blocks_table) { + appendStringInfo(&caseexpr, + "WHEN t1.col_name LIKE '%%' || chr(31) || '%%' THEN " + "(SELECT cloudsync_encode_value(blk.col_value) FROM %s.\"%s_cloudsync_blocks\" blk " + "WHERE blk.pk = t1.pk AND blk.col_name = t1.col_name) ", + quote_identifier(nsp), base); + } + appendStringInfoString(&caseexpr, + "ELSE CASE t1.col_name " + ); + + for (uint64 k = 0; k < ncols; k++) { + HeapTuple ct = SPI_tuptable->vals[k]; + TupleDesc cd = SPI_tuptable->tupdesc; + char *cname = SPI_getvalue(ct, cd, 1); + if (!cname) continue; + + appendStringInfo(&caseexpr, + "WHEN %s THEN cloudsync_encode_value(b.%s) ", + quote_literal_cstr(cname), + quote_identifier(cname) + ); + pfree(cname); + } + SPI_freetuptable(SPI_tuptable); + + appendStringInfoString(&caseexpr, + "ELSE " CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA " END END" + ); + + const char *quoted_base_ident = quote_identifier(base); + + appendStringInfo(&buf, + "SELECT * FROM (" + "SELECT %s AS tbl, t1.pk, t1.col_name, " + "%s AS col_value, " + "t1.col_version, t1.db_version, site_tbl.site_id, " + "COALESCE(t2.col_version, 1) AS cl, t1.seq " + "FROM %s.%s t1 " + "LEFT JOIN cloudsync_site_id site_tbl ON t1.site_id = site_tbl.id " + "LEFT JOIN %s.%s t2 " + " ON t1.pk = t2.pk AND t2.col_name = '%s' " + "LEFT JOIN %s.%s b ON %s " + ") s WHERE s.col_value IS DISTINCT FROM %s", + quoted_base, + caseexpr.data, + quoted_nsp, quoted_rel, + quoted_nsp, quoted_rel, + CLOUDSYNC_TOMBSTONE_VALUE, + quoted_nsp, quoted_base_ident, + joincond.data, + CLOUDSYNC_RLS_RESTRICTED_VALUE_BYTEA + ); + + // Only free quoted identifiers if they're different from the input + // (quote_identifier returns input pointer if no quoting needed) + if (quoted_base_ident != base) pfree((void*)quoted_base_ident); + pfree(joincond.data); + pfree(caseexpr.data); + + pfree(base); + pfree(base_lit); + + pfree(quoted_base); + pfree(nsp_lit); + bool nsp_was_quoted = (quoted_nsp != nsp); + pfree(nsp); + if (nsp_was_quoted) pfree((void *)quoted_nsp); + bool rel_was_quoted = (quoted_rel != rel); + pfree(rel); + if (rel_was_quoted) pfree((void *)quoted_rel); + } + if (nsp_list) pfree(nsp_list); + if (rel_list) pfree(rel_list); + + // Ensure result survives SPI_finish by allocating in the caller context. + MemoryContext old_ctx = MemoryContextSwitchTo(caller_ctx); + if (first) { + result = pstrdup( + "SELECT NULL::text AS tbl, NULL::bytea AS pk, NULL::text AS col_name, NULL::bytea AS col_value, " + "NULL::bigint AS col_version, NULL::bigint AS db_version, NULL::bytea AS site_id, " + "NULL::bigint AS cl, NULL::bigint AS seq WHERE false" + ); + } else { + result = pstrdup(buf.data); + } + MemoryContextSwitchTo(old_ctx); + + SPI_finish(); + } + PG_CATCH(); + { + SPI_finish(); + PG_RE_THROW(); + } + PG_END_TRY(); + + return result; +} + +PG_FUNCTION_INFO_V1(cloudsync_changes_select); +Datum cloudsync_changes_select(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + SRFState *st_local = NULL; + bool spi_connected_local = false; + + PG_TRY(); + { + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + int64 min_db_version = PG_GETARG_INT64(0); + bool site_is_null = PG_ARGISNULL(1); + bytea *filter_site_id = site_is_null ? NULL : PG_GETARG_BYTEA_PP(1); + + char *union_sql = build_union_sql(); + + StringInfoData q; + initStringInfo(&q); + appendStringInfo(&q, + "SELECT tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq " + "FROM ( %s ) u " + "WHERE db_version > $1 " + " AND ($2 IS NULL OR site_id = $2) " + "ORDER BY db_version, seq ASC", + union_sql + ); + + if (SPI_connect() != SPI_OK_CONNECT) { + ereport(ERROR, (errmsg("cloudsync: SPI_connect failed in SRF"))); + } + spi_connected_local = true; + + Oid argtypes[2] = {INT8OID, BYTEAOID}; + Datum values[2]; + char nulls[2] = {' ', ' '}; + + values[0] = Int64GetDatum(min_db_version); + if (site_is_null) { nulls[1] = 'n'; values[1] = (Datum)0; } + else values[1] = PointerGetDatum(filter_site_id); + + Portal portal = SPI_cursor_open_with_args(NULL, q.data, 2, argtypes, values, nulls, true, 0); + if (!portal) { + ereport(ERROR, (errmsg("cloudsync: SPI_cursor_open failed in SRF"))); + } + + TupleDesc outdesc; + if (get_call_result_type(fcinfo, NULL, &outdesc) != TYPEFUNC_COMPOSITE) { + ereport(ERROR, (errmsg("cloudsync: return type must be composite"))); + } + outdesc = BlessTupleDesc(outdesc); + + SRFState *st = palloc0(sizeof(SRFState)); + st->portal = portal; + st->outdesc = outdesc; + st->spi_connected = true; + funcctx->user_fctx = st; + st_local = st; + + pfree(union_sql); + pfree(q.data); + + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + SRFState *st = (SRFState *) funcctx->user_fctx; + st_local = st; + + SPI_cursor_fetch(st->portal, true, 1); + if (SPI_processed == 0) { + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + SPI_cursor_close(st->portal); + st->portal = NULL; + + SPI_finish(); + st->spi_connected = false; + + // SPI operations may leave us in multi_call_memory_ctx + // Must switch to a safe context before SRF_RETURN_DONE deletes it + MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + + SRF_RETURN_DONE(funcctx); + } + + HeapTuple tup = SPI_tuptable->vals[0]; + TupleDesc td = SPI_tuptable->tupdesc; + + Datum outvals[9]; + bool outnulls[9]; + for (int i = 0; i < 9; i++) { + outvals[i] = SPI_getbinval(tup, td, i+1, &outnulls[i]); + if (!outnulls[i]) { + Form_pg_attribute att = TupleDescAttr(td, i); + outvals[i] = datumCopy(outvals[i], att->attbyval, att->attlen); + } + } + + HeapTuple outtup = heap_form_tuple(st->outdesc, outvals, outnulls); + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(outtup)); + } + PG_CATCH(); + { + // Switch to function's context (safe, won't be deleted) + // Avoids assertion if we're currently in multi_call_memory_ctx + MemoryContextSwitchTo(fcinfo->flinfo->fn_mcxt); + + if (st_local && st_local->portal) { + SPI_cursor_close(st_local->portal); + st_local->portal = NULL; + } + + if (st_local && st_local->spi_connected) { + SPI_finish(); + st_local->spi_connected = false; + spi_connected_local = false; + } else if (spi_connected_local) { + SPI_finish(); + spi_connected_local = false; + } + + PG_RE_THROW(); + } + PG_END_TRY(); +} + +// Trigger INSERT + +PG_FUNCTION_INFO_V1(cloudsync_changes_insert_trigger); +Datum cloudsync_changes_insert_trigger (PG_FUNCTION_ARGS) { + // sanity check + bool spi_connected = false; + TriggerData *trigdata = (TriggerData *) fcinfo->context; + if (!CALLED_AS_TRIGGER(fcinfo)) ereport(ERROR, (errmsg("cloudsync_changes_insert_trigger must be called as trigger"))); + if (!TRIGGER_FIRED_BY_INSERT(trigdata->tg_event)) ereport(ERROR, (errmsg("Only INSERT allowed on cloudsync_changes"))); + + HeapTuple newtup = trigdata->tg_trigtuple; + pgvalue_t *col_value = NULL; + PG_TRY(); + { + TupleDesc desc = trigdata->tg_relation->rd_att; + bool isnull; + + char *insert_tbl = text_to_cstring((text*) DatumGetPointer(heap_getattr(newtup, 1, desc, &isnull))); + if (isnull) ereport(ERROR, (errmsg("tbl cannot be NULL"))); + + bytea *insert_pk = (bytea*) DatumGetPointer(heap_getattr(newtup, 2, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("pk cannot be NULL"))); + int insert_pk_len = (int)(VARSIZE_ANY_EXHDR(insert_pk)); + + Datum insert_name_datum = heap_getattr(newtup, 3, desc, &isnull); + char *insert_name = NULL; + bool insert_name_owned = false; + if (isnull) { + insert_name = CLOUDSYNC_TOMBSTONE_VALUE; + } else { + insert_name = text_to_cstring((text*) DatumGetPointer(insert_name_datum)); + insert_name_owned = true; + } + bool is_tombstone = (strcmp(insert_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0); + + // raw_insert_value is declared as bytea in the view (cloudsync-encoded value) + bytea *insert_value_encoded = (bytea*) DatumGetPointer(heap_getattr(newtup, 4, desc, &isnull)); + + int64 insert_col_version = DatumGetInt64(heap_getattr(newtup, 5, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("col_version cannot be NULL"))); + + int64 insert_db_version = DatumGetInt64(heap_getattr(newtup, 6, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("db_version cannot be NULL"))); + + bytea *insert_site_id = (bytea*) DatumGetPointer(heap_getattr(newtup, 7, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("site_id cannot be NULL"))); + int insert_site_id_len = (int)(VARSIZE_ANY_EXHDR(insert_site_id)); + + int64 insert_cl = DatumGetInt64(heap_getattr(newtup, 8, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("cl cannot be NULL"))); + + int64 insert_seq = DatumGetInt64(heap_getattr(newtup, 9, desc, &isnull)); + if (isnull) ereport(ERROR, (errmsg("seq cannot be NULL"))); + + // lookup algo in cloudsync_tables + cloudsync_context *data = get_cloudsync_context(); + cloudsync_table_context *table = table_lookup(data, insert_tbl); + if (!table) ereport(ERROR, (errmsg("Unable to find table"))); + + if (SPI_connect() != SPI_OK_CONNECT) ereport(ERROR, (errmsg("cloudsync: SPI_connect failed in trigger"))); + spi_connected = true; + + // Decode value to base type; SQL statement handles type casting via $n::typename. + // For non-NULL values, we get the decoded base type (INT8OID for integers, TEXTOID for text/UUID, etc). + // For NULL values, we must use the SAME decoded type that non-NULL values would use. + // This ensures type consistency across all calls, as SPI caches parameter types on first prepare. + if (!is_tombstone) { + bool value_is_null = false; + col_value = cloudsync_decode_bytea_to_pgvalue(insert_value_encoded, &value_is_null); + + // When value is NULL, create a typed NULL pgvalue with the decoded type. + // We map the column's actual Oid to the corresponding decoded Oid (e.g., INT4OID → INT8OID). + if (!col_value && value_is_null) { + Oid col_oid = get_column_oid(table_schema(table), insert_tbl, insert_name); + if (OidIsValid(col_oid)) { + Oid decoded_oid = map_column_oid_to_decoded_oid(col_oid); + col_value = pgvalue_create((Datum)0, decoded_oid, -1, InvalidOid, true); + } + } + } + + int rc = DBRES_OK; + int64_t rowid = 0; + if (table_algo_isgos(table)) { + rc = merge_insert_col(data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_name, col_value, (int64_t)insert_col_version, (int64_t)insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, (int64_t)insert_seq, &rowid); + } else { + rc = merge_insert (data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_cl, insert_name, col_value, insert_col_version, insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, insert_seq, &rowid); + } + if (rc != DBRES_OK) { + ereport(ERROR, (errmsg("Error during merge_insert: %s", database_errmsg(data)))); + } + + pgvalue_free(col_value); + pfree(insert_tbl); + if (insert_name_owned) pfree(insert_name); + + SPI_finish(); + spi_connected = false; + } + PG_CATCH(); + { + pgvalue_free(col_value); + if (spi_connected) { + SPI_finish(); + spi_connected = false; + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return PointerGetDatum(newtup); +} diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c new file mode 100644 index 0000000..3fc6310 --- /dev/null +++ b/src/postgresql/database_postgresql.c @@ -0,0 +1,3076 @@ +// +// database_postgresql.c +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +// PostgreSQL requires postgres.h to be included FIRST +// It sets up the entire environment including platform compatibility +#include "postgres.h" + +#include +#include +#include + +#include "../cloudsync.h" +#include "../database.h" +#include "../dbutils.h" +#include "../sql.h" +#include "../utils.h" + +// PostgreSQL SPI and other headers +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" + +#include "pgvalue.h" + +// ============================================================================ +// SPI CONNECTION REQUIREMENTS +// ============================================================================ +// +// IMPORTANT: This implementation requires an active SPI connection to function. +// The Extension Function that calls these functions MUST: +// +// 1. Call SPI_connect() before using any database functions +// 2. Call SPI_finish() before returning from the extension function +// +// ============================================================================ + +// MARK: - PREPARED STATEMENTS - + +// PostgreSQL SPI handles require knowing parameter count and types upfront. +// Solution: Defer actual SPI_prepare until first step(), after all bindings are set. +#define MAX_PARAMS 32 + +typedef struct { + // Prepared plan + SPIPlanPtr plan; + bool plan_is_prepared; + + // Cursor execution + Portal portal; // owned by statement + bool portal_open; + + // Current fetched batch (we fetch 1 row at a time, but SPI still returns a tuptable) + SPITupleTable *last_tuptable; // must SPI_freetuptable() before next fetch + HeapTuple current_tuple; + TupleDesc current_tupdesc; + + // Params + int nparams; + Oid types[MAX_PARAMS]; + Oid prepared_types[MAX_PARAMS]; // types used when plan was SPI_prepare'd + int prepared_nparams; // nparams at prepare time + Datum values[MAX_PARAMS]; + char nulls[MAX_PARAMS]; + bool executed_nonselect; // non-select executed already + + // Memory + MemoryContext stmt_mcxt; // lifetime = pg_stmt_t + MemoryContext bind_mcxt; // resettable region for parameters (cleared on clear_bindings/reset) + MemoryContext row_mcxt; // per-row scratch (cleared each step after consumer copies) + + // Context + const char *sql; + cloudsync_context *data; +} pg_stmt_t; + +static int database_refresh_snapshot (void); + +// MARK: - SQL - + +static char *sql_escape_character (const char *name, char *buffer, size_t bsize, char c) { + if (!name || !buffer || bsize < 1) { + if (buffer && bsize > 0) buffer[0] = '\0'; + return NULL; + } + + size_t i = 0, j = 0; + + while (name[i]) { + if (name[i] == c) { + // Need space for 2 chars (escaped c) + null + if (j >= bsize - 2) { + elog(WARNING, "Identifier name too long for buffer, truncated: %s", name); + break; + } + buffer[j++] = c; + buffer[j++] = c; + } else { + // Need space for 1 char + null + if (j >= bsize - 1) { + elog(WARNING, "Identifier name too long for buffer, truncated: %s", name); + break; + } + buffer[j++] = name[i]; + } + i++; + } + + buffer[j] = '\0'; + return buffer; +} + +static char *sql_escape_identifier (const char *name, char *buffer, size_t bsize) { + // PostgreSQL identifier escaping: double any embedded double quotes + // Does NOT add surrounding quotes (caller's responsibility) + // Similar to SQLite's %q behavior for escaping + return sql_escape_character(name, buffer, bsize, '"'); +} + +static char *sql_escape_literal (const char *name, char *buffer, size_t bsize) { + // Escapes single quotes for use inside SQL string literals: ' → '' + // Does NOT add surrounding quotes (caller's responsibility) + return sql_escape_character(name, buffer, bsize, '\''); +} + +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta) { + // Escape the table name (doubles any embedded quotes) + char escaped[512]; + sql_escape_identifier(table_name, escaped, sizeof(escaped)); + + // Add the surrounding quotes in the format string + if (is_meta) { + snprintf(buffer, bsize, "DROP TABLE IF EXISTS \"%s_cloudsync\";", escaped); + } else { + snprintf(buffer, bsize, "DROP TABLE IF EXISTS \"%s\";", escaped); + } + + return buffer; +} + +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_PK, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_DELETE_ROW_BY_PK, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_INSERT_PK_IGNORE, qualified); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_UPSERT_PK_AND_COL, qualified, colname, colname); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + if (ncolnames <= 0 || !colnames) return NULL; + + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + // Build VALUES list for column names: ('col_a',1),('col_b',2) + // Column names are SQL literals here, so escape single quotes + size_t values_cap = (size_t)ncolnames * 128 + 1; + char *col_values = cloudsync_memory_alloc(values_cap); + if (!col_values) { cloudsync_memory_free(qualified); return NULL; } + + size_t vpos = 0; + for (int i = 0; i < ncolnames; i++) { + char esc[1024]; + sql_escape_literal(colnames[i], esc, sizeof(esc)); + vpos += snprintf(col_values + vpos, values_cap - vpos, "%s('%s'::text,%d)", + i > 0 ? "," : "", esc, i + 1); + } + + // Build meta-query that generates the final INSERT...ON CONFLICT SQL with proper types + char *meta_sql = cloudsync_memory_mprintf( + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (SELECT count(*) AS n FROM pk), " + "cols AS (" + " SELECT u.colname, format_type(a.atttypid, a.atttypmod) AS coltype, u.ord " + " FROM (VALUES %s) AS u(colname, ord) " + " JOIN pg_attribute a ON a.attrelid = (SELECT oid FROM tbl) AND a.attname = u.colname " + " WHERE a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',' ORDER BY ord) FROM pk)" + " || ',' || (SELECT string_agg(format('%%I', colname), ',' ORDER BY ord) FROM cols) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',' ORDER BY ord) FROM pk)" + " || ',' || (SELECT string_agg(format('$%%s::%%s', (SELECT n FROM pk_count) + ord, coltype), ',' ORDER BY ord) FROM cols) || ')'" + " || ' ON CONFLICT (' || (SELECT string_agg(format('%%I', attname), ',' ORDER BY ord) FROM pk) || ')'" + " || ' DO UPDATE SET ' || (SELECT string_agg(format('%%I=EXCLUDED.%%I', colname, colname), ',' ORDER BY ord) FROM cols)" + " || ';';", + qualified, col_values + ); + + cloudsync_memory_free(qualified); + cloudsync_memory_free(col_values); + if (!meta_sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, meta_sql, &query); + cloudsync_memory_free(meta_sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + if (ncolnames <= 0 || !colnames) return NULL; + + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + // Build VALUES list for column names: ('col_a',1),('col_b',2) + size_t values_cap = (size_t)ncolnames * 128 + 1; + char *col_values = cloudsync_memory_alloc(values_cap); + if (!col_values) { cloudsync_memory_free(qualified); return NULL; } + + size_t vpos = 0; + for (int i = 0; i < ncolnames; i++) { + char esc[1024]; + sql_escape_literal(colnames[i], esc, sizeof(esc)); + vpos += snprintf(col_values + vpos, values_cap - vpos, "%s('%s'::text,%d)", + i > 0 ? "," : "", esc, i + 1); + } + + // Build meta-query that generates UPDATE ... SET col=$ WHERE pk=$ + char *meta_sql = cloudsync_memory_mprintf( + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (SELECT count(*) AS n FROM pk), " + "cols AS (" + " SELECT u.colname, format_type(a.atttypid, a.atttypmod) AS coltype, u.ord " + " FROM (VALUES %s) AS u(colname, ord) " + " JOIN pg_attribute a ON a.attrelid = (SELECT oid FROM tbl) AND a.attname = u.colname " + " WHERE a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'UPDATE ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' SET ' || (SELECT string_agg(format('%%I=$%%s::%%s', colname, (SELECT n FROM pk_count) + ord, coltype), ',' ORDER BY ord) FROM cols)" + " || ' WHERE ' || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';", + qualified, col_values + ); + + cloudsync_memory_free(qualified); + cloudsync_memory_free(col_values); + if (!meta_sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, meta_sql, &query); + cloudsync_memory_free(meta_sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(data); + char *qualified = database_build_base_ref(schema, table_name); + if (!qualified) return NULL; + + char *sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_COLS_BY_PK_FMT, qualified, colname); + cloudsync_memory_free(qualified); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col) { + char *meta_ref = database_build_meta_ref(cloudsync_schema(data), table_name); + if (!meta_ref) return NULL; + + char *result = cloudsync_memory_mprintf(SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL, meta_ref, except_col, meta_ref, meta_ref, except_col); + cloudsync_memory_free(meta_ref); + return result; +} + +char *database_table_schema (const char *table_name) { + if (!table_name) return NULL; + + // Build metadata table name + char meta_table[256]; + snprintf(meta_table, sizeof(meta_table), "%s_cloudsync", table_name); + + // Query system catalogs to find the schema of the metadata table. + // Rationale: The metadata table is created in the same schema as the base table, + // so finding its location tells us which schema the table belongs to. + const char *query = + "SELECT n.nspname " + "FROM pg_class c " + "JOIN pg_namespace n ON c.relnamespace = n.oid " + "WHERE c.relname = $1 " + "AND c.relkind = 'r'"; // 'r' = ordinary table + + char *schema = NULL; + + if (SPI_connect() != SPI_OK_CONNECT) { + return NULL; + } + + Oid argtypes[1] = {TEXTOID}; + Datum values[1] = {CStringGetTextDatum(meta_table)}; + char nulls[1] = {' '}; + + int rc = SPI_execute_with_args(query, 1, argtypes, values, nulls, true, 1); + + if (rc == SPI_OK_SELECT && SPI_processed > 0) { + TupleDesc tupdesc = SPI_tuptable->tupdesc; + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + + Datum datum = SPI_getbinval(tuple, tupdesc, 1, &isnull); + if (!isnull) { + // pg_namespace.nspname is type 'name', not 'text' + Name nspname = DatumGetName(datum); + schema = cloudsync_string_dup(NameStr(*nspname)); + } + } + + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + pfree(DatumGetPointer(values[0])); + SPI_finish(); + + // Returns NULL if metadata table doesn't exist yet (during initialization). + // Caller should fall back to cloudsync_schema() in this case. + return schema; +} + +char *database_build_meta_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s_cloudsync\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s_cloudsync\"", escaped_table); +} + +char *database_build_base_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s\"", escaped_table); +} + +char *database_build_blocks_ref (const char *schema, const char *table_name) { + char escaped_table[512]; + sql_escape_identifier(table_name, escaped_table, sizeof(escaped_table)); + if (schema) { + char escaped_schema[512]; + sql_escape_identifier(schema, escaped_schema, sizeof(escaped_schema)); + return cloudsync_memory_mprintf("\"%s\".\"%s_cloudsync_blocks\"", escaped_schema, escaped_table); + } + return cloudsync_memory_mprintf("\"%s_cloudsync_blocks\"", escaped_table); +} + +// Schema-aware SQL builder for PostgreSQL: deletes columns not in schema or pkcol. +// Schema parameter: pass empty string to fall back to current_schema() via SQL. +char *sql_build_delete_cols_not_in_schema_query (const char *schema, const char *table_name, const char *meta_ref, const char *pkcol) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "DELETE FROM %s WHERE col_name NOT IN (" + "SELECT column_name FROM information_schema.columns WHERE table_name = '%s' " + "AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "UNION SELECT '%s'" + ");", + meta_ref, esc_table, esc_schema, pkcol + ); +} + +// Builds query to get comma-separated list of primary key column names. +char *sql_build_pk_collist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(quote_ident(column_name), ',') " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", + esc_table, esc_schema + ); +} + +// Builds query to get SELECT list of decoded primary key columns. +char *sql_build_pk_decode_selectlist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(" + "'cloudsync_pk_decode(pk, ' || ordinal_position || ') AS ' || quote_ident(column_name), ',' ORDER BY ordinal_position" + ") " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", + esc_table, esc_schema + ); +} + +// Builds query to get qualified (schema.table.column) primary key column list. +char *sql_build_pk_qualified_collist_query (const char *schema, const char *table_name) { + const char *schema_param = schema ? schema : ""; + + char esc_table[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + return cloudsync_memory_mprintf( + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND constraint_name LIKE '%%_pkey';", esc_table, esc_schema + ); +} + +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, + const char *pkvalues_identifiers, + const char *base_ref, const char *meta_ref, + const char *filter) { + UNUSED_PARAMETER(schema); + + char esc_table[1024]; + sql_escape_literal(table_name, esc_table, sizeof(esc_table)); + + // PostgreSQL: Use NOT EXISTS with cloudsync_pk_encode to avoid EXCEPT type mismatch. + // + // CRITICAL: Pass PK columns directly to VARIADIC functions (NOT wrapped in ARRAY[]). + // This preserves each column's actual type (TEXT, INTEGER, etc.) for correct pk_encode. + // Using ARRAY[] would require all elements to be the same type, causing errors with + // mixed-type composite PKs (e.g., TEXT + INTEGER). + // + // Example: cloudsync_insert('table', col1, col2) where col1=TEXT, col2=INTEGER + // PostgreSQL's VARIADIC handling preserves each type and matches SQLite's encoding. + if (filter) { + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%s', %s) " + "FROM %s b " + "WHERE (%s) AND NOT EXISTS (" + " SELECT 1 FROM %s m WHERE m.pk = cloudsync_pk_encode(%s)" + ");", + esc_table, pkvalues_identifiers, base_ref, filter, meta_ref, pkvalues_identifiers + ); + } + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%s', %s) " + "FROM %s b " + "WHERE NOT EXISTS (" + " SELECT 1 FROM %s m WHERE m.pk = cloudsync_pk_encode(%s)" + ");", + esc_table, pkvalues_identifiers, base_ref, meta_ref, pkvalues_identifiers + ); +} + +// MARK: - HELPER FUNCTIONS - + +// Map SPI result codes to DBRES +static int map_spi_result (int rc) { + switch (rc) { + case SPI_OK_SELECT: + case SPI_OK_INSERT: + case SPI_OK_UPDATE: + case SPI_OK_DELETE: + case SPI_OK_UTILITY: + return DBRES_OK; + case SPI_OK_INSERT_RETURNING: + case SPI_OK_UPDATE_RETURNING: + case SPI_OK_DELETE_RETURNING: + return DBRES_ROW; + default: + return DBRES_ERROR; + } +} + +static void clear_fetch_batch (pg_stmt_t *stmt) { + if (!stmt) return; + if (stmt->last_tuptable) { + SPI_freetuptable(stmt->last_tuptable); + stmt->last_tuptable = NULL; + } + stmt->current_tuple = NULL; + stmt->current_tupdesc = NULL; + if (stmt->row_mcxt) MemoryContextReset(stmt->row_mcxt); +} + +static void close_portal (pg_stmt_t *stmt) { + if (!stmt) return; + + // Always clear portal_open first to maintain consistent state + stmt->portal_open = false; + + if (!stmt->portal) return; + + PG_TRY(); + { + SPI_cursor_close(stmt->portal); + } + PG_CATCH(); + { + // Log but don't throw - we're cleaning up + FlushErrorState(); + } + PG_END_TRY(); + stmt->portal = NULL; +} + +static inline Datum get_datum (pg_stmt_t *stmt, int col /* 0-based */, bool *isnull, Oid *type) { + if (!stmt || !stmt->current_tuple || !stmt->current_tupdesc) { + if (isnull) *isnull = true; + if (type) *type = 0; + return (Datum) 0; + } + if (type) *type = SPI_gettypeid(stmt->current_tupdesc, col + 1); + return SPI_getbinval(stmt->current_tuple, stmt->current_tupdesc, col + 1, isnull); +} + +// MARK: - PRIVATE - + +int database_select1_value (cloudsync_context *data, const char *sql, char **ptr_value, int64_t *int_value, DBTYPE expected_type) { + cloudsync_reset_error(data); + + // init values and sanity check expected_type + if (ptr_value) *ptr_value = NULL; + if (int_value) *int_value = 0; + if (expected_type != DBTYPE_INTEGER && expected_type != DBTYPE_TEXT && expected_type != DBTYPE_BLOB) { + return cloudsync_set_error(data, "Invalid expected_type", DBRES_MISUSE); + } + + int rc = SPI_execute(sql, true, 0); + if (rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute failed in database_select1_value", DBRES_ERROR); + goto cleanup; + } + + // ensure at least one column + if (!SPI_tuptable || !SPI_tuptable->tupdesc) { + rc = cloudsync_set_error(data, "No result table", DBRES_ERROR); + goto cleanup; + } + if (SPI_tuptable->tupdesc->natts < 1) { + rc = cloudsync_set_error(data, "No columns in result", DBRES_ERROR); + goto cleanup; + } + + // no rows OK + if (SPI_processed == 0) { + rc = DBRES_OK; + goto cleanup; + } + + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + Datum datum = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + + // NULL value is OK + if (isnull) { + rc = DBRES_OK; + goto cleanup; + } + + // Get type info + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 1); + + if (expected_type == DBTYPE_INTEGER) { + switch (typeid) { + case INT2OID: + *int_value = (int64_t)DatumGetInt16(datum); + break; + case INT4OID: + *int_value = (int64_t)DatumGetInt32(datum); + break; + case INT8OID: + *int_value = DatumGetInt64(datum); + break; + default: + rc = cloudsync_set_error(data, "Type mismatch: expected integer", DBRES_ERROR); + goto cleanup; + } + } else if (expected_type == DBTYPE_TEXT) { + char *val = SPI_getvalue(tuple, SPI_tuptable->tupdesc, 1); + if (val) { + size_t len = strlen(val); + char *ptr = cloudsync_memory_alloc(len + 1); + if (!ptr) { + pfree(val); + rc = cloudsync_set_error(data, "Memory allocation failed", DBRES_NOMEM); + goto cleanup; + } + memcpy(ptr, val, len); + ptr[len] = '\0'; + if (ptr_value) *ptr_value = ptr; + if (int_value) *int_value = (int64_t)len; + pfree(val); + } + } else if (expected_type == DBTYPE_BLOB) { + bytea *ba = DatumGetByteaP(datum); + int len = VARSIZE(ba) - VARHDRSZ; + if (len > 0) { + char *ptr = cloudsync_memory_alloc(len); + if (!ptr) { + rc = cloudsync_set_error(data, "Memory allocation failed", DBRES_NOMEM); + goto cleanup; + } + memcpy(ptr, VARDATA(ba), len); + if (ptr_value) *ptr_value = ptr; + if (int_value) *int_value = len; + } + } + + rc = DBRES_OK; + +cleanup: + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return rc; +} + +int database_select2_values (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + cloudsync_reset_error(data); + + // init values + *value = NULL; + *value2 = 0; + *len = 0; + + int rc = SPI_execute(sql, true, 0); + if (rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute failed in database_select2_values", DBRES_ERROR); + goto cleanup; + } + + if (!SPI_tuptable || !SPI_tuptable->tupdesc) { + rc = cloudsync_set_error(data, "No result table in database_select2_values", DBRES_ERROR); + goto cleanup; + } + if (SPI_tuptable->tupdesc->natts < 2) { + rc = cloudsync_set_error(data, "Result has fewer than 2 columns in database_select2_values", DBRES_ERROR); + goto cleanup; + } + if (SPI_processed == 0) { + rc = DBRES_OK; + goto cleanup; + } + + HeapTuple tuple = SPI_tuptable->vals[0]; + bool isnull; + + // First column - text/blob + Datum datum1 = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) { + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 1); + if (typeid == BYTEAOID) { + bytea *ba = DatumGetByteaP(datum1); + int blob_len = VARSIZE(ba) - VARHDRSZ; + if (blob_len > 0) { + char *ptr = cloudsync_memory_alloc(blob_len); + if (!ptr) { + rc = DBRES_NOMEM; + goto cleanup; + } + + memcpy(ptr, VARDATA(ba), blob_len); + *value = ptr; + *len = blob_len; + } + } else { + text *txt = DatumGetTextP(datum1); + int text_len = VARSIZE(txt) - VARHDRSZ; + if (text_len > 0) { + char *ptr = cloudsync_memory_alloc(text_len + 1); + if (!ptr) { + rc = DBRES_NOMEM; + goto cleanup; + } + + memcpy(ptr, VARDATA(txt), text_len); + ptr[text_len] = '\0'; + *value = ptr; + *len = text_len; + } + } + } + + // Second column - int + Datum datum2 = SPI_getbinval(tuple, SPI_tuptable->tupdesc, 2, &isnull); + if (!isnull) { + Oid typeid = SPI_gettypeid(SPI_tuptable->tupdesc, 2); + if (typeid == INT8OID) { + *value2 = DatumGetInt64(datum2); + } else if (typeid == INT4OID) { + *value2 = (int64_t)DatumGetInt32(datum2); + } + } + + rc = DBRES_OK; + +cleanup: + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return rc; +} + +static bool database_system_exists (cloudsync_context *data, const char *name, const char *type, bool force_public, const char *schema) { + if (!name || !type) return false; + cloudsync_reset_error(data); + + bool exists = false; + const char *query; + // Schema parameter: pass empty string to fall back to current_schema() via SQL + const char *schema_param = (schema && schema[0]) ? schema : ""; + + if (strcmp(type, "table") == 0) { + if (force_public) { + query = "SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = $1"; + } else { + query = "SELECT 1 FROM pg_tables WHERE schemaname = COALESCE(NULLIF($2, ''), current_schema()) AND tablename = $1"; + } + } else if (strcmp(type, "trigger") == 0) { + query = "SELECT 1 FROM pg_trigger WHERE tgname = $1"; + } else { + return false; + } + + bool need_schema_param = !force_public && strcmp(type, "trigger") != 0; + Datum datum_name = CStringGetTextDatum(name); + Datum datum_schema = need_schema_param ? CStringGetTextDatum(schema_param) : (Datum)0; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + if (!need_schema_param) { + // force_public or trigger: only need table/trigger name parameter + Oid argtypes[1] = {TEXTOID}; + Datum values[1] = {datum_name}; + char nulls[1] = {' '}; + int rc = SPI_execute_with_args(query, 1, argtypes, values, nulls, true, 0); + exists = (rc >= 0 && SPI_processed > 0); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + } else { + // table with schema parameter + Oid argtypes[2] = {TEXTOID, TEXTOID}; + Datum values[2] = {datum_name, datum_schema}; + char nulls[2] = {' ', ' '}; + int rc = SPI_execute_with_args(query, 2, argtypes, values, nulls, true, 0); + exists = (rc >= 0 && SPI_processed > 0); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + exists = false; + } + PG_END_TRY(); + + pfree(DatumGetPointer(datum_name)); + if (need_schema_param) pfree(DatumGetPointer(datum_schema)); + + elog(DEBUG1, "database_system_exists %s: %d", name, exists); + return exists; + } + +// MARK: - GENERAL - + +int database_exec (cloudsync_context *data, const char *sql) { + if (!sql) return cloudsync_set_error(data, "SQL statement is NULL", DBRES_ERROR); + cloudsync_reset_error(data); + + int rc; + bool is_error = false; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + rc = SPI_execute(sql, false, 0); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + } + is_error = true; + } + PG_END_TRY(); + + if (is_error) return rc; + + // Increment command counter to make changes visible + if (rc >= 0) { + database_refresh_snapshot(); + return map_spi_result(rc); + } + + return cloudsync_set_error(data, "SPI_execute failed", DBRES_ERROR); +} + +int database_exec_callback (cloudsync_context *data, const char *sql, int (*callback)(void *xdata, int argc, char **values, char **names), void *xdata) { + if (!sql) return cloudsync_set_error(data, "SQL statement is NULL", DBRES_ERROR); + cloudsync_reset_error(data); + + int rc; + bool is_error = false; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + rc = SPI_execute(sql, true, 0); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + is_error = true; + } + PG_END_TRY(); + + if (is_error) return rc; + if (rc < 0) return cloudsync_set_error(data, "SPI_execute failed", DBRES_ERROR); + + // Call callback for each row if provided + if (callback && SPI_tuptable) { + TupleDesc tupdesc = SPI_tuptable->tupdesc; + if (!tupdesc) { + SPI_freetuptable(SPI_tuptable); + return cloudsync_set_error(data, "Invalid tuple descriptor", DBRES_ERROR); + } + + int ncols = tupdesc->natts; + if (ncols <= 0) { + SPI_freetuptable(SPI_tuptable); + return DBRES_OK; + } + + // IMPORTANT: Save SPI state before any callback can modify it. + // Callbacks may execute SPI queries which overwrite global SPI_tuptable. + // We must copy all data we need BEFORE calling any callbacks. + uint64 nrows = SPI_processed; + SPITupleTable *saved_tuptable = SPI_tuptable; + + // No rows to process - free tuptable and return success + if (nrows == 0) { + SPI_freetuptable(saved_tuptable); + return DBRES_OK; + } + + // Allocate array for column names (shared across all rows) + char **names = cloudsync_memory_alloc(ncols * sizeof(char*)); + if (!names) { + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + // Get column names - make copies to avoid pointing to internal memory + for (int i = 0; i < ncols; i++) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + if (attr) { + names[i] = cloudsync_string_dup(NameStr(attr->attname)); + } else { + names[i] = NULL; + } + } + + // Pre-extract ALL row values before calling any callbacks. + // This prevents SPI state corruption when callbacks run queries. + char ***all_values = cloudsync_memory_alloc(nrows * sizeof(char**)); + if (!all_values) { + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + // Extract values from all tuples + for (uint64 row = 0; row < nrows; row++) { + HeapTuple tuple = saved_tuptable->vals[row]; + all_values[row] = cloudsync_memory_alloc(ncols * sizeof(char*)); + if (!all_values[row]) { + // Cleanup already allocated rows + for (uint64 r = 0; r < row; r++) { + for (int c = 0; c < ncols; c++) { + if (all_values[r][c]) pfree(all_values[r][c]); + } + cloudsync_memory_free(all_values[r]); + } + cloudsync_memory_free(all_values); + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + SPI_freetuptable(saved_tuptable); + return DBRES_NOMEM; + } + + if (!tuple) { + for (int i = 0; i < ncols; i++) all_values[row][i] = NULL; + continue; + } + + for (int i = 0; i < ncols; i++) { + bool isnull; + SPI_getbinval(tuple, tupdesc, i + 1, &isnull); + all_values[row][i] = (isnull) ? NULL : SPI_getvalue(tuple, tupdesc, i + 1); + } + } + + // Free SPI_tuptable BEFORE calling callbacks - we have all data we need + SPI_freetuptable(saved_tuptable); + SPI_tuptable = NULL; + + // Now process each row - callbacks can safely run SPI queries + int result = DBRES_OK; + for (uint64 row = 0; row < nrows; row++) { + int cb_rc = callback(xdata, ncols, all_values[row], names); + + if (cb_rc != 0) { + char errmsg[1024]; + snprintf(errmsg, sizeof(errmsg), "database_exec_callback aborted %d", cb_rc); + result = cloudsync_set_error(data, errmsg, DBRES_ABORT); + break; + } + } + + // Cleanup all extracted values + for (uint64 row = 0; row < nrows; row++) { + for (int i = 0; i < ncols; i++) { + if (all_values[row][i]) pfree(all_values[row][i]); + } + cloudsync_memory_free(all_values[row]); + } + cloudsync_memory_free(all_values); + + // Free column names + for (int i = 0; i < ncols; i++) { + if (names[i]) cloudsync_memory_free(names[i]); + } + cloudsync_memory_free(names); + + return result; + } + + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; +} + +int database_write (cloudsync_context *data, const char *sql, const char **bind_values, DBTYPE bind_types[], int bind_lens[], int bind_count) { + if (!sql) return cloudsync_set_error(data, "Invalid parameters to database_write", DBRES_ERROR); + cloudsync_reset_error(data); + + // Prepare statement + dbvm_t *stmt; + int rc = databasevm_prepare(data, sql, &stmt, 0); + if (rc != DBRES_OK) return rc; + + // Bind parameters + for (int i = 0; i < bind_count; i++) { + int param_idx = i + 1; + + switch (bind_types[i]) { + case DBTYPE_NULL: + rc = databasevm_bind_null(stmt, param_idx); + break; + case DBTYPE_INTEGER: { + int64_t val = strtoll(bind_values[i], NULL, 0); + rc = databasevm_bind_int(stmt, param_idx, val); + break; + } + case DBTYPE_FLOAT: { + double val = strtod(bind_values[i], NULL); + rc = databasevm_bind_double(stmt, param_idx, val); + break; + } + case DBTYPE_TEXT: + rc = databasevm_bind_text(stmt, param_idx, bind_values[i], bind_lens[i]); + break; + case DBTYPE_BLOB: + rc = databasevm_bind_blob(stmt, param_idx, bind_values[i], bind_lens[i]); + break; + default: + rc = DBRES_ERROR; + break; + } + + if (rc != DBRES_OK) { + databasevm_finalize(stmt); + return rc; + } + } + + // Execute + rc = databasevm_step(stmt); + databasevm_finalize(stmt); + + return (rc == DBRES_DONE) ? DBRES_OK : rc; +} + +int database_select_int (cloudsync_context *data, const char *sql, int64_t *value) { + return database_select1_value(data, sql, NULL, value, DBTYPE_INTEGER); +} + +int database_select_text (cloudsync_context *data, const char *sql, char **value) { + int64_t len = 0; + return database_select1_value(data, sql, value, &len, DBTYPE_TEXT); +} + +int database_select_blob (cloudsync_context *data, const char *sql, char **value, int64_t *len) { + return database_select1_value(data, sql, value, len, DBTYPE_BLOB); +} + +int database_select_blob_int (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + return database_select2_values(data, sql, value, len, value2); +} + +int database_cleanup (cloudsync_context *data) { + // NOOP + return DBRES_OK; +} + +// MARK: - STATUS - +int database_errcode (cloudsync_context *data) { + return cloudsync_errcode(data); +} + +const char *database_errmsg (cloudsync_context *data) { + return cloudsync_errmsg(data); +} + +bool database_in_transaction (cloudsync_context *data) { + // In SPI context, we're always in a transaction + return IsTransactionState(); +} + +bool database_table_exists (cloudsync_context *data, const char *name, const char *schema) { + return database_system_exists(data, name, "table", false, schema); +} + +bool database_internal_table_exists (cloudsync_context *data, const char *name) { + // Internal tables always in public schema + return database_system_exists(data, name, "table", true, NULL); +} + +bool database_trigger_exists (cloudsync_context *data, const char *name) { + // Triggers: extract table name to get schema + // Trigger names follow pattern:
_cloudsync_ + // For now, pass NULL to use current_schema() + return database_system_exists(data, name, "trigger", false, NULL); +} + +// MARK: - SCHEMA INFO - + +static int64_t database_count_bind (cloudsync_context *data, const char *sql, const char *table_name, const char *schema) { + // Schema parameter: pass empty string to fall back to current_schema() via SQL + const char *schema_param = (schema && schema[0]) ? schema : ""; + + Oid argtypes[2] = {TEXTOID, TEXTOID}; + Datum values[2] = {CStringGetTextDatum(table_name), CStringGetTextDatum(schema_param)}; + char nulls[2] = {' ', ' '}; + + int64_t count = 0; + int rc = SPI_execute_with_args(sql, 2, argtypes, values, nulls, true, 0); + if (rc >= 0 && SPI_processed > 0 && SPI_tuptable) { + bool isnull; + Datum d = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) count = DatumGetInt64(d); + } + + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + pfree(DatumGetPointer(values[0])); + pfree(DatumGetPointer(values[1])); + return count; +} + +int database_count_pk (cloudsync_context *data, const char *table_name, bool not_null, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY'"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "WHERE c.table_name = $1 " + "AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ")"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "JOIN information_schema.key_column_usage kcu ON c.column_name = kcu.column_name AND c.table_schema = kcu.table_schema AND c.table_name = kcu.table_name " + "JOIN information_schema.table_constraints tc ON kcu.constraint_name = tc.constraint_name AND kcu.table_schema = tc.table_schema " + "WHERE c.table_name = $1 AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY' " + "AND c.data_type IN ('smallint', 'integer', 'bigint')"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema) { + const char *sql = + "SELECT COUNT(*) FROM information_schema.columns c " + "WHERE c.table_name = $1 " + "AND c.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + "AND c.is_nullable = 'NO' " + "AND c.column_default IS NULL " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(NULLIF($2, ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ")"; + + return (int)database_count_bind(data, sql, table_name, schema); +} + +/* +int database_debug (db_t *db, bool print_result) { + // PostgreSQL debug information + if (print_result) { + elog(DEBUG1, "PostgreSQL SPI debug info:"); + elog(DEBUG1, " SPI_processed: %lu", (unsigned long)SPI_processed); + elog(DEBUG1, " In transaction: %d", IsTransactionState()); + } + return DBRES_OK; +} + */ + +// MARK: - METADATA TABLES - + +int database_create_metatable (cloudsync_context *data, const char *table_name) { + int rc; + const char *schema = cloudsync_schema(data); + + char *meta_ref = database_build_meta_ref(schema, table_name); + if (!meta_ref) return DBRES_NOMEM; + + char *sql2 = cloudsync_memory_mprintf( + "CREATE TABLE IF NOT EXISTS %s (" + "pk BYTEA NOT NULL," + "col_name TEXT NOT NULL," + "col_version BIGINT," + "db_version BIGINT NOT NULL DEFAULT 0," + "seq INTEGER NOT NULL DEFAULT 0," + "site_id BIGINT NOT NULL DEFAULT 0," + "PRIMARY KEY (pk, col_name)" + ");", + meta_ref); + if (!sql2) { cloudsync_memory_free(meta_ref); return DBRES_NOMEM; } + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) { cloudsync_memory_free(meta_ref); return rc; } + + // Create indices for performance + { + char escaped_tbl[512], escaped_sch[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + if (schema) { + sql_escape_identifier(schema, escaped_sch, sizeof(escaped_sch)); + sql2 = cloudsync_memory_mprintf( + "CREATE INDEX IF NOT EXISTS \"%s_cloudsync_db_version_idx\" " + "ON \"%s\".\"%s_cloudsync\" (db_version);", + escaped_tbl, escaped_sch, escaped_tbl); + } else { + sql2 = cloudsync_memory_mprintf( + "CREATE INDEX IF NOT EXISTS \"%s_cloudsync_db_version_idx\" " + "ON \"%s_cloudsync\" (db_version);", + escaped_tbl, escaped_tbl); + } + } + cloudsync_memory_free(meta_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +// MARK: - TRIGGERS - + +static int database_create_insert_trigger_internal (cloudsync_context *data, const char *table_name, char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_insert_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_insert_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg('NEW.' || quote_ident(kcu.column_name) || '::text', ',' ORDER BY kcu.ordinal_position) " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_schema_literal); + + char *pk_list = NULL; + int rc = database_select_text(data, sql, &pk_list); + if (rc != DBRES_OK) return rc; + if (!pk_list || pk_list[0] == '\0') { + if (pk_list) cloudsync_memory_free(pk_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN NEW; END IF; " + " PERFORM cloudsync_insert('%s', %s); " + " RETURN NEW; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, esc_tbl_literal, pk_list); + cloudsync_memory_free(pk_list); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER INSERT ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +static int database_create_update_trigger_gos_internal (cloudsync_context *data, const char *table_name, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_before_update_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_before_update_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + + char *sql = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " RAISE EXCEPTION 'Error: UPDATE operation is not allowed on table %s.'; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" BEFORE UPDATE ON %s " + "FOR EACH ROW WHEN (cloudsync_is_enabled('%s') = true) " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, esc_tbl_literal, func_name); + cloudsync_memory_free(base_ref); + if (!sql) return DBRES_NOMEM; + + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static int database_create_update_trigger_internal (cloudsync_context *data, const char *table_name, const char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_update_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_update_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg(" + " '(''%s'', NEW.' || quote_ident(kcu.column_name) || '::text, OLD.' || " + "quote_ident(kcu.column_name) || '::text)', " + " ', ' ORDER BY kcu.ordinal_position" + ") " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_tbl_literal, esc_schema_literal); + + char *pk_values_list = NULL; + int rc = database_select_text(data, sql, &pk_values_list); + if (rc != DBRES_OK) return rc; + if (!pk_values_list || pk_values_list[0] == '\0') { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + snprintf(sql, sizeof(sql), + "SELECT string_agg(" + " '(''%s'', NEW.' || quote_ident(c.column_name) || '::text, OLD.' || " + "quote_ident(c.column_name) || '::text)', " + " ', ' ORDER BY c.ordinal_position" + ") " + "FROM information_schema.columns c " + "WHERE c.table_name = '%s' " + "AND c.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND NOT EXISTS (" + " SELECT 1 FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = c.table_name " + " AND tc.table_schema = c.table_schema " + " AND tc.constraint_type = 'PRIMARY KEY' " + " AND kcu.column_name = c.column_name" + ");", + esc_tbl_literal, esc_tbl_literal, esc_schema_literal); + + char *col_values_list = NULL; + rc = database_select_text(data, sql, &col_values_list); + if (rc != DBRES_OK) { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return rc; + } + + char *values_query = NULL; + if (col_values_list && col_values_list[0] != '\0') { + values_query = cloudsync_memory_mprintf("VALUES %s, %s", pk_values_list, col_values_list); + } else { + values_query = cloudsync_memory_mprintf("VALUES %s", pk_values_list); + } + + if (pk_values_list) cloudsync_memory_free(pk_values_list); + if (col_values_list) cloudsync_memory_free(col_values_list); + if (!values_query) return DBRES_NOMEM; + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN NEW; END IF; " + " PERFORM cloudsync_update(table_name, new_value, old_value) " + " FROM (%s) AS v(table_name, new_value, old_value); " + " RETURN NEW; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, values_query); + cloudsync_memory_free(values_query); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER UPDATE ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +static int database_create_delete_trigger_gos_internal (cloudsync_context *data, const char *table_name, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_before_delete_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_before_delete_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + + char *sql = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " RAISE EXCEPTION 'Error: DELETE operation is not allowed on table %s.'; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal); + if (!sql) return DBRES_NOMEM; + + int rc = database_exec(data, sql); + cloudsync_memory_free(sql); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" BEFORE DELETE ON %s " + "FOR EACH ROW WHEN (cloudsync_is_enabled('%s') = true) " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, esc_tbl_literal, func_name); + cloudsync_memory_free(base_ref); + if (!sql) return DBRES_NOMEM; + + rc = database_exec(data, sql); + cloudsync_memory_free(sql); + return rc; +} + +static int database_create_delete_trigger_internal (cloudsync_context *data, const char *table_name, const char *trigger_when, const char *schema) { + if (!table_name) return DBRES_MISUSE; + + const char *schema_param = (schema && schema[0]) ? schema : ""; + + char trigger_name[1024]; + char func_name[1024]; + char escaped_tbl[512]; + sql_escape_identifier(table_name, escaped_tbl, sizeof(escaped_tbl)); + snprintf(trigger_name, sizeof(trigger_name), "cloudsync_after_delete_%s", escaped_tbl); + snprintf(func_name, sizeof(func_name), "cloudsync_after_delete_%s_fn", escaped_tbl); + + if (database_trigger_exists(data, trigger_name)) return DBRES_OK; + + char esc_tbl_literal[1024], esc_schema_literal[1024]; + sql_escape_literal(table_name, esc_tbl_literal, sizeof(esc_tbl_literal)); + sql_escape_literal(schema_param, esc_schema_literal, sizeof(esc_schema_literal)); + + char sql[2048]; + snprintf(sql, sizeof(sql), + "SELECT string_agg('OLD.' || quote_ident(kcu.column_name) || '::text', ',' ORDER BY kcu.ordinal_position) " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY';", + esc_tbl_literal, esc_schema_literal); + + char *pk_list = NULL; + int rc = database_select_text(data, sql, &pk_list); + if (rc != DBRES_OK) return rc; + if (!pk_list || pk_list[0] == '\0') { + if (pk_list) cloudsync_memory_free(pk_list); + return cloudsync_set_error(data, "No primary key columns found for table", DBRES_ERROR); + } + + char *sql2 = cloudsync_memory_mprintf( + "CREATE OR REPLACE FUNCTION \"%s\"() RETURNS trigger AS $$ " + "BEGIN " + " IF cloudsync_is_sync('%s') THEN RETURN OLD; END IF; " + " PERFORM cloudsync_delete('%s', %s); " + " RETURN OLD; " + "END; " + "$$ LANGUAGE plpgsql;", + func_name, esc_tbl_literal, esc_tbl_literal, pk_list); + cloudsync_memory_free(pk_list); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + if (rc != DBRES_OK) return rc; + + char *base_ref = database_build_base_ref(schema, table_name); + if (!base_ref) return DBRES_NOMEM; + + sql2 = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%s\" AFTER DELETE ON %s %s " + "EXECUTE FUNCTION \"%s\"();", + trigger_name, base_ref, trigger_when ? trigger_when : "", func_name); + cloudsync_memory_free(base_ref); + if (!sql2) return DBRES_NOMEM; + + rc = database_exec(data, sql2); + cloudsync_memory_free(sql2); + return rc; +} + +// Build trigger WHEN clauses, optionally incorporating a row-level filter. +// INSERT/UPDATE use NEW-prefixed filter, DELETE uses OLD-prefixed filter. +static void database_build_trigger_when( + cloudsync_context *data, const char *table_name, const char *filter, + const char *schema, + char *when_new, size_t when_new_size, + char *when_old, size_t when_old_size) +{ + char *new_filter_str = NULL; + char *old_filter_str = NULL; + + if (filter) { + const char *schema_param = (schema && schema[0]) ? schema : ""; + char esc_tbl[1024], esc_schema[1024]; + sql_escape_literal(table_name, esc_tbl, sizeof(esc_tbl)); + sql_escape_literal(schema_param, esc_schema, sizeof(esc_schema)); + + char col_sql[2048]; + snprintf(col_sql, sizeof(col_sql), + "SELECT column_name::text FROM information_schema.columns " + "WHERE table_name = '%s' AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "ORDER BY ordinal_position;", + esc_tbl, esc_schema); + + char *col_names[256]; + int ncols = 0; + + dbvm_t *col_vm = NULL; + int crc = databasevm_prepare(data, col_sql, &col_vm, 0); + if (crc == DBRES_OK) { + while (databasevm_step(col_vm) == DBRES_ROW && ncols < 256) { + const char *name = database_column_text(col_vm, 0); + if (name) col_names[ncols++] = cloudsync_memory_mprintf("%s", name); + } + databasevm_finalize(col_vm); + } + + if (ncols > 0) { + new_filter_str = cloudsync_filter_add_row_prefix(filter, "NEW", col_names, ncols); + old_filter_str = cloudsync_filter_add_row_prefix(filter, "OLD", col_names, ncols); + for (int i = 0; i < ncols; ++i) cloudsync_memory_free(col_names[i]); + } + } + + char esc_tbl[512]; + sql_escape_literal(table_name, esc_tbl, sizeof(esc_tbl)); + + if (new_filter_str) { + snprintf(when_new, when_new_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false AND (%s))", + esc_tbl, new_filter_str); + } else { + snprintf(when_new, when_new_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false)", + esc_tbl); + } + + if (old_filter_str) { + snprintf(when_old, when_old_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false AND (%s))", + esc_tbl, old_filter_str); + } else { + snprintf(when_old, when_old_size, + "FOR EACH ROW WHEN (cloudsync_is_sync('%s') = false)", + esc_tbl); + } + + if (new_filter_str) cloudsync_memory_free(new_filter_str); + if (old_filter_str) cloudsync_memory_free(old_filter_str); +} + +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter) { + if (!table_name) return DBRES_MISUSE; + + // Detect schema from metadata table if it exists, otherwise use cloudsync_schema() + // This is called before table_add_to_context(), so we can't rely on table lookup. + char *detected_schema = database_table_schema(table_name); + const char *schema = detected_schema ? detected_schema : cloudsync_schema(data); + + char trigger_when_new[4096]; + char trigger_when_old[4096]; + database_build_trigger_when(data, table_name, filter, schema, + trigger_when_new, sizeof(trigger_when_new), + trigger_when_old, sizeof(trigger_when_old)); + + int rc = database_create_insert_trigger_internal(data, table_name, trigger_when_new, schema); + if (rc != DBRES_OK) { + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; + } + + if (algo == table_algo_crdt_gos) { + rc = database_create_update_trigger_gos_internal(data, table_name, schema); + } else { + rc = database_create_update_trigger_internal(data, table_name, trigger_when_new, schema); + } + if (rc != DBRES_OK) { + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; + } + + if (algo == table_algo_crdt_gos) { + rc = database_create_delete_trigger_gos_internal(data, table_name, schema); + } else { + rc = database_create_delete_trigger_internal(data, table_name, trigger_when_old, schema); + } + + if (detected_schema) cloudsync_memory_free(detected_schema); + return rc; +} + +int database_delete_triggers (cloudsync_context *data, const char *table) { + char *base_ref = database_build_base_ref(cloudsync_schema(data), table); + if (!base_ref) return DBRES_NOMEM; + + char escaped_tbl[512]; + sql_escape_identifier(table, escaped_tbl, sizeof(escaped_tbl)); + + char *sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_insert_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_update_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_before_update_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%s\" ON %s;", + escaped_tbl, base_ref); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_after_delete_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + sql = cloudsync_memory_mprintf( + "DROP FUNCTION IF EXISTS \"cloudsync_before_delete_%s_fn\"() CASCADE;", + escaped_tbl); + if (sql) { database_exec(data, sql); cloudsync_memory_free(sql); } + + cloudsync_memory_free(base_ref); + return DBRES_OK; +} + +// MARK: - SCHEMA VERSIONING - + +int64_t database_schema_version (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, SQL_SCHEMA_VERSION, &value); + return (rc == DBRES_OK) ? value : 0; +} + +uint64_t database_schema_hash (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC LIMIT 1;", &value); + return (rc == DBRES_OK) ? (uint64_t)value : 0; +} + +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash) { + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = %" PRId64, (int64_t)hash); + + int64_t value = 0; + database_select_int(data, sql, &value); + return (value == 1); +} + +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash) { + // Build normalized schema string using only: column name (lowercase), type (SQLite affinity), pk flag + // Format: tablename:colname:affinity:pk,... (ordered by table name, then column ordinal position) + // This makes the hash resilient to formatting, quoting, case differences and portable across databases + // + // PostgreSQL type to SQLite affinity mapping: + // - integer, smallint, bigint, boolean → 'integer' + // - bytea → 'blob' + // - real, double precision → 'real' + // - numeric, decimal → 'numeric' + // - Everything else → 'text' (default) + // This includes: text, varchar, char, uuid, timestamp, timestamptz, date, time, + // interval, json, jsonb, inet, cidr, macaddr, geometric types, xml, enums, + // and any custom/extension types. Using 'text' as default ensures compatibility + // since most types serialize to text representation and SQLite stores unknown + // types as TEXT affinity. + + char *schema = NULL; + int rc = database_select_text(data, + "SELECT string_agg(" + " LOWER(c.table_name) || ':' || LOWER(c.column_name) || ':' || " + " CASE " + // Integer types (including boolean as 0/1) + " WHEN c.data_type IN ('integer', 'smallint', 'bigint', 'boolean') THEN 'integer' " + // Blob type + " WHEN c.data_type = 'bytea' THEN 'blob' " + // Real/float types + " WHEN c.data_type IN ('real', 'double precision') THEN 'real' " + // Numeric types (explicit precision/scale) + " WHEN c.data_type IN ('numeric', 'decimal') THEN 'numeric' " + // Default to text for everything else: + // - String types: text, character varying, character, name, uuid + // - Date/time: timestamp, date, time, interval, etc. + // - JSON: json, jsonb + // - Network: inet, cidr, macaddr + // - Geometric: point, line, box, etc. + // - Custom/extension types + " ELSE 'text' " + " END || ':' || " + " CASE WHEN kcu.column_name IS NOT NULL THEN '1' ELSE '0' END, " + " ',' ORDER BY c.table_name, c.ordinal_position" + ") " + "FROM information_schema.columns c " + "JOIN cloudsync_table_settings cts ON LOWER(c.table_name) = LOWER(cts.tbl_name) " + "LEFT JOIN information_schema.table_constraints tc " + " ON tc.table_name = c.table_name " + " AND tc.table_schema = c.table_schema " + " AND tc.constraint_type = 'PRIMARY KEY' " + "LEFT JOIN information_schema.key_column_usage kcu " + " ON kcu.table_name = c.table_name " + " AND kcu.column_name = c.column_name " + " AND kcu.table_schema = c.table_schema " + " AND kcu.constraint_name = tc.constraint_name " + "WHERE c.table_schema = COALESCE(cloudsync_schema(), current_schema())", + &schema); + + if (rc != DBRES_OK || !schema) return cloudsync_set_error(data, "database_update_schema_hash error 1", DBRES_ERROR); + + size_t schema_len = strlen(schema); + DEBUG_MERGE("database_update_schema_hash len %zu schema %s", schema_len, schema); + uint64_t h = fnv1a_hash(schema, schema_len); + cloudsync_memory_free(schema); + if (hash && *hash == h) return cloudsync_set_error(data, "database_update_schema_hash constraint", DBRES_CONSTRAINT); + + char sql[1024]; + snprintf(sql, sizeof(sql), + "INSERT INTO cloudsync_schema_versions (hash, seq) " + "VALUES (%" PRId64 ", COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " + "ON CONFLICT(hash) DO UPDATE SET " + "seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", + (int64_t)h); + rc = database_exec(data, sql); + if (rc == DBRES_OK) { + if (hash) *hash = h; + return rc; + } + + return cloudsync_set_error(data, "database_update_schema_hash error 2", DBRES_ERROR); +} + +// MARK: - PRIMARY KEY - + +int database_pk_rowid (cloudsync_context *data, const char *table_name, char ***names, int *count) { + // PostgreSQL doesn't have rowid concept like SQLite + // Use OID or primary key columns instead + return database_pk_names(data, table_name, names, count); +} + +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count) { + if (!table_name || !names || !count) return DBRES_MISUSE; + + const char *sql = + "SELECT kcu.column_name FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "WHERE tc.table_name = $1 AND tc.table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND tc.constraint_type = 'PRIMARY KEY' " + "ORDER BY kcu.ordinal_position"; + + Oid argtypes[1] = { TEXTOID }; + Datum values[1] = { CStringGetTextDatum(table_name) }; + char nulls[1] = { ' ' }; + + int rc = SPI_execute_with_args(sql, 1, argtypes, values, nulls, true, 0); + pfree(DatumGetPointer(values[0])); + + if (rc != SPI_OK_SELECT || SPI_processed == 0) { + *names = NULL; + *count = 0; + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; + } + + uint64_t n = SPI_processed; + char **pk_names = cloudsync_memory_zeroalloc(n * sizeof(char*)); + if (!pk_names) { + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_NOMEM; + } + + for (uint64_t i = 0; i < n; i++) { + HeapTuple tuple = SPI_tuptable->vals[i]; + bool isnull; + SPI_getbinval(tuple, SPI_tuptable->tupdesc, 1, &isnull); + if (!isnull) { + // SPI_getvalue returns a palloc'd string regardless of column type + char *name = SPI_getvalue(tuple, SPI_tuptable->tupdesc, 1); + pk_names[i] = (name) ? cloudsync_string_dup(name) : NULL; + if (name) pfree(name); + } + + // Cleanup on allocation failure + if (!isnull && pk_names[i] == NULL) { + for (uint64_t j = 0; j < i; j++) { + if (pk_names[j]) cloudsync_memory_free(pk_names[j]); + } + cloudsync_memory_free(pk_names); + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_NOMEM; + } + } + + *names = pk_names; + *count = (int)n; + if (SPI_tuptable) SPI_freetuptable(SPI_tuptable); + return DBRES_OK; +} + +// MARK: - VM - + +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags) { + if (!sql || !vm) { + return cloudsync_set_error(data, "Invalid parameters to databasevm_prepare", DBRES_ERROR); + } + *vm = NULL; + cloudsync_reset_error(data); + + // sanity check number of parameters + // int counter = count_params(sql); + // if (counter > MAX_PARAMS) return cloudsync_set_error(data, "Maximum number of parameters reached", DBRES_MISUSE); + + // create PostgreSQL VM statement + pg_stmt_t *stmt = (pg_stmt_t *)cloudsync_memory_zeroalloc(sizeof(pg_stmt_t)); + if (!stmt) return cloudsync_set_error(data, "Not enough memory to allocate a dbvm_t struct", DBRES_NOMEM); + stmt->data = data; + + int rc = DBRES_OK; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + MemoryContext parent = (flags & DBFLAG_PERSISTENT) ? TopMemoryContext : CurrentMemoryContext; + stmt->stmt_mcxt = AllocSetContextCreate(parent, "cloudsync stmt", ALLOCSET_DEFAULT_SIZES); + if (!stmt->stmt_mcxt) { + cloudsync_memory_free(stmt); + ereport(ERROR, (errmsg("Failed to create statement memory context"))); + } + stmt->bind_mcxt = AllocSetContextCreate(stmt->stmt_mcxt, "cloudsync binds", ALLOCSET_DEFAULT_SIZES); + stmt->row_mcxt = AllocSetContextCreate(stmt->stmt_mcxt, "cloudsync row", ALLOCSET_DEFAULT_SIZES); + + MemoryContext old = MemoryContextSwitchTo(stmt->stmt_mcxt); + stmt->sql = pstrdup(sql); + MemoryContextSwitchTo(old); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + if (stmt->stmt_mcxt) MemoryContextDelete(stmt->stmt_mcxt); + cloudsync_memory_free(stmt); + rc = DBRES_NOMEM; + stmt = NULL; + } + PG_END_TRY(); + + if (stmt) databasevm_clear_bindings((dbvm_t*)stmt); + *vm = (dbvm_t*)stmt; + + return rc; +} + +int databasevm_step0 (pg_stmt_t *stmt) { + cloudsync_context *data = stmt->data; + if (!data) return DBRES_ERROR; + + int rc = DBRES_OK; + MemoryContext oldcontext = CurrentMemoryContext; + + PG_TRY(); + { + if (!stmt->sql) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("databasevm_step0 invalid sql pointer"))); + } + + stmt->plan = SPI_prepare(stmt->sql, stmt->nparams, stmt->types); + if (stmt->plan == NULL) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to prepare SQL statement"))); + } + + SPI_keepplan(stmt->plan); + stmt->plan_is_prepared = true; + + // Save the types used for this plan so we can detect type changes + memcpy(stmt->prepared_types, stmt->types, sizeof(Oid) * stmt->nparams); + stmt->prepared_nparams = stmt->nparams; + } + PG_CATCH(); + { + // Switch to safe context for CopyErrorData (can't be ErrorContext) + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + + // Clean up partially prepared plan if needed + if (stmt->plan != NULL && !stmt->plan_is_prepared) { + PG_TRY(); + { + SPI_freeplan(stmt->plan); + } + PG_CATCH(); + { + FlushErrorState(); // Swallow errors during cleanup + } + PG_END_TRY(); + stmt->plan = NULL; + } + } + PG_END_TRY(); + + return rc; +} + +int databasevm_step (dbvm_t *vm) { + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt) return DBRES_MISUSE; + + cloudsync_context *data = stmt->data; + cloudsync_reset_error(data); + + // If plan is prepared but parameter types have changed since preparation, + // free the old plan and re-prepare with new types. This happens when the same + // prepared statement is reused with different PK encodings (e.g., integer vs text). + if (stmt->plan_is_prepared && stmt->plan) { + bool types_changed = (stmt->nparams != stmt->prepared_nparams); + if (!types_changed) { + for (int i = 0; i < stmt->nparams; i++) { + if (stmt->types[i] != stmt->prepared_types[i]) { + types_changed = true; + break; + } + } + } + if (types_changed) { + SPI_freeplan(stmt->plan); + stmt->plan = NULL; + stmt->plan_is_prepared = false; + } + } + + if (!stmt->plan_is_prepared) { + int rc = databasevm_step0(stmt); + if (rc != DBRES_OK) return rc; + } + if (!stmt->plan_is_prepared || !stmt->plan) return DBRES_ERROR; + + int rc = DBRES_DONE; + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + do { + // if portal is open, we fetch one row + if (stmt->portal_open) { + // free prior fetched row batch + clear_fetch_batch(stmt); + + SPI_cursor_fetch(stmt->portal, true, 1); + + if (SPI_processed == 0) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = DBRES_DONE; + break; + } + + // null check for SPI_tuptable + if (!SPI_tuptable || !SPI_tuptable->tupdesc || !SPI_tuptable->vals) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = cloudsync_set_error(data, "SPI_cursor_fetch returned invalid tuptable", DBRES_ERROR); + break; + } + + MemoryContextReset(stmt->row_mcxt); + + stmt->last_tuptable = SPI_tuptable; + stmt->current_tupdesc = stmt->last_tuptable->tupdesc; + stmt->current_tuple = stmt->last_tuptable->vals[0]; + rc = DBRES_ROW; + break; + } + + // First step: decide whether to use portal. + // Even for INSERT/UPDATE/DELETE ... RETURNING you WANT a portal. + // Strategy: + // - Only open a cursor if the plan supports it (avoid "cannot open INSERT query as cursor"). + // - Otherwise execute once as a non-row-returning statement. + if (!stmt->executed_nonselect) { + if (SPI_is_cursor_plan(stmt->plan)) { + // try cursor open + stmt->portal = NULL; + if (stmt->nparams == 0) stmt->portal = SPI_cursor_open(NULL, stmt->plan, NULL, NULL, false); + else stmt->portal = SPI_cursor_open(NULL, stmt->plan, stmt->values, stmt->nulls, false); + + if (stmt->portal != NULL) { + // Don't set portal_open until we successfully fetch first row + + // fetch first row + clear_fetch_batch(stmt); + SPI_cursor_fetch(stmt->portal, true, 1); + + if (SPI_processed == 0) { + // No rows - close portal, don't set portal_open + clear_fetch_batch(stmt); + close_portal(stmt); + rc = DBRES_DONE; + break; + } + + // null check for SPI_tuptable + if (!SPI_tuptable || !SPI_tuptable->tupdesc || !SPI_tuptable->vals) { + clear_fetch_batch(stmt); + close_portal(stmt); + rc = cloudsync_set_error(data, "SPI_cursor_fetch returned invalid tuptable", DBRES_ERROR); + break; + } + + MemoryContextReset(stmt->row_mcxt); + + stmt->last_tuptable = SPI_tuptable; + stmt->current_tupdesc = stmt->last_tuptable->tupdesc; + stmt->current_tuple = stmt->last_tuptable->vals[0]; + + // Only set portal_open AFTER everything succeeded + stmt->portal_open = true; + + rc = DBRES_ROW; + break; + } + } + + // Execute once (non-row-returning or cursor open failed). + int spi_rc; + if (stmt->nparams == 0) spi_rc = SPI_execute_plan(stmt->plan, NULL, NULL, false, 0); + else spi_rc = SPI_execute_plan(stmt->plan, stmt->values, stmt->nulls, false, 0); + if (spi_rc < 0) { + rc = cloudsync_set_error(data, "SPI_execute_plan failed", DBRES_ERROR); + break; + } + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + stmt->executed_nonselect = true; + rc = DBRES_DONE; + break; + } + + rc = DBRES_DONE; + } while (0); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + int err = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + + // free resources + clear_fetch_batch(stmt); + close_portal(stmt); + + rc = err; + } + PG_END_TRY(); + return rc; +} + +void databasevm_finalize (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + PG_TRY(); + { + clear_fetch_batch(stmt); + close_portal(stmt); + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + if (stmt->plan_is_prepared && stmt->plan) { + SPI_freeplan(stmt->plan); + stmt->plan = NULL; + stmt->plan_is_prepared = false; + } + } + PG_CATCH(); + { + /* don't throw from finalize; just swallow */ + FlushErrorState(); + } + PG_END_TRY(); + + if (stmt->stmt_mcxt) MemoryContextDelete(stmt->stmt_mcxt); + cloudsync_memory_free(stmt); +} + +void databasevm_reset (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + // Close any open cursor and clear fetched data + clear_fetch_batch(stmt); + close_portal(stmt); + + // Clear global SPI tuple table if any + if (SPI_tuptable) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = NULL; + } + + // Reset execution state + stmt->executed_nonselect = false; + + // Reset parameter values but keep the plan, types, and nparams intact. + // The prepared plan can be reused with new values of the same types, + // avoiding the cost of re-planning on every iteration. + if (stmt->bind_mcxt) MemoryContextReset(stmt->bind_mcxt); + for (int i = 0; i < stmt->nparams; i++) { + stmt->values[i] = (Datum) 0; + stmt->nulls[i] = 'n'; + } +} + +void databasevm_clear_bindings (dbvm_t *vm) { + if (!vm) return; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + + // Only clear the bound parameter values. + // Do NOT close portals, free fetch batches, or free the plan — + // those are execution state, not bindings. + if (stmt->bind_mcxt) MemoryContextReset(stmt->bind_mcxt); + stmt->nparams = 0; + + // Reset params array to defaults + for (int i = 0; i < MAX_PARAMS; i++) { + stmt->types[i] = UNKNOWNOID; + stmt->values[i] = (Datum) 0; + stmt->nulls[i] = 'n'; // default NULL + } +} + +const char *databasevm_sql (dbvm_t *vm) { + if (!vm) return NULL; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + return (char *)stmt->sql; +} + +// MARK: - BINDING - + +static int databasevm_bind_null_type (dbvm_t *vm, int index, Oid t) { + int rc = databasevm_bind_null(vm, index); + if (rc != DBRES_OK) return rc; + int idx = index - 1; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->types[idx] = t; + return rc; +} + +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size) { + if (!vm || index < 1) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, BYTEAOID); + + // validate size fits Size and won't overflow + if (size > (uint64) (MaxAllocSize - VARHDRSZ)) return DBRES_NOMEM; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + // Convert binary data to PostgreSQL bytea + bytea *ba = (bytea*)palloc(size + VARHDRSZ); + SET_VARSIZE(ba, size + VARHDRSZ); + memcpy(VARDATA(ba), value, size); + + stmt->values[idx] = PointerGetDatum(ba); + stmt->types[idx] = BYTEAOID; + stmt->nulls[idx] = ' '; + + MemoryContextSwitchTo(old); + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_double (dbvm_t *vm, int index, double value) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = Float8GetDatum(value); + stmt->types[idx] = FLOAT8OID; + stmt->nulls[idx] = ' '; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = Int64GetDatum(value); + stmt->types[idx] = INT8OID; + stmt->nulls[idx] = ' '; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_null (dbvm_t *vm, int index) { + if (!vm || index < 1) return DBRES_ERROR; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + stmt->values[idx] = (Datum)0; + stmt->types[idx] = TEXTOID; // TEXTOID has casts to most types + stmt->nulls[idx] = 'n'; + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size) { + if (!vm || index < 1) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, TEXTOID); + + // validate size fits Size and won't overflow + if (size < 0) size = (int)strlen(value); + if ((Size)size > MaxAllocSize - VARHDRSZ) return DBRES_NOMEM; + + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + text *t = cstring_to_text_with_len(value, size); + stmt->values[idx] = PointerGetDatum(t); + stmt->types[idx] = TEXTOID; + stmt->nulls[idx] = ' '; + + MemoryContextSwitchTo(old); + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value) { + if (!vm) return DBRES_ERROR; + if (!value) return databasevm_bind_null_type(vm, index, TEXTOID); + + // validate index bounds properly (1-based index) + if (index < 1) return DBRES_ERROR; + int idx = index - 1; + if (idx >= MAX_PARAMS) return DBRES_ERROR; + + pg_stmt_t *stmt = (pg_stmt_t*)vm; + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) { + stmt->values[idx] = (Datum)0; + // Use the actual column type if available, otherwise default to TEXTOID + stmt->types[idx] = (v && OidIsValid(v->typeid)) ? v->typeid : TEXTOID; + stmt->nulls[idx] = 'n'; + } else { + int16 typlen; + bool typbyval; + + get_typlenbyval(v->typeid, &typlen, &typbyval); + MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); + + Datum dcopy; + if (typbyval) { + // Pass-by-value: direct copy is safe + dcopy = v->datum; + } else { + // Pass-by-reference: need to copy the actual data + // Handle variable-length types (typlen == -1) and cstrings (typlen == -2) + if (typlen == -1) { + // Variable-length type (varlena): use VARSIZE_ANY to handle + // both short (1-byte) and regular (4-byte) varlena headers + Size len = VARSIZE_ANY(DatumGetPointer(v->datum)); + dcopy = PointerGetDatum(palloc(len)); + memcpy(DatumGetPointer(dcopy), DatumGetPointer(v->datum), len); + } else if (typlen == -2) { + // Null-terminated cstring + dcopy = CStringGetDatum(pstrdup(DatumGetCString(v->datum))); + } else { + // Fixed-length pass-by-reference + dcopy = datumCopy(v->datum, false, typlen); + } + } + + stmt->values[idx] = dcopy; + MemoryContextSwitchTo(old); + stmt->types[idx] = OidIsValid(v->typeid) ? v->typeid : TEXTOID; + stmt->nulls[idx] = ' '; + } + + if (stmt->nparams < idx + 1) stmt->nparams = idx + 1; + return DBRES_OK; +} + +// MARK: - COLUMN - + +Datum database_column_datum (dbvm_t *vm, int index) { + if (!vm) return (Datum)0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return (Datum)0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return (Datum)0; + + bool isnull = true; + Datum d = get_datum(stmt, index, &isnull, NULL); + return (isnull) ? (Datum)0 : d; +} + +const void *database_column_blob (dbvm_t *vm, int index, size_t *len) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Datum d = get_datum(stmt, index, &isnull, NULL); + if (isnull) return NULL; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + bytea *ba = DatumGetByteaP(d); + + // Validate VARSIZE before computing length + Size varsize = VARSIZE(ba); + if (varsize < VARHDRSZ) { + // Corrupt or invalid bytea - VARSIZE should always be >= VARHDRSZ + MemoryContextSwitchTo(old); + elog(WARNING, "database_column_blob: invalid bytea VARSIZE %zu", varsize); + return NULL; + } + + Size blen = VARSIZE(ba) - VARHDRSZ; + void *out = palloc(blen); + if (!out) { + MemoryContextSwitchTo(old); + return NULL; + } + + memcpy(out, VARDATA(ba), (size_t)blen); + MemoryContextSwitchTo(old); + + if (len) *len = (size_t)blen; + return out; +} + +double database_column_double (dbvm_t *vm, int index) { + if (!vm) return 0.0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0.0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0.0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0.0; + + switch (type) { + case FLOAT4OID: return (double)DatumGetFloat4(d); + case FLOAT8OID: return (double)DatumGetFloat8(d); + case NUMERICOID: return DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow, d)); + case INT2OID: return (double)DatumGetInt16(d); + case INT4OID: return (double)DatumGetInt32(d); + case INT8OID: return (double)DatumGetInt64(d); + case BOOLOID: return (double)DatumGetBool(d); + } + + return 0.0; +} + +int64_t database_column_int (dbvm_t *vm, int index) { + if (!vm) return 0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0; + + switch (type) { + case FLOAT4OID: return (int64_t)DatumGetFloat4(d); + case FLOAT8OID: return (int64_t)DatumGetFloat8(d); + case INT2OID: return (int64_t)DatumGetInt16(d); + case INT4OID: return (int64_t)DatumGetInt32(d); + case INT8OID: return (int64_t)DatumGetInt64(d); + case BOOLOID: return (int64_t)DatumGetBool(d); + } + + return 0; +} + +const char *database_column_text (dbvm_t *vm, int index) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return NULL; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + char *out = NULL; + + if (type == BYTEAOID) { + bytea *b = DatumGetByteaP(d); + int len = VARSIZE(b) - VARHDRSZ; + out = palloc(len + 1); + memcpy(out, VARDATA(b), len); + out[len] = 0; + } else if (type == TEXTOID || type == VARCHAROID || type == BPCHAROID) { + text *t = DatumGetTextP(d); + int len = VARSIZE(t) - VARHDRSZ; + out = palloc(len + 1); + memcpy(out, VARDATA(t), len); + out[len] = 0; + } else { + MemoryContextSwitchTo(old); + return NULL; + } + + MemoryContextSwitchTo(old); + + return out; +} + +dbvalue_t *database_column_value (dbvm_t *vm, int index) { + if (!vm) return NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return NULL; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + int32 typmod = TupleDescAttr(stmt->current_tupdesc, index)->atttypmod; + Oid collation = TupleDescAttr(stmt->current_tupdesc, index)->attcollation; + + pgvalue_t *v = pgvalue_create(d, type, typmod, collation, isnull); + if (v) pgvalue_ensure_detoast(v); + return (dbvalue_t*)v; +} + +int database_column_bytes (dbvm_t *vm, int index) { + if (!vm) return 0; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return 0; + if (index < 0 || index >= stmt->current_tupdesc->natts) return 0; + + bool isnull = true; + Oid type = 0; + Datum d = get_datum(stmt, index, &isnull, &type); + if (isnull) return 0; + + MemoryContext old = MemoryContextSwitchTo(stmt->row_mcxt); + + int bytes = 0; + if (type == BYTEAOID) { + // BLOB case + bytea *ba = DatumGetByteaP(d); + bytes = (int)(VARSIZE(ba) - VARHDRSZ); + } else if (type != TEXTOID && type != VARCHAROID && type != BPCHAROID) { + // any non-TEXT case should be discarded + bytes = 0; + } else { + // for text, return string length + text *txt = DatumGetTextP(d); + bytes = (int)(VARSIZE(txt) - VARHDRSZ); + } + MemoryContextSwitchTo(old); + + return bytes; +} + +int database_column_type (dbvm_t *vm, int index) { + if (!vm) return DBTYPE_NULL; + pg_stmt_t *stmt = (pg_stmt_t*)vm; + if (!stmt->last_tuptable || !stmt->current_tupdesc) return DBTYPE_NULL; + if (index < 0 || index >= stmt->current_tupdesc->natts) return DBTYPE_NULL; + + bool isnull = true; + Oid type = 0; + get_datum(stmt, index, &isnull, &type); + if (isnull) return DBTYPE_NULL; + + switch (type) { + case INT2OID: + case INT4OID: + case INT8OID: + return DBTYPE_INTEGER; + + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return DBTYPE_FLOAT; + + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + return DBTYPE_TEXT; + + case BYTEAOID: + return DBTYPE_BLOB; + } + + return DBTYPE_TEXT; +} + +// MARK: - VALUE - + +const void *database_value_blob (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return NULL; + + // Text types reuse blob accessor (pk encode reads text bytes directly). + // Exclude JSONB: its internal format is binary, not text — + // it must go through OidOutputFunctionCall to get the JSON text. + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + text *txt = (text *)DatumGetPointer(v->datum); + return VARDATA_ANY(txt); + } + + if (v->typeid == BYTEAOID) { + pgvalue_ensure_detoast(v); + bytea *ba = (bytea *)DatumGetPointer(v->datum); + return VARDATA_ANY(ba); + } + + // For unmapped types and JSONB (mapped to DBTYPE_TEXT), + // convert to text representation via the type's output function + const char *cstr = database_value_text(value); + if (cstr) return cstr; + + return NULL; +} + +double database_value_double (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0.0; + + switch (v->typeid) { + case FLOAT4OID: + return (double)DatumGetFloat4(v->datum); + case FLOAT8OID: + return DatumGetFloat8(v->datum); + case NUMERICOID: + return DatumGetFloat8(DirectFunctionCall1(numeric_float8_no_overflow, v->datum)); + case INT2OID: + return (double)DatumGetInt16(v->datum); + case INT4OID: + return (double)DatumGetInt32(v->datum); + case INT8OID: + return (double)DatumGetInt64(v->datum); + case BOOLOID: + return DatumGetBool(v->datum) ? 1.0 : 0.0; + default: + return 0.0; + } +} + +int64_t database_value_int (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0; + + switch (v->typeid) { + case INT2OID: + return (int64_t)DatumGetInt16(v->datum); + case INT4OID: + return (int64_t)DatumGetInt32(v->datum); + case INT8OID: + return DatumGetInt64(v->datum); + case BOOLOID: + return DatumGetBool(v->datum) ? 1 : 0; + default: + return 0; + } +} + +const char *database_value_text (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return NULL; + + if (!v->cstring && !v->owns_cstring) { + PG_TRY(); + { + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + v->cstring = text_to_cstring((text *)DatumGetPointer(v->datum)); + } else { + // Type output function for JSONB and other non-text types + Oid outfunc; + bool isvarlena; + getTypeOutputInfo(v->typeid, &outfunc, &isvarlena); + v->cstring = OidOutputFunctionCall(outfunc, v->datum); + } + v->owns_cstring = true; + } + PG_CATCH(); + { + MemoryContextSwitchTo(CurrentMemoryContext); + ErrorData *edata = CopyErrorData(); + elog(WARNING, "database_value_text: conversion failed for type %u: %s", v->typeid, edata->message); + FreeErrorData(edata); + FlushErrorState(); + v->cstring = NULL; + v->owns_cstring = true; // prevents retry of failed conversion + } + PG_END_TRY(); + } + + return v->cstring; +} + +int database_value_bytes (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v || v->isnull) return 0; + + // Exclude JSONB: binary internal format, must use OidOutputFunctionCall + if (pgvalue_is_text_type(v->typeid) && v->typeid != JSONBOID) { + pgvalue_ensure_detoast(v); + text *txt = (text *)DatumGetPointer(v->datum); + return VARSIZE_ANY_EXHDR(txt); + } + if (v->typeid == BYTEAOID) { + pgvalue_ensure_detoast(v); + bytea *ba = (bytea *)DatumGetPointer(v->datum); + return VARSIZE_ANY_EXHDR(ba); + } + // For unmapped types and JSONB (mapped to DBTYPE_TEXT), + // ensure the text representation is materialized + database_value_text(value); + if (v->cstring) { + return (int)strlen(v->cstring); + } + return 0; +} + +int database_value_type (dbvalue_t *value) { + return pgvalue_dbtype((pgvalue_t *)value); +} + +void database_value_free (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + pgvalue_free(v); +} + +void *database_value_dup (dbvalue_t *value) { + pgvalue_t *v = (pgvalue_t *)value; + if (!v) return NULL; + + pgvalue_t *copy = pgvalue_create(v->datum, v->typeid, v->typmod, v->collation, v->isnull); + + // Deep-copy pass-by-reference (varlena) datum data into TopMemoryContext + // so the copy survives SPI_finish() which destroys the caller's SPI context. + bool is_varlena = (v->typeid == BYTEAOID) || pgvalue_is_text_type(v->typeid); + if (is_varlena && !v->isnull) { + void *src = v->owned_detoast ? v->owned_detoast : DatumGetPointer(v->datum); + Size len = VARSIZE_ANY(src); + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + copy->owned_detoast = palloc(len); + MemoryContextSwitchTo(old); + memcpy(copy->owned_detoast, src, len); + copy->datum = PointerGetDatum(copy->owned_detoast); + copy->detoasted = true; + } + if (v->cstring) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + copy->cstring = pstrdup(v->cstring); + MemoryContextSwitchTo(old); + copy->owns_cstring = true; + } + return (void*)copy; +} + +// MARK: - SAVEPOINTS - + +static int database_refresh_snapshot (void) { + // Only manipulate snapshots in a valid transaction + if (!IsTransactionState()) { + return DBRES_OK; // Not in transaction, nothing to do + } + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + CommandCounterIncrement(); + + // Pop existing snapshot if any + if (ActiveSnapshotSet()) { + PopActiveSnapshot(); + } + + // Push fresh snapshot + PushActiveSnapshot(GetTransactionSnapshot()); + } + PG_CATCH(); + { + // Snapshot refresh failed - log warning but don't fail operation + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + elog(WARNING, "refresh_snapshot_after_command failed: %s", edata->message); + FreeErrorData(edata); + FlushErrorState(); + return DBRES_ERROR; + } + PG_END_TRY(); + + return DBRES_OK; +} + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + BeginInternalSubTransaction(NULL); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + } + PG_END_TRY(); + + return rc; +} + +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + if (GetCurrentTransactionNestLevel() <= 1) return DBRES_OK; + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + ReleaseCurrentSubTransaction(); + database_refresh_snapshot(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + rc = DBRES_ERROR; + } + PG_END_TRY(); + + return rc; +} + +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name) { + cloudsync_reset_error(data); + if (GetCurrentTransactionNestLevel() <= 1) return DBRES_OK; + int rc = DBRES_OK; + + MemoryContext oldcontext = CurrentMemoryContext; + PG_TRY(); + { + RollbackAndReleaseCurrentSubTransaction(); + database_refresh_snapshot(); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + ErrorData *edata = CopyErrorData(); + cloudsync_set_error(data, edata->message, DBRES_ERROR); + FreeErrorData(edata); + FlushErrorState(); + rc = DBRES_ERROR; + } + PG_END_TRY(); + + return rc; +} + +// MARK: - MEMORY - +// Use palloc in TopMemoryContext for PostgreSQL memory management integration. +// This provides memory tracking, debugging support, and proper cleanup on connection end. + +void *dbmem_alloc (uint64_t size) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *ptr = palloc(size); + MemoryContextSwitchTo(old); + return ptr; +} + +void *dbmem_zeroalloc (uint64_t size) { + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *ptr = palloc0(size); + MemoryContextSwitchTo(old); + return ptr; +} + +void *dbmem_realloc (void *ptr, uint64_t new_size) { + // repalloc doesn't accept NULL, unlike realloc + if (!ptr) return dbmem_alloc(new_size); + MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); + void *newptr = repalloc(ptr, new_size); + MemoryContextSwitchTo(old); + return newptr; +} + +char *dbmem_mprintf (const char *format, ...) { + if (!format) return NULL; + + va_list args; + va_start(args, format); + + // Calculate required buffer size + va_list args_copy; + va_copy(args_copy, args); + int len = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + + if (len < 0) { + va_end(args); + return NULL; + } + + // Allocate buffer in TopMemoryContext and format string + char *result = dbmem_alloc(len + 1); + if (!result) {va_end(args); return NULL;} + vsnprintf(result, len + 1, format, args); + + va_end(args); + return result; +} + +char *dbmem_vmprintf (const char *format, va_list list) { + if (!format) return NULL; + + // Calculate required buffer size + va_list args_copy; + va_copy(args_copy, list); + int len = vsnprintf(NULL, 0, format, args_copy); + va_end(args_copy); + + if (len < 0) return NULL; + + // Allocate buffer in TopMemoryContext and format string + char *result = dbmem_alloc(len + 1); + if (!result) return NULL; + vsnprintf(result, len + 1, format, list); + + return result; +} + +void dbmem_free (void *ptr) { + if (ptr) { + pfree(ptr); + } +} + +uint64_t dbmem_size (void *ptr) { + // palloc doesn't expose allocated size directly + // Return 0 as a safe default + return 0; +} + + diff --git a/src/postgresql/migrations/README.md b/src/postgresql/migrations/README.md new file mode 100644 index 0000000..497bf77 --- /dev/null +++ b/src/postgresql/migrations/README.md @@ -0,0 +1,74 @@ +# CloudSync PostgreSQL Migration Scripts + +This directory holds PostgreSQL extension upgrade scripts of the form: + + cloudsync----.sql + +PostgreSQL uses these to execute `ALTER EXTENSION cloudsync UPDATE` by chaining +one or more files to reach the target version. + +## Versioning model + +The PostgreSQL extension version (`default_version` in `cloudsync.control`) is +**`MAJOR.MINOR`** only — derived from the first two components of +`CLOUDSYNC_VERSION` in `src/cloudsync.h`. The full semver of the compiled +binary is reported by the `cloudsync_version()` SQL function. + +| Release kind | Example | EXTVERSION moves? | Upgrade script? | User action | +| -------------------------------- | --------------- | ----------------- | --------------- | ------------------ | +| PATCH bump (binary only) | 1.0.16 → 1.0.17 | No (stays `1.0`) | Not required | Swap the `.so`. | +| MINOR bump (SQL surface changes) | 1.0.x → 1.1.0 | Yes (`1.0` → `1.1`) | Required | `ALTER EXTENSION cloudsync UPDATE;` | +| MAJOR bump | 1.x → 2.0.0 | Yes (`1.x` → `2.0`) | Required | `ALTER EXTENSION cloudsync UPDATE;` | + +CI enforces this contract via `scripts/check-postgres-migration.sh`: + +- PATCH releases: the script diffs the current `cloudsync.sql.in` against the + previous release's install script. If they differ, the build fails — + accidental SQL surface drift in a PATCH release would silently break users + whose `pg_extension.extversion` would otherwise stay at the old EXTVERSION. +- MINOR/MAJOR releases: the script requires a matching + `cloudsync----.sql` in this directory. + +## When to add an upgrade script + +Add one **only when `EXTVERSION` changes** — i.e. when you bump MINOR or MAJOR +in `src/cloudsync.h`. The filename is literally +`cloudsync----.sql`, not the full semver. + +Examples: + + 1.0.17 -> 1.1.0 → cloudsync--1.0--1.1.sql + 1.1.5 -> 2.0.0 → cloudsync--1.1--2.0.sql + +PATCH-level releases (1.0.16 → 1.0.17 → 1.0.18 → …) require **no file** — the +catalog's `installed_version` stays at the MAJOR.MINOR, and the `.so` swap is +a transparent binary upgrade. + +## Rules for upgrade script content + +- Use `CREATE OR REPLACE` for every function — the underlying C symbol may + have changed even when the SQL signature didn't. +- Drop removed objects explicitly (`DROP FUNCTION IF EXISTS ...`). +- Never run `CREATE EXTENSION`-style bootstrap inside an upgrade script. +- Objects created inside an upgrade script are automatically attached to the + extension via `pg_depend`. +- Scripts are packaged and installed by `make postgres-install` / + `make postgres-package` via a wildcard; no need to list them anywhere. + +## Verifying the chain + +After rebuilding, inside the PG container: + + SELECT * FROM pg_extension_update_paths('cloudsync'); + +All `source -> target` rows should show a non-NULL `path`. + +## I intended a PATCH but the CI check says "SQL surface drift" + +Either: + +1. You meant to change SQL. Bump MINOR in `src/cloudsync.h` and add + `cloudsync----.sql`. +2. You didn't. Revert the change in `src/postgresql/cloudsync.sql.in`. + +The CI error message prints a unified diff to help you decide which. diff --git a/src/postgresql/pgvalue.c b/src/postgresql/pgvalue.c new file mode 100644 index 0000000..69fd626 --- /dev/null +++ b/src/postgresql/pgvalue.c @@ -0,0 +1,198 @@ +// +// pgvalue.c +// PostgreSQL-specific dbvalue_t helpers +// + +#include "pgvalue.h" + +#include "catalog/pg_type.h" +#include "utils/lsyscache.h" +#include "utils/builtins.h" +#include "../utils.h" + +bool pgvalue_is_text_type(Oid typeid) { + switch (typeid) { + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + case NAMEOID: + case JSONOID: + case JSONBOID: + case XMLOID: + return true; + default: + return false; + } +} + +static bool pgvalue_is_varlena(Oid typeid) { + return (typeid == BYTEAOID) || pgvalue_is_text_type(typeid); +} + +pgvalue_t *pgvalue_create(Datum datum, Oid typeid, int32 typmod, Oid collation, bool isnull) { + pgvalue_t *v = cloudsync_memory_zeroalloc(sizeof(pgvalue_t)); + if (!v) return NULL; + + v->datum = datum; + v->typeid = typeid; + v->typmod = typmod; + v->collation = collation; + v->isnull = isnull; + return v; +} + +void pgvalue_free (pgvalue_t *v) { + if (!v) return; + + if (v->owned_detoast) { + pfree(v->owned_detoast); + } + if (v->owns_cstring && v->cstring) { + pfree(v->cstring); + } + cloudsync_memory_free(v); +} + +void pgvalue_ensure_detoast(pgvalue_t *v) { + if (!v || v->detoasted) return; + if (!pgvalue_is_varlena(v->typeid) || v->isnull) return; + + v->owned_detoast = (void *)PG_DETOAST_DATUM_COPY(v->datum); + v->datum = PointerGetDatum(v->owned_detoast); + v->detoasted = true; +} + +int pgvalue_dbtype(pgvalue_t *v) { + if (!v || v->isnull) return DBTYPE_NULL; + switch (v->typeid) { + case INT2OID: + case INT4OID: + case INT8OID: + case BOOLOID: + case CHAROID: + case OIDOID: + return DBTYPE_INTEGER; + case FLOAT4OID: + case FLOAT8OID: + case NUMERICOID: + return DBTYPE_FLOAT; + case BYTEAOID: + return DBTYPE_BLOB; + default: + if (pgvalue_is_text_type(v->typeid)) { + return DBTYPE_TEXT; + } + return DBTYPE_TEXT; + } +} + +static bool pgvalue_vec_push(pgvalue_t ***arr, int *count, int *cap, pgvalue_t *val) { + if (*cap == 0) { + *cap = 8; + *arr = (pgvalue_t **)cloudsync_memory_zeroalloc(sizeof(pgvalue_t *) * (*cap)); + if (*arr == NULL) return false; + } else if (*count >= *cap) { + *cap *= 2; + *arr = (pgvalue_t **)cloudsync_memory_realloc(*arr, sizeof(pgvalue_t *) * (*cap)); + if (*arr == NULL) return false; + } + (*arr)[(*count)++] = val; + return true; +} + +pgvalue_t **pgvalues_from_array(ArrayType *array, int *out_count) { + if (out_count) *out_count = 0; + if (!array) return NULL; + + Oid elem_type = ARR_ELEMTYPE(array); + int16 elmlen; + bool elmbyval; + char elmalign; + get_typlenbyvalalign(elem_type, &elmlen, &elmbyval, &elmalign); + + Datum *elems = NULL; + bool *nulls = NULL; + int nelems = 0; + + deconstruct_array(array, elem_type, elmlen, elmbyval, elmalign, &elems, &nulls, &nelems); + + pgvalue_t **values = NULL; + int count = 0; + int cap = 0; + + for (int i = 0; i < nelems; i++) { + pgvalue_t *v = pgvalue_create(elems[i], elem_type, -1, InvalidOid, nulls ? nulls[i] : false); + pgvalue_vec_push(&values, &count, &cap, v); + } + + if (elems) pfree(elems); + if (nulls) pfree(nulls); + + if (out_count) *out_count = count; + return values; +} + +pgvalue_t **pgvalues_from_args(FunctionCallInfo fcinfo, int start_arg, int *out_count) { + if (out_count) *out_count = 0; + if (!fcinfo) return NULL; + + pgvalue_t **values = NULL; + int count = 0; + int cap = 0; + + for (int i = start_arg; i < PG_NARGS(); i++) { + Oid argtype = get_fn_expr_argtype(fcinfo->flinfo, i); + bool isnull = PG_ARGISNULL(i); + + // If the argument is an array (used for VARIADIC pk functions), expand it. + Oid elemtype = InvalidOid; + if (OidIsValid(argtype)) { + elemtype = get_element_type(argtype); + } + + if (OidIsValid(elemtype) && !isnull) { + ArrayType *array = PG_GETARG_ARRAYTYPE_P(i); + int subcount = 0; + pgvalue_t **subvals = pgvalues_from_array(array, &subcount); + for (int j = 0; j < subcount; j++) { + pgvalue_vec_push(&values, &count, &cap, subvals[j]); + } + if (subvals) cloudsync_memory_free(subvals); + continue; + } + + Datum datum = isnull ? (Datum)0 : PG_GETARG_DATUM(i); + pgvalue_t *v = pgvalue_create(datum, argtype, -1, fcinfo->fncollation, isnull); + pgvalue_vec_push(&values, &count, &cap, v); + } + + if (out_count) *out_count = count; + return values; +} + +void pgvalues_normalize_to_text(pgvalue_t **values, int count) { + // Convert all non-text pgvalues to text representation. + // This ensures PK encoding is consistent regardless of whether the caller + // passes native types (e.g., integer 1) or text representations (e.g., '1'). + // The UPDATE trigger casts all values to ::text, so INSERT trigger and + // SQL functions must do the same for PK encoding consistency. + if (!values) return; + + for (int i = 0; i < count; i++) { + pgvalue_t *v = values[i]; + if (!v || v->isnull) continue; + if (pgvalue_is_text_type(v->typeid)) continue; + + // Convert to text using the type's output function + const char *cstr = database_value_text((dbvalue_t *)v); + if (!cstr) continue; + + // Create a new text datum + text *t = cstring_to_text(cstr); + pgvalue_t *new_v = pgvalue_create(PointerGetDatum(t), TEXTOID, -1, v->collation, false); + if (new_v) { + pgvalue_free(v); + values[i] = new_v; + } + } +} diff --git a/src/postgresql/pgvalue.h b/src/postgresql/pgvalue.h new file mode 100644 index 0000000..3fbd28b --- /dev/null +++ b/src/postgresql/pgvalue.h @@ -0,0 +1,44 @@ +// pgvalue.h +// PostgreSQL-specific dbvalue_t wrapper + +#ifndef CLOUDSYNC_PGVALUE_H +#define CLOUDSYNC_PGVALUE_H + +// Define POSIX feature test macros before any includes +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "postgres.h" +#include "fmgr.h" +#include "utils/memutils.h" +#include "utils/array.h" +#include "../database.h" + +// dbvalue_t representation for PostgreSQL. We capture Datum + type metadata so +// value helpers can resolve type/length/ownership without relying on fcinfo lifetime. +typedef struct pgvalue_t { + Datum datum; + Oid typeid; + int32 typmod; + Oid collation; + bool isnull; + bool detoasted; + void *owned_detoast; + char *cstring; + bool owns_cstring; +} pgvalue_t; + +pgvalue_t *pgvalue_create(Datum datum, Oid typeid, int32 typmod, Oid collation, bool isnull); +void pgvalue_free (pgvalue_t *v); +void pgvalue_ensure_detoast(pgvalue_t *v); +bool pgvalue_is_text_type(Oid typeid); +int pgvalue_dbtype(pgvalue_t *v); +pgvalue_t **pgvalues_from_array(ArrayType *array, int *out_count); +pgvalue_t **pgvalues_from_args(FunctionCallInfo fcinfo, int start_arg, int *out_count); +void pgvalues_normalize_to_text(pgvalue_t **values, int count); + +#endif // CLOUDSYNC_PGVALUE_H diff --git a/src/postgresql/postgresql_log.h b/src/postgresql/postgresql_log.h new file mode 100644 index 0000000..94e0e12 --- /dev/null +++ b/src/postgresql/postgresql_log.h @@ -0,0 +1,27 @@ +// +// postgresql_log.h +// cloudsync +// +// PostgreSQL-specific logging implementation using elog() +// +// Note: This header requires _POSIX_C_SOURCE and _GNU_SOURCE to be defined +// before any includes. These are set as compiler flags in Makefile.postgresql. +// + +#ifndef __POSTGRESQL_LOG__ +#define __POSTGRESQL_LOG__ + +// setjmp.h is needed before postgres.h for sigjmp_buf type +#include + +// Include PostgreSQL headers +#include "postgres.h" +#include "utils/elog.h" + +// PostgreSQL logging macros using elog() +// DEBUG1 is the highest priority debug level in PostgreSQL +// LOG is for informational messages +#define CLOUDSYNC_LOG_DEBUG(...) elog(DEBUG1, __VA_ARGS__) +#define CLOUDSYNC_LOG_INFO(...) elog(LOG, __VA_ARGS__) + +#endif diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c new file mode 100644 index 0000000..44ea2c1 --- /dev/null +++ b/src/postgresql/sql_postgresql.c @@ -0,0 +1,451 @@ +// +// sql_postgresql.c +// cloudsync +// +// PostgreSQL-specific SQL queries +// Created by Claude Code on 22/12/25. +// + +#include "../sql.h" + +// MARK: Settings + +const char * const SQL_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_settings WHERE key=$1;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE = + "INSERT INTO cloudsync_settings (key, value) VALUES ($1, $2) " + "ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE = + "DELETE FROM cloudsync_settings WHERE key = $1;"; + +const char * const SQL_TABLE_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=$1 AND col_name=$2 AND key=$3);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE = + "DELETE FROM cloudsync_table_settings WHERE tbl_name=$1;"; + +const char * const SQL_TABLE_SETTINGS_REPLACE = + "INSERT INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES ($1, $2, $3, $4) " + "ON CONFLICT (tbl_name, col_name, key) DO UPDATE SET value = EXCLUDED.value;"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ONE = + "DELETE FROM cloudsync_table_settings WHERE (tbl_name=$1 AND col_name=$2 AND key=$3);"; + +const char * const SQL_TABLE_SETTINGS_COUNT_TABLES = + "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"; + +const char * const SQL_SETTINGS_LOAD_GLOBAL = + "SELECT key, value FROM cloudsync_settings;"; + +const char * const SQL_SETTINGS_LOAD_TABLE = + "SELECT lower(tbl_name), lower(col_name), key, value FROM cloudsync_table_settings ORDER BY tbl_name, col_name;"; + +const char * const SQL_CREATE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_settings (key TEXT PRIMARY KEY NOT NULL, value TEXT);" + "CREATE TABLE IF NOT EXISTS public.app_schema_version (" + "version BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY" + ");" + "CREATE OR REPLACE FUNCTION bump_app_schema_version() " + "RETURNS event_trigger AS $$ " + "BEGIN " + "INSERT INTO public.app_schema_version DEFAULT VALUES; " + "END;" + "$$ LANGUAGE plpgsql;" + "DROP EVENT TRIGGER IF EXISTS app_schema_change;" + "CREATE EVENT TRIGGER app_schema_change " + "ON ddl_command_end " + "EXECUTE FUNCTION bump_app_schema_version();"; + +// format strings (snprintf) are also static SQL templates +const char * const SQL_INSERT_SETTINGS_STR_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', '%s');"; + +const char * const SQL_INSERT_SETTINGS_INT_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', %lld);"; + +const char * const SQL_CREATE_SITE_ID_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_site_id (" + "id BIGSERIAL PRIMARY KEY, " + "site_id BYTEA UNIQUE NOT NULL" + ");"; + +const char * const SQL_INSERT_SITE_ID_ROWID = + "INSERT INTO cloudsync_site_id (id, site_id) VALUES ($1, $2);"; + +const char * const SQL_CREATE_TABLE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_table_settings (tbl_name TEXT NOT NULL, col_name TEXT NOT NULL, key TEXT NOT NULL, value TEXT, PRIMARY KEY(tbl_name,col_name,key));"; + +const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_schema_versions (hash BIGINT PRIMARY KEY, seq INTEGER NOT NULL)"; + +const char * const SQL_SETTINGS_CLEANUP_DROP_ALL = + "DROP TABLE IF EXISTS cloudsync_settings CASCADE; " + "DROP TABLE IF EXISTS cloudsync_site_id CASCADE; " + "DROP TABLE IF EXISTS cloudsync_table_settings CASCADE; " + "DROP TABLE IF EXISTS cloudsync_schema_versions CASCADE;"; + +// MARK: CloudSync + +const char * const SQL_DBVERSION_BUILD_QUERY = + "WITH table_names AS (" + "SELECT quote_ident(schemaname) || '.' || quote_ident(tablename) as tbl_name " + "FROM pg_tables " + "WHERE tablename LIKE '%_cloudsync'" + "), " + "query_parts AS (" + "SELECT tbl_name, " + "format('SELECT COALESCE(MAX(db_version), 0) FROM %s', tbl_name) as part " + "FROM table_names" + ") " + "SELECT string_agg(part, ' UNION ALL ') FROM query_parts;"; + +const char * const SQL_CHANGES_INSERT_ROW = + "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9);"; + +// MARK: Additional SQL constants for PostgreSQL + +const char * const SQL_SITEID_SELECT_ROWID0 = + "SELECT site_id FROM cloudsync_site_id WHERE id = 0;"; + +const char * const SQL_DATA_VERSION = + "SELECT txid_snapshot_xmin(txid_current_snapshot());"; // was "PRAGMA data_version" + +const char * const SQL_SCHEMA_VERSION = + "SELECT COALESCE(max(version), 0) FROM app_schema_version;"; // was "PRAGMA schema_version" + +const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID = + "INSERT INTO cloudsync_site_id (site_id) VALUES ($1) " + "ON CONFLICT(site_id) DO UPDATE SET site_id = EXCLUDED.site_id " + "RETURNING id;"; + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID = + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.columns " + "WHERE table_name = $1 AND column_name NOT IN (" + "SELECT column_name FROM information_schema.key_column_usage " + "WHERE table_name = $1 AND constraint_name LIKE '%_pkey'" + ");"; // TODO: build full SELECT ... WHERE ctid=? analog with ordered columns like SQLite + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "nonpk AS (" + " SELECT a.attname " + " FROM pg_attribute a " + " JOIN tbl t ON t.oid = a.attrelid " + " WHERE a.attnum > 0 AND NOT a.attisdropped " + " AND a.attnum NOT IN (" + " SELECT k.attnum " + " FROM pg_index x " + " JOIN tbl t2 ON t2.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) AS k(attnum) ON true " + " WHERE x.indisprimary" + " ) " + " ORDER BY a.attnum" + ") " + "SELECT " + " 'SELECT '" + " || (SELECT string_agg(format('%%I', attname), ',') FROM nonpk)" + " || ' FROM ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_DELETE_ROW_BY_ROWID = + "DELETE FROM %s WHERE ctid = $1;"; // TODO: consider using PK-based deletion; ctid is unstable + +const char * const SQL_BUILD_DELETE_ROW_BY_PK = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + ") " + "SELECT " + " 'DELETE FROM ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_INSERT_ROWID_IGNORE = + "INSERT INTO %s DEFAULT VALUES ON CONFLICT DO NOTHING;"; // TODO: adapt to explicit PK inserts (no rowid in PG) + +const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID = + "INSERT INTO %s (ctid, %s) VALUES ($1, $2) " + "ON CONFLICT DO UPDATE SET %s = $2;"; // TODO: align with SQLite upsert by rowid; avoid ctid + +const char * const SQL_BUILD_INSERT_PK_IGNORE = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',') FROM pk) || ')'" + " || ' ON CONFLICT DO NOTHING;';"; + +const char * const SQL_BUILD_UPSERT_PK_AND_COL = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS oid" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.oid = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "pk_count AS (" + " SELECT count(*) AS n FROM pk" + "), " + "col AS (" + " SELECT '%s'::text AS colname, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_attribute a " + " JOIN tbl t ON t.oid = a.attrelid " + " WHERE a.attname = '%s' AND a.attnum > 0 AND NOT a.attisdropped" + ") " + "SELECT " + " 'INSERT INTO ' || (SELECT (oid::regclass)::text FROM tbl)" + " || ' (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk)" + " || ',' || (SELECT format('%%I', colname) FROM col) || ')'" + " || ' VALUES (' || (SELECT string_agg(format('$%%s::%%s', ord, coltype), ',') FROM pk)" + " || ',' || (SELECT format('$%%s::%%s', (SELECT n FROM pk_count) + 1, coltype) FROM col) || ')'" + " || ' ON CONFLICT (' || (SELECT string_agg(format('%%I', attname), ',') FROM pk) || ')'" + " || ' DO UPDATE SET ' || (SELECT format('%%I', colname) FROM col)" + " || '=' || (SELECT format('$%%s::%%s', (SELECT n FROM pk_count) + 2, coltype) FROM col) || ';';"; + +const char * const SQL_SELECT_COLS_BY_ROWID_FMT = + "SELECT %s%s%s FROM %s WHERE ctid = $1;"; // TODO: align with PK/rowid selection builder + +const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT = + "WITH tbl AS (" + " SELECT to_regclass('%s') AS tblreg" + "), " + "pk AS (" + " SELECT a.attname, k.ord, format_type(a.atttypid, a.atttypmod) AS coltype " + " FROM pg_index x " + " JOIN tbl t ON t.tblreg = x.indrelid " + " JOIN LATERAL unnest(x.indkey) WITH ORDINALITY AS k(attnum, ord) ON true " + " JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = k.attnum " + " WHERE x.indisprimary " + " ORDER BY k.ord" + "), " + "col AS (" + " SELECT '%s'::text AS colname" + ") " + "SELECT " + " 'SELECT ' || (SELECT format('%%I', colname) FROM col) " + " || ' FROM ' || (SELECT tblreg::text FROM tbl)" + " || ' WHERE '" + " || (SELECT string_agg(format('%%I=$%%s::%%s', attname, ord, coltype), ' AND ' ORDER BY ord) FROM pk)" + " || ';';"; + +const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK = + "SELECT EXISTS(SELECT 1 FROM %s WHERE pk = $1 LIMIT 1);"; + +const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION = + "UPDATE %s " + "SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = $1, seq = $2, site_id = 0 " + "WHERE pk = $3 AND col_name = '%s';"; + +const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, '%s', 1, $2, $3, 0) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = CASE %s.col_version %% 2 WHEN 0 THEN %s.col_version + 1 ELSE %s.col_version + 2 END, " + "db_version = $2, seq = $3, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, $4, $5, 0) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = %s.col_version + 1, db_version = $6, seq = $7, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL = + "DELETE FROM %s WHERE pk = $1 AND col_name != '%s';"; // TODO: match SQLite delete semantics + +const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL = + "WITH moved AS (" + " SELECT col_name " + " FROM %s WHERE pk = $3 AND col_name != '%s'" + "), " + "upserted AS (" + " INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + " SELECT $1, col_name, 1, $2, cloudsync_seq(), 0 " + " FROM moved " + " ON CONFLICT (pk, col_name) DO UPDATE SET " + " col_version = 1, db_version = $2, seq = cloudsync_seq(), site_id = 0" + ") " + "DELETE FROM %s WHERE pk = $3 AND col_name != '%s';"; + +const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS = + "SELECT COALESCE(" + "(SELECT col_version FROM %s WHERE pk = $1 AND col_name = '%s'), " + "(SELECT 1 FROM %s WHERE pk = $1 LIMIT 1)" + ");"; + +const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID = + "INSERT INTO %s " + "(pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, cloudsync_db_version_next($4), $5, $6) " + "ON CONFLICT (pk, col_name) DO UPDATE SET " + "col_version = EXCLUDED.col_version, " + "db_version = cloudsync_db_version_next($4), " + "seq = EXCLUDED.seq, " + "site_id = EXCLUDED.site_id " + "RETURNING ((db_version::bigint << 30) | seq);"; // TODO: align RETURNING and bump logic with SQLite (version increments on conflict) + +const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL = + "UPDATE %s " + "SET col_version = 0, db_version = cloudsync_db_version_next($1) " + "WHERE pk = $2 AND col_name != '%s';"; // TODO: confirm tombstone semantics match SQLite + +const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL = + "SELECT col_version FROM %s WHERE pk = $1 AND col_name = $2;"; + +const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL = + "SELECT site_id FROM %s WHERE pk = $1 AND col_name = $2;"; + +const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID = + "SELECT c.column_name, c.ordinal_position " + "FROM information_schema.columns c " + "WHERE c.table_name = '%s' " + "AND c.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "AND c.column_name NOT IN (" + " SELECT kcu.column_name FROM information_schema.table_constraints tc " + " JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + " WHERE tc.table_name = '%s' AND tc.table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + " AND tc.constraint_type = 'PRIMARY KEY'" + ") " + "ORDER BY ordinal_position;"; + +const char * const SQL_DROP_CLOUDSYNC_TABLE = + "DROP TABLE IF EXISTS %s CASCADE;"; + +const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE = + "DELETE FROM %s;"; + +const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL = + "DELETE FROM %s WHERE col_name NOT IN (" + "SELECT column_name FROM information_schema.columns WHERE table_name = '%s' " + "AND table_schema = COALESCE(NULLIF('%s', ''), current_schema()) " + "UNION SELECT '%s'" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT = + "SELECT string_agg(quote_ident(column_name), ',' ORDER BY ordinal_position) " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK = + "DELETE FROM %s " + "WHERE (col_name != '%s' OR (col_name = '%s' AND col_version %% 2 != 0)) " + "AND NOT EXISTS (" + "SELECT 1 FROM %s " + "WHERE %s.pk = cloudsync_pk_encode(%s) LIMIT 1" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST = + "SELECT string_agg(quote_ident(column_name), ',') " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST = + "SELECT string_agg(" + "'cloudsync_pk_decode(pk, ' || ordinal_position || ') AS ' || quote_ident(column_name), ',' ORDER BY ordinal_position" + ") " + "FROM information_schema.key_column_usage " + "WHERE table_name = '%s' AND table_schema = COALESCE(cloudsync_schema(), current_schema()) " + "AND constraint_name LIKE '%%_pkey';"; + +const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC = + "SELECT cloudsync_insert('%s', %s) " + "FROM (SELECT %s FROM %s EXCEPT SELECT %s FROM %s);"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM %s) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM %s _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = $1" + ");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM %s WHERE (%s)) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM %s _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = $1" + ");"; + +// MARK: Blocks (block-level LWW) + +const char * const SQL_BLOCKS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS %s (" + "pk BYTEA NOT NULL, " + "col_name TEXT COLLATE \"C\" NOT NULL, " + "col_value TEXT, " + "PRIMARY KEY (pk, col_name))"; + +const char * const SQL_BLOCKS_UPSERT = + "INSERT INTO %s (pk, col_name, col_value) VALUES ($1, $2, $3) " + "ON CONFLICT (pk, col_name) DO UPDATE SET col_value = EXCLUDED.col_value"; + +const char * const SQL_BLOCKS_SELECT = + "SELECT col_value FROM %s WHERE pk = $1 AND col_name = $2"; + +const char * const SQL_BLOCKS_DELETE = + "DELETE FROM %s WHERE pk = $1 AND col_name = $2"; + +const char * const SQL_BLOCKS_LIST_ALIVE = + "SELECT b.col_value FROM %s b " + "JOIN %s m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.pk = $1 AND b.col_name LIKE $2 " + "AND m.pk = $3 AND m.col_name LIKE $4 AND m.col_version %% 2 = 1 " + "ORDER BY b.col_name COLLATE \"C\""; + +const char * const SQL_BLOCKS_INSERT_IGNORE = + "INSERT INTO %s (pk, col_name, col_value) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"; + +const char * const SQL_META_SCAN_COL_FOR_MIGRATION = + "SELECT DISTINCT m.pk FROM %s m " + "WHERE m.col_name = $1 AND m.col_version %% 2 = 1 " + "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE $2)"; + +const char * const SQL_META_INSERT_BLOCK_IGNORE = + "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES ($1, $2, $3, $4, $5, 0) ON CONFLICT DO NOTHING"; diff --git a/src/sql.h b/src/sql.h new file mode 100644 index 0000000..d9b9f0d --- /dev/null +++ b/src/sql.h @@ -0,0 +1,81 @@ +// +// sql.h +// cloudsync +// +// Created by Marco Bambini on 17/12/25. +// + +#ifndef __CLOUDSYNC_SQL__ +#define __CLOUDSYNC_SQL__ + +// SETTINGS +extern const char * const SQL_SETTINGS_GET_VALUE; +extern const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE; +extern const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE; +extern const char * const SQL_TABLE_SETTINGS_GET_VALUE; +extern const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE; +extern const char * const SQL_TABLE_SETTINGS_REPLACE; +extern const char * const SQL_TABLE_SETTINGS_DELETE_ONE; +extern const char * const SQL_TABLE_SETTINGS_COUNT_TABLES; +extern const char * const SQL_SETTINGS_LOAD_GLOBAL; +extern const char * const SQL_SETTINGS_LOAD_TABLE; +extern const char * const SQL_CREATE_SETTINGS_TABLE; +extern const char * const SQL_INSERT_SETTINGS_STR_FORMAT; +extern const char * const SQL_INSERT_SETTINGS_INT_FORMAT; +extern const char * const SQL_CREATE_SITE_ID_TABLE; +extern const char * const SQL_INSERT_SITE_ID_ROWID; +extern const char * const SQL_CREATE_TABLE_SETTINGS_TABLE; +extern const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE; +extern const char * const SQL_SETTINGS_CLEANUP_DROP_ALL; + +// CLOUDSYNC +extern const char * const SQL_DBVERSION_BUILD_QUERY; +extern const char * const SQL_SITEID_SELECT_ROWID0; +extern const char * const SQL_DATA_VERSION; +extern const char * const SQL_SCHEMA_VERSION; +extern const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID; +extern const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID; +extern const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK; +extern const char * const SQL_DELETE_ROW_BY_ROWID; +extern const char * const SQL_BUILD_DELETE_ROW_BY_PK; +extern const char * const SQL_INSERT_ROWID_IGNORE; +extern const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID; +extern const char * const SQL_BUILD_INSERT_PK_IGNORE; +extern const char * const SQL_BUILD_UPSERT_PK_AND_COL; +extern const char * const SQL_SELECT_COLS_BY_ROWID_FMT; +extern const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT; +extern const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK; +extern const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION; +extern const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION; +extern const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION; +extern const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS; +extern const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID; +extern const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL; +extern const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID; +extern const char * const SQL_DROP_CLOUDSYNC_TABLE; +extern const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE; +extern const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT; +extern const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST; +extern const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST; +extern const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC; +extern const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL; +extern const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED; +extern const char * const SQL_CHANGES_INSERT_ROW; + +// BLOCKS (block-level LWW) +extern const char * const SQL_BLOCKS_CREATE_TABLE; +extern const char * const SQL_BLOCKS_UPSERT; +extern const char * const SQL_BLOCKS_SELECT; +extern const char * const SQL_BLOCKS_DELETE; +extern const char * const SQL_BLOCKS_LIST_ALIVE; +extern const char * const SQL_BLOCKS_INSERT_IGNORE; +extern const char * const SQL_META_SCAN_COL_FOR_MIGRATION; +extern const char * const SQL_META_INSERT_BLOCK_IGNORE; + +#endif diff --git a/src/vtab.c b/src/sqlite/cloudsync_changes_sqlite.c similarity index 76% rename from src/vtab.c rename to src/sqlite/cloudsync_changes_sqlite.c index 09c2fb6..58a9b88 100644 --- a/src/vtab.c +++ b/src/sqlite/cloudsync_changes_sqlite.c @@ -1,5 +1,5 @@ // -// vtab.c +// cloudsync_changes_sqlite.c // cloudsync // // Created by Marco Bambini on 23/09/24. @@ -7,10 +7,10 @@ #include #include -#include "vtab.h" -#include "utils.h" -#include "dbutils.h" -#include "cloudsync.h" + +#include "cloudsync_changes_sqlite.h" +#include "../utils.h" +#include "../dbutils.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT3 @@ -19,7 +19,7 @@ SQLITE_EXTENSION_INIT3 typedef struct cloudsync_changes_vtab { sqlite3_vtab base; // base class, must be first sqlite3 *db; - void *aux; + cloudsync_context *data; } cloudsync_changes_vtab; typedef struct cloudsync_changes_cursor { @@ -49,7 +49,18 @@ bool force_vtab_filter_abort = false; // MARK: - -const char *opname_from_value (int value) { +int vtab_set_error (sqlite3_vtab *vtab, const char *format, ...) { + va_list arg; + va_start (arg, format); + char *err = sqlite3_vmprintf(format, arg); + va_end (arg); + + if (vtab->zErrMsg) sqlite3_free(vtab->zErrMsg); + vtab->zErrMsg = err; + return SQLITE_ERROR; +} + +const char *vtab_opname_from_value (int value) { switch (value) { case SQLITE_INDEX_CONSTRAINT_EQ: return "="; case SQLITE_INDEX_CONSTRAINT_GT: return ">"; @@ -79,7 +90,7 @@ const char *opname_from_value (int value) { return NULL; } -int colname_is_legal (const char *name) { +int vtab_colname_is_legal (const char *name) { int count = sizeof(cloudsync_changes_columns) / sizeof (char *); for (int i=0; idb = db; - vnew->aux = aux; + vnew->data = aux; *vtab = (sqlite3_vtab *)vnew; } @@ -284,8 +296,8 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id int idx = constraint->iColumn; uint8_t op = constraint->op; - const char *colname = (idx > 0) ? COLNAME_FROM_INDEX(idx) : "rowid"; - const char *opname = opname_from_value(op); + const char *colname = (idx >= 0 && idx < CLOUDSYNC_CHANGES_NCOLS) ? COLNAME_FROM_INDEX(idx) : "rowid"; + const char *opname = vtab_opname_from_value(op); if (!opname) continue; // build next constraint @@ -318,8 +330,8 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id if (i > 0) sindex += snprintf(s+sindex, slen-sindex, ", "); int idx = orderby->iColumn; - const char *colname = COLNAME_FROM_INDEX(idx); - if (!colname_is_legal(colname)) orderconsumed = 0; + const char *colname = (idx >= 0 && idx < CLOUDSYNC_CHANGES_NCOLS) ? COLNAME_FROM_INDEX(idx) : "rowid"; + if (!vtab_colname_is_legal(colname)) orderconsumed = 0; sindex += snprintf(s+sindex, slen-sindex, "%s %s", colname, orderby->desc ? " DESC" : " ASC"); } @@ -383,11 +395,26 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id int cloudsync_changesvtab_filter (sqlite3_vtab_cursor *cursor, int idxn, const char *idxs, int argc, sqlite3_value **argv) { DEBUG_VTAB("cloudsync_changesvtab_filter"); - + cloudsync_changes_cursor *c = (cloudsync_changes_cursor *)cursor; + cloudsync_context *data = c->vtab->data; sqlite3 *db = c->vtab->db; - char *sql = build_changes_sql(db, idxs); - if (sql == NULL) return SQLITE_NOMEM; + char *sql = vtab_build_changes_sql(data, idxs); + if (sql == NULL) { + // vtab_build_changes_sql returns NULL when no *_cloudsync meta-tables + // exist (cloudsync_init was never called, or the last configured table + // was cleaned up): the inner GROUP_CONCAT produces a NULL row and the + // outer SELECT yields a NULL final string. Distinguish this from a + // genuine OOM by checking whether cloudsync is configured, so the user + // gets an actionable message instead of "out of memory". + if (!cloudsync_config_exists(data) || dbutils_table_settings_count_tables(data) == 0) { + return vtab_set_error((sqlite3_vtab *)c->vtab, + "cloudsync has no tables configured for sync. Call " + "SELECT cloudsync_init('') to enable sync on a " + "table before querying cloudsync_changes."); + } + return SQLITE_NOMEM; + } // the xFilter method may be called multiple times on the same sqlite3_vtab_cursor* if (c->vm) sqlite3_finalize(c->vm); @@ -472,39 +499,94 @@ int cloudsync_changesvtab_rowid (sqlite3_vtab_cursor *cursor, sqlite3_int64 *row return SQLITE_OK; } +int cloudsync_changesvtab_insert_gos (sqlite3_vtab *vtab, cloudsync_context *data, cloudsync_table_context *table, const char *insert_pk, int insert_pk_len, const char *insert_name, sqlite3_value *insert_value, sqlite3_int64 insert_col_version, sqlite3_int64 insert_db_version, const char *insert_site_id, int insert_site_id_len, sqlite3_int64 insert_seq, int64_t *rowid) { + DEBUG_VTAB("cloudsync_changesvtab_insert_gos"); + + // Grow-Only Set (GOS) Algorithm: Only insertions are allowed, deletions and updates are prevented from a trigger. + int rc = merge_insert_col(data, table, insert_pk, insert_pk_len, insert_name, insert_value, (int64_t)insert_col_version, (int64_t)insert_db_version, insert_site_id, insert_site_id_len, (int64_t)insert_seq, rowid); + + if (rc != SQLITE_OK) { + vtab_set_error(vtab, "%s", cloudsync_errmsg(data)); + } + + return rc; +} + +int cloudsync_changesvtab_insert (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { + DEBUG_VTAB("cloudsync_changesvtab_insert"); + + // this function performs the merging logic for an insert in a cloud-synchronized table. It handles + // different scenarios including conflicts, causal lengths, delete operations, and resurrecting rows + // based on the incoming data (from remote nodes or clients) and the local database state + + // this function handles different CRDT algorithms (GOS, DWS, AWS, and CLS). + // the merging strategy is determined based on the table->algo value. + + // meta table declaration: + // tbl TEXT NOT NULL, pk BLOB NOT NULL, col_name TEXT NOT NULL," + // "col_value ANY, col_version INTEGER NOT NULL, db_version INTEGER NOT NULL," + // "site_id BLOB NOT NULL, cl INTEGER NOT NULL, seq INTEGER NOT NULL + + // meta information to retrieve from arguments: + // argv[0] -> table name (TEXT) + // argv[1] -> primary key (BLOB) + // argv[2] -> column name (TEXT or NULL if sentinel) + // argv[3] -> column value (ANY) + // argv[4] -> column version (INTEGER) + // argv[5] -> database version (INTEGER) + // argv[6] -> site ID (BLOB, identifies the origin of the update) + // argv[7] -> causal length (INTEGER, tracks the order of operations) + // argv[8] -> sequence number (INTEGER, unique per operation) + + // extract table name + const char *insert_tbl = (const char *)sqlite3_value_text(argv[0]); + + // lookup table + cloudsync_context *data = (cloudsync_context *)(((cloudsync_changes_vtab *)vtab)->data); + cloudsync_table_context *table = table_lookup(data, insert_tbl); + if (!table) return vtab_set_error(vtab, "Unable to find table %s,", insert_tbl); + + // extract the remaining fields from the input values + const char *insert_pk = (const char *)sqlite3_value_blob(argv[1]); + int insert_pk_len = sqlite3_value_bytes(argv[1]); + const char *insert_name = (sqlite3_value_type(argv[2]) == SQLITE_NULL) ? CLOUDSYNC_TOMBSTONE_VALUE : (const char *)sqlite3_value_text(argv[2]); + sqlite3_value *insert_value = argv[3]; + int64_t insert_col_version = (int64_t)sqlite3_value_int(argv[4]); + int64_t insert_db_version = (int64_t)sqlite3_value_int(argv[5]); + const char *insert_site_id = (const char *)sqlite3_value_blob(argv[6]); + int insert_site_id_len = sqlite3_value_bytes(argv[6]); + int64_t insert_cl = (int64_t)sqlite3_value_int(argv[7]); + int64_t insert_seq = (int64_t)sqlite3_value_int(argv[8]); + + // perform different logic for each different table algorithm + if (table_algo_isgos(table)) return cloudsync_changesvtab_insert_gos(vtab, data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, (int64_t *)rowid); + + int rc = merge_insert (data, table, insert_pk, insert_pk_len, insert_cl, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, (int64_t *)rowid); + if (rc != SQLITE_OK) { + return vtab_set_error(vtab, "%s", cloudsync_errmsg(data)); + } + + return SQLITE_OK; +} + int cloudsync_changesvtab_update (sqlite3_vtab *vtab, int argc, sqlite3_value **argv, sqlite3_int64 *rowid) { DEBUG_VTAB("cloudsync_changesvtab_update"); // only INSERT statements are allowed bool is_insert = (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL); if (!is_insert) { - cloudsync_vtab_set_error(vtab, "Only INSERT and SELECT statements are allowed against the cloudsync_changes table"); + vtab_set_error(vtab, "Only INSERT and SELECT statements are allowed against the cloudsync_changes table"); return SQLITE_MISUSE; } // argv[0] is set only in case of DELETE statement (it contains the rowid of a row in the virtual table to be deleted) // argv[1] is the rowid of a new row to be inserted into the virtual table (always NULL in our case) // so reduce the number of meaningful arguments by 2 - return cloudsync_merge_insert(vtab, argc-2, &argv[2], rowid); + return cloudsync_changesvtab_insert(vtab, argc-2, &argv[2], rowid); } // MARK: - -cloudsync_context *cloudsync_vtab_get_context (sqlite3_vtab *vtab) { - return (cloudsync_context *)(((cloudsync_changes_vtab *)vtab)->aux); -} - -int cloudsync_vtab_set_error (sqlite3_vtab *vtab, const char *format, ...) { - va_list arg; - va_start (arg, format); - char *err = cloudsync_memory_vmprintf(format, arg); - va_end (arg); - - if (vtab->zErrMsg) cloudsync_memory_free(vtab->zErrMsg); - vtab->zErrMsg = err; - return SQLITE_ERROR; -} - int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata) { static sqlite3_module cloudsync_changes_module = { /* iVersion */ 0, diff --git a/src/sqlite/cloudsync_changes_sqlite.h b/src/sqlite/cloudsync_changes_sqlite.h new file mode 100644 index 0000000..d6c284d --- /dev/null +++ b/src/sqlite/cloudsync_changes_sqlite.h @@ -0,0 +1,21 @@ +// +// cloudsync_changes_sqlite.h +// cloudsync +// +// Created by Marco Bambini on 23/09/24. +// + +#ifndef __CLOUDSYNC_CHANGES_SQLITE__ +#define __CLOUDSYNC_CHANGES_SQLITE__ + +#include "../cloudsync.h" + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata); + +#endif diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c new file mode 100644 index 0000000..bdff56b --- /dev/null +++ b/src/sqlite/cloudsync_sqlite.c @@ -0,0 +1,1542 @@ +// +// cloudsync_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 05/12/25. +// + +#include "cloudsync_sqlite.h" +#include "cloudsync_changes_sqlite.h" +#include "../pk.h" +#include "../cloudsync.h" +#include "../block.h" +#include "../database.h" +#include "../dbutils.h" + +#ifndef CLOUDSYNC_OMIT_NETWORK +#include "../network/network.h" +#endif + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT1 +#endif + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +#ifdef _WIN32 +#define APIEXPORT __declspec(dllexport) +#else +#define APIEXPORT +#endif + +typedef struct { + sqlite3_context *context; + int index; +} cloudsync_pk_decode_context; + +typedef struct { + sqlite3_value *table_name; + sqlite3_value **new_values; + sqlite3_value **old_values; + int count; + int capacity; +} cloudsync_update_payload; + +void dbsync_set_error (sqlite3_context *context, const char *format, ...) { + char buffer[2048]; + + va_list arg; + va_start (arg, format); + vsnprintf(buffer, sizeof(buffer), format, arg); + va_end (arg); + + if (context) sqlite3_result_error(context, buffer, -1); +} + +// MARK: - Public - + +void dbsync_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_version"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3_result_text(context, CLOUDSYNC_VERSION, -1, SQLITE_STATIC); +} + +void dbsync_siteid (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_siteid"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3_result_blob(context, cloudsync_siteid(data), UUID_LEN, SQLITE_STATIC); +} + +void dbsync_db_version (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_db_version"); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_dbversion_check_uptodate(data); + if (rc != SQLITE_OK) { + // When cloudsync_init was never called, data_version_stmt is NULL and + // database_errmsg() falls back to "not an error", producing the + // confusing "Unable to retrieve db_version (not an error)". Detect the + // uninitialized state and return an actionable message instead. The + // extra check only runs on the error branch, so it costs nothing on + // the sync hot path (merge operations keep going through the normal + // path where rc == SQLITE_OK). + if (!cloudsync_context_is_initialized(data)) { + dbsync_set_error(context, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_db_version()."); + } else { + dbsync_set_error(context, "Unable to retrieve db_version (%s).", database_errmsg(data)); + } + return; + } + + sqlite3_result_int64(context, cloudsync_dbversion(data)); +} + +void dbsync_db_version_next (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_db_version_next"); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + sqlite3_int64 merging_version = (argc == 1) ? database_value_int(argv[0]) : CLOUDSYNC_VALUE_NOTSET; + sqlite3_int64 value = cloudsync_dbversion_next(data, merging_version); + if (value == -1) { + if (!cloudsync_context_is_initialized(data)) { + dbsync_set_error(context, + "cloudsync is not initialized: call SELECT cloudsync_init('') " + "to enable sync on a table before calling cloudsync_db_version_next()."); + } else { + dbsync_set_error(context, "Unable to retrieve next_db_version (%s).", database_errmsg(data)); + } + return; + } + + sqlite3_result_int64(context, value); +} + +void dbsync_seq (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_seq"); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + sqlite3_result_int(context, cloudsync_bumpseq(data)); +} + +void dbsync_uuid (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_uuid"); + + char value[UUID_STR_MAXLEN]; + char *uuid = cloudsync_uuid_v7_string(value, true); + sqlite3_result_text(context, uuid, -1, SQLITE_TRANSIENT); +} + +// MARK: - + +void dbsync_set (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set"); + + // sanity check parameters + const char *key = (const char *)database_value_text(argv[0]); + const char *value = (const char *)database_value_text(argv[1]); + + // silently fails + if (key == NULL) return; + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + dbutils_settings_set_key_value(data, key, value); +} + +void dbsync_set_column (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_column"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *col = (const char *)database_value_text(argv[1]); + const char *key = (const char *)database_value_text(argv[2]); + const char *value = (const char *)database_value_text(argv[3]); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Handle block column setup: cloudsync_set_column('tbl', 'col', 'algo', 'block') + if (key && value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { + int rc = cloudsync_setup_block_column(data, tbl, col, NULL, true); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + } + return; + } + + // Handle delimiter setting: cloudsync_set_column('tbl', 'col', 'delimiter', '\n\n') + if (key && strcmp(key, "delimiter") == 0) { + cloudsync_table_context *table = table_lookup(data, tbl); + if (table) { + int col_idx = table_col_index(table, col); + if (col_idx >= 0 && table_col_algo(table, col_idx) == col_algo_block) { + table_set_col_delimiter(table, col_idx, value); + } + } + } + + dbutils_table_settings_set_key_value(data, tbl, col, key, value); +} + +void dbsync_set_table (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_table"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *key = (const char *)database_value_text(argv[1]); + const char *value = (const char *)database_value_text(argv[2]); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + dbutils_table_settings_set_key_value(data, tbl, "*", key, value); +} + +void dbsync_set_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_set_schema"); + + const char *schema = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_set_schema(data, schema); +} + +void dbsync_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_schema"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *schema = cloudsync_schema(data); + (schema) ? sqlite3_result_text(context, schema, -1, NULL) : sqlite3_result_null(context); +} + +void dbsync_table_schema (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_table_schema"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *table_name = (const char *)database_value_text(argv[0]); + const char *schema = cloudsync_table_schema(data, table_name); + (schema) ? sqlite3_result_text(context, schema, -1, NULL) : sqlite3_result_null(context); +} + +void dbsync_is_sync (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_is_sync"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + if (cloudsync_insync(data)) { + sqlite3_result_int(context, 1); + return; + } + + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + sqlite3_result_int(context, (table) ? (table_enabled(table) == 0) : 0); +} + +void dbsync_col_value (sqlite3_context *context, int argc, sqlite3_value **argv) { + // DEBUG_FUNCTION("cloudsync_col_value"); + + // argv[0] -> table name + // argv[1] -> column name + // argv[2] -> encoded pk + + // retrieve column name + const char *col_name = (const char *)database_value_text(argv[1]); + if (!col_name) { + dbsync_set_error(context, "Column name cannot be NULL"); + return; + } + + // check for special tombstone value + if (strcmp(col_name, CLOUDSYNC_TOMBSTONE_VALUE) == 0) { + sqlite3_result_null(context); + return; + } + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in clousdsync_colvalue.", table_name); + return; + } + + // Block column: if col_name contains \x1F, read from blocks table + if (block_is_block_colname(col_name) && table_has_block_cols(table)) { + dbvm_t *bvm = table_block_value_read_stmt(table); + if (!bvm) { + sqlite3_result_null(context); + return; + } + int rc = databasevm_bind_blob(bvm, 1, database_value_blob(argv[2]), database_value_bytes(argv[2])); + if (rc != DBRES_OK) { databasevm_reset(bvm); sqlite3_result_error(context, database_errmsg(data), -1); return; } + rc = databasevm_bind_text(bvm, 2, col_name, -1); + if (rc != DBRES_OK) { databasevm_reset(bvm); sqlite3_result_error(context, database_errmsg(data), -1); return; } + + rc = databasevm_step(bvm); + if (rc == SQLITE_ROW) { + sqlite3_result_value(context, database_column_value(bvm, 0)); + } else if (rc == SQLITE_DONE) { + sqlite3_result_null(context); + } else { + sqlite3_result_error(context, database_errmsg(data), -1); + } + databasevm_reset(bvm); + return; + } + + // extract the right col_value vm associated to the column name + sqlite3_stmt *vm = table_column_lookup(table, col_name, false, NULL); + if (!vm) { + sqlite3_result_error(context, "Unable to retrieve column value precompiled statement in clousdsync_colvalue.", -1); + return; + } + + // bind primary key values + int rc = pk_decode_prikey((char *)database_value_blob(argv[2]), (size_t)database_value_bytes(argv[2]), pk_decode_bind_callback, (void *)vm); + if (rc < 0) goto cleanup; + + // execute vm + rc = databasevm_step(vm); + if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + sqlite3_result_text(context, CLOUDSYNC_RLS_RESTRICTED_VALUE, -1, SQLITE_STATIC); + } else if (rc == SQLITE_ROW) { + // store value result + rc = SQLITE_OK; + sqlite3_result_value(context, database_column_value(vm, 0)); + } + +cleanup: + if (rc != SQLITE_OK) { + sqlite3_result_error(context, database_errmsg(data), -1); + } + databasevm_reset(vm); +} + +void dbsync_pk_encode (sqlite3_context *context, int argc, sqlite3_value **argv) { + size_t bsize = 0; + char *buffer = pk_encode_prikey((dbvalue_t **)argv, argc, NULL, &bsize); + if (!buffer || buffer == PRIKEY_NULL_CONSTRAINT_ERROR) { + sqlite3_result_null(context); + return; + } + sqlite3_result_blob(context, (const void *)buffer, (int)bsize, SQLITE_TRANSIENT); + cloudsync_memory_free(buffer); +} + +int dbsync_pk_decode_set_result_callback (void *xdata, int index, int type, int64_t ival, double dval, char *pval) { + cloudsync_pk_decode_context *decode_context = (cloudsync_pk_decode_context *)xdata; + // decode_context->index is 1 based + // index is 0 based + if (decode_context->index != index+1) return SQLITE_OK; + + int rc = 0; + sqlite3_context *context = decode_context->context; + switch (type) { + case SQLITE_INTEGER: + sqlite3_result_int64(context, ival); + break; + + case SQLITE_FLOAT: + sqlite3_result_double(context, dval); + break; + + case SQLITE_NULL: + sqlite3_result_null(context); + break; + + case SQLITE_TEXT: + sqlite3_result_text(context, pval, (int)ival, SQLITE_TRANSIENT); + break; + + case SQLITE_BLOB: + sqlite3_result_blob(context, pval, (int)ival, SQLITE_TRANSIENT); + break; + } + + return rc; +} + + +void dbsync_pk_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { + const char *pk = (const char *)database_value_blob(argv[0]); + int pk_len = database_value_bytes(argv[0]); + int i = (int)database_value_int(argv[1]); + + cloudsync_pk_decode_context xdata = {.context = context, .index = i}; + pk_decode_prikey((char *)pk, (size_t)pk_len, dbsync_pk_decode_set_result_callback, &xdata); +} + +// MARK: - + +void dbsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_insert %s", database_value_text(argv[0])); + // debug_values(argc-1, &argv[1]); + + // argv[0] is table name + // argv[1]..[N] is primary key(s) + + // table_cloudsync + // pk -> encode(argc-1, &argv[1]) + // col_name -> name + // col_version -> 0/1 +1 + // db_version -> check + // site_id 0 + // seq -> sqlite_master + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_insert.", table_name); + return; + } + + // encode the primary key values into a buffer + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[1], table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + return; + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Insert aborted because primary key in table %s contains NULL values.", table_name); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + + // check if a row with the same primary key already exists + // if so, this means the row might have been previously deleted (sentinel) + bool pk_exists = table_pk_exists(table, pk, pklen); + int rc = SQLITE_OK; + + if (table_count_cols(table) == 0) { + // if there are no columns other than primary keys, insert a sentinel record + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } else if (pk_exists){ + // if a row with the same primary key already exists, update the sentinel record + rc = local_update_sentinel(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + + // process each non-primary key column for insert or update + for (int i=0; icount); + if (positions) { + for (int b = 0; b < blocks->count; b++) { + char *block_cn = block_build_colname(col, positions[b]); + if (block_cn) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); + + // Store block value in blocks table + dbvm_t *wvm = table_block_value_write_stmt(table); + if (wvm && rc == SQLITE_OK) { + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + + cloudsync_memory_free(block_cn); + } + cloudsync_memory_free(positions[b]); + if (rc != SQLITE_OK) break; + } + cloudsync_memory_free(positions); + } + block_list_free(blocks); + } + } + databasevm_reset((dbvm_t *)val_vm); + if (rc == DBRES_ROW || rc == DBRES_DONE) rc = SQLITE_OK; + if (rc != SQLITE_OK) goto cleanup; + } else { + // Regular column: mark as inserted or updated in the metadata + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + } + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + // free memory if the primary key was dynamically allocated + if (pk != buffer) cloudsync_memory_free(pk); +} + +void dbsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_delete %s", database_value_text(argv[0])); + // debug_values(argc-1, &argv[1]); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_delete.", table_name); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + int rc = SQLITE_OK; + + // encode the primary key values into a buffer + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[1], table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + return; + } + + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Delete aborted because primary key in table %s contains NULL values.", table_name); + return; + } + + // mark the row as deleted by inserting a delete sentinel into the metadata + rc = local_mark_delete_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // remove any metadata related to the old rows associated with this primary key + rc = local_drop_meta(table, pk, pklen); + if (rc != SQLITE_OK) goto cleanup; + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + // free memory if the primary key was dynamically allocated + if (pk != buffer) cloudsync_memory_free(pk); +} + +// MARK: - + +void dbsync_update_payload_free (cloudsync_update_payload *payload) { + for (int i=0; icount; i++) { + database_value_free(payload->new_values[i]); + database_value_free(payload->old_values[i]); + } + cloudsync_memory_free(payload->new_values); + cloudsync_memory_free(payload->old_values); + database_value_free(payload->table_name); + payload->new_values = NULL; + payload->old_values = NULL; + payload->table_name = NULL; + payload->count = 0; + payload->capacity = 0; +} + +int dbsync_update_payload_append (cloudsync_update_payload *payload, sqlite3_value *v1, sqlite3_value *v2, sqlite3_value *v3) { + if (payload->count >= payload->capacity) { + int newcap = payload->capacity ? payload->capacity * 2 : 128; + + sqlite3_value **new_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->new_values, newcap * sizeof(*new_values_2)); + if (!new_values_2) return SQLITE_NOMEM; + + sqlite3_value **old_values_2 = (sqlite3_value **)cloudsync_memory_realloc(payload->old_values, newcap * sizeof(*old_values_2)); + if (!old_values_2) { + // new_values_2 succeeded but old_values failed; keep new_values_2 pointer + // (it's still valid, just larger) but don't update capacity + payload->new_values = new_values_2; + return SQLITE_NOMEM; + } + + payload->new_values = new_values_2; + payload->old_values = old_values_2; + payload->capacity = newcap; + } + + int index = payload->count; + if (payload->table_name == NULL) payload->table_name = database_value_dup(v1); + else if (dbutils_value_compare(payload->table_name, v1) != 0) return SQLITE_NOMEM; + + payload->new_values[index] = database_value_dup(v2); + payload->old_values[index] = database_value_dup(v3); + + // sanity check memory allocations before committing count + bool v1_can_be_null = (database_value_type(v1) == SQLITE_NULL); + bool v2_can_be_null = (database_value_type(v2) == SQLITE_NULL); + bool v3_can_be_null = (database_value_type(v3) == SQLITE_NULL); + + bool oom = false; + if ((payload->table_name == NULL) && (!v1_can_be_null)) oom = true; + if ((payload->new_values[index] == NULL) && (!v2_can_be_null)) oom = true; + if ((payload->old_values[index] == NULL) && (!v3_can_be_null)) oom = true; + + if (oom) { + // clean up partial allocations at this index to prevent leaks + if (payload->new_values[index]) { database_value_free(payload->new_values[index]); payload->new_values[index] = NULL; } + if (payload->old_values[index]) { database_value_free(payload->old_values[index]); payload->old_values[index] = NULL; } + return SQLITE_NOMEM; + } + + payload->count++; + return SQLITE_OK; +} + +void dbsync_update_step (sqlite3_context *context, int argc, sqlite3_value **argv) { + // argv[0] => table_name + // argv[1] => new_column_value + // argv[2] => old_column_value + + // allocate/get the update payload + cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); + if (!payload) {sqlite3_result_error_nomem(context); return;} + + if (dbsync_update_payload_append(payload, argv[0], argv[1], argv[2]) != SQLITE_OK) { + sqlite3_result_error_nomem(context); + } +} + +void dbsync_update_final (sqlite3_context *context) { + cloudsync_update_payload *payload = (cloudsync_update_payload *)sqlite3_aggregate_context(context, sizeof(cloudsync_update_payload)); + if (!payload || payload->count == 0) return; + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // lookup table + const char *table_name = (const char *)database_value_text(payload->table_name); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_update.", table_name); + dbsync_update_payload_free(payload); + return; + } + + // compute the next database version for tracking changes + int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); + int rc = SQLITE_OK; + + // Check if the primary key(s) have changed + bool prikey_changed = false; + for (int i=0; iold_values[i], payload->new_values[i]) != 0) { + prikey_changed = true; + break; + } + } + + // encode the NEW primary key values into a buffer (used later for indexing) + char buffer[1024]; + char buffer2[1024]; + size_t pklen = sizeof(buffer); + size_t oldpklen = sizeof(buffer2); + char *oldpk = NULL; + + char *pk = pk_encode_prikey((dbvalue_t **)payload->new_values, table_count_pks(table), buffer, &pklen); + if (!pk) { + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + dbsync_update_payload_free(payload); + return; + } + if (pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + dbsync_set_error(context, "Update aborted because primary key in table %s contains NULL values.", table_name); + dbsync_update_payload_free(payload); + return; + } + + if (prikey_changed) { + // if the primary key has changed, we need to handle the row differently: + // 1. mark the old row (OLD primary key) as deleted + // 2. create a new row (NEW primary key) + + // encode the OLD primary key into a buffer + oldpk = pk_encode_prikey((dbvalue_t **)payload->old_values, table_count_pks(table), buffer2, &oldpklen); + if (!oldpk) { + // no check here about PRIKEY_NULL_CONSTRAINT_ERROR because by design oldpk cannot contain NULL values + if (pk != buffer) cloudsync_memory_free(pk); + sqlite3_result_error(context, "Not enough memory to encode the primary key(s).", -1); + dbsync_update_payload_free(payload); + return; + } + + // mark the rows with the old primary key as deleted in the metadata (old row handling) + rc = local_mark_delete_meta(table, oldpk, oldpklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // move non-sentinel metadata entries from OLD primary key to NEW primary key + // handles the case where some metadata is retained across primary key change + // see https://github.com/sqliteai/sqlite-sync/blob/main/docs/PriKey.md for more details + rc = local_update_move_meta(table, pk, pklen, oldpk, oldpklen, db_version); + if (rc != SQLITE_OK) goto cleanup; + + // mark a new sentinel row with the new primary key in the metadata + rc = local_mark_insert_sentinel_meta(table, pk, pklen, db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + + // free memory if the OLD primary key was dynamically allocated + if (oldpk != buffer2) cloudsync_memory_free(oldpk); + oldpk = NULL; + } + + // compare NEW and OLD values (excluding primary keys) to handle column updates + for (int i=0; iold_values[col_index], payload->new_values[col_index]) != 0) { + if (table_col_algo(table, i) == col_algo_block) { + // Block column: diff old and new text, emit per-block metadata changes + const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); + const char *delim = table_col_delimiter(table, i); + const char *col = table_colname(table, i); + + // Read existing blocks from blocks table + block_list_t *old_blocks = block_list_create_empty(); + if (table_block_list_stmt(table)) { + char *like_pattern = block_build_colname(col, "%"); + if (like_pattern) { + // Query blocks table directly for existing block names and values + char *list_sql = cloudsync_memory_mprintf( + "SELECT col_name, col_value FROM %s WHERE pk = ?1 AND col_name LIKE ?2 ORDER BY col_name", + table_blocks_ref(table)); + if (list_sql) { + dbvm_t *list_vm = NULL; + if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { + databasevm_bind_blob(list_vm, 1, pk, (int)pklen); + databasevm_bind_text(list_vm, 2, like_pattern, -1); + while (databasevm_step(list_vm) == DBRES_ROW) { + const char *bcn = database_column_text(list_vm, 0); + const char *bval = database_column_text(list_vm, 1); + const char *pos = block_extract_position_id(bcn); + if (pos && old_blocks) { + block_list_add(old_blocks, bval ? bval : "", pos); + } + } + databasevm_finalize(list_vm); + } + cloudsync_memory_free(list_sql); + } + cloudsync_memory_free(like_pattern); + } + } + + // Split new text into parts (NULL text = all blocks removed) + block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); + if (new_blocks && old_blocks) { + // Build array of new content strings (NULL when count is 0) + const char **new_parts = NULL; + if (new_blocks->count > 0) { + new_parts = (const char **)cloudsync_memory_alloc( + (uint64_t)(new_blocks->count * sizeof(char *))); + if (new_parts) { + for (int b = 0; b < new_blocks->count; b++) { + new_parts[b] = new_blocks->entries[b].content; + } + } + } + + if (new_parts || new_blocks->count == 0) { + block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, + new_parts, new_blocks->count); + if (diff) { + for (int d = 0; d < diff->count; d++) { + block_diff_entry_t *de = &diff->entries[d]; + char *block_cn = block_build_colname(col, de->position_id); + if (!block_cn) continue; + + if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { + rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Store block value + if (rc == SQLITE_OK && table_block_value_write_stmt(table)) { + dbvm_t *wvm = table_block_value_write_stmt(table); + databasevm_bind_blob(wvm, 1, pk, (int)pklen); + databasevm_bind_text(wvm, 2, block_cn, -1); + databasevm_bind_text(wvm, 3, de->content, -1); + databasevm_step(wvm); + databasevm_reset(wvm); + } + } else if (de->type == BLOCK_DIFF_REMOVED) { + // Mark block as deleted in metadata (even col_version) + rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, + db_version, cloudsync_bumpseq(data)); + // Remove from blocks table + if (rc == SQLITE_OK) { + block_delete_value_external(data, table, pk, pklen, block_cn); + } + } + cloudsync_memory_free(block_cn); + if (rc != SQLITE_OK) break; + } + block_diff_free(diff); + } + if (new_parts) cloudsync_memory_free((void *)new_parts); + } + } + if (new_blocks) block_list_free(new_blocks); + if (old_blocks) block_list_free(old_blocks); + if (rc != SQLITE_OK) goto cleanup; + } else { + // Regular column: mark as updated in the metadata (columns are in cid order) + rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); + if (rc != SQLITE_OK) goto cleanup; + } + } + } + +cleanup: + if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + if (pk != buffer) cloudsync_memory_free(pk); + if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); + + dbsync_update_payload_free(payload); +} + +// MARK: - + +void dbsync_cleanup (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_cleanup"); + + const char *table = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_cleanup(data, table); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_enable_disable (sqlite3_context *context, const char *table_name, bool value) { + DEBUG_FUNCTION("cloudsync_enable_disable"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) return; + + table_set_enabled(table, value); +} + +void dbsync_enable (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_enable"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_enable_disable(context, table, true); +} + +void dbsync_disable (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_disable"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_enable_disable(context, table, false); +} + +void dbsync_is_enabled (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_is_enabled"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_table_context *table = table_lookup(data, table_name); + + int result = (table && table_enabled(table)) ? 1 : 0; + sqlite3_result_int(context, result); +} + +void dbsync_terminate (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_terminate"); + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_terminate(data); + sqlite3_result_int(context, rc); +} + +// MARK: - + +void dbsync_init (sqlite3_context *context, const char *table, const char *algo, CLOUDSYNC_INIT_FLAG init_flags) { + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = database_begin_savepoint(data, "cloudsync_init"); + if (rc != SQLITE_OK) { + dbsync_set_error(context, "Unable to create cloudsync_init savepoint. %s", database_errmsg(data)); + sqlite3_result_error_code(context, rc); + return; + } + + rc = cloudsync_init_table(data, table, algo, init_flags); + if (rc == SQLITE_OK) { + rc = database_commit_savepoint(data, "cloudsync_init"); + if (rc != SQLITE_OK) { + dbsync_set_error(context, "Unable to release cloudsync_init savepoint. %s", database_errmsg(data)); + sqlite3_result_error_code(context, rc); + } + } else { + // in case of error, rollback transaction + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + database_rollback_savepoint(data, "cloudsync_init"); + return; + } + + cloudsync_update_schema_hash(data); + + // returns site_id as TEXT + char buffer[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_stringify(cloudsync_siteid(data), buffer, false); + sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT); +} + +void dbsync_init3 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init2"); + + const char *table = (const char *)database_value_text(argv[0]); + const char *algo = (const char *)database_value_text(argv[1]); + int init_flags = database_value_int(argv[2]); + dbsync_init(context, table, algo, init_flags); +} + +void dbsync_init2 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init2"); + + const char *table = (const char *)database_value_text(argv[0]); + const char *algo = (const char *)database_value_text(argv[1]); + dbsync_init(context, table, algo, CLOUDSYNC_INIT_FLAG_NONE); +} + +void dbsync_init1 (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_init1"); + + const char *table = (const char *)database_value_text(argv[0]); + dbsync_init(context, table, NULL, CLOUDSYNC_INIT_FLAG_NONE); +} + +// MARK: - + +void dbsync_begin_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_begin_alter"); + + const char *table_name = (const char *)database_value_text(argv[0]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = database_begin_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + sqlite3_result_error(context, "Unable to create cloudsync_alter savepoint", -1); + sqlite3_result_error_code(context, rc); + return; + } + + rc = cloudsync_begin_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_commit_alter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_commit_alter"); + + //retrieve table argument + const char *table_name = (const char *)database_value_text(argv[0]); + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_commit_alter(data, table_name); + if (rc != DBRES_OK) { + database_rollback_savepoint(data, "cloudsync_alter"); + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + rc = database_commit_savepoint(data, "cloudsync_alter"); + if (rc != DBRES_OK) { + sqlite3_result_error(context, database_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + cloudsync_update_schema_hash(data); +} + +// MARK: - Payload - + +void dbsync_payload_encode_step (sqlite3_context *context, int argc, sqlite3_value **argv) { + // allocate/get the session context + cloudsync_payload_context *payload = (cloudsync_payload_context *)sqlite3_aggregate_context(context, (int)cloudsync_payload_context_size(NULL)); + if (!payload) { + sqlite3_result_error(context, "Not enough memory to allocate payload session context", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return; + } + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_payload_encode_step(payload, data, argc, (dbvalue_t **)argv); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + } +} + +void dbsync_payload_encode_final (sqlite3_context *context) { + // get the session context + cloudsync_payload_context *payload = (cloudsync_payload_context *)sqlite3_aggregate_context(context, (int)cloudsync_payload_context_size(NULL)); + if (!payload) { + sqlite3_result_error(context, "Unable to extract payload session context", -1); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return; + } + + // retrieve context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int rc = cloudsync_payload_encode_final(payload, data); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // result is OK so get BLOB and returns it + int64_t blob_size = 0; + char *blob = cloudsync_payload_blob (payload, &blob_size, NULL); + if (!blob) { + sqlite3_result_null(context); + } else { + sqlite3_result_blob64(context, blob, blob_size, SQLITE_TRANSIENT); + cloudsync_memory_free(blob); + } + + // from: https://sqlite.org/c3ref/aggregate_context.html + // SQLite automatically frees the memory allocated by sqlite3_aggregate_context() when the aggregate query concludes. +} + +void dbsync_payload_decode (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_decode"); + //debug_values(argc, argv); + + // sanity check payload type + if (database_value_type(argv[0]) != SQLITE_BLOB) { + sqlite3_result_error(context, "Error on cloudsync_payload_decode: value must be a BLOB.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + return; + } + + // sanity check payload size + int blen = database_value_bytes(argv[0]); + size_t header_size = 0; + cloudsync_payload_context_size(&header_size); + if (blen < (int)header_size) { + sqlite3_result_error(context, "Error on cloudsync_payload_decode: invalid input size.", -1); + sqlite3_result_error_code(context, SQLITE_MISUSE); + return; + } + + // obtain payload + const char *payload = (const char *)database_value_blob(argv[0]); + + // apply changes + int nrows = 0; + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_payload_apply(data, payload, blen, &nrows); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // returns number of applied rows + sqlite3_result_int(context, nrows); +} + +#ifdef CLOUDSYNC_DESKTOP_OS +void dbsync_payload_save (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_save"); + + // sanity check argument + if (database_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "Unable to retrieve file path.", -1); + return; + } + + // retrieve full path to file + const char *payload_path = (const char *)database_value_text(argv[0]); + + // retrieve global context + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + int blob_size = 0; + int rc = cloudsync_payload_save(data, payload_path, &blob_size); + if (rc == SQLITE_OK) { + // if OK then returns blob size + sqlite3_result_int64(context, (sqlite3_int64)blob_size); + return; + } + + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); +} + +void dbsync_payload_load (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("dbsync_payload_load"); + + // sanity check argument + if (database_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(context, "Unable to retrieve file path.", -1); + return; + } + + // retrieve full path to file + const char *path = (const char *)database_value_text(argv[0]); + + int64_t payload_size = 0; + char *payload = cloudsync_file_read(path, &payload_size); + if (!payload) { + if (payload_size < 0) { + sqlite3_result_error(context, "Unable to read payload from file path.", -1); + sqlite3_result_error_code(context, SQLITE_IOERR); + return; + } + // no rows affected but no error either + sqlite3_result_int(context, 0); + return; + } + + int nrows = 0; + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + int rc = cloudsync_payload_apply (data, payload, (int)payload_size, &nrows); + if (payload) cloudsync_memory_free(payload); + + if (rc != SQLITE_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); + return; + } + + // returns number of applied rows + sqlite3_result_int(context, nrows); +} +#endif + +// MARK: - Register - + +int dbsync_register_with_flags (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, int flags, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + + int rc = sqlite3_create_function_v2(db, name, nargs, flags, ctx, xfunc, xstep, xfinal, ctx_free); + + if (rc != SQLITE_OK) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Error creating function %s: %s", name, sqlite3_errmsg(db)); + return rc; + } + return SQLITE_OK; +} + +int dbsync_register (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_VOLATILE = SQLITE_UTF8; + DEBUG_DBFUNCTION("dbsync_register %s", name); + return dbsync_register_with_flags(db, name, xfunc, xstep, xfinal, nargs, FLAGS_VOLATILE, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + DEBUG_DBFUNCTION("dbsync_register_function %s", name); + return dbsync_register(db, name, xfunc, NULL, NULL, nargs, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_pure_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_PURE = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC; + DEBUG_DBFUNCTION("dbsync_register_pure_function %s", name); + return dbsync_register_with_flags(db, name, xfunc, NULL, NULL, nargs, FLAGS_PURE, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_trigger_function (sqlite3 *db, const char *name, void (*xfunc)(sqlite3_context*,int,sqlite3_value**), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_TRIGGER = SQLITE_UTF8 | SQLITE_INNOCUOUS; + DEBUG_DBFUNCTION("dbsync_register_trigger_function %s", name); + return dbsync_register_with_flags(db, name, xfunc, NULL, NULL, nargs, FLAGS_TRIGGER, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + DEBUG_DBFUNCTION("dbsync_register_aggregate %s", name); + return dbsync_register(db, name, NULL, xstep, xfinal, nargs, pzErrMsg, ctx, ctx_free); +} + +int dbsync_register_trigger_aggregate (sqlite3 *db, const char *name, void (*xstep)(sqlite3_context*,int,sqlite3_value**), void (*xfinal)(sqlite3_context*), int nargs, char **pzErrMsg, void *ctx, void (*ctx_free)(void *)) { + const int FLAGS_TRIGGER = SQLITE_UTF8 | SQLITE_INNOCUOUS; + DEBUG_DBFUNCTION("dbsync_register_trigger_aggregate %s", name); + return dbsync_register_with_flags(db, name, NULL, xstep, xfinal, nargs, FLAGS_TRIGGER, pzErrMsg, ctx, ctx_free); +} + +// MARK: - Block-level LWW - + +void dbsync_text_materialize (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_text_materialize"); + + // argv[0] -> table name + // argv[1] -> column name + // argv[2..N] -> primary key values + + if (argc < 3) { + sqlite3_result_error(context, "cloudsync_text_materialize requires at least 3 arguments: table, column, pk...", -1); + return; + } + + const char *table_name = (const char *)database_value_text(argv[0]); + const char *col_name = (const char *)database_value_text(argv[1]); + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + cloudsync_table_context *table = table_lookup(data, table_name); + if (!table) { + dbsync_set_error(context, "Unable to retrieve table name %s in cloudsync_text_materialize.", table_name); + return; + } + + int col_idx = table_col_index(table, col_name); + if (col_idx < 0 || table_col_algo(table, col_idx) != col_algo_block) { + dbsync_set_error(context, "Column %s in table %s is not configured as block-level.", col_name, table_name); + return; + } + + // Encode primary keys + int npks = table_count_pks(table); + if (argc - 2 != npks) { + sqlite3_result_error(context, "Wrong number of primary key values for cloudsync_text_materialize.", -1); + return; + } + + char buffer[1024]; + size_t pklen = sizeof(buffer); + char *pk = pk_encode_prikey((dbvalue_t **)&argv[2], npks, buffer, &pklen); + if (!pk || pk == PRIKEY_NULL_CONSTRAINT_ERROR) { + sqlite3_result_error(context, "Failed to encode primary key(s).", -1); + return; + } + + // Materialize the column + int rc = block_materialize_column(data, table, pk, (int)pklen, col_name); + if (rc != DBRES_OK) { + sqlite3_result_error(context, cloudsync_errmsg(data), -1); + } else { + sqlite3_result_int(context, 1); + } + + if (pk != buffer) cloudsync_memory_free(pk); +} + +// MARK: - Row Filter - + +void dbsync_set_filter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_set_filter"); + + const char *tbl = (const char *)database_value_text(argv[0]); + const char *filter_expr = (const char *)database_value_text(argv[1]); + if (!tbl || !filter_expr) { + dbsync_set_error(context, "cloudsync_set_filter: table and filter expression required"); + return; + } + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Guard against calling set_filter before the target table has been set + // up for sync: without this, we'd hit "no such table: + // cloudsync_table_settings" or "no such table: main." deep inside + // the trigger recreation path, which is not actionable. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + dbsync_set_error(context, + "cloudsync_set_filter: table '%s' is not configured for sync. " + "Call SELECT cloudsync_init('%s') first.", tbl, tbl); + return; + } + + // Store filter in table settings + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", filter_expr); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop and recreate triggers with the filter + database_delete_triggers(data, tbl); + int rc = database_create_triggers(data, tbl, algo, filter_expr); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_set_filter: error recreating triggers"); + sqlite3_result_error_code(context, rc); + return; + } + + // Clean and refill metatable with the new filter + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_set_filter: error resetting metatable"); + sqlite3_result_error_code(context, rc); + return; + } + + sqlite3_result_int(context, 1); +} + +void dbsync_clear_filter (sqlite3_context *context, int argc, sqlite3_value **argv) { + DEBUG_FUNCTION("cloudsync_clear_filter"); + + const char *tbl = (const char *)database_value_text(argv[0]); + if (!tbl) { + dbsync_set_error(context, "cloudsync_clear_filter: table name required"); + return; + } + + cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + + // Guard against calling clear_filter before the target table has been set + // up for sync — see dbsync_set_filter for the same rationale. + if (!cloudsync_context_is_initialized(data) || !table_lookup(data, tbl)) { + dbsync_set_error(context, + "cloudsync_clear_filter: table '%s' is not configured for sync. " + "Call SELECT cloudsync_init('%s') first.", tbl, tbl); + return; + } + + // Remove filter from table settings (set to NULL/empty) + dbutils_table_settings_set_key_value(data, tbl, "*", "filter", NULL); + + // Read current algo + table_algo algo = dbutils_table_settings_get_algo(data, tbl); + if (algo == table_algo_none) algo = table_algo_crdt_cls; + + // Drop and recreate triggers without filter + database_delete_triggers(data, tbl); + int rc = database_create_triggers(data, tbl, algo, NULL); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_clear_filter: error recreating triggers"); + sqlite3_result_error_code(context, rc); + return; + } + + // Clean and refill metatable without filter (all rows) + rc = cloudsync_reset_metatable(data, tbl); + if (rc != DBRES_OK) { + dbsync_set_error(context, "cloudsync_clear_filter: error resetting metatable"); + sqlite3_result_error_code(context, rc); + return; + } + + sqlite3_result_int(context, 1); +} + +int dbsync_register_functions (sqlite3 *db, char **pzErrMsg) { + int rc = SQLITE_OK; + + // there's no built-in way to verify if sqlite3_cloudsync_init has already been called + // for this specific database connection, we use a workaround: we attempt to retrieve the + // cloudsync_version and check for an error, an error indicates that initialization has not been performed + if (sqlite3_exec(db, "SELECT cloudsync_version();", NULL, NULL, NULL) == SQLITE_OK) return SQLITE_OK; + + // init memory debugger (NOOP in production) + cloudsync_memory_init(1); + + // set fractional-indexing allocator to use cloudsync memory + block_init_allocator(); + + // init context + void *ctx = cloudsync_context_create(db); + if (!ctx) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Not enought memory to create a database context"); + return SQLITE_NOMEM; + } + + // register functions + + // PUBLIC functions + rc = dbsync_register_pure_function(db, "cloudsync_version", dbsync_version, 0, pzErrMsg, ctx, cloudsync_context_free); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init1, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init2, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_init", dbsync_init3, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_enable", dbsync_enable, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_disable", dbsync_disable, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_is_enabled", dbsync_is_enabled, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_cleanup", dbsync_cleanup, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_terminate", dbsync_terminate, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set", dbsync_set, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_table", dbsync_set_table, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_filter", dbsync_set_filter, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_clear_filter", dbsync_clear_filter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_schema", dbsync_set_schema, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_schema", dbsync_schema, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_table_schema", dbsync_table_schema, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_set_column", dbsync_set_column, 4, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_siteid", dbsync_siteid, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version", dbsync_db_version, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version_next", dbsync_db_version_next, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_db_version_next", dbsync_db_version_next, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_begin_alter", dbsync_begin_alter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_commit_alter", dbsync_commit_alter, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_uuid", dbsync_uuid, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // PAYLOAD + rc = dbsync_register_aggregate(db, "cloudsync_payload_encode", dbsync_payload_encode_step, dbsync_payload_encode_final, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // alias + rc = dbsync_register_function(db, "cloudsync_payload_decode", dbsync_payload_decode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + rc = dbsync_register_function(db, "cloudsync_payload_apply", dbsync_payload_decode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + #ifdef CLOUDSYNC_DESKTOP_OS + rc = dbsync_register_function(db, "cloudsync_payload_save", dbsync_payload_save, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_payload_load", dbsync_payload_load, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + #endif + + // PRIVATE functions (used inside triggers — require SQLITE_INNOCUOUS) + rc = dbsync_register_trigger_function(db, "cloudsync_is_sync", dbsync_is_sync, 1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_function(db, "cloudsync_insert", dbsync_insert, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_aggregate(db, "cloudsync_update", dbsync_update_step, dbsync_update_final, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_trigger_function(db, "cloudsync_delete", dbsync_delete, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_col_value", dbsync_col_value, 3, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_pure_function(db, "cloudsync_pk_encode", dbsync_pk_encode, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_pure_function(db, "cloudsync_pk_decode", dbsync_pk_decode, 2, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_seq", dbsync_seq, 0, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + rc = dbsync_register_function(db, "cloudsync_text_materialize", dbsync_text_materialize, -1, pzErrMsg, ctx, NULL); + if (rc != SQLITE_OK) return rc; + + // NETWORK LAYER + #ifndef CLOUDSYNC_OMIT_NETWORK + rc = cloudsync_network_register(db, pzErrMsg, ctx); + if (rc != SQLITE_OK) return rc; + #endif + + cloudsync_context *data = (cloudsync_context *)ctx; + sqlite3_commit_hook(db, cloudsync_commit_hook, ctx); + sqlite3_rollback_hook(db, cloudsync_rollback_hook, ctx); + + // register eponymous only changes virtual table + rc = cloudsync_vtab_register_changes (db, data); + if (rc != SQLITE_OK) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("Error creating changes virtual table: %s", sqlite3_errmsg(db)); + return rc; + } + + // load config, if exists + if (cloudsync_config_exists(data)) { + if (cloudsync_context_init(data) == NULL) { + // Do not free ctx here: it is already owned by the cloudsync_version + // function (registered above with cloudsync_context_free as its + // destructor). SQLite will release it when the connection is closed. + // Freeing it manually would cause a double-free on sqlite3_close. + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("An error occurred while trying to initialize context"); + return SQLITE_ERROR; + } + + // update schema hash if upgrading from an older version + if (dbutils_settings_check_version(data, NULL) != 0) { + cloudsync_update_schema_hash(data); + } + + // make sure to update internal version to current version + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + } + + return SQLITE_OK; +} + +// MARK: - Main Entrypoint - + +APIEXPORT int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + DEBUG_FUNCTION("sqlite3_cloudsync_init"); + + #ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); + #endif + + return dbsync_register_functions(db, pzErrMsg); +} diff --git a/src/sqlite/cloudsync_sqlite.h b/src/sqlite/cloudsync_sqlite.h new file mode 100644 index 0000000..12127e4 --- /dev/null +++ b/src/sqlite/cloudsync_sqlite.h @@ -0,0 +1,19 @@ +// +// cloudsync_sqlite.h +// cloudsync +// +// Created by Marco Bambini on 05/12/25. +// + +#ifndef __CLOUDSYNC_SQLITE__ +#define __CLOUDSYNC_SQLITE__ + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +int sqlite3_cloudsync_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); + +#endif diff --git a/src/sqlite/database_sqlite.c b/src/sqlite/database_sqlite.c new file mode 100644 index 0000000..b7864bb --- /dev/null +++ b/src/sqlite/database_sqlite.c @@ -0,0 +1,1387 @@ +// +// database_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 03/12/25. +// + +#include "../cloudsync.h" +#include "../database.h" +#include "../dbutils.h" +#include "../utils.h" +#include "../sql.h" + +#include +#include +#include + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif + +// MARK: - SQL - + +char *sql_build_drop_table (const char *table_name, char *buffer, int bsize, bool is_meta) { + char *sql = NULL; + + if (is_meta) { + sql = sqlite3_snprintf(bsize, buffer, "DROP TABLE IF EXISTS \"%w_cloudsync\";", table_name); + } else { + sql = sqlite3_snprintf(bsize, buffer, "DROP TABLE IF EXISTS \"%w\";", table_name); + } + + return sql; +} + +char *sql_escape_identifier (const char *name, char *buffer, size_t bsize) { + return sqlite3_snprintf((int)bsize, buffer, "%q", name); +} + +char *sql_build_select_nonpk_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char *sql = NULL; + + /* + This SQL statement dynamically generates a SELECT query for a specified table. + It uses Common Table Expressions (CTEs) to construct the column names and + primary key conditions based on the table schema, which is obtained through + the `pragma_table_info` function. + + 1. `col_names` CTE: + - Retrieves a comma-separated list of non-primary key column names from + the specified table's schema. + + 2. `pk_where` CTE: + - Retrieves a condition string representing the primary key columns in the + format: "column1=? AND column2=? AND ...", used to create the WHERE clause + for selecting rows based on primary key values. + + 3. Final SELECT: + - Constructs the complete SELECT statement as a string, combining: + - Column names from `col_names`. + - The target table name. + - The WHERE clause conditions from `pk_where`. + + The resulting query can be used to select rows from the table based on primary + key values, and can be executed within the application to retrieve data dynamically. + */ + + // Unfortunately in SQLite column names (or table names) cannot be bound parameters in a SELECT statement + // otherwise we should have used something like SELECT 'SELECT ? FROM %w WHERE rowid=?'; + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + + #if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES + if (table->rowid_only) { + sql = memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID, table->name, table->name); + goto process_process; + } + #endif + + sql = cloudsync_memory_mprintf(SQL_BUILD_SELECT_NONPK_COLS_BY_PK, table_name, table_name, singlequote_escaped_table_name); + +#if !CLOUDSYNC_DISABLE_ROWIDONLY_TABLES +process_process: +#endif + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_delete_by_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *sql = cloudsync_memory_mprintf(SQL_BUILD_DELETE_ROW_BY_PK, table_name, singlequote_escaped_table_name); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_insert_pk_ignore (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *sql = cloudsync_memory_mprintf(SQL_BUILD_INSERT_PK_IGNORE, table_name, table_name, singlequote_escaped_table_name); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_col (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char buffer2[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *singlequote_escaped_col_name = sql_escape_identifier(colname, buffer2, sizeof(buffer2)); + char *sql = cloudsync_memory_mprintf( + SQL_BUILD_UPSERT_PK_AND_COL, + table_name, + table_name, + singlequote_escaped_table_name, + singlequote_escaped_col_name, + singlequote_escaped_col_name + ); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_upsert_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + UNUSED_PARAMETER(schema); + if (ncolnames <= 0 || !colnames) return NULL; + + // Get PK column names via pragma_table_info (same approach as database_pk_names) + char **pk_names = NULL; + int npks = 0; + int rc = database_pk_names(data, table_name, &pk_names, &npks); + if (rc != DBRES_OK || npks <= 0 || !pk_names) return NULL; + + // Build column list: "pk1","pk2","col_a","col_b" + char *col_list = cloudsync_memory_mprintf("\"%w\"", pk_names[0]); + if (!col_list) goto fail; + for (int i = 1; i < npks; i++) { + char *prev = col_list; + col_list = cloudsync_memory_mprintf("%s,\"%w\"", prev, pk_names[i]); + cloudsync_memory_free(prev); + if (!col_list) goto fail; + } + for (int i = 0; i < ncolnames; i++) { + char *prev = col_list; + col_list = cloudsync_memory_mprintf("%s,\"%w\"", prev, colnames[i]); + cloudsync_memory_free(prev); + if (!col_list) goto fail; + } + + // Build bind list: ?,?,?,? + int total = npks + ncolnames; + char *binds = (char *)cloudsync_memory_alloc(total * 2); + if (!binds) { cloudsync_memory_free(col_list); goto fail; } + int pos = 0; + for (int i = 0; i < total; i++) { + if (i > 0) binds[pos++] = ','; + binds[pos++] = '?'; + } + binds[pos] = '\0'; + + // Build excluded set: "col_a"=EXCLUDED."col_a","col_b"=EXCLUDED."col_b" + char *excl = cloudsync_memory_mprintf("\"%w\"=EXCLUDED.\"%w\"", colnames[0], colnames[0]); + if (!excl) { cloudsync_memory_free(col_list); cloudsync_memory_free(binds); goto fail; } + for (int i = 1; i < ncolnames; i++) { + char *prev = excl; + excl = cloudsync_memory_mprintf("%s,\"%w\"=EXCLUDED.\"%w\"", prev, colnames[i], colnames[i]); + cloudsync_memory_free(prev); + if (!excl) { cloudsync_memory_free(col_list); cloudsync_memory_free(binds); goto fail; } + } + + // Assemble final SQL + char *sql = cloudsync_memory_mprintf( + "INSERT INTO \"%w\" (%s) VALUES (%s) ON CONFLICT DO UPDATE SET %s;", + table_name, col_list, binds, excl + ); + + cloudsync_memory_free(col_list); + cloudsync_memory_free(binds); + cloudsync_memory_free(excl); + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + return sql; + +fail: + if (pk_names) { + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + } + return NULL; +} + +char *sql_build_update_pk_and_multi_cols (cloudsync_context *data, const char *table_name, const char **colnames, int ncolnames, const char *schema) { + UNUSED_PARAMETER(schema); + if (ncolnames <= 0 || !colnames) return NULL; + + // Get PK column names + char **pk_names = NULL; + int npks = 0; + int rc = database_pk_names(data, table_name, &pk_names, &npks); + if (rc != DBRES_OK || npks <= 0 || !pk_names) return NULL; + + // Build SET clause: "col_a"=?npks+1,"col_b"=?npks+2 + // Uses numbered parameters to match merge_flush_pending bind order: + // positions 1..npks are PKs, npks+1..npks+ncolnames are column values. + char *set_clause = cloudsync_memory_mprintf("\"%w\"=?%d", colnames[0], npks + 1); + if (!set_clause) goto fail; + for (int i = 1; i < ncolnames; i++) { + char *prev = set_clause; + set_clause = cloudsync_memory_mprintf("%s,\"%w\"=?%d", prev, colnames[i], npks + 1 + i); + cloudsync_memory_free(prev); + if (!set_clause) goto fail; + } + + // Build WHERE clause: "pk1"=?1 AND "pk2"=?2 + char *where_clause = cloudsync_memory_mprintf("\"%w\"=?%d", pk_names[0], 1); + if (!where_clause) { cloudsync_memory_free(set_clause); goto fail; } + for (int i = 1; i < npks; i++) { + char *prev = where_clause; + where_clause = cloudsync_memory_mprintf("%s AND \"%w\"=?%d", prev, pk_names[i], 1 + i); + cloudsync_memory_free(prev); + if (!where_clause) { cloudsync_memory_free(set_clause); goto fail; } + } + + // Assemble: UPDATE "table" SET ... WHERE ... + char *sql = cloudsync_memory_mprintf( + "UPDATE \"%w\" SET %s WHERE %s;", + table_name, set_clause, where_clause + ); + + cloudsync_memory_free(set_clause); + cloudsync_memory_free(where_clause); + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + return sql; + +fail: + if (pk_names) { + for (int i = 0; i < npks; i++) cloudsync_memory_free(pk_names[i]); + cloudsync_memory_free(pk_names); + } + return NULL; +} + +char *sql_build_select_cols_by_pk (cloudsync_context *data, const char *table_name, const char *colname, const char *schema) { + UNUSED_PARAMETER(schema); + char *colnamequote = "\""; + char buffer[1024]; + char buffer2[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + char *singlequote_escaped_col_name = sql_escape_identifier(colname, buffer2, sizeof(buffer2)); + char *sql = cloudsync_memory_mprintf( + SQL_BUILD_SELECT_COLS_BY_PK_FMT, + table_name, + colnamequote, + singlequote_escaped_col_name, + colnamequote, + singlequote_escaped_table_name + ); + if (!sql) return NULL; + + char *query = NULL; + int rc = database_select_text(data, sql, &query); + cloudsync_memory_free(sql); + + return (rc == DBRES_OK) ? query : NULL; +} + +char *sql_build_rekey_pk_and_reset_version_except_col (cloudsync_context *data, const char *table_name, const char *except_col) { + UNUSED_PARAMETER(data); + + char *meta_ref = database_build_meta_ref(NULL, table_name); + if (!meta_ref) return NULL; + + char *result = cloudsync_memory_mprintf(SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL, meta_ref, except_col); + cloudsync_memory_free(meta_ref); + return result; +} + +char *database_table_schema (const char *table_name) { + return NULL; +} + +char *database_build_meta_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_memory_mprintf("%s_cloudsync", table_name); +} + +char *database_build_base_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_string_dup(table_name); +} + +char *database_build_blocks_ref (const char *schema, const char *table_name) { + // schema unused in SQLite + return cloudsync_memory_mprintf("%s_cloudsync_blocks", table_name); +} + +// SQLite version: schema parameter unused (SQLite has no schemas). +char *sql_build_delete_cols_not_in_schema_query (const char *schema, const char *table_name, const char *meta_ref, const char *pkcol) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "DELETE FROM \"%w\" WHERE col_name NOT IN (" + "SELECT name FROM pragma_table_info('%q') " + "UNION SELECT '%s'" + ");", + meta_ref, table_name, pkcol + ); +} + +char *sql_build_pk_collist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", + table_name + ); +} + +char *sql_build_pk_decode_selectlist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + return cloudsync_memory_mprintf( + "SELECT group_concat(" + "'cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ','" + ") " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", + table_name + ); +} + +char *sql_build_pk_qualified_collist_query (const char *schema, const char *table_name) { + UNUSED_PARAMETER(schema); + + char buffer[1024]; + char *singlequote_escaped_table_name = sql_escape_identifier(table_name, buffer, sizeof(buffer)); + if (!singlequote_escaped_table_name) return NULL; + + return cloudsync_memory_mprintf( + "SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;", singlequote_escaped_table_name, singlequote_escaped_table_name + ); +} + +char *sql_build_insert_missing_pks_query(const char *schema, const char *table_name, + const char *pkvalues_identifiers, + const char *base_ref, const char *meta_ref, + const char *filter) { + UNUSED_PARAMETER(schema); + + // SQLite: Use NOT EXISTS with cloudsync_pk_encode (same approach as PostgreSQL). + // This avoids needing pk_decode select list which requires executing a query. + if (filter) { + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%q', %s) " + "FROM \"%w\" " + "WHERE (%s) AND NOT EXISTS (" + " SELECT 1 FROM \"%w\" WHERE pk = cloudsync_pk_encode(%s)" + ");", + table_name, pkvalues_identifiers, base_ref, filter, meta_ref, pkvalues_identifiers + ); + } + return cloudsync_memory_mprintf( + "SELECT cloudsync_insert('%q', %s) " + "FROM \"%w\" " + "WHERE NOT EXISTS (" + " SELECT 1 FROM \"%w\" WHERE pk = cloudsync_pk_encode(%s)" + ");", + table_name, pkvalues_identifiers, base_ref, meta_ref, pkvalues_identifiers + ); +} + +// MARK: - PRIVATE - + +static int database_select1_value (cloudsync_context *data, const char *sql, char **ptr_value, int64_t *int_value, DBTYPE expected_type) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + // init values and sanity check expected_type + if (ptr_value) *ptr_value = NULL; + *int_value = 0; + if (expected_type != DBTYPE_INTEGER && expected_type != DBTYPE_TEXT && expected_type != DBTYPE_BLOB) return SQLITE_MISUSE; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_select; + + // ensure at least one column + if (sqlite3_column_count(vm) < 1) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) {rc = SQLITE_OK; goto cleanup_select;} // no rows OK + if (rc != SQLITE_ROW) goto cleanup_select; + + // sanity check column type + DBTYPE type = (DBTYPE)sqlite3_column_type(vm, 0); + if (type == SQLITE_NULL) {rc = SQLITE_OK; goto cleanup_select;} + if (type != expected_type) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + if (expected_type == DBTYPE_INTEGER) { + *int_value = (int64_t)sqlite3_column_int64(vm, 0); + } else { + const void *value = (expected_type == DBTYPE_TEXT) ? (const void *)sqlite3_column_text(vm, 0) : (const void *)sqlite3_column_blob(vm, 0); + int len = sqlite3_column_bytes(vm, 0); + if (len) { + char *ptr = cloudsync_memory_alloc(len + 1); + if (!ptr) {rc = SQLITE_NOMEM; goto cleanup_select;} + + if (len > 0 && value) memcpy(ptr, value, len); + if (expected_type == DBTYPE_TEXT) ptr[len] = 0; // NULL terminate in case of TEXT + + *ptr_value = ptr; + *int_value = len; + } + } + rc = SQLITE_OK; + +cleanup_select: + if (vm) sqlite3_finalize(vm); + return rc; +} + +static int database_select2_values (cloudsync_context *data, const char *sql, char **value, int64_t *len, int64_t *value2) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + // init values and sanity check expected_type + *value = NULL; + *value2 = 0; + *len = 0; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_select; + + // ensure column count + if (sqlite3_column_count(vm) < 2) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + rc = sqlite3_step(vm); + if (rc == SQLITE_DONE) {rc = SQLITE_OK; goto cleanup_select;} // no rows OK + if (rc != SQLITE_ROW) goto cleanup_select; + + // sanity check column types + if (sqlite3_column_type(vm, 0) != SQLITE_BLOB) {rc = SQLITE_MISMATCH; goto cleanup_select;} + if (sqlite3_column_type(vm, 1) != SQLITE_INTEGER) {rc = SQLITE_MISMATCH; goto cleanup_select;} + + // 1st column is BLOB + const void *blob = (const void *)sqlite3_column_blob(vm, 0); + int blob_len = sqlite3_column_bytes(vm, 0); + if (blob_len) { + char *ptr = cloudsync_memory_alloc(blob_len); + if (!ptr) {rc = SQLITE_NOMEM; goto cleanup_select;} + + if (blob_len > 0 && blob) memcpy(ptr, blob, blob_len); + *value = ptr; + *len = blob_len; + } + + // 2nd column is INTEGER + *value2 = (int64_t)sqlite3_column_int64(vm, 1); + + rc = SQLITE_OK; + +cleanup_select: + if (vm) sqlite3_finalize(vm); + return rc; +} + +bool database_system_exists (cloudsync_context *data, const char *name, const char *type) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + bool result = false; + + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type='%s' AND name=?1 COLLATE NOCASE);", type); + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_bind_text(vm, 1, name, -1, SQLITE_STATIC); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_step(vm); + if (rc == SQLITE_ROW) { + result = (bool)sqlite3_column_int(vm, 0); + rc = SQLITE_OK; + } + +finalize: + if (rc != SQLITE_OK) DEBUG_ALWAYS("Error executing %s in dbutils_system_exists for type %s name %s (%s).", sql, type, name, sqlite3_errmsg(db)); + if (vm) sqlite3_finalize(vm); + return result; +} + +// MARK: - GENERAL - + +int database_exec (cloudsync_context *data, const char *sql) { + return sqlite3_exec((sqlite3 *)cloudsync_db(data), sql, NULL, NULL, NULL); +} + +int database_exec_callback (cloudsync_context *data, const char *sql, int (*callback)(void *xdata, int argc, char **values, char **names), void *xdata) { + return sqlite3_exec((sqlite3 *)cloudsync_db(data), sql, callback, xdata, NULL); +} + +int database_write (cloudsync_context *data, const char *sql, const char **bind_values, DBTYPE bind_types[], int bind_lens[], int bind_count) { + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2((sqlite3 *)db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup_write; + + for (int i=0; i0 AND \"notnull\"=1;", table_name); + } else { + sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk>0;", table_name); + } + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_nonpk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = NULL; + + sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0;", table_name); + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_int_pk (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=1 AND \"type\" LIKE '%%INT%%';", table_name); + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_count_notnull_without_default (cloudsync_context *data, const char *table_name, const char *schema) { + UNUSED_PARAMETER(schema); + char buffer[1024]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT count(*) FROM pragma_table_info('%q') WHERE pk=0 AND \"notnull\"=1 AND \"dflt_value\" IS NULL;", table_name); + + int64_t count = 0; + int rc = database_select_int(data, sql, &count); + if (rc != DBRES_OK) return -1; + return (int)count; +} + +int database_cleanup (cloudsync_context *data) { + char *sql = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'cloudsync_%' AND name NOT LIKE '%_cloudsync';"; + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + char **result = NULL; + char *errmsg = NULL; + int nrows, ncols; + int rc = sqlite3_get_table(db, sql, &result, &nrows, &ncols, &errmsg); + if (rc != SQLITE_OK) { + cloudsync_set_error(data, (errmsg) ? errmsg : "Error retrieving augmented tables", rc); + goto exit_cleanup; + } + + for (int i = ncols; i < nrows+ncols; i+=ncols) { + int rc2 = cloudsync_cleanup(data, result[i]); + if (rc2 != SQLITE_OK) {rc = rc2; goto exit_cleanup;} + } + +exit_cleanup: + if (result) sqlite3_free_table(result); + if (errmsg) sqlite3_free(errmsg); + return rc; +} + +// MARK: - TRIGGERS and META - + +int database_create_metatable (cloudsync_context *data, const char *table_name) { + DEBUG_DBFUNCTION("database_create_metatable %s", table_name); + + // table_name cannot be longer than 512 characters so static buffer size is computed accordling to that value + char buffer[2048]; + + // WITHOUT ROWID is available starting from SQLite version 3.8.2 (2013-12-06) and later + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "CREATE TABLE IF NOT EXISTS \"%w_cloudsync\" (pk BLOB NOT NULL, col_name TEXT NOT NULL, col_version INTEGER, db_version INTEGER, site_id INTEGER DEFAULT 0, seq INTEGER, PRIMARY KEY (pk, col_name)) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS \"%w_cloudsync_db_idx\" ON \"%w_cloudsync\" (db_version);", table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_insert_trigger (cloudsync_context *data, const char *table_name, char *trigger_when) { + // NEW.prikey1, NEW.prikey2... + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_insert_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('NEW.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); + + char *pkclause = NULL; + int rc = database_select_text(data, sql2, &pkclause); + if (rc != SQLITE_OK) return rc; + char *pkvalues = (pkclause) ? pkclause : "NEW.rowid"; + + char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER INSERT ON \"%w\" %s BEGIN SELECT cloudsync_insert('%q', %s); END", trigger_name, table_name, trigger_when, table_name, pkvalues); + if (pkclause) cloudsync_memory_free(pkclause); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +int database_create_update_trigger_gos (cloudsync_context *data, const char *table_name) { + // Grow Only Set + // In a grow-only set, the update operation is not allowed. + // A grow-only set is a type of CRDT (Conflict-free Replicated Data Type) where the only permissible operation is to add elements to the set, + // without ever removing or modifying them. + // Once an element is added to the set, it remains there permanently, which guarantees that the set only grows over time. + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_before_update_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048+512]; + char *sql = sqlite3_snprintf(sizeof(buffer2), buffer2, "CREATE TRIGGER \"%w\" BEFORE UPDATE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: UPDATE operation is not allowed on table %w.'); END", trigger_name, table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_update_trigger (cloudsync_context *data, const char *table_name, const char *trigger_when) { + // NEW.prikey1, NEW.prikey2, OLD.prikey1, OLD.prikey2, NEW.col1, OLD.col1, NEW.col2, OLD.col2... + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_update_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + // generate VALUES clause for all columns using a CTE to avoid compound SELECT limits + // first, get all primary key columns in order + char buffer2[2048]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name, table_name); + + char *pk_values_list = NULL; + int rc = database_select_text(data, sql2, &pk_values_list); + if (rc != SQLITE_OK) return rc; + + // then get all regular columns in order + sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('('||quote('%q')||', NEW.\"' || format('%%w', name) || '\", OLD.\"' || format('%%w', name) || '\")', ', ') FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;", table_name, table_name); + + char *col_values_list = NULL; + rc = database_select_text(data, sql2, &col_values_list); + if (rc != SQLITE_OK) { + if (pk_values_list) cloudsync_memory_free(pk_values_list); + return rc; + } + + // build the complete VALUES query + char *values_query = NULL; + if (col_values_list && strlen(col_values_list) > 0) { + // Table has both primary keys and regular columns + values_query = cloudsync_memory_mprintf( + "WITH column_data(table_name, new_value, old_value) AS (VALUES %s, %s) " + "SELECT table_name, new_value, old_value FROM column_data", + pk_values_list, col_values_list); + } else { + // Table has only primary keys + values_query = cloudsync_memory_mprintf( + "WITH column_data(table_name, new_value, old_value) AS (VALUES %s) " + "SELECT table_name, new_value, old_value FROM column_data", + pk_values_list); + } + + if (pk_values_list) cloudsync_memory_free(pk_values_list); + if (col_values_list) cloudsync_memory_free(col_values_list); + if (!values_query) return SQLITE_NOMEM; + + // create the trigger with aggregate function + char *sql = cloudsync_memory_mprintf( + "CREATE TRIGGER \"%w\" AFTER UPDATE ON \"%w\" %s BEGIN " + "SELECT cloudsync_update(table_name, new_value, old_value) FROM (%s); " + "END", + trigger_name, table_name, trigger_when, values_query); + + cloudsync_memory_free(values_query); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +int database_create_delete_trigger_gos (cloudsync_context *data, const char *table_name) { + // Grow Only Set + // In a grow-only set, the delete operation is not allowed. + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_before_delete_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[2048+512]; + char *sql = sqlite3_snprintf(sizeof(buffer2), buffer2, "CREATE TRIGGER \"%w\" BEFORE DELETE ON \"%w\" FOR EACH ROW WHEN cloudsync_is_enabled('%q') = 1 BEGIN SELECT RAISE(ABORT, 'Error: DELETE operation is not allowed on table %w.'); END", trigger_name, table_name, table_name, table_name); + + int rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + return rc; +} + +int database_create_delete_trigger (cloudsync_context *data, const char *table_name, const char *trigger_when) { + // OLD.prikey1, OLD.prikey2... + + char buffer[1024]; + char *trigger_name = sqlite3_snprintf(sizeof(buffer), buffer, "cloudsync_after_delete_%s", table_name); + if (database_trigger_exists(data, trigger_name)) return SQLITE_OK; + + char buffer2[1024]; + char *sql2 = sqlite3_snprintf(sizeof(buffer2), buffer2, "SELECT group_concat('OLD.\"' || format('%%w', name) || '\"', ',') FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;", table_name); + + char *pkclause = NULL; + int rc = database_select_text(data, sql2, &pkclause); + if (rc != SQLITE_OK) return rc; + char *pkvalues = (pkclause) ? pkclause : "OLD.rowid"; + + char *sql = cloudsync_memory_mprintf("CREATE TRIGGER \"%w\" AFTER DELETE ON \"%w\" %s BEGIN SELECT cloudsync_delete('%q',%s); END", trigger_name, table_name, trigger_when, table_name, pkvalues); + if (pkclause) cloudsync_memory_free(pkclause); + if (!sql) return SQLITE_NOMEM; + + rc = database_exec(data, sql); + DEBUG_SQL("\n%s", sql); + cloudsync_memory_free(sql); + return rc; +} + +// Build trigger WHEN clauses, optionally incorporating a row-level filter. +// INSERT/UPDATE use NEW-prefixed filter, DELETE uses OLD-prefixed filter. +// Returns dynamically-allocated strings that must be freed with cloudsync_memory_free. +static void database_build_trigger_when( + cloudsync_context *data, const char *table_name, const char *filter, + char **when_new_out, char **when_old_out) +{ + char *new_filter_str = NULL; + char *old_filter_str = NULL; + + if (filter) { + char sql_cols[1024]; + sqlite3_snprintf(sizeof(sql_cols), sql_cols, + "SELECT name FROM pragma_table_info('%q') ORDER BY cid;", table_name); + + char *col_names[256]; + int ncols = 0; + + sqlite3_stmt *col_vm = NULL; + int col_rc = sqlite3_prepare_v2((sqlite3 *)cloudsync_db(data), sql_cols, -1, &col_vm, NULL); + if (col_rc == SQLITE_OK) { + while (sqlite3_step(col_vm) == SQLITE_ROW && ncols < 256) { + const char *name = (const char *)sqlite3_column_text(col_vm, 0); + if (name) { + char *dup = cloudsync_memory_mprintf("%s", name); + if (!dup) break; + col_names[ncols++] = dup; + } + } + sqlite3_finalize(col_vm); + } + + if (ncols > 0) { + new_filter_str = cloudsync_filter_add_row_prefix(filter, "NEW", col_names, ncols); + old_filter_str = cloudsync_filter_add_row_prefix(filter, "OLD", col_names, ncols); + for (int i = 0; i < ncols; ++i) cloudsync_memory_free(col_names[i]); + } + } + + if (new_filter_str) { + *when_new_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0 AND (%s)", table_name, new_filter_str); + } else { + *when_new_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table_name); + } + + if (old_filter_str) { + *when_old_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0 AND (%s)", table_name, old_filter_str); + } else { + *when_old_out = cloudsync_memory_mprintf( + "FOR EACH ROW WHEN cloudsync_is_sync('%q') = 0", table_name); + } + + if (new_filter_str) cloudsync_memory_free(new_filter_str); + if (old_filter_str) cloudsync_memory_free(old_filter_str); +} + +int database_create_triggers (cloudsync_context *data, const char *table_name, table_algo algo, const char *filter) { + DEBUG_DBFUNCTION("database_create_triggers %s", table_name); + + if (dbutils_settings_check_version(data, "0.8.25") <= 0) { + database_delete_triggers(data, table_name); + } + + char *trigger_when_new = NULL; + char *trigger_when_old = NULL; + database_build_trigger_when(data, table_name, filter, + &trigger_when_new, &trigger_when_old); + + if (!trigger_when_new || !trigger_when_old) { + if (trigger_when_new) cloudsync_memory_free(trigger_when_new); + if (trigger_when_old) cloudsync_memory_free(trigger_when_old); + return SQLITE_NOMEM; + } + + // INSERT TRIGGER (uses NEW prefix) + int rc = database_create_insert_trigger(data, table_name, trigger_when_new); + if (rc != SQLITE_OK) goto done; + + // UPDATE TRIGGER (uses NEW prefix) + if (algo == table_algo_crdt_gos) rc = database_create_update_trigger_gos(data, table_name); + else rc = database_create_update_trigger(data, table_name, trigger_when_new); + if (rc != SQLITE_OK) goto done; + + // DELETE TRIGGER (uses OLD prefix) + if (algo == table_algo_crdt_gos) rc = database_create_delete_trigger_gos(data, table_name); + else rc = database_create_delete_trigger(data, table_name, trigger_when_old); + +done: + if (rc != SQLITE_OK) DEBUG_ALWAYS("database_create_triggers error %s (%d)", sqlite3_errmsg(cloudsync_db(data)), rc); + cloudsync_memory_free(trigger_when_new); + cloudsync_memory_free(trigger_when_old); + return rc; +} + +int database_delete_triggers (cloudsync_context *data, const char *table) { + DEBUG_DBFUNCTION("database_delete_triggers %s", table); + + // from cloudsync_table_sanity_check we already know that 2048 is OK + char buffer[2048]; + size_t blen = sizeof(buffer); + int rc = SQLITE_ERROR; + + char *sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_update_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_before_delete_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_insert_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_update_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + + sql = sqlite3_snprintf((int)blen, buffer, "DROP TRIGGER IF EXISTS \"cloudsync_after_delete_%w\";", table); + rc = database_exec(data, sql); + if (rc != SQLITE_OK) goto finalize; + +finalize: + if (rc != SQLITE_OK) DEBUG_ALWAYS("dbutils_delete_triggers error %s (%s)", database_errmsg(data), sql); + return rc; +} + +// MARK: - SCHEMA - + +int64_t database_schema_version (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, SQL_SCHEMA_VERSION, &value); + return (rc == DBRES_OK) ? value : 0; +} + +uint64_t database_schema_hash (cloudsync_context *data) { + int64_t value = 0; + int rc = database_select_int(data, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC limit 1;", &value); + return (rc == DBRES_OK) ? (uint64_t)value : 0; +} + +bool database_check_schema_hash (cloudsync_context *data, uint64_t hash) { + // a change from the current version of the schema or from previous known schema can be applied + // a change from a newer schema version not yet applied to this peer cannot be applied + // so a schema hash is valid if it exists in the cloudsync_schema_versions table + + // the idea is to allow changes on stale peers and to be able to apply these changes on peers with newer schema, + // but it requires alter table operation on augmented tables only add new columns and never drop columns for backward compatibility + char sql[1024]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%" PRId64 ")", (int64_t)hash); + + int64_t value = 0; + database_select_int(data, sql, &value); + return (value == 1); +} + +int database_update_schema_hash (cloudsync_context *data, uint64_t *hash) { + // Build normalized schema string using only: column name (lowercase), type (SQLite affinity), pk flag + // Format: tablename:colname:affinity:pk,... (ordered by table name, then column id) + // This makes the hash resilient to formatting, quoting, case differences and portable across databases + // + // Type mapping (simplified from SQLite affinity rules for cross-database compatibility): + // - Types containing 'INT' → 'integer' + // - Types containing 'CHAR', 'CLOB', 'TEXT' → 'text' + // - Types containing 'BLOB' or empty → 'blob' + // - Types containing 'REAL', 'FLOA', 'DOUB' → 'real' + // - Types exactly 'NUMERIC' or 'DECIMAL' → 'numeric' + // - Everything else → 'text' (default) + // + // NOTE: This deviates from SQLite's actual affinity rules where unknown types get NUMERIC affinity. + // We use 'text' as default to improve cross-database compatibility with PostgreSQL, where types + // like TIMESTAMPTZ, UUID, JSON, etc. are commonly used and map to 'text' in the PostgreSQL + // implementation. This ensures schemas with PostgreSQL-specific type names in SQLite DDL + // will hash consistently across both databases. + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + + char **tables = NULL; + int ntables, tcols; + int rc = sqlite3_get_table(db, "SELECT DISTINCT tbl_name FROM cloudsync_table_settings ORDER BY tbl_name;", + &tables, &ntables, &tcols, NULL); + if (rc != SQLITE_OK || ntables == 0) { + if (tables) sqlite3_free_table(tables); + return SQLITE_ERROR; + } + + char *schema = NULL; + size_t schema_len = 0; + size_t schema_cap = 0; + + for (int t = 1; t <= ntables; t++) { + const char *tbl_name = tables[t]; + + // Query pragma_table_info for this table with normalized type + char *col_sql = cloudsync_memory_mprintf( + "SELECT LOWER(name), " + "CASE " + " WHEN UPPER(type) LIKE '%%INT%%' THEN 'integer' " + " WHEN UPPER(type) LIKE '%%CHAR%%' OR UPPER(type) LIKE '%%CLOB%%' OR UPPER(type) LIKE '%%TEXT%%' THEN 'text' " + " WHEN UPPER(type) LIKE '%%BLOB%%' OR type = '' THEN 'blob' " + " WHEN UPPER(type) LIKE '%%REAL%%' OR UPPER(type) LIKE '%%FLOA%%' OR UPPER(type) LIKE '%%DOUB%%' THEN 'real' " + " WHEN UPPER(type) IN ('NUMERIC', 'DECIMAL') THEN 'numeric' " + " ELSE 'text' " + "END, " + "CASE WHEN pk > 0 THEN '1' ELSE '0' END " + "FROM pragma_table_info('%q') ORDER BY cid;", tbl_name); + + if (!col_sql) { + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(tables); + return SQLITE_NOMEM; + } + + char **cols = NULL; + int nrows, ncols; + rc = sqlite3_get_table(db, col_sql, &cols, &nrows, &ncols, NULL); + cloudsync_memory_free(col_sql); + + if (rc != SQLITE_OK || ncols != 3) { + if (cols) sqlite3_free_table(cols); + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(tables); + return SQLITE_ERROR; + } + + // Append each column: tablename:colname:affinity:pk + for (int r = 1; r <= nrows; r++) { + const char *col_name = cols[r * 3]; + const char *col_type = cols[r * 3 + 1]; + const char *col_pk = cols[r * 3 + 2]; + + // Calculate required size: tbl_name:col_name:col_type:col_pk, + size_t entry_len = strlen(tbl_name) + 1 + strlen(col_name) + 1 + strlen(col_type) + 1 + strlen(col_pk) + 1; + + if (schema_len + entry_len + 1 > schema_cap) { + schema_cap = (schema_cap == 0) ? 1024 : schema_cap * 2; + if (schema_cap < schema_len + entry_len + 1) schema_cap = schema_len + entry_len + 1; + char *new_schema = cloudsync_memory_realloc(schema, schema_cap); + if (!new_schema) { + if (schema) cloudsync_memory_free(schema); + sqlite3_free_table(cols); + sqlite3_free_table(tables); + return SQLITE_NOMEM; + } + schema = new_schema; + } + + int written = snprintf(schema + schema_len, schema_cap - schema_len, "%s:%s:%s:%s,", + tbl_name, col_name, col_type, col_pk); + schema_len += written; + } + + sqlite3_free_table(cols); + } + + sqlite3_free_table(tables); + + if (!schema || schema_len == 0) return SQLITE_ERROR; + + // Remove trailing comma + if (schema_len > 0 && schema[schema_len - 1] == ',') { + schema[schema_len - 1] = '\0'; + schema_len--; + } + + DEBUG_MERGE("database_update_schema_hash len %zu schema %s", schema_len, schema); + sqlite3_uint64 h = fnv1a_hash(schema, schema_len); + cloudsync_memory_free(schema); + if (hash && *hash == h) return SQLITE_CONSTRAINT; + + char sql[1024]; + snprintf(sql, sizeof(sql), "INSERT INTO cloudsync_schema_versions (hash, seq) " + "VALUES (%" PRId64 ", COALESCE((SELECT MAX(seq) FROM cloudsync_schema_versions), 0) + 1) " + "ON CONFLICT(hash) DO UPDATE SET " + " seq = (SELECT COALESCE(MAX(seq), 0) + 1 FROM cloudsync_schema_versions);", (int64_t)h); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc == SQLITE_OK && hash) *hash = h; + return rc; +} + +// MARK: - VM - + +int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags) { + return sqlite3_prepare_v3((sqlite3 *)cloudsync_db(data), sql, -1, flags, (sqlite3_stmt **)vm, NULL); +} + +int databasevm_step (dbvm_t *vm) { + return sqlite3_step((sqlite3_stmt *)vm); +} + +void databasevm_finalize (dbvm_t *vm) { + sqlite3_finalize((sqlite3_stmt *)vm); +} + +void databasevm_reset (dbvm_t *vm) { + sqlite3_reset((sqlite3_stmt *)vm); +} + +void databasevm_clear_bindings (dbvm_t *vm) { + sqlite3_clear_bindings((sqlite3_stmt *)vm); +} + +const char *databasevm_sql (dbvm_t *vm) { + return sqlite3_sql((sqlite3_stmt *)vm); + // the following allocates memory that needs to be freed + // return sqlite3_expanded_sql((sqlite3_stmt *)vm); +} + +static int database_pk_rowid (sqlite3 *db, const char *table_name, char ***names, int *count) { + char buffer[2048]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT rowid FROM %Q LIMIT 0;", table_name); + if (!sql) return SQLITE_NOMEM; + + sqlite3_stmt *vm = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + { + char **r = (char**)cloudsync_memory_alloc(sizeof(char*)); + if (!r) {rc = SQLITE_NOMEM; goto cleanup;} + r[0] = cloudsync_string_dup("rowid"); + if (!r[0]) {cloudsync_memory_free(r); rc = SQLITE_NOMEM; goto cleanup;} + *names = r; + *count = 1; + } + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +int database_pk_names (cloudsync_context *data, const char *table_name, char ***names, int *count) { + char buffer[2048]; + char *sql = sqlite3_snprintf(sizeof(buffer), buffer, "SELECT name FROM pragma_table_info(%Q) WHERE pk > 0 ORDER BY pk;", table_name); + if (!sql) return SQLITE_NOMEM; + + sqlite3 *db = (sqlite3 *)cloudsync_db(data); + sqlite3_stmt *vm = NULL; + + int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // count PK columns + int rows = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) rows++; + if (rc != SQLITE_DONE) goto cleanup; + + if (rows == 0) { + sqlite3_finalize(vm); + // no declared PKs so check for rowid availability + return database_pk_rowid(db, table_name, names, count); + } + + // reset vm to read PKs again + rc = sqlite3_reset(vm); + if (rc != SQLITE_OK) goto cleanup; + + // allocate array + char **r = (char**)cloudsync_memory_zeroalloc(sizeof(char*) * rows); + if (!r) {rc = SQLITE_NOMEM; goto cleanup;} + + int i = 0; + while ((rc = sqlite3_step(vm)) == SQLITE_ROW) { + const char *txt = (const char*)sqlite3_column_text(vm, 0); + if (!txt) {rc = SQLITE_ERROR; goto cleanup_r;} + r[i] = cloudsync_string_dup(txt); + if (!r[i]) { rc = SQLITE_NOMEM; goto cleanup_r;} + i++; + } + if (rc == SQLITE_DONE) rc = SQLITE_OK; + + *names = r; + *count = rows; + goto cleanup; + +cleanup_r: + for (int j = 0; j < i; j++) { + if (r[j]) cloudsync_memory_free(r[j]); + } + cloudsync_memory_free(r); + +cleanup: + if (vm) sqlite3_finalize(vm); + return rc; +} + +// MARK: - BINDING - + +int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size) { + return sqlite3_bind_blob64((sqlite3_stmt *)vm, index, value, size, SQLITE_STATIC); +} + +int databasevm_bind_double (dbvm_t *vm, int index, double value) { + return sqlite3_bind_double((sqlite3_stmt *)vm, index, value); +} + +int databasevm_bind_int (dbvm_t *vm, int index, int64_t value) { + return sqlite3_bind_int64((sqlite3_stmt *)vm, index, value); +} + +int databasevm_bind_null (dbvm_t *vm, int index) { + return sqlite3_bind_null((sqlite3_stmt *)vm, index); +} + +int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size) { + return sqlite3_bind_text((sqlite3_stmt *)vm, index, value, size, SQLITE_STATIC); +} + +int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value) { + return sqlite3_bind_value((sqlite3_stmt *)vm, index, (const sqlite3_value *)value); +} + +// MARK: - VALUE - + +const void *database_value_blob (dbvalue_t *value) { + return sqlite3_value_blob((sqlite3_value *)value); +} + +double database_value_double (dbvalue_t *value) { + return sqlite3_value_double((sqlite3_value *)value); +} + +int64_t database_value_int (dbvalue_t *value) { + return (int64_t)sqlite3_value_int64((sqlite3_value *)value); +} + +const char *database_value_text (dbvalue_t *value) { + return (const char *)sqlite3_value_text((sqlite3_value *)value); +} + +int database_value_bytes (dbvalue_t *value) { + return sqlite3_value_bytes((sqlite3_value *)value); +} + +int database_value_type (dbvalue_t *value) { + return sqlite3_value_type((sqlite3_value *)value); +} + +void database_value_free (dbvalue_t *value) { + sqlite3_value_free((sqlite3_value *)value); +} + +void *database_value_dup (dbvalue_t *value) { + return sqlite3_value_dup((const sqlite3_value *)value); +} + + +// MARK: - COLUMN - + +const void *database_column_blob (dbvm_t *vm, int index, size_t *len) { + if (len) *len = sqlite3_column_bytes((sqlite3_stmt *)vm, index); + return sqlite3_column_blob((sqlite3_stmt *)vm, index); +} + +double database_column_double (dbvm_t *vm, int index) { + return sqlite3_column_double((sqlite3_stmt *)vm, index); +} + +int64_t database_column_int (dbvm_t *vm, int index) { + return (int64_t)sqlite3_column_int64((sqlite3_stmt *)vm, index); +} + +const char *database_column_text (dbvm_t *vm, int index) { + return (const char *)sqlite3_column_text((sqlite3_stmt *)vm, index); +} + +dbvalue_t *database_column_value (dbvm_t *vm, int index) { + return (dbvalue_t *)sqlite3_column_value((sqlite3_stmt *)vm, index); +} + +int database_column_bytes (dbvm_t *vm, int index) { + return sqlite3_column_bytes((sqlite3_stmt *)vm, index); +} + +int database_column_type (dbvm_t *vm, int index) { + return sqlite3_column_type((sqlite3_stmt *)vm, index); +} + +// MARK: - SAVEPOINT - + +int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "SAVEPOINT %s;", savepoint_name); + return database_exec(data, sql); +} + +int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "RELEASE %s;", savepoint_name); + return database_exec(data, sql); +} + +int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name) { + char sql[1024]; + snprintf(sql, sizeof(sql), "ROLLBACK TO %s; RELEASE %s;", savepoint_name, savepoint_name); + return database_exec(data, sql); +} + +// MARK: - MEMORY - + +void *dbmem_alloc (uint64_t size) { + return sqlite3_malloc64((sqlite3_uint64)size); +} + +void *dbmem_zeroalloc (uint64_t size) { + void *ptr = (void *)dbmem_alloc(size); + if (!ptr) return NULL; + + memset(ptr, 0, (size_t)size); + return ptr; +} + +void *dbmem_realloc (void *ptr, uint64_t new_size) { + return sqlite3_realloc64(ptr, (sqlite3_uint64)new_size); +} + +char *dbmem_vmprintf (const char *format, va_list list) { + return sqlite3_vmprintf(format, list); +} + +char *dbmem_mprintf(const char *format, ...) { + va_list ap; + char *z; + + va_start(ap, format); + z = dbmem_vmprintf(format, ap); + va_end(ap); + + return z; +} + +void dbmem_free (void *ptr) { + sqlite3_free(ptr); +} + +uint64_t dbmem_size (void *ptr) { + return (uint64_t)sqlite3_msize(ptr); +} + + diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c new file mode 100644 index 0000000..471ae9b --- /dev/null +++ b/src/sqlite/sql_sqlite.c @@ -0,0 +1,318 @@ +// +// sql_sqlite.c +// cloudsync +// +// Created by Marco Bambini on 17/12/25. +// + +#include "../sql.h" + +// MARK: Settings + +const char * const SQL_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_settings WHERE key=?1;"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_REPLACE = + "REPLACE INTO cloudsync_settings (key, value) VALUES (?1, ?2);"; + +const char * const SQL_SETTINGS_SET_KEY_VALUE_DELETE = + "DELETE FROM cloudsync_settings WHERE key = ?1;"; + +const char * const SQL_TABLE_SETTINGS_GET_VALUE = + "SELECT value FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ALL_FOR_TABLE = + "DELETE FROM cloudsync_table_settings WHERE tbl_name=?1;"; + +const char * const SQL_TABLE_SETTINGS_REPLACE = + "REPLACE INTO cloudsync_table_settings (tbl_name, col_name, key, value) VALUES (?1, ?2, ?3, ?4);"; + +const char * const SQL_TABLE_SETTINGS_DELETE_ONE = + "DELETE FROM cloudsync_table_settings WHERE (tbl_name=?1 AND col_name=?2 AND key=?3);"; + +const char * const SQL_TABLE_SETTINGS_COUNT_TABLES = + "SELECT count(*) FROM cloudsync_table_settings WHERE key='algo';"; + +const char * const SQL_SETTINGS_LOAD_GLOBAL = + "SELECT key, value FROM cloudsync_settings;"; + +const char * const SQL_SETTINGS_LOAD_TABLE = + "SELECT lower(tbl_name), lower(col_name), key, value FROM cloudsync_table_settings ORDER BY tbl_name, col_name;"; + +const char * const SQL_CREATE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_settings (key TEXT PRIMARY KEY NOT NULL COLLATE NOCASE, value TEXT);"; + +// format strings (sqlite3_snprintf) are also static SQL templates +const char * const SQL_INSERT_SETTINGS_STR_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%q', '%q');"; + +const char * const SQL_INSERT_SETTINGS_INT_FORMAT = + "INSERT INTO cloudsync_settings (key, value) VALUES ('%s', %lld);"; + +const char * const SQL_CREATE_SITE_ID_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_site_id (site_id BLOB UNIQUE NOT NULL);"; + +const char * const SQL_INSERT_SITE_ID_ROWID = + "INSERT INTO cloudsync_site_id (rowid, site_id) VALUES (?, ?);"; + +const char * const SQL_CREATE_TABLE_SETTINGS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_table_settings (tbl_name TEXT NOT NULL COLLATE NOCASE, col_name TEXT NOT NULL COLLATE NOCASE, key TEXT, value TEXT, PRIMARY KEY(tbl_name,col_name,key));"; + +const char * const SQL_CREATE_SCHEMA_VERSIONS_TABLE = + "CREATE TABLE IF NOT EXISTS cloudsync_schema_versions (hash INTEGER PRIMARY KEY, seq INTEGER NOT NULL)"; + +const char * const SQL_SETTINGS_CLEANUP_DROP_ALL = + "DROP TABLE IF EXISTS cloudsync_settings; " + "DROP TABLE IF EXISTS cloudsync_site_id; " + "DROP TABLE IF EXISTS cloudsync_table_settings; " + "DROP TABLE IF EXISTS cloudsync_schema_versions; "; + +// MARK: CloudSync + +const char * const SQL_DBVERSION_BUILD_QUERY = + "WITH table_names AS (" + "SELECT format('%w', name) as tbl_name " + "FROM sqlite_master " + "WHERE type='table' " + "AND name LIKE '%_cloudsync'" + "), " + "query_parts AS (" + "SELECT 'SELECT max(db_version) as version FROM \"' || tbl_name || '\"' as part FROM table_names" + "), " + "combined_query AS (" + "SELECT GROUP_CONCAT(part, ' UNION ALL ') || ' UNION SELECT value as version FROM cloudsync_settings WHERE key = ''pre_alter_dbversion''' as full_query FROM query_parts" + ") " + "SELECT 'SELECT max(version) as version FROM (' || full_query || ');' FROM combined_query;"; + +const char * const SQL_SITEID_SELECT_ROWID0 = + "SELECT site_id FROM cloudsync_site_id WHERE rowid=0;"; + +const char * const SQL_DATA_VERSION = + "PRAGMA data_version;"; + +const char * const SQL_SCHEMA_VERSION = + "PRAGMA schema_version;"; + +const char * const SQL_SITEID_GETSET_ROWID_BY_SITEID = + "INSERT INTO cloudsync_site_id (site_id) VALUES (?) " + "ON CONFLICT(site_id) DO UPDATE SET site_id = site_id " + "RETURNING rowid;"; + +// Format +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_ROWID = + "WITH col_names AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols " + "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid" + ") " + "SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE rowid=?;'"; + +const char * const SQL_BUILD_SELECT_NONPK_COLS_BY_PK = + "WITH col_names AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') AS cols " + "FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid" + "), " + "pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'SELECT ' || (SELECT cols FROM col_names) || ' FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_DELETE_ROW_BY_ROWID = + "DELETE FROM \"%w\" WHERE rowid=?;"; + +const char * const SQL_BUILD_DELETE_ROW_BY_PK = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'DELETE FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_INSERT_ROWID_IGNORE = + "INSERT OR IGNORE INTO \"%w\" (rowid) VALUES (?);"; + +const char * const SQL_UPSERT_ROWID_AND_COL_BY_ROWID = + "INSERT INTO \"%w\" (rowid, \"%w\") VALUES (?, ?) ON CONFLICT DO UPDATE SET \"%w\"=?;"; + +const char * const SQL_BUILD_INSERT_PK_IGNORE = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + "), " + "pk_bind AS (" + "SELECT group_concat('?') AS pk_binding " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'INSERT OR IGNORE INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ') VALUES (' || (SELECT pk_binding FROM pk_bind) || ');'"; + +const char * const SQL_BUILD_UPSERT_PK_AND_COL = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"') AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + "), " + "pk_bind AS (" + "SELECT group_concat('?') AS pk_binding " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'INSERT INTO \"%w\" (' || (SELECT pk_clause FROM pk_where) || ',\"%w\") VALUES (' || (SELECT pk_binding FROM pk_bind) || ',?) ON CONFLICT DO UPDATE SET \"%w\"=?;'"; + +const char * const SQL_SELECT_COLS_BY_ROWID_FMT = + "SELECT %s%w%s FROM \"%w\" WHERE rowid=?;"; + +const char * const SQL_BUILD_SELECT_COLS_BY_PK_FMT = + "WITH pk_where AS (" + "SELECT group_concat('\"' || format('%%w', name) || '\"', '=? AND ') || '=?' AS pk_clause " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk" + ") " + "SELECT 'SELECT %s%w%s FROM \"%w\" WHERE ' || (SELECT pk_clause FROM pk_where) || ';'"; + +const char * const SQL_CLOUDSYNC_ROW_EXISTS_BY_PK = + "SELECT EXISTS(SELECT 1 FROM \"%w\" WHERE pk = ? LIMIT 1);"; + +const char * const SQL_CLOUDSYNC_UPDATE_COL_BUMP_VERSION = + "UPDATE \"%w\" " + "SET col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = ?, seq = ?, site_id = 0 " + "WHERE pk = ? AND col_name = '%s';"; + +const char * const SQL_CLOUDSYNC_UPSERT_COL_INIT_OR_BUMP_VERSION = + "INSERT INTO \"%w\" (pk, col_name, col_version, db_version, seq, site_id) " + "SELECT ?, '%s', 1, ?, ?, 0 " + "WHERE 1 " + "ON CONFLICT DO UPDATE SET " + "col_version = CASE col_version %% 2 WHEN 0 THEN col_version + 1 ELSE col_version + 2 END, " + "db_version = ?, seq = ?, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_UPSERT_RAW_COLVERSION = + "INSERT INTO \"%w\" (pk, col_name, col_version, db_version, seq, site_id ) " + "SELECT ?, ?, ?, ?, ?, 0 " + "WHERE 1 " + "ON CONFLICT DO UPDATE SET " + "col_version = \"%w\".col_version + 1, db_version = ?, seq = ?, site_id = 0;"; + +const char * const SQL_CLOUDSYNC_DELETE_PK_EXCEPT_COL = + "DELETE FROM \"%w\" WHERE pk=? AND col_name!='%s';"; + +const char * const SQL_CLOUDSYNC_REKEY_PK_AND_RESET_VERSION_EXCEPT_COL = + "UPDATE OR REPLACE \"%w\" " + "SET pk=?, db_version=?, col_version=1, seq=cloudsync_seq(), site_id=0 " + "WHERE (pk=? AND col_name!='%s');"; + +const char * const SQL_CLOUDSYNC_GET_COL_VERSION_OR_ROW_EXISTS = + "SELECT COALESCE(" + "(SELECT col_version FROM \"%w\" WHERE pk=? AND col_name='%s'), " + "(SELECT 1 FROM \"%w\" WHERE pk=?)" + ");"; + +const char * const SQL_CLOUDSYNC_INSERT_RETURN_CHANGE_ID = + "INSERT OR REPLACE INTO \"%w\" " + "(pk, col_name, col_version, db_version, seq, site_id) " + "VALUES (?, ?, ?, cloudsync_db_version_next(?), ?, ?) " + "RETURNING ((db_version << 30) | seq);"; + +const char * const SQL_CLOUDSYNC_TOMBSTONE_PK_EXCEPT_COL = + "UPDATE \"%w\" " + "SET col_version = 0, db_version = cloudsync_db_version_next(?) " + "WHERE pk=? AND col_name!='%s';"; + +const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL = + "SELECT col_version FROM \"%w\" WHERE pk=? AND col_name=?;"; + +const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL = + "SELECT site_id FROM \"%w\" WHERE pk=? AND col_name=?;"; + +const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID = + "SELECT name, cid FROM pragma_table_info('%q') WHERE pk=0 ORDER BY cid;"; + +const char * const SQL_DROP_CLOUDSYNC_TABLE = + "DROP TABLE IF EXISTS \"%w\";"; + +const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE = + "DELETE FROM \"%w\";"; + +const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL = + "DELETE FROM \"%w\" WHERE \"col_name\" NOT IN (" + "SELECT name FROM pragma_table_info('%q') UNION SELECT '%s'" + ")"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT = + "SELECT group_concat('\"%w\".\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%s') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK = + "DELETE FROM \"%w\" " + "WHERE (\"col_name\" != '%s' OR (\"col_name\" = '%s' AND col_version %% 2 != 0)) " + "AND NOT EXISTS (" + "SELECT 1 FROM \"%w\" " + "WHERE \"%w\".pk = cloudsync_pk_encode(%s) LIMIT 1" + ");"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_COLLIST = + "SELECT group_concat('\"' || format('%%w', name) || '\"', ',') " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_PRAGMA_TABLEINFO_PK_DECODE_SELECTLIST = + "SELECT group_concat(" + "'cloudsync_pk_decode(pk, ' || pk || ') AS ' || '\"' || format('%%w', name) || '\"', ','" + ") " + "FROM pragma_table_info('%q') WHERE pk>0 ORDER BY pk;"; + +const char * const SQL_CLOUDSYNC_INSERT_MISSING_PKS_FROM_BASE_EXCEPT_SYNC = + "SELECT cloudsync_insert('%q', %s) " + "FROM (SELECT %s FROM \"%w\" EXCEPT SELECT %s FROM \"%w\");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\") " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM \"%w\" _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?" + ");"; + +const char * const SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED = + "WITH _cstemp1 AS (SELECT cloudsync_pk_encode(%s) AS pk FROM \"%w\" WHERE (%s)) " + "SELECT _cstemp1.pk FROM _cstemp1 " + "WHERE NOT EXISTS (" + "SELECT 1 FROM \"%w\" _cstemp2 " + "WHERE _cstemp2.pk = _cstemp1.pk AND _cstemp2.col_name = ?" + ");"; + +const char * const SQL_CHANGES_INSERT_ROW = + "INSERT INTO cloudsync_changes(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) " + "VALUES (?,?,?,?,?,?,?,?,?);"; + +// MARK: Blocks (block-level LWW) + +const char * const SQL_BLOCKS_CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS %s (" + "pk BLOB NOT NULL, " + "col_name TEXT NOT NULL, " + "col_value BLOB, " + "PRIMARY KEY (pk, col_name)) WITHOUT ROWID"; + +const char * const SQL_BLOCKS_UPSERT = + "INSERT OR REPLACE INTO %s (pk, col_name, col_value) VALUES (?1, ?2, ?3)"; + +const char * const SQL_BLOCKS_SELECT = + "SELECT col_value FROM %s WHERE pk = ?1 AND col_name = ?2"; + +const char * const SQL_BLOCKS_DELETE = + "DELETE FROM %s WHERE pk = ?1 AND col_name = ?2"; + +const char * const SQL_BLOCKS_LIST_ALIVE = + "SELECT b.col_value FROM %s b " + "JOIN %s m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.pk = ?1 AND b.col_name LIKE ?2 " + "AND m.pk = ?3 AND m.col_name LIKE ?4 AND m.col_version %% 2 = 1 " + "ORDER BY b.col_name"; + +const char * const SQL_BLOCKS_INSERT_IGNORE = + "INSERT OR IGNORE INTO %s (pk, col_name, col_value) VALUES (?1, ?2, ?3)"; + +const char * const SQL_META_SCAN_COL_FOR_MIGRATION = + "SELECT DISTINCT m.pk FROM %s m " + "WHERE m.col_name = ?1 AND m.col_version %% 2 = 1 " + "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE ?2)"; + +const char * const SQL_META_INSERT_BLOCK_IGNORE = + "INSERT OR IGNORE INTO %s (pk, col_name, col_version, db_version, seq, site_id) " + "VALUES (?1, ?2, ?3, ?4, ?5, 0)"; diff --git a/src/sqlite3ext.h b/src/sqlite/sqlite3ext.h similarity index 100% rename from src/sqlite3ext.h rename to src/sqlite/sqlite3ext.h diff --git a/src/utils.c b/src/utils.c index b5e0de7..fff6cdd 100644 --- a/src/utils.c +++ b/src/utils.c @@ -18,7 +18,7 @@ #define file_close _close #else #include -#if defined(__APPLE__) +#if defined(__APPLE__) && !defined(CLOUDSYNC_POSTGRESQL_BUILD) #include #elif !defined(__ANDROID__) #include @@ -33,15 +33,11 @@ #include #endif -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif - #define FNV_OFFSET_BASIS 0xcbf29ce484222325ULL #define FNV_PRIME 0x100000001b3ULL #define HASH_CHAR(_c) do { h ^= (uint8_t)(_c); h *= FNV_PRIME; h_final = h;} while (0) -// MARK: UUIDv7 - +// MARK: - UUIDv7 - /* UUIDv7 is a 128-bit unique identifier like it's older siblings, such as the widely used UUIDv4. @@ -61,9 +57,12 @@ int cloudsync_uuid_v7 (uint8_t value[UUID_LEN]) { // fill the buffer with high-quality random data #ifdef _WIN32 if (BCryptGenRandom(NULL, (BYTE*)value, UUID_LEN, BCRYPT_USE_SYSTEM_PREFERRED_RNG) != STATUS_SUCCESS) return -1; - #elif defined(__APPLE__) + #elif defined(__APPLE__) && !defined(CLOUDSYNC_POSTGRESQL_BUILD) // Use SecRandomCopyBytes for macOS/iOS if (SecRandomCopyBytes(kSecRandomDefault, UUID_LEN, value) != errSecSuccess) return -1; + #elif defined(__APPLE__) && defined(CLOUDSYNC_POSTGRESQL_BUILD) + // PostgreSQL build: use getentropy to avoid Security.framework type conflicts + if (getentropy(value, UUID_LEN) != 0) return -1; #elif defined(__ANDROID__) //arc4random_buf doesn't have a return value to check for success arc4random_buf(value, UUID_LEN); @@ -113,15 +112,17 @@ char *cloudsync_uuid_v7_stringify (uint8_t uuid[UUID_LEN], char value[UUID_STR_M char *cloudsync_uuid_v7_string (char value[UUID_STR_MAXLEN], bool dash_format) { uint8_t uuid[UUID_LEN]; - if (cloudsync_uuid_v7(uuid) != 0) return NULL; + if (cloudsync_uuid_v7(uuid) != 0) return NULL; return cloudsync_uuid_v7_stringify(uuid, value, dash_format); } int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN]) { // reconstruct the timestamp by reversing the bit shifts and combining the bytes - uint64_t t1 = ((uint64_t)value1[0] << 40) | ((uint64_t)value1[1] << 32) | ((uint64_t)value1[2] << 24) | ((uint64_t)value1[3] << 16) | ((uint64_t)value1[4] << 8) | ((uint64_t)value1[5]); - uint64_t t2 = ((uint64_t)value2[0] << 40) | ((uint64_t)value2[1] << 32) | ((uint64_t)value2[2] << 24) | ((uint64_t)value2[3] << 16) | ((uint64_t)value2[4] << 8) | ((uint64_t)value2[5]); + uint64_t t1 = ((uint64_t)value1[0] << 40) | ((uint64_t)value1[1] << 32) | ((uint64_t)value1[2] << 24) | + ((uint64_t)value1[3] << 16) | ((uint64_t)value1[4] << 8) | ((uint64_t)value1[5]); + uint64_t t2 = ((uint64_t)value2[0] << 40) | ((uint64_t)value2[1] << 32) | ((uint64_t)value2[2] << 24) | + ((uint64_t)value2[3] << 16) | ((uint64_t)value2[4] << 8) | ((uint64_t)value2[5]); if (t1 == t2) return memcmp(value1, value2, UUID_LEN); return (t1 > t2) ? 1 : -1; @@ -129,24 +130,16 @@ int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN // MARK: - General - -void *cloudsync_memory_zeroalloc (uint64_t size) { - void *ptr = (void *)cloudsync_memory_alloc((sqlite3_uint64)size); - if (!ptr) return NULL; - - memset(ptr, 0, (size_t)size); - return ptr; -} - -char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase) { +char *cloudsync_string_ndup_v2 (const char *str, size_t len, bool lowercase) { if (str == NULL) return NULL; - char *s = (char *)cloudsync_memory_alloc((sqlite3_uint64)(len + 1)); + char *s = (char *)cloudsync_memory_alloc((uint64_t)(len + 1)); if (!s) return NULL; if (lowercase) { // convert each character to lowercase and copy it to the new string for (size_t i = 0; i < len; i++) { - s[i] = tolower(str[i]); + s[i] = (char)tolower(str[i]); } } else { memcpy(s, str, len); @@ -158,35 +151,42 @@ char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase) { return s; } -char *cloudsync_string_dup (const char *str, bool lowercase) { - if (str == NULL) return NULL; - - size_t len = strlen(str); - return cloudsync_string_ndup(str, len, lowercase); +char *cloudsync_string_ndup (const char *str, size_t len) { + return cloudsync_string_ndup_v2(str, len, false); +} + +char *cloudsync_string_ndup_lowercase (const char *str, size_t len) { + return cloudsync_string_ndup_v2(str, len, true); +} + +char *cloudsync_string_dup (const char *str) { + return cloudsync_string_ndup_v2(str, (str) ? strlen(str) : 0, false); +} + +char *cloudsync_string_dup_lowercase (const char *str) { + return cloudsync_string_ndup_v2(str, (str) ? strlen(str) : 0, true); } int cloudsync_blob_compare(const char *blob1, size_t size1, const char *blob2, size_t size2) { - if (size1 != size2) { - return (int)(size1 - size2); // Blobs are different if sizes are different - } - return memcmp(blob1, blob2, size1); // Use memcmp for byte-by-byte comparison + if (size1 != size2) return (size1 > size2) ? 1 : -1; // blobs are different if sizes are different + return memcmp(blob1, blob2, size1); // use memcmp for byte-by-byte comparison } -void cloudsync_rowid_decode (sqlite3_int64 rowid, sqlite3_int64 *db_version, sqlite3_int64 *seq) { +void cloudsync_rowid_decode (int64_t rowid, int64_t *db_version, int64_t *seq) { // use unsigned 64-bit integer for intermediate calculations // when db_version is large enough, it can cause overflow, leading to negative values // to handle this correctly, we need to ensure the calculations are done in an unsigned 64-bit integer context - // before converting back to sqlite3_int64 as needed + // before converting back to int64_t as needed uint64_t urowid = (uint64_t)rowid; // define the bit mask for seq (30 bits) const uint64_t SEQ_MASK = 0x3FFFFFFF; // (2^30 - 1) // extract seq by masking the lower 30 bits - *seq = (sqlite3_int64)(urowid & SEQ_MASK); + *seq = (int64_t)(urowid & SEQ_MASK); // extract db_version by shifting 30 bits to the right - *db_version = (sqlite3_int64)(urowid >> 30); + *db_version = (int64_t)(urowid >> 30); } char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement) { @@ -196,13 +196,13 @@ char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *rep size_t replacement_len = strlen(replacement); if (strncmp(input, prefix, prefix_len) == 0) { - // Allocate memory for new string + // allocate memory for new string size_t input_len = strlen(input); size_t new_len = input_len - prefix_len + replacement_len; char *result = cloudsync_memory_alloc(new_len + 1); // +1 for null terminator if (!result) return NULL; - // Copy replacement and the rest of the input string + // copy replacement and the rest of the input string strcpy(result, replacement); strcpy(result + replacement_len, input + prefix_len); return result; @@ -213,7 +213,7 @@ char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *rep } /* - Compute a normalized hash of a SQLite CREATE TABLE statement. + Compute a normalized hash of a CREATE TABLE statement. * Normalization: * - Skips comments (-- and / * ) @@ -322,7 +322,7 @@ static bool cloudsync_file_read_all (int fd, char *buf, size_t n) { return true; } -char *cloudsync_file_read (const char *path, sqlite3_int64 *len) { +char *cloudsync_file_read (const char *path, int64_t *len) { int fd = -1; char *buffer = NULL; @@ -409,31 +409,6 @@ bool cloudsync_file_write (const char *path, const char *buffer, size_t len) { #endif -// MARK: - CRDT algos - - -table_algo crdt_algo_from_name (const char *algo_name) { - if (algo_name == NULL) return table_algo_none; - - if ((strcasecmp(algo_name, "CausalLengthSet") == 0) || (strcasecmp(algo_name, "cls") == 0)) return table_algo_crdt_cls; - if ((strcasecmp(algo_name, "GrowOnlySet") == 0) || (strcasecmp(algo_name, "gos") == 0)) return table_algo_crdt_gos; - if ((strcasecmp(algo_name, "DeleteWinsSet") == 0) || (strcasecmp(algo_name, "dws") == 0)) return table_algo_crdt_dws; - if ((strcasecmp(algo_name, "AddWinsSet") == 0) || (strcasecmp(algo_name, "aws") == 0)) return table_algo_crdt_aws; - - // if nothing is found - return table_algo_none; -} - -const char *crdt_algo_name (table_algo algo) { - switch (algo) { - case table_algo_crdt_cls: return "cls"; - case table_algo_crdt_gos: return "gos"; - case table_algo_crdt_dws: return "dws"; - case table_algo_crdt_aws: return "aws"; - case table_algo_none: return NULL; - } - return NULL; -} - // MARK: - Memory Debugger - #if CLOUDSYNC_DEBUG_MEMORY @@ -620,10 +595,10 @@ void memdebug_finalize (void) { } } -void *memdebug_alloc (sqlite3_uint64 size) { - void *ptr = sqlite3_malloc64(size); +void *memdebug_alloc (uint64_t size) { + void *ptr = dbmem_alloc(size); if (!ptr) { - BUILD_ERROR("Unable to allocated a block of %lld bytes", size); + BUILD_ERROR("Unable to allocated a block of %" PRIu64" bytes", size); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, NULL); return NULL; @@ -632,7 +607,15 @@ void *memdebug_alloc (sqlite3_uint64 size) { return ptr; } -void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { +void *memdebug_zeroalloc (uint64_t size) { + void *ptr = memdebug_alloc(size); + if (!ptr) return NULL; + + memset(ptr, 0, (size_t)size); + return ptr; +} + +void *memdebug_realloc (void *ptr, uint64_t new_size) { if (!ptr) return memdebug_alloc(new_size); mem_slot *slot = _ptr_lookup(ptr); @@ -644,9 +627,9 @@ void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { } void *back_ptr = ptr; - void *new_ptr = sqlite3_realloc64(ptr, new_size); + void *new_ptr = dbmem_realloc(ptr, new_size); if (!new_ptr) { - BUILD_ERROR("Unable to reallocate a block of %lld bytes.", new_size); + BUILD_ERROR("Unable to reallocate a block of %" PRIu64 " bytes.", new_size); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, slot); return NULL; @@ -657,15 +640,15 @@ void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size) { } char *memdebug_vmprintf (const char *format, va_list list) { - char *ptr = sqlite3_vmprintf(format, list); + char *ptr = dbmem_vmprintf(format, list); if (!ptr) { - BUILD_ERROR("Unable to allocated for sqlite3_vmprintf with format %s", format); + BUILD_ERROR("Unable to allocated for dbmem_vmprintf with format %s", format); BUILD_STACK(n, stack); memdebug_report(current_error, stack, n, NULL); return NULL; } - _ptr_add(ptr, sqlite3_msize(ptr)); + _ptr_add(ptr, dbmem_size(ptr)); return ptr; } @@ -680,8 +663,8 @@ char *memdebug_mprintf(const char *format, ...) { return z; } -sqlite3_uint64 memdebug_msize (void *ptr) { - return sqlite3_msize(ptr); +uint64_t memdebug_msize (void *ptr) { + return dbmem_size(ptr); } void memdebug_free (void *ptr) { @@ -709,7 +692,7 @@ void memdebug_free (void *ptr) { } _ptr_remove(ptr); - sqlite3_free(ptr); + dbmem_free(ptr); } #endif diff --git a/src/utils.h b/src/utils.h index d526d86..3f0e098 100644 --- a/src/utils.h +++ b/src/utils.h @@ -14,6 +14,7 @@ #include #include #include +#include "database.h" // CLOUDSYNC_DESKTOP_OS = 1 if compiling for macOS, Linux (desktop), or Windows // Not set for iOS, Android, WebAssembly, or other platforms @@ -28,12 +29,6 @@ #define CLOUDSYNC_DESKTOP_OS 1 #endif -#ifndef SQLITE_CORE -#include "sqlite3ext.h" -#else -#include "sqlite3.h" -#endif - #define CLOUDSYNC_DEBUG_FUNCTIONS 0 #define CLOUDSYNC_DEBUG_DBFUNCTIONS 0 #define CLOUDSYNC_DEBUG_SETTINGS 0 @@ -43,49 +38,60 @@ #define CLOUDSYNC_DEBUG_STMT 0 #define CLOUDSYNC_DEBUG_MERGE 0 -#define DEBUG_RUNTIME(...) do {if (data->debug) printf(__VA_ARGS__ );} while (0) -#define DEBUG_PRINTLN(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) -#define DEBUG_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) -#define DEBUG_PRINT(...) do {printf(__VA_ARGS__ );} while (0) +// Debug macros - platform-specific logging +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + // PostgreSQL build - use elog() for logging + #include "postgresql/postgresql_log.h" + #define DEBUG_RUNTIME(...) do {if (data->debug) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__ );} while (0) + #define DEBUG_PRINTLN(...) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__) + #define DEBUG_ALWAYS(...) CLOUDSYNC_LOG_INFO(__VA_ARGS__) + #define DEBUG_PRINT(...) CLOUDSYNC_LOG_DEBUG(__VA_ARGS__) +#else + // SQLite and other platforms use printf() + #define DEBUG_RUNTIME(...) do {if (data->debug) printf(__VA_ARGS__ );} while (0) + #define DEBUG_PRINTLN(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) + #define DEBUG_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) + #define DEBUG_PRINT(...) do {printf(__VA_ARGS__ );} while (0) +#endif #if CLOUDSYNC_DEBUG_FUNCTIONS -#define DEBUG_FUNCTION(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_FUNCTION(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_FUNCTION(...) #endif -#if CLOUDSYNC_DEBUG_DBFUNCTION -#define DEBUG_DBFUNCTION(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#if CLOUDSYNC_DEBUG_DBFUNCTIONS +#define DEBUG_DBFUNCTION(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_DBFUNCTION(...) #endif #if CLOUDSYNC_DEBUG_SETTINGS -#define DEBUG_SETTINGS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_SETTINGS(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_SETTINGS(...) #endif #if CLOUDSYNC_DEBUG_SQL -#define DEBUG_SQL(...) do {printf(__VA_ARGS__ );printf("\n\n");} while (0) +#define DEBUG_SQL(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_SQL(...) #endif #if CLOUDSYNC_DEBUG_VTAB -#define DEBUG_VTAB(...) do {printf(__VA_ARGS__ );printf("\n\n");} while (0) +#define DEBUG_VTAB(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_VTAB(...) #endif #if CLOUDSYNC_DEBUG_STMT -#define DEBUG_STMT(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_STMT(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_STMT(...) #endif #if CLOUDSYNC_DEBUG_MERGE -#define DEBUG_MERGE(...) do {printf(__VA_ARGS__ );printf("\n");} while (0) +#define DEBUG_MERGE(...) DEBUG_PRINTLN(__VA_ARGS__) #else #define DEBUG_MERGE(...) #endif @@ -94,64 +100,55 @@ #define cloudsync_memory_init(_once) memdebug_init(_once) #define cloudsync_memory_finalize memdebug_finalize #define cloudsync_memory_alloc memdebug_alloc +#define cloudsync_memory_zeroalloc memdebug_zeroalloc #define cloudsync_memory_free memdebug_free #define cloudsync_memory_realloc memdebug_realloc #define cloudsync_memory_size memdebug_msize -#define cloudsync_memory_vmprintf memdebug_vmprintf #define cloudsync_memory_mprintf memdebug_mprintf void memdebug_init (int once); void memdebug_finalize (void); -void *memdebug_alloc (sqlite3_uint64 size); -void *memdebug_realloc (void *ptr, sqlite3_uint64 new_size); +void *memdebug_alloc (uint64_t size); +void *memdebug_zeroalloc (uint64_t size); +void *memdebug_realloc (void *ptr, uint64_t new_size); char *memdebug_vmprintf (const char *format, va_list list); char *memdebug_mprintf(const char *format, ...); void memdebug_free (void *ptr); -sqlite3_uint64 memdebug_msize (void *ptr); +uint64_t memdebug_msize (void *ptr); #else #define cloudsync_memory_init(_once) #define cloudsync_memory_finalize() -#define cloudsync_memory_alloc sqlite3_malloc64 -#define cloudsync_memory_free sqlite3_free -#define cloudsync_memory_realloc sqlite3_realloc64 -#define cloudsync_memory_size sqlite3_msize -#define cloudsync_memory_vmprintf sqlite3_vmprintf -#define cloudsync_memory_mprintf sqlite3_mprintf +#define cloudsync_memory_alloc dbmem_alloc +#define cloudsync_memory_zeroalloc dbmem_zeroalloc +#define cloudsync_memory_free dbmem_free +#define cloudsync_memory_realloc dbmem_realloc +#define cloudsync_memory_size dbmem_size +#define cloudsync_memory_mprintf dbmem_mprintf #endif #define UUID_STR_MAXLEN 37 #define UUID_LEN 16 -// The type of CRDT chosen for a table controls what rows are included or excluded when merging tables together from different databases -typedef enum { - table_algo_none = 0, - table_algo_crdt_cls = 100, // CausalLengthSet - table_algo_crdt_gos, // GrowOnlySet - table_algo_crdt_dws, // DeleteWinsSet - table_algo_crdt_aws // AddWinsSet -} table_algo; - -table_algo crdt_algo_from_name (const char *name); -const char *crdt_algo_name (table_algo algo); - int cloudsync_uuid_v7 (uint8_t value[UUID_LEN]); int cloudsync_uuid_v7_compare (uint8_t value1[UUID_LEN], uint8_t value2[UUID_LEN]); char *cloudsync_uuid_v7_string (char value[UUID_STR_MAXLEN], bool dash_format); char *cloudsync_uuid_v7_stringify (uint8_t uuid[UUID_LEN], char value[UUID_STR_MAXLEN], bool dash_format); -char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement); uint64_t fnv1a_hash(const char *data, size_t len); -void *cloudsync_memory_zeroalloc (uint64_t size); -char *cloudsync_string_ndup (const char *str, size_t len, bool lowercase); -char *cloudsync_string_dup (const char *str, bool lowercase); +char *cloudsync_string_replace_prefix(const char *input, char *prefix, char *replacement); +char *cloudsync_string_dup (const char *str); +char *cloudsync_string_dup_lowercase (const char *str); +char *cloudsync_string_ndup (const char *str, size_t len); +char *cloudsync_string_ndup_lowercase (const char *str, size_t len); + int cloudsync_blob_compare(const char *blob1, size_t size1, const char *blob2, size_t size2); -void cloudsync_rowid_decode (sqlite3_int64 rowid, sqlite3_int64 *db_version, sqlite3_int64 *seq); +void cloudsync_rowid_decode (int64_t rowid, int64_t *db_version, int64_t *seq); -// available only on Desktop OS +// available only on Desktop OS (no WASM, no mobile) #ifdef CLOUDSYNC_DESKTOP_OS bool cloudsync_file_delete (const char *path); -char *cloudsync_file_read (const char *path, sqlite3_int64 *len); +char *cloudsync_file_read (const char *path, int64_t *len); bool cloudsync_file_write (const char *path, const char *buffer, size_t len); #endif diff --git a/src/vtab.h b/src/vtab.h deleted file mode 100644 index 0c9bd64..0000000 --- a/src/vtab.h +++ /dev/null @@ -1,18 +0,0 @@ -// -// vtab.h -// cloudsync -// -// Created by Marco Bambini on 23/09/24. -// - -#ifndef __CLOUDSYNC_VTAB__ -#define __CLOUDSYNC_VTAB__ - -#include "cloudsync.h" -#include "cloudsync_private.h" - -int cloudsync_vtab_register_changes (sqlite3 *db, cloudsync_context *xdata); -cloudsync_context *cloudsync_vtab_get_context (sqlite3_vtab *vtab); -int cloudsync_vtab_set_error (sqlite3_vtab *vtab, const char *format, ...); - -#endif diff --git a/test/main.c b/test/integration.c similarity index 62% rename from test/main.c rename to test/integration.c index a540c9f..6966fde 100644 --- a/test/main.c +++ b/test/integration.c @@ -1,9 +1,9 @@ // -// main.c +// integration.c // cloudsync // // Created by Gioele Cantoni on 05/06/25. -// Set CONNECTION_STRING, APIKEY and WEBLITE environment variables before running this test. +// Set INTEGRATION_TEST_OFFLINE_DATABASE_ID and INTEGRATION_TEST_DATABASE_ID environment variables before running this test. // #include @@ -31,6 +31,7 @@ #ifdef CLOUDSYNC_LOAD_FROM_SOURCES #include "cloudsync.h" +#include "cloudsync_sqlite.h" #endif #define DB_PATH "health-track.sqlite" @@ -40,7 +41,7 @@ #define TERMINATE if (db) { db_exec(db, "SELECT cloudsync_terminate();"); } #define ABORT_TEST abort_test: ERROR_MSG TERMINATE if (db) sqlite3_close(db); return rc; -typedef enum { PRINT, NOPRINT, INTGR, GT0 } expected_type; +typedef enum { PRINT, NOPRINT, INTGR, GT0, STR } expected_type; typedef struct { expected_type type; @@ -86,6 +87,15 @@ static int callback(void *data, int argc, char **argv, char **names) { } else goto multiple_columns; break; + case STR: + if(argc == 1){ + if(!argv[0] || strcmp(argv[0], expect->value.s) != 0){ + printf("Error: expected from %s: \"%s\", got \"%s\"\n", names[0], expect->value.s, argv[0] ? argv[0] : "NULL"); + return SQLITE_ERROR; + } + } else goto multiple_columns; + break; + default: printf("Error: unknown expect type\n"); return SQLITE_ERROR; @@ -135,6 +145,16 @@ int db_expect_gt0 (sqlite3 *db, const char *sql) { return rc; } +int db_expect_str (sqlite3 *db, const char *sql, const char *expect) { + expected_t data; + data.type = STR; + data.value.s = expect; + + int rc = sqlite3_exec(db, sql, callback, &data, NULL); + if (rc != SQLITE_OK) printf("Error while executing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; +} + int open_load_ext(const char *db_path, sqlite3 **out_db) { sqlite3 *db = NULL; int rc = sqlite3_open(db_path, &db); @@ -204,17 +224,31 @@ int test_init (const char *db_path, int init) { rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK - // init network with connection string + apikey - char network_init[512]; - const char* conn_str = getenv("CONNECTION_STRING"); - const char* apikey = getenv("APIKEY"); - if (!conn_str || !apikey) { - fprintf(stderr, "Error: CONNECTION_STRING or APIKEY not set.\n"); + // init network + char network_init[1024]; + const char* test_db_id = getenv("INTEGRATION_TEST_DATABASE_ID"); + if (!test_db_id) { + fprintf(stderr, "Error: INTEGRATION_TEST_DATABASE_ID not set.\n"); exit(1); } - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s?apikey=%s');", conn_str, apikey); + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", test_db_id); + } rc = db_exec(db, network_init); RCHECK + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + rc = db_expect_int(db, "SELECT COUNT(*) as count FROM activities;", 0); RCHECK rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK char value[UUID_STR_MAXLEN]; @@ -223,7 +257,7 @@ int test_init (const char *db_path, int init) { snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); rc = db_exec(db, sql); RCHECK rc = db_expect_int(db, "SELECT COUNT(*) as count FROM users;", 1); RCHECK - rc = db_expect_gt0(db, "SELECT cloudsync_network_sync(250,10);"); RCHECK + rc = db_expect_gt0(db, "SELECT cloudsync_network_sync(250,10) ->> '$.receive.rows';"); RCHECK rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM users;"); RCHECK rc = db_expect_gt0(db, "SELECT COUNT(*) as count FROM activities;"); RCHECK rc = db_expect_int(db, "SELECT COUNT(*) as count FROM workouts;", 0); RCHECK @@ -261,30 +295,48 @@ int test_enable_disable(const char *db_path) { cloudsync_uuid_v7_string(value, true); char sql[256]; - rc = db_exec(db, "SELECT cloudsync_init('*');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('workouts');"); RCHECK rc = db_exec(db, "SELECT cloudsync_disable('users');"); RCHECK snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s', '%s');", value, value); - rc = db_exec(db, sql); RCHECK + //rc = db_exec(db, sql); RCHECK rc = db_exec(db, "SELECT cloudsync_enable('users');"); RCHECK snprintf(sql, sizeof(sql), "INSERT INTO users (id, name) VALUES ('%s-should-sync', '%s-should-sync');", value, value); rc = db_exec(db, sql); RCHECK - // init network with connection string + apikey - char network_init[512]; - const char* conn_str = getenv("CONNECTION_STRING"); - const char* apikey = getenv("APIKEY"); - if (!conn_str || !apikey) { - fprintf(stderr, "Error: CONNECTION_STRING or APIKEY not set.\n"); + // init network + char network_init[1024]; + const char* test_db_id = getenv("INTEGRATION_TEST_DATABASE_ID"); + if (!test_db_id) { + fprintf(stderr, "Error: INTEGRATION_TEST_DATABASE_ID not set.\n"); exit(1); } - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s?apikey=%s');", conn_str, apikey); + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", test_db_id); + } rc = db_exec(db, network_init); RCHECK + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + rc = db_exec(db, "SELECT cloudsync_network_send_changes();"); RCHECK - rc = db_exec(db, "SELECT cloudsync_cleanup('*');"); + rc = db_exec(db, "SELECT cloudsync_cleanup('users');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('activities');"); RCHECK + rc = db_exec(db, "SELECT cloudsync_cleanup('workouts');"); RCHECK // give the server the time to apply the latest sent changes, it is an async job sqlite3_sleep(5000); @@ -293,12 +345,21 @@ int test_enable_disable(const char *db_path) { rc = open_load_ext(":memory:", &db2); RCHECK rc = db_init(db2); RCHECK - rc = db_exec(db2, "SELECT cloudsync_init('*');"); RCHECK + rc = db_exec(db2, "SELECT cloudsync_init('users');"); RCHECK + rc = db_exec(db2, "SELECT cloudsync_init('activities');"); RCHECK + rc = db_exec(db2, "SELECT cloudsync_init('workouts');"); RCHECK // init network with connection string + apikey rc = db_exec(db2, network_init); RCHECK - rc = db_expect_gt0(db2, "SELECT cloudsync_network_sync(250,10);"); RCHECK + if (apikey) { + char set_apikey2[512]; + snprintf(set_apikey2, sizeof(set_apikey2), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db2, set_apikey2); RCHECK + } + + rc = db_expect_gt0(db2, "SELECT cloudsync_network_sync(250,10) ->> '$.receive.rows';"); RCHECK snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM users WHERE name='%s';", value); rc = db_expect_int(db2, sql, 0); RCHECK @@ -327,19 +388,35 @@ int test_offline_error(const char *db_path) { rc = db_exec(db, "INSERT INTO test_table (id, value) VALUES (cloudsync_uuid(), 'test1'), (cloudsync_uuid(), 'test2');"); RCHECK - // Initialize network with offline connection string - const char* offline_conn_str = getenv("CONNECTION_STRING_OFFLINE_PROJECT"); - if (!offline_conn_str) { - printf("Skipping offline error test: CONNECTION_STRING_OFFLINE_PROJECT not set.\n"); + // Initialize network with offline database ID + const char* offline_db_id = getenv("INTEGRATION_TEST_OFFLINE_DATABASE_ID"); + if (!offline_db_id) { + printf("Skipping offline error test: INTEGRATION_TEST_OFFLINE_DATABASE_ID not set.\n"); rc = SQLITE_OK; goto abort_test; } char network_init[512]; - snprintf(network_init, sizeof(network_init), "SELECT cloudsync_network_init('%s');", offline_conn_str); + const char* custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (custom_address) { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, offline_db_id); + } else { + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init('%s');", offline_db_id); + } rc = db_exec(db, network_init); RCHECK + const char* apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); + RCHECK + } + // Try to sync - this should fail with the expected error char *errmsg = NULL; rc = sqlite3_exec(db, "SELECT cloudsync_network_sync();", NULL, NULL, &errmsg); @@ -350,17 +427,30 @@ int test_offline_error(const char *db_path) { goto abort_test; } - // Verify the error message contains the expected text - const char *expected_error = "cloudsync_network_send_changes unable to upload BLOB changes to remote host"; - if (!errmsg || strstr(errmsg, expected_error) == NULL) { - printf("Error: Expected error message containing '%s', but got '%s'\n", - expected_error, errmsg ? errmsg : "NULL"); - if (errmsg) sqlite3_free(errmsg); + // Verify the error JSON contains expected fields using SQLite JSON extraction + if (!errmsg) { + printf("Error: Expected an error message, but got NULL\n"); rc = SQLITE_ERROR; goto abort_test; } - if (errmsg) sqlite3_free(errmsg); + char verify_sql[1024]; + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].status');", errmsg); + rc = db_expect_str(db, verify_sql, "503"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected status in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].code');", errmsg); + rc = db_expect_str(db, verify_sql, "database_paused"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected code in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + snprintf(verify_sql, sizeof(verify_sql), + "SELECT json_extract('%s', '$.errors[0].title');", errmsg); + rc = db_expect_str(db, verify_sql, "Database paused"); + if (rc != SQLITE_OK) { printf("Offline error: unexpected title in: %s\n", errmsg); sqlite3_free(errmsg); goto abort_test; } + + sqlite3_free(errmsg); rc = SQLITE_OK; ABORT_TEST @@ -398,6 +488,84 @@ int test_double_empty_network_init(const char *db_path) { ABORT_TEST } +// Failure-path integration test. +// +// Targets a cloudsync database (INTEGRATION_TEST_FAILURE_DATABASE_ID) +// configured server-side to fail apply and check jobs. Verifies that the +// new failures.{apply,check} response shape is correctly parsed and emitted as +// send.lastFailure (cloudsync_network_send_changes) and receive.lastFailure +// (cloudsync_network_check_changes), and that cloudsync_network_sync surfaces +// at least one of them. +// +// First invocation primes the server (sends data, queues a check) — server-side +// async jobs may not have failed yet. After a sleep, the second invocation must +// see lastFailure populated. +int test_failure_path (const char *db_path) { + int rc = SQLITE_OK; + sqlite3 *db = NULL; + + const char *test_db_id = getenv("INTEGRATION_TEST_FAILURE_DATABASE_ID"); + if (!test_db_id) { + printf("(INTEGRATION_TEST_FAILURE_DATABASE_ID not set, skipping) "); + return SQLITE_OK; + } + const char *custom_address = getenv("INTEGRATION_TEST_CLOUDSYNC_ADDRESS"); + if (!custom_address) { + printf("(INTEGRATION_TEST_CLOUDSYNC_ADDRESS not set, skipping) "); + return SQLITE_OK; + } + + rc = open_load_ext(db_path, &db); RCHECK + + rc = db_exec(db, "CREATE TABLE IF NOT EXISTS failure_users (id TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL DEFAULT '', value BLOB);"); RCHECK + rc = db_exec(db, "SELECT cloudsync_init('failure_users');"); RCHECK + + char network_init[1024]; + snprintf(network_init, sizeof(network_init), + "SELECT cloudsync_network_init_custom('%s', '%s');", custom_address, test_db_id); + rc = db_exec(db, network_init); RCHECK + + const char *apikey = getenv("INTEGRATION_TEST_APIKEY"); + if (apikey) { + char set_apikey[512]; + snprintf(set_apikey, sizeof(set_apikey), + "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, set_apikey); RCHECK + } + + // Insert a row so cloudsync_network_send_changes has a payload to upload. + // Insert a 1MB value to skip the fast-lane and force using the normal s3 path with async job, + // otherwise the error would be immediately returned by the apply endpoint. + char value[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(value, true); + char sql[256]; + snprintf(sql, sizeof(sql), "INSERT INTO failure_users (id, name, value) VALUES ('%s', '%s', randomblob(1048576));", value, value); + rc = db_exec(db, sql); RCHECK + + // First invocation — primes the server. Failures may not yet be reported. + rc = db_exec(db, "SELECT cloudsync_network_send_changes();"); RCHECK + rc = db_exec(db, "SELECT cloudsync_network_check_changes();"); RCHECK + rc = db_exec(db, "SELECT cloudsync_network_sync(250, 1);"); RCHECK + + // Give the server time to process and fail the queued apply/check jobs. + sqlite3_sleep(5000); + + // Second invocation — failures must surface now. + // jobId is always > 0 when failure object is present, so ->> + GT0 doubles as + // an existence check (NULL → atoi returns 0 → fails GT0). + rc = db_expect_gt0(db, + "SELECT cloudsync_network_send_changes() ->> '$.send.lastFailure.jobId';"); RCHECK + rc = db_expect_gt0(db, + "SELECT cloudsync_network_check_changes() ->> '$.receive.lastFailure.jobId';"); RCHECK + // sync must surface at least one of the two; instr() catches either path. + rc = db_expect_gt0(db, + "SELECT instr(cloudsync_network_sync(250, 1), '\"lastFailure\":');"); RCHECK + + rc = db_exec(db, "SELECT cloudsync_terminate();"); + +ABORT_TEST +} + int version(void){ sqlite3 *db = NULL; int rc = open_load_ext(":memory:", &db); @@ -467,6 +635,7 @@ int main (void) { rc += test_report("Enable Disable Test:", test_enable_disable(DB_PATH)); rc += test_report("Offline Error Test:", test_offline_error(":memory:")); rc += test_report("Double Empty Init Test:", test_double_empty_network_init(":memory:")); + rc += test_report("Failure Path Test:", test_failure_path(":memory:")); remove(DB_PATH); // remove the database file @@ -562,4 +731,4 @@ int main (void) { printf("\n"); return rc; -} +} \ No newline at end of file diff --git a/test/postgresql/01_unittest.sql b/test/postgresql/01_unittest.sql new file mode 100644 index 0000000..ad14d72 --- /dev/null +++ b/test/postgresql/01_unittest.sql @@ -0,0 +1,322 @@ +-- 'Unittest' + +\set testid '01' + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_1; +CREATE DATABASE cloudsync_test_1; + +\connect cloudsync_test_1 +\ir helper_psql_conn_setup.sql + +-- Reset extension and install +-- DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- 'Test version visibility' +SELECT cloudsync_version() AS version \gset +SELECT current_setting('server_version') AS pg_version \gset +\echo [PASS] (:testid) Test cloudsync_version: :version (PostgreSQL :pg_version) + +-- Test uuid generation +SELECT cloudsync_uuid() AS uuid1 \gset +SELECT pg_sleep(0.1) \gset +SELECT cloudsync_uuid() AS uuid2 \gset + +-- Test 1: Format check (UUID v7 has standard format: xxxxxxxx-xxxx-7xxx-xxxx-xxxxxxxxxxxx) +SELECT (:'uuid1' ~ '^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') AS uuid_format_ok \gset +\if :uuid_format_ok +\echo [PASS] (:testid) UUID format valid (UUIDv7 pattern) +\else +\echo [FAIL] (:testid) UUID format invalid - Got: :uuid1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Uniqueness check +SELECT (:'uuid1' != :'uuid2') AS uuid_unique_ok \gset +\if :uuid_unique_ok +\echo [PASS] (:testid) UUID uniqueness (two calls generated different UUIDs) +\else +\echo [FAIL] (:testid) UUID uniqueness - Both calls returned: :uuid1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Monotonicity check (UUIDv7 should be sortable by timestamp) +SELECT (:'uuid1' < :'uuid2') AS uuid_monotonic_ok \gset +\if :uuid_monotonic_ok +\echo [PASS] (:testid) UUID monotonicity (UUIDs are time-ordered) +\else +\echo [FAIL] (:testid) UUID monotonicity - uuid1: :uuid1, uuid2: :uuid2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Type check (ensure it's actually UUID type, not text) +SELECT (pg_typeof(cloudsync_uuid())::text = 'uuid') AS uuid_type_ok \gset +\if :uuid_type_ok +\echo [PASS] (:testid) UUID type is correct (uuid, not text or bytea) +\else +\echo [FAIL] (:testid) UUID type incorrect - Got: (pg_typeof(cloudsync_uuid())::text) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test init on a simple table' +SELECT cloudsync_cleanup('smoke_tbl') AS _cleanup_ok \gset +SELECT (cloudsync_is_sync('smoke_tbl') = false) AS init_cleanup_ok \gset +\if :init_cleanup_ok +\echo [PASS] (:testid) Test init cleanup +\else +\echo [FAIL] (:testid) Test init cleanup +SELECT (:fail::int + 1) AS fail \gset +\endif +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id \gset +SELECT (to_regclass('public.smoke_tbl_cloudsync') IS NOT NULL) AS init_create_ok \gset +\if :init_create_ok +\echo [PASS] (:testid) Test init create +\else +\echo [FAIL] (:testid) Test init create +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert metadata row creation' +SELECT cloudsync_uuid() AS smoke_id \gset +INSERT INTO smoke_tbl (id, val) VALUES (:'smoke_id', 'hello'); +SELECT (COUNT(*) = 1) AS insert_meta_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :insert_meta_ok +\echo [PASS] (:testid) Test insert metadata row creation +\else +\echo [FAIL] (:testid) Test insert metadata row creation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert metadata fields' +SELECT (db_version > 0 AND seq >= 0) AS insert_meta_fields_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :insert_meta_fields_ok +\echo [PASS] (:testid) Test insert metadata fields +\else +\echo [FAIL] (:testid) Test insert metadata fields +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update val only' +SELECT col_version AS val_ver_before +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +UPDATE smoke_tbl SET val = 'hello2' WHERE id = :'smoke_id'; +SELECT col_version AS val_ver_after +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +SELECT (:val_ver_after::bigint > :val_ver_before::bigint) AS update_val_ok \gset +\if :update_val_ok +\echo [PASS] (:testid) Test update val only +\else +\echo [FAIL] (:testid) Test update val only +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update id only' +SELECT cloudsync_uuid() AS smoke_id2 \gset +UPDATE smoke_tbl SET id = :'smoke_id2' WHERE id = :'smoke_id'; +SELECT (COUNT(*) = 1) AS update_id_old_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_id_old_tombstone_ok +\echo [PASS] (:testid) Test update id only (old tombstone) +\else +\echo [FAIL] (:testid) Test update id only (old tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 0) AS update_id_old_val_gone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id']::text[]) + AND col_name = 'val' \gset +\if :update_id_old_val_gone_ok +\echo [PASS] (:testid) Test update id only (old val gone) +\else +\echo [FAIL] (:testid) Test update id only (old val gone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_id_new_val_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = 'val' \gset +\if :update_id_new_val_ok +\echo [PASS] (:testid) Test update id only (new val) +\else +\echo [FAIL] (:testid) Test update id only (new val) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_id_new_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_id_new_tombstone_ok +\echo [PASS] (:testid) Test update id only (new tombstone) +\else +\echo [FAIL] (:testid) Test update id only (new tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test update id and val' +SELECT cloudsync_uuid() AS smoke_id3 \gset +UPDATE smoke_tbl SET id = :'smoke_id3', val = 'hello3' WHERE id = :'smoke_id2'; +SELECT (COUNT(*) = 1) AS update_both_old_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_both_old_tombstone_ok +\echo [PASS] (:testid) Test update id and val (old tombstone) +\else +\echo [FAIL] (:testid) Test update id and val (old tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 0) AS update_both_old_val_gone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id2']::text[]) + AND col_name = 'val' \gset +\if :update_both_old_val_gone_ok +\echo [PASS] (:testid) Test update id and val (old val gone) +\else +\echo [FAIL] (:testid) Test update id and val (old val gone) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_both_new_val_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = 'val' \gset +\if :update_both_new_val_ok +\echo [PASS] (:testid) Test update id and val (new val) +\else +\echo [FAIL] (:testid) Test update id and val (new val) +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (COUNT(*) = 1) AS update_both_new_tombstone_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :update_both_new_tombstone_ok +\echo [PASS] (:testid) Test update id and val (new tombstone) +\else +\echo [FAIL] (:testid) Test update id and val (new tombstone) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete metadata tombstone' +DELETE FROM smoke_tbl WHERE id = :'smoke_id3'; +SELECT (COUNT(*) = 1) AS delete_meta_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :delete_meta_ok +\echo [PASS] (:testid) Test delete metadata tombstone +\else +\echo [FAIL] (:testid) Test delete metadata tombstone +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete metadata fields' +SELECT (db_version > 0 AND seq >= 0) AS delete_meta_fields_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name = '__[RIP]__' \gset +\if :delete_meta_fields_ok +\echo [PASS] (:testid) Test delete metadata fields +\else +\echo [FAIL] (:testid) Test delete metadata fields +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test delete removes non-tombstone metadata' +SELECT (COUNT(*) = 0) AS delete_meta_only_ok +FROM smoke_tbl_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id3']::text[]) + AND col_name != '__[RIP]__' \gset +\if :delete_meta_only_ok +\echo [PASS] (:testid) Test delete removes non-tombstone metadata +\else +\echo [FAIL] (:testid) Test delete removes non-tombstone metadata +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write' +SELECT cloudsync_uuid() AS smoke_id4 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'smoke_tbl', + cloudsync_pk_encode(VARIADIC ARRAY[:'smoke_id4']::text[]), + 'val', + -- "change_write" encoded as cloudsync text value (type 0x61 + len 0x0c) + decode('0b0c6368616e67655f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_row_ok +FROM smoke_tbl +WHERE id = :'smoke_id4' AND val = 'change_write' \gset +\if :changes_write_row_ok +\echo [PASS] (:testid) Test cloudsync_changes view write +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read' +SELECT COUNT(*) AS changes_view_count +FROM cloudsync_changes +WHERE tbl = 'smoke_tbl' \gset +SELECT COUNT(*) AS changes_meta_count +FROM smoke_tbl_cloudsync \gset +SELECT (:changes_view_count::int = :changes_meta_count::int) AS changes_read_ok \gset +\if :changes_read_ok +\echo [PASS] (:testid) Test cloudsync_changes view read +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test site id visibility' +SELECT cloudsync_siteid() AS site_id \gset +\echo [PASS] (:testid) Test site id visibility :site_id + +-- 'Test site id encoding' +SELECT (length(encode(cloudsync_siteid()::bytea, 'hex')) > 0) AS sid_ok \gset +\if :sid_ok +\echo [PASS] (:testid) Test site id encoding +\else +\echo [FAIL] (:testid) Test site id encoding +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test double init no-op' +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id2 \gset +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id3 \gset +\echo [PASS] (:testid) Test double init no-op + +-- 'Test payload encode signature' +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash +FROM smoke_tbl \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset +SELECT (length(:'payload_hex') > 0 AND substring(:'payload_hex' from 1 for 8) = '434c5359') AS payload_sig_ok \gset +\if :payload_sig_ok +\echo [PASS] (:testid) Test payload encode signature +\else +\echo [FAIL] (:testid) Test payload encode signature +SELECT (:fail::int + 1) AS fail \gset +\endif \ No newline at end of file diff --git a/test/postgresql/02_roundtrip.sql b/test/postgresql/02_roundtrip.sql new file mode 100644 index 0000000..5a7ecbf --- /dev/null +++ b/test/postgresql/02_roundtrip.sql @@ -0,0 +1,38 @@ +-- '2 db roundtrip test' + +\set testid '02' +\ir helper_test_init.sql + +\connect cloudsync_test_1 +\ir helper_psql_conn_setup.sql +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +DROP DATABASE IF EXISTS cloudsync_test_2; +CREATE DATABASE cloudsync_test_2; +\connect cloudsync_test_2 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply_ok \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset +SELECT (:'smoke_hash' = :'smoke_hash_b') AS payload_roundtrip_ok \gset +\if :payload_roundtrip_ok +\echo [PASS] (:testid) Test payload roundtrip to another database +\else +\echo [FAIL] (:testid) Test payload roundtrip to another database +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_1; +DROP DATABASE IF EXISTS cloudsync_test_2; +\else +\echo [INFO] !!!!! +\endif \ No newline at end of file diff --git a/test/postgresql/03_multiple_roundtrip.sql b/test/postgresql/03_multiple_roundtrip.sql new file mode 100644 index 0000000..18541d3 --- /dev/null +++ b/test/postgresql/03_multiple_roundtrip.sql @@ -0,0 +1,302 @@ +-- 'Test multi-db roundtrip with concurrent updates' + +\set testid '03' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_03_a; +DROP DATABASE IF EXISTS cloudsync_test_03_b; +DROP DATABASE IF EXISTS cloudsync_test_03_c; +CREATE DATABASE cloudsync_test_03_a; +CREATE DATABASE cloudsync_test_03_b; +CREATE DATABASE cloudsync_test_03_c; + +\connect cloudsync_test_03_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_03_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_03_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: independent inserts on each database +\connect cloudsync_test_03_a +INSERT INTO smoke_tbl VALUES ('id1', 'a1'); +INSERT INTO smoke_tbl VALUES ('id2', 'a2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +INSERT INTO smoke_tbl VALUES ('id3', 'b3'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +INSERT INTO smoke_tbl VALUES ('id4', 'c4'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 1 apply: fan-out changes +\connect cloudsync_test_03_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent updates on the same row + mixed operations +\connect cloudsync_test_03_a +UPDATE smoke_tbl SET val = 'a1_a' WHERE id = 'id1'; +DELETE FROM smoke_tbl WHERE id = 'id2'; +INSERT INTO smoke_tbl VALUES ('id5', 'a5'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +UPDATE smoke_tbl SET val = 'a1_b' WHERE id = 'id1'; +UPDATE smoke_tbl SET val = 'b3_b' WHERE id = 'id3'; +INSERT INTO smoke_tbl VALUES ('id6', 'b6'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +UPDATE smoke_tbl SET val = 'a1_c' WHERE id = 'id1'; +DELETE FROM smoke_tbl WHERE id = 'id4'; +INSERT INTO smoke_tbl VALUES ('id7', 'c7'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 2 apply: fan-out changes +\connect cloudsync_test_03_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Round 3: additional operations to force another sync cycle +\connect cloudsync_test_03_a +UPDATE smoke_tbl SET val = 'b3_a' WHERE id = 'id3'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_b +DELETE FROM smoke_tbl WHERE id = 'id5'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_03_c +UPDATE smoke_tbl SET val = 'b6_c' WHERE id = 'id6'; +INSERT INTO smoke_tbl VALUES ('id8', 'c8'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3 apply: final fan-out +\connect cloudsync_test_03_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_03_b +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_03_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_03_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_03_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_03_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test multi-db roundtrip with concurrent updates +\else +\echo [FAIL] (:testid) Test multi-db roundtrip with concurrent updates +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_03_a; +DROP DATABASE IF EXISTS cloudsync_test_03_b; +DROP DATABASE IF EXISTS cloudsync_test_03_c; +\endif \ No newline at end of file diff --git a/test/postgresql/04_colversion_skew.sql b/test/postgresql/04_colversion_skew.sql new file mode 100644 index 0000000..b2d1000 --- /dev/null +++ b/test/postgresql/04_colversion_skew.sql @@ -0,0 +1,325 @@ +-- 'Test multi-db roundtrip with skewed col_version updates' +-- - concurrent update pattern where A/B/C perform 2/1/3 updates respectively on id1 before syncing. +-- - It follows the same apply order as the existing 3‑DB test and verifies final convergence across all three databases + +\set testid '04' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_04_a; +DROP DATABASE IF EXISTS cloudsync_test_04_b; +DROP DATABASE IF EXISTS cloudsync_test_04_c; +CREATE DATABASE cloudsync_test_04_a; +CREATE DATABASE cloudsync_test_04_b; +CREATE DATABASE cloudsync_test_04_c; + +\connect cloudsync_test_04_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_04_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_04_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on a single database, then sync +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a INSERT id1=seed_a1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_a1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 1 apply: fan-out changes +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: skewed concurrent updates on id1 +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a UPDATE id1=a1_u1' +\endif +UPDATE smoke_tbl SET val = 'a1_u1' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_a UPDATE id1=a1_u2' +\endif +UPDATE smoke_tbl SET val = 'a1_u2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_b UPDATE id1=b1_u1' +\endif +UPDATE smoke_tbl SET val = 'b1_u1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u1' +\endif +UPDATE smoke_tbl SET val = 'c1_u1' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u2' +\endif +UPDATE smoke_tbl SET val = 'c1_u2' WHERE id = 'id1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_04_c UPDATE id1=c1_u3' +\endif +UPDATE smoke_tbl SET val = 'c1_u3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 2 apply: fan-out changes +\connect cloudsync_test_04_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_04_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_04_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_04_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_04_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_04_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test multi-db roundtrip with skewed col_version updates +\else +\echo [FAIL] (:testid) Test multi-db roundtrip with skewed col_version updates +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_04_a; +DROP DATABASE IF EXISTS cloudsync_test_04_b; +DROP DATABASE IF EXISTS cloudsync_test_04_c; +\endif \ No newline at end of file diff --git a/test/postgresql/05_delete_recreate_cycle.sql b/test/postgresql/05_delete_recreate_cycle.sql new file mode 100644 index 0000000..0e3a313 --- /dev/null +++ b/test/postgresql/05_delete_recreate_cycle.sql @@ -0,0 +1,784 @@ +-- 'Test delete/recreate/update/delete/reinsert cycle across multiple DBs' +-- 1. A inserts +-- 2. B deletes +-- 3. C recreates with new value +-- 4. A updates +-- 5. B deletes again +-- 6. C reinserts with another value + + +\set testid '05' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_05_a; +DROP DATABASE IF EXISTS cloudsync_test_05_b; +DROP DATABASE IF EXISTS cloudsync_test_05_c; +CREATE DATABASE cloudsync_test_05_a; +CREATE DATABASE cloudsync_test_05_b; +CREATE DATABASE cloudsync_test_05_c; + +\connect cloudsync_test_05_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_05_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_05_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_a INSERT id1=seed_v1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: B deletes id1, sync +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_b DELETE id1' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 3: C recreates id1, sync +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_c INSERT id1=recreate_v2' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'recreate_v2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 4: A updates id1, sync +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_a UPDATE id1=update_v3' +\endif +UPDATE smoke_tbl SET val = 'update_v3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r4, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r4_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r4', 3), 'hex')) AS _apply_a_r4_b \gset +\else +SELECT 0 AS _apply_a_r4_b \gset +\endif +\if :payload_c_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r4', 3), 'hex')) AS _apply_a_r4_c \gset +\else +SELECT 0 AS _apply_a_r4_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r4', 3), 'hex')) AS _apply_b_r4_a \gset +\else +SELECT 0 AS _apply_b_r4_a \gset +\endif +\if :payload_c_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r4', 3), 'hex')) AS _apply_b_r4_c \gset +\else +SELECT 0 AS _apply_b_r4_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r4', 3), 'hex')) AS _apply_c_r4_a \gset +\else +SELECT 0 AS _apply_c_r4_a \gset +\endif +\if :payload_b_r4_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r4', 3), 'hex')) AS _apply_c_r4_b \gset +\else +SELECT 0 AS _apply_c_r4_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 5: B deletes id1, sync +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_b DELETE id1 (round5)' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r5, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r5_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r5', 3), 'hex')) AS _apply_a_r5_b \gset +\else +SELECT 0 AS _apply_a_r5_b \gset +\endif +\if :payload_c_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r5', 3), 'hex')) AS _apply_a_r5_c \gset +\else +SELECT 0 AS _apply_a_r5_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r5', 3), 'hex')) AS _apply_b_r5_a \gset +\else +SELECT 0 AS _apply_b_r5_a \gset +\endif +\if :payload_c_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r5', 3), 'hex')) AS _apply_b_r5_c \gset +\else +SELECT 0 AS _apply_b_r5_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r5', 3), 'hex')) AS _apply_c_r5_a \gset +\else +SELECT 0 AS _apply_c_r5_a \gset +\endif +\if :payload_b_r5_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round5 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r5', 3), 'hex')) AS _apply_c_r5_b \gset +\else +SELECT 0 AS _apply_c_r5_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round5 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 6: C re-inserts id1, sync +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_05_c INSERT id1=reinsert_v4' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'reinsert_v4'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r6, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r6_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_05_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r6', 3), 'hex')) AS _apply_a_r6_b \gset +\else +SELECT 0 AS _apply_a_r6_b \gset +\endif +\if :payload_c_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r6', 3), 'hex')) AS _apply_a_r6_c \gset +\else +SELECT 0 AS _apply_a_r6_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r6', 3), 'hex')) AS _apply_b_r6_a \gset +\else +SELECT 0 AS _apply_b_r6_a \gset +\endif +\if :payload_c_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r6', 3), 'hex')) AS _apply_b_r6_c \gset +\else +SELECT 0 AS _apply_b_r6_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_05_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 before merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r6', 3), 'hex')) AS _apply_c_r6_a \gset +\else +SELECT 0 AS _apply_c_r6_a \gset +\endif +\if :payload_b_r6_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round6 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r6', 3), 'hex')) AS _apply_c_r6_b \gset +\else +SELECT 0 AS _apply_c_r6_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round6 after merge cloudsync_test_05_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_05_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_05_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_05_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test delete/recreate/update/delete/reinsert cycle +\else +\echo [FAIL] (:testid) Test delete/recreate/update/delete/reinsert cycle +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_05_a; +DROP DATABASE IF EXISTS cloudsync_test_05_b; +DROP DATABASE IF EXISTS cloudsync_test_05_c; +\endif \ No newline at end of file diff --git a/test/postgresql/06_out_of_order_delivery.sql b/test/postgresql/06_out_of_order_delivery.sql new file mode 100644 index 0000000..607fd25 --- /dev/null +++ b/test/postgresql/06_out_of_order_delivery.sql @@ -0,0 +1,288 @@ +-- 'Test out-of-order payload delivery across multiple DBs' +-- - Seeds id1 +-- - Produces round2 and round3 concurrent updates +-- - Applies round3 before round2 on C, while A/B apply round2 then round3 +-- - Verifies convergence across all three DBs + +\set testid '06' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_06_a; +DROP DATABASE IF EXISTS cloudsync_test_06_b; +DROP DATABASE IF EXISTS cloudsync_test_06_c; +CREATE DATABASE cloudsync_test_06_a; +CREATE DATABASE cloudsync_test_06_b; +CREATE DATABASE cloudsync_test_06_c; + +\connect cloudsync_test_06_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_06_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_06_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_06_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_06_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_06_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent updates +\connect cloudsync_test_06_a +UPDATE smoke_tbl SET val = 'a1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +UPDATE smoke_tbl SET val = 'b1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +UPDATE smoke_tbl SET val = 'c1_r2' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3: further updates (newer payloads) +\connect cloudsync_test_06_a +UPDATE smoke_tbl SET val = 'a1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_b +UPDATE smoke_tbl SET val = 'b1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_06_c +UPDATE smoke_tbl SET val = 'c1_r3' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Out-of-order apply: apply round3 before round2 on C, and round2 before round3 on A/B +\connect cloudsync_test_06_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_06_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_06_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_06_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_06_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_06_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Test out-of-order payload delivery +\else +\echo [FAIL] (:testid) Test out-of-order payload delivery +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_06_a; +DROP DATABASE IF EXISTS cloudsync_test_06_b; +DROP DATABASE IF EXISTS cloudsync_test_06_c; +\endif \ No newline at end of file diff --git a/test/postgresql/07_delete_vs_update.sql b/test/postgresql/07_delete_vs_update.sql new file mode 100644 index 0000000..cfa3f75 --- /dev/null +++ b/test/postgresql/07_delete_vs_update.sql @@ -0,0 +1,290 @@ +-- Concurrent delete vs update +-- Steps: +-- 1) Seed id1 on A, sync to B/C +-- 2) B deletes id1 while C updates id1, then sync +-- 3) A updates id1 after merge, then sync + +\set testid '07' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_07_a; +DROP DATABASE IF EXISTS cloudsync_test_07_b; +DROP DATABASE IF EXISTS cloudsync_test_07_c; +CREATE DATABASE cloudsync_test_07_a; +CREATE DATABASE cloudsync_test_07_b; +CREATE DATABASE cloudsync_test_07_c; + +\connect cloudsync_test_07_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_07_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_07_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on A, sync to B/C +\connect cloudsync_test_07_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: B deletes id1, C updates id1, then sync +\connect cloudsync_test_07_b +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +UPDATE smoke_tbl SET val = 'c1_update' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Round 3: A updates id1 after merge, then sync +\connect cloudsync_test_07_a +UPDATE smoke_tbl SET val = 'a1_post_merge' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_07_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif + +\connect cloudsync_test_07_b +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif + +\connect cloudsync_test_07_c +\if :payload_a_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_07_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_07_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_07_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Concurrent delete vs update +\else +\echo [FAIL] (:testid) Concurrent delete vs update +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_07_a; +DROP DATABASE IF EXISTS cloudsync_test_07_b; +DROP DATABASE IF EXISTS cloudsync_test_07_c; +\endif \ No newline at end of file diff --git a/test/postgresql/08_resurrect_delayed_delete.sql b/test/postgresql/08_resurrect_delayed_delete.sql new file mode 100644 index 0000000..c714a1c --- /dev/null +++ b/test/postgresql/08_resurrect_delayed_delete.sql @@ -0,0 +1,363 @@ +-- Resurrect after delete with delayed payload +-- Steps: +-- 1) Seed id1 on A, sync to B/C +-- 2) A deletes id1 and generates delete payload (do not apply yet on B) +-- 3) B recreates id1 with new value, sync to A/C +-- 4) Apply delayed delete payload from A to B/C +-- 5) Verify convergence + +\set testid '08' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_08_a; +DROP DATABASE IF EXISTS cloudsync_test_08_b; +DROP DATABASE IF EXISTS cloudsync_test_08_c; +CREATE DATABASE cloudsync_test_08_a; +CREATE DATABASE cloudsync_test_08_b; +CREATE DATABASE cloudsync_test_08_c; + +\connect cloudsync_test_08_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_08_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_08_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed id1 on A, sync to B/C +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_a INSERT id1=seed_v1' +\endif +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 2: A deletes id1 (payload delayed for B/C) +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_a DELETE id1' +\endif +DELETE FROM smoke_tbl WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Round 3: B recreates id1, sync to A/C (but A's delete still not applied on B/C) +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_08_b UPSERT id1=recreate_v2' +\endif +INSERT INTO smoke_tbl (id, val) +VALUES ('id1', 'recreate_v2') +ON CONFLICT (id) DO UPDATE SET val = EXCLUDED.val; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_08_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_a smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Round 4: apply delayed delete payload from A to B/C +\connect cloudsync_test_08_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply delayed a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r4_a_delayed \gset +\else +SELECT 0 AS _apply_b_r4_a_delayed \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_08_b smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +\connect cloudsync_test_08_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 before merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round4 apply delayed a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r4_a_delayed \gset +\else +SELECT 0 AS _apply_c_r4_a_delayed \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round4 after merge cloudsync_test_08_c smoke_tbl' +SELECT * FROM smoke_tbl ORDER BY id; +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_08_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_08_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_08_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Resurrect after delete with delayed payload +\else +\echo [FAIL] (:testid) Resurrect after delete with delayed payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_08_a; +DROP DATABASE IF EXISTS cloudsync_test_08_b; +DROP DATABASE IF EXISTS cloudsync_test_08_c; +\endif \ No newline at end of file diff --git a/test/postgresql/09_multicol_concurrent_edits.sql b/test/postgresql/09_multicol_concurrent_edits.sql new file mode 100644 index 0000000..b3142f3 --- /dev/null +++ b/test/postgresql/09_multicol_concurrent_edits.sql @@ -0,0 +1,217 @@ +-- Multi-column concurrent edits +-- Steps: +-- 1) Create table with two data columns, seed row on A, sync to B/C +-- 2) B updates col_a while C updates col_b concurrently +-- 3) Sync and verify both columns are preserved on all DBs + +\set testid '09' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_09_a; +DROP DATABASE IF EXISTS cloudsync_test_09_b; +DROP DATABASE IF EXISTS cloudsync_test_09_c; +CREATE DATABASE cloudsync_test_09_a; +CREATE DATABASE cloudsync_test_09_b; +CREATE DATABASE cloudsync_test_09_c; + +\connect cloudsync_test_09_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_09_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_09_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, col_a TEXT, col_b TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Round 1: seed row on A, sync to B/C +\connect cloudsync_test_09_a +INSERT INTO smoke_tbl VALUES ('id1', 'a0', 'b0'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_a +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif + +\connect cloudsync_test_09_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif + +\connect cloudsync_test_09_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif + +-- Round 2: concurrent edits on different columns +\connect cloudsync_test_09_b +UPDATE smoke_tbl SET col_a = 'a1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_c +UPDATE smoke_tbl SET col_b = 'b1' WHERE id = 'id1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_09_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply round 2 payloads +\connect cloudsync_test_09_a +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif + +\connect cloudsync_test_09_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif + +\connect cloudsync_test_09_c +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif + +-- Final consistency check across all three databases (both columns) +\connect cloudsync_test_09_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_09_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_09_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(col_a, '') || ':' || COALESCE(col_b, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Multi-column concurrent edits +\else +\echo [FAIL] (:testid) Multi-column concurrent edits +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_09_a; +DROP DATABASE IF EXISTS cloudsync_test_09_b; +DROP DATABASE IF EXISTS cloudsync_test_09_c; +\endif \ No newline at end of file diff --git a/test/postgresql/10_empty_payload_noop.sql b/test/postgresql/10_empty_payload_noop.sql new file mode 100644 index 0000000..ba65f0d --- /dev/null +++ b/test/postgresql/10_empty_payload_noop.sql @@ -0,0 +1,217 @@ +-- Empty payload + no-op merge +-- Steps: +-- 1) Setup three DBs and table +-- 2) Attempt to encode/apply empty payloads +-- 3) Verify data unchanged and hashes match + +\set testid '10' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_10_a; +DROP DATABASE IF EXISTS cloudsync_test_10_b; +DROP DATABASE IF EXISTS cloudsync_test_10_c; +CREATE DATABASE cloudsync_test_10_a; +CREATE DATABASE cloudsync_test_10_b; +CREATE DATABASE cloudsync_test_10_c; + +\connect cloudsync_test_10_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_a \gset + +\connect cloudsync_test_10_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_b \gset + +\connect cloudsync_test_10_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS smoke_tbl; +CREATE TABLE smoke_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('smoke_tbl', 'CLS', 1) AS _init_site_id_c \gset + +-- Seed a stable row so hashes are meaningful +\connect cloudsync_test_10_a +INSERT INTO smoke_tbl VALUES ('id1', 'seed_v1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_seed, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_seed_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply seed payloads so all DBs start in sync +\connect cloudsync_test_10_a +\if :payload_b_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_seed', 3), 'hex')) AS _apply_a_seed_b \gset +\else +SELECT 0 AS _apply_a_seed_b \gset +\endif +\if :payload_c_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_seed', 3), 'hex')) AS _apply_a_seed_c \gset +\else +SELECT 0 AS _apply_a_seed_c \gset +\endif + +\connect cloudsync_test_10_b +\if :payload_a_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_seed', 3), 'hex')) AS _apply_b_seed_a \gset +\else +SELECT 0 AS _apply_b_seed_a \gset +\endif +\if :payload_c_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_seed', 3), 'hex')) AS _apply_b_seed_c \gset +\else +SELECT 0 AS _apply_b_seed_c \gset +\endif + +\connect cloudsync_test_10_c +\if :payload_a_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_seed', 3), 'hex')) AS _apply_c_seed_a \gset +\else +SELECT 0 AS _apply_c_seed_a \gset +\endif +\if :payload_b_seed_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_seed', 3), 'hex')) AS _apply_c_seed_b \gset +\else +SELECT 0 AS _apply_c_seed_b \gset +\endif + +-- Encode payloads with no changes (expected empty) +\connect cloudsync_test_10_a +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_10_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_empty, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_empty_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply empty payloads (should be no-ops) +\connect cloudsync_test_10_a +\if :payload_b_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_empty', 3), 'hex')) AS _apply_a_empty_b \gset +\else +SELECT 0 AS _apply_a_empty_b \gset +\endif +\if :payload_c_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_empty', 3), 'hex')) AS _apply_a_empty_c \gset +\else +SELECT 0 AS _apply_a_empty_c \gset +\endif + +\connect cloudsync_test_10_b +\if :payload_a_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_empty', 3), 'hex')) AS _apply_b_empty_a \gset +\else +SELECT 0 AS _apply_b_empty_a \gset +\endif +\if :payload_c_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_empty', 3), 'hex')) AS _apply_b_empty_c \gset +\else +SELECT 0 AS _apply_b_empty_c \gset +\endif + +\connect cloudsync_test_10_c +\if :payload_a_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_empty', 3), 'hex')) AS _apply_c_empty_a \gset +\else +SELECT 0 AS _apply_c_empty_a \gset +\endif +\if :payload_b_empty_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_empty', 3), 'hex')) AS _apply_c_empty_b \gset +\else +SELECT 0 AS _apply_c_empty_b \gset +\endif + +-- Final consistency check across all three databases +\connect cloudsync_test_10_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_a +FROM smoke_tbl \gset + +\connect cloudsync_test_10_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_b +FROM smoke_tbl \gset + +\connect cloudsync_test_10_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS smoke_hash_c +FROM smoke_tbl \gset + +SELECT (:'smoke_hash_a' = :'smoke_hash_b' AND :'smoke_hash_a' = :'smoke_hash_c') AS multi_db_roundtrip_ok \gset +\if :multi_db_roundtrip_ok +\echo [PASS] (:testid) Empty payload + no-op merge +\else +\echo [FAIL] (:testid) Empty payload + no-op merge +SELECT (:fail::int + 1) AS fail \gset +\endif + + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_10_a; +DROP DATABASE IF EXISTS cloudsync_test_10_b; +DROP DATABASE IF EXISTS cloudsync_test_10_c; +\endif \ No newline at end of file diff --git a/test/postgresql/11_multi_table_multi_columns_rounds.sql b/test/postgresql/11_multi_table_multi_columns_rounds.sql new file mode 100644 index 0000000..a94550f --- /dev/null +++ b/test/postgresql/11_multi_table_multi_columns_rounds.sql @@ -0,0 +1,717 @@ +-- 'Test multi-table multi-db roundtrip' +-- simulate the sport tracker app from examples +-- Steps: +-- 1) Create three databases, initialize users/activities/workouts and cloudsync +-- 2) Round 1: seed base data on A and sync to B/C +-- 3) Round 2: concurrent updates/inserts on A/B/C, then sync +-- 4) Round 3: more concurrent edits, then sync +-- 5) Verify convergence per table across all three databases + +\set testid '11' +\ir helper_test_init.sql + +-- Step 1: setup databases and schema +\if :{?DEBUG_MERGE} +\echo '[STEP 1] Setup databases and schema' +\endif +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_11_a; +DROP DATABASE IF EXISTS cloudsync_test_11_b; +DROP DATABASE IF EXISTS cloudsync_test_11_c; +CREATE DATABASE cloudsync_test_11_a; +CREATE DATABASE cloudsync_test_11_b; +CREATE DATABASE cloudsync_test_11_c; + +\connect cloudsync_test_11_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_a \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_a \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_a \gset + +\connect cloudsync_test_11_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_b \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_b \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_b \gset + +\connect cloudsync_test_11_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS workouts; +DROP TABLE IF EXISTS activities; +DROP TABLE IF EXISTS users; +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL DEFAULT '' +); +CREATE TABLE IF NOT EXISTS activities ( + id TEXT PRIMARY KEY NOT NULL, + type TEXT NOT NULL DEFAULT 'runnning', + duration INTEGER, + distance DOUBLE PRECISION, + calories INTEGER, + date TEXT, + notes TEXT, + user_id TEXT REFERENCES users (id) +); +CREATE TABLE IF NOT EXISTS workouts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + type TEXT, + duration INTEGER, + exercises TEXT, + date TEXT, + completed INTEGER DEFAULT 0, + user_id TEXT +); +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_c \gset +SELECT cloudsync_init('activities', 'CLS', 1) AS _init_activities_c \gset +SELECT cloudsync_init('workouts', 'CLS', 1) AS _init_workouts_c \gset + +-- Step 2: Round 1 seed base data on A, sync to B/C +\if :{?DEBUG_MERGE} +\echo '[STEP 2] Round 1 seed base data on A, sync to B/C' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT users u1=alice' +\endif +INSERT INTO users (id, name) VALUES ('u1', 'alice'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT activities act1' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act1', 'running', 30, 5.0, 200, '2026-01-01', 'seed', 'u1'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT workouts w1' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w1', 'base', 'cardio', 30, 'run', '2026-01-01', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_a_r1_b \gset +\else +SELECT 0 AS _apply_a_r1_b \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_a_r1_c \gset +\else +SELECT 0 AS _apply_a_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> b' +\echo ### payload_a_r1: :payload_a_r1 +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_b_r1_a \gset +\else +SELECT 0 AS _apply_b_r1_a \gset +\endif +\if :payload_c_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r1', 3), 'hex')) AS _apply_b_r1_c \gset +\else +SELECT 0 AS _apply_b_r1_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply_c_r1_a \gset +\else +SELECT 0 AS _apply_c_r1_a \gset +\endif +\if :payload_b_r1_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round1 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r1', 3), 'hex')) AS _apply_c_r1_b \gset +\else +SELECT 0 AS _apply_c_r1_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round1 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round1 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 3: Round 2 concurrent updates and inserts across nodes +\if :{?DEBUG_MERGE} +\echo '[STEP 3] Round 2 concurrent updates and inserts across nodes' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE users u1=alice_a2' +\endif +UPDATE users SET name = 'alice_a2' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE activities act1 duration/calories' +\endif +UPDATE activities SET duration = 35, calories = 220 WHERE id = 'act1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a INSERT workouts w2' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w2', 'tempo', 'cardio', 40, 'run', '2026-01-02', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE users u1=alice_b2' +\endif +UPDATE users SET name = 'alice_b2' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE workouts w1 completed=1' +\endif +UPDATE workouts SET completed = 1 WHERE id = 'w1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b INSERT users u2=bob' +\endif +INSERT INTO users (id, name) VALUES ('u2', 'bob'); +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b INSERT activities act2' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act2', 'cycling', 60, 20.0, 500, '2026-01-02', 'b_seed', 'u2'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE activities act1 notes=c_note' +\endif +UPDATE activities SET notes = 'c_note' WHERE id = 'act1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE workouts w1 type=strength' +\endif +UPDATE workouts SET type = 'strength' WHERE id = 'w1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c INSERT workouts w3' +\endif +INSERT INTO workouts (id, name, type, duration, exercises, date, completed, user_id) +VALUES ('w3', 'lift', 'strength', 45, 'squat', '2026-01-02', 0, 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_a_r2_b \gset +\else +SELECT 0 AS _apply_a_r2_b \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_a_r2_c \gset +\else +SELECT 0 AS _apply_a_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_b_r2_a \gset +\else +SELECT 0 AS _apply_b_r2_a \gset +\endif +\if :payload_c_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r2', 3), 'hex')) AS _apply_b_r2_c \gset +\else +SELECT 0 AS _apply_b_r2_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_c_r2_a \gset +\else +SELECT 0 AS _apply_c_r2_a \gset +\endif +\if :payload_b_r2_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round2 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r2', 3), 'hex')) AS _apply_c_r2_b \gset +\else +SELECT 0 AS _apply_c_r2_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round2 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round2 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 4: Round 3 more concurrent edits +\if :{?DEBUG_MERGE} +\echo '[STEP 4] Round 3 more concurrent edits' +\endif +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_a UPDATE workouts w2 completed=1' +\endif +UPDATE workouts SET completed = 1 WHERE id = 'w2'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_b UPDATE activities act1 distance=6.5' +\endif +UPDATE activities SET distance = 6.5 WHERE id = 'act1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c UPDATE users u1=alice_c3' +\endif +UPDATE users SET name = 'alice_c3' WHERE id = 'u1'; +\if :{?DEBUG_MERGE} +\echo '[INFO] cloudsync_test_11_c INSERT activities act3' +\endif +INSERT INTO activities (id, type, duration, distance, calories, date, notes, user_id) +VALUES ('act3', 'yoga', 45, 0.0, 150, '2026-01-03', 'c_seed', 'u1'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_c_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_c_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_11_a +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_a_r3_b \gset +\else +SELECT 0 AS _apply_a_r3_b \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> a' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_a_r3_c \gset +\else +SELECT 0 AS _apply_a_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_a users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_a activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_a workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_b +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_b_r3_a \gset +\else +SELECT 0 AS _apply_b_r3_a \gset +\endif +\if :payload_c_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply c -> b' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_c_r3', 3), 'hex')) AS _apply_b_r3_c \gset +\else +SELECT 0 AS _apply_b_r3_c \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_b users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_b activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_b workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +\connect cloudsync_test_11_c +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 before merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 before merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif +\if :payload_a_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply a -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r3', 3), 'hex')) AS _apply_c_r3_a \gset +\else +SELECT 0 AS _apply_c_r3_a \gset +\endif +\if :payload_b_r3_ok +\if :{?DEBUG_MERGE} +\echo '[MERGE] round3 apply b -> c' +\endif +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_c_r3_b \gset +\else +SELECT 0 AS _apply_c_r3_b \gset +\endif +\if :{?DEBUG_MERGE} +\echo '[INFO] round3 after merge cloudsync_test_11_c users' +SELECT * FROM users ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_c activities' +SELECT * FROM activities ORDER BY id; +\echo '[INFO] round3 after merge cloudsync_test_11_c workouts' +SELECT * FROM workouts ORDER BY id; +\endif + +-- Step 5: final consistency check across all three databases +\if :{?DEBUG_MERGE} +\echo '[STEP 5] Final consistency check across all three databases' +\endif +\connect cloudsync_test_11_a +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_a +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_a +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_a +FROM workouts \gset + +\connect cloudsync_test_11_b +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_b +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_b +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_b +FROM workouts \gset + +\connect cloudsync_test_11_c +SELECT md5(COALESCE(string_agg(id || ':' || name, ',' ORDER BY id), '')) AS users_hash_c +FROM users \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(type, '') || ':' || COALESCE(duration::text, '') || ':' || + COALESCE(distance::text, '') || ':' || COALESCE(calories::text, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(notes, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS activities_hash_c +FROM activities \gset +SELECT md5(COALESCE(string_agg( + id || ':' || COALESCE(name, '') || ':' || COALESCE(type, '') || ':' || + COALESCE(duration::text, '') || ':' || COALESCE(exercises, '') || ':' || + COALESCE(date, '') || ':' || COALESCE(completed::text, '') || ':' || COALESCE(user_id, ''), + ',' ORDER BY id +), '')) AS workouts_hash_c +FROM workouts \gset + +SELECT (:'users_hash_a' = :'users_hash_b' AND :'users_hash_a' = :'users_hash_c') AS users_ok \gset +\if :users_ok +\echo [PASS] (:testid) Multi-table users convergence +\else +\echo [FAIL] (:testid) Multi-table users convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'activities_hash_a' = :'activities_hash_b' AND :'activities_hash_a' = :'activities_hash_c') AS activities_ok \gset +\if :activities_ok +\echo [PASS] (:testid) Multi-table activities convergence +\else +\echo [FAIL] (:testid) Multi-table activities convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'workouts_hash_a' = :'workouts_hash_b' AND :'workouts_hash_a' = :'workouts_hash_c') AS workouts_ok \gset +\if :workouts_ok +\echo [PASS] (:testid) Multi-table workouts convergence +\else +\echo [FAIL] (:testid) Multi-table workouts convergence +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_11_a; +DROP DATABASE IF EXISTS cloudsync_test_11_b; +DROP DATABASE IF EXISTS cloudsync_test_11_c; +\endif \ No newline at end of file diff --git a/test/postgresql/12_repeated_table_multi_schemas.sql b/test/postgresql/12_repeated_table_multi_schemas.sql new file mode 100644 index 0000000..d362824 --- /dev/null +++ b/test/postgresql/12_repeated_table_multi_schemas.sql @@ -0,0 +1,228 @@ +\set testid '12' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_12; +CREATE DATABASE cloudsync_test_12; + +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- Reset extension and install +DROP EXTENSION IF EXISTS cloudsync CASCADE; +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- 'Test multi-schema table init (setup)' +CREATE SCHEMA IF NOT EXISTS test_schema; +DROP TABLE IF EXISTS public.repeated_table; +DROP TABLE IF EXISTS test_schema.repeated_table; +CREATE TABLE public.repeated_table (id TEXT PRIMARY KEY, data TEXT); +CREATE TABLE test_schema.repeated_table (id TEXT PRIMARY KEY, data TEXT); + +-- Reset the connection to test if we load the configuration correctly +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- 'Test init on table that exists in multiple schemas (default: public)' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated \gset +SELECT cloudsync_init('repeated_table', 'CLS', 1) AS _init_repeated_public \gset +SELECT cloudsync_table_schema('repeated_table') AS repeated_schema_public \gset +SELECT (:'repeated_schema_public' = 'public') AS repeated_schema_public_ok \gset +\if :repeated_schema_public_ok +\echo [PASS] (:testid) Test cloudsync_table_schema returns public for repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_table_schema returns public for repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (to_regclass('public.repeated_table_cloudsync') IS NOT NULL) AS init_repeated_public_ok \gset +\if :init_repeated_public_ok +\echo [PASS] (:testid) Test init on repeated_table in public schema +\else +\echo [FAIL] (:testid) Test init on repeated_table in public schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test insert on repeated_table in public schema' +SELECT cloudsync_uuid() AS repeated_id1 \gset +INSERT INTO public.repeated_table (id, data) VALUES (:'repeated_id1', 'public_data'); +SELECT (COUNT(*) = 1) AS insert_repeated_public_ok +FROM public.repeated_table_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id1']::text[]) + AND col_name = 'data' \gset +\if :insert_repeated_public_ok +\echo [PASS] (:testid) Test insert metadata on repeated_table in public +\else +\echo [FAIL] (:testid) Test insert metadata on repeated_table in public +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read for public.repeated_table' +SELECT COUNT(*) AS changes_view_repeated_count +FROM cloudsync_changes +WHERE tbl = 'repeated_table' \gset +SELECT COUNT(*) AS changes_meta_repeated_count +FROM public.repeated_table_cloudsync \gset +SELECT (:changes_view_repeated_count::int = :changes_meta_repeated_count::int) AS changes_read_repeated_ok \gset +\if :changes_read_repeated_ok +\echo [PASS] (:testid) Test cloudsync_changes view read for public.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read for public.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write for public.repeated_table' +SELECT cloudsync_uuid() AS repeated_id2 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'repeated_table', + cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id2']::text[]), + 'data', + -- "public_write" encoded as cloudsync text value (type 0x0b + len 0x0c) + decode('0b0c7075626c69635f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_repeated_ok +FROM public.repeated_table +WHERE id = :'repeated_id2' AND data = 'public_write' \gset +\if :changes_write_repeated_ok +\echo [PASS] (:testid) Test cloudsync_changes view write for public.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write for public.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cleanup on table with ambiguous name' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated2 \gset +SELECT (to_regclass('public.repeated_table_cloudsync') IS NULL) AS cleanup_repeated_ok \gset +\if :cleanup_repeated_ok +\echo [PASS] (:testid) Test cleanup on repeated_table +\else +\echo [FAIL] (:testid) Test cleanup on repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_set_schema and init on test_schema' +SELECT cloudsync_set_schema('test_schema') AS _set_schema \gset +SELECT cloudsync_init('repeated_table', 'CLS', 1) AS _init_repeated_test_schema \gset +SELECT cloudsync_table_schema('repeated_table') AS repeated_schema_test_schema \gset +SELECT (:'repeated_schema_test_schema' = 'test_schema') AS repeated_schema_test_schema_ok \gset +\if :repeated_schema_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_table_schema returns test_schema for repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_table_schema returns test_schema for repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (to_regclass('test_schema.repeated_table_cloudsync') IS NOT NULL) AS init_repeated_test_schema_ok \gset +\if :init_repeated_test_schema_ok +\echo [PASS] (:testid) Test init on repeated_table in test_schema +\else +\echo [FAIL] (:testid) Test init on repeated_table in test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test that public.repeated_table_cloudsync was not recreated' +SELECT (to_regclass('public.repeated_table_cloudsync') IS NULL) AS public_still_clean_ok \gset +\if :public_still_clean_ok +\echo [PASS] (:testid) Test public.repeated_table_cloudsync still cleaned up +\else +\echo [FAIL] (:testid) Test public.repeated_table_cloudsync should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- reset the current schema to check if the next connection load the correct configuration +SELECT cloudsync_set_schema('public') AS _reset_schema \gset + +-- Reset the connection to test if if loads the correct configuration for the table on the correct schema +\connect cloudsync_test_12 +\ir helper_psql_conn_setup.sql + +-- 'Test insert on repeated_table in test_schema' +SELECT cloudsync_uuid() AS repeated_id3 \gset +INSERT INTO test_schema.repeated_table (id, data) VALUES (:'repeated_id3', 'test_schema_data'); +SELECT (COUNT(*) = 1) AS insert_repeated_test_schema_ok +FROM test_schema.repeated_table_cloudsync +WHERE pk = cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id3']::text[]) + AND col_name = 'data' \gset +\if :insert_repeated_test_schema_ok +\echo [PASS] (:testid) Test insert metadata on repeated_table in test_schema +\else +\echo [FAIL] (:testid) Test insert metadata on repeated_table in test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view read for test_schema.repeated_table' +SELECT COUNT(*) AS changes_view_test_schema_count +FROM cloudsync_changes +WHERE tbl = 'repeated_table' \gset +SELECT COUNT(*) AS changes_meta_test_schema_count +FROM test_schema.repeated_table_cloudsync \gset +SELECT (:changes_view_test_schema_count::int = :changes_meta_test_schema_count::int) AS changes_read_test_schema_ok \gset +\if :changes_read_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_changes view read for test_schema.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view read for test_schema.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cloudsync_changes view write for test_schema.repeated_table' +SELECT cloudsync_uuid() AS repeated_id4 \gset +INSERT INTO cloudsync_changes (tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) +VALUES ( + 'repeated_table', + cloudsync_pk_encode(VARIADIC ARRAY[:'repeated_id4']::text[]), + 'data', + -- "testschema_write" encoded as cloudsync text value (type 0x0b + len 0x10) + decode('0b1074657374736368656d615f7772697465', 'hex'), + 1, + cloudsync_db_version_next(), + cloudsync_siteid(), + 1, + 0 +); +SELECT (COUNT(*) = 1) AS changes_write_test_schema_ok +FROM test_schema.repeated_table +WHERE id = :'repeated_id4' AND data = 'testschema_write' \gset +\if :changes_write_test_schema_ok +\echo [PASS] (:testid) Test cloudsync_changes view write for test_schema.repeated_table +\else +\echo [FAIL] (:testid) Test cloudsync_changes view write for test_schema.repeated_table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Test cleanup on repeated_table on test_schema' +SELECT cloudsync_cleanup('repeated_table') AS _cleanup_repeated3 \gset +SELECT (to_regclass('test_schema.repeated_table_cloudsync') IS NULL) AS cleanup_repeated3_ok \gset +\if :cleanup_repeated3_ok +\echo [PASS] (:testid) Test cleanup on repeated_table on test_schema +\else +\echo [FAIL] (:testid) Test cleanup on repeated_table on test_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 'Reset schema to public for subsequent tests' +SELECT cloudsync_set_schema('public') AS _reset_schema \gset +SELECT current_schema() AS current_schema_after_reset \gset +SELECT (:'current_schema_after_reset' = 'public') AS schema_reset_ok \gset +\if :schema_reset_ok +\echo [PASS] (:testid) Test schema reset to public +\else +\echo [FAIL] (:testid) Test schema reset to public +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :{?DEBUG_MERGE} +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_12; +\endif + +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_12; +\endif \ No newline at end of file diff --git a/test/postgresql/13_per_table_schema_tracking.sql b/test/postgresql/13_per_table_schema_tracking.sql new file mode 100644 index 0000000..8729fe6 --- /dev/null +++ b/test/postgresql/13_per_table_schema_tracking.sql @@ -0,0 +1,234 @@ +-- Per-Table Schema Tracking Tests +-- Tests from plans/PLAN_per_table_schema_tracking.md + +\set testid '13' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup any existing test databases and schemas +DROP DATABASE IF EXISTS cloudsync_test_13; +CREATE DATABASE cloudsync_test_13; + +\connect cloudsync_test_13 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- ============================================================================ +-- Test 1: Basic Schema Detection +-- ============================================================================ + +CREATE SCHEMA test_schema; +CREATE TABLE test_schema.products (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); +SELECT cloudsync_set_schema('test_schema') AS _set_schema \gset +SELECT cloudsync_init('products', 'CLS', 1) AS _init_products \gset + +-- Test: Verify schema is detected correctly +SELECT cloudsync_table_schema('products') AS detected_schema \gset +SELECT (:'detected_schema' = 'test_schema') AS basic_schema_detection_ok \gset +\if :basic_schema_detection_ok +\echo [PASS] (:testid) Basic Schema Detection +\else +\echo [FAIL] (:testid) Basic Schema Detection - Expected 'test_schema', got '':detected_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 2: Same-Connection Duplicate Prevention +-- ============================================================================ + +CREATE TABLE public.users (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); + +SELECT cloudsync_set_schema('public') AS _set_public \gset +SELECT cloudsync_init('users', 'CLS', 1) AS _init_users_public \gset + +-- Attempt to init again in same connection (should FAIL) +DO $$ +BEGIN + PERFORM cloudsync_init('users', 'CLS', 1); + RAISE EXCEPTION 'Expected error but init succeeded'; +EXCEPTION + WHEN OTHERS THEN + -- Expected to fail + NULL; +END $$; + +\echo [PASS] (:testid) Same-Connection Duplicate Prevention + +-- ============================================================================ +-- Test 3: Schema Not Found (NULL handling) +-- ============================================================================ + +CREATE TABLE orphan_table (id TEXT PRIMARY KEY); + +-- Test: Querying schema of non-initialized table should return NULL or empty +DO $$ +DECLARE + orphan_schema TEXT; +BEGIN + SELECT cloudsync_table_schema('orphan_table') INTO orphan_schema; + IF orphan_schema IS NOT NULL THEN + RAISE EXCEPTION 'Expected NULL for orphan table schema, got %', orphan_schema; + END IF; +END $$; + +\echo [PASS] (:testid) Schema Not Found (NULL handling) + +-- ============================================================================ +-- Test 4: Schema Setting Does Not Affect Existing Tables +-- ============================================================================ + +CREATE SCHEMA schema_a; +CREATE SCHEMA schema_b; +CREATE TABLE schema_a.orders (id TEXT PRIMARY KEY, total TEXT NOT NULL DEFAULT '0'); +CREATE TABLE schema_b.products_b (id TEXT PRIMARY KEY, name TEXT NOT NULL DEFAULT ''); + +-- Initialize in schema_a +SELECT cloudsync_set_schema('schema_a') AS _set_schema_a \gset +SELECT cloudsync_init('orders', 'CLS', 1) AS _init_orders \gset + +-- Verify schema +SELECT cloudsync_table_schema('orders') AS orders_schema_before \gset +SELECT (:'orders_schema_before' = 'schema_a') AS orders_schema_before_ok \gset +\if :orders_schema_before_ok +\echo [PASS] (:testid) Schema Setting - Initial schema correct (schema_a) +\else +\echo [FAIL] (:testid) Schema Setting - Initial schema incorrect. Expected 'schema_a', got '':orders_schema_before +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Change global schema setting +SELECT cloudsync_set_schema('schema_b') AS _set_schema_b \gset + +-- Test: Existing table still uses original schema +SELECT cloudsync_table_schema('orders') AS orders_schema_after \gset +SELECT (:'orders_schema_after' = 'schema_a') AS orders_schema_unchanged_ok \gset +\if :orders_schema_unchanged_ok +\echo [PASS] (:testid) Schema Setting Does Not Affect Existing Tables +\else +\echo [FAIL] (:testid) Schema Setting Does Not Affect Existing Tables - Expected 'schema_a', got '':orders_schema_after +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test: New initialization uses new schema +SELECT cloudsync_init('products_b', 'CLS', 1) AS _init_products_b \gset +SELECT cloudsync_table_schema('products_b') AS products_schema \gset +SELECT (:'products_schema' = 'schema_b') AS new_table_uses_new_schema_ok \gset +\if :new_table_uses_new_schema_ok +\echo [PASS] (:testid) New table uses new schema setting (schema_b) +\else +\echo [FAIL] (:testid) New table uses new schema setting - Expected 'schema_b', got '':products_schema +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 5: Multi-Schema Database with Different Table Names +-- ============================================================================ + +CREATE SCHEMA sales; +CREATE SCHEMA analytics; + +CREATE TABLE sales.orders_sales (id TEXT PRIMARY KEY, total TEXT NOT NULL DEFAULT '0'); +CREATE TABLE analytics.reports (id TEXT PRIMARY KEY, data TEXT NOT NULL DEFAULT '{}'); + +SELECT cloudsync_set_schema('sales') AS _set_sales \gset +SELECT cloudsync_init('orders_sales', 'CLS', 1) AS _init_orders_sales \gset + +SELECT cloudsync_set_schema('analytics') AS _set_analytics \gset +SELECT cloudsync_init('reports', 'CLS', 1) AS _init_reports \gset + +-- Both should work independently +INSERT INTO sales.orders_sales VALUES (cloudsync_uuid()::text, '100.00'); +INSERT INTO analytics.reports VALUES (cloudsync_uuid()::text, '{"type":"summary"}'); + +-- Verify changes tracked in correct schemas +SELECT + (SELECT COUNT(*) > 0 FROM sales.orders_sales_cloudsync) AND + (SELECT COUNT(*) > 0 FROM analytics.reports_cloudsync) AS multi_schema_ok \gset +\if :multi_schema_ok +\echo [PASS] (:testid) Multi-Schema Database with Different Table Names +\else +\echo [FAIL] (:testid) Multi-Schema Database with Different Table Names +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 6: System Tables in Public Schema +-- ============================================================================ + +CREATE SCHEMA custom_schema; +SELECT cloudsync_set_schema('custom_schema') AS _set_custom \gset +CREATE TABLE custom_schema.test_table (id TEXT PRIMARY KEY, val TEXT NOT NULL DEFAULT ''); +SELECT cloudsync_init('test_table', 'CLS', 1) AS _init_test_table \gset + +-- System tables should still be in public +SELECT COUNT(*) AS system_tables_in_public FROM pg_tables +WHERE tablename IN ('cloudsync_settings', 'cloudsync_site_id', 'cloudsync_table_settings', 'cloudsync_schema_versions') + AND schemaname = 'public' \gset + +-- Metadata table should be in custom_schema +SELECT schemaname AS metadata_schema FROM pg_tables +WHERE tablename = 'test_table_cloudsync' \gset + +SELECT (:system_tables_in_public = 4 AND :'metadata_schema' = 'custom_schema') AS system_tables_ok \gset +\if :system_tables_ok +\echo [PASS] (:testid) System Tables in Public Schema +\else +\echo [FAIL] (:testid) System Tables in Public Schema - System tables: :system_tables_in_public/4, metadata schema: ':metadata_schema' +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 7: Metadata Table Location Detection +-- ============================================================================ + +-- Verify that metadata tables are created in the correct schemas +SELECT COUNT(*) AS products_meta FROM pg_tables +WHERE tablename = 'products_cloudsync' AND schemaname = 'test_schema' \gset + +SELECT COUNT(*) AS users_meta FROM pg_tables +WHERE tablename = 'users_cloudsync' AND schemaname = 'public' \gset + +SELECT COUNT(*) AS orders_meta FROM pg_tables +WHERE tablename = 'orders_cloudsync' AND schemaname = 'schema_a' \gset + +SELECT (:products_meta = 1 AND :users_meta = 1 AND :orders_meta = 1) AS metadata_locations_ok \gset +\if :metadata_locations_ok +\echo [PASS] (:testid) Metadata Table Location Detection +\else +\echo [FAIL] (:testid) Metadata Table Location Detection - products: :products_meta, users: :users_meta, orders: :orders_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 8: Schema-Qualified Queries Work Correctly +-- ============================================================================ + +-- Insert data into different schemas and verify they're independent +INSERT INTO test_schema.products VALUES (cloudsync_uuid()::text, 'Product A'); +INSERT INTO public.users VALUES (cloudsync_uuid()::text, 'User A'); +INSERT INTO schema_a.orders VALUES (cloudsync_uuid()::text, '50.00'); + +-- Count rows in each table +SELECT COUNT(*) AS products_count FROM test_schema.products \gset +SELECT COUNT(*) AS users_count FROM public.users \gset +SELECT COUNT(*) AS orders_count FROM schema_a.orders \gset + +-- All should have at least one row +SELECT (:products_count > 0 AND :users_count > 0 AND :orders_count > 0) AS qualified_queries_ok \gset +\if :qualified_queries_ok +\echo [PASS] (:testid) Schema-Qualified Queries Work Correctly +\else +\echo [FAIL] (:testid) Schema-Qualified Queries Work Correctly - products: :products_count, users: :users_count, orders: :orders_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_13; +\endif diff --git a/test/postgresql/14_datatype_roundtrip.sql b/test/postgresql/14_datatype_roundtrip.sql new file mode 100644 index 0000000..a139310 --- /dev/null +++ b/test/postgresql/14_datatype_roundtrip.sql @@ -0,0 +1,404 @@ +-- DBTYPE Roundtrip Test +-- Tests encoding/decoding of all DBTYPEs (INTEGER, FLOAT, TEXT, BLOB, NULL) +-- in the internal value representation during database synchronization + +\set testid '14' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_14a; +DROP DATABASE IF EXISTS cloudsync_test_14b; +CREATE DATABASE cloudsync_test_14a; +CREATE DATABASE cloudsync_test_14b; + +-- ============================================================================ +-- Setup Database A with comprehensive table +-- ============================================================================ + +\connect cloudsync_test_14a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with composite primary key and all data types +CREATE TABLE all_types ( + -- Composite primary key (TEXT columns as required by CloudSync) + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + + -- INTEGER columns + col_int_notnull INTEGER NOT NULL DEFAULT 0, + col_int_nullable INTEGER, + + -- FLOAT columns (using DOUBLE PRECISION in PostgreSQL) + col_float_notnull DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_float_nullable DOUBLE PRECISION, + + -- TEXT columns + col_text_notnull TEXT NOT NULL DEFAULT '', + col_text_nullable TEXT, + + -- BLOB columns (BYTEA in PostgreSQL) + col_blob_notnull BYTEA NOT NULL DEFAULT E'\\x00', + col_blob_nullable BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('all_types', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various values for each type +-- ============================================================================ + +-- Row 1: All non-null values +INSERT INTO all_types VALUES ( + 'pk1', 'pk2', + -- INTEGER + 42, 100, + -- FLOAT + 3.14159, 2.71828, + -- TEXT + 'hello world', 'test string', + -- BLOB + E'\\xDEADBEEF', E'\\xCAFEBABE' +); + +-- Row 2: Mix of null and non-null +INSERT INTO all_types (id1, id2, col_int_notnull, col_float_notnull, col_text_notnull, col_blob_notnull) +VALUES ( + 'pk3', 'pk4', + -999, + -123.456, + 'only required fields', + E'\\x0102030405' +); + +-- Row 3: Edge cases - zeros, empty strings, single byte blob +INSERT INTO all_types VALUES ( + 'pk5', 'pk6', + 0, 0, + 0.0, 0.0, + '', '', + E'\\x00', E'\\x00' +); + +-- Row 4: Large values +INSERT INTO all_types VALUES ( + 'pk7', 'pk8', + 2147483647, -2147483648, -- INT max and min + 1.7976931348623157e+308, -1.7976931348623157e+308, -- Large floats + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ' || + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + 'Another long text with special chars: café, naïve, 日本語, emoji: 🚀', + E'\\xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', -- Large blob (16 bytes of 0xFF) + E'\\x0102030405060708090A0B0C0D0E0F10' -- Sequential bytes +); + +-- Row 5: Special characters in text +INSERT INTO all_types VALUES ( + 'pk9', 'pk10', + 1, 2, + 1.5, 2.5, + E'Special\nchars:\t\r\nand\\backslash', -- Escaped characters + E'Quote''s and "double" quotes', + E'\\x5C00', -- Backslash byte and null byte + E'\\x0D0A' -- CR LF +); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_int_notnull::text, 'NULL') || ':' || + COALESCE(col_int_nullable::text, 'NULL') || ':' || + COALESCE(col_float_notnull::text, 'NULL') || ':' || + COALESCE(col_float_nullable::text, 'NULL') || ':' || + COALESCE(col_text_notnull, 'NULL') || ':' || + COALESCE(col_text_nullable, 'NULL') || ':' || + COALESCE(encode(col_blob_notnull, 'hex'), 'NULL') || ':' || + COALESCE(encode(col_blob_nullable, 'hex'), 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_a FROM all_types \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_14b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE all_types ( + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + col_int_notnull INTEGER NOT NULL DEFAULT 0, + col_int_nullable INTEGER, + col_float_notnull DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_float_nullable DOUBLE PRECISION, + col_text_notnull TEXT NOT NULL DEFAULT '', + col_text_nullable TEXT, + col_blob_notnull BYTEA NOT NULL DEFAULT E'\\x00', + col_blob_nullable BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('all_types', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_int_notnull::text, 'NULL') || ':' || + COALESCE(col_int_nullable::text, 'NULL') || ':' || + COALESCE(col_float_notnull::text, 'NULL') || ':' || + COALESCE(col_float_nullable::text, 'NULL') || ':' || + COALESCE(col_text_notnull, 'NULL') || ':' || + COALESCE(col_text_nullable, 'NULL') || ':' || + COALESCE(encode(col_blob_notnull, 'hex'), 'NULL') || ':' || + COALESCE(encode(col_blob_nullable, 'hex'), 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_b FROM all_types \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_a FROM all_types \gset +\connect cloudsync_test_14a +SELECT COUNT(*) AS count_a_orig FROM all_types \gset + +\connect cloudsync_test_14b +SELECT (:count_a = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_a rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific data type preservation +-- ============================================================================ + +-- Verify INTEGER values +SELECT + (SELECT col_int_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 42 AND + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 100 AND + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS integers_ok \gset +\if :integers_ok +\echo [PASS] (:testid) INTEGER type preservation +\else +\echo [FAIL] (:testid) INTEGER type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify FLOAT values (with tolerance for floating point) +SELECT + ABS((SELECT col_float_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') - 3.14159) < 0.00001 AND + ABS((SELECT col_float_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') - 2.71828) < 0.00001 AND + (SELECT col_float_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS floats_ok \gset +\if :floats_ok +\echo [PASS] (:testid) FLOAT type preservation +\else +\echo [FAIL] (:testid) FLOAT type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify TEXT values +SELECT + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 'hello world' AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2') = 'test string' AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = '' +AS text_ok \gset +\if :text_ok +\echo [PASS] (:testid) TEXT type preservation +\else +\echo [FAIL] (:testid) TEXT type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify BLOB values +SELECT + encode((SELECT col_blob_notnull FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2'), 'hex') = 'deadbeef' AND + encode((SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk1' AND id2 = 'pk2'), 'hex') = 'cafebabe' AND + (SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS blobs_ok \gset +\if :blobs_ok +\echo [PASS] (:testid) BLOB type preservation +\else +\echo [FAIL] (:testid) BLOB type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NULL handling +SELECT + (SELECT col_int_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_float_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_text_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL AND + (SELECT col_blob_nullable FROM all_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS nulls_ok \gset +\if :nulls_ok +\echo [PASS] (:testid) NULL type preservation +\else +\echo [FAIL] (:testid) NULL type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test special characters and edge cases +-- ============================================================================ + +-- Verify special characters in TEXT +SELECT + (SELECT col_text_notnull FROM all_types WHERE id1 = 'pk9' AND id2 = 'pk10') = E'Special\nchars:\t\r\nand\\backslash' +AS special_chars_ok \gset +\if :special_chars_ok +\echo [PASS] (:testid) Special characters in TEXT preserved +\else +\echo [FAIL] (:testid) Special characters in TEXT not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify zero values +SELECT + (SELECT col_int_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = 0 AND + (SELECT col_float_notnull FROM all_types WHERE id1 = 'pk5' AND id2 = 'pk6') = 0.0 +AS zero_values_ok \gset +\if :zero_values_ok +\echo [PASS] (:testid) Zero values preserved +\else +\echo [FAIL] (:testid) Zero values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +-- Verify all primary key combinations are present +SELECT COUNT(DISTINCT (id1, id2)) = 5 AS pk_count_ok FROM all_types \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_14b + +-- Add a new row in Database B +INSERT INTO all_types VALUES ( + 'pkB1', 'pkB2', + 999, 888, + 9.99, 8.88, + 'from database B', 'bidirectional test', + E'\\xBBBBBBBB', E'\\xAAAAAAAA' +); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_14a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM all_types +WHERE id1 = 'pkB1' AND id2 = 'pkB2' AND col_text_notnull = 'from database B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_14a; +DROP DATABASE IF EXISTS cloudsync_test_14b; +\endif diff --git a/test/postgresql/15_datatype_roundtrip_unmapped.sql b/test/postgresql/15_datatype_roundtrip_unmapped.sql new file mode 100644 index 0000000..cfc2fcf --- /dev/null +++ b/test/postgresql/15_datatype_roundtrip_unmapped.sql @@ -0,0 +1,388 @@ +-- DBTYPE Roundtrip Test (Unmapped PostgreSQL Types) +-- Tests encoding/decoding for types that are not explicitly mapped to +-- DBTYPE_INTEGER/FLOAT/TEXT/BLOB/NULL in the common layer. + +\set testid '15' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_15a; +DROP DATABASE IF EXISTS cloudsync_test_15b; +CREATE DATABASE cloudsync_test_15a; +CREATE DATABASE cloudsync_test_15b; + +-- ============================================================================ +-- Setup Database A with unmapped types table +-- ============================================================================ + +\connect cloudsync_test_15a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with composite primary key and unmapped types +CREATE TABLE unmapped_types ( + -- Composite primary key (TEXT columns as required by CloudSync) + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + + -- JSONB columns + col_jsonb_notnull JSONB NOT NULL DEFAULT '{}'::jsonb, + col_jsonb_nullable JSONB, + + -- UUID columns + col_uuid_notnull UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + col_uuid_nullable UUID, + + -- INET columns + col_inet_notnull INET NOT NULL DEFAULT '0.0.0.0', + col_inet_nullable INET, + + -- CIDR columns + col_cidr_notnull CIDR NOT NULL DEFAULT '0.0.0.0/0', + col_cidr_nullable CIDR, + + -- RANGE columns + col_int4range_notnull INT4RANGE NOT NULL DEFAULT 'empty'::int4range, + col_int4range_nullable INT4RANGE +); + +-- Initialize CloudSync +SELECT cloudsync_init('unmapped_types', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various values for each type +-- ============================================================================ + +-- Row 1: All non-null values +INSERT INTO unmapped_types VALUES ( + 'pk1', 'pk2', + '{"a":1,"b":[1,2]}'::jsonb, '{"k":"v"}'::jsonb, + '11111111-1111-1111-1111-111111111111', '22222222-2222-2222-2222-222222222222', + '192.168.1.10', '10.1.2.3', + '10.0.0.0/24', '192.168.0.0/16', + '[1,10)', '[20,30)' +); + +-- Row 2: Mix of null and non-null +INSERT INTO unmapped_types ( + id1, id2, + col_jsonb_notnull, + col_uuid_notnull, + col_inet_notnull, + col_cidr_notnull, + col_int4range_notnull +) VALUES ( + 'pk3', 'pk4', + '{"only":"required"}'::jsonb, + '33333333-3333-3333-3333-333333333333', + '127.0.0.1', + '127.0.0.0/8', + '[0,1)' +); + +-- Row 3: Edge cases - empty JSON, empty range +INSERT INTO unmapped_types VALUES ( + 'pk5', 'pk6', + '{}'::jsonb, '[]'::jsonb, + '44444444-4444-4444-4444-444444444444', '55555555-5555-5555-5555-555555555555', + '0.0.0.0', '255.255.255.255', + '0.0.0.0/0', '255.255.255.0/24', + 'empty'::int4range, 'empty'::int4range +); + +-- Row 4: IPv6 + negative range +INSERT INTO unmapped_types VALUES ( + 'pk7', 'pk8', + '{"ipv6":true}'::jsonb, '{"note":"range"}'::jsonb, + '66666666-6666-6666-6666-666666666666', '77777777-7777-7777-7777-777777777777', + '2001:db8::1', '2001:db8::2', + '2001:db8::/32', '2001:db8:abcd::/48', + '[-5,5]', '[-10,10)' +); + +-- Row 5: Nested JSON +INSERT INTO unmapped_types VALUES ( + 'pk9', 'pk10', + '{"obj":{"x":1,"y":[2,3]}}'::jsonb, '{"arr":[{"a":1},{"b":2}]}'::jsonb, + '88888888-8888-8888-8888-888888888888', '99999999-9999-9999-9999-999999999999', + '172.16.0.1', '172.16.0.2', + '172.16.0.0/12', '172.16.1.0/24', + '[100,200)', '[200,300)' +); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_jsonb_notnull::text, 'NULL') || ':' || + COALESCE(col_jsonb_nullable::text, 'NULL') || ':' || + COALESCE(col_uuid_notnull::text, 'NULL') || ':' || + COALESCE(col_uuid_nullable::text, 'NULL') || ':' || + COALESCE(col_inet_notnull::text, 'NULL') || ':' || + COALESCE(col_inet_nullable::text, 'NULL') || ':' || + COALESCE(col_cidr_notnull::text, 'NULL') || ':' || + COALESCE(col_cidr_nullable::text, 'NULL') || ':' || + COALESCE(col_int4range_notnull::text, 'NULL') || ':' || + COALESCE(col_int4range_nullable::text, 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_a FROM unmapped_types \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_15b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE unmapped_types ( + id1 TEXT NOT NULL, + id2 TEXT NOT NULL, + PRIMARY KEY (id1, id2), + col_jsonb_notnull JSONB NOT NULL DEFAULT '{}'::jsonb, + col_jsonb_nullable JSONB, + col_uuid_notnull UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + col_uuid_nullable UUID, + col_inet_notnull INET NOT NULL DEFAULT '0.0.0.0', + col_inet_nullable INET, + col_cidr_notnull CIDR NOT NULL DEFAULT '0.0.0.0/0', + col_cidr_nullable CIDR, + col_int4range_notnull INT4RANGE NOT NULL DEFAULT 'empty'::int4range, + col_int4range_nullable INT4RANGE +); + +-- Initialize CloudSync +SELECT cloudsync_init('unmapped_types', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id1 || ':' || id2 || ':' || + COALESCE(col_jsonb_notnull::text, 'NULL') || ':' || + COALESCE(col_jsonb_nullable::text, 'NULL') || ':' || + COALESCE(col_uuid_notnull::text, 'NULL') || ':' || + COALESCE(col_uuid_nullable::text, 'NULL') || ':' || + COALESCE(col_inet_notnull::text, 'NULL') || ':' || + COALESCE(col_inet_nullable::text, 'NULL') || ':' || + COALESCE(col_cidr_notnull::text, 'NULL') || ':' || + COALESCE(col_cidr_nullable::text, 'NULL') || ':' || + COALESCE(col_int4range_notnull::text, 'NULL') || ':' || + COALESCE(col_int4range_nullable::text, 'NULL'), + '|' ORDER BY id1, id2 + ), + '' + ) +) AS hash_b FROM unmapped_types \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM unmapped_types \gset +\connect cloudsync_test_15a +SELECT COUNT(*) AS count_a_orig FROM unmapped_types \gset + +\connect cloudsync_test_15b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific data type preservation +-- ============================================================================ + +-- JSONB values +SELECT + (SELECT col_jsonb_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '{"a":1,"b":[1,2]}'::jsonb AND + (SELECT col_jsonb_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '{"k":"v"}'::jsonb AND + (SELECT col_jsonb_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS jsonb_ok \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB type preservation +\else +\echo [FAIL] (:testid) JSONB type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- UUID values +SELECT + (SELECT col_uuid_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '11111111-1111-1111-1111-111111111111'::uuid AND + (SELECT col_uuid_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '22222222-2222-2222-2222-222222222222'::uuid AND + (SELECT col_uuid_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS uuid_ok \gset +\if :uuid_ok +\echo [PASS] (:testid) UUID type preservation +\else +\echo [FAIL] (:testid) UUID type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- INET values +SELECT + (SELECT col_inet_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '192.168.1.10'::inet AND + (SELECT col_inet_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '10.1.2.3'::inet AND + (SELECT col_inet_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS inet_ok \gset +\if :inet_ok +\echo [PASS] (:testid) INET type preservation +\else +\echo [FAIL] (:testid) INET type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- CIDR values +SELECT + (SELECT col_cidr_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '10.0.0.0/24'::cidr AND + (SELECT col_cidr_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '192.168.0.0/16'::cidr AND + (SELECT col_cidr_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS cidr_ok \gset +\if :cidr_ok +\echo [PASS] (:testid) CIDR type preservation +\else +\echo [FAIL] (:testid) CIDR type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- RANGE values +SELECT + (SELECT col_int4range_notnull FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '[1,10)'::int4range AND + (SELECT col_int4range_nullable FROM unmapped_types WHERE id1 = 'pk1' AND id2 = 'pk2') = '[20,30)'::int4range AND + (SELECT col_int4range_nullable FROM unmapped_types WHERE id1 = 'pk3' AND id2 = 'pk4') IS NULL +AS ranges_ok \gset +\if :ranges_ok +\echo [PASS] (:testid) RANGE type preservation +\else +\echo [FAIL] (:testid) RANGE type preservation +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +-- Verify all primary key combinations are present +SELECT COUNT(DISTINCT (id1, id2)) = 5 AS pk_count_ok FROM unmapped_types \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_15b + +-- Add a new row in Database B +INSERT INTO unmapped_types VALUES ( + 'pkB1', 'pkB2', + '{"from":"database B"}'::jsonb, '{"bidirectional":true}'::jsonb, + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', + '10.10.10.10', '10.10.10.11', + '10.10.0.0/16', '10.10.10.0/24', + '[50,60)', '[60,70)' +); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_15a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM unmapped_types +WHERE id1 = 'pkB1' AND id2 = 'pkB2' AND col_jsonb_notnull = '{"from":"database B"}'::jsonb \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_15a; +DROP DATABASE IF EXISTS cloudsync_test_15b; +\endif diff --git a/test/postgresql/16_composite_pk_text_int_roundtrip.sql b/test/postgresql/16_composite_pk_text_int_roundtrip.sql new file mode 100644 index 0000000..c2fcb8e --- /dev/null +++ b/test/postgresql/16_composite_pk_text_int_roundtrip.sql @@ -0,0 +1,217 @@ +-- Composite PK Roundtrip Test (TEXT + INTEGER) +-- Tests roundtrip with a composite primary key that mixes TEXT and INTEGER. + +\set testid '16' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_16a; +DROP DATABASE IF EXISTS cloudsync_test_16b; +CREATE DATABASE cloudsync_test_16a; +CREATE DATABASE cloudsync_test_16b; + +-- ============================================================================ +-- Setup Database A with composite PK (TEXT + INTEGER) +-- ============================================================================ + +\connect cloudsync_test_16a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE mixed_pk ( + id_text TEXT NOT NULL, + id_int INTEGER NOT NULL, + PRIMARY KEY (id_text, id_int), + + col_text TEXT NOT NULL DEFAULT '', + col_int INTEGER NOT NULL DEFAULT 0, + col_float DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_blob BYTEA +); + +-- Initialize CloudSync (skip int pk check for this test) +SELECT cloudsync_init('mixed_pk', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data +-- ============================================================================ + +INSERT INTO mixed_pk VALUES ('pkA', 1, 'hello', 42, 3.14, E'\\xDEADBEEF'); +INSERT INTO mixed_pk VALUES ('pkA', 2, 'world', 7, 2.71, NULL); +INSERT INTO mixed_pk VALUES ('pkB', 1, '', 0, 0.0, E'\\x00'); +INSERT INTO mixed_pk VALUES ('pkC', 10, 'edge', -1, -1.5, E'\\xCAFEBABE'); +INSERT INTO mixed_pk VALUES ('pkD', 20, 'more', 999, 123.456, E'\\x010203'); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id_text || ':' || id_int::text || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_int::text, 'NULL') || ':' || + COALESCE(col_float::text, 'NULL') || ':' || + COALESCE(encode(col_blob, 'hex'), 'NULL'), + '|' ORDER BY id_text, id_int + ), + '' + ) +) AS hash_a FROM mixed_pk \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_16b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE mixed_pk ( + id_text TEXT NOT NULL, + id_int INTEGER NOT NULL, + PRIMARY KEY (id_text, id_int), + col_text TEXT NOT NULL DEFAULT '', + col_int INTEGER NOT NULL DEFAULT 0, + col_float DOUBLE PRECISION NOT NULL DEFAULT 0.0, + col_blob BYTEA +); + +-- Initialize CloudSync (skip int pk check for this test) +SELECT cloudsync_init('mixed_pk', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id_text || ':' || id_int::text || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_int::text, 'NULL') || ':' || + COALESCE(col_float::text, 'NULL') || ':' || + COALESCE(encode(col_blob, 'hex'), 'NULL'), + '|' ORDER BY id_text, id_int + ), + '' + ) +) AS hash_b FROM mixed_pk \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM mixed_pk \gset +\connect cloudsync_test_16a +SELECT COUNT(*) AS count_a_orig FROM mixed_pk \gset + +\connect cloudsync_test_16b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test composite primary key encoding +-- ============================================================================ + +SELECT COUNT(DISTINCT (id_text, id_int)) = 5 AS pk_count_ok FROM mixed_pk \gset +\if :pk_count_ok +\echo [PASS] (:testid) Composite primary keys preserved +\else +\echo [FAIL] (:testid) Composite primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_16b + +INSERT INTO mixed_pk VALUES ('pkB', 99, 'from B', 123, 9.99, E'\\xBEEF'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_16a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM mixed_pk +WHERE id_text = 'pkB' AND id_int = 99 AND col_text = 'from B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_16a; +DROP DATABASE IF EXISTS cloudsync_test_16b; +\endif diff --git a/test/postgresql/17_uuid_pk_roundtrip.sql b/test/postgresql/17_uuid_pk_roundtrip.sql new file mode 100644 index 0000000..ceb7952 --- /dev/null +++ b/test/postgresql/17_uuid_pk_roundtrip.sql @@ -0,0 +1,230 @@ +-- UUID Primary Key Roundtrip Test +-- Tests roundtrip with a UUID primary key (single column). + +\set testid '17' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_17a; +DROP DATABASE IF EXISTS cloudsync_test_17b; +CREATE DATABASE cloudsync_test_17a; +CREATE DATABASE cloudsync_test_17b; + +-- ============================================================================ +-- Setup Database A with UUID primary key +-- ============================================================================ + +\connect cloudsync_test_17a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + stock INTEGER NOT NULL DEFAULT 0, + metadata BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('products', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with UUIDs +-- ============================================================================ + +INSERT INTO products VALUES ('550e8400-e29b-41d4-a716-446655440000', 'Product A', 99.99, 100, E'\\xDEADBEEF'); +INSERT INTO products VALUES ('6ba7b810-9dad-11d1-80b4-00c04fd430c8', 'Product B', 49.50, 50, NULL); +INSERT INTO products VALUES ('6ba7b811-9dad-11d1-80b4-00c04fd430c8', 'Product C', 0.0, 0, E'\\x00'); +INSERT INTO products VALUES ('6ba7b812-9dad-11d1-80b4-00c04fd430c8', 'Product D', 123.45, 999, E'\\xCAFEBABE'); +INSERT INTO products VALUES ('6ba7b813-9dad-11d1-80b4-00c04fd430c8', '', -1.0, -1, E'\\x010203'); + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(stock::text, 'NULL') || ':' || + COALESCE(encode(metadata, 'hex'), 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM products \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_17b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + stock INTEGER NOT NULL DEFAULT 0, + metadata BYTEA +); + +-- Initialize CloudSync +SELECT cloudsync_init('products', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(stock::text, 'NULL') || ':' || + COALESCE(encode(metadata, 'hex'), 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM products \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM products \gset +\connect cloudsync_test_17a +SELECT COUNT(*) AS count_a_orig FROM products \gset + +\connect cloudsync_test_17b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify UUID primary keys preserved +-- ============================================================================ + +SELECT COUNT(DISTINCT id) = 5 AS uuid_count_ok FROM products \gset +\if :uuid_count_ok +\echo [PASS] (:testid) UUID primary keys preserved +\else +\echo [FAIL] (:testid) UUID primary keys not all preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test specific UUID values +-- ============================================================================ + +SELECT COUNT(*) = 1 AS uuid_test_ok +FROM products +WHERE id = '550e8400-e29b-41d4-a716-446655440000' + AND name = 'Product A' + AND price = 99.99 \gset +\if :uuid_test_ok +\echo [PASS] (:testid) Specific UUID record verified +\else +\echo [FAIL] (:testid) Specific UUID record not found or incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +\connect cloudsync_test_17b + +INSERT INTO products VALUES ('7ba7b814-9dad-11d1-80b4-00c04fd430c8', 'From B', 77.77, 777, E'\\xBEEF'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_17a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM products +WHERE id = '7ba7b814-9dad-11d1-80b4-00c04fd430c8' + AND name = 'From B' + AND price = 77.77 \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_17a; +DROP DATABASE IF EXISTS cloudsync_test_17b; +\endif diff --git a/test/postgresql/18_bulk_insert_performance.sql b/test/postgresql/18_bulk_insert_performance.sql new file mode 100644 index 0000000..8913221 --- /dev/null +++ b/test/postgresql/18_bulk_insert_performance.sql @@ -0,0 +1,244 @@ +-- Bulk Insert Performance Roundtrip Test +-- Tests roundtrip with 1000 rows and measures time for each operation. + +\set testid '18' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_18a; +DROP DATABASE IF EXISTS cloudsync_test_18b; +CREATE DATABASE cloudsync_test_18a; +CREATE DATABASE cloudsync_test_18b; + +-- ============================================================================ +-- Setup Database A +-- ============================================================================ + +\connect cloudsync_test_18a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID NOT NULL PRIMARY KEY DEFAULT cloudsync_uuid(), + name TEXT NOT NULL DEFAULT '', + value DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + description TEXT +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Record start time +-- ============================================================================ + +SELECT clock_timestamp() AS test_start_time \gset +\echo [INFO] (:testid) Test started at :test_start_time + +-- ============================================================================ +-- Insert 1000 rows and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS insert_start_time \gset + +INSERT INTO items (name, value, quantity, description) +SELECT + 'Item ' || i, + (random() * 1000)::DOUBLE PRECISION, + (random() * 100)::INTEGER, + 'Description for item ' || i || ' with some additional text to simulate real data' +FROM generate_series(1, 1000) AS i; + +SELECT clock_timestamp() AS insert_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'insert_end_time'::timestamp - :'insert_start_time'::timestamp)) * 1000 AS insert_time_ms \gset +\echo [INFO] (:testid) Insert 1000 rows: :insert_time_ms ms + +-- ============================================================================ +-- Verify row count in Database A +-- ============================================================================ + +SELECT COUNT(*) AS count_a FROM items \gset +SELECT (:count_a = 1000) AS insert_count_ok \gset +\if :insert_count_ok +\echo [PASS] (:testid) Inserted 1000 rows successfully +\else +\echo [FAIL] (:testid) Expected 1000 rows, got :count_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM items \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS encode_start_time \gset + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT clock_timestamp() AS encode_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'encode_end_time'::timestamp - :'encode_start_time'::timestamp)) * 1000 AS encode_time_ms \gset +\echo [INFO] (:testid) Encode payload: :encode_time_ms ms + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Report payload size +SELECT length(:'payload_a_hex') / 2 AS payload_size_bytes \gset +\echo [INFO] (:testid) Payload size: :payload_size_bytes bytes + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_18b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID NOT NULL PRIMARY KEY DEFAULT cloudsync_uuid(), + name TEXT NOT NULL DEFAULT '', + value DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + description TEXT +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B and measure time +-- ============================================================================ + +SELECT clock_timestamp() AS apply_start_time \gset + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +SELECT clock_timestamp() AS apply_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'apply_end_time'::timestamp - :'apply_start_time'::timestamp)) * 1000 AS apply_time_ms \gset +\echo [INFO] (:testid) Apply payload: :apply_time_ms ms + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM items \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM items \gset + +SELECT (:count_b = :count_a) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify sample data integrity +-- ============================================================================ + +SELECT COUNT(*) = 1 AS sample_check_ok +FROM items +WHERE name = 'Item 500' \gset +\if :sample_check_ok +\echo [PASS] (:testid) Sample row verified (name='Item 500') +\else +\echo [FAIL] (:testid) Sample row verification failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Calculate and report total elapsed time +-- ============================================================================ + +SELECT clock_timestamp() AS test_end_time \gset + +SELECT EXTRACT(EPOCH FROM (:'test_end_time'::timestamp - :'test_start_time'::timestamp)) * 1000 AS total_time_ms \gset + +\echo [INFO] (:testid) Performance summary: +\echo [INFO] (:testid) - Insert 1000 rows: :insert_time_ms ms +\echo [INFO] (:testid) - Encode payload: :encode_time_ms ms +\echo [INFO] (:testid) - Apply payload: :apply_time_ms ms +\echo [INFO] (:testid) - Total elapsed time: :total_time_ms ms + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_18a; +DROP DATABASE IF EXISTS cloudsync_test_18b; +\endif diff --git a/test/postgresql/19_uuid_pk_with_unmapped_cols.sql b/test/postgresql/19_uuid_pk_with_unmapped_cols.sql new file mode 100644 index 0000000..9111c8e --- /dev/null +++ b/test/postgresql/19_uuid_pk_with_unmapped_cols.sql @@ -0,0 +1,681 @@ +-- UUID PK with Unmapped Non-PK Columns Test +-- Tests comprehensive CRUD operations with UUID primary key and unmapped PostgreSQL types. +-- Covers: INSERT, UPDATE non-PK, UPDATE mapped cols, UPDATE PK, DELETE, RESURRECT, bidirectional sync. + +\set testid '19' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_19a; +DROP DATABASE IF EXISTS cloudsync_test_19b; +CREATE DATABASE cloudsync_test_19a; +CREATE DATABASE cloudsync_test_19b; + +-- ============================================================================ +-- Setup Database A with UUID PK and unmapped non-PK columns +-- ============================================================================ + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + related_id UUID, + ip_address INET NOT NULL DEFAULT '0.0.0.0', + network CIDR, + name TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 0 +); + +-- Initialize CloudSync +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- ROUND 1: Initial INSERT (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 1: Initial INSERT (A -> B) === + +INSERT INTO items VALUES ( + '11111111-1111-1111-1111-111111111111', + '{"type":"widget","tags":["a","b"]}'::jsonb, + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + '192.168.1.1', + '192.168.0.0/16', + 'Widget One', + 100 +); + +INSERT INTO items VALUES ( + '22222222-2222-2222-2222-222222222222', + '{"type":"gadget"}'::jsonb, + NULL, + '10.0.0.1', + '10.0.0.0/8', + 'Gadget Two', + 200 +); + +INSERT INTO items VALUES ( + '33333333-3333-3333-3333-333333333333', + '{}'::jsonb, + 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', + '127.0.0.1', + NULL, + '', + 0 +); + +-- Compute hash for round 1 +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r1 FROM items \gset + +\echo [INFO] (:testid) Round 1 - Database A hash: :hash_a_r1 + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup Database B +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + related_id UUID, + ip_address INET NOT NULL DEFAULT '0.0.0.0', + network CIDR, + name TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- Apply round 1 payload +SELECT cloudsync_payload_apply(decode(:'payload_a_r1', 'hex')) AS apply_r1 \gset + +SELECT (:apply_r1 >= 0) AS r1_applied \gset +\if :r1_applied +\echo [PASS] (:testid) Round 1 payload applied +\else +\echo [FAIL] (:testid) Round 1 payload apply failed: :apply_r1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify hash match +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r1 FROM items \gset + +SELECT (:'hash_a_r1' = :'hash_b_r1') AS r1_match \gset +\if :r1_match +\echo [PASS] (:testid) Round 1 hashes match +\else +\echo [FAIL] (:testid) Round 1 hash mismatch: A=:hash_a_r1 B=:hash_b_r1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 2: UPDATE unmapped non-PK columns (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 2: UPDATE unmapped non-PK columns (A -> B) === + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +-- Update JSONB +UPDATE items SET metadata = '{"type":"widget","tags":["a","b","c"],"updated":true}'::jsonb +WHERE id = '11111111-1111-1111-1111-111111111111'; + +-- Update UUID non-PK +UPDATE items SET related_id = 'cccccccc-cccc-cccc-cccc-cccccccccccc' +WHERE id = '22222222-2222-2222-2222-222222222222'; + +-- Update INET +UPDATE items SET ip_address = '172.16.0.1' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Update CIDR +UPDATE items SET network = '172.16.0.0/12' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r2 FROM items \gset + +\echo [INFO] (:testid) Round 2 - Database A hash: :hash_a_r2 + +-- Sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r2', 'hex')) AS apply_r2 \gset + +SELECT (:apply_r2 >= 0) AS r2_applied \gset +\if :r2_applied +\echo [PASS] (:testid) Round 2 payload applied +\else +\echo [FAIL] (:testid) Round 2 payload apply failed: :apply_r2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r2 FROM items \gset + +SELECT (:'hash_a_r2' = :'hash_b_r2') AS r2_match \gset +\if :r2_match +\echo [PASS] (:testid) Round 2 hashes match (unmapped cols updated) +\else +\echo [FAIL] (:testid) Round 2 hash mismatch: A=:hash_a_r2 B=:hash_b_r2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 3: UPDATE mapped columns (B -> A) - Bidirectional +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 3: UPDATE mapped columns (B -> A) === + +-- Update TEXT and INTEGER columns in B +UPDATE items SET name = 'Updated Widget', count = 150 +WHERE id = '11111111-1111-1111-1111-111111111111'; + +UPDATE items SET name = 'Updated Gadget', count = 250 +WHERE id = '22222222-2222-2222-2222-222222222222'; + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r3 FROM items \gset + +\echo [INFO] (:testid) Round 3 - Database B hash: :hash_b_r3 + +-- Sync B -> A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_r3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_b_r3', 'hex')) AS apply_r3 \gset + +SELECT (:apply_r3 >= 0) AS r3_applied \gset +\if :r3_applied +\echo [PASS] (:testid) Round 3 payload applied (B -> A) +\else +\echo [FAIL] (:testid) Round 3 payload apply failed: :apply_r3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r3 FROM items \gset + +SELECT (:'hash_a_r3' = :'hash_b_r3') AS r3_match \gset +\if :r3_match +\echo [PASS] (:testid) Round 3 hashes match (mapped cols updated B->A) +\else +\echo [FAIL] (:testid) Round 3 hash mismatch: A=:hash_a_r3 B=:hash_b_r3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 4: DELETE row (A -> B) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 4: DELETE row (A -> B) === + +DELETE FROM items WHERE id = '33333333-3333-3333-3333-333333333333'; + +SELECT COUNT(*) AS count_a_r4 FROM items \gset +\echo [INFO] (:testid) Round 4 - Database A row count: :count_a_r4 + +-- Sync deletion +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r4', 'hex')) AS apply_r4 \gset + +SELECT (:apply_r4 >= 0) AS r4_applied \gset +\if :r4_applied +\echo [PASS] (:testid) Round 4 payload applied +\else +\echo [FAIL] (:testid) Round 4 payload apply failed: :apply_r4 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS count_b_r4 FROM items \gset + +SELECT (:count_a_r4 = :count_b_r4) AS r4_count_match \gset +\if :r4_count_match +\echo [PASS] (:testid) Round 4 row counts match after DELETE (:count_b_r4 rows) +\else +\echo [FAIL] (:testid) Round 4 row count mismatch: A=:count_a_r4 B=:count_b_r4 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify deleted row is gone +SELECT COUNT(*) = 0 AS deleted_row_gone +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset +\if :deleted_row_gone +\echo [PASS] (:testid) Deleted row not present in Database B +\else +\echo [FAIL] (:testid) Deleted row still exists in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 5: RESURRECT row (B -> A) +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 5: RESURRECT row (B -> A) === + +-- Re-insert the deleted row with different values +INSERT INTO items VALUES ( + '33333333-3333-3333-3333-333333333333', + '{"resurrected":true}'::jsonb, + 'dddddddd-dddd-dddd-dddd-dddddddddddd', + '8.8.8.8', + '8.8.0.0/16', + 'Resurrected Item', + 999 +); + +SELECT COUNT(*) AS count_b_r5 FROM items \gset +\echo [INFO] (:testid) Round 5 - Database B row count: :count_b_r5 + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r5 FROM items \gset + +\echo [INFO] (:testid) Round 5 - Database B hash: :hash_b_r5 + +-- Sync resurrection B -> A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_r5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_b_r5', 'hex')) AS apply_r5 \gset + +SELECT (:apply_r5 >= 0) AS r5_applied \gset +\if :r5_applied +\echo [PASS] (:testid) Round 5 payload applied (resurrection) +\else +\echo [FAIL] (:testid) Round 5 payload apply failed: :apply_r5 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r5 FROM items \gset + +SELECT (:'hash_a_r5' = :'hash_b_r5') AS r5_match \gset +\if :r5_match +\echo [PASS] (:testid) Round 5 hashes match (row resurrected) +\else +\echo [FAIL] (:testid) Round 5 hash mismatch: A=:hash_a_r5 B=:hash_b_r5 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify resurrected row exists with correct values +SELECT COUNT(*) = 1 AS resurrected_ok +FROM items +WHERE id = '33333333-3333-3333-3333-333333333333' + AND metadata = '{"resurrected":true}'::jsonb + AND name = 'Resurrected Item' + AND count = 999 \gset +\if :resurrected_ok +\echo [PASS] (:testid) Resurrected row verified with correct values +\else +\echo [FAIL] (:testid) Resurrected row has incorrect values +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 6: UPDATE primary key (A -> B) +-- Note: PK update = DELETE old row + INSERT new row in CRDT systems +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 6: UPDATE primary key (A -> B) === + +-- Change the UUID PK of the resurrected row +-- This should result in old PK being deleted and new PK being inserted +UPDATE items SET id = '55555555-5555-5555-5555-555555555555' +WHERE id = '33333333-3333-3333-3333-333333333333'; + +SELECT COUNT(*) AS count_a_r6 FROM items \gset +\echo [INFO] (:testid) Round 6 - Database A row count after PK update: :count_a_r6 + +-- Verify old PK is gone and new PK exists in A +SELECT COUNT(*) = 0 AS old_pk_gone_a +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset + +SELECT COUNT(*) = 1 AS new_pk_exists_a +FROM items WHERE id = '55555555-5555-5555-5555-555555555555' \gset + +-- Compute hash +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r6 FROM items \gset + +\echo [INFO] (:testid) Round 6 - Database A hash: :hash_a_r6 + +-- Sync PK update A -> B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r6', 'hex')) AS apply_r6 \gset + +SELECT (:apply_r6 >= 0) AS r6_applied \gset +\if :r6_applied +\echo [PASS] (:testid) Round 6 payload applied (PK update) +\else +\echo [FAIL] (:testid) Round 6 payload apply failed: :apply_r6 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify old PK is gone in B +SELECT COUNT(*) = 0 AS old_pk_gone_b +FROM items WHERE id = '33333333-3333-3333-3333-333333333333' \gset +\if :old_pk_gone_b +\echo [PASS] (:testid) Old UUID PK removed from Database B +\else +\echo [FAIL] (:testid) Old UUID PK still exists in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify new PK exists in B with correct data +SELECT COUNT(*) = 1 AS new_pk_exists_b +FROM items +WHERE id = '55555555-5555-5555-5555-555555555555' + AND metadata = '{"resurrected":true}'::jsonb + AND name = 'Resurrected Item' + AND count = 999 \gset +\if :new_pk_exists_b +\echo [PASS] (:testid) New UUID PK exists with correct data in Database B +\else +\echo [FAIL] (:testid) New UUID PK missing or has incorrect data in Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify hash match +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r6 FROM items \gset + +SELECT (:'hash_a_r6' = :'hash_b_r6') AS r6_match \gset +\if :r6_match +\echo [PASS] (:testid) Round 6 hashes match (PK updated) +\else +\echo [FAIL] (:testid) Round 6 hash mismatch: A=:hash_a_r6 B=:hash_b_r6 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ROUND 7: INSERT new row (A -> B) - Final verification +-- ============================================================================ + +\echo [INFO] (:testid) === ROUND 7: INSERT new row (A -> B) === + +\connect cloudsync_test_19a +\ir helper_psql_conn_setup.sql + +INSERT INTO items VALUES ( + '44444444-4444-4444-4444-444444444444', + '{"final":"test"}'::jsonb, + 'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', + '1.1.1.1', + '1.0.0.0/8', + 'Final Item', + 444 +); + +-- Compute hash for round 7 +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_r7 FROM items \gset + +\echo [INFO] (:testid) Round 7 - Database A hash: :hash_a_r7 + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_r7 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_19b +\ir helper_psql_conn_setup.sql + +SELECT cloudsync_payload_apply(decode(:'payload_a_r7', 'hex')) AS apply_r7 \gset + +SELECT (:apply_r7 >= 0) AS r7_applied \gset +\if :r7_applied +\echo [PASS] (:testid) Round 7 payload applied +\else +\echo [FAIL] (:testid) Round 7 payload apply failed: :apply_r7 +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(metadata::text, 'NULL') || ':' || + COALESCE(related_id::text, 'NULL') || ':' || + COALESCE(ip_address::text, 'NULL') || ':' || + COALESCE(network::text, 'NULL') || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(count::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_r7 FROM items \gset + +SELECT (:'hash_a_r7' = :'hash_b_r7') AS r7_match \gset +\if :r7_match +\echo [PASS] (:testid) Round 7 hashes match - all CRUD operations verified +\else +\echo [FAIL] (:testid) Round 7 hash mismatch: A=:hash_a_r7 B=:hash_b_r7 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Final row count verification +SELECT COUNT(*) AS final_count FROM items \gset +SELECT (:final_count = 4) AS final_count_ok \gset +\if :final_count_ok +\echo [PASS] (:testid) Final row count correct (:final_count rows) +\else +\echo [FAIL] (:testid) Final row count incorrect: expected 4, got :final_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_19a; +DROP DATABASE IF EXISTS cloudsync_test_19b; +\endif diff --git a/test/postgresql/20_init_with_existing_data.sql b/test/postgresql/20_init_with_existing_data.sql new file mode 100644 index 0000000..57e249e --- /dev/null +++ b/test/postgresql/20_init_with_existing_data.sql @@ -0,0 +1,298 @@ +-- Init With Existing Data Test +-- Tests cloudsync_init on a table that already contains data. +-- This verifies that cloudsync_refill_metatable correctly creates +-- metadata entries for pre-existing rows. + +\set testid '20' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_20a; +DROP DATABASE IF EXISTS cloudsync_test_20b; +CREATE DATABASE cloudsync_test_20a; +CREATE DATABASE cloudsync_test_20b; + +-- ============================================================================ +-- Setup Database A - INSERT DATA BEFORE cloudsync_init +-- ============================================================================ + +\connect cloudsync_test_20a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with UUID primary key (required for CRDT replication) +CREATE TABLE items ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + metadata JSONB +); + +-- ============================================================================ +-- INSERT DATA BEFORE CALLING cloudsync_init +-- This is the key difference from other tests - data exists before sync setup +-- ============================================================================ + +INSERT INTO items VALUES ('11111111-1111-1111-1111-111111111111', 'Pre-existing Item 1', 10.99, 100, '{"pre": true}'); +INSERT INTO items VALUES ('22222222-2222-2222-2222-222222222222', 'Pre-existing Item 2', 20.50, 200, '{"pre": true, "id": 2}'); +INSERT INTO items VALUES ('33333333-3333-3333-3333-333333333333', 'Pre-existing Item 3', 30.00, 300, NULL); +INSERT INTO items VALUES ('44444444-4444-4444-4444-444444444444', 'Pre-existing Item 4', 0.0, 0, '[]'); +INSERT INTO items VALUES ('55555555-5555-5555-5555-555555555555', 'Pre-existing Item 5', -5.50, -10, '{"nested": {"key": "value"}}'); + +-- Verify data exists before init +SELECT COUNT(*) AS pre_init_count FROM items \gset +\echo [INFO] (:testid) Rows before cloudsync_init: :pre_init_count + +-- ============================================================================ +-- NOW call cloudsync_init - this should trigger cloudsync_refill_metatable +-- ============================================================================ + +SELECT cloudsync_init('items', 'CLS', 0) AS _init_a \gset + +-- ============================================================================ +-- Verify metadata was created for existing rows +-- ============================================================================ + +-- Check that metadata table exists and has entries +SELECT COUNT(*) AS metadata_count FROM items_cloudsync \gset + +SELECT (:metadata_count > 0) AS metadata_created \gset +\if :metadata_created +\echo [PASS] (:testid) Metadata table populated after init (:metadata_count entries) +\else +\echo [FAIL] (:testid) Metadata table empty after init - cloudsync_refill_metatable may have failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(metadata::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM items \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A (pre-existing data) +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema (empty) +-- ============================================================================ + +\connect cloudsync_test_20b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE items ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0, + metadata JSONB +); + +-- Initialize CloudSync on empty table +SELECT cloudsync_init('items', 'CLS', 0) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(metadata::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM items \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM items \gset +\connect cloudsync_test_20a +SELECT COUNT(*) AS count_a_orig FROM items \gset + +\connect cloudsync_test_20b +SELECT (:count_b = :count_a_orig) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (:count_b rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Database A: :count_a_orig, Database B: :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific pre-existing data was synced correctly +-- ============================================================================ + +SELECT COUNT(*) = 1 AS item1_ok +FROM items +WHERE id = '11111111-1111-1111-1111-111111111111' + AND name = 'Pre-existing Item 1' + AND price = 10.99 + AND quantity = 100 \gset +\if :item1_ok +\echo [PASS] (:testid) Pre-existing item 1 synced correctly +\else +\echo [FAIL] (:testid) Pre-existing item 1 not found or incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify JSONB data +SELECT COUNT(*) = 1 AS jsonb_ok +FROM items +WHERE id = '55555555-5555-5555-5555-555555555555' AND metadata = '{"nested": {"key": "value"}}'::jsonb \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB data synced correctly +\else +\echo [FAIL] (:testid) JSONB data not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test: Add new data AFTER init, verify it also syncs +-- ============================================================================ + +\connect cloudsync_test_20a + +-- Add new row after init +INSERT INTO items VALUES ('66666666-6666-6666-6666-666666666666', 'Post-init Item', 66.66, 666, '{"post": true}'); + +-- Encode new changes +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_20b +SELECT cloudsync_payload_apply(decode(:'payload_a2_hex', 'hex')) AS apply_result2 \gset + +SELECT COUNT(*) = 1 AS post_init_ok +FROM items +WHERE id = '66666666-6666-6666-6666-666666666666' AND name = 'Post-init Item' \gset +\if :post_init_ok +\echo [PASS] (:testid) Post-init data syncs correctly +\else +\echo [FAIL] (:testid) Post-init data failed to sync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +INSERT INTO items VALUES ('77777777-7777-7777-7777-777777777777', 'From B', 77.77, 777, '{"from": "B"}'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_20a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) = 1 AS bidirectional_ok +FROM items +WHERE id = '77777777-7777-7777-7777-777777777777' AND name = 'From B' \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Final verification: total row count should be 7 in both databases +-- ============================================================================ + +SELECT COUNT(*) AS final_count_a FROM items \gset +\connect cloudsync_test_20b +SELECT COUNT(*) AS final_count_b FROM items \gset + +SELECT (:final_count_a = 7 AND :final_count_b = 7) AS final_counts_ok \gset +\if :final_counts_ok +\echo [PASS] (:testid) Final row counts correct (7 rows each) +\else +\echo [FAIL] (:testid) Final row counts incorrect - A: :final_count_a, B: :final_count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup: Drop test databases if not in DEBUG mode and no failures +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_20a; +DROP DATABASE IF EXISTS cloudsync_test_20b; +\endif diff --git a/test/postgresql/21_null_value_sync.sql b/test/postgresql/21_null_value_sync.sql new file mode 100644 index 0000000..8b75d18 --- /dev/null +++ b/test/postgresql/21_null_value_sync.sql @@ -0,0 +1,194 @@ +-- Test: NULL Value Sync Parameter Binding +-- This test verifies that syncing NULL values works correctly in all scenarios: +-- 1. Insert with NULL value first, then non-NULL +-- 2. Update existing row to NULL +-- +-- ISSUE: When a NULL value is synced first, PostgreSQL SPI prepares a statement with +-- only the PK parameters. Subsequent non-NULL syncs fail with "there is no parameter $3". +-- +-- The test uses payload_encode/payload_apply to simulate cross-database sync. + +\set testid '21' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_21a; +DROP DATABASE IF EXISTS cloudsync_test_21b; +CREATE DATABASE cloudsync_test_21a; +CREATE DATABASE cloudsync_test_21b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_21a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a simple table with a nullable column +CREATE TABLE test_null_sync ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT -- Nullable column +); + +-- Initialize CloudSync +SELECT cloudsync_init('test_null_sync', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B - Target database (same schema) +-- ============================================================================ + +\connect cloudsync_test_21b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE test_null_sync ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT +); + +SELECT cloudsync_init('test_null_sync', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Test 1: Insert NULL value first, then sync to B +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Insert row with NULL value +INSERT INTO test_null_sync (id, value) VALUES ('row1', NULL); + +-- Encode payload from Database A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_null_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with NULL value +SELECT cloudsync_payload_apply(decode(:'payload_null_hex', 'hex')) AS apply_null_result \gset + +SELECT (:apply_null_result >= 0) AS null_applied \gset +\if :null_applied +\echo [PASS] (:testid) NULL value payload applied successfully +\else +\echo [FAIL] (:testid) NULL value payload failed to apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the NULL value was synced +SELECT COUNT(*) = 1 AS null_row_exists FROM test_null_sync WHERE id = 'row1' AND value IS NULL \gset +\if :null_row_exists +\echo [PASS] (:testid) NULL value synced correctly +\else +\echo [FAIL] (:testid) NULL value not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 2: Insert non-NULL value, then sync to B +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Insert row with non-NULL value +INSERT INTO test_null_sync (id, value) VALUES ('row2', 'hello world'); + +-- Encode payload from Database A (includes new row) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_nonnull_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with non-NULL value +SELECT cloudsync_payload_apply(decode(:'payload_nonnull_hex', 'hex')) AS apply_nonnull_result \gset + +SELECT (:apply_nonnull_result >= 0) AS nonnull_applied \gset +\if :nonnull_applied +\echo [PASS] (:testid) Non-NULL value payload applied successfully after NULL +\else +\echo [FAIL] (:testid) Non-NULL value payload failed to apply (parameter binding issue) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the non-NULL value was synced +SELECT COUNT(*) = 1 AS nonnull_row_exists FROM test_null_sync WHERE id = 'row2' AND value = 'hello world' \gset +\if :nonnull_row_exists +\echo [PASS] (:testid) Non-NULL value synced correctly +\else +\echo [FAIL] (:testid) Non-NULL value not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 3: Verify both rows exist in Database B +-- ============================================================================ + +SELECT COUNT(*) AS total_rows FROM test_null_sync \gset +SELECT (:total_rows = 2) AS both_rows_exist \gset +\if :both_rows_exist +\echo [PASS] (:testid) Both rows synced successfully +\else +\echo [FAIL] (:testid) Expected 2 rows, found :total_rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test 4: Update existing row to NULL, then sync to B +-- This tests that updating a column from non-NULL to NULL works correctly. +-- ============================================================================ + +\connect cloudsync_test_21a + +-- Update row2 to set value to NULL +UPDATE test_null_sync SET value = NULL WHERE id = 'row2'; + +-- Encode payload from Database A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update_null_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_21b + +-- Apply payload with updated NULL value +SELECT cloudsync_payload_apply(decode(:'payload_update_null_hex', 'hex')) AS apply_update_null_result \gset + +SELECT (:apply_update_null_result >= 0) AS update_null_applied \gset +\if :update_null_applied +\echo [PASS] (:testid) Update to NULL payload applied successfully +\else +\echo [FAIL] (:testid) Update to NULL payload failed to apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the update to NULL was synced +SELECT COUNT(*) = 1 AS update_null_synced FROM test_null_sync WHERE id = 'row2' AND value IS NULL \gset +\if :update_null_synced +\echo [PASS] (:testid) Update to NULL synced correctly +\else +\echo [FAIL] (:testid) Update to NULL not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_21a; +DROP DATABASE IF EXISTS cloudsync_test_21b; +\endif diff --git a/test/postgresql/22_null_column_roundtrip.sql b/test/postgresql/22_null_column_roundtrip.sql new file mode 100644 index 0000000..0a601a9 --- /dev/null +++ b/test/postgresql/22_null_column_roundtrip.sql @@ -0,0 +1,347 @@ +-- Test: NULL Column Roundtrip +-- This test verifies that syncing rows with various NULL column combinations works correctly. +-- Tests all permutations: NULL in first column, second column, both, and neither. + +\set testid '22' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_22a; +DROP DATABASE IF EXISTS cloudsync_test_22b; +CREATE DATABASE cloudsync_test_22a; +CREATE DATABASE cloudsync_test_22b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_22a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with nullable columns (no DEFAULT values) +CREATE TABLE null_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER +); + +-- Initialize CloudSync +SELECT cloudsync_init('null_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Insert test data with various NULL combinations +-- ============================================================================ + +-- Row 1: NULL in value column only +INSERT INTO null_sync_test (id, name, value) VALUES ('pg1', 'name1', NULL); + +-- Row 2: NULL in name column only +INSERT INTO null_sync_test (id, name, value) VALUES ('pg2', NULL, 42); + +-- Row 3: NULL in both columns +INSERT INTO null_sync_test (id, name, value) VALUES ('pg3', NULL, NULL); + +-- Row 4: No NULLs (both columns have values) +INSERT INTO null_sync_test (id, name, value) VALUES ('pg4', 'name4', 100); + +-- Row 5: Empty string (not NULL) and zero +INSERT INTO null_sync_test (id, name, value) VALUES ('pg5', '', 0); + +-- Row 6: Another NULL in value +INSERT INTO null_sync_test (id, name, value) VALUES ('pg6', 'name6', NULL); + +-- ============================================================================ +-- Verify source data +-- ============================================================================ + +SELECT COUNT(*) = 6 AS source_row_count_ok FROM null_sync_test \gset +\if :source_row_count_ok +\echo [PASS] (:testid) Source database has 6 rows +\else +\echo [FAIL] (:testid) Source database row count incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM null_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Verify payload was created +SELECT (length(:'payload_a_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_22b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create identical table schema +CREATE TABLE null_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER +); + +-- Initialize CloudSync +SELECT cloudsync_init('null_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B (result: :apply_result) +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM null_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM null_sync_test \gset +SELECT (:count_b = 6) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (6 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 6, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific NULL patterns +-- ============================================================================ + +-- pg1: name='name1', value=NULL +SELECT (SELECT name = 'name1' AND value IS NULL FROM null_sync_test WHERE id = 'pg1') AS pg1_ok \gset +\if :pg1_ok +\echo [PASS] (:testid) pg1: name='name1', value=NULL preserved +\else +\echo [FAIL] (:testid) pg1: NULL in value column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg2: name=NULL, value=42 +SELECT (SELECT name IS NULL AND value = 42 FROM null_sync_test WHERE id = 'pg2') AS pg2_ok \gset +\if :pg2_ok +\echo [PASS] (:testid) pg2: name=NULL, value=42 preserved +\else +\echo [FAIL] (:testid) pg2: NULL in name column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg3: name=NULL, value=NULL +SELECT (SELECT name IS NULL AND value IS NULL FROM null_sync_test WHERE id = 'pg3') AS pg3_ok \gset +\if :pg3_ok +\echo [PASS] (:testid) pg3: name=NULL, value=NULL preserved +\else +\echo [FAIL] (:testid) pg3: Both NULLs not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg4: name='name4', value=100 (no NULLs) +SELECT (SELECT name = 'name4' AND value = 100 FROM null_sync_test WHERE id = 'pg4') AS pg4_ok \gset +\if :pg4_ok +\echo [PASS] (:testid) pg4: name='name4', value=100 preserved +\else +\echo [FAIL] (:testid) pg4: Non-NULL values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg5: name='', value=0 (empty string and zero, not NULL) +SELECT (SELECT name = '' AND value = 0 FROM null_sync_test WHERE id = 'pg5') AS pg5_ok \gset +\if :pg5_ok +\echo [PASS] (:testid) pg5: empty string and zero preserved (not NULL) +\else +\echo [FAIL] (:testid) pg5: Empty string or zero incorrectly converted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg6: name='name6', value=NULL +SELECT (SELECT name = 'name6' AND value IS NULL FROM null_sync_test WHERE id = 'pg6') AS pg6_ok \gset +\if :pg6_ok +\echo [PASS] (:testid) pg6: name='name6', value=NULL preserved +\else +\echo [FAIL] (:testid) pg6: NULL in value column not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test bidirectional sync (B -> A) +-- ============================================================================ + +-- Add a new row in Database B with NULLs +INSERT INTO null_sync_test (id, name, value) VALUES ('pgB1', NULL, 999); + +-- Encode payload from Database B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply to Database A +\connect cloudsync_test_22a +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +-- Verify the new row exists in Database A with correct NULL +SELECT (SELECT name IS NULL AND value = 999 FROM null_sync_test WHERE id = 'pgB1') AS bidirectional_ok \gset +\if :bidirectional_ok +\echo [PASS] (:testid) Bidirectional sync works (B to A with NULL) +\else +\echo [FAIL] (:testid) Bidirectional sync failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test UPDATE to NULL +-- ============================================================================ + +-- Update pg4 to set name to NULL +UPDATE null_sync_test SET name = NULL WHERE id = 'pg4'; + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_22b +SELECT cloudsync_payload_apply(decode(:'payload_update_hex', 'hex')) AS apply_update \gset + +-- Verify pg4 now has NULL name +SELECT (SELECT name IS NULL AND value = 100 FROM null_sync_test WHERE id = 'pg4') AS update_to_null_ok \gset +\if :update_to_null_ok +\echo [PASS] (:testid) UPDATE to NULL synced correctly +\else +\echo [FAIL] (:testid) UPDATE to NULL failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Test UPDATE from NULL to value +-- ============================================================================ + +-- Update pg3 to set both columns to non-NULL values +\connect cloudsync_test_22a +UPDATE null_sync_test SET name = 'updated', value = 123 WHERE id = 'pg3'; + +-- Encode and sync to B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_update2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_22b +SELECT cloudsync_payload_apply(decode(:'payload_update2_hex', 'hex')) AS apply_update2 \gset + +-- Verify pg3 now has non-NULL values +SELECT (SELECT name = 'updated' AND value = 123 FROM null_sync_test WHERE id = 'pg3') AS update_from_null_ok \gset +\if :update_from_null_ok +\echo [PASS] (:testid) UPDATE from NULL to value synced correctly +\else +\echo [FAIL] (:testid) UPDATE from NULL to value failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Final verification - both databases should have 7 rows with matching content +-- ============================================================================ + +SELECT COUNT(*) AS final_count_b FROM null_sync_test \gset +\connect cloudsync_test_22a +SELECT COUNT(*) AS final_count_a FROM null_sync_test \gset + +\connect cloudsync_test_22b +SELECT (:final_count_a = 7 AND :final_count_b = 7) AS final_counts_ok \gset +\if :final_counts_ok +\echo [PASS] (:testid) Final row counts correct (7 rows each) +\else +\echo [FAIL] (:testid) Final row counts incorrect - A: :final_count_a, B: :final_count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_22a; +DROP DATABASE IF EXISTS cloudsync_test_22b; +\endif diff --git a/test/postgresql/23_uuid_column_roundtrip.sql b/test/postgresql/23_uuid_column_roundtrip.sql new file mode 100644 index 0000000..38adbdc --- /dev/null +++ b/test/postgresql/23_uuid_column_roundtrip.sql @@ -0,0 +1,359 @@ +-- Test: UUID Column Roundtrip +-- This test verifies that syncing rows with UUID columns (not as PK) works correctly. +-- Tests various combinations of NULL and non-NULL UUID values alongside other nullable columns. +-- +-- IMPORTANT: This test is structured to isolate whether NULL UUID values trigger encoding bugs: +-- Step 1: Sync a single row with non-NULL UUID only (baseline) +-- Step 2: Sync a row with NULL UUID, then a row with non-NULL UUID (test NULL trigger) +-- Step 3: Sync remaining rows with mixed NULL/non-NULL UUIDs + +\set testid '23' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_23a; +DROP DATABASE IF EXISTS cloudsync_test_23b; +CREATE DATABASE cloudsync_test_23a; +CREATE DATABASE cloudsync_test_23b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_23a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with UUID column and other nullable columns +CREATE TABLE uuid_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER, + id2 UUID +); + +-- Initialize CloudSync +SELECT cloudsync_init('uuid_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B with same schema (before any inserts) +-- ============================================================================ + +\connect cloudsync_test_23b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE uuid_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT, + value INTEGER, + id2 UUID +); + +SELECT cloudsync_init('uuid_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Sync a single row with non-NULL UUID only (baseline test) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: Single row with non-NULL UUID === + +\connect cloudsync_test_23a + +-- Insert only one row with a non-NULL UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step1', 'baseline', 1, 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'); + +-- Encode payload +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_23b +SELECT cloudsync_payload_apply(decode(:'payload_step1_hex', 'hex')) AS apply_step1 \gset + +-- Verify step 1 +SELECT (SELECT id2 = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' FROM uuid_sync_test WHERE id = 'step1') AS step1_ok \gset +\if :step1_ok +\echo [PASS] (:testid) Step 1: Single non-NULL UUID preserved correctly +\else +\echo [FAIL] (:testid) Step 1: Single non-NULL UUID NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT id, name, value, id2::text FROM uuid_sync_test WHERE id = 'step1'; +\endif + +-- ============================================================================ +-- STEP 2: Sync NULL UUID row, then non-NULL UUID row (test NULL trigger) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: NULL UUID followed by non-NULL UUID === + +\connect cloudsync_test_23a + +-- Insert a row with NULL UUID first +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step2a', 'null_uuid', 2, NULL); + +-- Then insert a row with non-NULL UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('step2b', 'after_null', 3, 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'); + +-- Encode payload (should contain both rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_23b +SELECT cloudsync_payload_apply(decode(:'payload_step2_hex', 'hex')) AS apply_step2 \gset + +-- Verify step 2a (NULL UUID) +SELECT (SELECT id2 IS NULL FROM uuid_sync_test WHERE id = 'step2a') AS step2a_ok \gset +\if :step2a_ok +\echo [PASS] (:testid) Step 2a: NULL UUID preserved correctly +\else +\echo [FAIL] (:testid) Step 2a: NULL UUID NOT preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify step 2b (non-NULL UUID after NULL) +SELECT (SELECT id2 = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' FROM uuid_sync_test WHERE id = 'step2b') AS step2b_ok \gset +\if :step2b_ok +\echo [PASS] (:testid) Step 2b: Non-NULL UUID after NULL preserved correctly +\else +\echo [FAIL] (:testid) Step 2b: Non-NULL UUID after NULL NOT preserved (NULL may have triggered bug) +SELECT (:fail::int + 1) AS fail \gset +SELECT id, name, value, id2::text FROM uuid_sync_test WHERE id = 'step2b'; +\endif + +-- ============================================================================ +-- STEP 3: Sync remaining rows with mixed NULL/non-NULL UUIDs +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: Mixed NULL/non-NULL UUIDs === + +\connect cloudsync_test_23a + +-- Row with NULL in value and id2 +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg1', 'name1', NULL, NULL); + +-- Row with NULL in name, has UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg2', NULL, 42, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); + +-- Row with all nullable columns NULL +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg3', NULL, NULL, NULL); + +-- Row with no NULLs - all columns have values +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg4', 'name4', 100, 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22'); + +-- Row with only id2 NULL +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg5', 'name5', 55, NULL); + +-- Row with only name NULL, has different UUID +INSERT INTO uuid_sync_test (id, name, value, id2) VALUES ('pg6', NULL, 66, 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33'); + +-- ============================================================================ +-- Verify source data +-- ============================================================================ + +SELECT COUNT(*) = 9 AS source_row_count_ok FROM uuid_sync_test \gset +\if :source_row_count_ok +\echo [PASS] (:testid) Source database has 9 rows +\else +\echo [FAIL] (:testid) Source database row count incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Compute hash of Database A data +-- ============================================================================ + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(id2::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM uuid_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +-- ============================================================================ +-- Encode payload from Database A (step 3 rows only) +-- ============================================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM uuid_sync_test_cloudsync \gset + +-- Verify payload was created +SELECT (length(:'payload_step3_hex') > 0) AS payload_created \gset +\if :payload_created +\echo [PASS] (:testid) Payload encoded from Database A +\else +\echo [FAIL] (:testid) Payload encoded from Database A - Empty payload +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Apply payload to Database B +-- ============================================================================ + +\connect cloudsync_test_23b + +SELECT cloudsync_payload_apply(decode(:'payload_step3_hex', 'hex')) AS apply_result \gset + +-- Verify application succeeded +SELECT (:apply_result >= 0) AS payload_applied \gset +\if :payload_applied +\echo [PASS] (:testid) Payload applied to Database B (result: :apply_result) +\else +\echo [FAIL] (:testid) Payload applied to Database B - Apply returned :apply_result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify data integrity after roundtrip +-- ============================================================================ + +-- Compute hash of Database B data (should match Database A) +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(value::text, 'NULL') || ':' || + COALESCE(id2::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM uuid_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +-- Compare hashes +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - Database A hash: :hash_a, Database B hash: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify row count +-- ============================================================================ + +SELECT COUNT(*) AS count_b FROM uuid_sync_test \gset +SELECT (:count_b = 9) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (9 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 9, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify specific UUID and NULL patterns +-- ============================================================================ + +-- pg1: name='name1', value=NULL, id2=NULL +SELECT (SELECT name = 'name1' AND value IS NULL AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg1') AS pg1_ok \gset +\if :pg1_ok +\echo [PASS] (:testid) pg1: name='name1', value=NULL, id2=NULL preserved +\else +\echo [FAIL] (:testid) pg1: NULL values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg2: name=NULL, value=42, id2='a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' +SELECT (SELECT name IS NULL AND value = 42 AND id2 = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' FROM uuid_sync_test WHERE id = 'pg2') AS pg2_ok \gset +\if :pg2_ok +\echo [PASS] (:testid) pg2: name=NULL, value=42, UUID preserved +\else +\echo [FAIL] (:testid) pg2: UUID or NULL not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg3: all nullable columns NULL +SELECT (SELECT name IS NULL AND value IS NULL AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg3') AS pg3_ok \gset +\if :pg3_ok +\echo [PASS] (:testid) pg3: all NULLs preserved +\else +\echo [FAIL] (:testid) pg3: all NULLs not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg4: no NULLs, id2='b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' +SELECT (SELECT name = 'name4' AND value = 100 AND id2 = 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' FROM uuid_sync_test WHERE id = 'pg4') AS pg4_ok \gset +\if :pg4_ok +\echo [PASS] (:testid) pg4: all values including UUID preserved +\else +\echo [FAIL] (:testid) pg4: values not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg5: name='name5', value=55, id2=NULL +SELECT (SELECT name = 'name5' AND value = 55 AND id2 IS NULL FROM uuid_sync_test WHERE id = 'pg5') AS pg5_ok \gset +\if :pg5_ok +\echo [PASS] (:testid) pg5: values with NULL UUID preserved +\else +\echo [FAIL] (:testid) pg5: NULL UUID not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pg6: name=NULL, value=66, id2='c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33' +SELECT (SELECT name IS NULL AND value = 66 AND id2 = 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33' FROM uuid_sync_test WHERE id = 'pg6') AS pg6_ok \gset +\if :pg6_ok +\echo [PASS] (:testid) pg6: NULL name with UUID preserved +\else +\echo [FAIL] (:testid) pg6: UUID c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a33 not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Show actual data for debugging if there are failures +-- ============================================================================ + +\if :{?DEBUG} +\echo [INFO] (:testid) Database A data: +\connect cloudsync_test_23a +SELECT id, name, value, id2::text FROM uuid_sync_test ORDER BY id; + +\echo [INFO] (:testid) Database B data: +\connect cloudsync_test_23b +SELECT id, name, value, id2::text FROM uuid_sync_test ORDER BY id; +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_23a; +DROP DATABASE IF EXISTS cloudsync_test_23b; +\endif diff --git a/test/postgresql/24_nullable_types_roundtrip.sql b/test/postgresql/24_nullable_types_roundtrip.sql new file mode 100644 index 0000000..2ac4082 --- /dev/null +++ b/test/postgresql/24_nullable_types_roundtrip.sql @@ -0,0 +1,495 @@ +-- Test: Nullable Types Roundtrip +-- This test verifies that syncing rows with various nullable column types works correctly. +-- Tests the type mapping for NULL values: INT2/4/8 → INT8, FLOAT4/8/NUMERIC → FLOAT8, BYTEA → BYTEA, others → TEXT +-- +-- IMPORTANT: This test inserts NULL values FIRST to trigger SPI plan caching with decoded types, +-- then inserts non-NULL values to verify the type mapping is consistent. + +\set testid '24' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_24a; +DROP DATABASE IF EXISTS cloudsync_test_24b; +CREATE DATABASE cloudsync_test_24a; +CREATE DATABASE cloudsync_test_24b; + +-- ============================================================================ +-- Setup Database A - Source database +-- ============================================================================ + +\connect cloudsync_test_24a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with various nullable column types +-- NOTE: BOOLEAN is excluded because it encodes as INTEGER but PostgreSQL can't cast INT8 to BOOLEAN. +-- This is a known limitation that requires changes to the encoding layer. +CREATE TABLE types_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + -- Integer types (all map to INT8OID in decoding) + col_int2 SMALLINT, + col_int4 INTEGER, + col_int8 BIGINT, + -- Float types (all map to FLOAT8OID in decoding) + col_float4 REAL, + col_float8 DOUBLE PRECISION, + col_numeric NUMERIC(10,2), + -- Binary type (maps to BYTEAOID in decoding) + col_bytea BYTEA, + -- Text types (all map to TEXTOID in decoding) + col_text TEXT, + col_varchar VARCHAR(100), + col_char CHAR(10), + -- Other types that map to TEXTOID + col_uuid UUID, + col_json JSON, + col_jsonb JSONB, + col_date DATE, + col_timestamp TIMESTAMP +); + +-- Initialize CloudSync +SELECT cloudsync_init('types_sync_test', 'CLS', 1) AS _init_a \gset + +-- ============================================================================ +-- Setup Database B with same schema (before any inserts) +-- ============================================================================ + +\connect cloudsync_test_24b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE types_sync_test ( + id TEXT PRIMARY KEY NOT NULL, + col_int2 SMALLINT, + col_int4 INTEGER, + col_int8 BIGINT, + col_float4 REAL, + col_float8 DOUBLE PRECISION, + col_numeric NUMERIC(10,2), + col_bytea BYTEA, + col_text TEXT, + col_varchar VARCHAR(100), + col_char CHAR(10), + col_uuid UUID, + col_json JSON, + col_jsonb JSONB, + col_date DATE, + col_timestamp TIMESTAMP +); + +SELECT cloudsync_init('types_sync_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Insert row with ALL NULL values first (triggers SPI plan caching) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: Insert row with ALL NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'null_row', NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL +); + +-- Encode payload +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step1_hex', 'hex')) AS apply_step1 \gset + +-- Verify step 1: all values should be NULL +SELECT (SELECT + col_int2 IS NULL AND col_int4 IS NULL AND col_int8 IS NULL AND + col_float4 IS NULL AND col_float8 IS NULL AND col_numeric IS NULL AND + col_bytea IS NULL AND col_text IS NULL AND col_varchar IS NULL AND + col_char IS NULL AND col_uuid IS NULL AND col_json IS NULL AND + col_jsonb IS NULL AND col_date IS NULL AND col_timestamp IS NULL +FROM types_sync_test WHERE id = 'null_row') AS step1_ok \gset + +\if :step1_ok +\echo [PASS] (:testid) Step 1: All NULL values preserved correctly +\else +\echo [FAIL] (:testid) Step 1: NULL values NOT preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 2: Insert row with ALL non-NULL values (tests type consistency) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: Insert row with ALL non-NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'full_row', + 32767, -- INT2 max + 2147483647, -- INT4 max + 9223372036854775807, -- INT8 max + 3.14159, -- FLOAT4 + 3.141592653589793, -- FLOAT8 + 12345.67, -- NUMERIC + '\xDEADBEEF', -- BYTEA + 'Hello, World!', -- TEXT + 'varchar_val', -- VARCHAR + 'char_val', -- CHAR (will be padded) + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', -- UUID + '{"key": "value"}', -- JSON + '{"nested": {"array": [1, 2, 3]}}', -- JSONB + '2024-01-15', -- DATE + '2024-01-15 10:30:00' -- TIMESTAMP +); + +-- Encode payload (only new rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step2_hex', 'hex')) AS apply_step2 \gset + +-- Verify step 2: Integer types +SELECT (SELECT col_int2 = 32767 FROM types_sync_test WHERE id = 'full_row') AS int2_ok \gset +\if :int2_ok +\echo [PASS] (:testid) INT2 (SMALLINT) value preserved: 32767 +\else +\echo [FAIL] (:testid) INT2 (SMALLINT) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int2 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_int4 = 2147483647 FROM types_sync_test WHERE id = 'full_row') AS int4_ok \gset +\if :int4_ok +\echo [PASS] (:testid) INT4 (INTEGER) value preserved: 2147483647 +\else +\echo [FAIL] (:testid) INT4 (INTEGER) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int4 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_int8 = 9223372036854775807 FROM types_sync_test WHERE id = 'full_row') AS int8_ok \gset +\if :int8_ok +\echo [PASS] (:testid) INT8 (BIGINT) value preserved: 9223372036854775807 +\else +\echo [FAIL] (:testid) INT8 (BIGINT) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_int8 FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Float types (use approximate comparison for floats) +SELECT (SELECT abs(col_float4 - 3.14159) < 0.0001 FROM types_sync_test WHERE id = 'full_row') AS float4_ok \gset +\if :float4_ok +\echo [PASS] (:testid) FLOAT4 (REAL) value preserved: ~3.14159 +\else +\echo [FAIL] (:testid) FLOAT4 (REAL) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_float4 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT abs(col_float8 - 3.141592653589793) < 0.000000000001 FROM types_sync_test WHERE id = 'full_row') AS float8_ok \gset +\if :float8_ok +\echo [PASS] (:testid) FLOAT8 (DOUBLE PRECISION) value preserved: ~3.141592653589793 +\else +\echo [FAIL] (:testid) FLOAT8 (DOUBLE PRECISION) value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_float8 FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_numeric = 12345.67 FROM types_sync_test WHERE id = 'full_row') AS numeric_ok \gset +\if :numeric_ok +\echo [PASS] (:testid) NUMERIC value preserved: 12345.67 +\else +\echo [FAIL] (:testid) NUMERIC value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_numeric FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: BYTEA type +SELECT (SELECT col_bytea = '\xDEADBEEF' FROM types_sync_test WHERE id = 'full_row') AS bytea_ok \gset +\if :bytea_ok +\echo [PASS] (:testid) BYTEA value preserved: DEADBEEF +\else +\echo [FAIL] (:testid) BYTEA value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT encode(col_bytea, 'hex') FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Text types +SELECT (SELECT col_text = 'Hello, World!' FROM types_sync_test WHERE id = 'full_row') AS text_ok \gset +\if :text_ok +\echo [PASS] (:testid) TEXT value preserved: Hello, World! +\else +\echo [FAIL] (:testid) TEXT value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_text FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_varchar = 'varchar_val' FROM types_sync_test WHERE id = 'full_row') AS varchar_ok \gset +\if :varchar_ok +\echo [PASS] (:testid) VARCHAR value preserved: varchar_val +\else +\echo [FAIL] (:testid) VARCHAR value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_varchar FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT trim(col_char) = 'char_val' FROM types_sync_test WHERE id = 'full_row') AS char_ok \gset +\if :char_ok +\echo [PASS] (:testid) CHAR value preserved: char_val +\else +\echo [FAIL] (:testid) CHAR value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_char FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- Verify step 2: Other types mapped to TEXT +SELECT (SELECT col_uuid = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' FROM types_sync_test WHERE id = 'full_row') AS uuid_ok \gset +\if :uuid_ok +\echo [PASS] (:testid) UUID value preserved: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +\else +\echo [FAIL] (:testid) UUID value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_uuid FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_json::text = '{"key": "value"}' FROM types_sync_test WHERE id = 'full_row') AS json_ok \gset +\if :json_ok +\echo [PASS] (:testid) JSON value preserved: {"key": "value"} +\else +\echo [FAIL] (:testid) JSON value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_json FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_jsonb @> '{"nested": {"array": [1, 2, 3]}}' FROM types_sync_test WHERE id = 'full_row') AS jsonb_ok \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB value preserved: {"nested": {"array": [1, 2, 3]}} +\else +\echo [FAIL] (:testid) JSONB value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_jsonb FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_date = '2024-01-15' FROM types_sync_test WHERE id = 'full_row') AS date_ok \gset +\if :date_ok +\echo [PASS] (:testid) DATE value preserved: 2024-01-15 +\else +\echo [FAIL] (:testid) DATE value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_date FROM types_sync_test WHERE id = 'full_row'; +\endif + +SELECT (SELECT col_timestamp = '2024-01-15 10:30:00' FROM types_sync_test WHERE id = 'full_row') AS timestamp_ok \gset +\if :timestamp_ok +\echo [PASS] (:testid) TIMESTAMP value preserved: 2024-01-15 10:30:00 +\else +\echo [FAIL] (:testid) TIMESTAMP value NOT preserved +SELECT (:fail::int + 1) AS fail \gset +SELECT col_timestamp FROM types_sync_test WHERE id = 'full_row'; +\endif + +-- ============================================================================ +-- STEP 3: Insert row with mixed NULL/non-NULL values +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: Insert row with mixed NULL/non-NULL values === + +\connect cloudsync_test_24a + +INSERT INTO types_sync_test ( + id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric, + col_bytea, col_text, col_varchar, col_char, col_uuid, col_json, col_jsonb, + col_date, col_timestamp +) VALUES ( + 'mixed_row', + NULL, -- INT2 NULL + 42, -- INT4 non-NULL + NULL, -- INT8 NULL + NULL, -- FLOAT4 NULL + 2.718281828, -- FLOAT8 non-NULL (e) + NULL, -- NUMERIC NULL + '\xCAFEBABE', -- BYTEA non-NULL + NULL, -- TEXT NULL + 'mixed', -- VARCHAR non-NULL + NULL, -- CHAR NULL + 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22', -- UUID non-NULL + NULL, -- JSON NULL + '{"mixed": true}', -- JSONB non-NULL + NULL, -- DATE NULL + '2024-06-15 14:00:00' -- TIMESTAMP non-NULL +); + +-- Encode payload (only new rows) +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_step3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM types_sync_test_cloudsync \gset + +-- Apply to Database B +\connect cloudsync_test_24b +SELECT cloudsync_payload_apply(decode(:'payload_step3_hex', 'hex')) AS apply_step3 \gset + +-- Verify mixed row +SELECT (SELECT + col_int2 IS NULL AND + col_int4 = 42 AND + col_int8 IS NULL AND + col_float4 IS NULL AND + abs(col_float8 - 2.718281828) < 0.000001 AND + col_numeric IS NULL AND + col_bytea = '\xCAFEBABE' AND + col_text IS NULL AND + col_varchar = 'mixed' AND + col_char IS NULL AND + col_uuid = 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22' AND + col_json IS NULL AND + col_jsonb @> '{"mixed": true}' AND + col_date IS NULL AND + col_timestamp = '2024-06-15 14:00:00' +FROM types_sync_test WHERE id = 'mixed_row') AS mixed_ok \gset + +\if :mixed_ok +\echo [PASS] (:testid) Mixed NULL/non-NULL row preserved correctly +\else +\echo [FAIL] (:testid) Mixed NULL/non-NULL row NOT preserved correctly +SELECT (:fail::int + 1) AS fail \gset +SELECT * FROM types_sync_test WHERE id = 'mixed_row'; +\endif + +-- ============================================================================ +-- STEP 4: Verify data integrity with hash comparison +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 4: Verify data integrity === + +\connect cloudsync_test_24a + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(col_int2::text, 'NULL') || ':' || + COALESCE(col_int4::text, 'NULL') || ':' || + COALESCE(col_int8::text, 'NULL') || ':' || + COALESCE(col_float8::text, 'NULL') || ':' || + COALESCE(col_numeric::text, 'NULL') || ':' || + COALESCE(encode(col_bytea, 'hex'), 'NULL') || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_varchar, 'NULL') || ':' || + COALESCE(col_uuid::text, 'NULL') || ':' || + COALESCE(col_jsonb::text, 'NULL') || ':' || + COALESCE(col_date::text, 'NULL') || ':' || + COALESCE(col_timestamp::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM types_sync_test \gset + +\echo [INFO] (:testid) Database A hash: :hash_a + +\connect cloudsync_test_24b + +SELECT md5( + COALESCE( + string_agg( + id || ':' || + COALESCE(col_int2::text, 'NULL') || ':' || + COALESCE(col_int4::text, 'NULL') || ':' || + COALESCE(col_int8::text, 'NULL') || ':' || + COALESCE(col_float8::text, 'NULL') || ':' || + COALESCE(col_numeric::text, 'NULL') || ':' || + COALESCE(encode(col_bytea, 'hex'), 'NULL') || ':' || + COALESCE(col_text, 'NULL') || ':' || + COALESCE(col_varchar, 'NULL') || ':' || + COALESCE(col_uuid::text, 'NULL') || ':' || + COALESCE(col_jsonb::text, 'NULL') || ':' || + COALESCE(col_date::text, 'NULL') || ':' || + COALESCE(col_timestamp::text, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM types_sync_test \gset + +\echo [INFO] (:testid) Database B hash: :hash_b + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed - hashes do not match +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS count_b FROM types_sync_test \gset +SELECT (:count_b = 3) AS row_counts_match \gset +\if :row_counts_match +\echo [PASS] (:testid) Row counts match (3 rows) +\else +\echo [FAIL] (:testid) Row counts mismatch - Expected 3, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Show actual data for debugging if there are failures +-- ============================================================================ + +\if :{?DEBUG} +\echo [INFO] (:testid) Database A data: +\connect cloudsync_test_24a +SELECT id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric FROM types_sync_test ORDER BY id; +SELECT id, encode(col_bytea, 'hex') as col_bytea, col_text, col_varchar, trim(col_char) as col_char FROM types_sync_test ORDER BY id; +SELECT id, col_uuid, col_json, col_jsonb, col_date, col_timestamp FROM types_sync_test ORDER BY id; + +\echo [INFO] (:testid) Database B data: +\connect cloudsync_test_24b +SELECT id, col_int2, col_int4, col_int8, col_float4, col_float8, col_numeric FROM types_sync_test ORDER BY id; +SELECT id, encode(col_bytea, 'hex') as col_bytea, col_text, col_varchar, trim(col_char) as col_char FROM types_sync_test ORDER BY id; +SELECT id, col_uuid, col_json, col_jsonb, col_date, col_timestamp FROM types_sync_test ORDER BY id; +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_24a; +DROP DATABASE IF EXISTS cloudsync_test_24b; +\endif diff --git a/test/postgresql/25_boolean_type_issue.sql b/test/postgresql/25_boolean_type_issue.sql new file mode 100644 index 0000000..b9fb6bb --- /dev/null +++ b/test/postgresql/25_boolean_type_issue.sql @@ -0,0 +1,241 @@ +-- Test: BOOLEAN Type Roundtrip +-- This test verifies that BOOLEAN columns sync correctly. +-- BOOLEAN values are encoded as INT8 in sync payloads. The cloudsync extension +-- provides a custom cast (bigint AS boolean) to enable this. +-- +-- See plans/ANALYSIS_BOOLEAN_TYPE_CONVERSION.md for details. + +\set testid '25' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_25a; +DROP DATABASE IF EXISTS cloudsync_test_25b; +CREATE DATABASE cloudsync_test_25a; +CREATE DATABASE cloudsync_test_25b; + +-- Setup Database A +\connect cloudsync_test_25a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE bool_test ( + id TEXT PRIMARY KEY NOT NULL, + flag BOOLEAN, + name TEXT +); + +SELECT cloudsync_init('bool_test', 'CLS', 1) AS _init_a \gset + +-- Setup Database B +\connect cloudsync_test_25b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE bool_test ( + id TEXT PRIMARY KEY NOT NULL, + flag BOOLEAN, + name TEXT +); + +SELECT cloudsync_init('bool_test', 'CLS', 1) AS _init_b \gset + +-- ============================================================================ +-- STEP 1: Insert NULL BOOLEAN first (triggers SPI plan caching) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 1: NULL BOOLEAN === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row1', NULL, 'null_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload1_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload1_hex', 'hex')) AS apply1 \gset + +SELECT (SELECT flag IS NULL AND name = 'null_flag' FROM bool_test WHERE id = 'row1') AS step1_ok \gset +\if :step1_ok +\echo [PASS] (:testid) Step 1: NULL BOOLEAN preserved +\else +\echo [FAIL] (:testid) Step 1: NULL BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 2: Insert TRUE BOOLEAN (tests INT8 -> BOOLEAN cast after NULL) +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 2: TRUE BOOLEAN after NULL === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row2', true, 'true_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload2_hex', 'hex')) AS apply2 \gset + +SELECT (SELECT flag = true AND name = 'true_flag' FROM bool_test WHERE id = 'row2') AS step2_ok \gset +\if :step2_ok +\echo [PASS] (:testid) Step 2: TRUE BOOLEAN preserved after NULL +\else +\echo [FAIL] (:testid) Step 2: TRUE BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 3: Insert FALSE BOOLEAN +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 3: FALSE BOOLEAN === + +\connect cloudsync_test_25a +INSERT INTO bool_test (id, flag, name) VALUES ('row3', false, 'false_flag'); + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload3_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload3_hex', 'hex')) AS apply3 \gset + +SELECT (SELECT flag = false AND name = 'false_flag' FROM bool_test WHERE id = 'row3') AS step3_ok \gset +\if :step3_ok +\echo [PASS] (:testid) Step 3: FALSE BOOLEAN preserved +\else +\echo [FAIL] (:testid) Step 3: FALSE BOOLEAN not preserved +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 4: Update TRUE to FALSE +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 4: Update TRUE to FALSE === + +\connect cloudsync_test_25a +UPDATE bool_test SET flag = false WHERE id = 'row2'; + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload4_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload4_hex', 'hex')) AS apply4 \gset + +SELECT (SELECT flag = false FROM bool_test WHERE id = 'row2') AS step4_ok \gset +\if :step4_ok +\echo [PASS] (:testid) Step 4: Update TRUE to FALSE synced +\else +\echo [FAIL] (:testid) Step 4: Update TRUE to FALSE not synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 5: Update NULL to TRUE +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 5: Update NULL to TRUE === + +\connect cloudsync_test_25a +UPDATE bool_test SET flag = true WHERE id = 'row1'; + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload5_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() AND db_version > :db_version \gset + +SELECT max(db_version) AS db_version FROM bool_test_cloudsync \gset + +\connect cloudsync_test_25b +SELECT cloudsync_payload_apply(decode(:'payload5_hex', 'hex')) AS apply5 \gset + +SELECT (SELECT flag = true FROM bool_test WHERE id = 'row1') AS step5_ok \gset +\if :step5_ok +\echo [PASS] (:testid) Step 5: Update NULL to TRUE synced +\else +\echo [FAIL] (:testid) Step 5: Update NULL to TRUE not synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- STEP 6: Verify final state with hash comparison +-- ============================================================================ + +\echo [INFO] (:testid) === STEP 6: Verify data integrity === + +\connect cloudsync_test_25a +SELECT md5( + COALESCE( + string_agg( + id || ':' || COALESCE(flag::text, 'NULL') || ':' || COALESCE(name, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a FROM bool_test \gset + +\connect cloudsync_test_25b +SELECT md5( + COALESCE( + string_agg( + id || ':' || COALESCE(flag::text, 'NULL') || ':' || COALESCE(name, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b FROM bool_test \gset + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Data integrity verified - hashes match +\else +\echo [FAIL] (:testid) Data integrity check failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS count_b FROM bool_test \gset +SELECT (:count_b = 3) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) Row count correct (3 rows) +\else +\echo [FAIL] (:testid) Row count incorrect - expected 3, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +\connect postgres +DROP DATABASE IF EXISTS cloudsync_test_25a; +DROP DATABASE IF EXISTS cloudsync_test_25b; +\endif diff --git a/test/postgresql/26_row_filter.sql b/test/postgresql/26_row_filter.sql new file mode 100644 index 0000000..01c9dde --- /dev/null +++ b/test/postgresql/26_row_filter.sql @@ -0,0 +1,105 @@ +-- 'Row-level filter (conditional sync) test' + +\set testid '26' +\ir helper_test_init.sql + +-- Create first database +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_26_a; +CREATE DATABASE cloudsync_test_26_a; + +\connect cloudsync_test_26_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table, init, set filter +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_site_id_a \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _set_filter_ok \gset + +-- Insert matching rows (user_id = 1) and non-matching rows (user_id = 2) +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); + +-- Test 1: Verify only matching rows are tracked in _cloudsync metadata +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 2) AS filter_insert_ok \gset +\if :filter_insert_ok +\echo [PASS] (:testid) Only matching rows tracked after INSERT (2 of 3) +\else +\echo [FAIL] (:testid) Expected 2 tracked PKs after INSERT, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Update non-matching row → no metadata change +SELECT COUNT(*) AS meta_before FROM tasks_cloudsync \gset +UPDATE tasks SET title = 'Task B Updated' WHERE id = 'b'; +SELECT COUNT(*) AS meta_after FROM tasks_cloudsync \gset +SELECT (:meta_before = :meta_after) AS filter_update_nonmatch_ok \gset +\if :filter_update_nonmatch_ok +\echo [PASS] (:testid) Non-matching UPDATE did not change metadata +\else +\echo [FAIL] (:testid) Non-matching UPDATE changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Delete non-matching row → no metadata change +SELECT COUNT(*) AS meta_before FROM tasks_cloudsync \gset +DELETE FROM tasks WHERE id = 'b'; +SELECT COUNT(*) AS meta_after FROM tasks_cloudsync \gset +SELECT (:meta_before = :meta_after) AS filter_delete_nonmatch_ok \gset +\if :filter_delete_nonmatch_ok +\echo [PASS] (:testid) Non-matching DELETE did not change metadata +\else +\echo [FAIL] (:testid) Non-matching DELETE changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Roundtrip - sync to second database, verify only filtered rows transfer +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_26_b; +CREATE DATABASE cloudsync_test_26_b; + +\connect cloudsync_test_26_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_site_id_b \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _set_filter_b_ok \gset +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply_ok \gset + +-- Verify: db2 should have only the matching rows +SELECT COUNT(*) AS task_count FROM tasks \gset +SELECT (:task_count = 2) AS roundtrip_count_ok \gset +\if :roundtrip_count_ok +\echo [PASS] (:testid) Roundtrip: correct number of rows synced (2) +\else +\echo [FAIL] (:testid) Roundtrip: expected 2 rows, got :task_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row 'c' exists with user_id = 1 +SELECT COUNT(*) AS c_exists FROM tasks WHERE id = 'c' AND user_id = 1 \gset +SELECT (:c_exists = 1) AS roundtrip_row_ok \gset +\if :roundtrip_row_ok +\echo [PASS] (:testid) Roundtrip: task 'c' with user_id=1 present +\else +\echo [FAIL] (:testid) Roundtrip: task 'c' with user_id=1 not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_26_a; +DROP DATABASE IF EXISTS cloudsync_test_26_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/27_rls_batch_merge.sql b/test/postgresql/27_rls_batch_merge.sql new file mode 100644 index 0000000..2ab51bf --- /dev/null +++ b/test/postgresql/27_rls_batch_merge.sql @@ -0,0 +1,356 @@ +-- 'RLS batch merge test' +-- Verifies that the deferred column-batch merge produces complete rows +-- that work correctly with PostgreSQL Row Level Security policies. +-- +-- Tests 1-3: cloudsync_payload_apply runs as superuser (service-role pattern). +-- RLS is enforced at the query layer when users access data. +-- +-- Tests 4-6: cloudsync_payload_apply runs as non-superuser (authenticated-role +-- pattern). RLS is enforced during the write itself. + +\set testid '27' +\ir helper_test_init.sql + +\set USER1 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +\set USER2 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + +-- ============================================================ +-- DB A: source database (no RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_27_a; +CREATE DATABASE cloudsync_test_27_a; + +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE documents ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + content TEXT +); +SELECT cloudsync_init('documents') AS _init_site_id_a \gset + +-- ============================================================ +-- DB B: target database (with RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_27_b; +CREATE DATABASE cloudsync_test_27_b; + +-- Create non-superuser role (ignore error if it already exists) +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test_rls_user') THEN + CREATE ROLE test_rls_user LOGIN; + END IF; +END $$; + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE documents ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + content TEXT +); +SELECT cloudsync_init('documents') AS _init_site_id_b \gset + +-- Auth mock: auth.uid() reads from session variable app.current_user_id +CREATE SCHEMA IF NOT EXISTS auth; +CREATE OR REPLACE FUNCTION auth.uid() RETURNS UUID + LANGUAGE sql STABLE +AS $$ SELECT NULLIF(current_setting('app.current_user_id', true), '')::UUID; $$; + +-- Enable RLS +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON documents FOR SELECT + USING (auth.uid() = user_id); +CREATE POLICY "insert_own" ON documents FOR INSERT + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "update_own" ON documents FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "delete_own" ON documents FOR DELETE + USING (auth.uid() = user_id); + +-- Grant permissions to test_rls_user +GRANT USAGE ON SCHEMA public TO test_rls_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO test_rls_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO test_rls_user; +GRANT USAGE ON SCHEMA auth TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA auth TO test_rls_user; + +-- ============================================================ +-- Test 1: Batch merge produces complete row — user1 doc synced +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc1', :'USER1'::UUID, 'Title 1', 'Content 1'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Save high-water mark so subsequent encodes only pick up new changes +SELECT COALESCE(max(db_version), 0) AS max_dbv_1 FROM cloudsync_changes \gset + +-- Apply as superuser (service-role pattern) +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_1', 'hex')) AS apply_1 \gset + +-- 1 row × 3 non-PK columns = 3 column-change entries +SELECT (:apply_1::int = 3) AS apply_1_ok \gset +\if :apply_1_ok +\echo [PASS] (:testid) RLS: apply returned :apply_1 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_1 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify complete row written (all columns present) +SELECT COUNT(*) AS doc1_count FROM documents WHERE id = 'doc1' AND title = 'Title 1' AND content = 'Content 1' AND user_id = :'USER1'::UUID \gset +SELECT (:doc1_count::int = 1) AS test1_ok \gset +\if :test1_ok +\echo [PASS] (:testid) RLS: batch merge writes complete row +\else +\echo [FAIL] (:testid) RLS: batch merge writes complete row — got :doc1_count matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Sync user2 doc, then verify RLS hides it from user1 +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc2', :'USER2'::UUID, 'Title 2', 'Content 2'); + +-- Encode only changes newer than test 1 (doc2 only) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_1 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_2 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_2', 'hex')) AS apply_2 \gset + +-- 1 row × 3 non-PK columns = 3 entries +SELECT (:apply_2::int = 3) AS apply_2_ok \gset +\if :apply_2_ok +\echo [PASS] (:testid) RLS: apply returned :apply_2 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_2 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc2 exists (superuser sees all) +SELECT COUNT(*) AS doc2_exists FROM documents WHERE id = 'doc2' \gset + +-- Now check as user1: RLS should hide doc2 (owned by user2) +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT COUNT(*) AS doc2_visible FROM documents WHERE id = 'doc2' \gset +RESET ROLE; + +SELECT (:doc2_exists::int = 1 AND :doc2_visible::int = 0) AS test2_ok \gset +\if :test2_ok +\echo [PASS] (:testid) RLS: user2 doc synced but hidden from user1 +\else +\echo [FAIL] (:testid) RLS: user2 doc synced but hidden from user1 — exists=:doc2_exists visible=:doc2_visible +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Update doc1, verify user1 sees update via RLS +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +UPDATE documents SET title = 'Title 1 Updated' WHERE id = 'doc1'; + +-- Encode only changes newer than test 2 (doc1 update only) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_2 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_3 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_3', 'hex')) AS apply_3 \gset + +-- 1 row × 1 changed column (title) = 1 entry +SELECT (:apply_3::int = 1) AS apply_3_ok \gset +\if :apply_3_ok +\echo [PASS] (:testid) RLS: apply returned :apply_3 +\else +\echo [FAIL] (:testid) RLS: apply returned :apply_3 (expected 1) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify update applied (superuser check) +SELECT COUNT(*) AS doc1_updated FROM documents WHERE id = 'doc1' AND title = 'Title 1 Updated' \gset + +-- Verify user1 can see the updated row via RLS +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT COUNT(*) AS doc1_visible FROM documents WHERE id = 'doc1' AND title = 'Title 1 Updated' \gset +RESET ROLE; + +SELECT (:doc1_updated::int = 1 AND :doc1_visible::int = 1) AS test3_ok \gset +\if :test3_ok +\echo [PASS] (:testid) RLS: update synced and visible to owner +\else +\echo [FAIL] (:testid) RLS: update synced and visible to owner — updated=:doc1_updated visible=:doc1_visible +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Authenticated insert allowed (own row) +-- cloudsync_payload_apply as non-superuser with matching user_id +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc3', :'USER1'::UUID, 'Title 3', 'Content 3'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_3 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset +RESET ROLE; + +-- 1 row × 3 non-PK columns = 3 entries +SELECT (:apply_4::int = 3) AS apply_4_ok \gset +\if :apply_4_ok +\echo [PASS] (:testid) RLS auth: apply returned :apply_4 +\else +\echo [FAIL] (:testid) RLS auth: apply returned :apply_4 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc3 exists with all columns correct +SELECT COUNT(*) AS doc3_count FROM documents WHERE id = 'doc3' AND title = 'Title 3' AND content = 'Content 3' AND user_id = :'USER1'::UUID \gset +SELECT (:doc3_count::int = 1) AS test4_ok \gset +\if :test4_ok +\echo [PASS] (:testid) RLS auth: insert own row allowed +\else +\echo [FAIL] (:testid) RLS auth: insert own row allowed — got :doc3_count matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Authenticated insert denied (other user's row) +-- cloudsync_payload_apply as non-superuser with mismatched user_id +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +INSERT INTO documents VALUES ('doc4', :'USER2'::UUID, 'Title 4', 'Content 4'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_4 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (doc4 owned by USER2) +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 3 non-PK columns = 3 entries (returned even if denied) +SELECT (:apply_5::int = 3) AS apply_5_ok \gset +\if :apply_5_ok +\echo [PASS] (:testid) RLS auth: denied apply returned :apply_5 +\else +\echo [FAIL] (:testid) RLS auth: denied apply returned :apply_5 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc4 does NOT exist (superuser check) +SELECT COUNT(*) AS doc4_count FROM documents WHERE id = 'doc4' \gset +SELECT (:doc4_count::int = 0) AS test5_ok \gset +\if :test5_ok +\echo [PASS] (:testid) RLS auth: insert other user row denied +\else +\echo [FAIL] (:testid) RLS auth: insert other user row denied — got :doc4_count rows (expected 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Authenticated update allowed (own row) +-- cloudsync_payload_apply as non-superuser updating own row +-- ============================================================ +\connect cloudsync_test_27_a +\ir helper_psql_conn_setup.sql +UPDATE documents SET title = 'Title 3 Updated' WHERE id = 'doc3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_5 \gset + +\connect cloudsync_test_27_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset +RESET ROLE; + +-- 1 row × 1 changed column (title) = 1 entry +SELECT (:apply_6::int = 1) AS apply_6_ok \gset +\if :apply_6_ok +\echo [PASS] (:testid) RLS auth: apply returned :apply_6 +\else +\echo [FAIL] (:testid) RLS auth: apply returned :apply_6 (expected 1) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify doc3 title was updated +SELECT COUNT(*) AS doc3_updated FROM documents WHERE id = 'doc3' AND title = 'Title 3 Updated' \gset +SELECT (:doc3_updated::int = 1) AS test6_ok \gset +\if :test6_ok +\echo [PASS] (:testid) RLS auth: update own row allowed +\else +\echo [FAIL] (:testid) RLS auth: update own row allowed — got :doc3_updated matching rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_27_a; +DROP DATABASE IF EXISTS cloudsync_test_27_b; +DROP ROLE IF EXISTS test_rls_user; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/28_db_version_tracking.sql b/test/postgresql/28_db_version_tracking.sql new file mode 100644 index 0000000..2baa69b --- /dev/null +++ b/test/postgresql/28_db_version_tracking.sql @@ -0,0 +1,275 @@ +-- Test db_version/seq tracking in cloudsync_changes after payload apply +-- PostgreSQL equivalent of SQLite unit tests: +-- "Merge Test db_version 1" (do_test_merge_check_db_version) +-- "Merge Test db_version 2" (do_test_merge_check_db_version_2) + +\set testid '28' +\ir helper_test_init.sql + +-- ============================================================ +-- Setup: create databases A and B with the todo table +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_28_a; +DROP DATABASE IF EXISTS cloudsync_test_28_b; +CREATE DATABASE cloudsync_test_28_a; +CREATE DATABASE cloudsync_test_28_b; + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_a \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_b \gset + +-- ============================================================ +-- Test 1: One-way merge (A -> B), mixed insert patterns +-- Mirrors do_test_merge_check_db_version from test/unit.c +-- ============================================================ + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql + +-- Autocommit insert (db_version 1) +INSERT INTO todo VALUES ('ID1', 'Buy groceries', 'in_progress1'); + +-- Multi-row insert (db_version 2 — single statement) +INSERT INTO todo VALUES ('ID2', 'Buy bananas', 'in_progress2'), ('ID3', 'Buy vegetables', 'in_progress3'); + +-- Autocommit insert (db_version 3) +INSERT INTO todo VALUES ('ID4', 'Buy apples', 'in_progress4'); + +-- Transaction with 3 inserts (db_version 4 — one transaction) +BEGIN; +INSERT INTO todo VALUES ('ID5', 'Buy oranges', 'in_progress5'); +INSERT INTO todo VALUES ('ID6', 'Buy lemons', 'in_progress6'); +INSERT INTO todo VALUES ('ID7', 'Buy pizza', 'in_progress7'); +COMMIT; + +-- Encode payload +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_t1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_t1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply to B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +\if :payload_a_t1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_t1', 3), 'hex')) AS _apply_t1 \gset +\endif + +-- Verify data matches +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_b_t1 +FROM todo \gset + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_a_t1 +FROM todo \gset + +SELECT (:'hash_a_t1' = :'hash_b_t1') AS t1_data_ok \gset +\if :t1_data_ok +\echo [PASS] (:testid) db_version test 1: data roundtrip matches +\else +\echo [FAIL] (:testid) db_version test 1: data roundtrip mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_b_t1 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_b_t1::int = 0) AS t1_no_dups_b \gset +\if :t1_no_dups_b +\echo [PASS] (:testid) db_version test 1: no duplicate (db_version, seq) on B +\else +\echo [FAIL] (:testid) db_version test 1: duplicate (db_version, seq) on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS row_count_b_t1 FROM todo \gset +SELECT (:row_count_b_t1::int = 7) AS t1_count_ok \gset +\if :t1_count_ok +\echo [PASS] (:testid) db_version test 1: row count correct (7) +\else +\echo [FAIL] (:testid) db_version test 1: expected 7 rows, got :row_count_b_t1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Bidirectional merge (A -> B, B -> A), mixed patterns +-- Mirrors do_test_merge_check_db_version_2 from test/unit.c +-- ============================================================ + +-- Reset: drop and recreate databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_28_a; +DROP DATABASE IF EXISTS cloudsync_test_28_b; +CREATE DATABASE cloudsync_test_28_a; +CREATE DATABASE cloudsync_test_28_b; + +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_a2 \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE todo (id TEXT PRIMARY KEY NOT NULL, title TEXT, status TEXT); +SELECT cloudsync_init('todo', 'CLS', 1) AS _init_b2 \gset + +-- DB A: two autocommit inserts (db_version 1, 2) +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +INSERT INTO todo VALUES ('ID1', 'Buy groceries', 'in_progress'); +INSERT INTO todo VALUES ('ID2', 'Foo', 'Bar'); + +-- DB B: two autocommit inserts + one transaction with 2 inserts +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +INSERT INTO todo VALUES ('ID3', 'Foo3', 'Bar3'); +INSERT INTO todo VALUES ('ID4', 'Foo4', 'Bar4'); +BEGIN; +INSERT INTO todo VALUES ('ID5', 'Foo5', 'Bar5'); +INSERT INTO todo VALUES ('ID6', 'Foo6', 'Bar6'); +COMMIT; + +-- Encode A's payload +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_t2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_t2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Encode B's payload +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_t2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_t2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Apply A -> B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +\if :payload_a_t2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_t2', 3), 'hex')) AS _apply_a_to_b \gset +\endif + +-- Apply B -> A +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +\if :payload_b_t2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_t2', 3), 'hex')) AS _apply_b_to_a \gset +\endif + +-- Verify data matches between A and B +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_a_t2 +FROM todo \gset + +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(title, '') || ':' || COALESCE(status, ''), ',' ORDER BY id), '')) AS hash_b_t2 +FROM todo \gset + +SELECT (:'hash_a_t2' = :'hash_b_t2') AS t2_data_ok \gset +\if :t2_data_ok +\echo [PASS] (:testid) db_version test 2: bidirectional data matches +\else +\echo [FAIL] (:testid) db_version test 2: bidirectional data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count (6 rows: ID1-ID6) +SELECT COUNT(*) AS row_count_t2 FROM todo \gset +SELECT (:row_count_t2::int = 6) AS t2_count_ok \gset +\if :t2_count_ok +\echo [PASS] (:testid) db_version test 2: row count correct (6) +\else +\echo [FAIL] (:testid) db_version test 2: expected 6 rows, got :row_count_t2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on A +\connect cloudsync_test_28_a +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_a_t2 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_a_t2::int = 0) AS t2_no_dups_a \gset +\if :t2_no_dups_a +\echo [PASS] (:testid) db_version test 2: no duplicate (db_version, seq) on A +\else +\echo [FAIL] (:testid) db_version test 2: duplicate (db_version, seq) on A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify no repeated (db_version, seq) tuples on B +\connect cloudsync_test_28_b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS dup_count_b_t2 +FROM ( + SELECT db_version, seq, COUNT(*) AS cnt + FROM cloudsync_changes + GROUP BY db_version, seq + HAVING COUNT(*) > 1 +) AS dups \gset + +SELECT (:dup_count_b_t2::int = 0) AS t2_no_dups_b \gset +\if :t2_no_dups_b +\echo [PASS] (:testid) db_version test 2: no duplicate (db_version, seq) on B +\else +\echo [FAIL] (:testid) db_version test 2: duplicate (db_version, seq) on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +-- DROP DATABASE IF EXISTS cloudsync_test_28_a; +-- DROP DATABASE IF EXISTS cloudsync_test_28_b; +\endif diff --git a/test/postgresql/29_rls_multicol.sql b/test/postgresql/29_rls_multicol.sql new file mode 100644 index 0000000..de8f304 --- /dev/null +++ b/test/postgresql/29_rls_multicol.sql @@ -0,0 +1,435 @@ +-- 'RLS multi-column batch merge test' +-- Extends test 27 with more column types (INTEGER, BOOLEAN) and additional +-- test cases: update-denied, mixed payloads (per-PK savepoint isolation), +-- and NULL handling. +-- +-- Tests 1-2: superuser (service-role pattern) +-- Tests 3-8: authenticated-role pattern + +\set testid '29' +\ir helper_test_init.sql + +\set USER1 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +\set USER2 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + +-- ============================================================ +-- DB A: source database (no RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_29_a; +CREATE DATABASE cloudsync_test_29_a; + +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + description TEXT, + priority INTEGER, + is_complete BOOLEAN +); +SELECT cloudsync_init('tasks') AS _init_site_id_a \gset + +-- ============================================================ +-- DB B: target database (with RLS) +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_29_b; +CREATE DATABASE cloudsync_test_29_b; + +-- Create non-superuser role (ignore error if it already exists) +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test_rls_user') THEN + CREATE ROLE test_rls_user LOGIN; + END IF; +END $$; + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks ( + id TEXT PRIMARY KEY NOT NULL, + user_id UUID, + title TEXT, + description TEXT, + priority INTEGER, + is_complete BOOLEAN +); +SELECT cloudsync_init('tasks') AS _init_site_id_b \gset + +-- Auth mock: auth.uid() reads from session variable app.current_user_id +CREATE SCHEMA IF NOT EXISTS auth; +CREATE OR REPLACE FUNCTION auth.uid() RETURNS UUID + LANGUAGE sql STABLE +AS $$ SELECT NULLIF(current_setting('app.current_user_id', true), '')::UUID; $$; + +-- Enable RLS +ALTER TABLE tasks ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "select_own" ON tasks FOR SELECT + USING (auth.uid() = user_id); +CREATE POLICY "insert_own" ON tasks FOR INSERT + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "update_own" ON tasks FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); +CREATE POLICY "delete_own" ON tasks FOR DELETE + USING (auth.uid() = user_id); + +-- Grant permissions to test_rls_user +GRANT USAGE ON SCHEMA public TO test_rls_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO test_rls_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO test_rls_user; +GRANT USAGE ON SCHEMA auth TO test_rls_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA auth TO test_rls_user; + +-- ============================================================ +-- Test 1: Superuser multi-row insert with varied types +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t1', :'USER1'::UUID, 'Task 1', 'Desc 1', 3, false); +INSERT INTO tasks VALUES ('t2', :'USER1'::UUID, 'Task 2', 'Desc 2', 1, true); +INSERT INTO tasks VALUES ('t3', :'USER2'::UUID, 'Task 3', 'Desc 3', 5, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_1 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_1', 'hex')) AS apply_1 \gset + +-- 3 rows × 5 non-PK columns = 15 column-change entries +SELECT (:apply_1::int = 15) AS apply_1_ok \gset +\if :apply_1_ok +\echo [PASS] (:testid) RLS multicol: superuser multi-row apply returned :apply_1 +\else +\echo [FAIL] (:testid) RLS multicol: superuser multi-row apply returned :apply_1 (expected 15) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify all 3 rows with correct column values +SELECT COUNT(*) AS t1_ok FROM tasks WHERE id = 't1' AND user_id = :'USER1'::UUID AND title = 'Task 1' AND description = 'Desc 1' AND priority = 3 AND is_complete = false \gset +SELECT COUNT(*) AS t2_ok FROM tasks WHERE id = 't2' AND user_id = :'USER1'::UUID AND title = 'Task 2' AND description = 'Desc 2' AND priority = 1 AND is_complete = true \gset +SELECT COUNT(*) AS t3_ok FROM tasks WHERE id = 't3' AND user_id = :'USER2'::UUID AND title = 'Task 3' AND description = 'Desc 3' AND priority = 5 AND is_complete = false \gset +SELECT (:t1_ok::int = 1 AND :t2_ok::int = 1 AND :t3_ok::int = 1) AS test1_ok \gset +\if :test1_ok +\echo [PASS] (:testid) RLS multicol: superuser multi-row insert with varied types +\else +\echo [FAIL] (:testid) RLS multicol: superuser multi-row insert with varied types — t1=:t1_ok t2=:t2_ok t3=:t3_ok +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Superuser multi-column partial update +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +UPDATE tasks SET title = 'Task 1 Updated', priority = 10, is_complete = true WHERE id = 't1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_1 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_2 FROM cloudsync_changes \gset + +-- Apply as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_hex_2', 'hex')) AS apply_2 \gset + +-- 1 row × 3 changed columns (title, priority, is_complete) = 3 entries +SELECT (:apply_2::int = 3) AS apply_2_ok \gset +\if :apply_2_ok +\echo [PASS] (:testid) RLS multicol: superuser partial update apply returned :apply_2 +\else +\echo [FAIL] (:testid) RLS multicol: superuser partial update apply returned :apply_2 (expected 3) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify updated columns changed and description preserved +SELECT COUNT(*) AS t1_updated FROM tasks WHERE id = 't1' AND title = 'Task 1 Updated' AND description = 'Desc 1' AND priority = 10 AND is_complete = true \gset +SELECT (:t1_updated::int = 1) AS test2_ok \gset +\if :test2_ok +\echo [PASS] (:testid) RLS multicol: superuser partial update preserves unchanged columns +\else +\echo [FAIL] (:testid) RLS multicol: superuser partial update preserves unchanged columns — got :t1_updated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Authenticated insert own row (all columns) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t4', :'USER1'::UUID, 'Task 4', 'Desc 4', 2, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_3 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_2 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_3 FROM cloudsync_changes \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_3', 'hex')) AS apply_3 \gset +RESET ROLE; + +-- 1 row × 5 non-PK columns = 5 entries +SELECT (:apply_3::int = 5) AS apply_3_ok \gset +\if :apply_3_ok +\echo [PASS] (:testid) RLS multicol auth: insert own row apply returned :apply_3 +\else +\echo [FAIL] (:testid) RLS multicol auth: insert own row apply returned :apply_3 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row exists with all columns correct +SELECT COUNT(*) AS t4_count FROM tasks WHERE id = 't4' AND user_id = :'USER1'::UUID AND title = 'Task 4' AND description = 'Desc 4' AND priority = 2 AND is_complete = false \gset +SELECT (:t4_count::int = 1) AS test3_ok \gset +\if :test3_ok +\echo [PASS] (:testid) RLS multicol auth: insert own row allowed +\else +\echo [FAIL] (:testid) RLS multicol auth: insert own row allowed — got :t4_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Authenticated insert denied (other user's row) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t5', :'USER2'::UUID, 'Task 5', 'Desc 5', 7, true); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_4 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_3 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (t5 owned by USER2) +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 5 columns = 5 entries in payload (returned even if denied) +SELECT (:apply_4::int = 5) AS apply_4_ok \gset +\if :apply_4_ok +\echo [PASS] (:testid) RLS multicol auth: denied insert apply returned :apply_4 +\else +\echo [FAIL] (:testid) RLS multicol auth: denied insert apply returned :apply_4 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify t5 does NOT exist (superuser check) +SELECT COUNT(*) AS t5_count FROM tasks WHERE id = 't5' \gset +SELECT (:t5_count::int = 0) AS test4_ok \gset +\if :test4_ok +\echo [PASS] (:testid) RLS multicol auth: insert other user row denied +\else +\echo [FAIL] (:testid) RLS multicol auth: insert other user row denied — got :t5_count rows (expected 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Authenticated update own row (multiple columns) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +UPDATE tasks SET title = 'Task 4 Updated', priority = 9 WHERE id = 't4'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_5 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_4 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset +RESET ROLE; + +-- 1 row × 2 changed columns (title, priority) = 2 entries +SELECT (:apply_5::int = 2) AS apply_5_ok \gset +\if :apply_5_ok +\echo [PASS] (:testid) RLS multicol auth: update own row apply returned :apply_5 +\else +\echo [FAIL] (:testid) RLS multicol auth: update own row apply returned :apply_5 (expected 2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify both columns changed, others preserved +SELECT COUNT(*) AS t4_updated FROM tasks WHERE id = 't4' AND title = 'Task 4 Updated' AND description = 'Desc 4' AND priority = 9 AND is_complete = false \gset +SELECT (:t4_updated::int = 1) AS test5_ok \gset +\if :test5_ok +\echo [PASS] (:testid) RLS multicol auth: update own row allowed +\else +\echo [FAIL] (:testid) RLS multicol auth: update own row allowed — got :t4_updated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Authenticated update denied (other user's row) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +-- t3 is owned by USER2, update it on A +UPDATE tasks SET title = 'Task 3 Hacked', priority = 99 WHERE id = 't3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_6 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_5 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_6 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity — should be denied (t3 owned by USER2) +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset + +-- Reconnect for clean state after expected RLS denial +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 1 row × 2 changed columns (title, priority) = 2 entries in payload +SELECT (:apply_6::int = 2) AS apply_6_ok \gset +\if :apply_6_ok +\echo [PASS] (:testid) RLS multicol auth: denied update apply returned :apply_6 +\else +\echo [FAIL] (:testid) RLS multicol auth: denied update apply returned :apply_6 (expected 2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify t3 still has original values (superuser check) +SELECT COUNT(*) AS t3_unchanged FROM tasks WHERE id = 't3' AND title = 'Task 3' AND priority = 5 \gset +SELECT (:t3_unchanged::int = 1) AS test6_ok \gset +\if :test6_ok +\echo [PASS] (:testid) RLS multicol auth: update other user row denied +\else +\echo [FAIL] (:testid) RLS multicol auth: update other user row denied — got :t3_unchanged (expected 1 unchanged) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Mixed payload — own + other user's rows (per-PK savepoint) +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t6', :'USER1'::UUID, 'Task 6', 'Desc 6', 4, false); +INSERT INTO tasks VALUES ('t7', :'USER2'::UUID, 'Task 7', 'Desc 7', 8, true); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_7 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_6 \gset + +SELECT COALESCE(max(db_version), 0) AS max_dbv_7 FROM cloudsync_changes \gset + +-- Apply as test_rls_user with USER1 identity +-- Per-PK savepoint: t6 (USER1) should succeed, t7 (USER2) should be denied +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_7', 'hex')) AS apply_7 \gset + +-- Reconnect for clean verification as superuser +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql + +-- 2 rows × 5 columns = 10 entries in payload +SELECT (:apply_7::int = 10) AS apply_7_ok \gset +\if :apply_7_ok +\echo [PASS] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 +\else +\echo [FAIL] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 (expected 10) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- t6 (own row) should exist, t7 (other's row) should NOT +SELECT COUNT(*) AS t6_exists FROM tasks WHERE id = 't6' AND user_id = :'USER1'::UUID AND title = 'Task 6' \gset +SELECT COUNT(*) AS t7_exists FROM tasks WHERE id = 't7' \gset +SELECT (:t6_exists::int = 1 AND :t7_exists::int = 0) AS test7_ok \gset +\if :test7_ok +\echo [PASS] (:testid) RLS multicol auth: mixed payload — per-PK savepoint isolation +\else +\echo [FAIL] (:testid) RLS multicol auth: mixed payload — t6=:t6_exists (expect 1) t7=:t7_exists (expect 0) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: NULL in non-ownership columns +-- ============================================================ +\connect cloudsync_test_29_a +\ir helper_psql_conn_setup.sql +INSERT INTO tasks VALUES ('t8', :'USER1'::UUID, 'Task 8', NULL, NULL, false); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex_8 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() + AND db_version > :max_dbv_7 \gset + +\connect cloudsync_test_29_b +\ir helper_psql_conn_setup.sql +SET app.current_user_id = :'USER1'; +SET ROLE test_rls_user; +SELECT cloudsync_payload_apply(decode(:'payload_hex_8', 'hex')) AS apply_8 \gset +RESET ROLE; + +-- 1 row × 5 non-PK columns = 5 entries +SELECT (:apply_8::int = 5) AS apply_8_ok \gset +\if :apply_8_ok +\echo [PASS] (:testid) RLS multicol auth: NULL columns apply returned :apply_8 +\else +\echo [FAIL] (:testid) RLS multicol auth: NULL columns apply returned :apply_8 (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NULLs preserved +SELECT COUNT(*) AS t8_count FROM tasks WHERE id = 't8' AND user_id = :'USER1'::UUID AND title = 'Task 8' AND description IS NULL AND priority IS NULL AND is_complete = false \gset +SELECT (:t8_count::int = 1) AS test8_ok \gset +\if :test8_ok +\echo [PASS] (:testid) RLS multicol auth: NULL in non-ownership columns preserved +\else +\echo [FAIL] (:testid) RLS multicol auth: NULL in non-ownership columns preserved — got :t8_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_29_a; +DROP DATABASE IF EXISTS cloudsync_test_29_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/30_null_prikey_insert.sql b/test/postgresql/30_null_prikey_insert.sql new file mode 100644 index 0000000..4e0bc08 --- /dev/null +++ b/test/postgresql/30_null_prikey_insert.sql @@ -0,0 +1,68 @@ +-- Test: NULL Primary Key Insert Rejection +-- Verifies that inserting a NULL primary key into a cloudsync-enabled table fails +-- and that the metatable only contains rows for valid inserts. + +\set testid '30' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test database +DROP DATABASE IF EXISTS cloudsync_test_30; +CREATE DATABASE cloudsync_test_30; + +\connect cloudsync_test_30 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create table with primary key and init cloudsync +CREATE TABLE t_null_pk ( + id TEXT NOT NULL PRIMARY KEY, + value TEXT +); + +SELECT cloudsync_init('t_null_pk', 'CLS', 1) AS _init \gset + +-- Test 1: INSERT with NULL primary key should fail +DO $$ +BEGIN + INSERT INTO t_null_pk (id, value) VALUES (NULL, 'test'); + RAISE EXCEPTION 'INSERT with NULL PK should have failed'; +EXCEPTION WHEN not_null_violation THEN + -- Expected +END $$; + +SELECT (COUNT(*) = 0) AS null_pk_rejected FROM t_null_pk \gset +\if :null_pk_rejected +\echo [PASS] (:testid) NULL PK insert rejected +\else +\echo [FAIL] (:testid) NULL PK insert was not rejected +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: INSERT with valid (non-NULL) primary key should succeed +INSERT INTO t_null_pk (id, value) VALUES ('valid_id', 'test'); + +SELECT (COUNT(*) = 1) AS valid_insert_ok FROM t_null_pk WHERE id = 'valid_id' \gset +\if :valid_insert_ok +\echo [PASS] (:testid) Valid PK insert succeeded +\else +\echo [FAIL] (:testid) Valid PK insert failed +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Metatable should have exactly 1 row (from the valid insert only) +SELECT (COUNT(*) = 1) AS meta_row_ok FROM t_null_pk_cloudsync \gset +\if :meta_row_ok +\echo [PASS] (:testid) Metatable has exactly 1 row +\else +\echo [FAIL] (:testid) Metatable row count mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_30; +\endif diff --git a/test/postgresql/31_alter_table_sync.sql b/test/postgresql/31_alter_table_sync.sql new file mode 100644 index 0000000..365c356 --- /dev/null +++ b/test/postgresql/31_alter_table_sync.sql @@ -0,0 +1,383 @@ +-- Alter Table Sync Test +-- Tests cloudsync_begin_alter and cloudsync_commit_alter functions. +-- Verifies that schema changes (add column) are handled correctly +-- and data syncs after alteration. + +\set testid '31' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +-- Cleanup and create test databases +DROP DATABASE IF EXISTS cloudsync_test_31a; +DROP DATABASE IF EXISTS cloudsync_test_31b; +CREATE DATABASE cloudsync_test_31a; +CREATE DATABASE cloudsync_test_31b; + +-- ============================================================================ +-- Setup Database A +-- ============================================================================ + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('products', 'CLS', 0) AS _init_a \gset + +INSERT INTO products VALUES ('11111111-1111-1111-1111-111111111111', 'Product A1', 10.99, 100); +INSERT INTO products VALUES ('22222222-2222-2222-2222-222222222222', 'Product A2', 20.50, 200); + +-- ============================================================================ +-- Setup Database B with same schema +-- ============================================================================ + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE products ( + id UUID PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + price DOUBLE PRECISION NOT NULL DEFAULT 0.0, + quantity INTEGER NOT NULL DEFAULT 0 +); + +SELECT cloudsync_init('products', 'CLS', 0) AS _init_b \gset + +INSERT INTO products VALUES ('33333333-3333-3333-3333-333333333333', 'Product B1', 30.00, 300); +INSERT INTO products VALUES ('44444444-4444-4444-4444-444444444444', 'Product B2', 40.75, 400); + +-- ============================================================================ +-- Initial Sync: A -> B and B -> A +-- ============================================================================ + +\echo [INFO] (:testid) === Initial Sync Before ALTER === + +-- Encode payload from A +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A's payload to B, encode B's payload +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS apply_a_to_b \gset + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply B's payload to A, verify initial sync +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b_hex', 'hex')) AS apply_b_to_a \gset + +SELECT COUNT(*) AS count_a_initial FROM products \gset + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT COUNT(*) AS count_b_initial FROM products \gset + +SELECT (:count_a_initial = 4 AND :count_b_initial = 4) AS initial_sync_ok \gset +\if :initial_sync_ok +\echo [PASS] (:testid) Initial sync complete - both databases have 4 rows +\else +\echo [FAIL] (:testid) Initial sync failed - A: :count_a_initial, B: :count_b_initial +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ALTER TABLE on Database A (begin_alter + ALTER + commit_alter on SAME connection) +-- ============================================================================ + +\echo [INFO] (:testid) === ALTER TABLE on Database A === + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset + +SELECT cloudsync_begin_alter('products') AS begin_alter_a \gset +\if :begin_alter_a +\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database A +\else +\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; + +SELECT cloudsync_commit_alter('products') AS commit_alter_a \gset +\if :commit_alter_a +\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database A +\else +\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Insert and update post-ALTER data on A +INSERT INTO products (id, name, price, quantity, description) +VALUES ('55555555-5555-5555-5555-555555555555', 'New Product A', 55.55, 555, 'Added after alter on A'); + +UPDATE products SET description = 'Updated on A' WHERE id = '11111111-1111-1111-1111-111111111111'; +UPDATE products SET quantity = 150 WHERE id = '11111111-1111-1111-1111-111111111111'; + +-- Encode post-ALTER payload from A +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +SELECT (length(:'payload_a2_hex') > 0) AS payload_a2_created \gset +\if :payload_a2_created +\echo [PASS] (:testid) Post-alter payload encoded from Database A +\else +\echo [FAIL] (:testid) Post-alter payload empty from Database A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- ALTER TABLE on Database B (begin_alter + ALTER + commit_alter on SAME connection) +-- Apply A's payload, insert/update, encode B's payload +-- ============================================================================ + +\echo [INFO] (:testid) === ALTER TABLE on Database B === + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset + +SELECT cloudsync_begin_alter('products') AS begin_alter_b \gset +\if :begin_alter_b +\echo [PASS] (:testid) cloudsync_begin_alter succeeded on Database B +\else +\echo [FAIL] (:testid) cloudsync_begin_alter failed on Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +ALTER TABLE products ADD COLUMN description TEXT NOT NULL DEFAULT ''; + +SELECT cloudsync_commit_alter('products') AS commit_alter_b \gset +\if :commit_alter_b +\echo [PASS] (:testid) cloudsync_commit_alter succeeded on Database B +\else +\echo [FAIL] (:testid) cloudsync_commit_alter failed on Database B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Insert and update post-ALTER data on B +INSERT INTO products (id, name, price, quantity, description) +VALUES ('66666666-6666-6666-6666-666666666666', 'New Product B', 66.66, 666, 'Added after alter on B'); + +UPDATE products SET description = 'Updated on B' WHERE id = '33333333-3333-3333-3333-333333333333'; +UPDATE products SET quantity = 350 WHERE id = '33333333-3333-3333-3333-333333333333'; + +-- Apply A's post-alter payload to B +SELECT cloudsync_payload_apply(decode(:'payload_a2_hex', 'hex')) AS apply_a2_to_b \gset + +SELECT (:apply_a2_to_b >= 0) AS apply_a2_ok \gset +\if :apply_a2_ok +\echo [PASS] (:testid) Post-alter payload from A applied to B +\else +\echo [FAIL] (:testid) Post-alter payload from A failed to apply to B: :apply_a2_to_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Encode post-ALTER payload from B +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_b2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- ============================================================================ +-- Apply B's payload to A, then verify final state +-- ============================================================================ + +\echo [INFO] (:testid) === Apply B payload to A and verify === + +\connect cloudsync_test_31a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('products', 'CLS', 0) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b2_hex', 'hex')) AS apply_b2_to_a \gset + +SELECT (:apply_b2_to_a >= 0) AS apply_b2_ok \gset +\if :apply_b2_ok +\echo [PASS] (:testid) Post-alter payload from B applied to A +\else +\echo [FAIL] (:testid) Post-alter payload from B failed to apply to A: :apply_b2_to_a +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Verify final state +-- ============================================================================ + +\echo [INFO] (:testid) === Verify Final State === + +-- Compute hash of Database A +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_a_final FROM products \gset + +\echo [INFO] (:testid) Database A final hash: :hash_a_final + +-- Row count on A +SELECT COUNT(*) AS count_a_final FROM products \gset + +-- Verify new row from B exists in A +SELECT COUNT(*) = 1 AS new_row_b_ok +FROM products +WHERE id = '66666666-6666-6666-6666-666666666666' + AND name = 'New Product B' + AND price = 66.66 + AND quantity = 666 + AND description = 'Added after alter on B' \gset + +-- Verify updated row from B synced to A +SELECT COUNT(*) = 1 AS updated_row_b_ok +FROM products +WHERE id = '33333333-3333-3333-3333-333333333333' + AND description = 'Updated on B' + AND quantity = 350 \gset + +\connect cloudsync_test_31b +\ir helper_psql_conn_setup.sql + +-- Compute hash of Database B +SELECT md5( + COALESCE( + string_agg( + id::text || ':' || + COALESCE(name, 'NULL') || ':' || + COALESCE(price::text, 'NULL') || ':' || + COALESCE(quantity::text, 'NULL') || ':' || + COALESCE(description, 'NULL'), + '|' ORDER BY id + ), + '' + ) +) AS hash_b_final FROM products \gset + +\echo [INFO] (:testid) Database B final hash: :hash_b_final + +-- Row count on B +SELECT COUNT(*) AS count_b_final FROM products \gset + +-- Verify new row from A exists in B +SELECT COUNT(*) = 1 AS new_row_a_ok +FROM products +WHERE id = '55555555-5555-5555-5555-555555555555' + AND name = 'New Product A' + AND price = 55.55 + AND quantity = 555 + AND description = 'Added after alter on A' \gset + +-- Verify updated row from A synced to B +SELECT COUNT(*) = 1 AS updated_row_a_ok +FROM products +WHERE id = '11111111-1111-1111-1111-111111111111' + AND description = 'Updated on A' + AND quantity = 150 \gset + +-- Verify new column exists +SELECT COUNT(*) = 1 AS description_column_exists +FROM information_schema.columns +WHERE table_name = 'products' AND column_name = 'description' \gset + +-- ============================================================================ +-- Report results +-- ============================================================================ + +-- Compare final hashes +SELECT (:'hash_a_final' = :'hash_b_final') AS final_hashes_match \gset +\if :final_hashes_match +\echo [PASS] (:testid) Final data integrity verified - hashes match after ALTER +\else +\echo [FAIL] (:testid) Final data integrity check failed - A: :hash_a_final, B: :hash_b_final +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:count_a_final = 6 AND :count_b_final = 6) AS row_counts_ok \gset +\if :row_counts_ok +\echo [PASS] (:testid) Row counts match (6 rows each) +\else +\echo [FAIL] (:testid) Row counts mismatch - A: :count_a_final, B: :count_b_final +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :new_row_a_ok +\echo [PASS] (:testid) New row from A synced to B with new schema +\else +\echo [FAIL] (:testid) New row from A not found or incorrect in B +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :new_row_b_ok +\echo [PASS] (:testid) New row from B synced to A with new schema +\else +\echo [FAIL] (:testid) New row from B not found or incorrect in A +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :updated_row_a_ok +\echo [PASS] (:testid) Updated row from A synced with new column values +\else +\echo [FAIL] (:testid) Updated row from A not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :updated_row_b_ok +\echo [PASS] (:testid) Updated row from B synced with new column values +\else +\echo [FAIL] (:testid) Updated row from B not synced correctly +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :description_column_exists +\echo [PASS] (:testid) Added column 'description' exists +\else +\echo [FAIL] (:testid) Added column 'description' not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================================ +-- Cleanup +-- ============================================================================ + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_31a; +DROP DATABASE IF EXISTS cloudsync_test_31b; +\endif diff --git a/test/postgresql/32_block_lww.sql b/test/postgresql/32_block_lww.sql new file mode 100644 index 0000000..389408f --- /dev/null +++ b/test/postgresql/32_block_lww.sql @@ -0,0 +1,146 @@ +-- 'Block-level LWW test' + +\set testid '32' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_test_a; +CREATE DATABASE cloudsync_block_test_a; + +\connect cloudsync_block_test_a +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a table with a text column for block-level LWW +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); + +-- Initialize cloudsync for the table +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset + +-- Configure body column as block-level +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol \gset + +-- Test 1: INSERT text, verify blocks table populated +INSERT INTO docs (id, body) VALUES ('doc1', 'line1 +line2 +line3'); + +-- Verify blocks table was created +SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'docs_cloudsync_blocks') AS blocks_table_exists \gset +\if :blocks_table_exists +\echo [PASS] (:testid) Blocks table created +\else +\echo [FAIL] (:testid) Blocks table not created +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify blocks have been stored (3 lines = 3 blocks) +SELECT count(*) AS block_count FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:block_count::int = 3) AS insert_blocks_ok \gset +\if :insert_blocks_ok +\echo [PASS] (:testid) Block insert: 3 blocks created +\else +\echo [FAIL] (:testid) Block insert: expected 3 blocks, got :block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify metadata has block entries (col_name contains \x1F separator) +SELECT count(*) AS meta_block_count FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_block_count::int = 3) AS meta_blocks_ok \gset +\if :meta_blocks_ok +\echo [PASS] (:testid) Block metadata: 3 block entries in _cloudsync +\else +\echo [FAIL] (:testid) Block metadata: expected 3 entries, got :meta_block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: UPDATE text (modify one line, add one line) +UPDATE docs SET body = 'line1 +line2_modified +line3 +line4' WHERE id = 'doc1'; + +-- Verify blocks updated (should now have 4 blocks) +SELECT count(*) AS block_count2 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:block_count2::int = 4) AS update_blocks_ok \gset +\if :update_blocks_ok +\echo [PASS] (:testid) Block update: 4 blocks after update +\else +\echo [FAIL] (:testid) Block update: expected 4 blocks, got :block_count2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Materialize and verify round-trip +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat \gset +SELECT body AS materialized_body FROM docs WHERE id = 'doc1' \gset + +SELECT (:'materialized_body' = 'line1 +line2_modified +line3 +line4') AS materialize_ok \gset +\if :materialize_ok +\echo [PASS] (:testid) Text materialize: reconstructed text matches +\else +\echo [FAIL] (:testid) Text materialize: text mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Verify col_value works for block entries +SELECT count(*) AS col_value_count FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' +AND cloudsync_col_value('docs', col_name, pk) IS NOT NULL \gset +SELECT (:col_value_count::int > 0) AS col_value_ok \gset +\if :col_value_ok +\echo [PASS] (:testid) col_value works for block entries +\else +\echo [FAIL] (:testid) col_value returned NULL for block entries +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Sync roundtrip - encode payload from db A before disconnecting +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS block_payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_test_b; +CREATE DATABASE cloudsync_block_test_b; +\connect cloudsync_block_test_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +SELECT cloudsync_payload_apply(decode(:'block_payload_hex', 'hex')) AS _apply_b \gset + +-- Materialize on db B +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_b \gset +SELECT body AS body_b FROM docs WHERE id = 'doc1' \gset + +SELECT (:'body_b' = 'line1 +line2_modified +line3 +line4') AS sync_ok \gset +\if :sync_ok +\echo [PASS] (:testid) Block sync roundtrip: text matches after apply + materialize +\else +\echo [FAIL] (:testid) Block sync roundtrip: text mismatch on db B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_test_a; +DROP DATABASE IF EXISTS cloudsync_block_test_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/33_block_lww_extended.sql b/test/postgresql/33_block_lww_extended.sql new file mode 100644 index 0000000..8b5de48 --- /dev/null +++ b/test/postgresql/33_block_lww_extended.sql @@ -0,0 +1,339 @@ +-- 'Block-level LWW extended tests: DELETE, empty text, multi-update, conflict' + +\set testid '33' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_ext_a; +DROP DATABASE IF EXISTS cloudsync_block_ext_b; +CREATE DATABASE cloudsync_block_ext_a; +CREATE DATABASE cloudsync_block_ext_b; + +-- ============================================================ +-- Setup db A +-- ============================================================ +\connect cloudsync_block_ext_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_a \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_a \gset + +-- ============================================================ +-- Test 1: DELETE marks tombstone, block metadata dropped +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc1', 'line1 +line2 +line3'); + +-- Verify 3 block metadata entries exist +SELECT count(*) AS meta_before FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_before::int = 3) AS meta_before_ok \gset +\if :meta_before_ok +\echo [PASS] (:testid) Delete pre-check: 3 block metadata entries +\else +\echo [FAIL] (:testid) Delete pre-check: expected 3 metadata, got :meta_before +SELECT (:fail::int + 1) AS fail \gset +\endif + +DELETE FROM docs WHERE id = 'doc1'; + +-- Tombstone should exist with even version (deleted) +SELECT count(*) AS tombstone_count FROM docs_cloudsync WHERE col_name = '__[RIP]__' AND col_version % 2 = 0 \gset +SELECT (:tombstone_count::int = 1) AS tombstone_ok \gset +\if :tombstone_ok +\echo [PASS] (:testid) Delete: tombstone exists with even version +\else +\echo [FAIL] (:testid) Delete: expected 1 tombstone, got :tombstone_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Block metadata should be dropped +SELECT count(*) AS meta_after FROM docs_cloudsync WHERE col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_after::int = 0) AS meta_dropped_ok \gset +\if :meta_dropped_ok +\echo [PASS] (:testid) Delete: block metadata dropped +\else +\echo [FAIL] (:testid) Delete: expected 0 metadata after delete, got :meta_after +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Row should be gone from base table +SELECT count(*) AS row_after FROM docs WHERE id = 'doc1' \gset +SELECT (:row_after::int = 0) AS row_gone_ok \gset +\if :row_gone_ok +\echo [PASS] (:testid) Delete: row removed from base table +\else +\echo [FAIL] (:testid) Delete: row still in base table +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Empty text creates single block +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc_empty', ''); + +SELECT count(*) AS empty_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_empty') \gset +SELECT (:empty_blocks::int = 1) AS empty_block_ok \gset +\if :empty_block_ok +\echo [PASS] (:testid) Empty text: 1 block created +\else +\echo [FAIL] (:testid) Empty text: expected 1 block, got :empty_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update from empty to multi-line +UPDATE docs SET body = 'NewLine1 +NewLine2' WHERE id = 'doc_empty'; + +SELECT count(*) AS updated_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_empty') \gset +SELECT (:updated_blocks::int = 2) AS update_from_empty_ok \gset +\if :update_from_empty_ok +\echo [PASS] (:testid) Empty text: 2 blocks after update +\else +\echo [FAIL] (:testid) Empty text: expected 2 blocks after update, got :updated_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Multi-update block counts +-- ============================================================ +INSERT INTO docs (id, body) VALUES ('doc_multi', 'A +B +C'); + +-- Update 1: remove middle line +UPDATE docs SET body = 'A +C' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks1 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks1::int = 2) AS multi1_ok \gset +\if :multi1_ok +\echo [PASS] (:testid) Multi-update: 2 blocks after removing middle +\else +\echo [FAIL] (:testid) Multi-update: expected 2, got :blocks1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 2: add two lines +UPDATE docs SET body = 'A +X +C +Y' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks2 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks2::int = 4) AS multi2_ok \gset +\if :multi2_ok +\echo [PASS] (:testid) Multi-update: 4 blocks after adding lines +\else +\echo [FAIL] (:testid) Multi-update: expected 4, got :blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 3: collapse to single line +UPDATE docs SET body = 'SINGLE' WHERE id = 'doc_multi'; + +SELECT count(*) AS blocks3 FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_multi') \gset +SELECT (:blocks3::int = 1) AS multi3_ok \gset +\if :multi3_ok +\echo [PASS] (:testid) Multi-update: 1 block after collapse +\else +\echo [FAIL] (:testid) Multi-update: expected 1, got :blocks3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Materialize and verify +SELECT cloudsync_text_materialize('docs', 'body', 'doc_multi') AS _mat_multi \gset +SELECT body AS multi_body FROM docs WHERE id = 'doc_multi' \gset +SELECT (:'multi_body' = 'SINGLE') AS multi_mat_ok \gset +\if :multi_mat_ok +\echo [PASS] (:testid) Multi-update: materialize matches +\else +\echo [FAIL] (:testid) Multi-update: materialize mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Two-database conflict on same block +-- ============================================================ + +-- Setup db B +\connect cloudsync_block_ext_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +-- Insert initial doc on db A +\connect cloudsync_block_ext_a +INSERT INTO docs (id, body) VALUES ('doc_conflict', 'Same +Middle +End'); + +-- Sync A -> B (round 1) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a_r1 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_a_r1', 'hex')) AS _apply_b_r1 \gset + +-- Materialize on B to get body +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_b_init \gset + +-- Verify B has the initial doc +SELECT body AS body_b_init FROM docs WHERE id = 'doc_conflict' \gset +SELECT (:'body_b_init' = 'Same +Middle +End') AS init_sync_ok \gset +\if :init_sync_ok +\echo [PASS] (:testid) Conflict: initial sync to B matches +\else +\echo [FAIL] (:testid) Conflict: initial sync to B mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Site A edits first line +\connect cloudsync_block_ext_a +UPDATE docs SET body = 'SiteA +Middle +End' WHERE id = 'doc_conflict'; + +-- Site B edits first line (conflict!) +\connect cloudsync_block_ext_b +UPDATE docs SET body = 'SiteB +Middle +End' WHERE id = 'doc_conflict'; + +-- Collect payloads from both sites +\connect cloudsync_block_ext_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b_r2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A's changes to B +SELECT cloudsync_payload_apply(decode(:'payload_a_r2', 'hex')) AS _apply_b_r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_b_r2 \gset + +-- Apply B's changes to A +\connect cloudsync_block_ext_a +SELECT cloudsync_payload_apply(decode(:'payload_b_r2', 'hex')) AS _apply_a_r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_conflict') AS _mat_a_r2 \gset + +-- Both should converge +SELECT body AS body_a_final FROM docs WHERE id = 'doc_conflict' \gset + +\connect cloudsync_block_ext_b +SELECT body AS body_b_final FROM docs WHERE id = 'doc_conflict' \gset + +-- Bodies must match (convergence) +SELECT (:'body_a_final' = :'body_b_final') AS converge_ok \gset +\if :converge_ok +\echo [PASS] (:testid) Conflict: databases converge after sync +\else +\echo [FAIL] (:testid) Conflict: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Unchanged lines must be preserved +SELECT (position('Middle' in :'body_a_final') > 0) AS has_middle \gset +\if :has_middle +\echo [PASS] (:testid) Conflict: unchanged line 'Middle' preserved +\else +\echo [FAIL] (:testid) Conflict: 'Middle' missing from result +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('End' in :'body_a_final') > 0) AS has_end \gset +\if :has_end +\echo [PASS] (:testid) Conflict: unchanged line 'End' preserved +\else +\echo [FAIL] (:testid) Conflict: 'End' missing from result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- One of the conflicting edits must win +SELECT (position('SiteA' in :'body_a_final') > 0 OR position('SiteB' in :'body_a_final') > 0) AS has_winner \gset +\if :has_winner +\echo [PASS] (:testid) Conflict: one site edit won (LWW) +\else +\echo [FAIL] (:testid) Conflict: neither SiteA nor SiteB in result +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: DELETE then re-INSERT (reinsert) +-- ============================================================ +\connect cloudsync_block_ext_a + +INSERT INTO docs (id, body) VALUES ('doc_reinsert', 'Old1 +Old2'); +DELETE FROM docs WHERE id = 'doc_reinsert'; + +-- Block metadata should be dropped after delete +SELECT count(*) AS meta_reinsert_del FROM docs_cloudsync +WHERE pk = cloudsync_pk_encode('doc_reinsert') +AND col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_reinsert_del::int = 0) AS reinsert_meta_del_ok \gset +\if :reinsert_meta_del_ok +\echo [PASS] (:testid) Reinsert: metadata dropped after delete +\else +\echo [FAIL] (:testid) Reinsert: expected 0 metadata, got :meta_reinsert_del +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Re-insert with new content +INSERT INTO docs (id, body) VALUES ('doc_reinsert', 'New1 +New2 +New3'); + +SELECT count(*) AS meta_reinsert_new FROM docs_cloudsync +WHERE pk = cloudsync_pk_encode('doc_reinsert') +AND col_name LIKE 'body' || chr(31) || '%' \gset +SELECT (:meta_reinsert_new::int = 3) AS reinsert_meta_ok \gset +\if :reinsert_meta_ok +\echo [PASS] (:testid) Reinsert: 3 block metadata after re-insert +\else +\echo [FAIL] (:testid) Reinsert: expected 3 metadata, got :meta_reinsert_new +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B and materialize +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_reinsert +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_ext_b +SELECT cloudsync_payload_apply(decode(:'payload_reinsert', 'hex')) AS _apply_reinsert \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_reinsert') AS _mat_reinsert \gset +SELECT body AS body_reinsert FROM docs WHERE id = 'doc_reinsert' \gset + +SELECT (:'body_reinsert' = 'New1 +New2 +New3') AS reinsert_sync_ok \gset +\if :reinsert_sync_ok +\echo [PASS] (:testid) Reinsert: sync roundtrip matches +\else +\echo [FAIL] (:testid) Reinsert: sync mismatch on db B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_ext_a; +DROP DATABASE IF EXISTS cloudsync_block_ext_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/34_block_lww_advanced.sql b/test/postgresql/34_block_lww_advanced.sql new file mode 100644 index 0000000..f8753ff --- /dev/null +++ b/test/postgresql/34_block_lww_advanced.sql @@ -0,0 +1,698 @@ +-- 'Block-level LWW advanced tests: noconflict, add+edit, three-way, mixed cols, NULL->text, interleaved, custom delimiter, large text, rapid updates' + +\set testid '34' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_adv_a; +DROP DATABASE IF EXISTS cloudsync_block_adv_b; +DROP DATABASE IF EXISTS cloudsync_block_adv_c; +CREATE DATABASE cloudsync_block_adv_a; +CREATE DATABASE cloudsync_block_adv_b; +CREATE DATABASE cloudsync_block_adv_c; + +-- ============================================================ +-- Test 1: Non-conflicting edits on different blocks +-- Site A edits line 1, Site B edits line 3 — BOTH should survive +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_a \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_a \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_b \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_b \gset + +-- Insert initial on A +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc1', 'Line1 +Line2 +Line3'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_init +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_init', 'hex')) AS _apply_init \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_init \gset + +-- Site A: edit first line +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'EditedByA +Line2 +Line3' WHERE id = 'doc1'; + +-- Site B: edit third line (no conflict — different block) +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'Line1 +Line2 +EditedByB' WHERE id = 'doc1'; + +-- Collect payloads +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +-- Apply A -> B, B -> A +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS _apply_ab \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_b', 'hex')) AS _apply_ba \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat_a \gset + +-- Both should converge +SELECT body AS body_a FROM docs WHERE id = 'doc1' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_b FROM docs WHERE id = 'doc1' \gset + +SELECT (:'body_a' = :'body_b') AS converge_ok \gset +\if :converge_ok +\echo [PASS] (:testid) NoConflict: databases converge +\else +\echo [FAIL] (:testid) NoConflict: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Both edits should be preserved +SELECT (position('EditedByA' in :'body_a') > 0) AS has_a_edit \gset +\if :has_a_edit +\echo [PASS] (:testid) NoConflict: Site A edit preserved +\else +\echo [FAIL] (:testid) NoConflict: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('EditedByB' in :'body_a') > 0) AS has_b_edit \gset +\if :has_b_edit +\echo [PASS] (:testid) NoConflict: Site B edit preserved +\else +\echo [FAIL] (:testid) NoConflict: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Line2' in :'body_a') > 0) AS has_middle \gset +\if :has_middle +\echo [PASS] (:testid) NoConflict: unchanged line preserved +\else +\echo [FAIL] (:testid) NoConflict: unchanged line missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Concurrent add + edit +-- Site A adds a line, Site B modifies an existing line +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc2', 'Alpha +Bravo'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_d2_init', 'hex')) AS _apply_d2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2 \gset + +-- Site A: add a new line at end +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'Alpha +Bravo +Charlie' WHERE id = 'doc2'; + +-- Site B: modify first line +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'AlphaEdited +Bravo' WHERE id = 'doc2'; + +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_d2b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +SELECT cloudsync_payload_apply(decode(:'payload_d2a', 'hex')) AS _apply_d2ab \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_d2b', 'hex')) AS _apply_d2ba \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat_d2a \gset + +SELECT body AS body_d2a FROM docs WHERE id = 'doc2' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_d2b FROM docs WHERE id = 'doc2' \gset + +SELECT (:'body_d2a' = :'body_d2b') AS d2_converge \gset +\if :d2_converge +\echo [PASS] (:testid) Add+Edit: databases converge +\else +\echo [FAIL] (:testid) Add+Edit: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Charlie' in :'body_d2a') > 0) AS has_charlie \gset +\if :has_charlie +\echo [PASS] (:testid) Add+Edit: added line Charlie preserved +\else +\echo [FAIL] (:testid) Add+Edit: added line Charlie missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Bravo' in :'body_d2a') > 0) AS has_bravo \gset +\if :has_bravo +\echo [PASS] (:testid) Add+Edit: unchanged Bravo preserved +\else +\echo [FAIL] (:testid) Add+Edit: Bravo missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Three-way sync — 3 databases, each edits a different line +-- ============================================================ +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init_c \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _setcol_c \gset + +-- Insert initial on A +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc3', 'L1 +L2 +L3 +L4'); + +-- Sync A -> B, A -> C +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3init', 'hex')) AS _apply_3b \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3b \gset + +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3init', 'hex')) AS _apply_3c \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3c \gset + +-- A edits line 1 +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'S0 +L2 +L3 +L4' WHERE id = 'doc3'; + +-- B edits line 2 +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'L1 +S1 +L3 +L4' WHERE id = 'doc3'; + +-- C edits line 4 +\connect cloudsync_block_adv_c +UPDATE docs SET body = 'L1 +L2 +L3 +S2' WHERE id = 'doc3'; + +-- Collect all payloads +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_c +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3c +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +-- Full mesh apply: each site receives from the other two +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_3b', 'hex')) AS _3ab \gset +SELECT cloudsync_payload_apply(decode(:'payload_3c', 'hex')) AS _3ac \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3a_final \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3a', 'hex')) AS _3ba \gset +SELECT cloudsync_payload_apply(decode(:'payload_3c', 'hex')) AS _3bc \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3b_final \gset + +\connect cloudsync_block_adv_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3a', 'hex')) AS _3ca \gset +SELECT cloudsync_payload_apply(decode(:'payload_3b', 'hex')) AS _3cb \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat_3c_final \gset + +-- All three should converge +\connect cloudsync_block_adv_a +SELECT body AS body_3a FROM docs WHERE id = 'doc3' \gset +\connect cloudsync_block_adv_b +SELECT body AS body_3b FROM docs WHERE id = 'doc3' \gset +\connect cloudsync_block_adv_c +SELECT body AS body_3c FROM docs WHERE id = 'doc3' \gset + +SELECT (:'body_3a' = :'body_3b' AND :'body_3b' = :'body_3c') AS three_converge \gset +\if :three_converge +\echo [PASS] (:testid) Three-way: all 3 databases converge +\else +\echo [FAIL] (:testid) Three-way: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S0' in :'body_3a') > 0) AS has_s0 \gset +\if :has_s0 +\echo [PASS] (:testid) Three-way: Site A edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S1' in :'body_3a') > 0) AS has_s1 \gset +\if :has_s1 +\echo [PASS] (:testid) Three-way: Site B edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('S2' in :'body_3a') > 0) AS has_s2 \gset +\if :has_s2 +\echo [PASS] (:testid) Three-way: Site C edit preserved +\else +\echo [FAIL] (:testid) Three-way: Site C edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Mixed block + normal columns +-- ============================================================ +\connect cloudsync_block_adv_a +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init_notes_a \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _setcol_notes_a \gset + +\connect cloudsync_block_adv_b +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init_notes_b \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _setcol_notes_b \gset + +\connect cloudsync_block_adv_a +INSERT INTO notes (id, body, title) VALUES ('n1', 'Line1 +Line2 +Line3', 'My Title'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_notes_init', 'hex')) AS _apply_notes \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes \gset + +-- A: edit block line 1 + title +\connect cloudsync_block_adv_a +UPDATE notes SET body = 'EditedLine1 +Line2 +Line3', title = 'Title From A' WHERE id = 'n1'; + +-- B: edit block line 3 + title (title conflicts via normal LWW) +\connect cloudsync_block_adv_b +UPDATE notes SET body = 'Line1 +Line2 +EditedLine3', title = 'Title From B' WHERE id = 'n1'; + +\connect cloudsync_block_adv_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_adv_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_notes_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +SELECT cloudsync_payload_apply(decode(:'payload_notes_a', 'hex')) AS _apply_notes_ab \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes_b \gset + +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_notes_b', 'hex')) AS _apply_notes_ba \gset +SELECT cloudsync_text_materialize('notes', 'body', 'n1') AS _mat_notes_a \gset + +SELECT body AS notes_body_a FROM notes WHERE id = 'n1' \gset +SELECT title AS notes_title_a FROM notes WHERE id = 'n1' \gset +\connect cloudsync_block_adv_b +SELECT body AS notes_body_b FROM notes WHERE id = 'n1' \gset +SELECT title AS notes_title_b FROM notes WHERE id = 'n1' \gset + +SELECT (:'notes_body_a' = :'notes_body_b') AS mixed_body_ok \gset +\if :mixed_body_ok +\echo [PASS] (:testid) MixedCols: body converges +\else +\echo [FAIL] (:testid) MixedCols: body diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('EditedLine1' in :'notes_body_a') > 0 AND position('EditedLine3' in :'notes_body_a') > 0) AS both_edits \gset +\if :both_edits +\echo [PASS] (:testid) MixedCols: both block edits preserved +\else +\echo [FAIL] (:testid) MixedCols: block edits missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'notes_title_a' = :'notes_title_b') AS mixed_title_ok \gset +\if :mixed_title_ok +\echo [PASS] (:testid) MixedCols: title converges (normal LWW) +\else +\echo [FAIL] (:testid) MixedCols: title diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: NULL to text transition +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc_null', NULL); + +-- Verify 1 block for NULL +SELECT count(*) AS null_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_null') \gset +SELECT (:null_blocks::int = 1) AS null_block_ok \gset +\if :null_block_ok +\echo [PASS] (:testid) NULL->Text: 1 block for NULL body +\else +\echo [FAIL] (:testid) NULL->Text: expected 1 block, got :null_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update to multi-line +UPDATE docs SET body = 'Hello +World +Foo' WHERE id = 'doc_null'; + +SELECT count(*) AS text_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_null') \gset +SELECT (:text_blocks::int = 3) AS text_block_ok \gset +\if :text_block_ok +\echo [PASS] (:testid) NULL->Text: 3 blocks after update +\else +\echo [FAIL] (:testid) NULL->Text: expected 3 blocks, got :text_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_null +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_null', 'hex')) AS _apply_null \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_null') AS _mat_null \gset + +SELECT body AS body_null FROM docs WHERE id = 'doc_null' \gset +SELECT (:'body_null' = 'Hello +World +Foo') AS null_text_ok \gset +\if :null_text_ok +\echo [PASS] (:testid) NULL->Text: sync roundtrip matches +\else +\echo [FAIL] (:testid) NULL->Text: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Interleaved inserts — multiple rounds between existing lines +-- ============================================================ +\connect cloudsync_block_adv_a +INSERT INTO docs (id, body) VALUES ('doc_inter', 'A +B'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_init', 'hex')) AS _apply_inter \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_inter \gset + +-- Round 1: A inserts between A and B +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'A +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_r1', 'hex')) AS _r1 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r1 \gset + +-- Round 2: B inserts between A and C +\connect cloudsync_block_adv_b +UPDATE docs SET body = 'A +D +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_a +SELECT cloudsync_payload_apply(decode(:'payload_inter_r2', 'hex')) AS _r2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r2 \gset + +-- Round 3: A inserts between D and C +\connect cloudsync_block_adv_a +UPDATE docs SET body = 'A +D +E +C +B' WHERE id = 'doc_inter'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_inter_r3 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_inter_r3', 'hex')) AS _r3 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc_inter') AS _mat_r3 \gset + +\connect cloudsync_block_adv_a +SELECT body AS inter_body_a FROM docs WHERE id = 'doc_inter' \gset +\connect cloudsync_block_adv_b +SELECT body AS inter_body_b FROM docs WHERE id = 'doc_inter' \gset + +SELECT (:'inter_body_a' = :'inter_body_b') AS inter_converge \gset +\if :inter_converge +\echo [PASS] (:testid) Interleaved: databases converge +\else +\echo [FAIL] (:testid) Interleaved: databases diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS inter_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc_inter') \gset +SELECT (:inter_blocks::int = 5) AS inter_count_ok \gset +\if :inter_count_ok +\echo [PASS] (:testid) Interleaved: 5 blocks after 3 rounds +\else +\echo [FAIL] (:testid) Interleaved: expected 5 blocks, got :inter_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Custom delimiter (paragraph separator: double newline) +-- ============================================================ +\connect cloudsync_block_adv_a +DROP TABLE IF EXISTS paragraphs; +CREATE TABLE paragraphs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('paragraphs', 'CLS', 1) AS _init_para \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'algo', 'block') AS _setcol_para \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'delimiter', E'\n\n') AS _setdelim \gset + +\connect cloudsync_block_adv_b +DROP TABLE IF EXISTS paragraphs; +CREATE TABLE paragraphs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('paragraphs', 'CLS', 1) AS _init_para_b \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'algo', 'block') AS _setcol_para_b \gset +SELECT cloudsync_set_column('paragraphs', 'body', 'delimiter', E'\n\n') AS _setdelim_b \gset + +\connect cloudsync_block_adv_a +INSERT INTO paragraphs (id, body) VALUES ('p1', E'Para one line1\nline2\n\nPara two\n\nPara three'); + +-- Should produce 3 blocks (3 paragraphs) +SELECT count(*) AS para_blocks FROM paragraphs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('p1') \gset +SELECT (:para_blocks::int = 3) AS para_ok \gset +\if :para_ok +\echo [PASS] (:testid) CustomDelim: 3 paragraph blocks +\else +\echo [FAIL] (:testid) CustomDelim: expected 3 blocks, got :para_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify roundtrip +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_para +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'paragraphs' \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_para', 'hex')) AS _apply_para \gset +SELECT cloudsync_text_materialize('paragraphs', 'body', 'p1') AS _mat_para \gset + +SELECT body AS para_body FROM paragraphs WHERE id = 'p1' \gset +SELECT (:'para_body' = E'Para one line1\nline2\n\nPara two\n\nPara three') AS para_roundtrip \gset +\if :para_roundtrip +\echo [PASS] (:testid) CustomDelim: sync roundtrip matches +\else +\echo [FAIL] (:testid) CustomDelim: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Large text — 200 lines +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +INSERT INTO docs (id, body) +SELECT 'bigdoc', string_agg('Line ' || lpad(i::text, 3, '0') || ' content', E'\n' ORDER BY i) +FROM generate_series(0, 199) AS s(i); + +SELECT count(*) AS big_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('bigdoc') \gset +SELECT (:big_blocks::int = 200) AS big_ok \gset +\if :big_ok +\echo [PASS] (:testid) LargeText: 200 blocks created +\else +\echo [FAIL] (:testid) LargeText: expected 200 blocks, got :big_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- All positions unique +SELECT count(DISTINCT col_name) AS big_distinct FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' +AND pk = cloudsync_pk_encode('bigdoc') \gset +SELECT (:big_distinct::int = 200) AS big_unique \gset +\if :big_unique +\echo [PASS] (:testid) LargeText: 200 unique position IDs +\else +\echo [FAIL] (:testid) LargeText: expected 200 unique positions, got :big_distinct +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_big +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_big', 'hex')) AS _apply_big \gset +SELECT cloudsync_text_materialize('docs', 'body', 'bigdoc') AS _mat_big \gset + +SELECT body AS big_body_b FROM docs WHERE id = 'bigdoc' \gset +\connect cloudsync_block_adv_a +SELECT body AS big_body_a FROM docs WHERE id = 'bigdoc' \gset + +SELECT (:'big_body_a' = :'big_body_b') AS big_match \gset +\if :big_match +\echo [PASS] (:testid) LargeText: sync roundtrip matches +\else +\echo [FAIL] (:testid) LargeText: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 9: Rapid sequential updates — 50 updates on same row +-- ============================================================ +\connect cloudsync_block_adv_a +\ir helper_psql_conn_setup.sql +INSERT INTO docs (id, body) VALUES ('rapid', 'Start'); + +DO $$ +DECLARE + i INT; + new_body TEXT := ''; +BEGIN + FOR i IN 0..49 LOOP + IF i > 0 THEN new_body := new_body || E'\n'; END IF; + new_body := new_body || 'Update' || i; + UPDATE docs SET body = new_body WHERE id = 'rapid'; + END LOOP; +END $$; + +SELECT count(*) AS rapid_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('rapid') \gset +SELECT (:rapid_blocks::int = 50) AS rapid_ok \gset +\if :rapid_ok +\echo [PASS] (:testid) RapidUpdates: 50 blocks after 50 updates +\else +\echo [FAIL] (:testid) RapidUpdates: expected 50 blocks, got :rapid_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync and verify +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_rapid +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_adv_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_rapid', 'hex')) AS _apply_rapid \gset +SELECT cloudsync_text_materialize('docs', 'body', 'rapid') AS _mat_rapid \gset + +SELECT body AS rapid_body_b FROM docs WHERE id = 'rapid' \gset +\connect cloudsync_block_adv_a +SELECT body AS rapid_body_a FROM docs WHERE id = 'rapid' \gset + +SELECT (:'rapid_body_a' = :'rapid_body_b') AS rapid_match \gset +\if :rapid_match +\echo [PASS] (:testid) RapidUpdates: sync roundtrip matches +\else +\echo [FAIL] (:testid) RapidUpdates: sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Update0' in :'rapid_body_a') > 0) AS has_first \gset +\if :has_first +\echo [PASS] (:testid) RapidUpdates: first update present +\else +\echo [FAIL] (:testid) RapidUpdates: first update missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (position('Update49' in :'rapid_body_a') > 0) AS has_last \gset +\if :has_last +\echo [PASS] (:testid) RapidUpdates: last update present +\else +\echo [FAIL] (:testid) RapidUpdates: last update missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_adv_a; +DROP DATABASE IF EXISTS cloudsync_block_adv_b; +DROP DATABASE IF EXISTS cloudsync_block_adv_c; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/35_block_lww_edge_cases.sql b/test/postgresql/35_block_lww_edge_cases.sql new file mode 100644 index 0000000..362d063 --- /dev/null +++ b/test/postgresql/35_block_lww_edge_cases.sql @@ -0,0 +1,420 @@ +-- 'Block-level LWW edge cases: unicode, special chars, delete vs edit, two block cols, text->NULL, payload sync, idempotent, ordering' + +\set testid '35' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_edge_a; +DROP DATABASE IF EXISTS cloudsync_block_edge_b; +CREATE DATABASE cloudsync_block_edge_a; +CREATE DATABASE cloudsync_block_edge_b; + +-- ============================================================ +-- Test 1: Unicode / multibyte content (emoji, CJK, accented) +-- ============================================================ +\connect cloudsync_block_edge_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert unicode text on A +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc1', E'Hello \U0001F600\nBonjour caf\u00e9\n\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc1') AS _mat \gset + +SELECT (body LIKE E'Hello %') AS unicode_ok FROM docs WHERE id = 'doc1' \gset +\if :unicode_ok +\echo [PASS] (:testid) Unicode: body starts with Hello +\else +\echo [FAIL] (:testid) Unicode: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Check line count (3 lines = 2 newlines) +SELECT (length(body) - length(replace(body, E'\n', '')) = 2) AS unicode_lines FROM docs WHERE id = 'doc1' \gset +\if :unicode_lines +\echo [PASS] (:testid) Unicode: 3 lines present +\else +\echo [FAIL] (:testid) Unicode: wrong line count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Special characters (tabs, backslashes, quotes) +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc2', E'line\twith\ttabs\nback\\\\slash\nO''Brien said "hi"'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc2') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc2') AS _mat \gset + +SELECT (body LIKE E'%\t%') AS special_tabs FROM docs WHERE id = 'doc2' \gset +\if :special_tabs +\echo [PASS] (:testid) SpecialChars: tabs preserved +\else +\echo [FAIL] (:testid) SpecialChars: tabs lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Brien%') AS special_quotes FROM docs WHERE id = 'doc2' \gset +\if :special_quotes +\echo [PASS] (:testid) SpecialChars: quotes preserved +\else +\echo [FAIL] (:testid) SpecialChars: quotes lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Delete vs edit — A deletes block 1, B edits block 2 +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc3', E'Alpha\nBeta\nGamma'); + +-- Sync initial to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc3') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat \gset + +-- A: remove first line +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'Beta\nGamma' WHERE id = 'doc3'; + +-- B: edit second line +\connect cloudsync_block_edge_b +UPDATE docs SET body = E'Alpha\nBetaEdited\nGamma' WHERE id = 'doc3'; + +-- Sync A -> B +\connect cloudsync_block_edge_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc3') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc3') AS _mat \gset + +-- B should have: Alpha removed (A wins), BetaEdited kept (B's edit) +SELECT (body NOT LIKE '%Alpha%') AS dve_no_alpha FROM docs WHERE id = 'doc3' \gset +\if :dve_no_alpha +\echo [PASS] (:testid) DelVsEdit: Alpha removed +\else +\echo [FAIL] (:testid) DelVsEdit: Alpha still present +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%BetaEdited%') AS dve_beta FROM docs WHERE id = 'doc3' \gset +\if :dve_beta +\echo [PASS] (:testid) DelVsEdit: BetaEdited present +\else +\echo [FAIL] (:testid) DelVsEdit: BetaEdited missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Gamma%') AS dve_gamma FROM docs WHERE id = 'doc3' \gset +\if :dve_gamma +\echo [PASS] (:testid) DelVsEdit: Gamma present +\else +\echo [FAIL] (:testid) DelVsEdit: Gamma missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Two block columns on the same table (body + notes) +-- ============================================================ +\connect cloudsync_block_edge_a +DROP TABLE IF EXISTS articles; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, body TEXT, notes TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('articles', 'body', 'algo', 'block') AS _sc1 \gset +SELECT cloudsync_set_column('articles', 'notes', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_edge_b +DROP TABLE IF EXISTS articles; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, body TEXT, notes TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('articles', 'body', 'algo', 'block') AS _sc1 \gset +SELECT cloudsync_set_column('articles', 'notes', 'algo', 'block') AS _sc2 \gset + +-- Insert on A +\connect cloudsync_block_edge_a +INSERT INTO articles (id, body, notes) VALUES ('art1', E'Body line 1\nBody line 2', E'Note 1\nNote 2'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'articles' \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'body', 'art1') AS _mb \gset +SELECT cloudsync_text_materialize('articles', 'notes', 'art1') AS _mn \gset + +SELECT (body = E'Body line 1\nBody line 2') AS twocol_body FROM articles WHERE id = 'art1' \gset +\if :twocol_body +\echo [PASS] (:testid) TwoBlockCols: body matches +\else +\echo [FAIL] (:testid) TwoBlockCols: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (notes = E'Note 1\nNote 2') AS twocol_notes FROM articles WHERE id = 'art1' \gset +\if :twocol_notes +\echo [PASS] (:testid) TwoBlockCols: notes matches +\else +\echo [FAIL] (:testid) TwoBlockCols: notes mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit body on A, notes on B — then sync +\connect cloudsync_block_edge_a +UPDATE articles SET body = E'Body EDITED\nBody line 2' WHERE id = 'art1'; + +\connect cloudsync_block_edge_b +UPDATE articles SET notes = E'Note 1\nNote EDITED' WHERE id = 'art1'; + +-- Sync A -> B +\connect cloudsync_block_edge_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'articles' \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'body', 'art1') AS _mb \gset +SELECT cloudsync_text_materialize('articles', 'notes', 'art1') AS _mn \gset + +SELECT (body LIKE '%Body EDITED%') AS twocol_body_ed FROM articles WHERE id = 'art1' \gset +\if :twocol_body_ed +\echo [PASS] (:testid) TwoBlockCols: body edited +\else +\echo [FAIL] (:testid) TwoBlockCols: body edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (notes LIKE '%Note EDITED%') AS twocol_notes_ed FROM articles WHERE id = 'art1' \gset +\if :twocol_notes_ed +\echo [PASS] (:testid) TwoBlockCols: notes kept +\else +\echo [FAIL] (:testid) TwoBlockCols: notes edit lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Text -> NULL (update to NULL removes all blocks) +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc5', E'Line1\nLine2\nLine3'); + +-- Verify blocks created +SELECT (count(*) = 3) AS blk_ok FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc5') \gset +\if :blk_ok +\echo [PASS] (:testid) TextToNull: 3 blocks created +\else +\echo [FAIL] (:testid) TextToNull: wrong initial block count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update to NULL +UPDATE docs SET body = NULL WHERE id = 'doc5'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload5 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc5') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload5', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc5') AS _mat \gset + +SELECT (body IS NULL) AS null_remote FROM docs WHERE id = 'doc5' \gset +\if :null_remote +\echo [PASS] (:testid) TextToNull: body is NULL on remote +\else +\echo [FAIL] (:testid) TextToNull: body not NULL on remote +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Payload-based sync with non-conflicting edits +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc6', E'First\nSecond\nThird'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc6') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc6') AS _mat \gset + +-- A edits line 1 +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'FirstEdited\nSecond\nThird' WHERE id = 'doc6'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc6') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc6') AS _mat \gset + +SELECT (body = E'FirstEdited\nSecond\nThird') AS payload_ok FROM docs WHERE id = 'doc6' \gset +\if :payload_ok +\echo [PASS] (:testid) PayloadSync: body matches +\else +\echo [FAIL] (:testid) PayloadSync: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Idempotent apply — same payload twice is a no-op +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc7', E'AAA\nBBB\nCCC'); + +-- Sync initial +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc7') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc7') AS _mat \gset + +-- A edits +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'AAA-edited\nBBB\nCCC' WHERE id = 'doc7'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7e +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc7') \gset + +-- Apply TWICE to B +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7e', 'hex')) AS _app1 \gset +SELECT cloudsync_payload_apply(decode(:'payload7e', 'hex')) AS _app2 \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc7') AS _mat \gset + +SELECT (body LIKE '%AAA-edited%') AS idemp_ok FROM docs WHERE id = 'doc7' \gset +\if :idemp_ok +\echo [PASS] (:testid) Idempotent: body matches after double apply +\else +\echo [FAIL] (:testid) Idempotent: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Block position ordering — sequential inserts preserve order after sync +-- ============================================================ +\connect cloudsync_block_edge_a +INSERT INTO docs (id, body) VALUES ('doc8', E'Top\nBottom'); + +-- Sync initial to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc8') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc8') AS _mat \gset + +-- A: add two lines between Top and Bottom +\connect cloudsync_block_edge_a +UPDATE docs SET body = E'Top\nMiddle1\nMiddle2\nBottom' WHERE id = 'doc8'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' +AND pk = cloudsync_pk_encode('doc8') \gset + +\connect cloudsync_block_edge_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'doc8') AS _mat \gset + +SELECT (body LIKE 'Top%') AS ord_top FROM docs WHERE id = 'doc8' \gset +\if :ord_top +\echo [PASS] (:testid) Ordering: Top first +\else +\echo [FAIL] (:testid) Ordering: Top not first +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Bottom') AS ord_bottom FROM docs WHERE id = 'doc8' \gset +\if :ord_bottom +\echo [PASS] (:testid) Ordering: Bottom last +\else +\echo [FAIL] (:testid) Ordering: Bottom not last +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Middle1 should come before Middle2 +SELECT (position('Middle1' IN body) < position('Middle2' IN body)) AS ord_correct FROM docs WHERE id = 'doc8' \gset +\if :ord_correct +\echo [PASS] (:testid) Ordering: Middle1 before Middle2 +\else +\echo [FAIL] (:testid) Ordering: wrong order +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = E'Top\nMiddle1\nMiddle2\nBottom') AS ord_exact FROM docs WHERE id = 'doc8' \gset +\if :ord_exact +\echo [PASS] (:testid) Ordering: exact match +\else +\echo [FAIL] (:testid) Ordering: content mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_edge_a; +DROP DATABASE IF EXISTS cloudsync_block_edge_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/36_block_lww_round3.sql b/test/postgresql/36_block_lww_round3.sql new file mode 100644 index 0000000..196ca80 --- /dev/null +++ b/test/postgresql/36_block_lww_round3.sql @@ -0,0 +1,476 @@ +-- 'Block-level LWW round 3: composite PK, empty vs null, delete+reinsert, integer PK, multi-row, non-overlapping add, long line, whitespace' + +\set testid '36' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r3_a; +DROP DATABASE IF EXISTS cloudsync_block_r3_b; +CREATE DATABASE cloudsync_block_r3_a; +CREATE DATABASE cloudsync_block_r3_b; + +-- ============================================================ +-- Test 1: Composite primary key (text + int) with block column +-- ============================================================ +\connect cloudsync_block_r3_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert on A +\connect cloudsync_block_r3_a +INSERT INTO docs (owner, seq, body) VALUES ('alice', 1, E'Line1\nLine2\nLine3'); + +SELECT count(*) AS cpk_blocks FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('alice', 1) \gset +SELECT (:'cpk_blocks'::int = 3) AS cpk_blk_ok \gset +\if :cpk_blk_ok +\echo [PASS] (:testid) CompositePK: 3 blocks created +\else +\echo [FAIL] (:testid) CompositePK: expected 3 blocks, got :cpk_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1) AS _mat \gset + +SELECT (body = E'Line1\nLine2\nLine3') AS cpk_body_ok FROM docs WHERE owner = 'alice' AND seq = 1 \gset +\if :cpk_body_ok +\echo [PASS] (:testid) CompositePK: body matches on B +\else +\echo [FAIL] (:testid) CompositePK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit on B, sync back +UPDATE docs SET body = E'Line1\nEdited2\nLine3' WHERE owner = 'alice' AND seq = 1; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload1b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'docs' \gset + +\connect cloudsync_block_r3_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload1b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('docs', 'body', 'alice', 1) AS _mat \gset + +SELECT (body = E'Line1\nEdited2\nLine3') AS cpk_rev_ok FROM docs WHERE owner = 'alice' AND seq = 1 \gset +\if :cpk_rev_ok +\echo [PASS] (:testid) CompositePK: reverse sync body matches +\else +\echo [FAIL] (:testid) CompositePK: reverse sync body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Empty string vs NULL +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS edocs; +CREATE TABLE edocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('edocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('edocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS edocs; +CREATE TABLE edocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('edocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('edocs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert empty string on A +\connect cloudsync_block_r3_a +INSERT INTO edocs (id, body) VALUES ('doc1', ''); + +SELECT count(*) AS evn_blocks FROM edocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'evn_blocks'::int = 1) AS evn_blk_ok \gset +\if :evn_blk_ok +\echo [PASS] (:testid) EmptyVsNull: 1 block for empty string +\else +\echo [FAIL] (:testid) EmptyVsNull: expected 1 block, got :evn_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'edocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('edocs', 'body', 'doc1') AS _mat \gset + +SELECT (body IS NOT NULL AND body = '') AS evn_empty_ok FROM edocs WHERE id = 'doc1' \gset +\if :evn_empty_ok +\echo [PASS] (:testid) EmptyVsNull: body is empty string (not NULL) +\else +\echo [FAIL] (:testid) EmptyVsNull: body should be empty string +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: DELETE row then re-insert with different content +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS rdocs; +CREATE TABLE rdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('rdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS rdocs; +CREATE TABLE rdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('rdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rdocs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert and sync +\connect cloudsync_block_r3_a +INSERT INTO rdocs (id, body) VALUES ('doc1', E'Old1\nOld2\nOld3'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3i', 'hex')) AS _app \gset + +-- Delete on A +\connect cloudsync_block_r3_a +DELETE FROM rdocs WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3d +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3d', 'hex')) AS _app \gset + +SELECT (count(*) = 0) AS dr_deleted FROM rdocs WHERE id = 'doc1' \gset +\if :dr_deleted +\echo [PASS] (:testid) DelReinsert: row deleted on B +\else +\echo [FAIL] (:testid) DelReinsert: row not deleted on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Re-insert with different content on A +\connect cloudsync_block_r3_a +INSERT INTO rdocs (id, body) VALUES ('doc1', E'New1\nNew2'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload3r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload3r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('rdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'New1\nNew2') AS dr_body_ok FROM rdocs WHERE id = 'doc1' \gset +\if :dr_body_ok +\echo [PASS] (:testid) DelReinsert: body matches after re-insert +\else +\echo [FAIL] (:testid) DelReinsert: body mismatch after re-insert +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: INTEGER primary key with block column +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id INTEGER PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS notes; +CREATE TABLE notes (id INTEGER PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('notes', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO notes (id, body) VALUES (42, E'First\nSecond\nThird'); + +SELECT count(*) AS ipk_blocks FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode(42) \gset +SELECT (:'ipk_blocks'::int = 3) AS ipk_blk_ok \gset +\if :ipk_blk_ok +\echo [PASS] (:testid) IntegerPK: 3 blocks created +\else +\echo [FAIL] (:testid) IntegerPK: expected 3 blocks, got :ipk_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload4 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload4', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('notes', 'body', 42) AS _mat \gset + +SELECT (body = E'First\nSecond\nThird') AS ipk_body_ok FROM notes WHERE id = 42 \gset +\if :ipk_body_ok +\echo [PASS] (:testid) IntegerPK: body matches on B +\else +\echo [FAIL] (:testid) IntegerPK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Multiple rows with block columns in a single sync +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS mdocs; +CREATE TABLE mdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('mdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS mdocs; +CREATE TABLE mdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('mdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO mdocs (id, body) VALUES ('r1', E'R1-Line1\nR1-Line2'); +INSERT INTO mdocs (id, body) VALUES ('r2', E'R2-Alpha\nR2-Beta\nR2-Gamma'); +INSERT INTO mdocs (id, body) VALUES ('r3', 'R3-Only'); +UPDATE mdocs SET body = E'R1-Edited\nR1-Line2' WHERE id = 'r1'; +UPDATE mdocs SET body = 'R3-Changed' WHERE id = 'r3'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload5 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload5', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r1') AS _m1 \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r2') AS _m2 \gset +SELECT cloudsync_text_materialize('mdocs', 'body', 'r3') AS _m3 \gset + +SELECT (body = E'R1-Edited\nR1-Line2') AS mr_r1 FROM mdocs WHERE id = 'r1' \gset +\if :mr_r1 +\echo [PASS] (:testid) MultiRow: r1 matches +\else +\echo [FAIL] (:testid) MultiRow: r1 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = E'R2-Alpha\nR2-Beta\nR2-Gamma') AS mr_r2 FROM mdocs WHERE id = 'r2' \gset +\if :mr_r2 +\echo [PASS] (:testid) MultiRow: r2 matches +\else +\echo [FAIL] (:testid) MultiRow: r2 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body = 'R3-Changed') AS mr_r3 FROM mdocs WHERE id = 'r3' \gset +\if :mr_r3 +\echo [PASS] (:testid) MultiRow: r3 matches +\else +\echo [FAIL] (:testid) MultiRow: r3 mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Concurrent add at non-overlapping positions (top vs bottom) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS ndocs; +CREATE TABLE ndocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ndocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ndocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS ndocs; +CREATE TABLE ndocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ndocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ndocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO ndocs (id, body) VALUES ('doc1', E'A\nB\nC'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ndocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ndocs', 'body', 'doc1') AS _mat \gset + +-- A: add at top -> X A B C +\connect cloudsync_block_r3_a +UPDATE ndocs SET body = E'X\nA\nB\nC' WHERE id = 'doc1'; + +-- B: add at bottom -> A B C Y +\connect cloudsync_block_r3_b +UPDATE ndocs SET body = E'A\nB\nC\nY' WHERE id = 'doc1'; + +-- Sync A -> B +\connect cloudsync_block_r3_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload6a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ndocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload6a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ndocs', 'body', 'doc1') AS _mat \gset + +SELECT (body LIKE '%X%') AS no_x FROM ndocs WHERE id = 'doc1' \gset +\if :no_x +\echo [PASS] (:testid) NonOverlap: X present +\else +\echo [FAIL] (:testid) NonOverlap: X missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE '%Y%') AS no_y FROM ndocs WHERE id = 'doc1' \gset +\if :no_y +\echo [PASS] (:testid) NonOverlap: Y present +\else +\echo [FAIL] (:testid) NonOverlap: Y missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (body LIKE 'X%' OR body LIKE E'%\nX\n%') AS no_x_before FROM ndocs WHERE id = 'doc1' \gset +\if :no_x_before +\echo [PASS] (:testid) NonOverlap: X before A +\else +\echo [FAIL] (:testid) NonOverlap: X not before A +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 7: Very long single line (10K chars) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS ldocs; +CREATE TABLE ldocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ldocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ldocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS ldocs; +CREATE TABLE ldocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('ldocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('ldocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +INSERT INTO ldocs (id, body) VALUES ('doc1', repeat('ABCDEFGHIJ', 1000)); + +SELECT count(*) AS ll_blocks FROM ldocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'ll_blocks'::int = 1) AS ll_blk_ok \gset +\if :ll_blk_ok +\echo [PASS] (:testid) LongLine: 1 block for 10K char line +\else +\echo [FAIL] (:testid) LongLine: expected 1 block, got :ll_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload7 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'ldocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload7', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('ldocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = repeat('ABCDEFGHIJ', 1000)) AS ll_body_ok FROM ldocs WHERE id = 'doc1' \gset +\if :ll_body_ok +\echo [PASS] (:testid) LongLine: body matches on B +\else +\echo [FAIL] (:testid) LongLine: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Whitespace and empty lines (delimiter edge cases) +-- ============================================================ +\connect cloudsync_block_r3_a +DROP TABLE IF EXISTS wdocs; +CREATE TABLE wdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('wdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('wdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_b +DROP TABLE IF EXISTS wdocs; +CREATE TABLE wdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('wdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('wdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r3_a +-- Text: "Line1\n\n spaces \n\t\ttabs\n\nLine6\n" = 7 blocks +INSERT INTO wdocs (id, body) VALUES ('doc1', E'Line1\n\n spaces \n\t\ttabs\n\nLine6\n'); + +SELECT count(*) AS ws_blocks FROM wdocs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'ws_blocks'::int = 7) AS ws_blk_ok \gset +\if :ws_blk_ok +\echo [PASS] (:testid) Whitespace: 7 blocks with empty/whitespace lines +\else +\echo [FAIL] (:testid) Whitespace: expected 7 blocks, got :ws_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8 +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'wdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('wdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Line1\n\n spaces \n\t\ttabs\n\nLine6\n') AS ws_body_ok FROM wdocs WHERE id = 'doc1' \gset +\if :ws_body_ok +\echo [PASS] (:testid) Whitespace: body matches with whitespace preserved +\else +\echo [FAIL] (:testid) Whitespace: body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit: remove empty lines +\connect cloudsync_block_r3_a +UPDATE wdocs SET body = E'Line1\n spaces \n\t\ttabs\nLine6' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload8b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'wdocs' \gset + +\connect cloudsync_block_r3_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload8b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('wdocs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Line1\n spaces \n\t\ttabs\nLine6') AS ws_edit_ok FROM wdocs WHERE id = 'doc1' \gset +\if :ws_edit_ok +\echo [PASS] (:testid) Whitespace: edited body matches +\else +\echo [FAIL] (:testid) Whitespace: edited body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r3_a; +DROP DATABASE IF EXISTS cloudsync_block_r3_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/37_block_lww_round4.sql b/test/postgresql/37_block_lww_round4.sql new file mode 100644 index 0000000..b04713e --- /dev/null +++ b/test/postgresql/37_block_lww_round4.sql @@ -0,0 +1,500 @@ +-- 'Block-level LWW round 4: UUID PK, RLS+blocks, multi-table, 3-site convergence, custom delimiter sync, mixed column updates' + +\set testid '37' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r4_a; +DROP DATABASE IF EXISTS cloudsync_block_r4_b; +DROP DATABASE IF EXISTS cloudsync_block_r4_c; +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +CREATE DATABASE cloudsync_block_r4_a; +CREATE DATABASE cloudsync_block_r4_b; +CREATE DATABASE cloudsync_block_r4_c; + +-- ============================================================ +-- Test 1: UUID primary key with block column +-- ============================================================ +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS uuid_docs; +CREATE TABLE uuid_docs (id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), body TEXT); +SELECT cloudsync_init('uuid_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('uuid_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS uuid_docs; +CREATE TABLE uuid_docs (id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), body TEXT); +SELECT cloudsync_init('uuid_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('uuid_docs', 'body', 'algo', 'block') AS _sc \gset + +-- Insert on A with explicit UUID +\connect cloudsync_block_r4_a +INSERT INTO uuid_docs (id, body) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', E'UUID-Line1\nUUID-Line2\nUUID-Line3'); + +SELECT count(*) AS uuid_blocks FROM uuid_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') \gset +SELECT (:'uuid_blocks'::int = 3) AS uuid_blk_ok \gset +\if :uuid_blk_ok +\echo [PASS] (:testid) UUID_PK: 3 blocks created +\else +\echo [FAIL] (:testid) UUID_PK: expected 3 blocks, got :uuid_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_uuid +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'uuid_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_uuid', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('uuid_docs', 'body', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') AS _mat \gset + +SELECT (body = E'UUID-Line1\nUUID-Line2\nUUID-Line3') AS uuid_body_ok FROM uuid_docs WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' \gset +\if :uuid_body_ok +\echo [PASS] (:testid) UUID_PK: body matches on B +\else +\echo [FAIL] (:testid) UUID_PK: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit on B, reverse sync +\connect cloudsync_block_r4_b +UPDATE uuid_docs SET body = E'UUID-Line1\nUUID-Edited\nUUID-Line3' WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_uuid_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'uuid_docs' \gset + +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_uuid_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('uuid_docs', 'body', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11') AS _mat \gset + +SELECT (body = E'UUID-Line1\nUUID-Edited\nUUID-Line3') AS uuid_rev_ok FROM uuid_docs WHERE id = 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11' \gset +\if :uuid_rev_ok +\echo [PASS] (:testid) UUID_PK: reverse sync matches +\else +\echo [FAIL] (:testid) UUID_PK: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: RLS filter + block columns +-- Only rows matching filter should have block tracking +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS rls_docs; +CREATE TABLE rls_docs (id TEXT PRIMARY KEY NOT NULL, owner_id INTEGER, body TEXT); +SELECT cloudsync_init('rls_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rls_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_filter('rls_docs', 'owner_id = 1') AS _sf \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS rls_docs; +CREATE TABLE rls_docs (id TEXT PRIMARY KEY NOT NULL, owner_id INTEGER, body TEXT); +SELECT cloudsync_init('rls_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('rls_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_filter('rls_docs', 'owner_id = 1') AS _sf \gset + +-- Insert matching row (owner_id=1) and non-matching row (owner_id=2) +\connect cloudsync_block_r4_a +INSERT INTO rls_docs (id, owner_id, body) VALUES ('match1', 1, E'Filtered-Line1\nFiltered-Line2'); +INSERT INTO rls_docs (id, owner_id, body) VALUES ('nomatch', 2, E'Hidden-Line1\nHidden-Line2'); + +-- Check: matching row has blocks, non-matching does not +SELECT count(*) AS rls_match_blocks FROM rls_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('match1') \gset +SELECT count(*) AS rls_nomatch_blocks FROM rls_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('nomatch') \gset + +SELECT (:'rls_match_blocks'::int = 2) AS rls_match_ok \gset +\if :rls_match_ok +\echo [PASS] (:testid) RLS+Blocks: matching row has 2 blocks +\else +\echo [FAIL] (:testid) RLS+Blocks: expected 2 blocks for matching row, got :rls_match_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (:'rls_nomatch_blocks'::int = 0) AS rls_nomatch_ok \gset +\if :rls_nomatch_ok +\echo [PASS] (:testid) RLS+Blocks: non-matching row has 0 blocks +\else +\echo [FAIL] (:testid) RLS+Blocks: expected 0 blocks for non-matching row, got :rls_nomatch_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync: only matching row should appear in changes +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_rls +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'rls_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_rls', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('rls_docs', 'body', 'match1') AS _mat \gset + +SELECT (body = E'Filtered-Line1\nFiltered-Line2') AS rls_sync_ok FROM rls_docs WHERE id = 'match1' \gset +\if :rls_sync_ok +\echo [PASS] (:testid) RLS+Blocks: matching row synced with correct body +\else +\echo [FAIL] (:testid) RLS+Blocks: matching row body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- non-matching row should NOT exist on B +SELECT (count(*) = 0) AS rls_norow_ok FROM rls_docs WHERE id = 'nomatch' \gset +\if :rls_norow_ok +\echo [PASS] (:testid) RLS+Blocks: non-matching row not synced +\else +\echo [FAIL] (:testid) RLS+Blocks: non-matching row should not exist on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Multi-table blocks — two tables with block columns in same payload +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS articles; +DROP TABLE IF EXISTS comments; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, content TEXT); +CREATE TABLE comments (id TEXT PRIMARY KEY NOT NULL, text_body TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_init('comments', 'CLS', 1) AS _init2 \gset +SELECT cloudsync_set_column('articles', 'content', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('comments', 'text_body', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS articles; +DROP TABLE IF EXISTS comments; +CREATE TABLE articles (id TEXT PRIMARY KEY NOT NULL, content TEXT); +CREATE TABLE comments (id TEXT PRIMARY KEY NOT NULL, text_body TEXT); +SELECT cloudsync_init('articles', 'CLS', 1) AS _init \gset +SELECT cloudsync_init('comments', 'CLS', 1) AS _init2 \gset +SELECT cloudsync_set_column('articles', 'content', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('comments', 'text_body', 'algo', 'block') AS _sc2 \gset + +\connect cloudsync_block_r4_a +INSERT INTO articles (id, content) VALUES ('art1', E'Para1\nPara2\nPara3'); +INSERT INTO comments (id, text_body) VALUES ('cmt1', E'Comment-Line1\nComment-Line2'); +UPDATE articles SET content = E'Para1-Edited\nPara2\nPara3' WHERE id = 'art1'; + +-- Single payload containing changes from both tables +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mt +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mt', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('articles', 'content', 'art1') AS _m1 \gset +SELECT cloudsync_text_materialize('comments', 'text_body', 'cmt1') AS _m2 \gset + +SELECT (content = E'Para1-Edited\nPara2\nPara3') AS mt_art_ok FROM articles WHERE id = 'art1' \gset +\if :mt_art_ok +\echo [PASS] (:testid) MultiTable: articles content matches +\else +\echo [FAIL] (:testid) MultiTable: articles content mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (text_body = E'Comment-Line1\nComment-Line2') AS mt_cmt_ok FROM comments WHERE id = 'cmt1' \gset +\if :mt_cmt_ok +\echo [PASS] (:testid) MultiTable: comments text_body matches +\else +\echo [FAIL] (:testid) MultiTable: comments text_body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Three-site convergence with block columns +-- All three sites make different edits, pairwise sync, verify convergence +-- Uses dedicated databases so all 3 have identical schema +-- ============================================================ +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +CREATE DATABASE cloudsync_block_3s_a; +CREATE DATABASE cloudsync_block_3s_b; +CREATE DATABASE cloudsync_block_3s_c; + +\connect cloudsync_block_3s_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE tdocs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('tdocs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('tdocs', 'body', 'algo', 'block') AS _sc \gset + +-- Initial insert on A, sync to B and C +\connect cloudsync_block_3s_a +INSERT INTO tdocs (id, body) VALUES ('doc1', E'Line1\nLine2\nLine3\nLine4\nLine5'); + +-- Full changes from A (includes schema info) +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_init +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_init', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_init', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Each site edits a DIFFERENT line (no conflicts) +-- A edits line 1 +\connect cloudsync_block_3s_a +UPDATE tdocs SET body = E'Line1-A\nLine2\nLine3\nLine4\nLine5' WHERE id = 'doc1'; + +-- B edits line 3 +\connect cloudsync_block_3s_b +UPDATE tdocs SET body = E'Line1\nLine2\nLine3-B\nLine4\nLine5' WHERE id = 'doc1'; + +-- C edits line 5 +\connect cloudsync_block_3s_c +UPDATE tdocs SET body = E'Line1\nLine2\nLine3\nLine4\nLine5-C' WHERE id = 'doc1'; + +-- Collect ALL changes from each site (not filtered by site_id) +-- This includes the schema info that recipients need +\connect cloudsync_block_3s_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_a +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_b +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +\connect cloudsync_block_3s_c +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_3s_c +FROM cloudsync_changes WHERE tbl = 'tdocs' \gset + +-- Apply all to A (B's and C's changes) +\connect cloudsync_block_3s_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_b', 'hex')) AS _app_b \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_c', 'hex')) AS _app_c \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Apply all to B (A's and C's changes) +\connect cloudsync_block_3s_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_a', 'hex')) AS _app_a \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_c', 'hex')) AS _app_c \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- Apply all to C (A's and B's changes) +\connect cloudsync_block_3s_c +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_3s_a', 'hex')) AS _app_a \gset +SELECT cloudsync_payload_apply(decode(:'payload_3s_b', 'hex')) AS _app_b \gset +SELECT cloudsync_text_materialize('tdocs', 'body', 'doc1') AS _mat \gset + +-- All three should converge +\connect cloudsync_block_3s_a +SELECT body AS body_a FROM tdocs WHERE id = 'doc1' \gset +\connect cloudsync_block_3s_b +SELECT body AS body_b FROM tdocs WHERE id = 'doc1' \gset +\connect cloudsync_block_3s_c +SELECT body AS body_c FROM tdocs WHERE id = 'doc1' \gset + +SELECT (:'body_a' = :'body_b') AS ab_match \gset +SELECT (:'body_b' = :'body_c') AS bc_match \gset + +\if :ab_match +\echo [PASS] (:testid) 3-Site: A and B converge +\else +\echo [FAIL] (:testid) 3-Site: A and B diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :bc_match +\echo [PASS] (:testid) 3-Site: B and C converge +\else +\echo [FAIL] (:testid) 3-Site: B and C diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- All edits should be present (non-conflicting) +SELECT (position('Line1-A' in :'body_a') > 0) AS has_a \gset +SELECT (position('Line3-B' in :'body_a') > 0) AS has_b \gset +SELECT (position('Line5-C' in :'body_a') > 0) AS has_c \gset + +\if :has_a +\echo [PASS] (:testid) 3-Site: Site A edit present +\else +\echo [FAIL] (:testid) 3-Site: Site A edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :has_b +\echo [PASS] (:testid) 3-Site: Site B edit present +\else +\echo [FAIL] (:testid) 3-Site: Site B edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :has_c +\echo [PASS] (:testid) 3-Site: Site C edit present +\else +\echo [FAIL] (:testid) 3-Site: Site C edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Custom delimiter sync roundtrip +-- Uses paragraph delimiter (double newline), edits, syncs +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS para_docs; +CREATE TABLE para_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('para_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('para_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('para_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS para_docs; +CREATE TABLE para_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('para_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('para_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('para_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r4_a +INSERT INTO para_docs (id, body) VALUES ('doc1', E'First paragraph.\n\nSecond paragraph.\n\nThird paragraph.'); + +SELECT count(*) AS pd_blocks FROM para_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'pd_blocks'::int = 3) AS pd_blk_ok \gset +\if :pd_blk_ok +\echo [PASS] (:testid) CustomDelimSync: 3 paragraph blocks +\else +\echo [FAIL] (:testid) CustomDelimSync: expected 3 blocks, got :pd_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pd +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'para_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pd', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('para_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'First paragraph.\n\nSecond paragraph.\n\nThird paragraph.') AS pd_sync_ok FROM para_docs WHERE id = 'doc1' \gset +\if :pd_sync_ok +\echo [PASS] (:testid) CustomDelimSync: body matches on B +\else +\echo [FAIL] (:testid) CustomDelimSync: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit paragraph 2 on B, sync back +\connect cloudsync_block_r4_b +UPDATE para_docs SET body = E'First paragraph.\n\nEdited second paragraph.\n\nThird paragraph.' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pd_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'para_docs' \gset + +\connect cloudsync_block_r4_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pd_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('para_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'First paragraph.\n\nEdited second paragraph.\n\nThird paragraph.') AS pd_rev_ok FROM para_docs WHERE id = 'doc1' \gset +\if :pd_rev_ok +\echo [PASS] (:testid) CustomDelimSync: reverse sync matches +\else +\echo [FAIL] (:testid) CustomDelimSync: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Block column + regular LWW column — mixed update +-- Single UPDATE changes both block col and regular col +-- ============================================================ +\connect cloudsync_block_r4_a +DROP TABLE IF EXISTS mixed_docs; +CREATE TABLE mixed_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('mixed_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mixed_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_b +DROP TABLE IF EXISTS mixed_docs; +CREATE TABLE mixed_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT, title TEXT); +SELECT cloudsync_init('mixed_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('mixed_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r4_a +INSERT INTO mixed_docs (id, body, title) VALUES ('doc1', E'Body-Line1\nBody-Line2', 'Original Title'); + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mix_i +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mixed_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mix_i', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mixed_docs', 'body', 'doc1') AS _mat \gset + +-- Update BOTH columns simultaneously on A +\connect cloudsync_block_r4_a +UPDATE mixed_docs SET body = E'Body-Edited1\nBody-Line2', title = 'New Title' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_mix_u +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'mixed_docs' \gset + +\connect cloudsync_block_r4_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_mix_u', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('mixed_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Body-Edited1\nBody-Line2') AS mix_body_ok FROM mixed_docs WHERE id = 'doc1' \gset +\if :mix_body_ok +\echo [PASS] (:testid) MixedUpdate: block column body matches +\else +\echo [FAIL] (:testid) MixedUpdate: block column body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (title = 'New Title') AS mix_title_ok FROM mixed_docs WHERE id = 'doc1' \gset +\if :mix_title_ok +\echo [PASS] (:testid) MixedUpdate: regular column title matches +\else +\echo [FAIL] (:testid) MixedUpdate: regular column title mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r4_a; +DROP DATABASE IF EXISTS cloudsync_block_r4_b; +DROP DATABASE IF EXISTS cloudsync_block_r4_c; +DROP DATABASE IF EXISTS cloudsync_block_3s_a; +DROP DATABASE IF EXISTS cloudsync_block_3s_b; +DROP DATABASE IF EXISTS cloudsync_block_3s_c; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/38_block_lww_round5.sql b/test/postgresql/38_block_lww_round5.sql new file mode 100644 index 0000000..93b6a16 --- /dev/null +++ b/test/postgresql/38_block_lww_round5.sql @@ -0,0 +1,433 @@ +-- 'Block-level LWW round 5: large blocks, payload idempotency composite PK, init with existing data, drop/re-add block config, delimiter-in-content' + +\set testid '38' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_r5_a; +DROP DATABASE IF EXISTS cloudsync_block_r5_b; +CREATE DATABASE cloudsync_block_r5_a; +CREATE DATABASE cloudsync_block_r5_b; + +-- ============================================================ +-- Test 7: Large number of blocks (200+ lines) +-- Verify diff and materialize work correctly at scale +-- ============================================================ +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS big_docs; +CREATE TABLE big_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('big_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('big_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +DROP TABLE IF EXISTS big_docs; +CREATE TABLE big_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('big_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('big_docs', 'body', 'algo', 'block') AS _sc \gset + +-- Generate 250-line text +\connect cloudsync_block_r5_a +INSERT INTO big_docs (id, body) +SELECT 'doc1', string_agg('Line-' || gs::text, E'\n' ORDER BY gs) +FROM generate_series(1, 250) gs; + +SELECT count(*) AS big_blocks FROM big_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'big_blocks'::int = 250) AS big_blk_ok \gset +\if :big_blk_ok +\echo [PASS] (:testid) LargeBlocks: 250 blocks created +\else +\echo [FAIL] (:testid) LargeBlocks: expected 250 blocks, got :big_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit a few lines scattered through the document +UPDATE big_docs SET body = ( + SELECT string_agg( + CASE + WHEN gs = 50 THEN 'EDITED-50' + WHEN gs = 150 THEN 'EDITED-150' + WHEN gs = 200 THEN 'EDITED-200' + ELSE 'Line-' || gs::text + END, + E'\n' ORDER BY gs + ) FROM generate_series(1, 250) gs +) WHERE id = 'doc1'; + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_big +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'big_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_big', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('big_docs', 'body', 'doc1') AS _mat \gset + +-- Verify edited lines are present +SELECT (position('EDITED-50' in body) > 0) AS big_e50 FROM big_docs WHERE id = 'doc1' \gset +SELECT (position('EDITED-150' in body) > 0) AS big_e150 FROM big_docs WHERE id = 'doc1' \gset +SELECT (position('EDITED-200' in body) > 0) AS big_e200 FROM big_docs WHERE id = 'doc1' \gset + +\if :big_e50 +\echo [PASS] (:testid) LargeBlocks: EDITED-50 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-50 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :big_e150 +\echo [PASS] (:testid) LargeBlocks: EDITED-150 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-150 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :big_e200 +\echo [PASS] (:testid) LargeBlocks: EDITED-200 present +\else +\echo [FAIL] (:testid) LargeBlocks: EDITED-200 missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify block count still 250 (edits don't change count) +SELECT count(*) AS big_blocks2 FROM big_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'big_blocks2'::int = 250) AS big_cnt_ok \gset +\if :big_cnt_ok +\echo [PASS] (:testid) LargeBlocks: block count stable after sync +\else +\echo [FAIL] (:testid) LargeBlocks: expected 250 blocks after sync, got :big_blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 8: Payload idempotency with composite PK +-- Apply same payload twice, verify no duplication or corruption +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS idem_docs; +CREATE TABLE idem_docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('idem_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('idem_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS idem_docs; +CREATE TABLE idem_docs (owner TEXT NOT NULL, seq INTEGER NOT NULL, body TEXT, PRIMARY KEY(owner, seq)); +SELECT cloudsync_init('idem_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('idem_docs', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_a +INSERT INTO idem_docs (owner, seq, body) VALUES ('bob', 1, E'Idem-Line1\nIdem-Line2\nIdem-Line3'); +UPDATE idem_docs SET body = E'Idem-Line1\nIdem-Edited\nIdem-Line3' WHERE owner = 'bob' AND seq = 1; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_idem +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'idem_docs' \gset + +-- Apply on B — first time +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_idem', 'hex')) AS _app1 \gset +SELECT cloudsync_text_materialize('idem_docs', 'body', 'bob', 1) AS _mat1 \gset + +SELECT (body = E'Idem-Line1\nIdem-Edited\nIdem-Line3') AS idem1_ok FROM idem_docs WHERE owner = 'bob' AND seq = 1 \gset +\if :idem1_ok +\echo [PASS] (:testid) Idempotent: first apply correct +\else +\echo [FAIL] (:testid) Idempotent: first apply mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS idem_meta1 FROM idem_docs_cloudsync WHERE pk = cloudsync_pk_encode('bob', 1) \gset + +-- Apply SAME payload again — second time (idempotent) +SELECT cloudsync_payload_apply(decode(:'payload_idem', 'hex')) AS _app2 \gset +SELECT cloudsync_text_materialize('idem_docs', 'body', 'bob', 1) AS _mat2 \gset + +SELECT (body = E'Idem-Line1\nIdem-Edited\nIdem-Line3') AS idem2_ok FROM idem_docs WHERE owner = 'bob' AND seq = 1 \gset +\if :idem2_ok +\echo [PASS] (:testid) Idempotent: second apply still correct +\else +\echo [FAIL] (:testid) Idempotent: body corrupted after double apply +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Metadata count should not change +SELECT count(*) AS idem_meta2 FROM idem_docs_cloudsync WHERE pk = cloudsync_pk_encode('bob', 1) \gset +SELECT (:'idem_meta1' = :'idem_meta2') AS idem_meta_ok \gset +\if :idem_meta_ok +\echo [PASS] (:testid) Idempotent: metadata count unchanged after double apply +\else +\echo [FAIL] (:testid) Idempotent: metadata count changed (:idem_meta1 vs :idem_meta2) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 9: Init with pre-existing data, then enable block column +-- Table has rows before cloudsync_set_column algo=block +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS predata; +CREATE TABLE predata (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('predata', 'CLS', 1) AS _init \gset + +-- Insert rows BEFORE enabling block algorithm +INSERT INTO predata (id, body) VALUES ('pre1', E'Pre-Line1\nPre-Line2'); +INSERT INTO predata (id, body) VALUES ('pre2', E'Pre-Alpha\nPre-Beta\nPre-Gamma'); + +-- Now enable block on the column +SELECT cloudsync_set_column('predata', 'body', 'algo', 'block') AS _sc \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS predata; +CREATE TABLE predata (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('predata', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('predata', 'body', 'algo', 'block') AS _sc \gset + +-- Update a pre-existing row on A to trigger block creation +\connect cloudsync_block_r5_a +UPDATE predata SET body = E'Pre-Line1\nPre-Edited2' WHERE id = 'pre1'; + +SELECT count(*) AS pre_blocks FROM predata_cloudsync_blocks WHERE pk = cloudsync_pk_encode('pre1') \gset +SELECT (:'pre_blocks'::int >= 2) AS pre_blk_ok \gset +\if :pre_blk_ok +\echo [PASS] (:testid) PreExisting: blocks created after update +\else +\echo [FAIL] (:testid) PreExisting: expected >= 2 blocks, got :pre_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_pre +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'predata' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_pre', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('predata', 'body', 'pre1') AS _mat \gset + +SELECT (body = E'Pre-Line1\nPre-Edited2') AS pre_sync_ok FROM predata WHERE id = 'pre1' \gset +\if :pre_sync_ok +\echo [PASS] (:testid) PreExisting: synced body matches after late block enable +\else +\echo [FAIL] (:testid) PreExisting: synced body mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- pre2 should also sync (as regular LWW or with insert sentinel) +SELECT (count(*) = 1) AS pre2_exists FROM predata WHERE id = 'pre2' \gset +\if :pre2_exists +\echo [PASS] (:testid) PreExisting: pre2 row synced +\else +\echo [FAIL] (:testid) PreExisting: pre2 row missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 10: Remove block algo then re-add +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS toggle_docs; +CREATE TABLE toggle_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('toggle_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc1 \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS toggle_docs; +CREATE TABLE toggle_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('toggle_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc1 \gset + +-- Insert with blocks on A +\connect cloudsync_block_r5_a +INSERT INTO toggle_docs (id, body) VALUES ('doc1', E'Toggle-Line1\nToggle-Line2'); + +SELECT count(*) AS tog_blocks1 FROM toggle_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'tog_blocks1'::int = 2) AS tog_blk1_ok \gset +\if :tog_blk1_ok +\echo [PASS] (:testid) Toggle: blocks created initially +\else +\echo [FAIL] (:testid) Toggle: expected 2 blocks initially, got :tog_blocks1 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Remove block algo (set to default LWW) +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'lww') AS _sc2 \gset + +-- Update while in LWW mode — should NOT create new blocks +UPDATE toggle_docs SET body = E'Toggle-LWW-Updated' WHERE id = 'doc1'; + +-- Re-enable block algo +SELECT cloudsync_set_column('toggle_docs', 'body', 'algo', 'block') AS _sc3 \gset + +-- Update with blocks re-enabled +UPDATE toggle_docs SET body = E'Toggle-Block-Again1\nToggle-Block-Again2\nToggle-Block-Again3' WHERE id = 'doc1'; + +SELECT count(*) AS tog_blocks2 FROM toggle_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'tog_blocks2'::int = 3) AS tog_blk2_ok \gset +\if :tog_blk2_ok +\echo [PASS] (:testid) Toggle: 3 blocks after re-enable +\else +\echo [FAIL] (:testid) Toggle: expected 3 blocks after re-enable, got :tog_blocks2 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync to B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_tog +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'toggle_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_tog', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('toggle_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Toggle-Block-Again1\nToggle-Block-Again2\nToggle-Block-Again3') AS tog_sync_ok FROM toggle_docs WHERE id = 'doc1' \gset +\if :tog_sync_ok +\echo [PASS] (:testid) Toggle: body matches after re-enable and sync +\else +\echo [FAIL] (:testid) Toggle: body mismatch after re-enable and sync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 11: Text containing the delimiter character as content +-- Default delimiter is \n — content has no real structure, just embedded newlines +-- ============================================================ +\connect cloudsync_block_r5_a +DROP TABLE IF EXISTS delim_docs; +CREATE TABLE delim_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('delim_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'algo', 'block') AS _sc \gset +-- Use paragraph delimiter (double newline) +SELECT cloudsync_set_column('delim_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +\connect cloudsync_block_r5_b +DROP TABLE IF EXISTS delim_docs; +CREATE TABLE delim_docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('delim_docs', 'CLS', 1) AS _init \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'algo', 'block') AS _sc \gset +SELECT cloudsync_set_column('delim_docs', 'body', 'delimiter', E'\n\n') AS _sd \gset + +-- Content with single newlines inside paragraphs (not delimiters) +\connect cloudsync_block_r5_a +INSERT INTO delim_docs (id, body) VALUES ('doc1', E'Paragraph one\nstill paragraph one.\n\nParagraph two\nstill para two.\n\nParagraph three.'); + +-- Should be 3 blocks (split by double newline) +SELECT count(*) AS dc_blocks FROM delim_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1') \gset +SELECT (:'dc_blocks'::int = 3) AS dc_blk_ok \gset +\if :dc_blk_ok +\echo [PASS] (:testid) DelimContent: 3 paragraph blocks (single newlines inside) +\else +\echo [FAIL] (:testid) DelimContent: expected 3 blocks, got :dc_blocks +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Sync A -> B +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Paragraph one\nstill paragraph one.\n\nParagraph two\nstill para two.\n\nParagraph three.') AS dc_sync_ok FROM delim_docs WHERE id = 'doc1' \gset +\if :dc_sync_ok +\echo [PASS] (:testid) DelimContent: body matches on B (embedded newlines preserved) +\else +\echo [FAIL] (:testid) DelimContent: body mismatch on B +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Edit paragraph 2 on B (change only the second paragraph), sync back +\connect cloudsync_block_r5_b +UPDATE delim_docs SET body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nParagraph three.' WHERE id = 'doc1'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_r +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_r', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +SELECT (body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nParagraph three.') AS dc_rev_ok FROM delim_docs WHERE id = 'doc1' \gset +\if :dc_rev_ok +\echo [PASS] (:testid) DelimContent: reverse sync matches (paragraph edit) +\else +\echo [FAIL] (:testid) DelimContent: reverse sync mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Concurrent edit: A edits para 1, B edits para 3 +\connect cloudsync_block_r5_a +UPDATE delim_docs SET body = E'Edited para one by A.\n\nEdited paragraph two.\n\nParagraph three.' WHERE id = 'doc1'; + +\connect cloudsync_block_r5_b +UPDATE delim_docs SET body = E'Paragraph one\nstill paragraph one.\n\nEdited paragraph two.\n\nEdited para three by B.' WHERE id = 'doc1'; + +\connect cloudsync_block_r5_a +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_a +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +\connect cloudsync_block_r5_b +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_dc_b +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'delim_docs' \gset + +-- Apply cross +\connect cloudsync_block_r5_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_b', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +\connect cloudsync_block_r5_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload_dc_a', 'hex')) AS _app \gset +SELECT cloudsync_text_materialize('delim_docs', 'body', 'doc1') AS _mat \gset + +-- Both should converge and both edits should be present +\connect cloudsync_block_r5_a +SELECT md5(body) AS dc_md5_a FROM delim_docs WHERE id = 'doc1' \gset +\connect cloudsync_block_r5_b +SELECT md5(body) AS dc_md5_b FROM delim_docs WHERE id = 'doc1' \gset + +SELECT (:'dc_md5_a' = :'dc_md5_b') AS dc_converge \gset +\if :dc_converge +\echo [PASS] (:testid) DelimContent: concurrent paragraph edits converge +\else +\echo [FAIL] (:testid) DelimContent: concurrent paragraph edits diverged +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect cloudsync_block_r5_a +SELECT (position('Edited para one by A.' in body) > 0) AS dc_has_a FROM delim_docs WHERE id = 'doc1' \gset +SELECT (position('Edited para three by B.' in body) > 0) AS dc_has_b FROM delim_docs WHERE id = 'doc1' \gset + +\if :dc_has_a +\echo [PASS] (:testid) DelimContent: site A paragraph edit present +\else +\echo [FAIL] (:testid) DelimContent: site A paragraph edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +\if :dc_has_b +\echo [PASS] (:testid) DelimContent: site B paragraph edit present +\else +\echo [FAIL] (:testid) DelimContent: site B paragraph edit missing +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Cleanup +-- ============================================================ +\ir helper_test_cleanup.sql +\if :should_cleanup +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_block_r5_a; +DROP DATABASE IF EXISTS cloudsync_block_r5_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/39_concurrent_write_apply.sql b/test/postgresql/39_concurrent_write_apply.sql new file mode 100644 index 0000000..d84397e --- /dev/null +++ b/test/postgresql/39_concurrent_write_apply.sql @@ -0,0 +1,179 @@ +-- 'Test concurrent write lock during payload apply' +-- NOTE: The lock-contention portion requires dblink with table access. +-- On environments where dblink cannot lock the table (e.g. Supabase), +-- the lock test is skipped and only apply + consistency are verified. + +\set testid '39' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_39_a; +DROP DATABASE IF EXISTS cloudsync_test_39_b; +CREATE DATABASE cloudsync_test_39_a; +CREATE DATABASE cloudsync_test_39_b; + +-- Setup db_a +\connect cloudsync_test_39_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('concurrent_tbl', 'CLS', 1) AS _init_a \gset + +-- Setup db_b +\connect cloudsync_test_39_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('concurrent_tbl', 'CLS', 1) AS _init_b \gset + +-- Insert row1 on db_a and sync to db_b +\connect cloudsync_test_39_a +INSERT INTO concurrent_tbl VALUES ('row1', 'val_a'); + +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_init, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_init_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_39_b +\if :payload_init_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_init', 3), 'hex')) AS _apply_init \gset +\endif + +-- Update row1 on db_a +\connect cloudsync_test_39_a +UPDATE concurrent_tbl SET val = 'val_a_updated' WHERE id = 'row1'; + +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_upd, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_upd_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Try to set up dblink and acquire a table lock +\connect cloudsync_test_39_b +CREATE EXTENSION IF NOT EXISTS dblink; + +SELECT dblink_connect('locker', 'dbname=cloudsync_test_39_b') AS _conn \gset +SELECT dblink_exec('locker', 'BEGIN') AS _begin \gset + +-- Try to acquire EXCLUSIVE lock — if this fails (e.g. permission denied on +-- Supabase), _lock won't be set and we skip the lock-contention test +\unset _lock +SELECT dblink_exec('locker', 'LOCK TABLE concurrent_tbl IN EXCLUSIVE MODE') AS _lock \gset + +\if :{?_lock} +-- ===== Lock acquired — run lock-contention test ===== + +BEGIN; +\set ON_ERROR_ROLLBACK on +SET LOCAL lock_timeout = '500ms'; + +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _blocked_apply \gset +\endif + +COMMIT; +\set ON_ERROR_ROLLBACK off + +-- row1 should still have the OLD value because the apply was blocked +SELECT val AS row1_val_check FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val_check' = 'val_a') AS blocked_ok \gset +\if :blocked_ok +\echo [PASS] (:testid) Apply correctly blocked by concurrent table lock +\else +\echo [FAIL] (:testid) Expected val_a (blocked), got :'row1_val_check' +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Release the table lock +SELECT dblink_exec('locker', 'COMMIT') AS _release \gset +SELECT dblink_disconnect('locker') AS _disconn \gset + +-- Retry apply — should succeed now +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _apply_retry \gset +\endif + +SELECT val AS row1_val FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val' = 'val_a_updated') AS retry_ok \gset +\if :retry_ok +\echo [PASS] (:testid) Apply succeeded after lock released +\else +\echo [FAIL] (:testid) Apply after unlock - expected val_a_updated, got :'row1_val' +SELECT (:fail::int + 1) AS fail \gset +\endif + +\else +-- ===== Lock failed — skip contention test, apply directly ===== +\echo [SKIP] (:testid) Lock-contention test skipped (dblink cannot lock table) + +-- Clean up the dblink connection (transaction is aborted) +SELECT dblink_exec('locker', 'ROLLBACK') AS _rollback \gset +SELECT dblink_disconnect('locker') AS _disconn \gset + +\if :payload_upd_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _apply_direct \gset +\endif + +SELECT val AS row1_val FROM concurrent_tbl WHERE id = 'row1' \gset +SELECT (:'row1_val' = 'val_a_updated') AS direct_ok \gset +\if :direct_ok +\echo [PASS] (:testid) Apply succeeded (no lock contention) +\else +\echo [FAIL] (:testid) Apply failed - expected val_a_updated, got :'row1_val' +SELECT (:fail::int + 1) AS fail \gset +\endif + +\endif + +-- Full cross-sync for consistency +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_final, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_final_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_39_a +\if :payload_b_final_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_final', 3), 'hex')) AS _apply_final \gset +\endif + +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_a +FROM concurrent_tbl \gset + +\connect cloudsync_test_39_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_b +FROM concurrent_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS consistency_ok \gset +\if :consistency_ok +\echo [PASS] (:testid) Cross-database consistency verified +\else +\echo [FAIL] (:testid) Consistency failed (hash_a=:'hash_a' hash_b=:'hash_b') +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_39_a; +DROP DATABASE IF EXISTS cloudsync_test_39_b; +\endif diff --git a/test/postgresql/40_unsupported_algorithms.sql b/test/postgresql/40_unsupported_algorithms.sql new file mode 100644 index 0000000..fcbf0f6 --- /dev/null +++ b/test/postgresql/40_unsupported_algorithms.sql @@ -0,0 +1,80 @@ +-- Test unsupported CRDT algorithms (DWS, AWS) +-- Verifies that cloudsync_init rejects DWS and AWS with clear errors +-- and that no metadata tables are created. + +\set testid '40' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_40; +CREATE DATABASE cloudsync_test_40; + +\connect cloudsync_test_40 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE test_dws (id TEXT PRIMARY KEY, val TEXT); +CREATE TABLE test_aws (id TEXT PRIMARY KEY, val TEXT); + +-- Test DWS rejection +DO $$ +BEGIN + PERFORM cloudsync_init('test_dws', 'dws', 1); + RAISE EXCEPTION 'cloudsync_init with dws should have failed'; +EXCEPTION WHEN OTHERS THEN + IF SQLERRM NOT LIKE '%not yet supported%' THEN + RAISE EXCEPTION 'Unexpected error for dws: %', SQLERRM; + END IF; +END $$; + +-- Verify no companion table was created for DWS +SELECT COUNT(*) = 0 AS no_dws_meta +FROM information_schema.tables +WHERE table_name = 'test_dws_cloudsync' \gset +\if :no_dws_meta +\echo [PASS] (:testid) DWS rejected - no metadata table created +\else +\echo [FAIL] (:testid) DWS metadata table should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test AWS rejection +DO $$ +BEGIN + PERFORM cloudsync_init('test_aws', 'aws', 1); + RAISE EXCEPTION 'cloudsync_init with aws should have failed'; +EXCEPTION WHEN OTHERS THEN + IF SQLERRM NOT LIKE '%not yet supported%' THEN + RAISE EXCEPTION 'Unexpected error for aws: %', SQLERRM; + END IF; +END $$; + +-- Verify no companion table was created for AWS +SELECT COUNT(*) = 0 AS no_aws_meta +FROM information_schema.tables +WHERE table_name = 'test_aws_cloudsync' \gset +\if :no_aws_meta +\echo [PASS] (:testid) AWS rejected - no metadata table created +\else +\echo [FAIL] (:testid) AWS metadata table should not exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify CLS still works (sanity check) +SELECT cloudsync_init('test_dws', 'cls', 1) AS _init_cls \gset +SELECT COUNT(*) = 1 AS cls_meta_ok +FROM information_schema.tables +WHERE table_name = 'test_dws_cloudsync' \gset +\if :cls_meta_ok +\echo [PASS] (:testid) CLS init works after DWS/AWS rejection +\else +\echo [FAIL] (:testid) CLS init should work +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_40; +\endif diff --git a/test/postgresql/41_corrupted_payload.sql b/test/postgresql/41_corrupted_payload.sql new file mode 100644 index 0000000..235c79d --- /dev/null +++ b/test/postgresql/41_corrupted_payload.sql @@ -0,0 +1,130 @@ +-- Test corrupted payload handling +-- Verifies that cloudsync_payload_apply rejects corrupted payloads +-- without crashing or corrupting state. + +\set testid '41' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_41_src; +DROP DATABASE IF EXISTS cloudsync_test_41_dst; +CREATE DATABASE cloudsync_test_41_src; +CREATE DATABASE cloudsync_test_41_dst; + +-- Setup source database with data +\connect cloudsync_test_41_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'value1'); +INSERT INTO test_tbl VALUES ('id2', 'value2'); + +-- Get a valid payload +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS valid_payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup destination database +\connect cloudsync_test_41_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Record initial state +SELECT COUNT(*) AS initial_count FROM test_tbl \gset + +-- Test 1: Empty blob (zero bytes) +DO $$ +BEGIN + PERFORM cloudsync_payload_apply(''::bytea); + -- If it returns without error with 0 rows, that's also acceptable +EXCEPTION WHEN OTHERS THEN + -- Expected: error on empty payload + NULL; +END $$; + +SELECT COUNT(*) AS count_after_empty FROM test_tbl \gset +SELECT (:count_after_empty::int = :initial_count::int) AS empty_blob_ok \gset +\if :empty_blob_ok +\echo [PASS] (:testid) Empty blob rejected - table unchanged +\else +\echo [FAIL] (:testid) Empty blob corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Random garbage bytes +DO $$ +BEGIN + PERFORM cloudsync_payload_apply(decode('deadbeefcafebabe0102030405060708', 'hex')); +EXCEPTION WHEN OTHERS THEN + -- Expected: error on garbage payload + NULL; +END $$; + +SELECT COUNT(*) AS count_after_garbage FROM test_tbl \gset +SELECT (:count_after_garbage::int = :initial_count::int) AS garbage_ok \gset +\if :garbage_ok +\echo [PASS] (:testid) Garbage bytes rejected - table unchanged +\else +\echo [FAIL] (:testid) Garbage bytes corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Truncated payload (first 10 bytes of valid payload) +-- Build truncated hex at top level using psql variable interpolation +SELECT substr(:'valid_payload_hex', 1, 20) AS truncated_hex \gset +-- Expected: error on truncated payload — locally disable ON_ERROR_STOP +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'truncated_hex', 'hex')) AS _apply_truncated \gset +\set ON_ERROR_STOP on + +SELECT COUNT(*) AS count_after_truncated FROM test_tbl \gset +SELECT (:count_after_truncated::int = :initial_count::int) AS truncated_ok \gset +\if :truncated_ok +\echo [PASS] (:testid) Truncated payload rejected - table unchanged +\else +\echo [FAIL] (:testid) Truncated payload corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: Valid payload with flipped byte in the middle +-- Compute corrupted payload at top level: flip one byte via XOR with FF +SELECT + substr(:'valid_payload_hex', 1, length(:'valid_payload_hex') / 2 - 1) + || lpad(to_hex(get_byte(decode(substr(:'valid_payload_hex', length(:'valid_payload_hex') / 2, 2), 'hex'), 0) # 255), 2, '0') + || substr(:'valid_payload_hex', length(:'valid_payload_hex') / 2 + 2) + AS corrupted_hex \gset +-- Expected: error on corrupted payload — locally disable ON_ERROR_STOP +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'corrupted_hex', 'hex')) AS _apply_corrupted \gset +\set ON_ERROR_STOP on + +SELECT COUNT(*) AS count_after_flipped FROM test_tbl \gset +SELECT (:count_after_flipped::int = :initial_count::int) AS flipped_ok \gset +\if :flipped_ok +\echo [PASS] (:testid) Flipped-byte payload rejected - table unchanged +\else +\echo [FAIL] (:testid) Flipped-byte payload corrupted table state +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Now apply the VALID payload to confirm it still works +SELECT cloudsync_payload_apply(decode(:'valid_payload_hex', 'hex')) AS valid_apply \gset +SELECT COUNT(*) AS count_after_valid FROM test_tbl \gset +SELECT (:count_after_valid::int = 2) AS valid_ok \gset +\if :valid_ok +\echo [PASS] (:testid) Valid payload applied successfully after corrupted attempts +\else +\echo [FAIL] (:testid) Valid payload failed after corrupted attempts - count: :count_after_valid +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_41_src; +DROP DATABASE IF EXISTS cloudsync_test_41_dst; +\endif diff --git a/test/postgresql/42_payload_idempotency.sql b/test/postgresql/42_payload_idempotency.sql new file mode 100644 index 0000000..86b6262 --- /dev/null +++ b/test/postgresql/42_payload_idempotency.sql @@ -0,0 +1,88 @@ +-- Test payload apply idempotency +-- Applying the same payload multiple times must produce identical results. + +\set testid '42' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_42_src; +DROP DATABASE IF EXISTS cloudsync_test_42_dst; +CREATE DATABASE cloudsync_test_42_src; +CREATE DATABASE cloudsync_test_42_dst; + +-- Setup source with data +\connect cloudsync_test_42_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT, num INTEGER); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'hello', 10); +INSERT INTO test_tbl VALUES ('id2', 'world', 20); +UPDATE test_tbl SET val = 'hello_updated' WHERE id = 'id1'; + +-- Encode payload +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup destination +\connect cloudsync_test_42_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT, num INTEGER); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Apply #1 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_1 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_1 +FROM test_tbl \gset +SELECT COUNT(*) AS count_1 FROM test_tbl \gset + +-- Apply #2 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_2 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_2 +FROM test_tbl \gset +SELECT COUNT(*) AS count_2 FROM test_tbl \gset + +-- Apply #3 +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_3 \gset +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, '') || ':' || COALESCE(num::text, ''), ',' ORDER BY id), '')) AS hash_3 +FROM test_tbl \gset +SELECT COUNT(*) AS count_3 FROM test_tbl \gset + +-- Verify row count stays constant +SELECT (:count_1::int = :count_2::int AND :count_2::int = :count_3::int) AS count_stable \gset +\if :count_stable +\echo [PASS] (:testid) Row count stable across 3 applies (:count_1 rows) +\else +\echo [FAIL] (:testid) Row count changed: :count_1 -> :count_2 -> :count_3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify data hash is identical after each apply +SELECT (:'hash_1' = :'hash_2' AND :'hash_2' = :'hash_3') AS hash_stable \gset +\if :hash_stable +\echo [PASS] (:testid) Data hash identical across 3 applies +\else +\echo [FAIL] (:testid) Data hash changed: :hash_1 -> :hash_2 -> :hash_3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify data values are correct +SELECT COUNT(*) = 1 AS data_ok +FROM test_tbl +WHERE id = 'id1' AND val = 'hello_updated' AND num = 10 \gset +\if :data_ok +\echo [PASS] (:testid) Data values correct after idempotent applies +\else +\echo [FAIL] (:testid) Data values incorrect +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_42_src; +DROP DATABASE IF EXISTS cloudsync_test_42_dst; +\endif diff --git a/test/postgresql/43_delete_resurrect_ordering.sql b/test/postgresql/43_delete_resurrect_ordering.sql new file mode 100644 index 0000000..a95f27a --- /dev/null +++ b/test/postgresql/43_delete_resurrect_ordering.sql @@ -0,0 +1,147 @@ +-- Test delete/resurrect with out-of-order payload delivery +-- Verifies CRDT causal length parity handles resurrection correctly +-- even when payloads arrive in non-causal order. + +\set testid '43' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_43_a; +DROP DATABASE IF EXISTS cloudsync_test_43_b; +DROP DATABASE IF EXISTS cloudsync_test_43_c; +CREATE DATABASE cloudsync_test_43_a; +CREATE DATABASE cloudsync_test_43_b; +CREATE DATABASE cloudsync_test_43_c; + +-- Setup all three databases +\connect cloudsync_test_43_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_a \gset + +\connect cloudsync_test_43_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_b \gset + +\connect cloudsync_test_43_c +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_c \gset + +-- Round 1: A inserts row, sync to all +\connect cloudsync_test_43_a +INSERT INTO test_tbl VALUES ('row1', 'original'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r1, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r1_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +\connect cloudsync_test_43_b +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply \gset +\endif + +\connect cloudsync_test_43_c +\if :payload_a_r1_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r1', 3), 'hex')) AS _apply \gset +\endif + +-- Round 2: A deletes row (CL goes 1->2) +\connect cloudsync_test_43_a +DELETE FROM test_tbl WHERE id = 'row1'; +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_a_r2, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_a_r2_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- Sync delete to B +\connect cloudsync_test_43_b +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply \gset +\endif + +-- Round 3: B re-inserts (CL goes 2->3, resurrection) +\connect cloudsync_test_43_b +INSERT INTO test_tbl VALUES ('row1', 'resurrected_by_b'); +SELECT CASE WHEN payload IS NULL OR octet_length(payload) = 0 + THEN '' + ELSE '\x' || encode(payload, 'hex') + END AS payload_b_r3, + (payload IS NOT NULL AND octet_length(payload) > 0) AS payload_b_r3_ok +FROM ( + SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload + FROM cloudsync_changes + WHERE site_id = cloudsync_siteid() +) AS p \gset + +-- C receives payloads in REVERSE order: B's resurrection FIRST, then A's delete +\connect cloudsync_test_43_c +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_b \gset +\endif +\if :payload_a_r2_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_a_r2', 3), 'hex')) AS _apply_a \gset +\endif + +-- A receives B's resurrection +\connect cloudsync_test_43_a +\if :payload_b_r3_ok +SELECT cloudsync_payload_apply(decode(substr(:'payload_b_r3', 3), 'hex')) AS _apply_b \gset +\endif + +-- Final convergence check: all three should have row1 with 'resurrected_by_b' +\connect cloudsync_test_43_a +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_a +FROM test_tbl \gset + +\connect cloudsync_test_43_b +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_b +FROM test_tbl \gset + +\connect cloudsync_test_43_c +SELECT md5(COALESCE(string_agg(id || ':' || COALESCE(val, ''), ',' ORDER BY id), '')) AS hash_c +FROM test_tbl \gset + +SELECT (:'hash_a' = :'hash_b' AND :'hash_b' = :'hash_c') AS all_converge \gset +\if :all_converge +\echo [PASS] (:testid) All 3 databases converge after out-of-order delete/resurrect +\else +\echo [FAIL] (:testid) Databases diverged - A: :hash_a, B: :hash_b, C: :hash_c +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify the row exists (resurrection won) +SELECT COUNT(*) = 1 AS row_exists +FROM test_tbl +WHERE id = 'row1' \gset +\if :row_exists +\echo [PASS] (:testid) Resurrected row exists on C (received out of order) +\else +\echo [FAIL] (:testid) Resurrected row missing on C +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_43_a; +DROP DATABASE IF EXISTS cloudsync_test_43_b; +DROP DATABASE IF EXISTS cloudsync_test_43_c; +\endif diff --git a/test/postgresql/44_large_composite_pk.sql b/test/postgresql/44_large_composite_pk.sql new file mode 100644 index 0000000..316b9f0 --- /dev/null +++ b/test/postgresql/44_large_composite_pk.sql @@ -0,0 +1,142 @@ +-- Test large composite primary key (5 columns) +-- Verifies pk_encode/pk_decode handles complex multi-column PKs correctly. + +\set testid '44' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_44_a; +DROP DATABASE IF EXISTS cloudsync_test_44_b; +CREATE DATABASE cloudsync_test_44_a; +CREATE DATABASE cloudsync_test_44_b; + +-- Setup Database A +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE composite_pk_tbl ( + pk_text1 TEXT NOT NULL, + pk_int1 INTEGER NOT NULL, + pk_text2 TEXT NOT NULL, + pk_int2 INTEGER NOT NULL, + pk_text3 TEXT NOT NULL, + data_col TEXT, + num_col INTEGER, + PRIMARY KEY (pk_text1, pk_int1, pk_text2, pk_int2, pk_text3) +); + +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _init_a \gset + +INSERT INTO composite_pk_tbl VALUES ('alpha', 1, 'beta', 100, 'gamma', 'data_a1', 42); +INSERT INTO composite_pk_tbl VALUES ('alpha', 2, 'beta', 200, 'delta', 'data_a2', 84); +INSERT INTO composite_pk_tbl VALUES ('x', 999, 'y', -1, 'z', 'edge_case', 0); + +-- Setup Database B +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE composite_pk_tbl ( + pk_text1 TEXT NOT NULL, + pk_int1 INTEGER NOT NULL, + pk_text2 TEXT NOT NULL, + pk_int2 INTEGER NOT NULL, + pk_text3 TEXT NOT NULL, + data_col TEXT, + num_col INTEGER, + PRIMARY KEY (pk_text1, pk_int1, pk_text2, pk_int2, pk_text3) +); + +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _init_b \gset + +INSERT INTO composite_pk_tbl VALUES ('alpha', 1, 'beta', 100, 'gamma', 'data_b1', 99); +INSERT INTO composite_pk_tbl VALUES ('foo', 3, 'bar', 300, 'baz', 'data_b2', 77); + +-- Encode and exchange payloads +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_b +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply A -> B +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS apply_a_to_b \gset + +-- Apply B -> A +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_b', 'hex')) AS apply_b_to_a \gset + +-- Update a row on A +UPDATE composite_pk_tbl SET data_col = 'updated_on_a' WHERE pk_text1 = 'foo' AND pk_int1 = 3 AND pk_text2 = 'bar' AND pk_int2 = 300 AND pk_text3 = 'baz'; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a2 +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_44_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('composite_pk_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a2', 'hex')) AS apply_a2_to_b \gset + +-- Final hash comparison +SELECT md5(COALESCE(string_agg( + pk_text1 || ':' || pk_int1::text || ':' || pk_text2 || ':' || pk_int2::text || ':' || pk_text3 || ':' || + COALESCE(data_col, 'NULL') || ':' || COALESCE(num_col::text, 'NULL'), + '|' ORDER BY pk_text1, pk_int1, pk_text2, pk_int2, pk_text3 +), '')) AS hash_b FROM composite_pk_tbl \gset + +\connect cloudsync_test_44_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + pk_text1 || ':' || pk_int1::text || ':' || pk_text2 || ':' || pk_int2::text || ':' || pk_text3 || ':' || + COALESCE(data_col, 'NULL') || ':' || COALESCE(num_col::text, 'NULL'), + '|' ORDER BY pk_text1, pk_int1, pk_text2, pk_int2, pk_text3 +), '')) AS hash_a FROM composite_pk_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS hashes_match \gset +\if :hashes_match +\echo [PASS] (:testid) Large composite PK (5 cols) roundtrip and update +\else +\echo [FAIL] (:testid) Hash mismatch - A: :hash_a, B: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify row count +SELECT COUNT(*) AS row_count FROM composite_pk_tbl \gset +SELECT (:row_count::int = 4) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) Row count correct (4 rows with 5-col composite PK) +\else +\echo [FAIL] (:testid) Expected 4 rows, got :row_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify update propagated +SELECT COUNT(*) = 1 AS update_ok +FROM composite_pk_tbl +WHERE pk_text1 = 'foo' AND pk_int1 = 3 AND data_col = 'updated_on_a' \gset +\if :update_ok +\echo [PASS] (:testid) Update propagated correctly for composite PK row +\else +\echo [FAIL] (:testid) Update not propagated +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_44_a; +DROP DATABASE IF EXISTS cloudsync_test_44_b; +\endif diff --git a/test/postgresql/45_pg_specific_types.sql b/test/postgresql/45_pg_specific_types.sql new file mode 100644 index 0000000..70ff701 --- /dev/null +++ b/test/postgresql/45_pg_specific_types.sql @@ -0,0 +1,176 @@ +-- Test PostgreSQL-specific type roundtrips +-- Covers JSONB, TIMESTAMPTZ, NUMERIC with precision, BYTEA + +\set testid '45' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_45_a; +DROP DATABASE IF EXISTS cloudsync_test_45_b; +CREATE DATABASE cloudsync_test_45_a; +CREATE DATABASE cloudsync_test_45_b; + +-- Setup Database A +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE typed_tbl ( + id TEXT PRIMARY KEY, + json_col JSONB, + ts_col TIMESTAMPTZ, + num_col NUMERIC(18, 6), + bin_col BYTEA +); + +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _init_a \gset + +INSERT INTO typed_tbl VALUES ( + 'row1', + '{"key": "value", "nested": {"arr": [1, 2, 3]}}', + '2025-01-15 10:30:00+00', + 123456.789012, + '\x48656c6c6f' +); + +INSERT INTO typed_tbl VALUES ( + 'row2', + '[1, "two", null, true, false]', + '2024-06-30 23:59:59.999999+05:30', + -999999.123456, + '\xdeadbeef' +); + +INSERT INTO typed_tbl VALUES ( + 'row3', + 'null', + '1970-01-01 00:00:00+00', + 0.000000, + '\x' +); + +-- Setup Database B +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE typed_tbl ( + id TEXT PRIMARY KEY, + json_col JSONB, + ts_col TIMESTAMPTZ, + num_col NUMERIC(18, 6), + bin_col BYTEA +); + +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _init_b \gset + +-- Encode and apply A -> B +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_a +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('typed_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_a', 'hex')) AS apply_a_to_b \gset + +-- Verify row count +SELECT COUNT(*) AS count_b FROM typed_tbl \gset +SELECT (:count_b::int = 3) AS count_ok \gset +\if :count_ok +\echo [PASS] (:testid) All 3 rows synced to B +\else +\echo [FAIL] (:testid) Expected 3 rows, got :count_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify JSONB roundtrip +SELECT COUNT(*) = 1 AS jsonb_ok +FROM typed_tbl +WHERE id = 'row1' + AND json_col @> '{"key": "value"}' + AND json_col -> 'nested' -> 'arr' = '[1, 2, 3]'::jsonb \gset +\if :jsonb_ok +\echo [PASS] (:testid) JSONB roundtrip correct +\else +\echo [FAIL] (:testid) JSONB data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify TIMESTAMPTZ roundtrip +SELECT COUNT(*) = 1 AS ts_ok +FROM typed_tbl +WHERE id = 'row1' + AND ts_col = '2025-01-15 10:30:00+00'::timestamptz \gset +\if :ts_ok +\echo [PASS] (:testid) TIMESTAMPTZ roundtrip correct +\else +\echo [FAIL] (:testid) TIMESTAMPTZ data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify NUMERIC roundtrip +SELECT COUNT(*) = 1 AS num_ok +FROM typed_tbl +WHERE id = 'row1' + AND num_col = 123456.789012 \gset +\if :num_ok +\echo [PASS] (:testid) NUMERIC(18,6) roundtrip correct +\else +\echo [FAIL] (:testid) NUMERIC data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify BYTEA roundtrip +SELECT COUNT(*) = 1 AS bytea_ok +FROM typed_tbl +WHERE id = 'row1' + AND bin_col = '\x48656c6c6f'::bytea \gset +\if :bytea_ok +\echo [PASS] (:testid) BYTEA roundtrip correct +\else +\echo [FAIL] (:testid) BYTEA data mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify full hash match +\connect cloudsync_test_45_a +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + id || ':' || + COALESCE(json_col::text, 'NULL') || ':' || + COALESCE(ts_col::text, 'NULL') || ':' || + COALESCE(num_col::text, 'NULL') || ':' || + COALESCE(encode(bin_col, 'hex'), 'NULL'), + '|' ORDER BY id +), '')) AS hash_a FROM typed_tbl \gset + +\connect cloudsync_test_45_b +\ir helper_psql_conn_setup.sql +SELECT md5(COALESCE(string_agg( + id || ':' || + COALESCE(json_col::text, 'NULL') || ':' || + COALESCE(ts_col::text, 'NULL') || ':' || + COALESCE(num_col::text, 'NULL') || ':' || + COALESCE(encode(bin_col, 'hex'), 'NULL'), + '|' ORDER BY id +), '')) AS hash_b FROM typed_tbl \gset + +SELECT (:'hash_a' = :'hash_b') AS hash_match \gset +\if :hash_match +\echo [PASS] (:testid) Full data hash matches for PG-specific types +\else +\echo [FAIL] (:testid) Hash mismatch - A: :hash_a, B: :hash_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_45_a; +DROP DATABASE IF EXISTS cloudsync_test_45_b; +\endif diff --git a/test/postgresql/46_schema_hash_mismatch.sql b/test/postgresql/46_schema_hash_mismatch.sql new file mode 100644 index 0000000..6aa1602 --- /dev/null +++ b/test/postgresql/46_schema_hash_mismatch.sql @@ -0,0 +1,95 @@ +-- Test schema hash mismatch during merge +-- Verifies detection when ALTER TABLE is done without cloudsync_begin/commit_alter. + +\set testid '46' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_46_src; +DROP DATABASE IF EXISTS cloudsync_test_46_dst; +CREATE DATABASE cloudsync_test_46_src; +CREATE DATABASE cloudsync_test_46_dst; + +-- Setup source +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_src \gset +INSERT INTO test_tbl VALUES ('id1', 'value1'); + +-- Setup destination with same schema +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE test_tbl (id TEXT PRIMARY KEY, val TEXT); +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _init_dst \gset + +-- Initial sync to get both in sync +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_initial +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +SELECT cloudsync_payload_apply(decode(:'payload_initial', 'hex')) AS _apply_initial \gset + +-- Now ALTER TABLE on destination WITHOUT using cloudsync_begin/commit_alter +ALTER TABLE test_tbl ADD COLUMN extra TEXT DEFAULT 'default'; + +-- Insert new data on source +\connect cloudsync_test_46_src +\ir helper_psql_conn_setup.sql +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit \gset +INSERT INTO test_tbl VALUES ('id2', 'value2'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_post_alter +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Apply payload from pre-alter source to post-alter destination +-- This should detect schema mismatch +\connect cloudsync_test_46_dst +\ir helper_psql_conn_setup.sql + +-- Reinit to pick up new schema +SELECT cloudsync_init('test_tbl', 'CLS', 1) AS _reinit_dst \gset + +-- The apply may error due to schema mismatch, or succeed silently. +-- Either outcome is acceptable — the key is no corruption (verified below). +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'payload_post_alter', 'hex')) AS _apply_mismatch \gset +\set ON_ERROR_STOP on + +-- Verify database is in a consistent state (not corrupted) +SELECT COUNT(*) AS final_count FROM test_tbl \gset +SELECT (:final_count::int >= 1) AS state_ok \gset +\if :state_ok +\echo [PASS] (:testid) Database consistent after schema mismatch scenario (rows: :final_count) +\else +\echo [FAIL] (:testid) Database corrupted after schema mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify original data is intact +SELECT COUNT(*) = 1 AS original_ok +FROM test_tbl +WHERE id = 'id1' AND val = 'value1' \gset +\if :original_ok +\echo [PASS] (:testid) Original data intact after schema mismatch +\else +\echo [FAIL] (:testid) Original data corrupted +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_46_src; +DROP DATABASE IF EXISTS cloudsync_test_46_dst; +\endif diff --git a/test/postgresql/47_row_filter_advanced.sql b/test/postgresql/47_row_filter_advanced.sql new file mode 100644 index 0000000..c5c8996 --- /dev/null +++ b/test/postgresql/47_row_filter_advanced.sql @@ -0,0 +1,194 @@ +-- 'Row-level filter advanced tests (clear, complex expressions, row transitions, filter change)' + +\set testid '47' +\ir helper_test_init.sql + +-- Create database +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_47_a; +CREATE DATABASE cloudsync_test_47_a; + +\connect cloudsync_test_47_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- ============================================================ +-- Test 1: cloudsync_clear_filter lifecycle +-- ============================================================ +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf \gset + +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); + +-- Only matching rows tracked +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 2) AS clear_t1a_ok \gset +\if :clear_t1a_ok +\echo [PASS] (:testid) clear_filter: 2 PKs tracked before clear +\else +\echo [FAIL] (:testid) clear_filter: expected 2 tracked PKs before clear, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Clear filter +SELECT cloudsync_clear_filter('tasks') AS _cf \gset + +-- Insert non-matching row — should now be tracked (no filter) +-- clear_filter refilled metatable with all 3 existing rows (a, b, c) + insert d = 4 +INSERT INTO tasks VALUES ('d', 'Task D', 2); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS clear_t1b_ok \gset +\if :clear_t1b_ok +\echo [PASS] (:testid) clear_filter: non-matching row tracked after clear (4 PKs) +\else +\echo [FAIL] (:testid) clear_filter: expected 4 PKs after clear+insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update row 'b' — already tracked by clear_filter refill, meta count unchanged +UPDATE tasks SET title = 'Task B Updated' WHERE id = 'b'; +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS clear_t1c_ok \gset +\if :clear_t1c_ok +\echo [PASS] (:testid) clear_filter: update on 'b' still 4 PKs +\else +\echo [FAIL] (:testid) clear_filter: expected 4 PKs after update on 'b', got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Complex filter — AND + comparison operators +-- ============================================================ +DROP TABLE IF EXISTS items; +CREATE TABLE items (id TEXT PRIMARY KEY NOT NULL, status TEXT, priority INTEGER, category TEXT, user_id INTEGER); +SELECT cloudsync_init('items') AS _init \gset +SELECT cloudsync_set_filter('items', 'user_id = 1 AND priority > 3') AS _sf \gset + +INSERT INTO items VALUES ('a', 'active', 5, 'work', 1); -- matches +INSERT INTO items VALUES ('b', 'active', 2, 'work', 1); -- fails priority +INSERT INTO items VALUES ('c', 'active', 5, 'work', 2); -- fails user_id + +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM items_cloudsync \gset +SELECT (:meta_pk_count = 1) AS complex_t2_ok \gset +\if :complex_t2_ok +\echo [PASS] (:testid) complex_filter: AND+comparison tracked 1 of 3 rows +\else +\echo [FAIL] (:testid) complex_filter: expected 1 tracked PK, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: IS NULL filter +-- ============================================================ +SELECT cloudsync_clear_filter('items') AS _cf \gset +SELECT cloudsync_set_filter('items', 'category IS NULL') AS _sf \gset + +SELECT COUNT(DISTINCT pk) AS meta_before FROM items_cloudsync \gset +INSERT INTO items VALUES ('f', 'x', 1, NULL, 1); -- matches +INSERT INTO items VALUES ('g', 'x', 1, 'work', 1); -- fails +SELECT COUNT(DISTINCT pk) AS meta_after FROM items_cloudsync \gset +SELECT ((:meta_after::int - :meta_before::int) = 1) AS null_t3_ok \gset +\if :null_t3_ok +\echo [PASS] (:testid) IS NULL filter: only NULL-category row tracked +\else +\echo [FAIL] (:testid) IS NULL filter: expected 1 new PK, got (:meta_after - :meta_before) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Row exits filter (matching -> non-matching via UPDATE) +-- ============================================================ +DROP TABLE IF EXISTS trans; +CREATE TABLE trans (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('trans') AS _init \gset +SELECT cloudsync_set_filter('trans', 'user_id = 1') AS _sf \gset + +INSERT INTO trans VALUES ('a', 'Task A', 1); + +SELECT COUNT(*) AS meta_before FROM trans_cloudsync \gset +UPDATE trans SET user_id = 2 WHERE id = 'a'; +SELECT COUNT(*) AS meta_after FROM trans_cloudsync \gset +SELECT (:meta_before = :meta_after) AS exit_t4_ok \gset +\if :exit_t4_ok +\echo [PASS] (:testid) row_exit: UPDATE out of filter did not change metadata +\else +\echo [FAIL] (:testid) row_exit: UPDATE out of filter changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Row enters filter (non-matching -> matching via UPDATE) +-- ============================================================ +INSERT INTO trans VALUES ('b', 'Task B', 2); + +SELECT COUNT(DISTINCT pk) AS meta_before FROM trans_cloudsync \gset +UPDATE trans SET user_id = 1 WHERE id = 'b'; +SELECT COUNT(DISTINCT pk) AS meta_after FROM trans_cloudsync \gset +SELECT (:meta_after::int > :meta_before::int) AS enter_t5_ok \gset +\if :enter_t5_ok +\echo [PASS] (:testid) row_enter: UPDATE into filter created metadata +\else +\echo [FAIL] (:testid) row_enter: UPDATE into filter did not create metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 6: Filter change after data +-- ============================================================ +DROP TABLE IF EXISTS fchange; +CREATE TABLE fchange (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('fchange') AS _init \gset +SELECT cloudsync_set_filter('fchange', 'user_id = 1') AS _sf \gset + +INSERT INTO fchange VALUES ('a', 'A', 1); -- matches +INSERT INTO fchange VALUES ('b', 'B', 2); -- non-matching +INSERT INTO fchange VALUES ('c', 'C', 1); -- matches + +SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset +SELECT (:meta_count = 2) AS change_t6a_ok \gset +\if :change_t6a_ok +\echo [PASS] (:testid) filter_change: 2 PKs under initial filter +\else +\echo [FAIL] (:testid) filter_change: expected 2 PKs under initial filter, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Change filter — resets metatable to only rows matching new filter (user_id = 2) +-- Only 'b' (user_id=2) matches new filter → 1 PK from refill, then insert d → 2 +SELECT cloudsync_set_filter('fchange', 'user_id = 2') AS _sf2 \gset + +INSERT INTO fchange VALUES ('d', 'D', 2); -- matches new filter +INSERT INTO fchange VALUES ('e', 'E', 1); -- non-matching under new filter + +SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset +SELECT (:meta_count = 2) AS change_t6b_ok \gset +\if :change_t6b_ok +\echo [PASS] (:testid) filter_change: 2 PKs after filter change (metatable reset) +\else +\echo [FAIL] (:testid) filter_change: expected 2 PKs after filter change, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Update 'a' (user_id=1) should NOT generate new metadata under new filter (user_id=2) +SELECT COUNT(*) AS meta_before FROM fchange_cloudsync \gset +UPDATE fchange SET title = 'A Updated' WHERE id = 'a'; +SELECT COUNT(*) AS meta_after FROM fchange_cloudsync \gset +SELECT (:meta_before = :meta_after) AS change_t6c_ok \gset +\if :change_t6c_ok +\echo [PASS] (:testid) filter_change: update on 'a' not tracked under new filter +\else +\echo [FAIL] (:testid) filter_change: update on 'a' changed metadata (:meta_before -> :meta_after) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_47_a; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/48_row_filter_multi_table.sql b/test/postgresql/48_row_filter_multi_table.sql new file mode 100644 index 0000000..c2a0590 --- /dev/null +++ b/test/postgresql/48_row_filter_multi_table.sql @@ -0,0 +1,166 @@ +-- 'Row-level filter: multi-table with different filters and composite primary keys' + +\set testid '48' +\ir helper_test_init.sql + +-- Create databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_48_a; +DROP DATABASE IF EXISTS cloudsync_test_48_b; +CREATE DATABASE cloudsync_test_48_a; +CREATE DATABASE cloudsync_test_48_b; + +-- ============================================================ +-- Setup Database A +-- ============================================================ +\connect cloudsync_test_48_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Table 1: composite PK with simple filter +CREATE TABLE projects (org_id INTEGER NOT NULL, proj_id INTEGER NOT NULL, name TEXT, PRIMARY KEY(org_id, proj_id)); +SELECT cloudsync_init('projects') AS _init \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf \gset + +-- Table 2: composite PK with multi-column filter including string literal +CREATE TABLE members (org_id INTEGER NOT NULL, user_id INTEGER NOT NULL, role TEXT, PRIMARY KEY(org_id, user_id)); +SELECT cloudsync_init('members') AS _init \gset +SELECT cloudsync_set_filter('members', 'org_id = 1 AND role = ''admin''') AS _sf \gset + +-- ============================================================ +-- Test 1: Composite PK — only matching rows tracked +-- ============================================================ +INSERT INTO projects VALUES (1, 1, 'Proj A'); -- matches +INSERT INTO projects VALUES (2, 1, 'Proj B'); -- fails org_id +INSERT INTO projects VALUES (1, 2, 'Proj C'); -- matches + +SELECT COUNT(DISTINCT pk) AS proj_meta FROM projects_cloudsync \gset +SELECT (:proj_meta = 2) AS t1_proj_ok \gset +\if :t1_proj_ok +\echo [PASS] (:testid) composite_pk: 2 of 3 projects tracked +\else +\echo [FAIL] (:testid) composite_pk: expected 2 project PKs, got :proj_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: Multi-column filter with string literal — different table +-- ============================================================ +INSERT INTO members VALUES (1, 10, 'admin'); -- matches both conditions +INSERT INTO members VALUES (1, 20, 'viewer'); -- fails role +INSERT INTO members VALUES (2, 10, 'admin'); -- fails org_id + +SELECT COUNT(DISTINCT pk) AS mem_meta FROM members_cloudsync \gset +SELECT (:mem_meta = 1) AS t2_mem_ok \gset +\if :t2_mem_ok +\echo [PASS] (:testid) multi_filter: 1 of 3 members tracked +\else +\echo [FAIL] (:testid) multi_filter: expected 1 member PK, got :mem_meta +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: Roundtrip sync — only matching rows per table transfer +-- ============================================================ +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_48_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE projects (org_id INTEGER NOT NULL, proj_id INTEGER NOT NULL, name TEXT, PRIMARY KEY(org_id, proj_id)); +SELECT cloudsync_init('projects') AS _init \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf \gset + +CREATE TABLE members (org_id INTEGER NOT NULL, user_id INTEGER NOT NULL, role TEXT, PRIMARY KEY(org_id, user_id)); +SELECT cloudsync_init('members') AS _init \gset +SELECT cloudsync_set_filter('members', 'org_id = 1 AND role = ''admin''') AS _sf \gset + +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS _apply \gset + +-- Verify projects +SELECT COUNT(*) AS proj_count FROM projects \gset +SELECT (:proj_count = 2) AS t3_proj_ok \gset +\if :t3_proj_ok +\echo [PASS] (:testid) roundtrip: 2 projects synced to db_b +\else +\echo [FAIL] (:testid) roundtrip: expected 2 projects in db_b, got :proj_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify members +SELECT COUNT(*) AS mem_count FROM members \gset +SELECT (:mem_count = 1) AS t3_mem_ok \gset +\if :t3_mem_ok +\echo [PASS] (:testid) roundtrip: 1 member synced to db_b +\else +\echo [FAIL] (:testid) roundtrip: expected 1 member in db_b, got :mem_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify correct member identity +SELECT COUNT(*) AS admin_exists FROM members WHERE org_id = 1 AND user_id = 10 AND role = 'admin' \gset +SELECT (:admin_exists = 1) AS t3_admin_ok \gset +\if :t3_admin_ok +\echo [PASS] (:testid) roundtrip: correct admin member (1,10) present +\else +\echo [FAIL] (:testid) roundtrip: admin member (1,10) not found +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Non-matching rows should NOT exist +SELECT COUNT(*) AS bad_proj FROM projects WHERE org_id = 2 \gset +SELECT (:bad_proj = 0) AS t3_no_bad_proj \gset +\if :t3_no_bad_proj +\echo [PASS] (:testid) roundtrip: no org_id=2 projects in db_b +\else +\echo [FAIL] (:testid) roundtrip: unexpected org_id=2 projects found (:bad_proj) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Update and delete on composite PK, then re-sync +-- ============================================================ +\connect cloudsync_test_48_a +\ir helper_psql_conn_setup.sql + +UPDATE projects SET name = 'Proj A Updated' WHERE org_id = 1 AND proj_id = 1; +DELETE FROM projects WHERE org_id = 1 AND proj_id = 2; + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload2_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_48_b +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(decode(:'payload2_hex', 'hex')) AS _apply2 \gset + +SELECT COUNT(*) AS proj_count FROM projects \gset +SELECT (:proj_count = 1) AS t4_count_ok \gset +\if :t4_count_ok +\echo [PASS] (:testid) update_delete: 1 project remaining after sync +\else +\echo [FAIL] (:testid) update_delete: expected 1 project, got :proj_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT COUNT(*) AS updated_exists FROM projects WHERE org_id = 1 AND proj_id = 1 AND name = 'Proj A Updated' \gset +SELECT (:updated_exists = 1) AS t4_updated_ok \gset +\if :t4_updated_ok +\echo [PASS] (:testid) update_delete: 'Proj A Updated' present in db_b +\else +\echo [FAIL] (:testid) update_delete: 'Proj A Updated' not found in db_b +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_48_a; +DROP DATABASE IF EXISTS cloudsync_test_48_b; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/49_row_filter_prefill.sql b/test/postgresql/49_row_filter_prefill.sql new file mode 100644 index 0000000..a570856 --- /dev/null +++ b/test/postgresql/49_row_filter_prefill.sql @@ -0,0 +1,192 @@ +-- 'Row-level filter with pre-existing data (prefill tests)' +-- Tests that cloudsync_refill_metatable correctly handles the filter +-- when rows exist before cloudsync_init and cloudsync_set_filter. + +\set testid '49' +\ir helper_test_init.sql + +-- Create databases +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_49_a; +DROP DATABASE IF EXISTS cloudsync_test_49_b; +CREATE DATABASE cloudsync_test_49_a; +CREATE DATABASE cloudsync_test_49_b; + +-- ============================================================ +-- Setup Database A — insert data BEFORE cloudsync_init +-- ============================================================ + +\connect cloudsync_test_49_a +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); + +-- Pre-existing rows: 3 matching (user_id=1), 2 non-matching +INSERT INTO tasks VALUES ('a', 'Task A', 1); +INSERT INTO tasks VALUES ('b', 'Task B', 2); +INSERT INTO tasks VALUES ('c', 'Task C', 1); +INSERT INTO tasks VALUES ('d', 'Task D', 3); +INSERT INTO tasks VALUES ('e', 'Task E', 1); + +-- Init and set filter AFTER data exists +SELECT cloudsync_init('tasks') AS _init_a \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf_a \gset + +-- ============================================================ +-- Test 1: set_filter resets metatable to only matching rows +-- cloudsync_init filled all 5, then set_filter cleaned and refilled → 3 matching (a, c, e) +-- ============================================================ + +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t1_ok \gset +\if :prefill_t1_ok +\echo [PASS] (:testid) prefill: 3 matching rows have metadata after set_filter +\else +\echo [FAIL] (:testid) prefill: expected 3 tracked PKs after set_filter, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 2: New matching insert IS tracked +-- ============================================================ + +INSERT INTO tasks VALUES ('f', 'Task F', 1); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS prefill_t2_ok \gset +\if :prefill_t2_ok +\echo [PASS] (:testid) prefill: new matching insert tracked (4 PKs) +\else +\echo [FAIL] (:testid) prefill: expected 4 PKs after matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 3: New non-matching insert is NOT tracked +-- ============================================================ + +INSERT INTO tasks VALUES ('g', 'Task G', 2); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset +SELECT (:meta_pk_count = 4) AS prefill_t3_ok \gset +\if :prefill_t3_ok +\echo [PASS] (:testid) prefill: new non-matching insert not tracked (still 4 PKs) +\else +\echo [FAIL] (:testid) prefill: expected still 4 PKs after non-matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 4: Sync roundtrip — pre-existing + matching new rows transfer +-- ============================================================ + +SELECT encode( + cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), + 'hex' +) AS payload_a_hex +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +-- Setup Database B (empty) +\connect cloudsync_test_49_b +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER); +SELECT cloudsync_init('tasks') AS _init_b \gset +SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf_b \gset + +-- Apply payload +SELECT cloudsync_payload_apply(decode(:'payload_a_hex', 'hex')) AS _apply \gset + +-- Only matching rows (a, c, e, f) should arrive; non-matching (b, d, g) should not +SELECT COUNT(*) AS row_count FROM tasks \gset +SELECT (:row_count = 4) AS prefill_t4a_ok \gset +\if :prefill_t4a_ok +\echo [PASS] (:testid) prefill_sync: 4 matching rows synced to Database B +\else +\echo [FAIL] (:testid) prefill_sync: expected 4 rows in Database B, got :row_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify non-matching post-filter row 'g' did NOT sync +SELECT COUNT(*) AS g_count FROM tasks WHERE id = 'g' \gset +SELECT (:g_count = 0) AS prefill_t4b_ok \gset +\if :prefill_t4b_ok +\echo [PASS] (:testid) prefill_sync: non-matching post-filter row 'g' not synced +\else +\echo [FAIL] (:testid) prefill_sync: row 'g' should not have synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Verify pre-existing non-matching row 'b' did NOT sync (metadata removed by set_filter) +SELECT COUNT(*) AS b_count FROM tasks WHERE id = 'b' AND user_id = 2 \gset +SELECT (:b_count = 0) AS prefill_t4c_ok \gset +\if :prefill_t4c_ok +\echo [PASS] (:testid) prefill_sync: pre-existing non-matching row 'b' not synced +\else +\echo [FAIL] (:testid) prefill_sync: pre-existing non-matching row 'b' should not have synced +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- ============================================================ +-- Test 5: Composite PK with pre-existing data +-- ============================================================ + +\connect cloudsync_test_49_a + +CREATE TABLE projects ( + org_id INTEGER NOT NULL, + proj_id INTEGER NOT NULL, + name TEXT, + status TEXT, + PRIMARY KEY(org_id, proj_id) +); + +-- Pre-existing rows: 2 matching (org_id=1), 2 non-matching +INSERT INTO projects VALUES (1, 1, 'Alpha', 'active'); +INSERT INTO projects VALUES (1, 2, 'Beta', 'active'); +INSERT INTO projects VALUES (2, 1, 'Gamma', 'active'); +INSERT INTO projects VALUES (2, 2, 'Delta', 'active'); + +SELECT cloudsync_init('projects') AS _init_proj \gset +SELECT cloudsync_set_filter('projects', 'org_id = 1') AS _sf_proj \gset + +-- set_filter resets metatable: only 2 matching rows (org_id=1) should have metadata +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 2) AS prefill_t5_ok \gset +\if :prefill_t5_ok +\echo [PASS] (:testid) prefill_composite: 2 matching rows have metadata after set_filter +\else +\echo [FAIL] (:testid) prefill_composite: expected 2 tracked PKs, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- New matching insert tracked +INSERT INTO projects VALUES (1, 3, 'Epsilon', 'active'); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t5b_ok \gset +\if :prefill_t5b_ok +\echo [PASS] (:testid) prefill_composite: new matching row tracked (3 PKs) +\else +\echo [FAIL] (:testid) prefill_composite: expected 3 PKs after matching insert, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- New non-matching insert NOT tracked +INSERT INTO projects VALUES (3, 1, 'Zeta', 'active'); +SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM projects_cloudsync \gset +SELECT (:meta_pk_count = 3) AS prefill_t5c_ok \gset +\if :prefill_t5c_ok +\echo [PASS] (:testid) prefill_composite: new non-matching row not tracked (still 3 PKs) +\else +\echo [FAIL] (:testid) prefill_composite: expected still 3 PKs, got :meta_pk_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_49_a; +DROP DATABASE IF EXISTS cloudsync_test_49_b; +\endif diff --git a/test/postgresql/50_block_lww_existing_data.sql b/test/postgresql/50_block_lww_existing_data.sql new file mode 100644 index 0000000..485a43e --- /dev/null +++ b/test/postgresql/50_block_lww_existing_data.sql @@ -0,0 +1,92 @@ +-- 'Block-level LWW: migration of existing tracked rows when algo=block is enabled' +-- Mirrors the SQLite unit test: Block LWW Existing Data + +\set testid '50' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_block_existing_a; +CREATE DATABASE cloudsync_block_existing_a; + +\connect cloudsync_block_existing_a +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Create a table and init cloudsync WITHOUT block algo first +DROP TABLE IF EXISTS docs; +CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('docs', 'CLS', 1) AS _init \gset + +-- Insert rows BEFORE enabling block algorithm (they will be tracked as regular CLS rows) +INSERT INTO docs (id, body) VALUES ('d1', E'Line1\nLine2\nLine3'); +INSERT INTO docs (id, body) VALUES ('d2', E'Alpha\nBeta'); + +-- Now enable block algo on the column that already has data +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc \gset + +-- Test 1: Blocks table should have 5 entries (3 for d1, 2 for d2) immediately after set_column +SELECT count(*) AS block_count FROM docs_cloudsync_blocks \gset +SELECT (:block_count::int = 5) AS block_count_ok \gset +\if :block_count_ok +\echo [PASS] (:testid) Migration: 5 block entries after set_column on existing data +\else +\echo [FAIL] (:testid) Migration: expected 5 block entries, got :block_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 2: Metadata should have 5 alive block entries +SELECT count(*) AS meta_count FROM docs_cloudsync +WHERE col_name LIKE 'body' || chr(31) || '%' AND col_version % 2 = 1 \gset +SELECT (:meta_count::int = 5) AS meta_count_ok \gset +\if :meta_count_ok +\echo [PASS] (:testid) Migration: 5 alive block metadata entries +\else +\echo [FAIL] (:testid) Migration: expected 5 alive metadata entries, got :meta_count +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 3: Calling set_column again should be idempotent (count stays at 5) +SELECT cloudsync_set_column('docs', 'body', 'algo', 'block') AS _sc2 \gset + +SELECT count(*) AS block_count2 FROM docs_cloudsync_blocks \gset +SELECT (:block_count2::int = 5) AS idempotent_ok \gset +\if :idempotent_ok +\echo [PASS] (:testid) Migration: idempotent (still 5 blocks after second set_column) +\else +\echo [FAIL] (:testid) Migration: idempotency broken, got :block_count2 blocks (expected 5) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 4: UPDATE on d1 should still work correctly after migration +UPDATE docs SET body = E'Line1\nLine2-edited\nLine3' WHERE id = 'd1'; + +SELECT count(*) AS block_count3 FROM docs_cloudsync_blocks \gset +SELECT (:block_count3::int = 5) AS update_count_ok \gset +\if :update_count_ok +\echo [PASS] (:testid) Migration: 5 blocks after UPDATE (d1 edited in-place) +\else +\echo [FAIL] (:testid) Migration: expected 5 blocks after update, got :block_count3 +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Test 5: Materialized value should reflect the update +SELECT cloudsync_text_materialize('docs', 'body', 'd1') AS _mat \gset + +SELECT (body = E'Line1\nLine2-edited\nLine3') AS mat_ok FROM docs WHERE id = 'd1' \gset +\if :mat_ok +\echo [PASS] (:testid) Migration: materialized value correct after update +\else +\echo [FAIL] (:testid) Migration: materialized value mismatch +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_block_existing_a; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/51_stale_table_settings_dropped_meta.sql b/test/postgresql/51_stale_table_settings_dropped_meta.sql new file mode 100644 index 0000000..6d4880a --- /dev/null +++ b/test/postgresql/51_stale_table_settings_dropped_meta.sql @@ -0,0 +1,123 @@ +-- 'Stale cloudsync_table_settings with dropped meta-table' +-- Mirrors the SQLite unit test: Stale Table Settings Dropped Meta +-- +-- When a user drops a tracked table and its
_cloudsync meta-table +-- manually (without calling cloudsync_cleanup), cloudsync_table_settings is +-- left with stale rows while pg_tables no longer has any matching +-- *_cloudsync table. Before the fix in cloudsync_dbversion_rebuild, opening a +-- new backend and calling any cloudsync function caused +-- cloudsync_dbversion_build_query to produce a NULL SQL string (string_agg +-- over zero rows), which was misreported as DBRES_NOMEM, making +-- cloudsync_context_init fail and every cloudsync_* call ereport ERROR. + +\set testid '51' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql + +DROP DATABASE IF EXISTS cloudsync_test_51; +CREATE DATABASE cloudsync_test_51; + +\connect cloudsync_test_51 +\ir helper_psql_conn_setup.sql + +CREATE EXTENSION IF NOT EXISTS cloudsync; + +-- Phase 1: create a tracked table and initialize cloudsync on it. +DROP TABLE IF EXISTS stale_doc CASCADE; +CREATE TABLE stale_doc (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('stale_doc', 'CLS', 1) AS _init \gset + +-- Sanity: the meta-table exists and cloudsync_table_settings has a row for it. +SELECT count(*) AS meta_exists FROM pg_tables WHERE tablename = 'stale_doc_cloudsync' \gset +SELECT (:meta_exists::int = 1) AS meta_exists_ok \gset +\if :meta_exists_ok +\echo [PASS] (:testid) stale_doc_cloudsync meta-table created +\else +\echo [FAIL] (:testid) expected stale_doc_cloudsync to exist +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT count(*) AS settings_rows FROM cloudsync_table_settings WHERE tbl_name = 'stale_doc' \gset +SELECT (:settings_rows::int > 0) AS settings_rows_ok \gset +\if :settings_rows_ok +\echo [PASS] (:testid) cloudsync_table_settings has row(s) for stale_doc +\else +\echo [FAIL] (:testid) expected cloudsync_table_settings row for stale_doc +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Phase 2: drop BOTH the base table and the meta-table without calling +-- cloudsync_cleanup. cloudsync_table_settings still references stale_doc, +-- but pg_tables has no *_cloudsync tables at all now. +DROP TABLE stale_doc; +DROP TABLE stale_doc_cloudsync; + +SELECT count(*) AS cloudsync_meta_tables FROM pg_tables WHERE tablename LIKE '%_cloudsync' \gset +SELECT (:cloudsync_meta_tables::int = 0) AS no_meta_ok \gset +\if :no_meta_ok +\echo [PASS] (:testid) no *_cloudsync meta-tables remain in pg_tables +\else +\echo [FAIL] (:testid) expected zero *_cloudsync tables, got :cloudsync_meta_tables +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Phase 3: reconnect to force a fresh backend. pg_cloudsync_context is a +-- static per-process pointer, so a new backend means +-- cloudsync_pg_context_init runs again on the next cloudsync call — which +-- is exactly what used to fail under this bug. +\connect cloudsync_test_51 +\ir helper_psql_conn_setup.sql + +-- cloudsync_version is a pure function that does not touch the context, so +-- this call cannot fail even with the bug present. It's here only as a +-- trivial smoke check that the extension is still loadable. +SELECT cloudsync_version() IS NOT NULL AS version_ok \gset +\if :version_ok +\echo [PASS] (:testid) cloudsync_version() reachable after reopen +\else +\echo [FAIL] (:testid) cloudsync_version() failed after reopen +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- The real check: calling a function that goes through get_cloudsync_context() +-- must succeed. Before the fix, cloudsync_dbversion_rebuild returned +-- DBRES_NOMEM here because SQL_DBVERSION_BUILD_QUERY's string_agg over zero +-- rows produced a NULL SQL string, and the whole init path would ereport +-- ERROR — any cloudsync_* call below would abort the script. +CREATE TABLE stale_doc2 (id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('stale_doc2', 'CLS', 1) AS _init2 \gset + +-- The new table's meta-table exists. If cloudsync_init failed (pre-fix +-- behavior) this count will be 0, covering both the init-rc check and the +-- meta-table creation in a single assertion. +SELECT count(*) AS meta2_exists FROM pg_tables WHERE tablename = 'stale_doc2_cloudsync' \gset +SELECT (:meta2_exists::int = 1) AS meta2_exists_ok \gset +\if :meta2_exists_ok +\echo [PASS] (:testid) cloudsync_init succeeded and stale_doc2_cloudsync was created +\else +\echo [FAIL] (:testid) expected stale_doc2_cloudsync to exist after cloudsync_init +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- An insert via the new cloudsync_init'd table should produce a cloudsync +-- metadata entry — confirming the context is fully functional. +INSERT INTO stale_doc2 (id, body) VALUES ('a', 'hello'); + +SELECT count(*) AS meta_rows FROM stale_doc2_cloudsync \gset +SELECT (:meta_rows::int > 0) AS meta_rows_ok \gset +\if :meta_rows_ok +\echo [PASS] (:testid) stale_doc2_cloudsync has metadata after insert +\else +\echo [FAIL] (:testid) expected metadata rows in stale_doc2_cloudsync +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Cleanup +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_51; +\else +\echo [INFO] !!!!! +\endif diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql new file mode 100644 index 0000000..9ff000a --- /dev/null +++ b/test/postgresql/full_test.sql @@ -0,0 +1,74 @@ +-- usage: +-- - normal: `psql postgresql://postgres:postgres@localhost:5432/cloudsync_test -f test/postgresql/smoke_test.sql` +-- - debug: `psql -v DEBUG=1 postgresql://postgres:postgres@localhost:5432/cloudsync_test -f test/postgresql/smoke_test.sql` + +\echo 'Running smoke_test...' + +\ir helper_psql_conn_setup.sql +-- \set ON_ERROR_STOP on +\set fail 0 + +\ir 01_unittest.sql +\ir 02_roundtrip.sql +\ir 03_multiple_roundtrip.sql +\ir 04_colversion_skew.sql +\ir 05_delete_recreate_cycle.sql +\ir 06_out_of_order_delivery.sql +\ir 07_delete_vs_update.sql +\ir 08_resurrect_delayed_delete.sql +\ir 09_multicol_concurrent_edits.sql +\ir 10_empty_payload_noop.sql +\ir 11_multi_table_multi_columns_rounds.sql +\ir 12_repeated_table_multi_schemas.sql +\ir 13_per_table_schema_tracking.sql +\ir 14_datatype_roundtrip.sql +\ir 15_datatype_roundtrip_unmapped.sql +\ir 16_composite_pk_text_int_roundtrip.sql +\ir 17_uuid_pk_roundtrip.sql +\ir 18_bulk_insert_performance.sql +\ir 19_uuid_pk_with_unmapped_cols.sql +\ir 20_init_with_existing_data.sql +\ir 21_null_value_sync.sql +\ir 22_null_column_roundtrip.sql +\ir 23_uuid_column_roundtrip.sql +\ir 24_nullable_types_roundtrip.sql +\ir 25_boolean_type_issue.sql +\ir 26_row_filter.sql +\ir 27_rls_batch_merge.sql +\ir 28_db_version_tracking.sql +\ir 29_rls_multicol.sql +\ir 30_null_prikey_insert.sql +\ir 31_alter_table_sync.sql +\ir 32_block_lww.sql +\ir 33_block_lww_extended.sql +\ir 34_block_lww_advanced.sql +\ir 35_block_lww_edge_cases.sql +\ir 36_block_lww_round3.sql +\ir 37_block_lww_round4.sql +\ir 38_block_lww_round5.sql +\ir 39_concurrent_write_apply.sql +\ir 40_unsupported_algorithms.sql +\ir 41_corrupted_payload.sql +\ir 42_payload_idempotency.sql +\ir 43_delete_resurrect_ordering.sql +\ir 44_large_composite_pk.sql +\ir 45_pg_specific_types.sql +\ir 46_schema_hash_mismatch.sql +\ir 47_row_filter_advanced.sql +\ir 48_row_filter_multi_table.sql +\ir 49_row_filter_prefill.sql +\ir 50_block_lww_existing_data.sql +\ir 51_stale_table_settings_dropped_meta.sql + +-- 'Test summary' +\echo '\nTest summary:' +\echo - Failures: :fail +SELECT (:fail::int > 0) AS fail_any \gset +\if :fail_any +\echo smoke test failed: :fail test(s) failed +DO $$ BEGIN + RAISE EXCEPTION 'smoke test failed'; +END $$; +\else +\echo - Status: OK +\endif diff --git a/test/postgresql/helper_psql_conn_setup.sql b/test/postgresql/helper_psql_conn_setup.sql new file mode 100644 index 0000000..3205be4 --- /dev/null +++ b/test/postgresql/helper_psql_conn_setup.sql @@ -0,0 +1,11 @@ +\if :{?DEBUG} +\set QUIET 0 +SET client_min_messages = debug1; SET log_min_messages = debug1; SET log_error_verbosity = verbose; +\pset tuples_only off +\pset format aligned +\else +\set QUIET 1 +SET client_min_messages = warning; SET log_min_messages = warning; +\pset tuples_only on +\pset format unaligned +\endif diff --git a/test/postgresql/helper_test_cleanup.sql b/test/postgresql/helper_test_cleanup.sql new file mode 100644 index 0000000..f7f7358 --- /dev/null +++ b/test/postgresql/helper_test_cleanup.sql @@ -0,0 +1,24 @@ +-- Test Cleanup Helper +-- This script should be included at the end of each test file +-- It determines if cleanup should happen based on DEBUG mode and test failures +-- Usage: +-- \ir helper_test_cleanup.sql +-- \if :should_cleanup +-- DROP DATABASE IF EXISTS your_test_db; +-- \endif + +\connect postgres + +-- Determine if we should cleanup +\if :{?DEBUG} +\set should_cleanup false +\echo [INFO] (:testid) DEBUG mode enabled - keeping test databases for inspection +\else +SELECT (:fail::int > :initial_fail::int) AS has_test_failures \gset +\if :has_test_failures +\set should_cleanup false +\echo [INFO] (:testid) Test failures detected - keeping test databases for inspection +\else +\set should_cleanup true +\endif +\endif diff --git a/test/postgresql/helper_test_init.sql b/test/postgresql/helper_test_init.sql new file mode 100644 index 0000000..e7b2b4d --- /dev/null +++ b/test/postgresql/helper_test_init.sql @@ -0,0 +1,13 @@ +-- Test Initialization Helper +-- This script should be included at the beginning of each test file after setting testid +-- Usage: \ir helper_test_init.sql + +-- Initialize fail counter if not already set +\if :{?fail} +-- fail counter already exists from previous tests +\else +\set fail 0 +\endif + +-- Store initial fail count to detect failures in this test +SELECT :fail::int AS initial_fail \gset diff --git a/test/sync_bench.c b/test/sync_bench.c new file mode 100644 index 0000000..946fffa --- /dev/null +++ b/test/sync_bench.c @@ -0,0 +1,653 @@ +// +// sync_bench.c +// cloudsync +// +// Measures end-to-end sync latency from one local SQLite database to another. +// + +#include +#include +#include +#include +#ifdef CLOUDSYNC_NETWORK_TRACE +#include +#endif +#include "sqlite3.h" +#include "utils.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +#define DB_A_PATH "dist/sync-bench-a.sqlite" +#define DB_B_PATH "dist/sync-bench-b.sqlite" +#define EXT_PATH "./dist/cloudsync" +#define DEFAULT_POLL_DELAY_MS 250 +#define DEFAULT_MAX_POLLS 40 +#define DEFAULT_RANDOM_BLOB_SIZE_BYTES (100 * 1024) +#define DEFAULT_CLEANUP_OLDER_THAN_SECONDS (24 * 60 * 60) + +typedef struct { + const char *operation; + int attempt; + int sqlite_rc; + int rows_received; + double started_ms; + double ended_ms; + double elapsed_ms; + char *result_json; +} sync_bench_request; + +typedef struct { + int local_version; + int server_version; + char *status; +} sync_bench_send_summary; + +#ifdef CLOUDSYNC_NETWORK_TRACE +static void bench_trace(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + fprintf(stderr, "[sync-bench] "); + vfprintf(stderr, fmt, args); + fprintf(stderr, "\n"); + va_end(args); +} +#else +#define bench_trace(...) ((void)0) +#endif + +static double monotonic_ms(void) { +#ifdef _WIN32 + LARGE_INTEGER freq; + LARGE_INTEGER counter; + QueryPerformanceFrequency(&freq); + QueryPerformanceCounter(&counter); + return ((double)counter.QuadPart * 1000.0) / (double)freq.QuadPart; +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((double)ts.tv_sec * 1000.0) + ((double)ts.tv_nsec / 1000000.0); +#endif +} + +static char *str_dup(const char *value) { + if (!value) value = ""; + size_t len = strlen(value); + char *copy = (char *)malloc(len + 1); + if (!copy) return NULL; + memcpy(copy, value, len + 1); + return copy; +} + +static int env_int(const char *name, int default_value) { + const char *value = getenv(name); + if (!value || !*value) return default_value; + char *end = NULL; + long parsed = strtol(value, &end, 10); + if (!end || *end != '\0' || parsed < 0 || parsed > 1000000) return default_value; + return (int)parsed; +} + +static int db_exec(sqlite3 *db, const char *sql) { + char *errmsg = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while executing %s: %s\n", sql, errmsg ? errmsg : sqlite3_errmsg(db)); + sqlite3_free(errmsg); + } + return rc; +} + +static int query_text(sqlite3 *db, const char *sql, char **out) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing %s: %s\n", sql, sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + const unsigned char *value = sqlite3_column_text(stmt, 0); + *out = str_dup((const char *)value); + if (!*out) rc = SQLITE_NOMEM; + else rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *out = NULL; + rc = SQLITE_OK; + } else { + fprintf(stderr, "Error while stepping %s: %s\n", sql, sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + return rc; +} + +static int timed_query_text(sqlite3 *db, const char *sql, char **out, double *started_ms, double *ended_ms) { + *started_ms = monotonic_ms(); + int rc = query_text(db, sql, out); + *ended_ms = monotonic_ms(); + return rc; +} + +static int open_load_ext(const char *db_path, sqlite3 **out_db) { + bench_trace("step=open-load-extension db_path=%s begin", db_path); + sqlite3 *db = NULL; + int rc = sqlite3_open(db_path, &db); + if (rc != SQLITE_OK) { + fprintf(stderr, "Unable to open %s: %s\n", db_path, db ? sqlite3_errmsg(db) : "unknown error"); + if (db) sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + rc = sqlite3_enable_load_extension(db, 1); + if (rc != SQLITE_OK) { + fprintf(stderr, "Unable to enable load_extension for %s: %s\n", db_path, sqlite3_errmsg(db)); + sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + rc = db_exec(db, "SELECT load_extension('" EXT_PATH "');"); + if (rc != SQLITE_OK) { + sqlite3_close(db); + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, rc); + return rc; + } + + *out_db = db; + bench_trace("step=open-load-extension db_path=%s end rc=%d", db_path, SQLITE_OK); + return SQLITE_OK; +} + +static int init_schema(sqlite3 *db, const char *label) { + bench_trace("step=init-schema db=%s begin", label); + int rc = db_exec(db, + "CREATE TABLE IF NOT EXISTS sync_bench_items (" + "id TEXT PRIMARY KEY NOT NULL," + "payload TEXT NOT NULL DEFAULT ''," + "marker TEXT NOT NULL DEFAULT ''," + "random_blob BLOB NOT NULL DEFAULT X''," + "updated_at TEXT NOT NULL DEFAULT ''" + ");"); + if (rc != SQLITE_OK) { + bench_trace("step=init-schema db=%s end rc=%d", label, rc); + return rc; + } + + rc = db_exec(db, "SELECT cloudsync_init('sync_bench_items');"); + bench_trace("step=init-schema db=%s end rc=%d", label, rc); + return rc; +} + +static int init_network(sqlite3 *db, const char *label, const char *database_id, const char *address, const char *apikey) { + char sql[2048]; + if (address && *address) { + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_init_custom('%s', '%s');", address, database_id); + } else { + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_init('%s');", database_id); + } + bench_trace("step=network-init db=%s begin mode=%s", label, (address && *address) ? "custom-address" : "default-address"); + int rc = db_exec(db, sql); + bench_trace("step=network-init db=%s end rc=%d", label, rc); + if (rc != SQLITE_OK) return rc; + + if (apikey && *apikey) { + bench_trace("step=set-apikey db=%s begin", label); + snprintf(sql, sizeof(sql), "SELECT cloudsync_network_set_apikey('%s');", apikey); + rc = db_exec(db, sql); + bench_trace("step=set-apikey db=%s end rc=%d", label, rc); + if (rc != SQLITE_OK) return rc; + } + + bench_trace("step=pre-measure-sync db=%s begin sql=cloudsync_network_sync(500,4)", label); + rc = db_exec(db, "SELECT cloudsync_network_sync(500, 4);"); + bench_trace("step=pre-measure-sync db=%s end rc=%d", label, rc); + return rc; +} + +static int setup_database(const char *label, const char *path, const char *database_id, const char *address, const char *apikey, sqlite3 **out_db) { + bench_trace("step=setup-database db=%s path=%s begin", label, path); + int rc = open_load_ext(path, out_db); + if (rc != SQLITE_OK) { + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; + } + + rc = init_schema(*out_db, label); + if (rc != SQLITE_OK) { + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; + } + + rc = init_network(*out_db, label, database_id, address, apikey); + bench_trace("step=setup-database db=%s end rc=%d", label, rc); + return rc; +} + +static int verify_row(sqlite3 *db, const char *id, const char *payload, const char *marker, + const void *random_blob, int random_blob_size, bool *verified) { + sqlite3_stmt *stmt = NULL; + const char *sql = "SELECT payload, marker, random_blob FROM sync_bench_items WHERE id = ?;"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing verification query: %s\n", sqlite3_errmsg(db)); + return rc; + } + + sqlite3_bind_text(stmt, 1, id, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + const char *actual_payload = (const char *)sqlite3_column_text(stmt, 0); + const char *actual_marker = (const char *)sqlite3_column_text(stmt, 1); + const void *actual_blob = sqlite3_column_blob(stmt, 2); + int actual_blob_size = sqlite3_column_bytes(stmt, 2); + bool blob_matches = (actual_blob_size == random_blob_size) && + (random_blob_size == 0 || + (actual_blob && random_blob && memcmp(actual_blob, random_blob, (size_t)random_blob_size) == 0)); + *verified = actual_payload && actual_marker && + strcmp(actual_payload, payload) == 0 && + strcmp(actual_marker, marker) == 0 && + blob_matches; + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + *verified = false; + rc = SQLITE_OK; + } else { + fprintf(stderr, "Error while verifying row: %s\n", sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + return rc; +} + +static int insert_benchmark_row(sqlite3 *db, const char *id, const char *payload, const char *marker, + const void *random_blob, int random_blob_size) { + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d begin", id, random_blob_size); + sqlite3_stmt *stmt = NULL; + const char *sql = "INSERT INTO sync_bench_items (id, payload, marker, random_blob, updated_at) VALUES (?, ?, ?, ?, datetime('now'));"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing insert: %s\n", sqlite3_errmsg(db)); + bench_trace("step=insert-source-row db=db_a row_id=%s end rc=%d", id, rc); + return rc; + } + + rc = sqlite3_bind_text(stmt, 1, id, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_text(stmt, 2, payload, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_text(stmt, 3, marker, -1, SQLITE_TRANSIENT); + if (rc == SQLITE_OK) rc = sqlite3_bind_blob(stmt, 4, random_blob, random_blob_size, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while binding benchmark row: %s\n", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d end rc=%d", id, random_blob_size, rc); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) rc = SQLITE_OK; + else fprintf(stderr, "Error while inserting benchmark row: %s\n", sqlite3_errmsg(db)); + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + bench_trace("step=insert-source-row db=db_a row_id=%s random_blob_size_bytes=%d end rc=%d", id, random_blob_size, rc); + return rc; +} + +static int cleanup_old_benchmark_rows(sqlite3 *db, int older_than_seconds, int *deleted_count) { + if (deleted_count) *deleted_count = 0; + if (older_than_seconds <= 0) { + bench_trace("step=cleanup-old-source-rows db=db_a enabled=false"); + return SQLITE_OK; + } + + char modifier[64]; + snprintf(modifier, sizeof(modifier), "-%d seconds", older_than_seconds); + + bench_trace("step=cleanup-old-source-rows db=db_a older_than_seconds=%d begin", older_than_seconds); + sqlite3_stmt *stmt = NULL; + const char *sql = + "DELETE FROM sync_bench_items " + "WHERE marker LIKE 'sync-bench-%' " + "AND updated_at < datetime('now', ?);"; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while preparing cleanup delete: %s\n", sqlite3_errmsg(db)); + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=0", rc); + return rc; + } + + rc = sqlite3_bind_text(stmt, 1, modifier, -1, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) { + fprintf(stderr, "Error while binding cleanup delete: %s\n", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=0", rc); + return rc; + } + + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + if (deleted_count) *deleted_count = sqlite3_changes(db); + } else { + fprintf(stderr, "Error while deleting old benchmark rows: %s\n", sqlite3_errmsg(db)); + } + + int finalize_rc = sqlite3_finalize(stmt); + if (rc == SQLITE_OK && finalize_rc != SQLITE_OK) rc = finalize_rc; + bench_trace("step=cleanup-old-source-rows db=db_a end rc=%d deleted=%d", rc, deleted_count ? *deleted_count : 0); + return rc; +} + +static int json_int_at_path(sqlite3 *db, const char *json, const char *path, int default_value) { + sqlite3_stmt *stmt = NULL; + int value = default_value; + int rc = sqlite3_prepare_v2(db, "SELECT json_extract(?, ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) return default_value; + sqlite3_bind_text(stmt, 1, json ? json : "", -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, path, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL) value = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + return value; +} + +static char *json_text_at_path(sqlite3 *db, const char *json, const char *path) { + sqlite3_stmt *stmt = NULL; + char *value = NULL; + int rc = sqlite3_prepare_v2(db, "SELECT json_extract(?, ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) return NULL; + sqlite3_bind_text(stmt, 1, json ? json : "", -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, path, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL) { + value = str_dup((const char *)sqlite3_column_text(stmt, 0)); + } + sqlite3_finalize(stmt); + return value; +} + +static int timed_request(sqlite3 *db, sync_bench_request *request, const char *operation, int attempt, const char *sql) { + request->operation = operation; + request->attempt = attempt; + request->rows_received = -1; + request->result_json = NULL; + request->sqlite_rc = timed_query_text(db, sql, &request->result_json, &request->started_ms, &request->ended_ms); + request->elapsed_ms = request->ended_ms - request->started_ms; + if (strcmp(operation, "check") == 0 && request->result_json) { + request->rows_received = json_int_at_path(db, request->result_json, "$.receive.rows", -1); + } + return request->sqlite_rc; +} + +static void json_print_escaped(const char *value) { + putchar('"'); + if (value) { + for (const unsigned char *p = (const unsigned char *)value; *p; p++) { + switch (*p) { + case '\\': printf("\\\\"); break; + case '"': printf("\\\""); break; + case '\n': printf("\\n"); break; + case '\r': printf("\\r"); break; + case '\t': printf("\\t"); break; + default: + if (*p < 0x20) printf("\\u%04x", *p); + else putchar(*p); + } + } + } + putchar('"'); +} + +static void print_text_report(const char *database_id, int poll_delay_ms, int max_polls, int random_blob_size, + int cleanup_older_than_seconds, int cleanup_deleted_rows, + const char *row_id, bool applied, + int polls, double total_ms, double verify_ms, double request_ms, double poll_sleep_ms, + double measured_overhead_ms, sync_bench_send_summary send_summary, + sync_bench_request *requests, int request_count) { + printf("\nSync Performance Benchmark\n"); + printf("database_id: %s\n", database_id); + printf("poll_delay_ms: %d\n", poll_delay_ms); + printf("max_polls: %d\n", max_polls); + printf("random_blob_size_bytes: %d\n", random_blob_size); + printf("cleanup_older_than_seconds: %d\n", cleanup_older_than_seconds); + printf("cleanup_deleted_rows: %d\n", cleanup_deleted_rows); + printf("row_id: %s\n", row_id); + printf("\nRequests:\n"); + for (int i = 0; i < request_count; i++) { + if (strcmp(requests[i].operation, "check") == 0) { + printf("%s[%d] %.2f ms rc=%d rows=%d\n", requests[i].operation, requests[i].attempt, + requests[i].elapsed_ms, requests[i].sqlite_rc, requests[i].rows_received); + } else { + printf("%s[%d] %.2f ms rc=%d status=%s localVersion=%d serverVersion=%d\n", + requests[i].operation, requests[i].attempt, requests[i].elapsed_ms, requests[i].sqlite_rc, + send_summary.status ? send_summary.status : "unknown", + send_summary.local_version, send_summary.server_version); + } + } + printf("\nResult:\n"); + printf("applied: %s\n", applied ? "true" : "false"); + printf("polls: %d\n", polls); + printf("total_send_to_apply_check_end_ms: %.2f\n", total_ms); + printf("network_request_elapsed_ms: %.2f\n", request_ms); + printf("poll_sleep_elapsed_ms: %.2f\n", poll_sleep_ms); + printf("local_overhead_elapsed_ms: %.2f\n", measured_overhead_ms); + printf("verification_select_ms: %.2f\n", verify_ms); +} + +static void print_json_report(int poll_delay_ms, int max_polls, int random_blob_size, + int cleanup_older_than_seconds, int cleanup_deleted_rows, + const char *row_id, bool applied, + int polls, double total_ms, double verify_ms, double request_ms, double poll_sleep_ms, + double measured_overhead_ms, sync_bench_send_summary send_summary, + sync_bench_request *requests, int request_count) { + printf("{\n"); + printf(" \"applied\": %s,\n", applied ? "true" : "false"); + printf(" \"pollDelayMs\": %d,\n", poll_delay_ms); + printf(" \"maxPolls\": %d,\n", max_polls); + printf(" \"randomBlobSizeBytes\": %d,\n", random_blob_size); + printf(" \"cleanupOlderThanSeconds\": %d,\n", cleanup_older_than_seconds); + printf(" \"cleanupDeletedRows\": %d,\n", cleanup_deleted_rows); + printf(" \"polls\": %d,\n", polls); + printf(" \"rowId\": "); json_print_escaped(row_id); printf(",\n"); + printf(" \"totalSendToApplyCheckEndMs\": %.2f,\n", total_ms); + printf(" \"networkRequestElapsedMs\": %.2f,\n", request_ms); + printf(" \"pollSleepElapsedMs\": %.2f,\n", poll_sleep_ms); + printf(" \"localOverheadElapsedMs\": %.2f,\n", measured_overhead_ms); + printf(" \"verificationSelectMs\": %.2f,\n", verify_ms); + printf(" \"send\": {\"status\": "); json_print_escaped(send_summary.status); + printf(", \"localVersion\": %d, \"serverVersion\": %d},\n", send_summary.local_version, send_summary.server_version); + printf(" \"requests\": [\n"); + for (int i = 0; i < request_count; i++) { + printf(" {\"operation\": "); json_print_escaped(requests[i].operation); + printf(", \"attempt\": %d, \"sqliteRc\": %d, \"elapsedMs\": %.2f", requests[i].attempt, + requests[i].sqlite_rc, requests[i].elapsed_ms); + if (strcmp(requests[i].operation, "check") == 0) printf(", \"rows\": %d", requests[i].rows_received); + printf(", \"result\": "); json_print_escaped(requests[i].result_json); + printf("}%s\n", i + 1 == request_count ? "" : ","); + } + printf(" ]\n"); + printf("}\n"); +} + +static void free_requests(sync_bench_request *requests, int request_count) { + for (int i = 0; i < request_count; i++) free(requests[i].result_json); +} + +int main(void) { + int rc = SQLITE_OK; + sqlite3 *db_a = NULL; + sqlite3 *db_b = NULL; + sync_bench_request *requests = NULL; + int request_count = 0; + bool applied = false; + int polls = 0; + double total_ms = 0.0; + double verify_ms = 0.0; + double poll_sleep_ms = 0.0; + double request_ms = 0.0; + double measured_overhead_ms = 0.0; + sync_bench_send_summary send_summary = {-1, -1, NULL}; + int cleanup_deleted_rows = 0; + char row_id[UUID_STR_MAXLEN] = ""; + char marker[96] = ""; + char payload[128] = ""; + unsigned char empty_blob = 0; + void *random_blob = NULL; + + const char *database_id = getenv("SYNC_BENCH_DATABASE_ID"); + const char *address = getenv("SYNC_BENCH_CLOUDSYNC_ADDRESS"); + const char *apikey = getenv("SYNC_BENCH_APIKEY"); + const char *output = getenv("SYNC_BENCH_OUTPUT"); + int poll_delay_ms = env_int("SYNC_BENCH_POLL_DELAY_MS", DEFAULT_POLL_DELAY_MS); + int max_polls = env_int("SYNC_BENCH_MAX_POLLS", DEFAULT_MAX_POLLS); + int random_blob_size = env_int("SYNC_BENCH_RANDOM_BLOB_SIZE_BYTES", DEFAULT_RANDOM_BLOB_SIZE_BYTES); + int cleanup_older_than_seconds = env_int("SYNC_BENCH_CLEANUP_OLDER_THAN_SECONDS", DEFAULT_CLEANUP_OLDER_THAN_SECONDS); + + if (!database_id || !*database_id) { + fprintf(stderr, "Error: SYNC_BENCH_DATABASE_ID not set.\n"); + return SQLITE_MISUSE; + } + + requests = (sync_bench_request *)calloc((size_t)max_polls + 1, sizeof(sync_bench_request)); + if (!requests) return SQLITE_NOMEM; + + remove(DB_A_PATH); + remove(DB_B_PATH); + cloudsync_memory_init(1); + + bench_trace("step=benchmark-setup begin database_id=%s poll_delay_ms=%d max_polls=%d", database_id, poll_delay_ms, max_polls); + rc = setup_database("db_a", DB_A_PATH, database_id, address, apikey, &db_a); + if (rc != SQLITE_OK) goto cleanup; + rc = setup_database("db_b", DB_B_PATH, database_id, address, apikey, &db_b); + if (rc != SQLITE_OK) goto cleanup; + bench_trace("step=benchmark-setup end rc=%d", rc); + + rc = cleanup_old_benchmark_rows(db_a, cleanup_older_than_seconds, &cleanup_deleted_rows); + if (rc != SQLITE_OK) goto cleanup; + if (cleanup_deleted_rows > 0) { + bench_trace("step=cleanup-send db=db_a deleted=%d begin sql=cloudsync_network_send_changes", cleanup_deleted_rows); + rc = db_exec(db_a, "SELECT cloudsync_network_send_changes();"); + bench_trace("step=cleanup-send db=db_a deleted=%d end rc=%d", cleanup_deleted_rows, rc); + if (rc != SQLITE_OK) goto cleanup; + } + + cloudsync_uuid_v7_string(row_id, true); + snprintf(marker, sizeof(marker), "sync-bench-%s", row_id); + snprintf(payload, sizeof(payload), "payload-%s", row_id); + + if (random_blob_size > 0) { + random_blob = malloc((size_t)random_blob_size); + if (!random_blob) { + rc = SQLITE_NOMEM; + goto cleanup; + } + sqlite3_randomness(random_blob_size, random_blob); + } else { + random_blob = &empty_blob; + } + + rc = insert_benchmark_row(db_a, row_id, payload, marker, random_blob, random_blob_size); + if (rc != SQLITE_OK) goto cleanup; + + bench_trace("step=verify-before-send db=db_b row_id=%s begin", row_id); + rc = verify_row(db_b, row_id, payload, marker, random_blob, random_blob_size, &applied); + bench_trace("step=verify-before-send db=db_b row_id=%s end rc=%d applied=%s", row_id, rc, applied ? "true" : "false"); + if (rc != SQLITE_OK) goto cleanup; + if (applied) { + fprintf(stderr, "Error: benchmark row already exists in receiver before send.\n"); + rc = SQLITE_ERROR; + goto cleanup; + } + + bench_trace("step=send db=db_a row_id=%s begin sql=cloudsync_network_send_changes", row_id); + rc = timed_request(db_a, &requests[request_count++], "send", 1, "SELECT cloudsync_network_send_changes();"); + bench_trace("step=send db=db_a row_id=%s end rc=%d elapsed_ms=%.2f", row_id, rc, requests[request_count - 1].elapsed_ms); + if (rc != SQLITE_OK) goto cleanup; + send_summary.status = json_text_at_path(db_a, requests[0].result_json, "$.send.status"); + send_summary.local_version = json_int_at_path(db_a, requests[0].result_json, "$.send.localVersion", -1); + send_summary.server_version = json_int_at_path(db_a, requests[0].result_json, "$.send.serverVersion", -1); + double total_start_ms = requests[0].started_ms; + + for (int i = 0; i < max_polls; i++) { + if (i > 0 && poll_delay_ms > 0) { + bench_trace("step=poll-sleep attempt=%d delay_ms=%d begin", i + 1, poll_delay_ms); + double sleep_start_ms = monotonic_ms(); + sqlite3_sleep(poll_delay_ms); + double sleep_elapsed_ms = monotonic_ms() - sleep_start_ms; + poll_sleep_ms += sleep_elapsed_ms; + bench_trace("step=poll-sleep attempt=%d end elapsed_ms=%.2f", i + 1, sleep_elapsed_ms); + } + bench_trace("step=check db=db_b attempt=%d row_id=%s begin sql=cloudsync_network_check_changes", i + 1, row_id); + rc = timed_request(db_b, &requests[request_count++], "check", i + 1, "SELECT cloudsync_network_check_changes();"); + polls = i + 1; + bench_trace("step=check db=db_b attempt=%d row_id=%s end rc=%d rows=%d elapsed_ms=%.2f", i + 1, row_id, rc, requests[request_count - 1].rows_received, requests[request_count - 1].elapsed_ms); + if (rc != SQLITE_OK) goto cleanup; + + bench_trace("step=verify-after-check db=db_b attempt=%d row_id=%s begin", i + 1, row_id); + double verify_start_ms = monotonic_ms(); + rc = verify_row(db_b, row_id, payload, marker, random_blob, random_blob_size, &applied); + double verify_end_ms = monotonic_ms(); + verify_ms = verify_end_ms - verify_start_ms; + bench_trace("step=verify-after-check db=db_b attempt=%d row_id=%s end rc=%d applied=%s elapsed_ms=%.2f", i + 1, row_id, rc, applied ? "true" : "false", verify_ms); + if (rc != SQLITE_OK) goto cleanup; + + if (applied) { + total_ms = requests[request_count - 1].ended_ms - total_start_ms; + break; + } + } + + if (!applied) { + total_ms = request_count > 0 ? requests[request_count - 1].ended_ms - requests[0].started_ms : 0.0; + rc = SQLITE_BUSY; + } + + for (int i = 0; i < request_count; i++) request_ms += requests[i].elapsed_ms; + measured_overhead_ms = total_ms - request_ms - poll_sleep_ms; + if (measured_overhead_ms < 0.0 && measured_overhead_ms > -0.01) measured_overhead_ms = 0.0; + +cleanup: + bench_trace("step=report begin rc=%d applied=%s request_count=%d", rc, applied ? "true" : "false", request_count); + if (output && strcmp(output, "json") == 0) { + print_json_report(poll_delay_ms, max_polls, random_blob_size, cleanup_older_than_seconds, cleanup_deleted_rows, + row_id, applied, polls, total_ms, verify_ms, + request_ms, poll_sleep_ms, measured_overhead_ms, send_summary, + requests, request_count); + } else { + print_text_report(database_id, poll_delay_ms, max_polls, random_blob_size, cleanup_older_than_seconds, cleanup_deleted_rows, + row_id, applied, polls, total_ms, verify_ms, + request_ms, poll_sleep_ms, measured_overhead_ms, send_summary, + requests, request_count); + } + bench_trace("step=report end rc=%d", rc); + + if (!applied && rc == SQLITE_BUSY) { + fprintf(stderr, "Error: row was not applied to receiver after %d polls.\n", max_polls); + } + + if (db_a) { + bench_trace("step=terminate db=db_a begin"); + db_exec(db_a, "SELECT cloudsync_terminate();"); + sqlite3_close(db_a); + bench_trace("step=terminate db=db_a end"); + } + if (db_b) { + bench_trace("step=terminate db=db_b begin"); + db_exec(db_b, "SELECT cloudsync_terminate();"); + sqlite3_close(db_b); + bench_trace("step=terminate db=db_b end"); + } + free_requests(requests, request_count); + free(send_summary.status); + if (random_blob && random_blob != &empty_blob) free(random_blob); + free(requests); + cloudsync_memory_finalize(); + return rc == SQLITE_OK ? 0 : rc; +} diff --git a/test/unit.c b/test/unit.c index f21614f..05e9c95 100644 --- a/test/unit.c +++ b/test/unit.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "sqlite3.h" @@ -21,28 +22,32 @@ #include "pk.h" #include "dbutils.h" +#include "database.h" #include "cloudsync.h" -#include "cloudsync_private.h" +#include "cloudsync_sqlite.h" // declared only if macro CLOUDSYNC_UNITTEST is defined extern char *OUT_OF_MEMORY_BUFFER; extern bool force_vtab_filter_abort; extern bool force_uncompressed_blob; -// private prototypes -sqlite3_stmt *stmt_reset (sqlite3_stmt *stmt); -int stmt_count (sqlite3_stmt *stmt, const char *value, size_t len, int type); -int stmt_execute (sqlite3_stmt *stmt, void *data); +void dbvm_reset (dbvm_t *stmt); +int dbvm_count (dbvm_t *stmt, const char *value, size_t len, int type); +int dbvm_execute (dbvm_t *stmt, void *data); -sqlite3_int64 dbutils_select (sqlite3 *db, const char *sql, const char **values, int types[], int lens[], int count, int expected_type); +int dbutils_settings_get_value (cloudsync_context *data, const char *key, char *buffer, size_t *blen, int64_t *intvalue); int dbutils_settings_table_load_callback (void *xdata, int ncols, char **values, char **names); -int dbutils_settings_check_version (sqlite3 *db, const char *version); -bool dbutils_migrate (sqlite3 *db); -const char *opname_from_value (int value); -int colname_is_legal (const char *name); -int binary_comparison (int x, int y); +int dbutils_settings_check_version (cloudsync_context *data, const char *version); +bool dbutils_settings_migrate (cloudsync_context *data); +const char *vtab_opname_from_value (int value); +int vtab_colname_is_legal (const char *name); +int dbutils_binary_comparison (int x, int y); sqlite3 *do_create_database (void); +int cloudsync_table_sanity_check (cloudsync_context *data, const char *name, CLOUDSYNC_INIT_FLAG init_flags); +bool database_system_exists (cloudsync_context *data, const char *name, const char *type); +int cloudsync_dbversion_rebuild (cloudsync_context *data); + static int stdout_backup = -1; // Backup file descriptor for stdout static int dev_null_fd = -1; // File descriptor for /dev/null static int test_counter = 1; @@ -88,6 +93,136 @@ static const char *query_changes = "QUERY_CHANGES"; // MARK: - +typedef struct { + int type; + int len; + int rc; + union { + sqlite3_int64 intValue; + double doubleValue; + char *stringValue; + } value; +} DATABASE_RESULT; + +DATABASE_RESULT unit_exec (cloudsync_context *data, const char *sql, const char **values, int types[], int lens[], int count, DATABASE_RESULT results[], int expected_types[], int result_count) { + DEBUG_DBFUNCTION("unit_exec %s", sql); + + sqlite3_stmt *pstmt = NULL; + bool is_write = (result_count == 0); + int type = 0; + + // compile sql + int rc = databasevm_prepare(data, sql, (void **)&pstmt, 0); + if (rc != SQLITE_OK) goto unitexec_finalize; + // check bindings + for (int i=0; iin_savepoint == true) { - if (s->is_approved) rc = sqlite3_exec(db, "RELEASE unittest_payload_apply_transaction", NULL, NULL, NULL); - else rc = sqlite3_exec(db, "ROLLBACK TO unittest_payload_apply_transaction; RELEASE unittest_payload_apply_transaction", NULL, NULL, NULL); - if (rc == SQLITE_OK) s->in_savepoint = false; - } - if (create_new) { - rc = sqlite3_exec(db, "SAVEPOINT unittest_payload_apply_transaction", NULL, NULL, NULL); - if (rc == SQLITE_OK) s->in_savepoint = true; - } - return rc; -} - -bool unittest_payload_apply_rls_callback(void **xdata, cloudsync_pk_decode_bind_context *d, sqlite3 *db, cloudsync_context *data, int step, int rc) { - bool is_approved = false; - unittest_payload_apply_rls_status *s; - if (*xdata) { - s = (unittest_payload_apply_rls_status *)*xdata; - } else { - s = cloudsync_memory_zeroalloc(sizeof(unittest_payload_apply_rls_status)); - s->is_approved = true; - *xdata = s; - } - - // extract context info - int64_t colname_len = 0; - char *colname = cloudsync_pk_context_colname(d, &colname_len); - - int64_t tbl_len = 0; - char *tbl = cloudsync_pk_context_tbl(d, &tbl_len); - - int64_t pk_len = 0; - void *pk = cloudsync_pk_context_pk(d, &pk_len); - - int64_t cl = cloudsync_pk_context_cl(d); - int64_t db_version = cloudsync_pk_context_dbversion(d); - - switch (step) { - case CLOUDSYNC_PAYLOAD_APPLY_WILL_APPLY: { - // if the tbl name or the prikey has changed, then verify if the row is valid - // must use strncmp because strings in xdata are not zero-terminated - bool tbl_changed = (s->last_tbl && (strlen(s->last_tbl) != (size_t)tbl_len || strncmp(s->last_tbl, tbl, (size_t)tbl_len) != 0)); - bool pk_changed = (s->last_pk && pk && cloudsync_blob_compare(s->last_pk, s->last_pk_len, pk, pk_len) != 0); - if (s->is_approved - && !s->last_is_delete - && (tbl_changed || pk_changed)) { - s->is_approved = unittest_validate_changed_row(db, data, s->last_tbl, s->last_pk, s->last_pk_len); - } - - s->last_is_delete = ((size_t)colname_len == strlen(CLOUDSYNC_TOMBSTONE_VALUE) && - strncmp(colname, CLOUDSYNC_TOMBSTONE_VALUE, (size_t)colname_len) == 0 - ) && cl % 2 == 0; - - // update the last_tbl value, if needed - if (!s->last_tbl || - !tbl || - (strlen(s->last_tbl) != (size_t)tbl_len) || - strncmp(s->last_tbl, tbl, (size_t)tbl_len) != 0) { - if (s->last_tbl) cloudsync_memory_free(s->last_tbl); - if (tbl && tbl_len > 0) s->last_tbl = cloudsync_string_ndup(tbl, tbl_len, false); - else s->last_tbl = NULL; - } - - // update the last_prikey and len values, if needed - if (!s->last_pk || !pk || cloudsync_blob_compare(s->last_pk, s->last_pk_len, pk, pk_len) != 0) { - if (s->last_pk) cloudsync_memory_free(s->last_pk); - if (pk && pk_len > 0) { - s->last_pk = cloudsync_memory_alloc(pk_len); - memcpy(s->last_pk, pk, pk_len); - s->last_pk_len = pk_len; - } else { - s->last_pk = NULL; - s->last_pk_len = 0; - } - } - - // commit the previous transaction, if any - // begin new transacion, if needed - if (s->last_db_version != db_version) { - rc = unittest_payload_apply_reset_transaction(db, s, true); - if (rc != SQLITE_OK) printf("unittest_payload_apply error in reset_transaction: (%d) %s\n", rc, sqlite3_errmsg(db)); - - // reset local variables - s->last_db_version = db_version; - s->is_approved = true; - } - - is_approved = s->is_approved; - break; - } - case CLOUDSYNC_PAYLOAD_APPLY_DID_APPLY: - is_approved = s->is_approved; - break; - case CLOUDSYNC_PAYLOAD_APPLY_CLEANUP: - if (s->is_approved && !s->last_is_delete) s->is_approved = unittest_validate_changed_row(db, data, s->last_tbl, s->last_pk, s->last_pk_len); - rc = unittest_payload_apply_reset_transaction(db, s, false); - if (s->last_tbl) cloudsync_memory_free(s->last_tbl); - if (s->last_pk) { - cloudsync_memory_free(s->last_pk); - s->last_pk_len = 0; - } - is_approved = s->is_approved; - - cloudsync_memory_free(s); - *xdata = NULL; - break; - } - - return is_approved; -} -#endif - // MARK: - #ifndef CLOUDSYNC_OMIT_PRINT_RESULT @@ -758,7 +731,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { if (print_result) printf("TESTING DELETE on %s\n", table_name); char *sql = sqlite3_mprintf("DELETE FROM \"%w\" WHERE first_name='name5';", table_name); - int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); sqlite3_free(sql); if (rc != SQLITE_OK) goto finalize; @@ -772,7 +745,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { const char *table_name = CUSTOMERS_NOCOLS_TABLE; if (print_result) printf("TESTING DELETE on %s\n", table_name); - int rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100005';", NULL, NULL, NULL); + rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100005';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; rc = sqlite3_exec(db, "DELETE FROM \"" CUSTOMERS_NOCOLS_TABLE "\" WHERE first_name='name100007';", NULL, NULL, NULL); @@ -783,7 +756,7 @@ void do_delete (sqlite3 *db, int table_mask, bool print_result) { const char *table_name = "customers_noprikey"; if (print_result) printf("TESTING DELETE on %s\n", table_name); - int rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200005';", NULL, NULL, NULL); + rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200005';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; rc = sqlite3_exec(db, "DELETE FROM customers_noprikey WHERE first_name='name200007';", NULL, NULL, NULL); @@ -876,7 +849,8 @@ bool do_test_vtab2 (void) { finalize: if (rc != SQLITE_OK) printf("do_test_vtab2 error: %s\n", sqlite3_errmsg(db)); - db = close_db(db); + close_db(db); + db = NULL; return result; } @@ -934,13 +908,13 @@ bool do_test_vtab(sqlite3 *db) { rc = sqlite3_exec(db, "SELECT tbl FROM cloudsync_changes WHERE db_version LIKE 1;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - const char *name = opname_from_value (666); + const char *name = vtab_opname_from_value (666); if (name != NULL) goto finalize; - rc = colname_is_legal("db_version"); + rc = vtab_colname_is_legal("db_version"); if (rc != 1) goto finalize; - rc = colname_is_legal("non_existing_column"); + rc = vtab_colname_is_legal("non_existing_column"); if (rc != 0) goto finalize; return do_test_vtab2(); @@ -951,28 +925,47 @@ bool do_test_vtab(sqlite3 *db) { } bool do_test_functions (sqlite3 *db, bool print_results) { - int size = 0, rc2; - char *site_id = dbutils_blob_select(db, "SELECT cloudsync_siteid();", &size, NULL, &rc2); - if (site_id == NULL || size != 16) goto abort_test_functions; + char *site_id = NULL; + int64_t len = 0; + cloudsync_context *data = cloudsync_context_create(db); + if (!data) return false; + + int rc = database_select_blob(data, "SELECT cloudsync_siteid();", &site_id, &len); + if (rc != DBRES_OK || site_id == NULL || len != 16) { + if (site_id) cloudsync_memory_free(site_id); + goto abort_test_functions; + } cloudsync_memory_free(site_id); - char *site_id_str = dbutils_text_select(db, "SELECT quote(cloudsync_siteid());"); - if (site_id_str == NULL) goto abort_test_functions; + char *site_id_str = NULL; + rc = database_select_text(data, "SELECT quote(cloudsync_siteid());", &site_id_str); + if (rc != DBRES_OK || site_id_str == NULL) { + if (site_id_str) cloudsync_memory_free(site_id_str); + goto abort_test_functions; + } if (print_results) printf("Site ID: %s\n", site_id_str); cloudsync_memory_free(site_id_str); - char *version = dbutils_text_select(db, "SELECT cloudsync_version();"); - if (version == NULL) goto abort_test_functions; + char *version = NULL; + rc = database_select_text(data, "SELECT cloudsync_version();", &version); + if (rc != DBRES_OK || version == NULL) { + if (version) cloudsync_memory_free(version); + goto abort_test_functions; + } if (print_results) printf("Lib Version: %s\n", version); cloudsync_memory_free(version); - sqlite3_int64 db_version = dbutils_int_select(db, "SELECT cloudsync_db_version();"); - if (print_results) printf("DB Version: %lld\n", db_version); + int64_t db_version = 0; + rc = database_select_int(data, "SELECT cloudsync_db_version();", &db_version); + if (rc != DBRES_OK) goto abort_test_functions; + if (print_results) printf("DB Version: %" PRId64 "\n", db_version); - sqlite3_int64 db_version_next = dbutils_int_select(db, "SELECT cloudsync_db_version_next();"); - if (print_results) printf("DB Version Next: %lld\n", db_version_next); + int64_t db_version_next = 0; + rc = database_select_int(data, "SELECT cloudsync_db_version_next();", &db_version); + if (rc != DBRES_OK) goto abort_test_functions; + if (print_results) printf("DB Version Next: %" PRId64 "\n", db_version_next); - int rc = sqlite3_exec(db, "CREATE TABLE tbl1 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); + rc = sqlite3_exec(db, "CREATE TABLE tbl1 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; rc = sqlite3_exec(db, "CREATE TABLE tbl2 (col1 TEXT PRIMARY KEY NOT NULL, col2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; @@ -980,31 +973,46 @@ bool do_test_functions (sqlite3 *db, bool print_results) { rc = sqlite3_exec(db, "DROP TABLE IF EXISTS rowid_table; DROP TABLE IF EXISTS nonnull_prikey_table;", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_init('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_init('tbl1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto abort_test_functions; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_disable('tbl1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v1 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl1');"); + int64_t value = 0; + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl1');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v1 = (int)value; if (v1 == 1) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_disable('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_disable('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v2 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl2');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl2');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v2 = (int)value; if (v2 == 1) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_enable('tbl1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v3 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl1');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl1');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v3 = (int)value; if (v3 != 1) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_enable('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_enable('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - int v4 = (int)dbutils_int_select(db, "SELECT cloudsync_is_enabled('tbl2');"); + rc = database_select_int(data, "SELECT cloudsync_is_enabled('tbl2');", &value); + if (rc != DBRES_OK) goto abort_test_functions; + int v4 = (int)value; if (v4 != 1) goto abort_test_functions; rc = sqlite3_exec(db, "SELECT cloudsync_set('key1', 'value1');", NULL, NULL, NULL); @@ -1016,17 +1024,26 @@ bool do_test_functions (sqlite3 *db, bool print_results) { rc = sqlite3_exec(db, "SELECT cloudsync_set_column('tbl1', 'col1', 'key1', 'value1');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('*');", NULL, NULL, NULL); + // * disabled in 0.9.0 + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('tbl1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto abort_test_functions; + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('tbl2');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test_functions; - char *uuid = dbutils_text_select(db, "SELECT cloudsync_uuid();"); - if (uuid == NULL) goto abort_test_functions; + char *uuid = NULL; + rc = database_select_text(data, "SELECT cloudsync_uuid();", &uuid); + if (rc != DBRES_OK || uuid == NULL) { + if (uuid) cloudsync_memory_free(uuid); + goto abort_test_functions; + } if (print_results) printf("New uuid: %s\n", uuid); cloudsync_memory_free(uuid); + cloudsync_context_free(data); return true; abort_test_functions: + cloudsync_context_free(data); printf("Error in do_test_functions: %s\n", sqlite3_errmsg(db)); return false; } @@ -1196,18 +1213,18 @@ bool do_augment_tables (int table_mask, sqlite3 *db, table_algo algo) { char sql[512]; if (table_mask & TEST_PRIKEYS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_TABLE, crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_TABLE, cloudsync_algo_name(algo)); int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_augment_tables; } if (table_mask & TEST_NOCOLS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_NOCOLS_TABLE, crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('%q', '%s');", CUSTOMERS_NOCOLS_TABLE, cloudsync_algo_name(algo)); if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_augment_tables; } if (table_mask & TEST_NOPRIKEYS) { - sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('customers_noprikey', '%s');", crdt_algo_name(algo)); + sqlite3_snprintf(sizeof(sql), sql, "SELECT cloudsync_init('customers_noprikey', '%s');", cloudsync_algo_name(algo)); if (sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) goto abort_augment_tables; } @@ -1307,7 +1324,7 @@ bool do_test_pk_single_value (sqlite3 *db, int type, int64_t ivalue, double dval pklist[0].type = type; if (type == SQLITE_INTEGER) { - snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%lld);", ivalue); + snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%" PRId64 ");", ivalue); pklist[0].ivalue = ivalue; } else if (type == SQLITE_FLOAT) { snprintf(sql, sizeof(sql), "SELECT cloudsync_pk_encode(%f);", dvalue); @@ -1357,7 +1374,7 @@ bool do_test_pk_single_value (sqlite3 *db, int type, int64_t ivalue, double dval exit(-666); } if (stmt) sqlite3_finalize(stmt); - dbutils_debug_stmt(db, true); + unit_debug(db, true); return result; } @@ -1414,7 +1431,38 @@ bool do_test_pkbind_callback (sqlite3 *db) { exit(-666); } if (stmt) sqlite3_finalize(stmt); - dbutils_debug_stmt(db, true); + unit_debug(db, true); + return result; +} + +bool do_test_single_pk (bool print_result) { + bool result = false; + + sqlite3 *db = NULL; + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + + // manually load extension + sqlite3_cloudsync_init(db, NULL, NULL); + + rc = sqlite3_exec(db, "CREATE TABLE single_pk_test (col1 INTEGER PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // the following function should fail + rc = sqlite3_exec(db, "SELECT cloudsync_init('single_pk_test');", NULL, NULL, NULL); + if (rc == SQLITE_OK) return false; + + // the following function should succedd + rc = sqlite3_exec(db, "SELECT cloudsync_init('single_pk_test', 'cls', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + result = true; + + // cleanup newly created table + sqlite3_exec(db, "SELECT cloudsync_cleanup('single_pk_test');", NULL, NULL, NULL); + +cleanup: + if (rc != SQLITE_OK && print_result) printf("do_test_single_pk error: %s\n", sqlite3_errmsg(db)); + close_db(db); return result; } @@ -1502,9 +1550,9 @@ bool do_test_pk (sqlite3 *db, int ntest, bool print_result) { // cleanup memory sqlite3_finalize(stmt); stmt = NULL; - for (int i=0; i 0) goto finalize; - sqlite3_int64 db_version = dbutils_int_select(db, "SELECT cloudsync_db_version();"); + int64_t db_version = 0; + database_select_int(data, "SELECT cloudsync_db_version();", &db_version); char *site_id_blob; - int site_id_blob_size; - sqlite3_int64 dbver1, seq1; - rc = dbutils_blob_int_int_select(db, "SELECT cloudsync_siteid(), cloudsync_db_version(), cloudsync_seq();", &site_id_blob, &site_id_blob_size, &dbver1, &seq1); + int64_t site_id_blob_size; + int64_t dbver1; + rc = database_select_blob_int(data, "SELECT cloudsync_siteid(), cloudsync_db_version();", &site_id_blob, &site_id_blob_size, &dbver1); if (rc != SQLITE_OK || site_id_blob == NULL ||dbver1 != db_version) goto finalize; cloudsync_memory_free(site_id_blob); // force out-of-memory test - value1 = dbutils_settings_get_value(db, "key1", OUT_OF_MEMORY_BUFFER, 0); - if (value1 != NULL) goto finalize; + rc = dbutils_settings_get_value(data, "key1", NULL, 0, NULL); + if (rc != SQLITE_MISUSE) goto finalize; - value1 = dbutils_table_settings_get_value(db, "foo", NULL, "key1", OUT_OF_MEMORY_BUFFER, 0); - if (value1 != NULL) goto finalize; + rc = dbutils_table_settings_get_value(data, "foo", NULL, "key1", NULL, 0); + if (rc != DBRES_MISUSE) goto finalize; - char *p = NULL; - dbutils_select(db, "SELECT zeroblob(16);", NULL, NULL, NULL, 0, SQLITE_NOMEM); - if (p != NULL) goto finalize; + //char *p = NULL; + //dbutils_select(data, "SELECT zeroblob(16);", NULL, NULL, NULL, 0, SQLITE_BLOB); + //if (p != NULL) goto finalize; - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, "0.0.0"); - int cmp = dbutils_settings_check_version(db, NULL); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, "0.0.0"); + int cmp = dbutils_settings_check_version(data, NULL); if (cmp == 0) goto finalize; - dbutils_settings_set_key_value(db, NULL, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); - cmp = dbutils_settings_check_version(db, NULL); + dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_LIBVERSION, CLOUDSYNC_VERSION); + cmp = dbutils_settings_check_version(data, NULL); if (cmp != 0) goto finalize; - cmp = dbutils_settings_check_version(db, "0.8.25"); + cmp = dbutils_settings_check_version(data, "0.8.25"); if (cmp <= 0) goto finalize; //dbutils_settings_table_load_callback(NULL, 0, NULL, NULL); - dbutils_migrate(NULL); + dbutils_settings_migrate(NULL); - dbutils_settings_cleanup(db); + dbutils_settings_cleanup(data); int n1 = 1; int n2 = 2; - cmp = binary_comparison(n1, n2); + cmp = dbutils_binary_comparison(n1, n2); if (cmp != -1) goto finalize; - cmp = binary_comparison(n2, n1); + cmp = dbutils_binary_comparison(n2, n1); if (cmp != 1) goto finalize; - cmp = binary_comparison(n1, n1); + cmp = dbutils_binary_comparison(n1, n1); if (cmp != 0) goto finalize; rc = SQLITE_OK; finalize: if (rc != SQLITE_OK) printf("%s\n", sqlite3_errmsg(db)); - db = close_db(db); + close_db(db); + db = NULL; + if (data) cloudsync_context_free(data); return (rc == SQLITE_OK); } @@ -1923,10 +1982,10 @@ bool do_test_others (sqlite3 *db) { // test unfinalized statement just to increase code coverage sqlite3_stmt *stmt = NULL; sqlite3_prepare_v2(db, "SELECT 1;", -1, &stmt, NULL); - int count = dbutils_debug_stmt(db, false); + int count = unit_debug(db, false); sqlite3_finalize(stmt); // to increase code coverage - dbutils_context_result_error(NULL, "Test is: %s", "Hello World"); + // dbutils_set_error(NULL, "Test is: %s", "Hello World"); return (count == 1); } @@ -1936,7 +1995,7 @@ bool do_test_error_cases (sqlite3 *db) { // test cloudsync_init missing table sqlite3_prepare_v2(db, "SELECT cloudsync_init('missing_table');", -1, &stmt, NULL); - int res = stmt_execute(stmt, NULL); + int res = dbvm_execute(stmt, NULL); sqlite3_finalize(stmt); if (res != -1) return false; @@ -1959,6 +2018,43 @@ bool do_test_error_cases (sqlite3 *db) { return true; } +bool do_test_null_prikey_insert (sqlite3 *db) { + // Create a table with a primary key that allows NULL (no NOT NULL constraint) + const char *sql = "CREATE TABLE IF NOT EXISTS t_null_pk (id TEXT PRIMARY KEY, value TEXT);" + "SELECT cloudsync_init('t_null_pk');"; + int rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + // Attempt to insert a row with NULL primary key — should fail + char *errmsg = NULL; + sql = "INSERT INTO t_null_pk (id, value) VALUES (NULL, 'test');"; + rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc == SQLITE_OK) return false; // should have failed + if (!errmsg) return false; + + // Verify the error message matches the expected format + const char *expected = "Insert aborted because primary key in table t_null_pk contains NULL values."; + bool match = (strcmp(errmsg, expected) == 0); + sqlite3_free(errmsg); + if (!match) return false; + + // Verify that a non-NULL primary key insert succeeds + sql = "INSERT INTO t_null_pk (id, value) VALUES ('valid_id', 'test');"; + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) return false; + + // Verify the metatable has exactly 1 row (only the valid insert) + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM t_null_pk_cloudsync;", -1, &stmt, NULL); + if (rc != SQLITE_OK) return false; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); return false; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 1) return false; + + return true; +} + bool do_test_internal_functions (void) { sqlite3 *db = NULL; sqlite3_stmt *vm = NULL; @@ -1986,12 +2082,12 @@ bool do_test_internal_functions (void) { rc = sqlite3_prepare(db, sql, -1, &vm, NULL); if (rc != SQLITE_OK) goto abort_test; - int res = stmt_count(vm, NULL, 0, 0); + int res = dbvm_count(vm, NULL, 0, 0); if (res != 0) goto abort_test; if (vm) sqlite3_finalize(vm); vm = NULL; - // TEST 2 (stmt_execute returns an error) + // TEST 2 (dbvm_execute returns an error) sql = "INSERT INTO foo (name, age) VALUES ('Name1', 22)"; rc = sqlite3_exec(db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) goto abort_test; @@ -2000,7 +2096,7 @@ bool do_test_internal_functions (void) { if (rc != SQLITE_OK) goto abort_test; // this statement must fail - res = stmt_execute(vm, NULL); + res = dbvm_execute(vm, NULL); if (res != -1) goto abort_test; if (vm) sqlite3_finalize(vm); vm = NULL; @@ -2017,1158 +2113,5032 @@ bool do_test_string_replace_prefix(void) { char *host = "rejfwkr.sqlite.cloud"; char *prefix = "sqlitecloud://"; char *replacement = "https://"; - + char string[512]; snprintf(string, sizeof(string), "%s%s", prefix, host); char expected[512]; snprintf(expected, sizeof(expected), "%s%s", replacement, host); - + char *replaced = cloudsync_string_replace_prefix(string, prefix, replacement); if (string == replaced || strcmp(replaced, expected) != 0) return false; if (string != replaced) cloudsync_memory_free(replaced); - + replaced = cloudsync_string_replace_prefix(expected, prefix, replacement); if (expected != replaced) return false; - + return true; } -bool do_test_many_columns (int ncols, sqlite3 *db) { - char sql_create[10000]; - int pos = 0; - pos += snprintf(sql_create+pos, sizeof(sql_create)-pos, "CREATE TABLE IF NOT EXISTS test_many_columns (id TEXT PRIMARY KEY NOT NULL"); - for (int i=1; i sqlite3_step reports a different result: %d != %d\n", rc1, rc2); - goto finalize; - } - // rc(s) are equals here - if (rc1 == SQLITE_DONE) break; - if (rc1 != SQLITE_ROW) goto finalize; - - // we have a ROW here - for (int i=0; i_cloudsync +// meta-table before reopening. With a stale cloudsync_table_settings row and +// no matching *_cloudsync meta-table in sqlite_master, the dbversion query +// builder produces an empty (NULL) SQL string, causing sqlite3_cloudsync_init +// to fail on reopen — previously crashing in some environments. +bool do_test_stale_table_settings_dropped_meta(bool cleanup_databases) { bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; + char dbpath[256]; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + return SQLITE_OK; +} + +// Run a SQL statement expected to fail, and verify that its error message +// contains the stable substring "cloudsync_init" — which every uninitialized +// guard added by the fix points the caller at. Matching a single stable token +// rather than full text tolerates future rewordings of the user-facing string. +static bool expect_uninit_error (sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) { + // Prepare-time failures also land in sqlite3_errmsg — accept those. + const char *m = sqlite3_errmsg(db); + bool ok = (m != NULL && strstr(m, "cloudsync_init") != NULL); + if (stmt) sqlite3_finalize(stmt); + return ok; } - + rc = sqlite3_step(stmt); + bool failed = (rc != SQLITE_ROW && rc != SQLITE_DONE); + const char *msg = sqlite3_errmsg(db); + bool has_hint = (msg != NULL && strstr(msg, "cloudsync_init") != NULL); + sqlite3_finalize(stmt); + return failed && has_hint; +} + +// Regression test for the "uninit error messages" fix. Every function that +// previously leaked a misleading low-level symptom (out of memory, "not an +// error", silent -1, multi-line "no such table" dumps) must now point the +// caller at SELECT cloudsync_init(...). Match on a stable substring rather +// than full text so the test tolerates future rewordings. +bool do_test_uninit_error_messages (void) { + sqlite3 *db = NULL; + bool result = false; + + if (sqlite3_open(":memory:", &db) != SQLITE_OK) return false; + if (sqlite3_cloudsync_init(db, NULL, NULL) != SQLITE_OK) goto cleanup; + + // 96 bytes of zero padding — bigger than cloudsync_payload_header, so the + // size sanity check in dbsync_payload_decode passes and control reaches + // our guard inside cloudsync_payload_apply. + const char *payload_sql = + "SELECT cloudsync_payload_apply(zeroblob(96));"; + + if (!expect_uninit_error(db, "SELECT * FROM cloudsync_changes;")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_db_version();")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_db_version_next();")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_set_filter('foo','1=1');")) goto cleanup; + if (!expect_uninit_error(db, "SELECT cloudsync_clear_filter('foo');")) goto cleanup; + if (!expect_uninit_error(db, payload_sql)) goto cleanup; + + // Happy path: after cloudsync_init the same functions no longer fail with + // the uninitialized hint. cloudsync_db_version must now return a value. + if (sqlite3_exec(db, + "CREATE TABLE t (id TEXT PRIMARY KEY NOT NULL, v TEXT);" + "SELECT cloudsync_init('t');", + NULL, NULL, NULL) != SQLITE_OK) goto cleanup; + + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, "SELECT cloudsync_db_version();", -1, &stmt, NULL) != SQLITE_OK) goto cleanup; + int step_rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (step_rc != SQLITE_ROW) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + +cleanup: + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } return result; } -// Test a sequence of random different concurrent changes in various clients -// with changes to pk columns and non-pk columns. -bool do_test_merge_2 (int nclients, int table_mask, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Verify that cloudsync_dbversion_rebuild surfaces a real failure from +// database_select_text(SQL_DBVERSION_BUILD_QUERY, ...) instead of silently +// treating it as "no *_cloudsync meta-tables present" — which would leave +// db_version_stmt unset and cause writes to fall back to CLOUDSYNC_MIN_DB_VERSION. +bool do_test_dbversion_rebuild_error(void) { + sqlite3 *db = NULL; + cloudsync_context *ctx = NULL; bool result = false; - int rc = SQLITE_OK; - int nrows = NINSERT; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0 — the early-return-OK path is not taken). + rc = sqlite3_exec(db, "CREATE TABLE t (id TEXT PRIMARY KEY NOT NULL, v TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "SELECT cloudsync_init('t');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a secondary context on the same db and initialize it. This + // context is independent from the one registered by sqlite3_cloudsync_init, + // so we can call cloudsync_dbversion_rebuild on it directly without + // disturbing the registered functions. + ctx = cloudsync_context_create(db); + if (!ctx) goto cleanup; + if (cloudsync_context_init(ctx) == NULL) goto cleanup; + + // Install an authorizer that denies reads of sqlite_master. New prepares + // (including the one SQL_DBVERSION_BUILD_QUERY triggers inside + // database_select_text) will fail with SQLITE_AUTH. Already-prepared + // statements are unaffected, so the registered cloudsync_* functions + // still work for cleanup. + sqlite3_set_authorizer(db, deny_sqlite_master_authorizer, NULL); + + // Expect a non-OK result now that the build query cannot be prepared. + // Before the review fix this would incorrectly return DBRES_OK and leave + // db_version_stmt == NULL, silently masking the failure. + int rebuild_rc = cloudsync_dbversion_rebuild(ctx); + + // Remove authorizer before any further work so cleanup can run normally. + sqlite3_set_authorizer(db, NULL, NULL); + + if (rebuild_rc == DBRES_OK) goto cleanup; + + // The error must have been recorded on the context via cloudsync_set_dberror. + if (cloudsync_errcode(ctx) == DBRES_OK) goto cleanup; + const char *msg = cloudsync_errmsg(ctx); + if (!msg || msg[0] == 0) goto cleanup; + + result = true; + +cleanup: + sqlite3_set_authorizer(db, NULL, NULL); + if (ctx) cloudsync_context_free(ctx); + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } - - // insert data in the last customer - do_insert(db[nclients-1], table_mask, NINSERT, print_result); - - // update some random data in all clients except the last one - for (int i=0; i 0) { + g_deny_insert_remaining--; + return SQLITE_OK; } + return SQLITE_DENY; } - - // deleta data in the first customer - do_delete(db[0], table_mask, print_result); - - // merge all changes - if (do_merge(db, nclients, false) == false) { - goto finalize; - } - - // compare results - for (int i=1; i " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "s\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } + return SQLITE_OK; +} + +// Test that the error cleanup path in table_add_to_context_cb doesn't crash +// when databasevm_prepare fails for the merge statement. +// Before the zeroalloc fix, col_value_stmt[index] contained uninitialized +// garbage and databasevm_finalize was called on it → SIGSEGV. +bool do_test_context_cb_error_cleanup(void) { + sqlite3 *db = NULL; + cloudsync_context *ctx = NULL; + bool result = false; + + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + sqlite3_cloudsync_init(db, NULL, NULL); + + // Create table with PK and non-PK columns + rc = sqlite3_exec(db, + "CREATE TABLE ctx_err (id TEXT PRIMARY KEY NOT NULL, val TEXT, num INTEGER);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Init cloudsync on the table (creates meta table, triggers, settings) + rc = sqlite3_exec(db, "SELECT cloudsync_init('ctx_err');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a fresh context — ctx_err is not in its table list, + // so table_add_to_context won't short-circuit at table_lookup. + ctx = cloudsync_context_create(db); + if (!ctx) goto cleanup; + + // Install authorizer that denies INSERT on the base table. + // Allow 1 INSERT (the sentinel prepared in table_add_stmts), + // then deny the next (the merge UPSERT prepared in the callback). + // This causes databasevm_prepare to fail inside table_add_to_context_cb, + // triggering the error cleanup path. + g_deny_insert_table = "ctx_err"; + g_deny_insert_remaining = 1; + sqlite3_set_authorizer(db, deny_insert_authorizer, NULL); + + // Must return false (callback error) without crashing + bool added = table_add_to_context(ctx, table_algo_crdt_cls, "ctx_err"); + + // Remove authorizer before any further operations + sqlite3_set_authorizer(db, NULL, NULL); + g_deny_insert_table = NULL; + + if (added) goto cleanup; // should have failed + + result = true; + +cleanup: + sqlite3_set_authorizer(db, NULL, NULL); + g_deny_insert_table = NULL; + if (ctx) cloudsync_context_free(ctx); + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); } return result; } -// Test cloudsync_merge_insert where local col_version is equal to the insert col_version, -// the greater value must win. -bool do_test_merge_4 (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Test cloudsync_terminate function +bool do_test_terminate(void) { + sqlite3 *db = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i " CUSTOMERS_TABLE "\n"); - sqlite3_snprintf(sizeof(buf), buf, "SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], buf, query_table); - } - + + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE term_test (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('term_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Call terminate + rc = sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i, col_name:age, db_version:2, col_version:2, site_id:1 (remote change) -// b. pk:, col_name:TOMBSTONE, db_version:3, col_version:2, site_id:0 (local change, delete old pk) -// c. pk:, col_name:TOMBSTONE, db_version:3, col_version:1, site_id:0 (local change, updated row with new pk) -// 5. merge changes from the second client to the first client: -// - if the second client sends only local changes, then the change at 4a is not sent so the first client will show NULL -// at column "age" for the row with the new pk -// - if the second client sends all the changes (not only local changes), then the change at 4a is sent along with 4b and 4c -// so the first client will show the correct value for the "age" column -bool do_test_merge_5 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i size2 should give positive result + int r1 = cloudsync_blob_compare(blob1, 100, blob2, 1); + if (r1 <= 0) return false; + + // size1 < size2 should give negative result + int r2 = cloudsync_blob_compare(blob1, 1, blob2, 100); + if (r2 >= 0) return false; + + // Same size, different content + int r3 = cloudsync_blob_compare(blob1, 1, blob2, 1); + if (r3 == 0) return false; + + // Equal + int r4 = cloudsync_blob_compare(blob1, 1, blob1, 1); + if (r4 != 0) return false; + + return true; +} + +// Test that cloudsync_uuid() is non-deterministic (returns different values in same query) +bool do_test_deterministic_flags(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i=0; i customers\n"); - char *sql = "SELECT * FROM cloudsync_changes;"; - do_query(db[1], sql, query_changes); - } - + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // SELECT cloudsync_uuid(), cloudsync_uuid() — both values should differ + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid(), cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *u1 = (const char *)sqlite3_column_text(stmt, 0); + const char *u2 = (const char *)sqlite3_column_text(stmt, 1); + if (!u1 || !u2) goto cleanup; + + // Non-deterministic: same query, different results + if (strcmp(u1, u2) == 0) goto cleanup; + result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i=0; i 1;"; - sqlite3_stmt *vm = NULL; - rc = sqlite3_prepare_v2(db[i], sql, -1, &vm, NULL); - if (rc != SQLITE_OK) goto finalize; - rc = sqlite3_step(vm); - if (rc != SQLITE_DONE) { - printf("cloudsync_changes should not have repeated db_version and seq values, got: db_version=%d, seq=%d, count=%d\n", sqlite3_column_int(vm, 0), sqlite3_column_int(vm, 1), sqlite3_column_int(vm, 2)); - if (vm) sqlite3_finalize(vm); - goto finalize; - } - if (vm) sqlite3_finalize(vm); + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table — use TEXT pk to avoid single INTEGER pk warning + rc = sqlite3_exec(db, "CREATE TABLE t1 (id TEXT PRIMARY KEY NOT NULL, name TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('t1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Get the schema hash value by reading cloudsync_schema_versions + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db, "SELECT hash FROM cloudsync_schema_versions ORDER BY seq DESC LIMIT 1;", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(stmt); goto cleanup; } + + int64_t hash = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + + // Verify the hash can be looked up using the same int64 representation + // This tests the PRId64 format consistency fix + char sql[256]; + snprintf(sql, sizeof(sql), "SELECT 1 FROM cloudsync_schema_versions WHERE hash = (%" PRId64 ")", hash); + + stmt = NULL; + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(stmt); goto cleanup; } + + int found = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (found != 1) goto cleanup; } - - // check grouped values from cloudsync_changes - char *query_changes = "SELECT db_version, COUNT(distinct(seq)) AS cnt FROM cloudsync_changes GROUP BY db_version;"; - char *query_expected_results = "SELECT * FROM (VALUES (1,2),(2,2),(3,2),(4,2),(5,4));"; - if (do_compare_queries(db[0], query_changes, db[0], query_expected_results, -1, -1, print_result) == false) { - goto finalize; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test cloudsync_blob_compare function +bool do_test_blob_compare(void) { + // Test same content, same size + const char blob1[] = {0x01, 0x02, 0x03, 0x04}; + const char blob2[] = {0x01, 0x02, 0x03, 0x04}; + int result = cloudsync_blob_compare(blob1, 4, blob2, 4); + if (result != 0) return false; + + // Test different sizes (line 168 in utils.c) + const char blob3[] = {0x01, 0x02, 0x03}; + result = cloudsync_blob_compare(blob1, 4, blob3, 3); + if (result == 0) return false; // Should be non-zero (different sizes) + + // Test different content, same size + const char blob4[] = {0x01, 0x02, 0x03, 0x05}; + result = cloudsync_blob_compare(blob1, 4, blob4, 4); + if (result == 0) return false; // Should be non-zero (different content) + + // Test empty blobs + result = cloudsync_blob_compare("", 0, "", 0); + if (result != 0) return false; + + return true; +} + +// Test string duplication functions more thoroughly +bool do_test_string_functions(void) { + // Test cloudsync_string_ndup (non-lowercase path) + const char *test_str = "Hello World"; + char *dup = cloudsync_string_ndup(test_str, 5); // "Hello" + if (!dup) return false; + if (strcmp(dup, "Hello") != 0) { + cloudsync_memory_free(dup); + return false; } - - if (print_result) { - printf("\n-> customers\n"); - char *sql = "SELECT * FROM cloudsync_changes();"; - do_query(db[1], sql, query_changes); + cloudsync_memory_free(dup); + + // Test cloudsync_string_dup + dup = cloudsync_string_dup("Test String"); + if (!dup) return false; + if (strcmp(dup, "Test String") != 0) { + cloudsync_memory_free(dup); + return false; } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i local_col_version, even if db_version is smaller - rc = sqlite3_exec(db, "INSERT INTO cloudsync_changes (tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) VALUES ('todo',x'010B0133','status','this update must be applied',11,9,x'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',1,0);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // compare results - char *sql = "SELECT * FROM todo ORDER BY id;"; - char *query_expected_results = "SELECT * FROM (VALUES ('1','Buy groceries','finalized'),('2','Buy bananas','in_progress2'),('3','New Item','this update must be applied'));"; - if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { - goto finalize; - } - - // compare values in the changes vtab - sql = "SELECT col_name,col_value,col_version,db_version,cl,seq FROM cloudsync_changes;"; - query_expected_results = "SELECT * FROM (VALUES ('title','Buy groceries',1,1,1,0),('title','Buy bananas',1,2,1,0),('status','in_progress2',1,2,1,1),('title','New Item',10,10,1,0),('status','finalized',200,300,1,0),('status','this update must be applied',11,301,1,0));"; - if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { - goto finalize; - } - - if (print_result) { - printf("\n-> customers\n"); - char *sql = "SELECT * FROM todo;"; - do_query(db, sql, query_changes); - } - result = true; - rc = SQLITE_OK; - -finalize: - if (rc != SQLITE_OK && db && (sqlite3_errcode(db) != SQLITE_OK)) printf("do_test_insert_cloudsync_changes: %s\n", sqlite3_errmsg(db)); + +cleanup: + if (stmt) sqlite3_finalize(stmt); if (db) close_db(db); - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, 0, timestamp, saved_counter); - file_delete_internal(buf); - } return result; } -#define SKIP_SCHEMA_CHECK 1 - -bool do_test_merge_alter_schema_1 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +// Test SQL-level pk_decode function +bool do_test_sql_pk_decode(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 3.15) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_pk_decode for BLOB (index 4) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 4);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const unsigned char expected_blob[] = {0xDE, 0xAD, 0xBE, 0xEF}; + const void *blob_val = sqlite3_column_blob(stmt, 0); + int blob_len = sqlite3_column_bytes(stmt, 0); + if (blob_len != 4 || memcmp(blob_val, expected_blob, 4) != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; - // merge all changes, now it must work fine - if (do_merge(db, nclients, only_locals) == false) { - goto finalize; - } - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test negative integer encoding and decoding + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-12345);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + int pk_len = sqlite3_column_bytes(stmt, 0); + char pk_copy[1024]; + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Decode and verify + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t int_val = sqlite3_column_int64(stmt, 0); + if (int_val != -12345) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test negative float encoding and decoding + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-3.14159);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + pk_len = sqlite3_column_bytes(stmt, 0); + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + double float_val = sqlite3_column_double(stmt, 0); + if (float_val > -3.14 || float_val < -3.15) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test INT64_MIN (maximum negative integer) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(-9223372036854775808);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + pk_len = sqlite3_column_bytes(stmt, 0); + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_decode(?, 1);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int_val = sqlite3_column_int64(stmt, 0); + if (int_val != INT64_MIN) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test settings functions +bool do_test_settings_functions(void) { + sqlite3 *db = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set + rc = sqlite3_exec(db, "SELECT cloudsync_set('test_key', 'test_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create a table and test table-level settings + rc = sqlite3_exec(db, "CREATE TABLE settings_test (id TEXT PRIMARY KEY NOT NULL, data TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('settings_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set_table + rc = sqlite3_exec(db, "SELECT cloudsync_set_table('settings_test', 'table_key', 'table_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_set_column + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('settings_test', 'data', 'col_key', 'col_value');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test cloudsync_is_sync and cloudsync_is_enabled functions +bool do_test_sync_enabled_functions(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE sync_test (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 1 after init + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int enabled = sqlite3_column_int(stmt, 0); + if (enabled != 1) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_is_sync - should return 0 (not in sync mode) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + // Value depends on implementation + sqlite3_finalize(stmt); + stmt = NULL; + + // Disable sync + rc = sqlite3_exec(db, "SELECT cloudsync_disable('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 0 after disable + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Re-enable sync + rc = sqlite3_exec(db, "SELECT cloudsync_enable('sync_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_enabled - should be 1 again + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('sync_test');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 1) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test with non-existent table + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_enabled('non_existent_table');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + enabled = sqlite3_column_int(stmt, 0); + if (enabled != 0) goto cleanup; // Should be 0 for non-existent table + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_uuid SQL function +bool do_test_sql_uuid_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_uuid() SQL function + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *uuid1 = (const char *)sqlite3_column_text(stmt, 0); + if (!uuid1 || strlen(uuid1) != 36) goto cleanup; // UUID with dashes + + // Store the first UUID + char uuid1_copy[40]; + strncpy(uuid1_copy, uuid1, sizeof(uuid1_copy) - 1); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Get another UUID - should be different + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_uuid();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *uuid2 = (const char *)sqlite3_column_text(stmt, 0); + if (!uuid2 || strlen(uuid2) != 36) goto cleanup; + + // UUIDs should be different + if (strcmp(uuid1_copy, uuid2) == 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test pk_encode with empty values +bool do_test_pk_encode_edge_cases(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test encoding empty text + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode('');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding empty blob + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(X'');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding zero + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(0);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding 0.0 + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(0.0);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test encoding large integers + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode(9223372036854775807);", -1, &stmt, NULL); // INT64_MAX + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + pk = sqlite3_column_blob(stmt, 0); + if (!pk) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_col_value function +bool do_test_col_value_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE col_test (id TEXT PRIMARY KEY NOT NULL, data TEXT, num INTEGER);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('col_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert data + rc = sqlite3_exec(db, "INSERT INTO col_test (id, data, num) VALUES ('key1', 'value1', 42);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Get the pk for key1 + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_pk_encode('key1');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const void *pk = sqlite3_column_blob(stmt, 0); + int pk_len = sqlite3_column_bytes(stmt, 0); + char pk_copy[256]; + memcpy(pk_copy, pk, pk_len); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value for text column + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', 'data', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + const char *val = (const char *)sqlite3_column_text(stmt, 0); + if (!val || strcmp(val, "value1") != 0) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value for integer column + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', 'num', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int num_val = sqlite3_column_int(stmt, 0); + if (num_val != 42) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_col_value with TOMBSTONE value (should return NULL) + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_col_value('col_test', '__[RIP]__', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_bind_blob(stmt, 1, pk_copy, pk_len, SQLITE_STATIC); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + if (sqlite3_column_type(stmt, 0) != SQLITE_NULL) goto cleanup; + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_is_sync function +bool do_test_is_sync_function(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE sync_check (id TEXT PRIMARY KEY NOT NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('sync_check');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_is_sync + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('sync_check');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + // Result depends on internal state + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_is_sync with non-existent table + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_is_sync('non_existent');", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int is_sync = sqlite3_column_int(stmt, 0); + if (is_sync != 0) goto cleanup; // Should be 0 for non-existent table + + sqlite3_finalize(stmt); + stmt = NULL; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test cloudsync_db_version_next function +bool do_test_db_version_next(void) { + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and sync a table to properly initialize the context + rc = sqlite3_exec(db, "CREATE TABLE dbv_test (id TEXT PRIMARY KEY NOT NULL, val TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_sync('dbv_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Test cloudsync_db_version_next without argument + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_db_version_next();", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t v1 = sqlite3_column_int64(stmt, 0); + + sqlite3_finalize(stmt); + stmt = NULL; + + // Test cloudsync_db_version_next with merging_version argument + rc = sqlite3_prepare_v2(db, "SELECT cloudsync_db_version_next(100);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) goto cleanup; + + int64_t v2 = sqlite3_column_int64(stmt, 0); + + sqlite3_finalize(stmt); + stmt = NULL; + + // v2 should be greater or equal to v1 + if (v2 < v1) goto cleanup; + + result = true; + +cleanup: + if (stmt) sqlite3_finalize(stmt); + if (db) close_db(db); + return result; +} + +// Test various insert/update/delete scenarios through SQL +bool do_test_insert_update_delete_sql(void) { + sqlite3 *db = NULL; + bool result = false; + int rc; + + rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) return false; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Create and init a table + rc = sqlite3_exec(db, "CREATE TABLE iud_test (id TEXT PRIMARY KEY NOT NULL, val TEXT, num REAL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_init('iud_test');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert data + rc = sqlite3_exec(db, "INSERT INTO iud_test (id, val, num) VALUES ('id1', 'initial', 1.5);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Update data + rc = sqlite3_exec(db, "UPDATE iud_test SET val = 'updated', num = 2.5 WHERE id = 'id1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert more data + rc = sqlite3_exec(db, "INSERT INTO iud_test (id, val, num) VALUES ('id2', 'second', 3.5);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Delete data + rc = sqlite3_exec(db, "DELETE FROM iud_test WHERE id = 'id2';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Check changes table + rc = sqlite3_exec(db, "SELECT COUNT(*) FROM cloudsync_changes;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + result = true; + +cleanup: + if (db) close_db(db); + return result; +} + +// Test dbutils_binary_comparison function (already exposed for testing) +bool do_test_binary_comparison(void) { + // Test cases for dbutils_binary_comparison + int result1 = dbutils_binary_comparison(5, 3); // 5 > 3, should return 1 + if (result1 != 1) return false; + + int result2 = dbutils_binary_comparison(3, 5); // 3 < 5, should return -1 + if (result2 != -1) return false; + + int result3 = dbutils_binary_comparison(5, 5); // 5 == 5, should return 0 + if (result3 != 0) return false; + + int result4 = dbutils_binary_comparison(-10, 10); // -10 < 10 + if (result4 != -1) return false; + + int result5 = dbutils_binary_comparison(0, 0); // 0 == 0 + if (result5 != 0) return false; + + return true; +} + + +// Test pk_decode with various malformed inputs +bool do_test_pk_decode_malformed(void) { + // Test with empty buffer + int res = pk_decode_prikey(NULL, 0, NULL, NULL); + if (res != -1) return false; // Should fail for NULL buffer + + // Test with buffer but 0 length + char empty[1] = {0}; + res = pk_decode_prikey(empty, 0, NULL, NULL); + // This should also fail since count can't be read + if (res != -1) return false; + + // Test pk_decode with count specified but incomplete buffer + size_t seek = 0; + res = pk_decode(empty, 0, 5, &seek, -1, NULL, NULL); + if (res != -1) return false; // Should fail - can't read 5 elements from empty buffer + + return true; +} + + +bool do_test_many_columns (int ncols, sqlite3 *db) { + char sql_create[10000]; + int pos = 0; + pos += snprintf(sql_create+pos, sizeof(sql_create)-pos, "CREATE TABLE IF NOT EXISTS test_many_columns (id TEXT PRIMARY KEY NOT NULL"); + for (int i=1; i sqlite3_step reports a different result: %d != %d\n", rc1, rc2); + goto finalize; + } + // rc(s) are equals here + if (rc1 == SQLITE_DONE) break; + if (rc1 != SQLITE_ROW) goto finalize; + + // we have a ROW here + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); + } + } + + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test a sequence of random different concurrent changes in various clients +// with changes to pk columns and non-pk columns. +bool do_test_merge_2 (int nclients, int table_mask, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int nrows = NINSERT; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "s\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test cloudsync_merge_insert where local col_version is equal to the insert col_version, +// the greater value must win. +bool do_test_merge_4 (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + sqlite3_snprintf(sizeof(buf), buf, "SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], buf, query_table); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i, col_name:age, db_version:2, col_version:2, site_id:1 (remote change) +// b. pk:, col_name:TOMBSTONE, db_version:3, col_version:2, site_id:0 (local change, delete old pk) +// c. pk:, col_name:TOMBSTONE, db_version:3, col_version:1, site_id:0 (local change, updated row with new pk) +// 5. merge changes from the second client to the first client: +// - if the second client sends only local changes, then the change at 4a is not sent so the first client will show NULL +// at column "age" for the row with the new pk +// - if the second client sends all the changes (not only local changes), then the change at 4a is sent along with 4b and 4c +// so the first client will show the correct value for the "age" column +bool do_test_merge_5 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter++; + for (int i=0; i customers\n"); + do_query(db[1], "SELECT * FROM cloudsync_changes;", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter++; + for (int i=0; i 1;"; + sqlite3_stmt *vm = NULL; + rc = sqlite3_prepare_v2(db[i], sql, -1, &vm, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(vm); + if (rc != SQLITE_DONE) { + printf("cloudsync_changes should not have repeated db_version and seq values, got: db_version=%d, seq=%d, count=%d\n", sqlite3_column_int(vm, 0), sqlite3_column_int(vm, 1), sqlite3_column_int(vm, 2)); + if (vm) sqlite3_finalize(vm); + goto finalize; + } + if (vm) sqlite3_finalize(vm); + } + + // check grouped values from cloudsync_changes + char *sql_changes = "SELECT db_version, COUNT(distinct(seq)) AS cnt FROM cloudsync_changes GROUP BY db_version;"; + char *query_expected_results = "SELECT * FROM (VALUES (1,2),(2,2),(3,2),(4,2),(5,4));"; + if (do_compare_queries(db[0], sql_changes, db[0], query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + if (print_result) { + printf("\n-> customers\n"); + do_query(db[1], "SELECT * FROM cloudsync_changes();", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i local_col_version, even if db_version is smaller + rc = sqlite3_exec(db, "INSERT INTO cloudsync_changes (tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) VALUES ('todo',x'010B0133','status','this update must be applied',11,9,x'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',1,0);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; + + // compare results + char *sql = "SELECT * FROM todo ORDER BY id;"; + char *query_expected_results = "SELECT * FROM (VALUES ('1','Buy groceries','finalized'),('2','Buy bananas','in_progress2'),('3','New Item','this update must be applied'));"; + if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + // compare values in the changes vtab + sql = "SELECT col_name,col_value,col_version,db_version,cl,seq FROM cloudsync_changes;"; + query_expected_results = "SELECT * FROM (VALUES ('title','Buy groceries',1,1,1,0),('title','Buy bananas',1,2,1,0),('status','in_progress2',1,2,1,1),('title','New Item',10,10,1,0),('status','finalized',200,300,1,0),('status','this update must be applied',11,301,1,0));"; + if (do_compare_queries(db, sql, db, query_expected_results, -1, -1, print_result) == false) { + goto finalize; + } + + if (print_result) { + printf("\n-> customers\n"); + do_query(db, "SELECT * FROM todo;", query_changes); + } + + result = true; + rc = SQLITE_OK; + +finalize: + if (rc != SQLITE_OK && db && (sqlite3_errcode(db) != SQLITE_OK)) printf("do_test_insert_cloudsync_changes: %s\n", sqlite3_errmsg(db)); + if (db) close_db(db); + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, 0, timestamp, saved_counter); + file_delete_internal(buf); + } + return result; +} + +bool do_test_merge_alter_schema_1 (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + int table_mask = TEST_PRIKEYS; + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + // create databases and tables + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE "\n"); + sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + + result = true; + rc = SQLITE_OK; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_two_tables error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test conflicting primary key updates +bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i " CUSTOMERS_TABLE " (after conflict resolution)\n"); + sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_conflicting_pkeys error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test large dataset merge performance +bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + const int LARGE_N = 1000; // Insert 1000 records per client + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Large dataset merge completed\n"); + char *sql = sqlite3_mprintf("SELECT COUNT(*) as total_records FROM \"%w\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_large_dataset error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test nested transactions before merge +bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_nested_transactions error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test three-way merge pattern +bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; iB, then B->C, then verify A==C + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[1], db[2], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[2], db[0], false, true) == false) { + goto finalize; + } + + // final full merge to ensure consistency + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify all databases are identical + for (int i=1; i 0) { + result = false; + printf("do_test_merge_three_way error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test NULL value handling during merge +bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_null_values error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test BLOB data merge +bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i BLOB test table\n"); + do_query(db[0], "SELECT id, HEX(data), description FROM blob_test ORDER BY id;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_blob_data error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test mixed operations (INSERT/UPDATE/DELETE) in transactions +bool do_test_merge_mixed_operations (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_mixed_operations error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test hub-spoke merge pattern (central client merging with multiple peripherals) +bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + int table_mask = TEST_PRIKEYS; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 4) { + nclients = 4; + printf("Number of test merge increased to %d clients\n", 4); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i hub + for (int i=1; i all spokes + for (int i=1; i 0) { + result = false; + printf("do_test_merge_hub_spoke error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test timestamp precision conflicts +bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Timestamp precision test\n"); + do_query(db[0], "SELECT id, created_at, data FROM timestamp_test ORDER BY id, created_at;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_timestamp_precision error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test partial merge failure recovery +bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_partial_failure error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test transaction rollback scenarios +bool do_test_merge_rollback_scenarios (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + printf("Rollback failed - found %d rolled back records\n", rollback_count); + sqlite3_finalize(stmt); + goto finalize; + } + } + sqlite3_finalize(stmt); + } + + // merge changes + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify consistency - only committed data should be present + for (int i=1; i 0) { + result = false; + printf("do_test_merge_rollback_scenarios error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test circular merge (A->B->C->A) +bool do_test_merge_circular (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i1->2->0 + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[1], db[2], false, true) == false) { + goto finalize; + } + + if (do_merge_using_payload(db[2], db[0], false, true) == false) { + goto finalize; + } + + // complete the circle and ensure consistency + if (do_merge(db, nclients, false) == false) { + goto finalize; + } + + // verify all clients have all data + for (int i=1; i Circular merge result\n"); + do_query(db[0], "SELECT * FROM circular_test ORDER BY origin_client;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_circular error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test foreign key constraints during merge +bool do_test_merge_foreign_keys (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_foreign_keys error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test trigger interaction during merge +// Expected failure: TRIGGERs are not fully supported by this extension. +bool do_test_merge_triggers (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i Trigger test results\n"); + do_query(db[0], "SELECT id, data, update_count FROM trigger_test ORDER BY id;", query_table); + printf("\n-> Audit log\n"); + do_query(db[0], "SELECT COUNT(*) as audit_entries FROM audit_log;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_triggers error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test index consistency during merge +bool do_test_merge_index_consistency (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i 0) { + result = false; + printf("do_test_merge_index_consistency error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test JSON column merge +bool do_test_merge_json_columns (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { nclients = 2; printf("Number of test merge increased to %d clients\n", 2); } - // create databases and tables - int table_mask = TEST_PRIKEYS; time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i JSON test results\n"); + do_query(db[0], "SELECT id, json_extract(metadata, '$.name') as name, json_extract(metadata, '$.preferences.theme') as theme FROM json_test ORDER BY id;", query_table); + } + + result = true; + +finalize: + for (int i=0; i 0) { + result = false; + printf("do_test_merge_json_columns error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +// Test concurrent merge attempts +bool do_test_payload_apply_concurrent_write (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + sqlite3 *db_target2 = NULL; + sqlite3_stmt *select_stmt = NULL; + sqlite3_stmt *apply_stmt = NULL; + bool result = false; + int rc = SQLITE_OK; + + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients < 2) nclients = 2; + if (nclients > 2) nclients = 2; // this test uses exactly 2 databases + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + + // create two file-based databases: db[0]=src, db[1]=target + for (int i = 0; i < nclients; ++i) { db[i] = do_create_database_file(i, timestamp, test_counter++); - if (db[i] == false) return false; - - if (do_create_tables(table_mask, db[i]) == false) { + if (db[i] == NULL) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE concurrent_tbl (id TEXT PRIMARY KEY, val TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('concurrent_tbl', 'cls', 1);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + } + + // insert data on src (db[0]) + rc = sqlite3_exec(db[0], "INSERT INTO concurrent_tbl VALUES ('row1', 'hello');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO concurrent_tbl VALUES ('row2', 'world');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // extract payload from db[0] + const char *encode_sql = "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();"; + rc = sqlite3_prepare_v2(db[0], encode_sql, -1, &select_stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + + rc = sqlite3_step(select_stmt); + if (rc != SQLITE_ROW) goto finalize; + + const void *payload_data = sqlite3_column_blob(select_stmt, 0); + int payload_size = sqlite3_column_bytes(select_stmt, 0); + if (payload_data == NULL || payload_size == 0) goto finalize; + + // copy payload since we'll need it after finalizing select_stmt + void *payload_copy = malloc(payload_size); + if (payload_copy == NULL) goto finalize; + memcpy(payload_copy, payload_data, payload_size); + sqlite3_finalize(select_stmt); + select_stmt = NULL; + + // open second connection to same file as db[1] (target) + { + char buf[256]; + do_build_database_path(buf, 1, timestamp, saved_counter + 1); + rc = sqlite3_open(buf, &db_target2); + if (rc != SQLITE_OK) { + printf("Error opening db_target2: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); goto finalize; } - - if (do_augment_tables(TEST_PRIKEYS, db[i], table_algo_crdt_cls) == false) { + sqlite3_exec(db_target2, "PRAGMA journal_mode=WAL;", NULL, NULL, NULL); + sqlite3_cloudsync_init(db_target2, NULL, NULL); + } + + // on db[1] (target): begin immediate to hold write lock + rc = sqlite3_exec(db[1], "BEGIN IMMEDIATE;", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("BEGIN IMMEDIATE failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + rc = sqlite3_exec(db[1], "INSERT INTO concurrent_tbl VALUES ('blocker', 'blocking');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("Blocker INSERT failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + + // on db_target2: try to apply payload — should fail with BUSY + rc = sqlite3_prepare_v2(db_target2, "SELECT cloudsync_payload_decode(?);", -1, &apply_stmt, NULL); + if (rc != SQLITE_OK) { + printf("Prepare apply failed: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + rc = sqlite3_bind_blob(apply_stmt, 1, payload_copy, payload_size, SQLITE_STATIC); + if (rc != SQLITE_OK) { + printf("Bind failed: %s\n", sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + // set a short busy timeout so it doesn't wait forever (0 = fail immediately) + sqlite3_busy_timeout(db_target2, 0); + + rc = sqlite3_step(apply_stmt); + if (rc == SQLITE_ROW || rc == SQLITE_DONE) { + printf("Expected BUSY error but apply succeeded (rc=%d)\n", rc); + free(payload_copy); + goto finalize; + } + + // verify we got a BUSY-related error + int errcode = sqlite3_errcode(db_target2); + if (errcode != SQLITE_BUSY && errcode != SQLITE_LOCKED) { + printf("Expected SQLITE_BUSY or SQLITE_LOCKED but got error %d: %s\n", errcode, sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + if (print_result) { + printf(" Step 1: Apply blocked as expected (errcode=%d: %s)\n", errcode, sqlite3_errmsg(db_target2)); + } + + // release the write lock on db[1] + rc = sqlite3_exec(db[1], "COMMIT;", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + printf("COMMIT failed: %s\n", sqlite3_errmsg(db[1])); + free(payload_copy); + goto finalize; + } + + // retry: reset and step again — should succeed now + sqlite3_reset(apply_stmt); + sqlite3_busy_timeout(db_target2, 5000); // give it time now + + rc = sqlite3_step(apply_stmt); + if (rc != SQLITE_ROW) { + printf("Expected SQLITE_ROW on retry but got %d: %s\n", rc, sqlite3_errmsg(db_target2)); + free(payload_copy); + goto finalize; + } + + if (print_result) { + printf(" Step 2: Apply succeeded after lock released\n"); + } + + sqlite3_finalize(apply_stmt); + apply_stmt = NULL; + free(payload_copy); + payload_copy = NULL; + + // verify: db_target2 should have row1, row2 (from payload) + blocker (from db[1]) + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db_target2, "SELECT COUNT(*) FROM concurrent_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_step(count_stmt); + if (rc != SQLITE_ROW) { sqlite3_finalize(count_stmt); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 3) { + printf("Expected 3 rows but got %d\n", count); goto finalize; } + if (print_result) { + printf(" Step 3: Target has %d rows (expected 3)\n", count); + } } - - // insert, update and delete some data in the first client - do_insert(db[0], TEST_PRIKEYS, NINSERT, print_result); - - // alter table also on db0 - if (do_alter_tables(table_mask, db[0], 4) == false) { + + // full consistency: merge all databases using payload + // first close db_target2 to avoid lock conflicts during merge + close_db(db_target2); + db_target2 = NULL; + + // merge db[0] <-> db[1] in both directions + if (do_merge_using_payload(db[0], db[1], false, true) == false) { + printf("Merge src->target failed\n"); goto finalize; } - -#ifndef SKIP_SCHEMA_CHECK - // merge changes from db0 to db1, it should fail because db0 has a newer schema hash - if (do_merge_using_payload(db[0], db[1], only_locals, false) == true) { + if (do_merge_using_payload(db[1], db[0], false, true) == false) { + printf("Merge target->src failed\n"); goto finalize; } -#endif + + // verify consistency + { + const char *sql = "SELECT * FROM concurrent_tbl ORDER BY id;"; + bool cmp = do_compare_queries(db[0], sql, db[1], sql, -1, -1, print_result); + if (!cmp) { + printf("Consistency check failed between src and target\n"); + goto finalize; + } + } + + if (print_result) { + printf(" Step 4: Full consistency verified\n"); + } + + result = true; + +finalize: + if (select_stmt) sqlite3_finalize(select_stmt); + if (apply_stmt) sqlite3_finalize(apply_stmt); + if (db_target2) close_db(db_target2); + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_payload_apply_concurrent_write error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) { + if (sqlite3_get_autocommit(db[i]) == 0) { + result = false; + printf("do_test_payload_apply_concurrent_write error: db %d is in transaction\n", i); + } + int counter = close_db(db[i]); + if (counter > 0) { + result = false; + printf("do_test_payload_apply_concurrent_write error: db %d has %d unterminated statements\n", i, counter); + } + } + if (cleanup_databases) { + char buf[256]; + do_build_database_path(buf, i, timestamp, saved_counter++); + file_delete_internal(buf); + } + } + return result; +} + +bool do_test_merge_concurrent_attempts (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; + int rc = SQLITE_OK; - // insert a new value on db1 - do_insert_val(db[1], TEST_PRIKEYS, 123456, print_result); + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); + if (nclients >= MAX_SIMULATED_CLIENTS) { + nclients = MAX_SIMULATED_CLIENTS; + printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 3) { + nclients = 3; + printf("Number of test merge increased to %d clients\n", 3); + } + + time_t timestamp = time(NULL); + int saved_counter = test_counter; + for (int i=0; i1 and 1->2 simultaneously + bool merge1 = do_merge_using_payload(db[0], db[1], false, true); + bool merge2 = do_merge_using_payload(db[1], db[2], false, true); + + if (!merge1 || !merge2) { + printf("Concurrent merge phase 1 had issues\n"); + } + + // second: 2->0 and 0->1 simultaneously + bool merge3 = do_merge_using_payload(db[2], db[0], false, true); + bool merge4 = do_merge_using_payload(db[0], db[1], false, true); + + if (!merge3 || !merge4) { + printf("Concurrent merge phase 2 had issues\n"); + } + + // final consistency merge + if (do_merge(db, nclients, false) == false) { + printf("Final merge failed\n"); } - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + // verify consistency across all clients + for (int i=1; i 0) { + result = false; + printf("do_test_merge_concurrent_attempts error: db %d has %d unterminated statements\n", i, counter); + } + } if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3178,11 +7148,27 @@ bool do_test_merge_alter_schema_2 (int nclients, bool print_result, bool cleanup return result; } -bool do_test_merge_two_tables (int nclients, bool print_result, bool cleanup_databases) { +// Test with 10 clients using a simple table with composite primary key +// Summary of the test function: +// 1. Simple table with composite primary key: Creates a table with columns (id1, id2) as the composite primary key and (data1, data2) as non-primary key columns +// 2. 10 database simulation: Forces exactly 10 clients regardless of the parameter passed +// 3. CloudSync initialization: Each database initializes the cloudsync library for the test table +// 4. Initial data: Each client inserts 3 initial rows with unique data +// 5. Multiple operation rounds: Performs 5 rounds of operations where each client performs: +// - Inserts new rows (20%) +// - Updates local data (15%) - data inserted by the same client +// - Updates remote data (50%) - data inserted by other clients, tests cross-client conflicts +// - Deletes both local and remote data (15%) - minimal deletions to preserve data for updates +// 6. Partial merging: After each round, clients merge with only 2-3 other clients (not all), simulating real-world scenarios where not all clients sync immediately +// 7. Final convergence: +// - Merges all changes from all clients to the first client (client 0) +// - Propagates the final state from client 0 to all other clients +// 8. Verification: Compares all databases to ensure they have converged to identical final states +bool do_test_merge_composite_pk_10_clients (int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + const char *table_name = "simple_composite_test"; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3193,118 +7179,260 @@ bool do_test_merge_two_tables (int nclients, bool print_result, bool cleanup_dat printf("Number of test merge increased to %d clients\n", 2); } + // Force to 10 clients for this specific test + nclients = 10; + // create databases and tables time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0) { + printf("Client %d updated %d local rows in round %d\n", i, updated_rows, round + 1); + } + } + break; + + case 2: // UPDATE remote data (more aggressive - creates conflicts) + { + int target_client = (i + 1 + round) % nclients; // Vary target client + char *sql; + int updated_rows = 0; + + // First try: update rows from target client's range + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_targeting%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", + table_name, round, i, target_client, round * 2000 + i, target_client * 100, (target_client + 1) * 100 - 1); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + + // Second try: if no rows updated, target any rows not from current client + if (updated_rows == 0) { + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_any', data2 = %d WHERE id1 NOT BETWEEN %d AND %d LIMIT 2;", + table_name, round, i, round * 2000 + i, i * 100, (i + 1) * 100 - 1); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + } + + // Third try: if still no updates, target any available rows + if (updated_rows == 0) { + sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_fallback', data2 = %d WHERE rowid IN (SELECT rowid FROM %s LIMIT 2 OFFSET %d);", + table_name, round, i, round * 2000 + i, table_name, (i + round) % 10); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + updated_rows += sqlite3_changes(db[i]); + } + + if (rc != SQLITE_OK) { + printf("Update remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); + } else if (print_result && updated_rows > 0) { + printf("Client %d updated %d remote rows in round %d\n", i, updated_rows, round + 1); + } + } + break; + + case 3: // DELETE (favoring remote data deletion) + { + // 80% chance to delete remote data, 20% chance to delete local data + if ((i + round) % 5 == 0) { + // Delete local data (20% of delete operations) + char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row1';", + table_name, i * 100 + 1, i); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) { + printf("Delete local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); + } + } else { + // Delete remote data (80% of delete operations) + int target_client = (i + 2 + round) % nclients; // Vary target client + char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row0';", + table_name, target_client * 100, target_client); + rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) { + printf("Delete remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); + } + } + } + break; + } + } + + // Partial merge: each client merges with 2-3 other random clients (not all) + for (int i = 0; i < nclients; ++i) { + int merge_targets = 2 + (round % 2); // 2 or 3 targets + for (int j = 0; j < merge_targets; ++j) { + int target = (i + j + 1 + round) % nclients; + if (target != i) { + if (do_merge_using_payload(db[i], db[target], true, true) == false) { + if (print_result) printf("Partial merge failed from client %d to %d\n", i, target); + goto finalize; + } + } + } } - if (do_augment_tables(table_mask, db[i], table_algo_crdt_cls) == false) { - return false; - } + if (print_result) printf("Completed round %d operations and partial merges\n", round + 1); } - // perform transactions on both tables in client 0 - rc = sqlite3_exec(db[0], "BEGIN TRANSACTION;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - // insert data into both tables in a single transaction - char *sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('john', 'doe', 30, 'test note', 'stamp1');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") VALUES ('jane', 'smith');", CUSTOMERS_NOCOLS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[0], "COMMIT;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - // perform different transactions on both tables in client 1 - rc = sqlite3_exec(db[1], "BEGIN TRANSACTION;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('alice', 'jones', 25, 'another note', 'stamp2');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\") VALUES ('bob', 'wilson');", CUSTOMERS_NOCOLS_TABLE); - rc = sqlite3_exec(db[1], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - rc = sqlite3_exec(db[1], "COMMIT;", NULL, NULL, NULL); - if (rc != SQLITE_OK) goto finalize; + // Final convergence: merge all changes to client 0 + if (print_result) printf("Starting final convergence to client 0\n"); + for (int i = 1; i < nclients; ++i) { + if (do_merge_using_payload(db[i], db[0], true, true) == false) { + if (print_result) printf("Final merge to client 0 failed from client %d\n", i); + goto finalize; + } + } - // merge changes between the two clients - if (do_merge(db, nclients, false) == false) { - goto finalize; + // Merge changes from client 0 to all other clients + if (print_result) printf("Propagating final state from client 0 to all other clients\n"); + for (int i = 1; i < nclients; ++i) { + if (do_merge_using_payload(db[0], db[i], false, true) == false) { + if (print_result) printf("Final merge from client 0 failed to client %d\n", i); + goto finalize; + } } - // verify that both databases have the same content for both tables - for (int i=1; i " CUSTOMERS_TABLE "\n"); - sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + printf("\nFinal converged state:\n"); + char *display_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); + do_query(db[0], display_sql, NULL); + sqlite3_free(display_sql); } result = true; - rc = SQLITE_OK; finalize: - for (int i=0; i 0) { result = false; - printf("do_test_merge_two_tables error: db %d has %d unterminated statements\n", i, counter); + printf("do_test_merge_composite_pk_10_clients error: db %d has %d unterminated statements\n", i, counter); } } if (cleanup_databases) { char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); + do_build_database_path(buf, i, timestamp, saved_counter); file_delete_internal(buf); } } return result; } -// Test conflicting primary key updates -bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool cleanup_databases) { +bool do_test_prikey (int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3315,174 +7443,65 @@ bool do_test_merge_conflicting_pkeys (int nclients, bool print_result, bool clea printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i " CUSTOMERS_TABLE " (after conflict resolution)\n"); - sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_conflicting_pkeys error: db %d has %d unterminated statements\n", i, counter); - } - } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); - } - } - return result; -} - -// Test large dataset merge performance -bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - const int LARGE_N = 1000; // Insert 1000 records per client - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i foo (client1)\n"); + do_query(db[0], "SELECT * FROM foo ORDER BY a;", NULL); } - // verify all clients have same data + // send local changes to client1 for (int i=1; i Large dataset merge completed\n"); - char *sql = sqlite3_mprintf("SELECT COUNT(*) as total_records FROM \"%w\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + // compare results + for (int i=1; i 0) { - result = false; - printf("do_test_merge_large_dataset error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_prikey_null error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3492,208 +7511,138 @@ bool do_test_merge_large_dataset (int nclients, bool print_result, bool cleanup_ return result; } -// Test nested transactions before merge -bool do_test_merge_nested_transactions (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { +bool do_test_double_init(int nclients, bool cleanup_databases) { + if (nclients<2) { nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); + printf("Number of clients for test do_test_double_init increased to %d clients\n", 2); } + bool result = false; + sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i 0) { - result = false; - printf("do_test_merge_nested_transactions error: db %d has %d unterminated statements\n", i, counter); - } - } + if (!result && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_init error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); + do_build_database_path(buf, i, timestamp, test_counter); file_delete_internal(buf); } } + test_counter++; return result; } -// Test three-way merge pattern -bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_gos (int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + cloudsync_context *data[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); - } if (nclients >= MAX_SIMULATED_CLIENTS) { nclients = MAX_SIMULATED_CLIENTS; printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); + } else if (nclients < 2) { + nclients = 2; + printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; iB, then B->C, then verify A==C - if (do_merge_using_payload(db[0], db[1], false, true) == false) { - goto finalize; - } - - if (do_merge_using_payload(db[1], db[2], false, true) == false) { - goto finalize; - } + const char *sql = "INSERT INTO log (id, desc, counter) VALUES (?, ?, ?);"; + char buffer[UUID_STR_MAXLEN]; + char desc[256]; + int nentries = 10; - if (do_merge_using_payload(db[2], db[0], false, true) == false) { - goto finalize; + // insert unique log in each database + for (int i=0; i log\n"); + do_query(db[0], "SELECT * FROM log ORDER BY id;", NULL); } result = true; finalize: for (int i=0; i 0) { - result = false; - printf("do_test_merge_three_way error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_gos error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); + if (data[i]) cloudsync_context_free(data[i]); + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3703,12 +7652,14 @@ bool do_test_merge_three_way (int nclients, bool print_result, bool cleanup_data return result; } -// Test NULL value handling during merge -bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_network_encode_decode (int nclients, bool print_result, bool cleanup_databases, bool force_uncompressed) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; + cloudsync_context *data[MAX_SIMULATED_CLIENTS] = {NULL}; + bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3719,75 +7670,84 @@ bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_da printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i customers\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); sqlite3_free(sql); - if (comparison_result == false) goto finalize; } result = true; + rc = SQLITE_OK; finalize: for (int i=0; i 0) { - result = false; - printf("do_test_merge_null_values error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_merge error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); + if (data[i]) cloudsync_context_free(data[i]); + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3797,8 +7757,9 @@ bool do_test_merge_null_values (int nclients, bool print_result, bool cleanup_da return result; } -// Test BLOB data merge -bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_fill_initial_data(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; @@ -3812,70 +7773,87 @@ bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_data printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i BLOB test table\n"); - do_query(db[0], "SELECT id, HEX(data), description FROM blob_test ORDER BY id;", query_table); + if (table_mask & TEST_PRIKEYS) { + printf("\n-> " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + if (table_mask & TEST_NOPRIKEYS) { + printf("\n-> customers_noprikey\n"); + do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } } result = true; + rc = SQLITE_OK; finalize: for (int i=0; i 0) { result = false; - printf("do_test_merge_blob_data error: db %d has %d unterminated statements\n", i, counter); + printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); } } + if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -3885,12 +7863,10 @@ bool do_test_merge_blob_data (int nclients, bool print_result, bool cleanup_data return result; } -// Test mixed operations (INSERT/UPDATE/DELETE) in transactions -bool do_test_merge_mixed_operations (int nclients, bool print_result, bool cleanup_databases) { +bool do_test_alter(int nclients, int alter_version, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { @@ -3901,6 +7877,8 @@ bool do_test_merge_mixed_operations (int nclients, bool print_result, bool clean printf("Number of test merge increased to %d clients\n", 2); } + // create databases and tables + int table_mask = TEST_PRIKEYS | TEST_NOCOLS; time_t timestamp = time(NULL); int saved_counter = test_counter; for (int i=0; i " CUSTOMERS_TABLE "\n"); + char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); + do_query(db[0], sql, query_table); + sqlite3_free(sql); + } + if (table_mask & TEST_NOCOLS) { + printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); + do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } + if (table_mask & TEST_NOPRIKEYS) { + printf("\n-> customers_noprikey\n"); + do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); + } } result = true; + rc = SQLITE_OK; finalize: for (int i=0; i 0) { - result = false; - printf("do_test_merge_mixed_operations error: db %d has %d unterminated statements\n", i, counter); - } - } + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) printf("do_test_merge error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4013,92 +7980,238 @@ bool do_test_merge_mixed_operations (int nclients, bool print_result, bool clean return result; } -// Test hub-spoke merge pattern (central client merging with multiple peripherals) -bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - + +bool do_test_payload_buffer (size_t blob_size) { + const char *table_name = "payload_buffer_test"; + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + unsigned char *blob = NULL; + char *errmsg = NULL; + bool success = false; + int rc = sqlite3_open(":memory:", &db); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_cloudsync_init(db, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_exec(db, "SELECT cloudsync_version();", NULL, NULL, &errmsg); + if (rc != SQLITE_OK) goto cleanup; + if (errmsg) { sqlite3_free(errmsg); errmsg = NULL; } + + char *sql = sqlite3_mprintf("CREATE TABLE IF NOT EXISTS \"%w\" (" + "id TEXT PRIMARY KEY NOT NULL, " + "value BLOB, " + "created_at TEXT DEFAULT CURRENT_TIMESTAMP" + ");", table_name); + if (!sql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + sqlite3_free(sql); + if (rc != SQLITE_OK) goto cleanup; + if (errmsg) { sqlite3_free(errmsg); errmsg = NULL; } + + sql = sqlite3_mprintf("SELECT cloudsync_init('%q');", table_name); + if (!sql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + sqlite3_free(sql); + if (rc != SQLITE_OK) goto cleanup; + if (errmsg) { sqlite3_free(errmsg); errmsg = NULL; } + + sql = sqlite3_mprintf("INSERT INTO \"%w\" (id, value) VALUES (?, ?);", table_name); + if (!sql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); + sqlite3_free(sql); + if (rc != SQLITE_OK) goto cleanup; + + char dummy_id[UUID_STR_MAXLEN]; + cloudsync_uuid_v7_string(dummy_id, true); + + blob = sqlite3_malloc64(blob_size); + if (!blob) { + rc = SQLITE_NOMEM; + goto cleanup; + } + for (size_t i = 0; i < blob_size; ++i) { + blob[i] = (unsigned char)(i % 256); + } + + rc = sqlite3_bind_text(stmt, 1, dummy_id, -1, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_bind_blob(stmt, 2, blob, (int)blob_size, SQLITE_TRANSIENT); + if (rc != SQLITE_OK) goto cleanup; + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) goto cleanup; + rc = sqlite3_finalize(stmt); + stmt = NULL; + if (rc != SQLITE_OK) goto cleanup; + + sqlite3_free(blob); + blob = NULL; + + const char *payload_sql = "SELECT length(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq)) " + "FROM cloudsync_changes;"; + rc = sqlite3_prepare_v2(db, payload_sql, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int row_count = 0; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + (void)sqlite3_column_int64(stmt, 0); + row_count++; + } + if (rc != SQLITE_DONE || row_count == 0) goto cleanup; + + success = true; + +cleanup: + if (stmt) { + sqlite3_finalize(stmt); + } + if (blob) { + sqlite3_free(blob); + } + if (errmsg) { + fprintf(stderr, "do_test_android_initial_payload error: %s\n", errmsg); + sqlite3_free(errmsg); + } + if (db) close_db(db); + db = NULL; + + return success; +} + +// MARK: - Row Filter Test - + +static int64_t test_query_int(sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + int64_t value = -1; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) return -1; + if (sqlite3_step(stmt) == SQLITE_ROW) value = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + return value; +} + +bool do_test_row_filter(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - int table_mask = TEST_PRIKEYS; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 4) { - nclients = 4; - printf("Number of test merge increased to %d clients\n", 4); - } - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i %" PRId64 ")\n", before, after); + goto finalize; + } } - - // hub-spoke merge: all spokes -> hub - for (int i=1; i %" PRId64 ")\n", before, after); goto finalize; } } - - // hub inserts aggregated data - char *sql = sqlite3_mprintf("INSERT INTO \"%w\" (first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\", age, note, stamp) VALUES ('hub', 'aggregated', 50, 'hub_summary', 'hub_stamp');", CUSTOMERS_TABLE); - rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); - sqlite3_free(sql); + + // --- Test 5: Delete matching row → metadata should update (tombstone) --- + rc = sqlite3_exec(db[0], "DELETE FROM tasks WHERE id='a';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // hub -> all spokes - for (int i=1; i tasks (db[0])\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0])\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1])\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_hub_spoke error: db %d has %d unterminated statements\n", i, counter); - } - } + + result = true; + +finalize: + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4108,89 +8221,100 @@ bool do_test_merge_hub_spoke (int nclients, bool print_result, bool cleanup_data return result; } -// Test timestamp precision conflicts -bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Clear Filter - + +bool do_test_row_filter_clear(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i Timestamp precision test\n"); - do_query(db[0], "SELECT id, created_at, data FROM timestamp_test ORDER BY id, created_at;", query_table); + printf("\n-> tasks (db[0]) after clear:\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks (db[1]) after merge:\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_timestamp_precision error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_clear error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4200,88 +8324,144 @@ bool do_test_merge_timestamp_precision (int nclients, bool print_result, bool cl return result; } -// Test partial merge failure recovery -bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Complex Expressions - + +bool do_test_row_filter_complex_expressions(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i 3');", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - const char *sql2 = "INSERT INTO partial_test (id, data, value) VALUES ('valid2', 'more_data', 200);"; - rc = sqlite3_exec(db[0], sql2, NULL, NULL, NULL); + + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('a', 'active', 5, 'work', 1);", NULL, NULL, NULL); // matches if (rc != SQLITE_OK) goto finalize; - - // insert data in client 1 that might cause constraint issues during merge - const char *sql3 = "INSERT INTO partial_test (id, data, value) VALUES ('valid3', 'client1_data', 300);"; - rc = sqlite3_exec(db[1], sql3, NULL, NULL, NULL); + rc = sqlite3_exec(db[0], "INSERT INTO items VALUES('b', 'active', 2, 'work', 1);", NULL, NULL, NULL); // fails priority if (rc != SQLITE_OK) goto finalize; - - // attempt merge - should handle any constraint violations gracefully - bool merge_result = do_merge(db, nclients, false); - - // verify that databases are still in consistent state even if merge had issues - for (int i=0; i 3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Insert on db[1] matching and non-matching + rc = sqlite3_exec(db[1], "INSERT INTO items VALUES('j', 'test', 5, 'cat', 1);", NULL, NULL, NULL); // matches + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[1], "INSERT INTO items VALUES('k', 'test', 1, 'cat', 1);", NULL, NULL, NULL); // fails priority + if (rc != SQLITE_OK) goto finalize; + + // Sync db[1] -> db[0] + if (do_merge_using_payload(db[1], db[0], true, true) == false) goto finalize; + { + int64_t j_exists = test_query_int(db[0], "SELECT COUNT(*) FROM items WHERE id='j';"); + if (j_exists != 1) { + printf("do_test_row_filter_complex: Phase E expected 'j' in db[0], not found\n"); + goto finalize; + } + int64_t k_exists = test_query_int(db[0], "SELECT COUNT(*) FROM items WHERE id='k';"); + if (k_exists != 0) { + printf("do_test_row_filter_complex: Phase E expected 'k' NOT in db[0], but found\n"); + goto finalize; + } + } + + if (print_result) { + printf("\n-> items (db[0]):\n"); + do_query(db[0], "SELECT * FROM items ORDER BY id;", NULL); + printf("\n-> items (db[1]):\n"); + do_query(db[1], "SELECT * FROM items ORDER BY id;", NULL); + } + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_partial_failure error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_complex error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4291,106 +8471,132 @@ bool do_test_merge_partial_failure (int nclients, bool print_result, bool cleanu return result; } -// Test transaction rollback scenarios -bool do_test_merge_rollback_scenarios (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Row Transition - + +bool do_test_row_filter_row_transition(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i non-matching) --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('a', 'Task A', 1);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - const char *sql2 = "INSERT INTO rollback_test (id, data, status) VALUES ('tx2', 'more_tx_data', 2);"; - rc = sqlite3_exec(db[0], sql2, NULL, NULL, NULL); + + { + int64_t meta_before = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + // Update row to no longer match filter + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 2 WHERE id = 'a';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t meta_after = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + // UPDATE trigger should NOT fire (NEW.user_id=2 fails WHEN clause) + if (meta_after != meta_before) { + printf("do_test_row_filter_row_transition: Phase A - UPDATE out of filter changed meta (%" PRId64 " -> %" PRId64 ")\n", meta_before, meta_after); + goto finalize; + } + } + + // Sync to db[1]: payload reads current row values, so 'a' arrives with user_id=2 + // (the INSERT metadata triggers the sync, but the payload carries current column values) + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t a_uid = test_query_int(db[1], "SELECT user_id FROM tasks WHERE id='a';"); + if (a_uid != 2) { + printf("do_test_row_filter_row_transition: Phase A - db[1] expected user_id=2 for 'a', got %" PRId64 "\n", a_uid); + goto finalize; + } + } + + // --- Phase B: Row enters filter (non-matching -> matching) --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('b', 'Task B', 2);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // rollback transaction - rc = sqlite3_exec(db[0], "ROLLBACK;", NULL, NULL, NULL); + + { + int64_t meta_before = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + // Update row to now match filter + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 1 WHERE id = 'b';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t meta_after = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + // UPDATE trigger should fire (NEW.user_id=1 passes WHEN clause) + if (meta_after <= meta_before) { + printf("do_test_row_filter_row_transition: Phase B - UPDATE into filter did not create metadata (%" PRId64 " -> %" PRId64 ")\n", meta_before, meta_after); + goto finalize; + } + } + + // Sync to db[1]: should now get 'b' with user_id=1 + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t b_exists = test_query_int(db[1], "SELECT COUNT(*) FROM tasks WHERE id='b' AND user_id=1;"); + if (b_exists != 1) { + printf("do_test_row_filter_row_transition: Phase B - db[1] expected 'b' with user_id=1, not found\n"); + goto finalize; + } + } + + // --- Phase C: Bidirectional with transition --- + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('c', 'Task C', 1);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // insert committed data in client 0 - const char *sql3 = "INSERT INTO rollback_test (id, data, status) VALUES ('committed', 'final_data', 10);"; - rc = sqlite3_exec(db[0], sql3, NULL, NULL, NULL); + // Sync 'c' to db[1] + if (do_merge_using_payload(db[0], db[1], true, true) == false) goto finalize; + { + int64_t c_exists = test_query_int(db[1], "SELECT COUNT(*) FROM tasks WHERE id='c';"); + if (c_exists != 1) { + printf("do_test_row_filter_row_transition: Phase C - db[1] should have 'c'\n"); + goto finalize; + } + } + + // db[0]: move 'c' out of filter (no metadata generated) + rc = sqlite3_exec(db[0], "UPDATE tasks SET user_id = 2 WHERE id = 'c';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // insert different data in client 1 - const char *sql4 = "INSERT INTO rollback_test (id, data, status) VALUES ('client1', 'different_data', 20);"; - rc = sqlite3_exec(db[1], sql4, NULL, NULL, NULL); + + // db[1]: update 'c' title (still matching on db[1], metadata generated) + rc = sqlite3_exec(db[1], "UPDATE tasks SET title = 'Task C Edited' WHERE id = 'c';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - - // verify rolled back data is not present, then merge - sqlite3_stmt *stmt; - rc = sqlite3_prepare_v2(db[0], "SELECT COUNT(*) FROM rollback_test WHERE id IN ('tx1', 'tx2');", -1, &stmt, NULL); - if (rc == SQLITE_OK) { - if (sqlite3_step(stmt) == SQLITE_ROW) { - int rollback_count = sqlite3_column_int(stmt, 0); - if (rollback_count > 0) { - printf("Rollback failed - found %d rolled back records\n", rollback_count); - sqlite3_finalize(stmt); - goto finalize; - } + + // Sync db[1] -> db[0] + if (do_merge_using_payload(db[1], db[0], true, true) == false) goto finalize; + { + // db[0] should receive the title update from db[1] + int64_t c_title_match = test_query_int(db[0], "SELECT COUNT(*) FROM tasks WHERE id='c' AND title='Task C Edited';"); + if (c_title_match != 1) { + printf("do_test_row_filter_row_transition: Phase C - db[0] should have updated title for 'c'\n"); + goto finalize; } - sqlite3_finalize(stmt); - } - - // merge changes - if (do_merge(db, nclients, false) == false) { - goto finalize; } - - // verify consistency - only committed data should be present - for (int i=1; i tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_rollback_scenarios error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_row_transition error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4400,92 +8606,118 @@ bool do_test_merge_rollback_scenarios (int nclients, bool print_result, bool cle return result; } -// Test circular merge (A->B->C->A) -bool do_test_merge_circular (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Filter Change - + +bool do_test_row_filter_change(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); - } - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i1->2->0 - if (do_merge_using_payload(db[0], db[1], false, true) == false) { - goto finalize; - } - - if (do_merge_using_payload(db[1], db[2], false, true) == false) { - goto finalize; + + // Set initial filter on db[0] + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_filter('tasks', 'user_id = 1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Insert rows under initial filter + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('a', 'Task A', 1);", NULL, NULL, NULL); // matches + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('b', 'Task B', 2);", NULL, NULL, NULL); // non-matching + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('c', 'Task C', 1);", NULL, NULL, NULL); // matches + if (rc != SQLITE_OK) goto finalize; + + // Test 1: 2 PKs tracked under initial filter + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 2) { + printf("do_test_row_filter_change: expected 2 PKs under initial filter, got %" PRId64 "\n", meta_count); + goto finalize; + } } - - if (do_merge_using_payload(db[2], db[0], false, true) == false) { - goto finalize; + + // Change filter to user_id = 2 + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_filter('tasks', 'user_id = 2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + + // Insert rows under new filter + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('d', 'Task D', 2);", NULL, NULL, NULL); // matches new + if (rc != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES('e', 'Task E', 1);", NULL, NULL, NULL); // non-matching under new + if (rc != SQLITE_OK) goto finalize; + + // Test 2: set_filter('user_id = 2') reset metatable — only 'b' (user_id=2) from refill + 'd' from insert = 2 + { + int64_t meta_count = test_query_int(db[0], "SELECT COUNT(DISTINCT pk) FROM tasks_cloudsync;"); + if (meta_count != 2) { + printf("do_test_row_filter_change: expected 2 PKs after filter change, got %" PRId64 "\n", meta_count); + goto finalize; + } } - - // complete the circle and ensure consistency - if (do_merge(db, nclients, false) == false) { - goto finalize; + + // Test 3: Update row 'a' (user_id=1) — does NOT match new filter (user_id=2) + { + int64_t before = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + rc = sqlite3_exec(db[0], "UPDATE tasks SET title='Task A Updated' WHERE id='a';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + int64_t after = test_query_int(db[0], "SELECT COUNT(*) FROM tasks_cloudsync;"); + if (after != before) { + printf("do_test_row_filter_change: update on 'a' should not change meta under new filter (%" PRId64 " -> %" PRId64 ")\n", before, after); + goto finalize; + } } - - // verify all clients have all data - for (int i=1; i Circular merge result\n"); - do_query(db[0], "SELECT * FROM circular_test ORDER BY origin_client;", query_table); + printf("\n-> tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0]):\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_circular error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_change error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4495,124 +8727,135 @@ bool do_test_merge_circular (int nclients, bool print_result, bool cleanup_datab return result; } -// Test foreign key constraints during merge -bool do_test_merge_foreign_keys (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Composite PK + Multi-Table - + +bool do_test_row_filter_composite_pk_multi_table(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i projects (db[0]):\n"); + do_query(db[0], "SELECT * FROM projects ORDER BY org_id, proj_id;", NULL); + printf("\n-> members (db[0]):\n"); + do_query(db[0], "SELECT * FROM members ORDER BY org_id, user_id;", NULL); + printf("\n-> projects (db[1]):\n"); + do_query(db[1], "SELECT * FROM projects ORDER BY org_id, proj_id;", NULL); + printf("\n-> members (db[1]):\n"); + do_query(db[1], "SELECT * FROM members ORDER BY org_id, user_id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_foreign_keys error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_composite error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4622,107 +8865,121 @@ bool do_test_merge_foreign_keys (int nclients, bool print_result, bool cleanup_d return result; } -// Test trigger interaction during merge -// Expected failure: TRIGGERs are not fully supported by this extension. -bool do_test_merge_triggers (int nclients, bool print_result, bool cleanup_databases) { +// MARK: - Row Filter: Pre-existing Data (Prefill) - + +bool do_test_row_filter_prefill(int nclients, bool print_result, bool cleanup_databases) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - + if (nclients >= MAX_SIMULATED_CLIENTS) nclients = MAX_SIMULATED_CLIENTS; + if (nclients < 2) nclients = 2; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i Trigger test results\n"); - do_query(db[0], "SELECT id, data, update_count FROM trigger_test ORDER BY id;", query_table); - printf("\n-> Audit log\n"); - do_query(db[0], "SELECT COUNT(*) as audit_entries FROM audit_log;", query_table); + printf("\n-> tasks (db[0]):\n"); + do_query(db[0], "SELECT * FROM tasks ORDER BY id;", NULL); + printf("\n-> tasks_cloudsync (db[0]):\n"); + do_query(db[0], "SELECT hex(pk), col_name, col_version, db_version FROM tasks_cloudsync ORDER BY pk, col_name;", NULL); + printf("\n-> tasks (db[1]):\n"); + do_query(db[1], "SELECT * FROM tasks ORDER BY id;", NULL); } - + result = true; - + finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_triggers error: db %d has %d unterminated statements\n", i, counter); - } - } + for (int i = 0; i < nclients; ++i) { + if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) + printf("do_test_row_filter_prefill error: %s\n", sqlite3_errmsg(db[i])); + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -4732,131 +8989,218 @@ bool do_test_merge_triggers (int nclients, bool print_result, bool cleanup_datab return result; } -// Test index consistency during merge -bool do_test_merge_index_consistency (int nclients, bool print_result, bool cleanup_databases) { +// Test that BEFORE triggers with RAISE(ABORT) simulate RLS denial: +// per-PK savepoints isolate failures so allowed rows commit and denied rows roll back. +bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; int rc = SQLITE_OK; - + memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); if (nclients >= MAX_SIMULATED_CLIENTS) { nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); } else if (nclients < 2) { nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); } - + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i 0) { result = false; - printf("do_test_merge_index_consistency error: db %d has %d unterminated statements\n", i, counter); + printf("do_test_rls_trigger_denial error: db %d has %d unterminated statements\n", i, counter); } } if (cleanup_databases) { @@ -4868,722 +9212,2725 @@ bool do_test_merge_index_consistency (int nclients, bool print_result, bool clea return result; } -// Test JSON column merge -bool do_test_merge_json_columns (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +// MARK: - Block-level LWW Tests - + +static int64_t do_select_int(sqlite3 *db, const char *sql) { + sqlite3_stmt *stmt = NULL; + int64_t val = -1; + if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + val = sqlite3_column_int64(stmt, 0); + } } - + if (stmt) sqlite3_finalize(stmt); + return val; +} + +// Regression: persisted block-column settings must not cause an infinite loop +// when the extension is reloaded. Without the fix, dbutils_settings_table_load_callback +// (driven by sqlite3_exec over cloudsync_table_settings) would re-invoke +// cloudsync_setup_block_column, which REPLACEd the same row mid-iteration and +// re-fed it back into the cursor. +bool do_test_block_column_reload(bool cleanup_databases) { + bool result = false; + char dbpath[256]; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i JSON test results\n"); - do_query(db[0], "SELECT id, json_extract(metadata, '$.name') as name, json_extract(metadata, '$.preferences.theme') as theme FROM json_test ORDER BY id;", query_table); + if (stmt) sqlite3_finalize(stmt); + return val; +} + +// Test: enabling block-level LWW on a table that already contains rows migrates +// existing data into the blocks table so it is not silently ignored. +// Verifies: +// 1. Blocks table is populated for pre-existing rows after set_column('algo','block'). +// 2. A subsequent UPDATE correctly diffs against the migrated blocks, not from empty. +// 3. The migration is idempotent (calling set_column twice does not double the blocks). +bool do_test_block_lww_existing_data(bool cleanup_databases) { + sqlite3 *db = NULL; + bool result = false; + char dbpath[256]; + time_t timestamp = time(NULL); + + #ifdef __ANDROID__ + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", ".", timestamp); + #else + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", getenv("HOME"), timestamp); + #endif + + int rc = sqlite3_open(dbpath, &db); + if (rc != SQLITE_OK) return false; + sqlite3_exec(db, "PRAGMA journal_mode=WAL;", NULL, NULL, NULL); + sqlite3_cloudsync_init(db, NULL, NULL); + + // Create table and init sync BEFORE enabling block-level LWW + rc = sqlite3_exec(db, "CREATE TABLE docs (id TEXT PRIMARY KEY NOT NULL, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Insert rows via the sync layer (populates regular column metadata) + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('d1', 'line1\nline2\nline3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('d2', 'alpha\nbeta');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + // Sanity: no blocks table yet + int64_t blocks_table_before = do_select_int(db, + "SELECT count(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_table_before != 0) { + printf("block_existing_data: blocks table should not exist before set_column\n"); + goto cleanup; + } + + // NOW enable block-level LWW — this should migrate the existing rows + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_existing_data: set_column failed: %s\n", sqlite3_errmsg(db)); goto cleanup; } + + // Verify blocks table was created and is populated + int64_t blocks_table_after = do_select_int(db, + "SELECT count(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_table_after != 1) { + printf("block_existing_data: blocks table not created after set_column\n"); + goto cleanup; + } + + // d1 has 3 lines → 3 block entries; d2 has 2 lines → 2 block entries = 5 total + int64_t block_count = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (block_count != 5) { + printf("block_existing_data: expected 5 block entries after migration, got %" PRId64 "\n", block_count); + goto cleanup; + } + + // Metadata must contain the migrated block entries (alive, odd col_version) + int64_t meta_block_count = do_select_int(db, + "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body\x1f%' AND col_version % 2 = 1;"); + if (meta_block_count != 5) { + printf("block_existing_data: expected 5 block metadata entries, got %" PRId64 "\n", meta_block_count); + goto cleanup; + } + + // UPDATE d1: change one line — diff should reuse existing block positions + rc = sqlite3_exec(db, "UPDATE docs SET body='line1\nMODIFIED\nline3' WHERE id='d1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_existing_data: UPDATE failed: %s\n", sqlite3_errmsg(db)); goto cleanup; } + + // After the update: total alive blocks is still 5 (d1: 3, d2: 2) + int64_t alive_after_update = do_select_int(db, + "SELECT count(*) FROM docs_cloudsync_blocks b " + "JOIN docs_cloudsync m ON b.pk = m.pk AND b.col_name = m.col_name " + "WHERE b.col_name LIKE 'body\x1f%' AND m.col_version % 2 = 1;"); + if (alive_after_update != 5) { + printf("block_existing_data: expected 5 alive blocks after update, got %" PRId64 "\n", alive_after_update); + goto cleanup; + } + + // Verify materialized value is correct after update + char *val = do_select_text(db, "SELECT body FROM docs WHERE id='d1';"); + if (!val || strcmp(val, "line1\nMODIFIED\nline3") != 0) { + printf("block_existing_data: unexpected body after update: '%s'\n", val ? val : "(null)"); + if (val) sqlite3_free(val); + goto cleanup; + } + sqlite3_free(val); + + // Idempotency: calling set_column again must not duplicate block entries + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto cleanup; + + int64_t block_count_after_second_call = do_select_int(db, "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (block_count_after_second_call != block_count) { + printf("block_existing_data: idempotency broken — block count changed from %" PRId64 " to %" PRId64 " on second set_column\n", + block_count, block_count_after_second_call); + goto cleanup; + } + + result = true; + +cleanup: + if (db) { + sqlite3_exec(db, "SELECT cloudsync_terminate();", NULL, NULL, NULL); + sqlite3_close(db); + } + if (cleanup_databases) { + file_delete_internal(dbpath); + char walpath[280]; + snprintf(walpath, sizeof(walpath), "%s-wal", dbpath); + file_delete_internal(walpath); + snprintf(walpath, sizeof(walpath), "%s-shm", dbpath); + file_delete_internal(walpath); + } + return result; +} + +bool do_test_block_lww_insert(int nclients, bool print_result, bool cleanup_databases) { + // Test: INSERT into a table with a block column properly splits text into blocks + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: CREATE TABLE failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: cloudsync_init failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: set_column failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + } + + // Insert a document with 3 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line 1\nLine 2\nLine 3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_insert: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks were created in the blocks table + int64_t block_count = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (block_count != 3) { + printf("block_insert: expected 3 blocks, got %" PRId64 "\n", block_count); + goto fail; + } + + // Verify metadata entries for blocks (col_name contains \x1F) + int64_t meta_count = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_count != 3) { + printf("block_insert: expected 3 block metadata entries, got %" PRId64 "\n", meta_count); + goto fail; } - - result = true; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge_json_columns error: db %d has %d unterminated statements\n", i, counter); - } + + // Verify no metadata entry for the whole 'body' column + int64_t whole_meta = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name = 'body';"); + if (whole_meta != 0) { + printf("block_insert: expected 0 whole-column metadata entries, got %" PRId64 "\n", whole_meta); + goto fail; + } + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_update(int nclients, bool print_result, bool cleanup_databases) { + // Test: UPDATE on a block column performs block diff + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert initial text + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'AAA\nBBB\nCCC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_update: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + + // Update: change middle line and add a new line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'AAA\nXXX\nCCC\nDDD' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_update: UPDATE failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + int64_t blocks_after = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + + // Should have 4 blocks after update (AAA, XXX, CCC, DDD) + if (blocks_after != 4) { + printf("block_update: expected 4 blocks after update, got %" PRId64 " (before: %" PRId64 ")\n", blocks_after, blocks_before); + goto fail; + } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_sync(int nclients, bool print_result, bool cleanup_databases) { + // Test: Two sites edit different blocks of the same document; after sync, both edits are preserved + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts the initial document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line A\nLine B\nLine C');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: INSERT db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync initial state: db[0] -> db[1] so both have the same document + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("block_sync: initial merge 0->1 failed\n"); goto fail; } + + // Site 0: edit first line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'EDITED A\nLine B\nLine C' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: UPDATE db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Site 1: edit third line + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line A\nLine B\nEDITED C' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_sync: UPDATE db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + // Sync: db[0] -> db[1] (send site 0's edits) + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("block_sync: merge 0->1 failed\n"); goto fail; } + // Sync: db[1] -> db[0] (send site 1's edits) + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("block_sync: merge 1->0 failed\n"); goto fail; } + + // Both databases should now have the merged result: "EDITED A\nLine B\nEDITED C" + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_sync: could not read body from one or both databases\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_sync: bodies don't match after sync:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // Check that both edits were preserved + if (!strstr(body0, "EDITED A")) { + printf("block_sync: missing 'EDITED A' in result: %s\n", body0); + ok = false; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + if (!strstr(body0, "EDITED C")) { + printf("block_sync: missing 'EDITED C' in result: %s\n", body0); + ok = false; + } + if (!strstr(body0, "Line B")) { + printf("block_sync: missing 'Line B' in result: %s\n", body0); + ok = false; } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test concurrent merge attempts -bool do_test_merge_concurrent_attempts (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 3) { - nclients = 3; - printf("Number of test merge increased to %d clients\n", 3); +bool do_test_block_lww_delete(int nclients, bool print_result, bool cleanup_databases) { + // Test: DELETE on a row with block columns marks tombstone and block metadata is dropped + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert a document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line A\nLine B\nLine C');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_delete: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks and metadata exist + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks_before != 3) { + printf("block_delete: expected 3 blocks before delete, got %" PRId64 "\n", blocks_before); + goto fail; } - + int64_t meta_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_before != 3) { + printf("block_delete: expected 3 block metadata before delete, got %" PRId64 "\n", meta_before); + goto fail; + } + + // Delete the row + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_delete: DELETE failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify metadata tombstone exists (delete sentinel) + int64_t tombstone = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name = '__[RIP]__' AND col_version % 2 = 0;"); + if (tombstone != 1) { + printf("block_delete: expected 1 delete tombstone, got %" PRId64 "\n", tombstone); + goto fail; + } + + // Verify block metadata was dropped (local_drop_meta removes non-tombstone metadata) + int64_t meta_after = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_after != 0) { + printf("block_delete: expected 0 block metadata after delete, got %" PRId64 "\n", meta_after); + goto fail; + } + + // Row should be gone from base table + int64_t row_count = do_select_int(db[0], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + if (row_count != 0) { + printf("block_delete: row still in base table after delete\n"); + goto fail; + } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_materialize(int nclients, bool print_result, bool cleanup_databases) { + // Test: cloudsync_text_materialize reconstructs text from blocks after sync + // Sync to a second db where body column is empty, then materialize there + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i1 and 1->2 simultaneously - bool merge1 = do_merge_using_payload(db[0], db[1], false, true); - bool merge2 = do_merge_using_payload(db[1], db[2], false, true); - - if (!merge1 || !merge2) { - printf("Concurrent merge phase 1 had issues\n"); + if (strcmp(body, "Alpha\nBravo\nCharlie\nDelta\nEcho") != 0) { + printf("block_materialize: body mismatch: %s\n", body); + sqlite3_free(body); + goto fail; } - - // second: 2->0 and 0->1 simultaneously - bool merge3 = do_merge_using_payload(db[2], db[0], false, true); - bool merge4 = do_merge_using_payload(db[0], db[1], false, true); - - if (!merge3 || !merge4) { - printf("Concurrent merge phase 2 had issues\n"); + sqlite3_free(body); + + // Also test materialize on db[0] (where body already matches) + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_materialize: materialize on db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body0 || strcmp(body0, "Alpha\nBravo\nCharlie\nDelta\nEcho") != 0) { + printf("block_materialize: body0 mismatch: %s\n", body0 ? body0 : "NULL"); + if (body0) sqlite3_free(body0); + goto fail; } - - // final consistency merge - if (do_merge(db, nclients, false) == false) { - printf("Final merge failed\n"); + sqlite3_free(body0); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_empty_text(int nclients, bool print_result, bool cleanup_databases) { + // Test: INSERT with empty body creates a single empty block + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert empty text + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', '');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_empty: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Should have exactly 1 block (empty content) + int64_t block_count = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (block_count != 1) { + printf("block_empty: expected 1 block for empty text, got %" PRId64 "\n", block_count); + goto fail; } - - // verify all clients have same data count - int expected_count = nclients * 5; // each client inserted 5 records - for (int i=0; i 0) { - result = false; - printf("do_test_merge_concurrent_attempts error: db %d has %d unterminated statements\n", i, counter); - } + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_conflict(int nclients, bool print_result, bool cleanup_databases) { + // Test: Two sites edit the SAME line concurrently; LWW picks the later write + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial document + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Same\nMiddle\nEnd');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: INSERT db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync initial state: db[0] -> db[1] + if (!do_merge_values(db[0], db[1], false)) { printf("block_conflict: initial merge failed\n"); goto fail; } + + // Site 0: edit first line + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Site0\nMiddle\nEnd' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: UPDATE db[0] failed\n"); goto fail; } + + // Site 1: also edit first line (conflict!) + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Site1\nMiddle\nEnd' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: UPDATE db[1] failed\n"); goto fail; } + + // Sync both ways using row-by-row merge + if (!do_merge_values(db[0], db[1], true)) { printf("block_conflict: merge 0->1 failed\n"); goto fail; } + if (!do_merge_values(db[1], db[0], true)) { printf("block_conflict: merge 1->0 failed\n"); goto fail; } + + // Materialize on both databases to reconstruct body from blocks + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: materialize db[0] failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_conflict: materialize db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + // Both databases should converge (same value) + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_conflict: could not read body from databases\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_conflict: bodies don't match after sync:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // Should contain either "Site0" or "Site1" (LWW picks one), plus unchanged lines + if (!strstr(body0, "Middle")) { + printf("block_conflict: missing 'Middle' in result: %s\n", body0); + ok = false; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter++); - file_delete_internal(buf); + if (!strstr(body0, "End")) { + printf("block_conflict: missing 'End' in result: %s\n", body0); + ok = false; + } + // One of the conflicting edits should win + if (!strstr(body0, "Site0") && !strstr(body0, "Site1")) { + printf("block_conflict: neither 'Site0' nor 'Site1' in result: %s\n", body0); + ok = false; } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_multi_update(int nclients, bool print_result, bool cleanup_databases) { + // Test: Multiple successive updates correctly maintain block state + sqlite3 *db[1] = {NULL}; + time_t timestamp = time(NULL); + int rc; + + db[0] = do_create_database_file(0, timestamp, test_counter++); + if (!db[0]) return false; + + rc = sqlite3_exec(db[0], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Insert initial text (3 lines) + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB\nC');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: INSERT failed\n"); goto fail; } + + // Update 1: remove middle line (3 -> 2 blocks) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nC' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 1 failed\n"); goto fail; } + + int64_t blocks1 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks1 != 2) { printf("block_multi: expected 2 blocks after update 1, got %" PRId64 "\n", blocks1); goto fail; } + + // Update 2: add two lines (2 -> 4 blocks) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nX\nC\nY' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 2 failed\n"); goto fail; } + + int64_t blocks2 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks2 != 4) { printf("block_multi: expected 4 blocks after update 2, got %" PRId64 "\n", blocks2); goto fail; } + + // Update 3: change everything to a single line (4 -> 1 block) + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'SINGLE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: UPDATE 3 failed\n"); goto fail; } + + int64_t blocks3 = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks;"); + if (blocks3 != 1) { printf("block_multi: expected 1 block after update 3, got %" PRId64 "\n", blocks3); goto fail; } + + // Materialize and verify + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_multi: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "SINGLE") != 0) { + printf("block_multi: expected 'SINGLE', got '%s'\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + close_db(db[0]); + return true; + +fail: + if (db[0]) close_db(db[0]); + return false; +} + +bool do_test_block_lww_reinsert(int nclients, bool print_result, bool cleanup_databases) { + // Test: DELETE then re-INSERT recreates blocks properly + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert, delete, then re-insert with different content + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Old1\nOld2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_reinsert: initial INSERT failed\n"); goto fail; } + + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_reinsert: DELETE failed\n"); goto fail; } + + // Block metadata should be dropped (blocks table entries are orphaned by design) + int64_t meta_after_del = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_after_del != 0) { + printf("block_reinsert: expected 0 block metadata after delete, got %" PRId64 "\n", meta_after_del); + goto fail; + } + + // Re-insert with new content + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'New1\nNew2\nNew3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_reinsert: re-INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Check block metadata was recreated (3 new block entries) + int64_t meta_after_reinsert = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (meta_after_reinsert != 3) { + printf("block_reinsert: expected 3 block metadata after re-insert, got %" PRId64 "\n", meta_after_reinsert); + goto fail; + } + + // Sync to db[1] and verify + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("block_reinsert: merge failed\n"); goto fail; } + + // Materialize on db[1] + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_reinsert: materialize on db[1] failed: %s\n", sqlite3_errmsg(db[1])); goto fail; } + + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body1 || strcmp(body1, "New1\nNew2\nNew3") != 0) { + printf("block_reinsert: body mismatch on db[1]: %s\n", body1 ? body1 : "NULL"); + if (body1) sqlite3_free(body1); + goto fail; + } + sqlite3_free(body1); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +bool do_test_block_lww_add_lines(int nclients, bool print_result, bool cleanup_databases) { + // Test: Both sites add lines at different positions; after sync, all lines are present + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync initial: 0 -> 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) goto fail; + + // Site 0: append a line at the end + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Line1\nLine2\nAppended0' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: insert a line in the middle + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nInserted1\nLine2' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_using_payload(db[0], db[1], true, true)) goto fail; + if (!do_merge_using_payload(db[1], db[0], true, true)) goto fail; + + // Both should converge + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1) { + printf("block_add_lines: could not read body\n"); + ok = false; + } else if (strcmp(body0, body1) != 0) { + printf("block_add_lines: bodies don't match:\n db[0]: %s\n db[1]: %s\n", body0, body1); + ok = false; + } else { + // All original and added lines should be present + if (!strstr(body0, "Line1")) { printf("block_add_lines: missing Line1\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("block_add_lines: missing Line2\n"); ok = false; } + if (!strstr(body0, "Appended0")) { printf("block_add_lines: missing Appended0\n"); ok = false; } + if (!strstr(body0, "Inserted1")) { printf("block_add_lines: missing Inserted1\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -// Test with 10 clients using a simple table with composite primary key -// Summary of the test function: -// 1. Simple table with composite primary key: Creates a table with columns (id1, id2) as the composite primary key and (data1, data2) as non-primary key columns -// 2. 10 database simulation: Forces exactly 10 clients regardless of the parameter passed -// 3. CloudSync initialization: Each database initializes the cloudsync library for the test table -// 4. Initial data: Each client inserts 3 initial rows with unique data -// 5. Multiple operation rounds: Performs 5 rounds of operations where each client performs: -// - Inserts new rows (20%) -// - Updates local data (15%) - data inserted by the same client -// - Updates remote data (50%) - data inserted by other clients, tests cross-client conflicts -// - Deletes both local and remote data (15%) - minimal deletions to preserve data for updates -// 6. Partial merging: After each round, clients merge with only 2-3 other clients (not all), simulating real-world scenarios where not all clients sync immediately -// 7. Final convergence: -// - Merges all changes from all clients to the first client (client 0) -// - Propagates the final state from client 0 to all other clients -// 8. Verification: Compares all databases to ensure they have converged to identical final states -bool do_test_merge_composite_pk_10_clients (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - const char *table_name = "simple_composite_test"; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); +// Test 1: Non-conflicting edits on different blocks — both edits preserved +bool do_test_block_lww_noconflict(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0 inserts initial document with 3 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync initial: 0 -> 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit first line only + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'EditedByA\nLine2\nLine3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit third line only (no conflict — different block) + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nLine2\nEditedByB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize on both + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("noconflict: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // BOTH edits should be preserved (this is the key value of block-level LWW) + if (!strstr(body0, "EditedByA")) { printf("noconflict: missing EditedByA\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("noconflict: missing Line2\n"); ok = false; } + if (!strstr(body0, "EditedByB")) { printf("noconflict: missing EditedByB\n"); ok = false; } } - - // Force to 10 clients for this specific test - nclients = 10; - - // create databases and tables + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 2: Concurrent add + edit — Site A adds a line, Site B modifies an existing line +bool do_test_block_lww_add_and_edit(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter++; - for (int i = 0; i < nclients; ++i) { - db[i] = do_create_database_file_v2(i, timestamp, saved_counter, true); - if (db[i] == false) return false; - - // Create simple table with composite primary key (id1, id2) and two non-pk columns (data1, data2) - char *sql = sqlite3_mprintf("CREATE TABLE %s (id1 INTEGER NOT NULL, id2 TEXT NOT NULL, data1 TEXT, data2 INTEGER, PRIMARY KEY(id1, id2));", table_name); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - - // Initialize cloudsync for the table - sql = sqlite3_mprintf("SELECT cloudsync_init('%s', 'cls', 1);", table_name); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Alpha\nBravo');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: add a new line at the end + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Alpha\nBravo\nCharlie' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: modify first line + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'AlphaEdited\nBravo' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("add_and_edit: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // The added line and the edit should both be present + if (!strstr(body0, "Charlie")) { printf("add_and_edit: missing Charlie (added line)\n"); ok = false; } + if (!strstr(body0, "Bravo")) { printf("add_and_edit: missing Bravo\n"); ok = false; } + // First line: either AlphaEdited wins (from site 1) or Alpha (from site 0) — depends on LWW + // But the added line Charlie must survive regardless } - - if (print_result) printf("Created %d databases with composite primary key table '%s'\n", nclients, table_name); - - // Insert initial data in each client - for (int i = 0; i < nclients; ++i) { - for (int j = 0; j < 3; ++j) { - char *sql = sqlite3_mprintf("INSERT INTO %s (id1, id2, data1, data2) VALUES (%d, 'client%d_row%d', 'initial_data_%d_%d', %d);", - table_name, i * 100 + j, i, j, i, j, i * 10 + j); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) goto finalize; - } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 3: Three-way sync — 3 databases with overlapping edits converge +bool do_test_block_lww_three_way(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[3] = {NULL, NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 3; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - if (print_result) printf("Inserted initial data in all clients\n"); - - // Perform various operations across clients in multiple rounds - for (int round = 0; round < 20; ++round) { - if (print_result) printf("Starting round %d of operations\n", round + 1); - - // Each client performs different operations (favoring remote data operations for better conflict testing) - for (int i = 0; i < nclients; ++i) { - // Weighted operation selection: focus on updates, minimal deletes - int operation_selector = (i * 7 + round * 3) % 20; - int operation; - - if (operation_selector < 4) { - operation = 0; // INSERT (20%) - } else if (operation_selector < 7) { - operation = 1; // UPDATE local data (15%) - } else if (operation_selector < 17) { - operation = 2; // UPDATE remote data (50%) - } else { - operation = 3; // DELETE (15% - reduced from 30%) - } - - switch (operation) { - case 0: // INSERT - { - char *sql = sqlite3_mprintf("INSERT INTO %s (id1, id2, data1, data2) VALUES (%d, 'round%d_client%d', 'new_data_%d_%d', %d);", - table_name, 1000 + round * 100 + i, round, i, round, i, round * 100 + i); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK && rc != SQLITE_CONSTRAINT) { - printf("Insert failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } - } - break; - - case 1: // UPDATE local data (more aggressive targeting) - { - // Try multiple approaches to find rows to update - char *sql; - int updated_rows = 0; - - // First try: update rows originally from this client - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_local_round%d_client%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", - table_name, round, i, round * 1000 + i, i * 100, (i + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - - // Second try: if no rows updated, target any available rows - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_local_round%d_client%d_any', data2 = %d WHERE rowid = (SELECT rowid FROM %s LIMIT 1 OFFSET %d);", - table_name, round, i, round * 1000 + i, table_name, i % 10); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - if (rc != SQLITE_OK) { - printf("Update local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } else if (print_result && updated_rows > 0) { - printf("Client %d updated %d local rows in round %d\n", i, updated_rows, round + 1); - } - } - break; - - case 2: // UPDATE remote data (more aggressive - creates conflicts) - { - int target_client = (i + 1 + round) % nclients; // Vary target client - char *sql; - int updated_rows = 0; - - // First try: update rows from target client's range - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_targeting%d', data2 = %d WHERE id1 BETWEEN %d AND %d;", - table_name, round, i, target_client, round * 2000 + i, target_client * 100, (target_client + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - - // Second try: if no rows updated, target any rows not from current client - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_any', data2 = %d WHERE id1 NOT BETWEEN %d AND %d LIMIT 2;", - table_name, round, i, round * 2000 + i, i * 100, (i + 1) * 100 - 1); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - // Third try: if still no updates, target any available rows - if (updated_rows == 0) { - sql = sqlite3_mprintf("UPDATE %s SET data1 = 'updated_remote_round%d_client%d_fallback', data2 = %d WHERE rowid IN (SELECT rowid FROM %s LIMIT 2 OFFSET %d);", - table_name, round, i, round * 2000 + i, table_name, (i + round) % 10); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - updated_rows += sqlite3_changes(db[i]); - } - - if (rc != SQLITE_OK) { - printf("Update remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); - } else if (print_result && updated_rows > 0) { - printf("Client %d updated %d remote rows in round %d\n", i, updated_rows, round + 1); - } - } - break; - - case 3: // DELETE (favoring remote data deletion) - { - // 80% chance to delete remote data, 20% chance to delete local data - if ((i + round) % 5 == 0) { - // Delete local data (20% of delete operations) - char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row1';", - table_name, i * 100 + 1, i); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) { - printf("Delete local data failed on client %d: %s\n", i, sqlite3_errmsg(db[i])); - } - } else { - // Delete remote data (80% of delete operations) - int target_client = (i + 2 + round) % nclients; // Vary target client - char *sql = sqlite3_mprintf("DELETE FROM %s WHERE id1 = %d AND id2 = 'client%d_row0';", - table_name, target_client * 100, target_client); - rc = sqlite3_exec(db[i], sql, NULL, NULL, NULL); - sqlite3_free(sql); - if (rc != SQLITE_OK) { - printf("Delete remote data failed on client %d targeting client %d: %s\n", i, target_client, sqlite3_errmsg(db[i])); - } - } - } - break; - } - } - - // Partial merge: each client merges with 2-3 other random clients (not all) - for (int i = 0; i < nclients; ++i) { - int merge_targets = 2 + (round % 2); // 2 or 3 targets - for (int j = 0; j < merge_targets; ++j) { - int target = (i + j + 1 + round) % nclients; - if (target != i) { - if (do_merge_using_payload(db[i], db[target], true, true) == false) { - if (print_result) printf("Partial merge failed from client %d to %d\n", i, target); - goto finalize; - } - } - } + + // Site 0 creates initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'L1\nL2\nL3\nL4');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync 0 -> 1, 0 -> 2 + if (!do_merge_values(db[0], db[1], false)) goto fail; + if (!do_merge_values(db[0], db[2], false)) goto fail; + + // Site 0: edit line 1 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'S0\nL2\nL3\nL4' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit line 2 + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'L1\nS1\nL3\nL4' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 2: edit line 4 + rc = sqlite3_exec(db[2], "UPDATE docs SET body = 'L1\nL2\nL3\nS2' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Full mesh sync: each site sends to every other site + for (int src = 0; src < 3; src++) { + for (int dst = 0; dst < 3; dst++) { + if (src == dst) continue; + if (!do_merge_values(db[src], db[dst], true)) { printf("three_way: merge %d->%d failed\n", src, dst); goto fail; } } - - if (print_result) printf("Completed round %d operations and partial merges\n", round + 1); } - - // Final convergence: merge all changes to client 0 - if (print_result) printf("Starting final convergence to client 0\n"); - for (int i = 1; i < nclients; ++i) { - if (do_merge_using_payload(db[i], db[0], true, true) == false) { - if (print_result) printf("Final merge to client 0 failed from client %d\n", i); - goto finalize; - } + + // Materialize all + for (int i = 0; i < 3; i++) { + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("three_way: materialize db[%d] failed\n", i); goto fail; } + } + + // All three should converge + char *body[3]; + for (int i = 0; i < 3; i++) { + body[i] = do_select_text(db[i], "SELECT body FROM docs WHERE id = 'doc1';"); + } + + bool ok = true; + if (!body[0] || !body[1] || !body[2]) { printf("three_way: NULL body\n"); ok = false; } + else if (strcmp(body[0], body[1]) != 0 || strcmp(body[1], body[2]) != 0) { + printf("three_way: not converged:\n [0]: %s\n [1]: %s\n [2]: %s\n", body[0], body[1], body[2]); + ok = false; + } else { + // All three non-conflicting edits should be preserved + if (!strstr(body[0], "S0")) { printf("three_way: missing S0\n"); ok = false; } + if (!strstr(body[0], "S1")) { printf("three_way: missing S1\n"); ok = false; } + if (!strstr(body[0], "L3")) { printf("three_way: missing L3\n"); ok = false; } + if (!strstr(body[0], "S2")) { printf("three_way: missing S2\n"); ok = false; } + } + + for (int i = 0; i < 3; i++) { if (body[i]) sqlite3_free(body[i]); } + for (int i = 0; i < 3; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 3; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 4: Mixed block + normal columns — both work independently +bool do_test_block_lww_mixed_columns(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE notes (id TEXT NOT NULL PRIMARY KEY, body TEXT, title TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('notes');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + // body is block-level LWW, title is normal LWW + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('notes', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Site 0: insert row with multi-line body and title + rc = sqlite3_exec(db[0], "INSERT INTO notes (id, body, title) VALUES ('n1', 'Line1\nLine2\nLine3', 'My Title');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync 0 -> 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit block column (body line 1) AND normal column (title) + rc = sqlite3_exec(db[0], "UPDATE notes SET body = 'EditedLine1\nLine2\nLine3', title = 'Title From A' WHERE id = 'n1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit a different block (body line 3) AND normal column (title — will conflict via LWW) + rc = sqlite3_exec(db[1], "UPDATE notes SET body = 'Line1\nLine2\nEditedLine3', title = 'Title From B' WHERE id = 'n1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize block column + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('notes', 'body', 'n1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('notes', 'body', 'n1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM notes WHERE id = 'n1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM notes WHERE id = 'n1';"); + char *title0 = do_select_text(db[0], "SELECT title FROM notes WHERE id = 'n1';"); + char *title1 = do_select_text(db[1], "SELECT title FROM notes WHERE id = 'n1';"); + + bool ok = true; + + // Bodies should converge + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("mixed_columns: body diverged\n"); + ok = false; + } else { + // Both non-conflicting block edits should be preserved + if (!strstr(body0, "EditedLine1")) { printf("mixed_columns: missing EditedLine1\n"); ok = false; } + if (!strstr(body0, "Line2")) { printf("mixed_columns: missing Line2\n"); ok = false; } + if (!strstr(body0, "EditedLine3")) { printf("mixed_columns: missing EditedLine3\n"); ok = false; } + } + + // Titles should converge (normal LWW — one wins) + if (!title0 || !title1 || strcmp(title0, title1) != 0) { + printf("mixed_columns: title diverged: [%s] vs [%s]\n", title0 ? title0 : "NULL", title1 ? title1 : "NULL"); + ok = false; } - - // Merge changes from client 0 to all other clients - if (print_result) printf("Propagating final state from client 0 to all other clients\n"); - for (int i = 1; i < nclients; ++i) { - if (do_merge_using_payload(db[0], db[i], false, true) == false) { - if (print_result) printf("Final merge from client 0 failed to client %d\n", i); - goto finalize; - } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + if (title0) sqlite3_free(title0); + if (title1) sqlite3_free(title1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 5: NULL to text transition — INSERT with NULL body, then UPDATE to multi-line text +bool do_test_block_lww_null_to_text(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; } - - // Verify all databases have converged to the same state - if (print_result) printf("Verifying convergence across all clients\n"); - char *verification_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); - for (int i = 1; i < nclients; ++i) { - if (do_compare_queries(db[0], verification_sql, db[i], verification_sql, -1, -1, print_result) == false) { - if (print_result) printf("Convergence verification failed: client 0 vs client %d\n", i); - sqlite3_free(verification_sql); - goto finalize; - } + + // Insert with NULL body on site 0 + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', NULL);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: INSERT NULL failed\n"); goto fail; } + + // Sync to site 1 + if (!do_merge_values(db[0], db[1], false)) { printf("null_to_text: initial sync failed\n"); goto fail; } + + // Update to multi-line text on site 0 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Hello\nWorld\nFoo' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: UPDATE failed\n"); goto fail; } + + // Verify blocks created + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("null_to_text: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync update to site 1 + if (!do_merge_values(db[0], db[1], true)) { printf("null_to_text: sync update failed\n"); goto fail; } + + // Materialize on site 1 + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("null_to_text: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "Hello\nWorld\nFoo") != 0) { + printf("null_to_text: expected 'Hello\\nWorld\\nFoo', got '%s'\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; } - sqlite3_free(verification_sql); - - if (print_result) { - printf("\nFinal converged state:\n"); - char *display_sql = sqlite3_mprintf("SELECT * FROM %s ORDER BY id1, id2;", table_name); - do_query(db[0], display_sql, NULL); - sqlite3_free(display_sql); + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 6: Interleaved inserts — multiple rounds of inserting between existing lines +bool do_test_block_lww_interleaved(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Start with 2 lines + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Round 1: Site 0 inserts between A and B + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Round 2: Site 1 inserts between A and C + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nD\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Round 3: Site 0 inserts between D and C + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nD\nE\nC\nB' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Verify final state on both sites + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("interleaved: diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // All 5 lines should be present + if (!strstr(body0, "A")) { printf("interleaved: missing A\n"); ok = false; } + if (!strstr(body0, "D")) { printf("interleaved: missing D\n"); ok = false; } + if (!strstr(body0, "E")) { printf("interleaved: missing E\n"); ok = false; } + if (!strstr(body0, "C")) { printf("interleaved: missing C\n"); ok = false; } + if (!strstr(body0, "B")) { printf("interleaved: missing B\n"); ok = false; } + + // Verify 5 blocks + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 5) { printf("interleaved: expected 5 blocks, got %" PRId64 "\n", blocks); ok = false; } } - - result = true; - -finalize: - for (int i = 0; i < nclients; ++i) { - if (rc != SQLITE_OK && db[i] && (sqlite3_errcode(db[i]) != SQLITE_OK)) { - printf("do_test_merge_composite_pk_10_clients error: %s\n", sqlite3_errmsg(db[i])); - } - if (db[i]) { - if (sqlite3_get_autocommit(db[i]) == 0) { - result = false; - printf("do_test_merge_composite_pk_10_clients error: db %d is in transaction\n", i); - } - - int counter = close_db_v2(db[i]); - if (counter > 0) { - result = false; - printf("do_test_merge_composite_pk_10_clients error: db %d has %d unterminated statements\n", i, counter); - } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 7: Custom delimiter — paragraph separator instead of newline +bool do_test_block_lww_custom_delimiter(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + // Set custom delimiter: double newline (paragraph separator) + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'delimiter', '\n\n');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("custom_delim: set delimiter failed: %s\n", sqlite3_errmsg(db[i])); goto fail; } + } + + // Insert text with double-newline separated paragraphs + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Para one line1\nline2\n\nPara two\n\nPara three');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Should produce 3 blocks (3 paragraphs) + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("custom_delim: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync and materialize + if (!do_merge_values(db[0], db[1], false)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "Para one line1\nline2\n\nPara two\n\nPara three") != 0) { + printf("custom_delim: mismatch: [%s]\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 8: Large text — many lines to verify position ID distribution +bool do_test_block_lww_large_text(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Build a 200-line text + #define LARGE_NLINES 200 + char large_text[LARGE_NLINES * 20]; + int offset = 0; + for (int i = 0; i < LARGE_NLINES; i++) { + if (i > 0) large_text[offset++] = '\n'; + offset += snprintf(large_text + offset, sizeof(large_text) - offset, "Line %03d content", i); + } + + // Insert via prepared statement to avoid SQL escaping issues + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "INSERT INTO docs (id, body) VALUES ('bigdoc', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, large_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { printf("large_text: INSERT failed\n"); goto fail; } + + // Verify block count + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('bigdoc');"); + if (blocks != LARGE_NLINES) { printf("large_text: expected %d blocks, got %" PRId64 "\n", LARGE_NLINES, blocks); goto fail; } + + // Verify all position IDs are unique and ordered + int64_t distinct_positions = do_select_int(db[0], + "SELECT count(DISTINCT col_name) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + if (distinct_positions != LARGE_NLINES) { + printf("large_text: expected %d distinct positions, got %" PRId64 "\n", LARGE_NLINES, distinct_positions); + goto fail; + } + + // Sync and materialize + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("large_text: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'bigdoc');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("large_text: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'bigdoc';"); + if (!body || strcmp(body, large_text) != 0) { + printf("large_text: roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test 9: Rapid sequential updates — many updates on same row in quick succession +bool do_test_block_lww_rapid_updates(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert initial + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Start');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // 50 rapid updates, progressively adding lines + sqlite3_stmt *upd = NULL; + rc = sqlite3_prepare_v2(db[0], "UPDATE docs SET body = ? WHERE id = 'doc1';", -1, &upd, NULL); + if (rc != SQLITE_OK) goto fail; + + #define RAPID_ROUNDS 50 + char rapid_text[RAPID_ROUNDS * 20]; + int roff = 0; + for (int i = 0; i < RAPID_ROUNDS; i++) { + if (i > 0) rapid_text[roff++] = '\n'; + roff += snprintf(rapid_text + roff, sizeof(rapid_text) - roff, "Update%d", i); + + sqlite3_bind_text(upd, 1, rapid_text, roff, SQLITE_STATIC); + rc = sqlite3_step(upd); + if (rc != SQLITE_DONE) { printf("rapid: UPDATE %d failed\n", i); sqlite3_finalize(upd); goto fail; } + sqlite3_reset(upd); + } + sqlite3_finalize(upd); + + // Verify final block count matches line count + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != RAPID_ROUNDS) { + printf("rapid: expected %d blocks, got %" PRId64 "\n", RAPID_ROUNDS, blocks); + goto fail; + } + + // Sync and verify roundtrip + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("rapid: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("rapid: materialize failed\n"); goto fail; } + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("rapid: roundtrip mismatch\n"); + ok = false; + } else { + // Check first and last lines + if (!strstr(body0, "Update0")) { printf("rapid: missing Update0\n"); ok = false; } + if (!strstr(body0, "Update49")) { printf("rapid: missing Update49\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Unicode/multibyte content in blocks (emoji, CJK, accented chars) +bool do_test_block_lww_unicode(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert multi-line text with unicode content + const char *unicode_text = "Hello \xC3\xA9\xC3\xA0\xC3\xBC" "\n" // accented: éàü + "\xE4\xB8\xAD\xE6\x96\x87\xE6\xB5\x8B\xE8\xAF\x95" "\n" // CJK: 中文测试 + "\xF0\x9F\x98\x80\xF0\x9F\x8E\x89\xF0\x9F\x9A\x80"; // emoji: 😀🎉🚀 + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, unicode_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Should have 3 blocks + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("unicode: expected 3 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync and materialize + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("unicode: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("unicode: materialize failed\n"); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, unicode_text) != 0) { + printf("unicode: roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + + // Update: edit the emoji line + const char *updated_text = "Hello \xC3\xA9\xC3\xA0\xC3\xBC" "\n" + "\xE4\xB8\xAD\xE6\x96\x87\xE6\xB5\x8B\xE8\xAF\x95" "\n" + "\xF0\x9F\x92\xAF\xF0\x9F\x94\xA5"; // changed emoji: 💯🔥 + sqlite3_free(body); + + stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "UPDATE docs SET body = ? WHERE id = 'doc1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, updated_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Sync update + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("unicode: sync update failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, updated_text) != 0) { + printf("unicode: update roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Special characters (tabs, carriage returns, etc.) in blocks +bool do_test_block_lww_special_chars(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Text with tabs, carriage returns, and other special chars within lines + const char *special_text = "col1\tcol2\tcol3\n" // tabs within line + "line with\r\nembedded\n" // \r before \n delimiter + "back\\slash \"quotes\""; // backslash and quotes + + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', ?);", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto fail; + sqlite3_bind_text(stmt, 1, special_text, -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) goto fail; + + // Should split on \n: "col1\tcol2\tcol3", "line with\r", "embedded", "back\\slash \"quotes\"" + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 4) { printf("special: expected 4 blocks, got %" PRId64 "\n", blocks); goto fail; } + + // Sync and verify roundtrip + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("special: sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, special_text) != 0) { + printf("special: roundtrip mismatch\n"); + if (body) sqlite3_free(body); + goto fail; + } + sqlite3_free(body); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Concurrent delete vs edit on different blocks +// Site A deletes the row, Site B edits a line. After sync, delete wins. +bool do_test_block_lww_delete_vs_edit(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert initial doc + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync to site 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: DELETE the row + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: Edit line 2 + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Line1\nEdited\nLine3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Both should converge: either row deleted or row exists with some content + int64_t rows0 = do_select_int(db[0], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + int64_t rows1 = do_select_int(db[1], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (rows0 != rows1) { + printf("delete_vs_edit: row count diverged: db0=%" PRId64 " db1=%" PRId64 "\n", rows0, rows1); + ok = false; + } + + // If the row still exists, materialize and verify convergence + if (rows0 > 0 && rows1 > 0) { + sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (body0 && body1 && strcmp(body0, body1) != 0) { + printf("delete_vs_edit: bodies diverged\n"); + ok = false; } - if (cleanup_databases) { - char buf[256]; - do_build_database_path(buf, i, timestamp, saved_counter); - file_delete_internal(buf); + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + } + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Two block columns on same table +bool do_test_block_lww_two_block_cols(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT, notes TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'notes', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert with both block columns + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body, notes) VALUES ('doc1', 'B1\nB2\nB3', 'N1\nN2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Verify blocks created for both columns + int64_t body_blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'body' || x'1f' || '%';"); + int64_t notes_blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync WHERE col_name LIKE 'notes' || x'1f' || '%';"); + if (body_blocks != 3) { printf("two_block_cols: expected 3 body blocks, got %" PRId64 "\n", body_blocks); goto fail; } + if (notes_blocks != 2) { printf("two_block_cols: expected 2 notes blocks, got %" PRId64 "\n", notes_blocks); goto fail; } + + // Sync to site 1 + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: edit body line 1 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'B1_edited\nB2\nB3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: edit notes line 2 + rc = sqlite3_exec(db[1], "UPDATE docs SET notes = 'N1\nN2_edited' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync both ways + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + // Materialize both columns on both sites + for (int i = 0; i < 2; i++) { + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: materialize body db[%d] failed\n", i); goto fail; } + rc = sqlite3_exec(db[i], "SELECT cloudsync_text_materialize('docs', 'notes', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("two_block_cols: materialize notes db[%d] failed\n", i); goto fail; } + } + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + char *notes0 = do_select_text(db[0], "SELECT notes FROM docs WHERE id = 'doc1';"); + char *notes1 = do_select_text(db[1], "SELECT notes FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("two_block_cols: body diverged\n"); ok = false; + } else if (!strstr(body0, "B1_edited")) { + printf("two_block_cols: body edit missing\n"); ok = false; + } + + if (!notes0 || !notes1 || strcmp(notes0, notes1) != 0) { + printf("two_block_cols: notes diverged\n"); ok = false; + } else if (!strstr(notes0, "N2_edited")) { + printf("two_block_cols: notes edit missing\n"); ok = false; + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + if (notes0) sqlite3_free(notes0); + if (notes1) sqlite3_free(notes1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Update text to NULL (text->NULL transition) +bool do_test_block_lww_text_to_null(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert multi-line text + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t blocks_before = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks_before != 3) { printf("text_to_null: expected 3 blocks before, got %" PRId64 "\n", blocks_before); goto fail; } + + // Update to NULL + rc = sqlite3_exec(db[0], "UPDATE docs SET body = NULL WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("text_to_null: UPDATE to NULL failed\n"); goto fail; } + + // Verify body is NULL + int64_t is_null = do_select_int(db[0], "SELECT body IS NULL FROM docs WHERE id = 'doc1';"); + if (is_null != 1) { printf("text_to_null: body not NULL after update\n"); goto fail; } + + // Sync and verify + if (!do_merge_values(db[0], db[1], false)) { printf("text_to_null: sync failed\n"); goto fail; } + + // Materialize on site 1 + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + int64_t is_null_b = do_select_int(db[1], "SELECT body IS NULL FROM docs WHERE id = 'doc1';"); + if (is_null_b != 1) { printf("text_to_null: body not NULL on site 1 after sync\n"); goto fail; } + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Payload-based sync for block columns (vs row-by-row do_merge_values) +bool do_test_block_lww_payload_sync(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert and first sync via payload + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Alpha\nBravo\nCharlie');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("payload_sync: initial merge failed\n"); goto fail; } + + // Edit on both sites + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Alpha_A\nBravo\nCharlie' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'Alpha\nBravo\nCharlie_B' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync via payload both ways + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("payload_sync: merge 0->1 failed\n"); goto fail; } + if (!do_merge_using_payload(db[1], db[0], true, true)) { printf("payload_sync: merge 1->0 failed\n"); goto fail; } + + // Materialize + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("payload_sync: bodies diverged\n"); ok = false; + } else { + if (!strstr(body0, "Alpha_A")) { printf("payload_sync: missing Alpha_A\n"); ok = false; } + if (!strstr(body0, "Bravo")) { printf("payload_sync: missing Bravo\n"); ok = false; } + if (!strstr(body0, "Charlie_B")) { printf("payload_sync: missing Charlie_B\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Idempotent apply — applying the same payload twice is a no-op +bool do_test_block_lww_idempotent(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert and sync + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Line1\nLine2\nLine3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[0], db[1], false, true)) goto fail; + + // Edit on site 0 + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Edited1\nLine2\nLine3' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Apply payload to site 1 TWICE + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("idempotent: first apply failed\n"); goto fail; } + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("idempotent: second apply failed\n"); goto fail; } + + // Materialize + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + bool ok = true; + if (!body || strcmp(body, "Edited1\nLine2\nLine3") != 0) { + printf("idempotent: body mismatch: [%s]\n", body ? body : "NULL"); + ok = false; + } + + // Verify block count is still 3 (no duplicates from double apply) + int64_t blocks = do_select_int(db[1], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 3) { printf("idempotent: expected 3 blocks, got %" PRId64 "\n", blocks); ok = false; } + + if (body) sqlite3_free(body); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Block position ordering — after edits, materialized text has correct line order +bool do_test_block_lww_ordering(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Insert initial doc: A B C D E + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'A\nB\nC\nD\nE');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_values(db[0], db[1], false)) goto fail; + + // Site 0: insert X between B and C, remove D -> A B X C E + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'A\nB\nX\nC\nE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: insert Y between D and E -> A B C D Y E + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nB\nC\nD\nY\nE' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Sync + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("ordering: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // Verify ordering: A must come before B, B before C, etc. + // All lines that survived should maintain relative order + const char *pA = strstr(body0, "A"); + const char *pB = strstr(body0, "B"); + const char *pC = strstr(body0, "C"); + const char *pE = strstr(body0, "E"); + + if (!pA || !pB || !pC || !pE) { + printf("ordering: missing original lines\n"); ok = false; + } else { + if (pA >= pB) { printf("ordering: A not before B\n"); ok = false; } + if (pB >= pC) { printf("ordering: B not before C\n"); ok = false; } + if (pC >= pE) { printf("ordering: C not before E\n"); ok = false; } + } + + // X (inserted between B and C) should appear between B and C + const char *pX = strstr(body0, "X"); + if (pX) { + if (pX <= pB || pX >= pC) { printf("ordering: X not between B and C\n"); ok = false; } + } + + // Y should appear somewhere after C + const char *pY = strstr(body0, "Y"); + if (pY) { + if (pY <= pC) { printf("ordering: Y not after C\n"); ok = false; } } } - return result; + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -bool do_test_prikey (int nclients, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; - bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables +// Test: Composite primary key (text + int) with block column +bool do_test_block_lww_composite_pk(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; time_t timestamp = time(NULL); - int saved_counter = test_counter; - for (int i=0; i foo (client1)\n"); - do_query(db[0], "SELECT * FROM foo ORDER BY a;", NULL); + + // Insert and sync + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'Old1\nOld2\nOld3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + if (!do_merge_using_payload(db[0], db[1], false, true)) goto fail; + + // Delete the row + rc = sqlite3_exec(db[0], "DELETE FROM docs WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("del_reinsert: DELETE failed\n"); goto fail; } + + // Sync delete + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("del_reinsert: delete sync failed\n"); goto fail; } + + // Verify row gone on site 1 + int64_t count = do_select_int(db[1], "SELECT count(*) FROM docs WHERE id = 'doc1';"); + if (count != 0) { printf("del_reinsert: row should be deleted on site 1, count=%" PRId64 "\n", count); goto fail; } + + // Re-insert with different content + rc = sqlite3_exec(db[0], "INSERT INTO docs (id, body) VALUES ('doc1', 'New1\nNew2');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("del_reinsert: re-INSERT failed: %s\n", sqlite3_errmsg(db[0])); goto fail; } + + // Sync re-insert + if (!do_merge_using_payload(db[0], db[1], true, true)) { printf("del_reinsert: reinsert sync failed\n"); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body || strcmp(body, "New1\nNew2") != 0) { + printf("del_reinsert: body mismatch after reinsert: [%s]\n", body ? body : "NULL"); + if (body) sqlite3_free(body); + goto fail; } - - // send local changes to client1 - for (int i=1; i X A B C + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'X\nA\nB\nC' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Site 1: add line at bottom -> A B C Y + rc = sqlite3_exec(db[1], "UPDATE docs SET body = 'A\nB\nC\nY' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + // Bidirectional sync + if (!do_merge_values(db[0], db[1], true)) goto fail; + if (!do_merge_values(db[1], db[0], true)) goto fail; + + rc = sqlite3_exec(db[0], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body0 = do_select_text(db[0], "SELECT body FROM docs WHERE id = 'doc1';"); + char *body1 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + + bool ok = true; + if (!body0 || !body1 || strcmp(body0, body1) != 0) { + printf("nonoverlap: bodies diverged: [%s] vs [%s]\n", body0 ? body0 : "NULL", body1 ? body1 : "NULL"); + ok = false; + } else { + // X should be present, Y should be present, original A B C should be present + if (!strstr(body0, "X")) { printf("nonoverlap: X missing\n"); ok = false; } + if (!strstr(body0, "Y")) { printf("nonoverlap: Y missing\n"); ok = false; } + if (!strstr(body0, "A")) { printf("nonoverlap: A missing\n"); ok = false; } + if (!strstr(body0, "B")) { printf("nonoverlap: B missing\n"); ok = false; } + if (!strstr(body0, "C")) { printf("nonoverlap: C missing\n"); ok = false; } + + // Order: X before A, Y after C + const char *pX = strstr(body0, "X"); + const char *pA = strstr(body0, "A"); + const char *pC = strstr(body0, "C"); + const char *pY = strstr(body0, "Y"); + if (pX && pA && pX >= pA) { printf("nonoverlap: X not before A\n"); ok = false; } + if (pC && pY && pY <= pC) { printf("nonoverlap: Y not after C\n"); ok = false; } + } + + if (body0) sqlite3_free(body0); + if (body1) sqlite3_free(body1); + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return ok; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; +} + +// Test: Very long single line (10K chars, single block) +bool do_test_block_lww_long_line(int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[2] = {NULL, NULL}; + time_t timestamp = time(NULL); + int rc; + + for (int i = 0; i < 2; i++) { + db[i] = do_create_database_file(i, timestamp, test_counter++); + if (!db[i]) return false; + rc = sqlite3_exec(db[i], "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + rc = sqlite3_exec(db[i], "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + } + + // Build a 10,000-char single line + { + char *long_line = (char *)malloc(10001); + if (!long_line) goto fail; + for (int i = 0; i < 10000; i++) long_line[i] = 'A' + (i % 26); + long_line[10000] = '\0'; + + char *sql = sqlite3_mprintf("INSERT INTO docs (id, body) VALUES ('doc1', '%q');", long_line); + rc = sqlite3_exec(db[0], sql, NULL, NULL, NULL); + sqlite3_free(sql); + + if (rc != SQLITE_OK) { printf("long_line: INSERT failed: %s\n", sqlite3_errmsg(db[0])); free(long_line); goto fail; } + + // Should have 1 block (no newlines) + int64_t blocks = do_select_int(db[0], "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('doc1');"); + if (blocks != 1) { printf("long_line: expected 1 block, got %" PRId64 "\n", blocks); free(long_line); goto fail; } + + // Sync to site 1 + if (!do_merge_using_payload(db[0], db[1], false, true)) { printf("long_line: sync failed\n"); free(long_line); goto fail; } + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { free(long_line); goto fail; } + + char *body = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + bool match = (body && strcmp(body, long_line) == 0); + if (!match) printf("long_line: body mismatch (len=%zu vs expected 10000)\n", body ? strlen(body) : 0); + if (body) sqlite3_free(body); + free(long_line); + if (!match) goto fail; } - - // compare results - for (int i=1; i "Line1\n spaces \n\t\ttabs\nLine6" + rc = sqlite3_exec(db[0], "UPDATE docs SET body = 'Line1\n spaces \n\t\ttabs\nLine6' WHERE id = 'doc1';", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + if (!do_merge_using_payload(db[0], db[1], true, true)) goto fail; + rc = sqlite3_exec(db[1], "SELECT cloudsync_text_materialize('docs', 'body', 'doc1');", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto fail; + + char *body2 = do_select_text(db[1], "SELECT body FROM docs WHERE id = 'doc1';"); + if (!body2 || strcmp(body2, "Line1\n spaces \n\t\ttabs\nLine6") != 0) { + printf("whitespace: body2 mismatch: [%s]\n", body2 ? body2 : "NULL"); + if (body2) sqlite3_free(body2); + goto fail; } - return result; + sqlite3_free(body2); + + for (int i = 0; i < 2; i++) { close_db(db[i]); db[i] = NULL; } + return true; + +fail: + for (int i = 0; i < 2; i++) if (db[i]) close_db(db[i]); + return false; } -bool do_test_double_init(int nclients, bool cleanup_databases) { - if (nclients<2) { - nclients = 2; - printf("Number of clients for test do_test_double_init increased to %d clients\n", 2); - } - - bool result = false; - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +bool do_test_block_lww_cleanup(bool print_result, bool cleanup_databases) { + // Test: cloudsync_cleanup removes both the meta-table and the blocks table + // when a table has block LWW columns configured. + sqlite3 *db = NULL; time_t timestamp = time(NULL); - db[0] = do_create_database_file(0, timestamp, test_counter); + int rc; + bool result = false; - // configure cloudsync for a table on the first connection - int table_mask = TEST_PRIKEYS; - if (do_create_tables(table_mask, db[0]) == false) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - - // double load - sqlite3_cloudsync_init(db[0], NULL, NULL); - sqlite3_cloudsync_init(db[0], NULL, NULL); - - // double cloudsync-init - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; + db = do_create_database_file(0, timestamp, test_counter++); + if (!db) return false; - // double terminate - if (sqlite3_exec(db[0], "SELECT cloudsync_terminate();", NULL, NULL, NULL) != SQLITE_OK) goto finalize; - if (do_augment_tables(table_mask, db[0], table_algo_crdt_cls) == false) goto finalize; - if (sqlite3_exec(db[0], "SELECT cloudsync_terminate();", NULL, NULL, NULL) != SQLITE_OK) goto finalize; + rc = sqlite3_exec(db, "CREATE TABLE docs (id TEXT NOT NULL PRIMARY KEY, body TEXT);", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: CREATE TABLE failed: %s\n", sqlite3_errmsg(db)); goto fail; } - db[1] = do_create_database_file(1, timestamp, test_counter); + rc = sqlite3_exec(db, "SELECT cloudsync_init('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: cloudsync_init failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + rc = sqlite3_exec(db, "SELECT cloudsync_set_column('docs', 'body', 'algo', 'block');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: set_column failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Insert a row to populate the blocks table + rc = sqlite3_exec(db, "INSERT INTO docs (id, body) VALUES ('doc1', 'Line 1\nLine 2\nLine 3');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: INSERT failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Confirm docs_cloudsync and docs_cloudsync_blocks exist before cleanup + int64_t meta_exists = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync';"); + if (meta_exists != 1) { printf("block_cleanup: docs_cloudsync missing before cleanup\n"); goto fail; } + + int64_t blocks_exists = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_exists != 1) { printf("block_cleanup: docs_cloudsync_blocks missing before cleanup\n"); goto fail; } + + // Run cleanup + rc = sqlite3_exec(db, "SELECT cloudsync_cleanup('docs');", NULL, NULL, NULL); + if (rc != SQLITE_OK) { printf("block_cleanup: cloudsync_cleanup failed: %s\n", sqlite3_errmsg(db)); goto fail; } + + // Both companion tables must be gone after cleanup + int64_t meta_after = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync';"); + if (meta_after != 0) { printf("block_cleanup: docs_cloudsync still exists after cleanup\n"); goto fail; } + + int64_t blocks_after = do_select_int(db, "SELECT COUNT(*) FROM sqlite_master WHERE name='docs_cloudsync_blocks';"); + if (blocks_after != 0) { printf("block_cleanup: docs_cloudsync_blocks still exists after cleanup\n"); goto fail; } result = true; - -finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i log\n"); - do_query(db[0], "SELECT * FROM log ORDER BY id;", NULL); + + // Test 4: Valid payload with flipped byte in the middle + { + char *corrupted = (char *)malloc(valid_len); + memcpy(corrupted, payload_copy, valid_len); + corrupted[valid_len / 2] ^= 0xFF; + + sqlite3_stmt *dec_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &dec_stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(dec_stmt, 1, corrupted, valid_len, SQLITE_STATIC); + rc = sqlite3_step(dec_stmt); + sqlite3_finalize(dec_stmt); + } + free(corrupted); } - + + // Verify destination table is still empty (no corrupted data inserted) + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM test_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) { free(payload_copy); goto finalize; } + if (sqlite3_step(count_stmt) != SQLITE_ROW) { sqlite3_finalize(count_stmt); free(payload_copy); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 0) { printf("corrupted_payload: expected 0 rows but got %d\n", count); free(payload_copy); goto finalize; } + } + + // Test 5: Valid payload should still work + if (!do_merge_using_payload(db[0], db[1], false, true)) { + printf("corrupted_payload: valid payload failed after corrupted attempts\n"); + free(payload_copy); + goto finalize; + } + + { + sqlite3_stmt *count_stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM test_tbl;", -1, &count_stmt, NULL); + if (rc != SQLITE_OK) { free(payload_copy); goto finalize; } + if (sqlite3_step(count_stmt) != SQLITE_ROW) { sqlite3_finalize(count_stmt); free(payload_copy); goto finalize; } + int count = sqlite3_column_int(count_stmt, 0); + sqlite3_finalize(count_stmt); + if (count != 2) { printf("corrupted_payload: expected 2 rows after valid apply but got %d\n", count); free(payload_copy); goto finalize; } + } + + free(payload_copy); result = true; - + finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i= 0 && count != prev_count) { + printf("payload_idempotency: row count changed from %d to %d on apply #%d\n", prev_count, count, apply + 1); + goto finalize; } + prev_count = count; } - - if (force_uncompressed) force_uncompressed_blob = false; - - // compare results - for (int i=1; i customers\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); + + // Verify data values are correct + { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[1], "SELECT val FROM test_tbl WHERE id = 'id1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + const char *val = (const char *)sqlite3_column_text(stmt, 0); + if (!val || strcmp(val, "hello_updated") != 0) { + printf("payload_idempotency: expected 'hello_updated', got '%s'\n", val ? val : "NULL"); + sqlite3_finalize(stmt); + goto finalize; + } + sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - + + // Compare source and target + result = do_compare_queries(db[0], "SELECT * FROM test_tbl ORDER BY id;", + db[1], "SELECT * FROM test_tbl ORDER BY id;", + -1, -1, print_result); + finalize: - for (int i=0; i= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - if (table_mask & TEST_NOPRIKEYS) { - printf("\n-> customers_noprikey\n"); - do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } + + // All 3 must converge to the same value + const char *query = "SELECT val FROM test_tbl WHERE id = 'row1';"; + char *values[3] = {NULL, NULL, NULL}; + + for (int i = 0; i < 3; i++) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[i], query, -1, &stmt, NULL); + if (rc != SQLITE_OK) goto tiebreak_finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto tiebreak_finalize; } + const char *text = (const char *)sqlite3_column_text(stmt, 0); + values[i] = text ? strdup(text) : NULL; + sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - -finalize: - for (int i=0; i 0) { - result = false; - printf("do_test_merge error: db %d has %d unterminated statements\n", i, counter); - } - } - + + // Check convergence + if (values[0] && values[1] && values[2] && + strcmp(values[0], values[1]) == 0 && strcmp(values[1], values[2]) == 0) { + result = true; + } else { + printf("causal_length_tiebreak: databases diverged: '%s', '%s', '%s'\n", + values[0] ? values[0] : "NULL", + values[1] ? values[1] : "NULL", + values[2] ? values[2] : "NULL"); + } + +tiebreak_finalize: + for (int i = 0; i < 3; i++) { + if (values[i]) free(values[i]); + } + +finalize: + for (int i = 0; i < 3; i++) { + if (db[i]) close_db(db[i]); if (cleanup_databases) { char buf[256]; do_build_database_path(buf, i, timestamp, saved_counter++); @@ -5797,113 +12101,66 @@ bool do_test_fill_initial_data(int nclients, bool print_result, bool cleanup_dat return result; } -bool do_test_alter(int nclients, int alter_version, bool print_result, bool cleanup_databases) { - sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; +bool do_test_delete_resurrect_ordering (int nclients, bool print_result, bool cleanup_databases) { + sqlite3 *db[3] = {NULL, NULL, NULL}; bool result = false; - int rc = SQLITE_OK; - - memset(db, 0, sizeof(sqlite3 *) * MAX_SIMULATED_CLIENTS); - if (nclients >= MAX_SIMULATED_CLIENTS) { - nclients = MAX_SIMULATED_CLIENTS; - printf("Number of test merge reduced to %d clients\n", MAX_SIMULATED_CLIENTS); - } else if (nclients < 2) { - nclients = 2; - printf("Number of test merge increased to %d clients\n", 2); - } - - // create databases and tables - int table_mask = TEST_PRIKEYS | TEST_NOCOLS; + time_t timestamp = time(NULL); int saved_counter = test_counter; - for (int i=0; i2) + sqlite3_exec(db[0], "DELETE FROM test_tbl WHERE id = 'row1';", NULL, NULL, NULL); + + // Sync delete to B + do_merge_using_payload(db[0], db[1], true, true); + + // Site B: re-insert (CL 2->3, resurrection) + sqlite3_exec(db[1], "INSERT INTO test_tbl VALUES ('row1', 'resurrected_by_b');", NULL, NULL, NULL); + + // Site C receives payloads in REVERSE order: B's resurrection first, then A's delete + do_merge_using_payload(db[1], db[2], true, true); + do_merge_using_payload(db[0], db[2], true, true); + + // Site A receives B's resurrection + do_merge_using_payload(db[1], db[2], true, true); + do_merge_using_payload(db[1], db[0], true, true); + + // All 3 should converge: row exists + const char *query = "SELECT * FROM test_tbl ORDER BY id;"; + result = do_compare_queries(db[0], query, db[1], query, -1, -1, print_result); + if (result) result = do_compare_queries(db[0], query, db[2], query, -1, -1, print_result); + + // Verify the row exists (resurrection should win) + if (result) { + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db[2], "SELECT COUNT(*) FROM test_tbl WHERE id = 'row1';", -1, &stmt, NULL); + if (rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW) { + int count = sqlite3_column_int(stmt, 0); + if (count != 1) { + printf("delete_resurrect_ordering: expected row1 to exist on db[2], count=%d\n", count); + result = false; } - if (do_compare_queries(db[0], sql, db[i], sql, -1, -1, print_result) == false) goto finalize; - } - } - - if (print_result) { - if (table_mask & TEST_PRIKEYS) { - printf("\n-> " CUSTOMERS_TABLE "\n"); - char *sql = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", CUSTOMERS_TABLE); - do_query(db[0], sql, query_table); - sqlite3_free(sql); - } - if (table_mask & TEST_NOCOLS) { - printf("\n-> \"" CUSTOMERS_NOCOLS_TABLE "\"\n"); - do_query(db[0], "SELECT * FROM \"" CUSTOMERS_NOCOLS_TABLE "\" ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); - } - if (table_mask & TEST_NOPRIKEYS) { - printf("\n-> customers_noprikey\n"); - do_query(db[0], "SELECT * FROM customers_noprikey ORDER BY first_name, \"" CUSTOMERS_TABLE_COLUMN_LASTNAME "\";", query_table); } + if (stmt) sqlite3_finalize(stmt); } - - result = true; - rc = SQLITE_OK; - + finalize: - for (int i=0; iText:", do_test_block_lww_null_to_text(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Interleave:", do_test_block_lww_interleaved(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW CustomDelim:", do_test_block_lww_custom_delimiter(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Large Text:", do_test_block_lww_large_text(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Rapid Upd:", do_test_block_lww_rapid_updates(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Unicode:", do_test_block_lww_unicode(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW SpecialChars:", do_test_block_lww_special_chars(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Del vs Edit:", do_test_block_lww_delete_vs_edit(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW TwoBlockCols:", do_test_block_lww_two_block_cols(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Text->NULL:", do_test_block_lww_text_to_null(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW PayloadSync:", do_test_block_lww_payload_sync(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Idempotent:", do_test_block_lww_idempotent(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Ordering:", do_test_block_lww_ordering(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW CompositePK:", do_test_block_lww_composite_pk(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW EmptyVsNull:", do_test_block_lww_empty_vs_null(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW DelReinsert:", do_test_block_lww_delete_reinsert(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW IntegerPK:", do_test_block_lww_integer_pk(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW MultiRow:", do_test_block_lww_multi_row(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW NonOverlap:", do_test_block_lww_nonoverlap_add(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW LongLine:", do_test_block_lww_long_line(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Whitespace:", do_test_block_lww_whitespace(2, print_result, cleanup_databases)); + result += test_report("Test Block LWW Cleanup:", do_test_block_lww_cleanup(print_result, cleanup_databases)); + + // edge-case tests + result += test_report("Corrupted Payload Test:", do_test_corrupted_payload(2, print_result, cleanup_databases)); + result += test_report("Payload Idempotency Test:", do_test_payload_idempotency(2, print_result, cleanup_databases)); + result += test_report("CL Tiebreak Test:", do_test_causal_length_tiebreak(3, print_result, cleanup_databases)); + result += test_report("Delete/Resurrect Order:", do_test_delete_resurrect_ordering(3, print_result, cleanup_databases)); + result += test_report("Large Composite PK Test:", do_test_large_composite_pk(2, print_result, cleanup_databases)); + result += test_report("Schema Hash Mismatch:", do_test_schema_hash_mismatch(2, print_result, cleanup_databases)); + result += test_report("Stale Table Settings:", do_test_stale_table_settings(cleanup_databases)); + result += test_report("Stale Table Settings Dropped Meta:", do_test_stale_table_settings_dropped_meta(cleanup_databases)); + result += test_report("DBVersion Rebuild Error:", do_test_dbversion_rebuild_error()); + result += test_report("Uninit Error Messages:", do_test_uninit_error_messages()); + result += test_report("Block LWW Existing Data:", do_test_block_lww_existing_data(cleanup_databases)); + result += test_report("Block Column Reload:", do_test_block_column_reload(cleanup_databases)); + result += test_report("CB Error Cleanup:", do_test_context_cb_error_cleanup()); + finalize: - printf("\n"); if (rc != SQLITE_OK) printf("%s (%d)\n", (db) ? sqlite3_errmsg(db) : "N/A", rc); - db = close_db(db); + close_db(db); + db = NULL; cloudsync_memory_finalize(); - sqlite3_int64 memory_used = sqlite3_memory_used(); + int64_t memory_used = (int64_t)sqlite3_memory_used(); + result += test_report("Memory Leaks Check:", memory_used == 0); if (memory_used > 0) { - printf("Memory leaked: %lld B\n", memory_used); + printf("\tleaked: %" PRId64 " B\n", memory_used); result++; }