diff --git a/dbkit/sqlite/CHANGELOG.md b/dbkit/sqlite/CHANGELOG.md index d0bc857..49ee585 100644 --- a/dbkit/sqlite/CHANGELOG.md +++ b/dbkit/sqlite/CHANGELOG.md @@ -8,6 +8,26 @@ v0.x, minor releases may contain breaking changes. Releases of this module are tagged `dbkit/sqlite/vX.Y.Z`. +## [Unreleased] + +### Added + +- `Fingerprint`, a SHA-256 of the schema and the rows of the main database, read in one read-only transaction, without the rows a `Leave` names. +- `ErrNoSpace` and `SpaceError`, the error of a snapshot whose target folder has less free space than the copy and the margin. +- `ErrSnapshotExists`, which a snapshot error matches when a file already holds the target. +- `ErrSnapshotUnsynced`, which a snapshot error matches when the copy is in place but its folder failed to sync. +- `SnapshotOptions.Mode`, which narrows the permission bits of each copy below those of the live file and must keep the owner's read and write. + +### Changed + +- **Breaking.** `NewSnapshotter` takes `SnapshotOptions`. Pass `sqlite.SnapshotOptions{}` for no margin and the live file's mode. +- **Breaking.** A snapshot fails with `ErrNoSpace` before it writes the copy when the target folder has less free space than the used pages of the database plus `SnapshotOptions.Margin`. +- **Breaking.** `Snapshot` refuses a target folder on a network file system, or on Linux a FUSE one, before it takes the lock, so a missing folder now fails at that read unless the snapshot runs as root. +- **Breaking.** `Open` and `Snapshot` also refuse IBRIX, StorNext, Oracle ACFS and the VMware shared folders of the old vmhgfs kernel module on Linux, and every file system macOS does not mark local. +- A snapshot whose folder cannot move a file without replacing one fails with an error that names the folder and says to write snapshots to a local folder. +- The error of a target that already exists reads "dbkit: the snapshot target already exists: " and the target. +- The error of a copy in place whose folder failed to sync reads "dbkit: the snapshot is in place but its folder failed to sync: ", then the target and the cause. + ## [0.4.0] - 2026-10-31 ### Changed diff --git a/dbkit/sqlite/fingerprint.go b/dbkit/sqlite/fingerprint.go new file mode 100644 index 0000000..7246c51 --- /dev/null +++ b/dbkit/sqlite/fingerprint.go @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "cmp" + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "hash" + "math" + "slices" + "strings" +) + +const ( + // fingerprintVersion opens the hash of every fingerprint. + fingerprintVersion = "dbkit fingerprint 1" + // notInternal keeps out the objects SQLite keeps for itself, all but sqlite_sequence. + notInternal = "(name NOT LIKE 'sqlite\\_%' ESCAPE '\\' OR name = 'sqlite_sequence')" + // schemaRead reads each object of the main schema, in name order. + schemaRead = "SELECT type, name, tbl_name, ifnull(sql, '') FROM main.sqlite_schema WHERE " + notInternal + + " ORDER BY name, type" + // tableRead reads each table of the main schema with whether it has no rowid and its type, in name order. + tableRead = "SELECT name, wr, type FROM pragma_table_list WHERE schema = 'main' " + + "AND type IN ('table', 'shadow', 'virtual') AND " + notInternal + " ORDER BY name" + // columnRead reads the columns of the table its one argument names with their key position and hidden kind, in order. + columnRead = "SELECT name, pk, hidden FROM pragma_table_xinfo(?, 'main') ORDER BY cid" + // virtualType is the type pragma_table_list gives a virtual table. + virtualType = "virtual" + // shadowType is the type pragma_table_list gives a table that holds the data of a virtual table. + shadowType = "shadow" +) + +var ( + // rowAliases are the names of the rowid of a table, in the order Fingerprint tries them. + rowAliases = []string{"rowid", "_rowid_", "oid"} + // walked runs after Fingerprint reads each table, with the table's name. + walked = func(string) {} + // errNoFingerprintHandle refuses a fingerprint of a nil handle. + errNoFingerprintHandle = errors.New("dbkit: Fingerprint needs a database handle, got nil") +) + +// Leave names the tables and the rows Fingerprint leaves out, by names whose ASCII letters match in any case. +type Leave struct { + // Tables names the tables whose rows, not CREATE text, Fingerprint leaves out, a virtual table for its shadow tables. + Tables []string + // Rows names rows Fingerprint leaves out by the value of one column. + Rows []LeaveRows +} + +// LeaveRows names the rows of one table whose column holds one of some values. +type LeaveRows struct { + // Table is the table that holds the rows, never a virtual table. + Table string + // Column is the column whose value decides, where a NULL value never matches. + Column string + // Values are the values of Column whose rows Fingerprint leaves out, where NULL matches no row. + Values []any +} + +// fingerprintTable is one table Fingerprint reads. +type fingerprintTable struct { + // name is the table's name. + name string + // withoutRowid reports that the table has no rowid. + withoutRowid bool + // virtual reports that the table is a virtual table, whose rows its shadow tables hold. + virtual bool + // shadow reports that the table holds the data of a virtual table. + shadow bool +} + +// fingerprintColumn is one column of a table Fingerprint reads. +type fingerprintColumn struct { + // name is the column's name. + name string + // key is the column's position in the primary key, or 0 outside it. + key int64 + // generated reports that the column takes its value from an expression. + generated bool +} + +// statement is one query and its arguments. +type statement struct { + // text is the query. + text string + // args are the arguments of the query. + args []any +} + +// fingerprintRead is the transaction one fingerprint reads in and the hash it writes to. +type fingerprintRead struct { + // tx is the read-only transaction of the read. + tx *sql.Tx + // h is the hash of the fingerprint. + h hash.Hash +} + +// Fingerprint returns the SHA-256, in lowercase hex, of the schema and rows of the main database of db without leave. +func Fingerprint(ctx context.Context, db *sql.DB, leave Leave) (string, error) { + if db == nil { + return "", errNoFingerprintHandle + } + tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return "", fmt.Errorf("dbkit: begin the fingerprint read: %w", err) + } + defer func() { _ = tx.Rollback() }() + r := fingerprintRead{tx: tx, h: sha256.New()} + writeText(r.h, fingerprintVersion) + if err := r.hashSchema(ctx); err != nil { + return "", fmt.Errorf("dbkit: read the schema for the fingerprint: %w", err) + } + tables, err := r.readTables(ctx) + if err != nil { + return "", fmt.Errorf("dbkit: read the tables for the fingerprint: %w", err) + } + if err := leave.check(tables); err != nil { + return "", err + } + tables = slices.DeleteFunc(tables, func(table fingerprintTable) bool { return table.virtual }) + for _, table := range tables { + if err := r.hashTable(ctx, table, leave); err != nil { + return "", err + } + walked(table.name) + } + return hex.EncodeToString(r.h.Sum(nil)), nil +} + +// check returns the error for a table leave names that tables does not hold, or rows it names in a virtual table. +func (l Leave) check(tables []fingerprintTable) error { + for _, name := range l.Tables { + if _, err := findTable(tables, name); err != nil { + return err + } + } + return l.checkRows(tables) +} + +// checkRows returns the error for rows Rows names in a table tables does not hold, or in a virtual table. +func (l Leave) checkRows(tables []fingerprintTable) error { + for _, rows := range l.Rows { + table, err := findTable(tables, rows.Table) + if err != nil { + return err + } + if table.virtual { + return fmt.Errorf("dbkit: the fingerprint cannot leave out rows of the virtual table %q, "+ + "leave out the whole table instead", rows.Table) + } + } + return nil +} + +// findTable returns the table of tables named name, or the error that tables holds none. +func findTable(tables []fingerprintTable, name string) (fingerprintTable, error) { + i := slices.IndexFunc(tables, func(table fingerprintTable) bool { return sameName(table.name, name) }) + if i < 0 { + return fingerprintTable{}, fmt.Errorf("dbkit: the fingerprint leaves out the table %q, "+ + "which the database does not hold", name) + } + return tables[i], nil +} + +// leavesTable reports whether Tables names table, or the virtual table whose data table holds. +func (l Leave) leavesTable(table fingerprintTable) bool { + if l.namesTable(table.name) { + return true + } + return table.shadow && l.namesTable(shadowOwner(table.name)) +} + +// namesTable reports whether Tables names the table named table. +func (l Leave) namesTable(table string) bool { + return slices.ContainsFunc(l.Tables, func(name string) bool { return sameName(name, table) }) +} + +// shadowOwner returns the name of the virtual table whose data the shadow table named shadow holds. +func shadowOwner(shadow string) string { + return shadow[:max(strings.LastIndexByte(shadow, '_'), 0)] +} + +// hashSchema writes each object of the main schema to the hash. +func (r fingerprintRead) hashSchema(ctx context.Context) error { + var kind, name, table, text string + return r.eachRow(ctx, statement{text: schemaRead}, []any{&kind, &name, &table, &text}, func() { + writeTag(r.h, 'S') + for _, field := range []string{kind, name, table, text} { + writeText(r.h, field) + } + }) +} + +// readTables returns each table, shadow table and virtual table of the main schema, in name order. +func (r fingerprintRead) readTables(ctx context.Context) ([]fingerprintTable, error) { + var tables []fingerprintTable + var name, kind string + var withoutRowid int64 + err := r.eachRow(ctx, statement{text: tableRead}, []any{&name, &withoutRowid, &kind}, func() { + tables = append(tables, fingerprintTable{ + name: name, withoutRowid: withoutRowid != 0, virtual: kind == virtualType, shadow: kind == shadowType, + }) + }) + return tables, err +} + +// readColumns returns the columns of the table named table, generated ones too, in order. +func (r fingerprintRead) readColumns(ctx context.Context, table string) ([]fingerprintColumn, error) { + var columns []fingerprintColumn + var column fingerprintColumn + var hidden int64 + read := statement{text: columnRead, args: []any{table}} + err := r.eachRow(ctx, read, []any{&column.name, &column.key, &hidden}, func() { + column.generated = hidden != 0 + columns = append(columns, column) + }) + return columns, err +} + +// hashTable writes the name, the columns and the rows of table to the hash, without the rows leave keeps out. +func (r fingerprintRead) hashTable(ctx context.Context, table fingerprintTable, leave Leave) error { + columns, err := r.readColumns(ctx, table.name) + if err != nil { + return fmt.Errorf("dbkit: read the columns of %q for the fingerprint: %w", table.name, err) + } + stored := storedColumns(columns) + r.hashColumns(table.name, stored) + filter, err := leaveFilter(table.name, columns, leave.Rows) + if err != nil { + return err + } + if leave.leavesTable(table) { + return nil + } + order, err := rowOrder(table, columns) + if err != nil { + return err + } + read, width := rowRead(table, order, stored, filter) + if err := r.hashRows(ctx, read, width); err != nil { + return fmt.Errorf("dbkit: read the rows of %q for the fingerprint: %w", table.name, err) + } + return nil +} + +// hashColumns writes the name of the table named table and the names of its columns to the hash. +func (r fingerprintRead) hashColumns(table string, columns []fingerprintColumn) { + writeTag(r.h, 'T') + writeText(r.h, table) + writeCount(r.h, len(columns)) + for _, column := range columns { + writeText(r.h, column.name) + } +} + +// storedColumns returns the columns of columns that take no value from an expression. +func storedColumns(columns []fingerprintColumn) []fingerprintColumn { + return slices.DeleteFunc(slices.Clone(columns), func(c fingerprintColumn) bool { return c.generated }) +} + +// hashRows writes each row read reads to the hash, as a storage class and a raw value for each of width columns. +func (r fingerprintRead) hashRows(ctx context.Context, read statement, width int) error { + kinds := make([]string, width) + raws := make([]any, width) + dest := make([]any, 0, 2*width) + for i := range width { + dest = append(dest, &kinds[i], &raws[i]) + } + return r.eachRow(ctx, read, dest, func() { + writeTag(r.h, 'R') + for i := range width { + writeValue(r.h, kinds[i], raws[i]) + } + }) +} + +// rowRead returns the statement that reads each row of table filter keeps in order, and the values it reads per row. +func rowRead(table fingerprintTable, order string, columns []fingerprintColumn, filter statement) (statement, int) { + values, width := rawColumns(columns), len(columns) + if !table.withoutRowid { + values, width = "typeof("+order+"), "+order+", "+values, width+1 + } + text := "SELECT " + values + " FROM main." + quoteName(table.name) + filter.text + " ORDER BY " + order + return statement{text: text, args: filter.args}, width +} + +// rowOrder returns what orders the rows of table, its key columns with no rowid or a rowid alias no column hides. +func rowOrder(table fingerprintTable, columns []fingerprintColumn) (string, error) { + if table.withoutRowid { + return keyOrder(columns), nil + } + for _, alias := range rowAliases { + if !hasColumn(columns, alias) { + return alias, nil + } + } + return "", fmt.Errorf("dbkit: the fingerprint cannot order the rows of %q, whose columns hide rowid, _rowid_ and oid", + table.name) +} + +// keyOrder returns the key columns of columns, quoted, in key order. +func keyOrder(columns []fingerprintColumn) string { + keys := slices.DeleteFunc(slices.Clone(columns), func(c fingerprintColumn) bool { return c.key == 0 }) + slices.SortFunc(keys, func(a, b fingerprintColumn) int { return cmp.Compare(a.key, b.key) }) + names := make([]string, len(keys)) + for i, key := range keys { + names[i] = quoteName(key.name) + } + return strings.Join(names, ", ") +} + +// leaveFilter returns the WHERE clause that keeps out the rows leave names in the table named table, and its arguments. +func leaveFilter(table string, columns []fingerprintColumn, leave []LeaveRows) (statement, error) { + var clauses []string + var args []any + for _, rows := range leave { + if !sameName(rows.Table, table) { + continue + } + if !hasColumn(columns, rows.Column) { + return statement{}, fmt.Errorf("dbkit: the fingerprint leaves out rows by the column %q, "+ + "which the table %q does not hold", rows.Column, table) + } + clauses = append(clauses, leaveClause(rows)) + args = append(args, rows.Values...) + } + if len(clauses) == 0 { + return statement{}, nil + } + return statement{text: " WHERE " + strings.Join(clauses, " AND "), args: args}, nil +} + +// hasColumn reports whether columns holds a column named name. +func hasColumn(columns []fingerprintColumn, name string) bool { + return slices.ContainsFunc(columns, func(c fingerprintColumn) bool { return sameName(c.name, name) }) +} + +// sameName reports whether a and b name the same SQLite object, their ASCII letters matching in any case. +func sameName(a, b string) bool { + if len(a) != len(b) { + return false + } + for i := range len(a) { + if lowerASCII(a[i]) != lowerASCII(b[i]) { + return false + } + } + return true +} + +// lowerASCII returns c in lower case when it is an ASCII capital letter, and c itself otherwise. +func lowerASCII(c byte) byte { + if 'A' <= c && c <= 'Z' { + return c + 'a' - 'A' + } + return c +} + +// leaveClause returns the condition that keeps a row whose column holds none of the values of rows, NULL matching none. +func leaveClause(rows LeaveRows) string { + marks := strings.TrimSuffix(strings.Repeat("?, ", len(rows.Values)), ", ") + return "coalesce(" + quoteName(rows.Column) + " IN (" + marks + "), 0) = 0" +} + +// rawColumns returns the storage class and the raw value of each of columns, text read as its bytes. +func rawColumns(columns []fingerprintColumn) string { + parts := make([]string, len(columns)) + for i, column := range columns { + name := quoteName(column.name) + parts[i] = "typeof(" + name + "), CASE typeof(" + name + ") WHEN 'text' THEN CAST(" + name + + " AS BLOB) ELSE " + name + " END" + } + return strings.Join(parts, ", ") +} + +// quoteName returns name as an SQL identifier. +func quoteName(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +// eachRow runs s in the transaction, scans each row into dest and calls use, and returns the first error. +func (r fingerprintRead) eachRow(ctx context.Context, s statement, dest []any, use func()) (err error) { + rows, err := r.tx.QueryContext(ctx, s.text, s.args...) + if err != nil { + return err + } + defer func() { err = errors.Join(err, rows.Close()) }() + for rows.Next() { + if err := rows.Scan(dest...); err != nil { + return err + } + use() + } + return rows.Err() +} + +// writeValue writes one value to h, tagged with its storage class kind and read from raw. +func writeValue(h hash.Hash, kind string, raw any) { + switch kind { + case "null": + writeTag(h, 'n') + case "integer": + n, _ := raw.(int64) + writeField(h, 'i', binary.AppendVarint(nil, n)) + case "real": + f, _ := raw.(float64) + writeField(h, 'r', binary.BigEndian.AppendUint64(nil, math.Float64bits(f))) + default: + b, _ := raw.([]byte) + writeField(h, kind[0], b) + } +} + +// writeField writes tag and then b with its length to h. +func writeField(h hash.Hash, tag byte, b []byte) { + writeTag(h, tag) + writeCount(h, len(b)) + h.Write(b) +} + +// writeText writes s with its length to h. +func writeText(h hash.Hash, s string) { + writeCount(h, len(s)) + h.Write([]byte(s)) +} + +// writeCount writes n to h. +func writeCount(h hash.Hash, n int) { + h.Write(binary.AppendVarint(nil, int64(n))) +} + +// writeTag writes the one byte tag to h. +func writeTag(h hash.Hash, tag byte) { + h.Write([]byte{tag}) +} diff --git a/dbkit/sqlite/fingerprint_internal_test.go b/dbkit/sqlite/fingerprint_internal_test.go new file mode 100644 index 0000000..01fcec7 --- /dev/null +++ b/dbkit/sqlite/fingerprint_internal_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "errors" + "path/filepath" + "strings" + "testing" +) + +// internalFingerprintOpen returns a handle on a fresh file with the tables fp_a and fp_b, one row each. +func internalFingerprintOpen(t *testing.T) *sql.DB { + t.Helper() + db, err := Open("sqlite:"+filepath.Join(internalFolder(t), "site.db"), internalOptions()) + if err != nil { + t.Fatalf("Open() error = %v, want nil", err) + } + t.Cleanup(func() { _ = db.Close() }) + for _, query := range []string{ + "CREATE TABLE fp_a (a INTEGER NOT NULL)", + "CREATE TABLE fp_b (b INTEGER NOT NULL)", + "INSERT INTO fp_a VALUES (1)", + "INSERT INTO fp_b VALUES (1)", + } { + if _, err := db.ExecContext(t.Context(), query); err != nil { + t.Fatalf("ExecContext(%q) error = %v, want nil", query, err) + } + } + return db +} + +// internalAfterTable puts in place of walked one that runs then once, after the table first is read. +func internalAfterTable(t *testing.T, first string, then func()) { + t.Helper() + kept := walked + walked = func(table string) { + if table == first { + then() + } + } + t.Cleanup(func() { walked = kept }) +} + +// internalFingerprint returns the fingerprint of db and fails the test when there is none. +func internalFingerprint(t *testing.T, db *sql.DB) string { + t.Helper() + fp, err := Fingerprint(t.Context(), db, Leave{}) + if err != nil { + t.Fatalf("Fingerprint() error = %v, want nil", err) + } + return fp +} + +func TestAWriterCommitsDuringTheReadWithNoWait(t *testing.T) { + db := internalFingerprintOpen(t) + before := internalFingerprint(t, db) + var wrote error + internalAfterTable(t, "fp_a", func() { + ctx, cancel := context.WithTimeout(t.Context(), internalCheckpointDeadline) + defer cancel() + _, wrote = db.ExecContext(ctx, "INSERT INTO fp_b VALUES (2)") + }) + + during := internalFingerprint(t, db) + + if wrote != nil { + t.Errorf("the write during the read error = %v, want it committed within %v", wrote, internalCheckpointDeadline) + } + if during != before { + t.Errorf("Fingerprint() with a write during the read = %s, want %s from the start of the read", during, before) + } + if after := internalFingerprint(t, db); after == before { + t.Errorf("Fingerprint() after the write = %s, want it moved from %s", after, before) + } +} + +func TestAContextEndingDuringTheReadFails(t *testing.T) { + db := internalFingerprintOpen(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + internalAfterTable(t, "fp_a", cancel) + + fp, err := Fingerprint(ctx, db, Leave{}) + + if !errors.Is(err, context.Canceled) || fp != "" { + t.Errorf("Fingerprint() = %q, %v, want no fingerprint and an error matching context.Canceled", fp, err) + } +} + +func TestTwoTextsKeepTheirLengths(t *testing.T) { + t.Parallel() + + first, second := sha256.New(), sha256.New() + writeText(first, "ab") + writeText(first, "c") + writeText(second, "a") + writeText(second, "bc") + + if a, b := first.Sum(nil), second.Sum(nil); bytes.Equal(a, b) { + t.Errorf("the hash of the texts ab and c = %x, want it apart from the hash of a and bc", a) + } +} + +func TestEachRowAnswersAScanError(t *testing.T) { + db := internalFingerprintOpen(t) + tx, err := db.BeginTx(t.Context(), &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatalf("BeginTx() error = %v, want nil", err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + var a, b int64 + + err = fingerprintRead{tx: tx}.eachRow(t.Context(), statement{text: "SELECT a FROM fp_a"}, []any{&a, &b}, func() {}) + + if err == nil || !strings.Contains(err.Error(), "destination arguments") { + t.Errorf("eachRow() error = %v, want the scan error of two destinations for one column", err) + } +} diff --git a/dbkit/sqlite/fingerprint_test.go b/dbkit/sqlite/fingerprint_test.go new file mode 100644 index 0000000..38e18fd --- /dev/null +++ b/dbkit/sqlite/fingerprint_test.go @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "testing" + + "github.com/gopherium/framework/dbkit/sqlite" + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" +) + +const ( + // fpRowsTable creates a rowid table of synthetic rows. + fpRowsTable = "CREATE TABLE fp_rows (id INTEGER PRIMARY KEY, label TEXT, amount REAL)" + // fpRowsSeed fills fp_rows with two rows. + fpRowsSeed = "INSERT INTO fp_rows VALUES (1, 'first', 1.5), (2, 'second', 2.5)" + // fpSettingsTable creates a table of named settings. + fpSettingsTable = "CREATE TABLE fp_settings (name TEXT PRIMARY KEY, value TEXT)" + // fpSettingsSeed fills fp_settings with two kept settings, one rewritten setting and one with no name. + fpSettingsSeed = "INSERT INTO fp_settings (rowid, name, value) VALUES " + + "(1, 'site_name', 'synthetic'), (2, 'theme', 'plain'), (3, 'last_check', 'morning'), (4, NULL, 'unnamed')" + // fpJobsTable creates a table a job runner rewrites by itself. + fpJobsTable = "CREATE TABLE fp_jobs (id INTEGER PRIMARY KEY, lease TEXT)" + // fpSchemaRead is the statement Fingerprint reads the schema with. + fpSchemaRead = "SELECT type, name, tbl_name, ifnull(sql, '') FROM main.sqlite_schema " + + "WHERE (name NOT LIKE 'sqlite\\_%' ESCAPE '\\' OR name = 'sqlite_sequence') ORDER BY name, type" + // fpTableRead is the statement Fingerprint reads the table list with. + fpTableRead = "SELECT name, wr, type FROM pragma_table_list WHERE schema = 'main' " + + "AND type IN ('table', 'shadow', 'virtual') " + + "AND (name NOT LIKE 'sqlite\\_%' ESCAPE '\\' OR name = 'sqlite_sequence') ORDER BY name" + // fpColumnRead is the statement Fingerprint reads the columns of one table with. + fpColumnRead = "SELECT name, pk, hidden FROM pragma_table_xinfo(?, 'main') ORDER BY cid" + // fpOneTable creates a table of one column with no declared type. + fpOneTable = "CREATE TABLE fp_one (a)" + // fpOneRead is the statement Fingerprint reads the rows of fp_one with. + fpOneRead = `SELECT typeof(rowid), rowid, typeof("a"), CASE typeof("a") WHEN 'text' THEN CAST("a" AS BLOB) ` + + `ELSE "a" END FROM main."fp_one" ORDER BY rowid` + // fpSearchTable creates a full text search table of one column. + fpSearchTable = "CREATE VIRTUAL TABLE fp_search USING fts5(body)" + // fpGeneratedTable creates a table with a stored and a virtual generated column. + fpGeneratedTable = "CREATE TABLE fp_gen (a INTEGER, flag AS (a % 2) STORED, half AS (a / 2) VIRTUAL)" +) + +// fpHex is the form of a fingerprint. +var fpHex = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// fpOpen returns a handle on a fresh file after running statements on it. +func fpOpen(t *testing.T, statements ...string) *sql.DB { + t.Helper() + db := rulesOpen(t, testOptions()) + for _, statement := range statements { + rulesExec(t, db, statement) + } + return db +} + +// fpOf returns the fingerprint of db without leave and fails the test when there is none. +func fpOf(t *testing.T, db *sql.DB, leave sqlite.Leave) string { + t.Helper() + fp, err := sqlite.Fingerprint(t.Context(), db, leave) + if err != nil { + t.Fatalf("Fingerprint() error = %v, want nil", err) + } + if !fpHex.MatchString(fp) { + t.Fatalf("Fingerprint() = %q, want 64 lowercase hex digits", fp) + } + return fp +} + +// fpMustMove fails the test unless running change on db moves its fingerprint without leave. +func fpMustMove(t *testing.T, db *sql.DB, leave sqlite.Leave, change string) { + t.Helper() + before := fpOf(t, db, leave) + rulesExec(t, db, change) + if after := fpOf(t, db, leave); after == before { + t.Errorf("Fingerprint() after %q = %s, want it moved from %s", change, after, before) + } +} + +// fpMustStay fails the test unless running change on db keeps its fingerprint without leave. +func fpMustStay(t *testing.T, db *sql.DB, leave sqlite.Leave, change string) { + t.Helper() + before := fpOf(t, db, leave) + rulesExec(t, db, change) + if after := fpOf(t, db, leave); after != before { + t.Errorf("Fingerprint() after %q = %s, want it kept at %s", change, after, before) + } +} + +// fpMustFail fails the test unless Fingerprint of db without leave answers no fingerprint and the error want. +func fpMustFail(t *testing.T, db *sql.DB, leave sqlite.Leave, want string) { + t.Helper() + fp, err := sqlite.Fingerprint(t.Context(), db, leave) + if fmt.Sprint(err) != want || fp != "" { + t.Errorf("Fingerprint() = %q, %v, want no fingerprint and %q", fp, err, want) + } +} + +func TestTheSameDataGivesTheSameFingerprint(t *testing.T) { + t.Parallel() + + first := fpOpen(t, fpRowsTable, fpRowsSeed) + second := fpOpen(t, fpRowsTable, fpRowsSeed) + + if a, b := fpOf(t, first, sqlite.Leave{}), fpOf(t, second, sqlite.Leave{}); a != b { + t.Errorf("Fingerprint() of two files with the same data = %s and %s, want them equal", a, b) + } +} + +func TestAChangedRowMovesTheFingerprint(t *testing.T) { + t.Parallel() + + fpMustMove(t, fpOpen(t, fpRowsTable, fpRowsSeed), sqlite.Leave{}, "UPDATE fp_rows SET label = 'changed' WHERE id = 2") +} + +func TestAWithoutRowidTableHashesInKeyOrder(t *testing.T) { + t.Parallel() + + const keyed = "CREATE TABLE fp_keyed (code TEXT PRIMARY KEY, value INTEGER) WITHOUT ROWID" + backwards := fpOpen(t, keyed, "INSERT INTO fp_keyed VALUES ('b', 2)", "INSERT INTO fp_keyed VALUES ('a', 1)") + forwards := fpOpen(t, keyed, "INSERT INTO fp_keyed VALUES ('a', 1)", "INSERT INTO fp_keyed VALUES ('b', 2)") + + if a, b := fpOf(t, backwards, sqlite.Leave{}), fpOf(t, forwards, sqlite.Leave{}); a != b { + t.Errorf("Fingerprint() of the same rows written in two orders = %s and %s, want them equal", a, b) + } + fpMustMove(t, forwards, sqlite.Leave{}, "UPDATE fp_keyed SET value = 3 WHERE code = 'a'") +} + +func TestAnInsertIntoAVirtualTableMovesTheFingerprint(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSearchTable) + + fpMustMove(t, db, sqlite.Leave{}, "INSERT INTO fp_search VALUES ('synthetic words')") +} + +func TestAFingerprintNeverReadsAVirtualTable(t *testing.T) { + t.Parallel() + + fpOf(t, fpOpen(t, "CREATE VIRTUAL TABLE fp_outside USING fts5(body, content='fp_missing')"), sqlite.Leave{}) +} + +func TestALeftOutVirtualTableKeepsTheRowsOfItsShadowTablesOut(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSearchTable, "CREATE TABLE fp_search_notes (body TEXT)") + leave := sqlite.Leave{Tables: []string{"FP_SEARCH"}} + + fpMustStay(t, db, leave, "INSERT INTO fp_search VALUES ('synthetic words')") + fpMustMove(t, db, leave, "INSERT INTO fp_search_notes VALUES ('synthetic note')") +} + +func TestLeftOutRowsOfAVirtualTableFail(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSearchTable) + + fpMustFail(t, db, sqlite.Leave{Rows: []sqlite.LeaveRows{{Table: "fp_search", Column: "body"}}}, + `dbkit: the fingerprint cannot leave out rows of the virtual table "fp_search", leave out the whole table instead`) +} + +func TestLeftOutRowsMatchAGeneratedColumn(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + // column is the generated column whose value 1 leaves a row out. + column string + // stay changes a left-out row and keeps it out. + stay string + // move changes a kept row. + move string + }{ + {column: "flag", stay: "UPDATE fp_gen SET a = 3 WHERE a = 1", move: "UPDATE fp_gen SET a = 4 WHERE a = 2"}, + {column: "half", stay: "UPDATE fp_gen SET a = 3 WHERE a = 2", move: "UPDATE fp_gen SET a = 0 WHERE a = 1"}, + } { + t.Run(tc.column, func(t *testing.T) { + t.Parallel() + db := fpOpen(t, fpGeneratedTable, "INSERT INTO fp_gen (a) VALUES (1), (2)") + leave := sqlite.Leave{Rows: []sqlite.LeaveRows{{Table: "fp_gen", Column: tc.column, Values: []any{1}}}} + + fpMustStay(t, db, leave, tc.stay) + fpMustMove(t, db, leave, tc.move) + }) + } +} + +func TestAnIDTakenAndFreedMovesTheFingerprint(t *testing.T) { + t.Parallel() + + db := fpOpen(t, "CREATE TABLE fp_counted (id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT)") + before := fpOf(t, db, sqlite.Leave{}) + rulesExec(t, db, "INSERT INTO fp_counted (label) VALUES ('taken')") + rulesExec(t, db, "DELETE FROM fp_counted") + + if after := fpOf(t, db, sqlite.Leave{}); after == before { + t.Errorf("Fingerprint() after an id was taken and freed = %s, want it moved from %s", after, before) + } +} + +func TestStatisticsDoNotMoveTheFingerprint(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpRowsTable, fpRowsSeed, "CREATE INDEX fp_rows_label ON fp_rows (label)") + + fpMustStay(t, db, sqlite.Leave{}, "ANALYZE") + fpMustStay(t, db, sqlite.Leave{}, "PRAGMA optimize") +} + +func TestANewIndexMovesTheFingerprint(t *testing.T) { + t.Parallel() + + fpMustMove(t, fpOpen(t, fpRowsTable, fpRowsSeed), sqlite.Leave{}, "CREATE INDEX fp_rows_label ON fp_rows (label)") +} + +func TestAViewRecreatedWithAnotherBodyMovesTheFingerprint(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpRowsTable, fpRowsSeed, "CREATE VIEW fp_view AS SELECT label FROM fp_rows") + before := fpOf(t, db, sqlite.Leave{}) + rulesExec(t, db, "DROP VIEW fp_view") + rulesExec(t, db, "CREATE VIEW fp_view AS SELECT amount FROM fp_rows") + + if after := fpOf(t, db, sqlite.Leave{}); after == before { + t.Errorf("Fingerprint() after the view got another body = %s, want it moved from %s", after, before) + } +} + +func TestALeftOutTableKeepsItsRowsOut(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpRowsTable, fpRowsSeed, fpJobsTable) + leave := sqlite.Leave{Tables: []string{"fp_jobs"}} + + fpMustStay(t, db, leave, "INSERT INTO fp_jobs (lease) VALUES ('held')") + fpMustMove(t, db, leave, "UPDATE fp_rows SET amount = 9.5 WHERE id = 1") +} + +func TestLeftOutRowsStayOut(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed, fpRowsTable, fpRowsSeed) + leave := sqlite.Leave{Rows: []sqlite.LeaveRows{ + {Table: "fp_settings", Column: "name", Values: []any{"last_check", "last_serials"}}, + }} + + fpMustStay(t, db, leave, "UPDATE fp_settings SET value = 'evening' WHERE name = 'last_check'") + fpMustStay(t, db, leave, "INSERT INTO fp_settings VALUES ('last_serials', 'fresh')") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'dark' WHERE name = 'theme'") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'renamed' WHERE name IS NULL") + fpMustMove(t, db, leave, "UPDATE fp_rows SET label = 'changed' WHERE id = 1") +} + +func TestANullAmongTheLeftOutValuesMatchesNoRow(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed) + leave := sqlite.Leave{Rows: []sqlite.LeaveRows{ + {Table: "fp_settings", Column: "name", Values: []any{"last_check", nil}}, + }} + + fpMustStay(t, db, leave, "UPDATE fp_settings SET value = 'evening' WHERE name = 'last_check'") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'dark' WHERE name = 'theme'") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'renamed' WHERE name IS NULL") +} + +func TestLeaveNamesMatchTheirTablesAndColumnsInAnyCase(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed, fpJobsTable) + leave := sqlite.Leave{ + Tables: []string{"FP_JOBS"}, + Rows: []sqlite.LeaveRows{{Table: "Fp_Settings", Column: "NAME", Values: []any{"last_check"}}}, + } + + fpMustStay(t, db, leave, "INSERT INTO fp_jobs (lease) VALUES ('held')") + fpMustStay(t, db, leave, "UPDATE fp_settings SET value = 'evening' WHERE name = 'last_check'") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'dark' WHERE name = 'theme'") +} + +func TestTwoLeftOutRowsOnOneTableBothStayOut(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed) + leave := sqlite.Leave{Rows: []sqlite.LeaveRows{ + {Table: "fp_settings", Column: "name", Values: []any{"last_check"}}, + {Table: "fp_settings", Column: "name", Values: []any{"last_serials"}}, + }} + + fpMustStay(t, db, leave, "UPDATE fp_settings SET value = 'evening' WHERE name = 'last_check'") + fpMustStay(t, db, leave, "INSERT INTO fp_settings VALUES ('last_serials', 'fresh')") + fpMustMove(t, db, leave, "UPDATE fp_settings SET value = 'dark' WHERE name = 'theme'") +} + +func TestLeftOutRowsWithNoValuesLeaveNothingOut(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed) + none := sqlite.Leave{Rows: []sqlite.LeaveRows{{Table: "fp_settings", Column: "name"}}} + + if a, b := fpOf(t, db, none), fpOf(t, db, sqlite.Leave{}); a != b { + t.Errorf("Fingerprint() leaving out no values = %s, want %s as with no leave", a, b) + } +} + +func TestALeaveNameThatMatchesNothingFails(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpSettingsTable, fpSettingsSeed) + + fpMustFail(t, db, sqlite.Leave{Tables: []string{"fp_missing"}}, + `dbkit: the fingerprint leaves out the table "fp_missing", which the database does not hold`) + fpMustFail(t, db, sqlite.Leave{Rows: []sqlite.LeaveRows{{Table: "fp_missing", Column: "name"}}}, + `dbkit: the fingerprint leaves out the table "fp_missing", which the database does not hold`) + fpMustFail(t, db, sqlite.Leave{Rows: []sqlite.LeaveRows{{Table: "fp_settings", Column: "missing"}}}, + `dbkit: the fingerprint leaves out rows by the column "missing", which the table "fp_settings" does not hold`) + fpMustFail(t, db, sqlite.Leave{ + Tables: []string{"fp_settings"}, + Rows: []sqlite.LeaveRows{{Table: "fp_settings", Column: "missing"}}, + }, `dbkit: the fingerprint leaves out rows by the column "missing", which the table "fp_settings" does not hold`) +} + +func TestALeaveNameMatchesOnlyASCIILettersInAnyCase(t *testing.T) { + t.Parallel() + + db := fpOpen(t, `CREATE TABLE "fp_σ" (a)`) + + fpMustFail(t, db, sqlite.Leave{Tables: []string{"fp_Σ"}}, + `dbkit: the fingerprint leaves out the table "fp_Σ", which the database does not hold`) +} + +func TestAFingerprintOfANilHandleFails(t *testing.T) { + t.Parallel() + + fpMustFail(t, nil, sqlite.Leave{}, "dbkit: Fingerprint needs a database handle, got nil") +} + +func TestATableNameThatNeedsQuotesIsHashed(t *testing.T) { + t.Parallel() + + db := fpOpen(t, `CREATE TABLE "fp ""odd"" name" ("a ""b""" TEXT)`) + + fpMustMove(t, db, sqlite.Leave{}, `INSERT INTO "fp ""odd"" name" VALUES ('synthetic')`) +} + +func TestValuesKeepTheirStoredTypeAndBytes(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + // name names the pair. + name string + // column is the column of fp_values the pair goes into. + column string + // first and second are two values a fingerprint must tell apart. + first, second string + }{ + {name: "two reals that print alike", column: "raw", first: "0.1 + 0.2", second: "0.3"}, + {name: "two texts that read as the same time", column: "at", first: "'2026-11-08 10:00:00'", + second: "'2026-11-08T10:00:00Z'"}, + {name: "an empty text and NULL", column: "raw", first: "''", second: "NULL"}, + {name: "an integer and its text", column: "raw", first: "1", second: "'1'"}, + {name: "a text and its bytes", column: "raw", first: "'1'", second: "x'31'"}, + {name: "an empty text and an empty blob", column: "raw", first: "''", second: "x''"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + const table = "CREATE TABLE fp_values (id INTEGER PRIMARY KEY, at DATETIME, raw)" + insert := "INSERT INTO fp_values (id, " + tc.column + ") VALUES (1, " + first := fpOpen(t, table, insert+tc.first+")") + second := fpOpen(t, table, insert+tc.second+")") + + if a, b := fpOf(t, first, sqlite.Leave{}), fpOf(t, second, sqlite.Leave{}); a == b { + t.Errorf("Fingerprint() of %s and of %s = %s, want them apart", tc.first, tc.second, a) + } + }) + } +} + +func TestTwoValuesInARowKeepTheirLengths(t *testing.T) { + t.Parallel() + + const table = "CREATE TABLE fp_pair (x, y)" + first := fpOpen(t, table, "INSERT INTO fp_pair VALUES ('a', 'b')") + second := fpOpen(t, table, "INSERT INTO fp_pair VALUES ('at', x'')") + + if a, b := fpOf(t, first, sqlite.Leave{}), fpOf(t, second, sqlite.Leave{}); a == b { + t.Errorf("Fingerprint() of the rows ('a', 'b') and ('at', x'') = %s, want them apart", a) + } +} + +func TestARowidHiddenByEveryAliasFails(t *testing.T) { + t.Parallel() + + db := fpOpen(t, "CREATE TABLE fp_hidden (rowid TEXT, _rowid_ TEXT, oid TEXT)") + + fpMustFail(t, db, sqlite.Leave{}, + `dbkit: the fingerprint cannot order the rows of "fp_hidden", whose columns hide rowid, _rowid_ and oid`) +} + +func TestARowidHiddenByOneAliasOrdersByAnother(t *testing.T) { + t.Parallel() + + const shadowed = "CREATE TABLE fp_shadowed (rowid TEXT, label TEXT)" + db := fpOpen(t, shadowed, "INSERT INTO fp_shadowed VALUES ('same', 'first'), ('same', 'second')") + stored := fpOpen(t, shadowed, "INSERT INTO fp_shadowed VALUES ('b', 'x'), ('a', 'y')") + reversed := fpOpen(t, shadowed, "INSERT INTO fp_shadowed VALUES ('a', 'y'), ('b', 'x')") + + fpMustMove(t, db, sqlite.Leave{}, "UPDATE fp_shadowed SET label = 'changed' WHERE label = 'second'") + if a, b := fpOf(t, stored, sqlite.Leave{}), fpOf(t, reversed, sqlite.Leave{}); a == b { + t.Errorf("Fingerprint() of the same rows stored in two orders = %s, want them apart by the real rowid", a) + } +} + +func TestAFingerprintNeverComputesAGeneratedColumn(t *testing.T) { + t.Parallel() + + db := fpOpen(t, "CREATE TABLE fp_doc (body TEXT)", "INSERT INTO fp_doc (body) VALUES ('not json')", + "ALTER TABLE fp_doc ADD COLUMN title AS (json_extract(body, '$.title')) VIRTUAL") + + fpOf(t, db, sqlite.Leave{}) +} + +func TestANewRowidMovesTheFingerprint(t *testing.T) { + t.Parallel() + + fpMustMove(t, fpOpen(t, "CREATE TABLE fp_body (body TEXT)", "INSERT INTO fp_body VALUES ('a')"), sqlite.Leave{}, + "UPDATE fp_body SET rowid = 99") + fpMustMove(t, fpOpen(t, "CREATE TABLE fp_shadowed (rowid TEXT, label TEXT)", + "INSERT INTO fp_shadowed VALUES ('same', 'first')"), sqlite.Leave{}, "UPDATE fp_shadowed SET _rowid_ = 99") +} + +func TestAGeneratedColumnNamedRowidOrdersByTheRealRowid(t *testing.T) { + t.Parallel() + + const generated = "CREATE TABLE fp_genrow (a INTEGER, rowid AS (a * 10))" + stored := fpOpen(t, generated, "INSERT INTO fp_genrow (a) VALUES (2), (1)") + reversed := fpOpen(t, generated, "INSERT INTO fp_genrow (a) VALUES (1), (2)") + + if a, b := fpOf(t, stored, sqlite.Leave{}), fpOf(t, reversed, sqlite.Leave{}); a == b { + t.Errorf("Fingerprint() of the same rows stored in two orders = %s, want them apart by the real rowid", a) + } +} + +func TestAFailedReadAnswersNoFingerprint(t *testing.T) { + t.Parallel() + + for _, statement := range []string{fpSchemaRead, fpTableRead, fpColumnRead, fpOneRead} { + t.Run(statement, func(t *testing.T) { + t.Parallel() + failed := errors.New("fingerprint test: the read failed on purpose") + faults := &sqlitetest.Faults{} + db := sqlitetest.OpenWithFaults(t, testOptions(), faults) + rulesExec(t, db, fpOneTable) + faults.FailStatement(statement, failed) + + fp, err := sqlite.Fingerprint(t.Context(), db, sqlite.Leave{}) + + if !errors.Is(err, failed) || fp != "" { + t.Errorf("Fingerprint() = %q, %v, want no fingerprint and an error matching %v", fp, err, failed) + } + }) + } +} + +func TestAFingerprintUnderAnEndedContextFails(t *testing.T) { + t.Parallel() + + db := fpOpen(t, fpRowsTable, fpRowsSeed) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + fp, err := sqlite.Fingerprint(ctx, db, sqlite.Leave{}) + + if !errors.Is(err, context.Canceled) || fp != "" { + t.Errorf("Fingerprint() = %q, %v, want no fingerprint and an error matching context.Canceled", fp, err) + } +} diff --git a/dbkit/sqlite/fs_darwin.go b/dbkit/sqlite/fs_darwin.go index 9535bb5..02e1f73 100644 --- a/dbkit/sqlite/fs_darwin.go +++ b/dbkit/sqlite/fs_darwin.go @@ -11,18 +11,41 @@ import ( // statfs reads the file system facts of the file or folder at path. var statfs = unix.Statfs -// networkSystems holds each network file system Open refuses, by its Statfs_t.Fstypename. -var networkSystems = map[string]bool{"nfs": true, "smbfs": true, "afpfs": true, "webdav": true} +// remoteSystem returns the Fstypename of the file system of path when macOS does not mark it local, or "" when it does. +func remoteSystem(path string) (string, error) { + var st unix.Statfs_t + if err := statfs(path, &st); err != nil { + return "", err + } + if st.Flags&unix.MNT_LOCAL != 0 { + return "", nil + } + return unix.ByteSliceToString(st.Fstypename[:]), nil +} // checkFolder returns an error when the file or folder at path is on a network or unreadable file system. func checkFolder(path string) error { - var st unix.Statfs_t - if err := statfs(path, &st); err != nil { + name, err := remoteSystem(path) + switch { + case err != nil: return fmt.Errorf("dbkit: read the file system of %s: %w", path, err) - } - if name := unix.ByteSliceToString(st.Fstypename[:]); networkSystems[name] { + case name != "": return fmt.Errorf("dbkit: %s is on a network file system (%s), where SQLite locks are unreliable, "+ "keep the database on a local disk", path, name) } return nil } + +// snapshotSystem returns the Fstypename of the file system of folder when macOS does not mark it local, or "". +func snapshotSystem(folder string) (string, error) { + return remoteSystem(folder) +} + +// freeSpace returns the bytes the file system of path leaves free to a user who is not root. +func freeSpace(path string) (int64, error) { + var st unix.Statfs_t + if err := statfs(path, &st); err != nil { + return 0, err + } + return blockBytes(st.Bavail, uint64(st.Bsize)), nil +} diff --git a/dbkit/sqlite/fs_linux.go b/dbkit/sqlite/fs_linux.go index de9d522..fb3dacf 100644 --- a/dbkit/sqlite/fs_linux.go +++ b/dbkit/sqlite/fs_linux.go @@ -11,7 +11,7 @@ import ( // statfs reads the file system facts of the file or folder at path. var statfs = unix.Statfs -// networkSystems names each network file system Open refuses, by its Statfs_t.Type magic number. +// networkSystems names each network file system Open and Snapshot refuse, by its Statfs_t.Type magic number. var networkSystems = map[uint32]string{ unix.NFS_SUPER_MAGIC: "NFS", unix.SMB_SUPER_MAGIC: "SMB", @@ -32,17 +32,53 @@ var networkSystems = map[uint32]string{ 0x01161970: "GFS2", 0x20030528: "OrangeFS", 0xAAD7AAEA: "Panasas", + 0xBACBACBC: "vmhgfs", + 0x013111A8: "IBRIX", + 0xBEEFDEAD: "StorNext", + 0x61636673: "ACFS", +} + +// systemType returns the Statfs_t.Type magic number of the file system of the file or folder at path. +func systemType(path string) (uint32, error) { + var st unix.Statfs_t + err := statfs(path, &st) + return uint32(st.Type), err } // checkFolder returns an error when the file or folder at path is on a network or unreadable file system. func checkFolder(path string) error { - var st unix.Statfs_t - if err := statfs(path, &st); err != nil { + magic, err := systemType(path) + if err != nil { return fmt.Errorf("dbkit: read the file system of %s: %w", path, err) } - if name, ok := networkSystems[uint32(st.Type)]; ok { + if name, ok := networkSystems[magic]; ok { return fmt.Errorf("dbkit: %s is on a network file system (%s), where SQLite locks are unreliable, "+ "keep the database on a local disk", path, name) } return nil } + +// snapshotSystem returns the name of the network or FUSE file system of folder, or "" for any other. +func snapshotSystem(folder string) (string, error) { + magic, err := systemType(folder) + switch { + case err != nil: + return "", err + case magic == unix.FUSE_SUPER_MAGIC: + return "FUSE", nil + } + return networkSystems[magic], nil +} + +// freeSpace returns the bytes the file system of path leaves free to a user who is not root. +func freeSpace(path string) (int64, error) { + var st unix.Statfs_t + if err := statfs(path, &st); err != nil { + return 0, err + } + size := uint64(st.Frsize) + if size == 0 { + size = uint64(st.Bsize) + } + return blockBytes(st.Bavail, size), nil +} diff --git a/dbkit/sqlite/fs_linux_test.go b/dbkit/sqlite/fs_linux_test.go index 931ac19..e2d90ca 100644 --- a/dbkit/sqlite/fs_linux_test.go +++ b/dbkit/sqlite/fs_linux_test.go @@ -4,6 +4,7 @@ package sqlite import ( "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -13,6 +14,8 @@ import ( "testing" "golang.org/x/sys/unix" + + "github.com/gopherium/framework/dbkit" ) // fsReporting returns a statfs that reports the file system magic for every path. @@ -28,20 +31,29 @@ func fsSetType[T int32 | int64 | uint32](field *T, magic uint32) { *field = T(magic) } -// fsLocal is the ext4 magic number. -const fsLocal = 0xEF53 +const ( + // fsLocal is the ext4 magic number. + fsLocal = 0xEF53 + // fsLocalAdvice is what the error of a refused snapshot folder tells its owner to do. + fsLocalAdvice = "write snapshots to a local folder, then copy them off the machine" +) -// fsNetworkAt returns a statfs that reports NFS for path and ext4 for every other path. -func fsNetworkAt(path string) func(string, *unix.Statfs_t) error { +// fsReportingAt returns a statfs that reports the file system magic for path and ext4 for every other path. +func fsReportingAt(path string, magic uint32) func(string, *unix.Statfs_t) error { return func(p string, st *unix.Statfs_t) error { fsSetType(&st.Type, fsLocal) if p == path { - fsSetType(&st.Type, 0x6969) + fsSetType(&st.Type, magic) } return nil } } +// fsNetworkAt returns a statfs that reports NFS for path and ext4 for every other path. +func fsNetworkAt(path string) func(string, *unix.Statfs_t) error { + return fsReportingAt(path, 0x6969) +} + // fsRecording returns a statfs that reports ext4 for every path and records each path it reads. func fsRecording(read *[]string) func(string, *unix.Statfs_t) error { return func(p string, st *unix.Statfs_t) error { @@ -121,6 +133,10 @@ func TestNetworkFileSystemsAreRefused(t *testing.T) { {"GFS2", 0x01161970}, {"OrangeFS", 0x20030528}, {"Panasas", 0xAAD7AAEA}, + {"vmhgfs", 0xBACBACBC}, + {"IBRIX", 0x013111A8}, + {"StorNext", 0xBEEFDEAD}, + {"ACFS", 0x61636673}, } for _, c := range network { if got, ok := networkSystems[c.magic]; !ok || got != c.name { @@ -226,6 +242,18 @@ func TestOpenChecksAPlainLocalFileAndItsFolder(t *testing.T) { } } +func TestOpenAcceptsAFUSEFolder(t *testing.T) { + file := fsFile(t) + fsSwap(t, fsReporting(0x65735546)) + + db, err := Open("sqlite:"+file, internalOptions()) + + if err != nil { + t.Fatalf("Open() error = %v, want a FUSE folder accepted", err) + } + t.Cleanup(func() { _ = db.Close() }) +} + func TestOpenRefusesAFolderItCannotRead(t *testing.T) { t.Parallel() @@ -246,3 +274,41 @@ func TestALocalFolderIsAccepted(t *testing.T) { t.Errorf("checkFolder() error = %v, want the test's local folder accepted", err) } } + +func TestASnapshotRefusesANetworkOrFUSEFolderBeforeItsLock(t *testing.T) { + for _, tc := range []struct { + // name is the name the error gives the file system. + name string + // magic is the Statfs_t.Type of the file system. + magic uint32 + }{ + {name: "NFS", magic: 0x6969}, + {name: "StorNext", magic: 0xBEEFDEAD}, + {name: "FUSE", magic: 0x65735546}, + } { + t.Run(tc.name, func(t *testing.T) { + db, folder := internalSnapshotOpen(t) + fsSwap(t, fsReportingAt(folder, tc.magic)) + + got := internalSnapshotTo(t, db, filepath.Join(folder, "copy.db")) + + want := "dbkit: the snapshot folder " + folder + " is on a network or FUSE file system (" + tc.name + "), " + + fsLocalAdvice + if got.snap != (dbkit.Snapshot{}) || fmt.Sprint(got.err) != want { + t.Errorf("Snapshot() = %+v, %v, want no snapshot and %q", got.snap, got.err, want) + } + internalMustHoldOnly(t, folder) + }) + } +} + +func TestAFailedSnapshotFolderReadNamesTheFolder(t *testing.T) { + db, folder := internalSnapshotOpen(t) + fsSwap(t, func(string, *unix.Statfs_t) error { return unix.EIO }) + + got := internalSnapshotTo(t, db, filepath.Join(folder, "copy.db")) + + internalMustAnswer(t, got, internalAnswer{err: unix.EIO}, + "dbkit: read the file system of the snapshot folder "+folder+": "+unix.EIO.Error()) + internalMustHoldOnly(t, folder) +} diff --git a/dbkit/sqlite/fs_other.go b/dbkit/sqlite/fs_other.go index 49d0653..43c8c5b 100644 --- a/dbkit/sqlite/fs_other.go +++ b/dbkit/sqlite/fs_other.go @@ -14,3 +14,14 @@ func checkFolder(string) error { return fmt.Errorf("dbkit: Open checks for a network file system on linux and darwin only, and refuses %s", runtime.GOOS) } + +// snapshotSystem refuses every snapshot folder on a system with no network file system check. +func snapshotSystem(string) (string, error) { + return "", fmt.Errorf("dbkit: Snapshot checks for a network file system on linux and darwin only, and refuses %s", + runtime.GOOS) +} + +// freeSpace refuses to read the free space on a system with no free space reader. +func freeSpace(string) (int64, error) { + return 0, fmt.Errorf("dbkit: Snapshot reads the free space on linux and darwin only, and refuses %s", runtime.GOOS) +} diff --git a/dbkit/sqlite/lock_unix_test.go b/dbkit/sqlite/lock_unix_test.go index b6cf4d0..e7678cc 100644 --- a/dbkit/sqlite/lock_unix_test.go +++ b/dbkit/sqlite/lock_unix_test.go @@ -597,7 +597,7 @@ func TestAFailedSnapshotLockCallStopsTheSnapshot(t *testing.T) { flock = func(int, int) error { return unix.ENOLCK } t.Cleanup(func() { flock = kept }) - got, err := NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: take the snapshot lock " + filepath.Join(folder, snapshotLockFile) + ": " + unix.ENOLCK.Error() if !errors.Is(err, unix.ENOLCK) || err.Error() != want || got.Path != "" { @@ -692,7 +692,7 @@ func TestAHardLinkAtTheSnapshotLockIsRefused(t *testing.T) { lockAs(t, tc.uid) tries := lockTries(t) - got, err := NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: the snapshot lock " + lock + " must be a regular file with one link" if err == nil || err.Error() != want || got.Path != "" { @@ -735,7 +735,8 @@ func TestTheSnapshotLockOfARunAsRootBelongsToTheDatabaseOwner(t *testing.T) { folder := internalFolder(t) lockRootChain(t, folder) - if _, err := NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")); err != nil { + target := filepath.Join(folder, "copy.db") + if _, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target); err != nil { t.Fatalf("Snapshot() error = %v, want nil", err) } @@ -781,7 +782,7 @@ func TestARunAsRootKeepsTheOwnerOfALockFileThatExists(t *testing.T) { lockAs(t, lockRootUID) target := filepath.Join(folder, "copy.db") - if got, err := NewSnapshotter(db).Snapshot(t.Context(), target); err != nil || got.Path != target { + if got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target); err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) } @@ -935,7 +936,7 @@ func lockUnixMode(mode fs.FileMode) uint32 { func lockMustRefuseAsRoot(t *testing.T, db *sql.DB, target, want string) { t.Helper() tries := lockTries(t) - got, err := NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target) if err == nil || err.Error() != want || got.Path != "" { t.Errorf("Snapshot(%q) = %+v, %v, want no snapshot and %q", target, got, err, want) } @@ -945,7 +946,7 @@ func lockMustRefuseAsRoot(t *testing.T, db *sql.DB, target, want string) { // lockMustTakeAsRoot fails the test unless a snapshot of db to target answers the path target. func lockMustTakeAsRoot(t *testing.T, db *sql.DB, target string) { t.Helper() - if got, err := NewSnapshotter(db).Snapshot(t.Context(), target); err != nil || got.Path != target { + if got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target); err != nil || got.Path != target { t.Fatalf("Snapshot(%q) = %+v, %v, want the path %s", target, got, err, target) } } @@ -1143,7 +1144,7 @@ func TestALinkRemovedBeforeItIsReadStopsASnapshotAsRoot(t *testing.T) { lockAs(t, lockRootUID) tries := lockTries(t) - got, err := NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(link, "copy.db")) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), filepath.Join(link, "copy.db")) want := "dbkit: read the link " + link + ": " if !errors.Is(err, fs.ErrNotExist) || !strings.HasPrefix(err.Error(), want) || got.Path != "" { @@ -1160,7 +1161,7 @@ func TestASnapshotAsAnotherUserReadsNoFolder(t *testing.T) { lockAs(t, lockServiceUID) target := filepath.Join(folder, "copy.db") - got, err := NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) diff --git a/dbkit/sqlite/snapshot.go b/dbkit/sqlite/snapshot.go index 4dff6c2..82a939c 100644 --- a/dbkit/sqlite/snapshot.go +++ b/dbkit/sqlite/snapshot.go @@ -37,10 +37,20 @@ const ( snapshotLockFile = ".dbkit-snapshot.lock" // snapshotLockName names the snapshot lock in its errors. snapshotLockName = "snapshot lock" + // localFolderAdvice tells the owner of a refused snapshot folder what to do instead. + localFolderAdvice = "write snapshots to a local folder, then copy them off the machine" + // ownerReadWrite holds the owner's read and write permission bits. + ownerReadWrite fs.FileMode = 0o600 ) -// ErrSnapshotRunning is the error of a snapshot into a folder another snapshot is writing into. -var ErrSnapshotRunning = errors.New("dbkit: another snapshot is writing into this folder") +var ( + // ErrSnapshotRunning is the error of a snapshot into a folder another snapshot is writing into. + ErrSnapshotRunning = errors.New("dbkit: another snapshot is writing into this folder") + // ErrSnapshotExists is the error of a snapshot whose target a file already holds. + ErrSnapshotExists = errors.New("dbkit: the snapshot target already exists") + // ErrSnapshotUnsynced is the error of a snapshot whose copy is in place but whose folder failed to sync. + ErrSnapshotUnsynced = errors.New("dbkit: the snapshot is in place but its folder failed to sync") +) var ( // syncFile writes the data of an open file to its disk. @@ -51,15 +61,46 @@ var ( errNoSnapshotHandle = errors.New("dbkit: NewSnapshotter needs a database handle, got nil") ) +// SnapshotOptions configures the snapshots of one Snapshotter. +type SnapshotOptions struct { + // Margin is the bytes a snapshot leaves free in its target folder beside the copy, 0 or more. + Margin int64 + // Mode narrows the permission bits of each copy below the live file's and keeps the owner's read and write, or is 0. + Mode fs.FileMode +} + +// check returns the error for options Snapshot refuses. +func (o SnapshotOptions) check() error { + switch { + case o.Margin < 0: + return fmt.Errorf("dbkit: the snapshot option Margin must be 0 or more, got %d", o.Margin) + case o.Mode&^fs.ModePerm != 0: + return fmt.Errorf("dbkit: the snapshot option Mode must hold permission bits only, got %v", o.Mode) + case o.Mode != 0 && o.Mode&ownerReadWrite != ownerReadWrite: + return fmt.Errorf("dbkit: the snapshot option Mode must keep the owner's read and write bits, got %v", o.Mode) + } + return nil +} + +// copyMode returns the permission bits of a copy of a live file with the mode live. +func (o SnapshotOptions) copyMode(live fs.FileMode) fs.FileMode { + if o.Mode == 0 { + return live.Perm() + } + return live.Perm() & o.Mode +} + // snapshotter writes snapshots of the database of one handle. type snapshotter struct { // db is the handle whose database is copied. db *sql.DB + // opts configures each snapshot. + opts SnapshotOptions } -// NewSnapshotter returns a Snapshotter of the database of db. -func NewSnapshotter(db *sql.DB) dbkit.Snapshotter { - return &snapshotter{db: db} +// NewSnapshotter returns a Snapshotter of the database of db with opts. +func NewSnapshotter(db *sql.DB, opts SnapshotOptions) dbkit.Snapshotter { + return &snapshotter{db: db, opts: opts} } // Snapshot moves a checked copy of the database to target and checkpoints the log, with Path set once target holds it. @@ -67,11 +108,14 @@ func (s *snapshotter) Snapshot(ctx context.Context, target string) (dbkit.Snapsh if s.db == nil { return dbkit.Snapshot{}, errNoSnapshotHandle } + if err := s.opts.check(); err != nil { + return dbkit.Snapshot{}, err + } if err := s.copyLocked(ctx, target); err != nil { return dbkit.Snapshot{}, err } if err := syncPath(filepath.Dir(target)); err != nil { - return dbkit.Snapshot{Path: target}, fmt.Errorf("dbkit: sync the folder of the snapshot %s: %w", target, err) + return dbkit.Snapshot{Path: target}, fmt.Errorf("%w: %s: %w", ErrSnapshotUnsynced, target, err) } c, err := s.checkpoint(ctx) if err != nil { @@ -98,32 +142,36 @@ func (s *snapshotter) prepare(ctx context.Context, target string) (string, *file if err := checkTarget(target); err != nil { return "", nil, err } - if err := checkRootFolder(filepath.Dir(target)); err != nil { + folder := filepath.Dir(target) + if err := checkFolders(folder); err != nil { return "", nil, err } path, live, err := s.liveFile(ctx) if err != nil { return "", nil, err } - folder := filepath.Dir(target) locker := &fileLocker{name: snapshotLockName, path: filepath.Join(folder, snapshotLockFile), database: path} if err := locker.tryLock(ErrSnapshotRunning); err != nil { return "", nil, err } - partial, err := startCopy(target, live) + partial, err := s.startCopy(ctx, target, live) if err != nil { return "", nil, errors.Join(err, locker.unlock()) } return partial, locker, nil } -// startCopy clears the stale copies beside target and creates its empty copy, and returns the copy's path. -func startCopy(target string, live fs.FileInfo) (string, error) { - if err := removePartials(filepath.Dir(target), live); err != nil { +// startCopy clears the stale copies beside target, checks the free space, creates the empty copy and returns its path. +func (s *snapshotter) startCopy(ctx context.Context, target string, live fs.FileInfo) (string, error) { + folder := filepath.Dir(target) + if err := removePartials(folder, live); err != nil { + return "", err + } + if err := s.checkSpace(ctx, folder); err != nil { return "", err } partial := target + partialSuffix - if err := createEmpty(partial, live.Mode().Perm()); err != nil { + if err := createEmpty(partial, s.opts.copyMode(live.Mode())); err != nil { return "", fmt.Errorf("dbkit: create the snapshot copy %s: %w", partial, err) } return partial, nil @@ -166,6 +214,9 @@ func (s *snapshotter) writeCopy(ctx context.Context, partial, target string) err switch { case errors.Is(err, fs.ErrExist): return existingTarget(target) + case errors.Is(err, noReplaceUnsupported): + return fmt.Errorf("dbkit: the file system of the snapshot folder %s cannot move a file without replacing one, "+ + "%s: %w", filepath.Dir(target), localFolderAdvice, err) case err != nil: return fmt.Errorf("dbkit: move the snapshot copy to %s: %w", target, err) } @@ -174,7 +225,7 @@ func (s *snapshotter) writeCopy(ctx context.Context, partial, target string) err // existingTarget returns the error for a target a file already holds. func existingTarget(target string) error { - return fmt.Errorf("dbkit: the snapshot target %s already exists", target) + return fmt.Errorf("%w: %s", ErrSnapshotExists, target) } // checkTarget returns the error for a target Snapshot refuses. @@ -203,6 +254,22 @@ func checkTarget(target string) error { return nil } +// checkFolders returns the error for a snapshot folder Snapshot refuses, or for a folder above it as root. +func checkFolders(folder string) error { + if err := checkRootFolder(folder); err != nil { + return err + } + name, err := snapshotSystem(folder) + switch { + case err != nil: + return fmt.Errorf("dbkit: read the file system of the snapshot folder %s: %w", folder, err) + case name != "": + return fmt.Errorf("dbkit: the snapshot folder %s is on a network or FUSE file system (%s), %s", + folder, name, localFolderAdvice) + } + return nil +} + // removePartials removes the regular copies and copy journals in folder, never the live database or its journal. func removePartials(folder string, live fs.FileInfo) error { entries, err := os.ReadDir(folder) diff --git a/dbkit/sqlite/snapshot_darwin.go b/dbkit/sqlite/snapshot_darwin.go index 145890a..eeaa61b 100644 --- a/dbkit/sqlite/snapshot_darwin.go +++ b/dbkit/sqlite/snapshot_darwin.go @@ -4,6 +4,9 @@ package sqlite import "golang.org/x/sys/unix" +// noReplaceUnsupported is the error of a file system that cannot move a file without replacing one. +const noReplaceUnsupported = unix.ENOTSUP + // renameNoReplace moves the file at from to the name to, and fails with EEXIST when a file holds that name. func renameNoReplace(from, to string) error { return unix.RenamexNp(from, to, unix.RENAME_EXCL) diff --git a/dbkit/sqlite/snapshot_internal_test.go b/dbkit/sqlite/snapshot_internal_test.go index 318ce93..8456ada 100644 --- a/dbkit/sqlite/snapshot_internal_test.go +++ b/dbkit/sqlite/snapshot_internal_test.go @@ -26,6 +26,8 @@ const ( internalLateTarget = "a file written between the copy and the move" // internalCheckpointDeadline is the deadline of a checkpoint behind a held writer. internalCheckpointDeadline = 100 * time.Millisecond + // internalWorldReadMode is the mode of a live file its owner reads and writes and everyone reads. + internalWorldReadMode fs.FileMode = 0o644 ) var ( @@ -99,6 +101,36 @@ func internalMustHoldOnly(t *testing.T, folder string, want ...string) { } } +func TestTheEmptyCopyHasTheNarrowedModeBeforeItHoldsData(t *testing.T) { + db, folder := internalSnapshotOpen(t) + s := &snapshotter{db: db, opts: SnapshotOptions{Mode: ownerReadWrite}} + path, _, err := s.liveFile(t.Context()) + if err != nil { + t.Fatalf("liveFile() error = %v, want nil", err) + } + if err := os.Chmod(path, internalWorldReadMode); err != nil { + t.Fatalf("Chmod(%s) error = %v, want nil", path, err) + } + live, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat(%s) error = %v, want nil", path, err) + } + + partial, err := s.startCopy(t.Context(), filepath.Join(folder, "copy.db"), live) + + if err != nil { + t.Fatalf("startCopy() error = %v, want nil", err) + } + info, err := os.Lstat(partial) + if err != nil { + t.Fatalf("Lstat(%s) error = %v, want nil", partial, err) + } + if info.Mode().Perm() != ownerReadWrite || info.Size() != 0 { + t.Errorf("the copy has the mode %v and %d bytes, want the mode %v and none", info.Mode().Perm(), info.Size(), + ownerReadWrite) + } +} + // internalDriver returns the driver value of the nil function list. func internalDriver(t *testing.T) *modernc.Driver { t.Helper() @@ -212,7 +244,7 @@ type internalAnswer struct { // internalSnapshotTo runs a snapshot of db to target and returns what it answered. func internalSnapshotTo(t *testing.T, db *sql.DB, target string) internalAnswer { t.Helper() - snap, err := NewSnapshotter(db).Snapshot(t.Context(), target) + snap, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target) return internalAnswer{snap: snap, err: err} } @@ -255,7 +287,7 @@ func TestSnapshotSyncsTheCopyBeforeTheMoveAndTheFolderAfter(t *testing.T) { } t.Cleanup(func() { syncFile, rename = keptSync, keptRename }) - if _, err := NewSnapshotter(db).Snapshot(t.Context(), target); err != nil { + if _, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target); err != nil { t.Fatalf("Snapshot() error = %v, want nil", err) } @@ -307,7 +339,7 @@ func TestASecondSnapshotInTheFolderKeepsTheFirstCopy(t *testing.T) { t.Cleanup(func() { rename = kept }) result := make(chan error, 1) go func() { - _, err := NewSnapshotter(db).Snapshot(context.Background(), first) + _, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(context.Background(), first) result <- err }() select { @@ -339,14 +371,17 @@ func TestSnapshotKeepsATargetMadeDuringTheCopy(t *testing.T) { } t.Cleanup(func() { rename = kept }) - got, err := NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := NewSnapshotter(db, SnapshotOptions{}).Snapshot(t.Context(), target) if got != (dbkit.Snapshot{}) { t.Errorf("Snapshot() = %+v, want no snapshot", got) } - if read, want := fmt.Sprint(err), "dbkit: the snapshot target "+target+" already exists"; read != want { + if read, want := fmt.Sprint(err), "dbkit: the snapshot target already exists: "+target; read != want { t.Errorf("Snapshot() error = %s, want %q", read, want) } + if !errors.Is(err, ErrSnapshotExists) { + t.Errorf("Snapshot() error = %v, want one matching ErrSnapshotExists", err) + } if data, err := os.ReadFile(target); err != nil || string(data) != internalLateTarget { t.Errorf("the target holds %d bytes, %v after the snapshot, want the %d bytes written during the copy", len(data), err, len(internalLateTarget)) @@ -362,6 +397,9 @@ func TestAFailedFolderSyncKeepsTheCopy(t *testing.T) { got := internalSnapshotTo(t, db, target) internalMustAnswer(t, got, internalAnswer{snap: dbkit.Snapshot{Path: target}, err: errSnapshotSyncFailed}, - "dbkit: sync the folder of the snapshot "+target+": "+errSnapshotSyncFailed.Error()) + ErrSnapshotUnsynced.Error()+": "+target+": "+errSnapshotSyncFailed.Error()) + if !errors.Is(got.err, ErrSnapshotUnsynced) { + t.Errorf("Snapshot() error = %v, want one matching ErrSnapshotUnsynced", got.err) + } internalMustHoldOnly(t, folder, snapshotLockFile, "copy.db") } diff --git a/dbkit/sqlite/snapshot_linux.go b/dbkit/sqlite/snapshot_linux.go index dbdd1e3..5499c8d 100644 --- a/dbkit/sqlite/snapshot_linux.go +++ b/dbkit/sqlite/snapshot_linux.go @@ -4,6 +4,9 @@ package sqlite import "golang.org/x/sys/unix" +// noReplaceUnsupported is the error of a file system that cannot move a file without replacing one. +const noReplaceUnsupported = unix.EINVAL + // renameNoReplace moves the file at from to the name to, and fails with EEXIST when a file holds that name. func renameNoReplace(from, to string) error { return unix.Renameat2(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_NOREPLACE) diff --git a/dbkit/sqlite/snapshot_linux_test.go b/dbkit/sqlite/snapshot_linux_test.go new file mode 100644 index 0000000..eba9b82 --- /dev/null +++ b/dbkit/sqlite/snapshot_linux_test.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +func TestAFolderThatCannotMoveWithoutReplacingSaysSo(t *testing.T) { + db, folder := internalSnapshotOpen(t) + kept := rename + rename = func(string, string) error { return unix.EINVAL } + t.Cleanup(func() { rename = kept }) + + got := internalSnapshotTo(t, db, filepath.Join(folder, "copy.db")) + + internalMustAnswer(t, got, internalAnswer{err: unix.EINVAL}, + "dbkit: the file system of the snapshot folder "+folder+" cannot move a file without replacing one, "+ + fsLocalAdvice+": "+unix.EINVAL.Error()) + internalMustHoldOnly(t, folder, snapshotLockFile) +} diff --git a/dbkit/sqlite/snapshot_other.go b/dbkit/sqlite/snapshot_other.go index 248cb51..3a6f2f2 100644 --- a/dbkit/sqlite/snapshot_other.go +++ b/dbkit/sqlite/snapshot_other.go @@ -5,10 +5,14 @@ package sqlite import ( + "errors" "fmt" "runtime" ) +// noReplaceUnsupported is the error of a file system that cannot move a file without replacing one. +var noReplaceUnsupported = errors.ErrUnsupported + // renameNoReplace refuses every move outside linux and darwin. func renameNoReplace(string, string) error { return fmt.Errorf("dbkit: the snapshot move works on linux and darwin only, and refuses %s", runtime.GOOS) diff --git a/dbkit/sqlite/snapshot_test.go b/dbkit/sqlite/snapshot_test.go index d2f2b83..6e41251 100644 --- a/dbkit/sqlite/snapshot_test.go +++ b/dbkit/sqlite/snapshot_test.go @@ -40,6 +40,11 @@ const ( snapshotHeldWriterBusyTimeout = 500 * time.Millisecond // snapshotOwnerOnlyMode is the mode of a live file only its owner reads and writes. snapshotOwnerOnlyMode fs.FileMode = 0o600 + // snapshotGroupReadMode is the mode of a file its owner reads and writes and its group reads. + snapshotGroupReadMode fs.FileMode = 0o640 + // snapshotUsedBytes is the statement a snapshot reads the bytes of the used pages of the database with. + snapshotUsedBytes = "SELECT ((SELECT page_count FROM pragma_page_count) - " + + "(SELECT freelist_count FROM pragma_freelist_count)) * (SELECT page_size FROM pragma_page_size)" // snapshotReadOnlyFolderMode is the mode of a folder its owner may list and enter but never write in. snapshotReadOnlyFolderMode fs.FileMode = 0o500 // snapshotChildSource names the environment switch that makes the test binary snapshot the database at its value. @@ -142,6 +147,11 @@ func snapshotLinkFolder(t *testing.T, folder, name string) string { return parent } +// snapshotOf returns a Snapshotter of db with no margin and the live file's mode. +func snapshotOf(db *sql.DB) dbkit.Snapshotter { + return sqlite.NewSnapshotter(db, sqlite.SnapshotOptions{}) +} + // snapshotOpen returns a handle with opts on a fresh file, and the file's path. func snapshotOpen(t *testing.T, opts sqlite.Options) (*sql.DB, string) { t.Helper() @@ -269,7 +279,7 @@ func snapshotStart(t *testing.T, db *sql.DB, target string) <-chan snapshotResul done := make(chan struct{}) go func() { defer close(done) - snapshot, err := sqlite.NewSnapshotter(db).Snapshot(context.Background(), target) + snapshot, err := snapshotOf(db).Snapshot(context.Background(), target) result <- snapshotResult{snapshot: snapshot, err: err} }() t.Cleanup(func() { <-done }) @@ -417,8 +427,14 @@ func snapshotMustBeKilled(t *testing.T, err error) { // snapshotMustRefuse fails the test unless a snapshot of db to target answers no snapshot and the error want. func snapshotMustRefuse(t *testing.T, db *sql.DB, target, want string) { t.Helper() - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) - if err == nil || err.Error() != want || got != (dbkit.Snapshot{}) { + snapshotMustRefuseBy(t, snapshotOf(db), target, want) +} + +// snapshotMustRefuseBy fails the test unless a snapshot by s to target answers no snapshot and the error want. +func snapshotMustRefuseBy(t *testing.T, s dbkit.Snapshotter, target, want string) { + t.Helper() + got, err := s.Snapshot(t.Context(), target) + if fmt.Sprint(err) != want || got != (dbkit.Snapshot{}) { t.Errorf("Snapshot(%q) = %+v, %v, want no snapshot and %q", target, got, err, want) } } @@ -589,7 +605,7 @@ func TestSnapshotChildProcess(t *testing.T) { snapshotHoldChanges(db, hold) } - snapshot, err := sqlite.NewSnapshotter(db).Snapshot(context.Background(), os.Getenv(snapshotChildTarget)) + snapshot, err := snapshotOf(db).Snapshot(context.Background(), os.Getenv(snapshotChildTarget)) t.Fatalf("Snapshot() in the snapshot child = %+v, %v, want the parent to kill the child first", snapshot, err) } @@ -615,7 +631,7 @@ func TestSnapshotLeavesNoTargetAfterAKill(t *testing.T) { snapshotSeed(t, db) rulesExec(t, db, "CREATE VIEW snapshot_view AS SELECT n FROM snapshot_count") folder := snapshotFolder(t) - if _, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "earlier.db")); err != nil { + if _, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "earlier.db")); err != nil { t.Fatalf("Snapshot() before the kill error = %v, want nil", err) } target := filepath.Join(folder, "killed.db") @@ -628,7 +644,7 @@ func TestSnapshotLeavesNoTargetAfterAKill(t *testing.T) { snapshotMustHoldOnly(t, folder, tc.atKill...) t.Logf("the killed snapshot left a copy of %d bytes", snapshotSize(t, target+snapshotPartial)) next := filepath.Join(folder, "next.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), next) + got, err := snapshotOf(db).Snapshot(t.Context(), next) if err != nil || got.Path != next { t.Fatalf("Snapshot() after the kill = %+v, %v, want the path %s", got, err, next) } @@ -660,7 +676,7 @@ func TestSnapshotRefusesAFolderAnotherSnapshotWritesInto(t *testing.T) { folder := snapshotFolder(t) child := snapshotStartChild(t, path, filepath.Join(folder, "running.db"), tc.stage) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "second.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "second.db")) want := "dbkit: take the snapshot lock " + filepath.Join(folder, snapshotLock) + ": " + sqlite.ErrSnapshotRunning.Error() @@ -687,7 +703,7 @@ func TestSnapshotKeepsALiveDatabaseNamedLikeAStaleCopy(t *testing.T) { } target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -706,7 +722,7 @@ func TestSnapshotKeepsALiveDatabaseNamedAsItsCopy(t *testing.T) { rulesExec(t, db, "CREATE TABLE snapshot_rows (a INTEGER NOT NULL)") before := snapshotStat(t, live) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) want := "dbkit: create the snapshot copy " + live + ": " if !errors.Is(err, fs.ErrExist) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { @@ -730,7 +746,7 @@ func TestSnapshotKeepsFilesAnotherProgramNamedPartial(t *testing.T) { } target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -757,7 +773,7 @@ func TestSnapshotKeepsAFolderAndALinkNamedAsACopy(t *testing.T) { } target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -776,7 +792,7 @@ func TestSnapshotIsNoWiderThanTheLiveFile(t *testing.T) { } target := filepath.Join(snapshotFolder(t), "copy.db") - if _, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target); err != nil { + if _, err := snapshotOf(db).Snapshot(t.Context(), target); err != nil { t.Fatalf("Snapshot() error = %v, want nil", err) } @@ -785,6 +801,125 @@ func TestSnapshotIsNoWiderThanTheLiveFile(t *testing.T) { } } +func TestSnapshotTakesTheNarrowerOfItsModeAndTheLiveFiles(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + // name names the case. + name string + // live is the mode of the live file. + live fs.FileMode + // mode is the mode the snapshot options set. + mode fs.FileMode + }{ + {name: "a mode narrower than the live file", live: snapshotGroupReadMode, mode: snapshotOwnerOnlyMode}, + {name: "a mode wider than the live file", live: snapshotOwnerOnlyMode, mode: snapshotGroupReadMode}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + db, path := snapshotOpen(t, testOptions()) + rulesExec(t, db, "CREATE TABLE snapshot_rows (a INTEGER NOT NULL)") + if err := os.Chmod(path, tc.live); err != nil { + t.Fatalf("Chmod(%s) error = %v, want nil", path, err) + } + target := filepath.Join(snapshotFolder(t), "copy.db") + + _, err := sqlite.NewSnapshotter(db, sqlite.SnapshotOptions{Mode: tc.mode}).Snapshot(t.Context(), target) + + if err != nil { + t.Fatalf("Snapshot() error = %v, want nil", err) + } + if got := snapshotStat(t, target).Mode().Perm(); got != snapshotOwnerOnlyMode { + t.Errorf("the snapshot has the mode %v, want %v from the live mode %v and the mode %v", got, + snapshotOwnerOnlyMode, tc.live, tc.mode) + } + }) + } +} + +func TestSnapshotRefusesANegativeMargin(t *testing.T) { + t.Parallel() + + db, _ := snapshotOpen(t, testOptions()) + folder := snapshotFolder(t) + + snapshotMustRefuseBy(t, sqlite.NewSnapshotter(db, sqlite.SnapshotOptions{Margin: -1}), + filepath.Join(folder, "copy.db"), "dbkit: the snapshot option Margin must be 0 or more, got -1") + + snapshotMustBeEmpty(t, folder) +} + +func TestSnapshotRefusesAModeBeyondPermissionBits(t *testing.T) { + t.Parallel() + + db, _ := snapshotOpen(t, testOptions()) + folder := snapshotFolder(t) + mode := fs.ModeSetuid | snapshotOwnerOnlyMode + snapshotter := sqlite.NewSnapshotter(db, sqlite.SnapshotOptions{Mode: mode}) + + snapshotMustRefuseBy(t, snapshotter, filepath.Join(folder, "copy.db"), + "dbkit: the snapshot option Mode must hold permission bits only, got "+mode.String()) + + snapshotMustBeEmpty(t, folder) +} + +func TestSnapshotRefusesAModeThatTakesTheOwnersReadOrWrite(t *testing.T) { + t.Parallel() + + for _, mode := range []fs.FileMode{0o400, 0o200, 0o440} { + t.Run(mode.String(), func(t *testing.T) { + t.Parallel() + db, _ := snapshotOpen(t, testOptions()) + folder := snapshotFolder(t) + snapshotter := sqlite.NewSnapshotter(db, sqlite.SnapshotOptions{Mode: mode}) + + snapshotMustRefuseBy(t, snapshotter, filepath.Join(folder, "copy.db"), + "dbkit: the snapshot option Mode must keep the owner's read and write bits, got "+mode.String()) + + snapshotMustBeEmpty(t, folder) + }) + } +} + +func TestAnExistingTargetMatchesErrSnapshotExists(t *testing.T) { + t.Parallel() + + db, _ := snapshotOpen(t, testOptions()) + target := filepath.Join(snapshotFolder(t), "kept.db") + if err := os.WriteFile(target, nil, snapshotOwnerOnlyMode); err != nil { + t.Fatalf("WriteFile() error = %v, want nil", err) + } + + _, err := snapshotOf(db).Snapshot(t.Context(), target) + + if !errors.Is(err, sqlite.ErrSnapshotExists) { + t.Errorf("Snapshot() error = %v, want one matching ErrSnapshotExists", err) + } + if errors.Is(err, sqlite.ErrSnapshotRunning) { + t.Errorf("Snapshot() error = %v matches ErrSnapshotRunning, want an existing target apart", err) + } +} + +func TestAFailedSizeReadLeavesNoTarget(t *testing.T) { + t.Parallel() + + failed := errors.New("snapshot test: the size read failed on purpose") + faults := &sqlitetest.Faults{} + faults.FailStatement(snapshotUsedBytes, failed) + db := sqlitetest.OpenWithFaults(t, testOptions(), faults) + folder := snapshotFolder(t) + + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + + if want := "dbkit: read the size of the database: " + failed.Error(); fmt.Sprint(err) != want { + t.Errorf("Snapshot() error = %v, want %q", err, want) + } + if !errors.Is(err, failed) || got != (dbkit.Snapshot{}) { + t.Errorf("Snapshot() = %+v, %v, want no snapshot and an error matching %v", got, err, failed) + } + snapshotMustHoldOnly(t, folder, snapshotLock) +} + func TestSnapshotRefusesALiveFileThatIsGone(t *testing.T) { t.Parallel() @@ -795,7 +930,7 @@ func TestSnapshotRefusesALiveFileThatIsGone(t *testing.T) { } folder := snapshotFolder(t) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: read the database file " + path + ": " if !errors.Is(err, fs.ErrNotExist) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { @@ -815,7 +950,7 @@ func TestSnapshotRefusesAnExistingTarget(t *testing.T) { t.Fatalf("WriteFile() error = %v, want nil", err) } - snapshotMustRefuse(t, db, target, "dbkit: the snapshot target "+target+" already exists") + snapshotMustRefuse(t, db, target, "dbkit: the snapshot target already exists: "+target) if size := snapshotSize(t, target); size != 0 { t.Errorf("the empty target holds %d bytes after the refused snapshot, want 0", size) @@ -830,7 +965,7 @@ func TestSnapshotRefusesAnExistingTarget(t *testing.T) { t.Fatalf("WriteFile() error = %v, want nil", err) } - snapshotMustRefuse(t, db, target, "dbkit: the snapshot target "+target+" already exists") + snapshotMustRefuse(t, db, target, "dbkit: the snapshot target already exists: "+target) if data, err := os.ReadFile(target); err != nil || string(data) != "kept bytes" { t.Errorf("the target holds %d bytes, %v after the refused snapshot, want its own %d bytes", @@ -848,7 +983,7 @@ func TestSnapshotRefusesAnExistingTarget(t *testing.T) { t.Fatalf("Symlink() error = %v, want nil", err) } - snapshotMustRefuse(t, db, target, "dbkit: the snapshot target "+target+" already exists") + snapshotMustRefuse(t, db, target, "dbkit: the snapshot target already exists: "+target) if got, err := os.Readlink(target); err != nil || got != missing { t.Errorf("the target links to %q, %v after the refused snapshot, want %q", got, err, missing) @@ -931,7 +1066,7 @@ func TestSnapshotTakesACleanTargetBehindALink(t *testing.T) { parent := snapshotLinkFolder(t, folder, "link") target := filepath.Join(folder, "link", "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -949,7 +1084,7 @@ func TestSnapshotRefusesATargetUnderAFile(t *testing.T) { t.Fatalf("WriteFile() error = %v, want nil", err) } - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(plain, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(plain, "copy.db")) if !errors.Is(err, syscall.ENOTDIR) || !strings.HasPrefix(err.Error(), "dbkit: check the snapshot target: ") || got != (dbkit.Snapshot{}) { @@ -966,9 +1101,9 @@ func TestSnapshotRefusesAMissingFolder(t *testing.T) { db, _ := snapshotOpen(t, testOptions()) missing := filepath.Join(snapshotFolder(t), "missing") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(missing, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(missing, "copy.db")) - want := "dbkit: open the snapshot lock " + filepath.Join(missing, snapshotLock) + ": " + want := "dbkit: read the file system of the snapshot folder " + missing + ": " if !errors.Is(err, fs.ErrNotExist) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { t.Errorf("Snapshot() = %+v, %v, want no snapshot and an error marked ErrNotExist starting %q", got, err, want) } @@ -984,7 +1119,7 @@ func TestSnapshotStopsAtAStaleCopyItCannotRemove(t *testing.T) { db, _ := snapshotOpen(t, testOptions()) rulesExec(t, db, "CREATE TABLE snapshot_rows (a INTEGER NOT NULL)") folder := snapshotFolder(t) - if _, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "earlier.db")); err != nil { + if _, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "earlier.db")); err != nil { t.Fatalf("Snapshot() before the stale copy error = %v, want nil", err) } stale := "stale.db" + snapshotPartial @@ -996,7 +1131,7 @@ func TestSnapshotStopsAtAStaleCopyItCannotRemove(t *testing.T) { } t.Cleanup(func() { _ = os.Chmod(folder, 0o700) }) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: remove the stale snapshot copy " + filepath.Join(folder, stale) + ": " if !errors.Is(err, fs.ErrPermission) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { @@ -1058,7 +1193,7 @@ func TestABusyCheckpointReportsAndKeepsTheCopy(t *testing.T) { rulesExec(t, db, "INSERT INTO snapshot_rows (body) VALUES (randomblob(64))") target := filepath.Join(snapshotFolder(t), "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s and no error while a reader holds the log", got, err, target) @@ -1114,7 +1249,7 @@ func TestAFailedCopyLeavesNoTarget(t *testing.T) { folder := snapshotFolder(t) target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) want := "dbkit: copy the database to " + target + snapshotPartial + ": " + failed.Error() if !errors.Is(err, failed) || err.Error() != want || got != (dbkit.Snapshot{}) { @@ -1132,7 +1267,7 @@ func TestAFailedPathReadLeavesNoTarget(t *testing.T) { db := sqlitetest.OpenWithFaults(t, testOptions(), faults) folder := snapshotFolder(t) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: read the database path: " + failed.Error() if !errors.Is(err, failed) || err.Error() != want || got != (dbkit.Snapshot{}) { @@ -1151,7 +1286,7 @@ func TestAFailedCheckpointKeepsTheCopy(t *testing.T) { rulesExec(t, db, "CREATE TABLE snapshot_rows (a INTEGER NOT NULL)") target := filepath.Join(snapshotFolder(t), "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) want := "dbkit: checkpoint the write-ahead log after the snapshot " + target + ": " + failed.Error() if !errors.Is(err, failed) || err.Error() != want || got.Path != target { @@ -1172,7 +1307,7 @@ func TestACancelledSnapshotLeavesNoTarget(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - got, err := sqlite.NewSnapshotter(db).Snapshot(ctx, target) + got, err := snapshotOf(db).Snapshot(ctx, target) want := "dbkit: read the database path: " + context.Canceled.Error() if !errors.Is(err, context.Canceled) || err.Error() != want || got != (dbkit.Snapshot{}) { diff --git a/dbkit/sqlite/snapshot_unix_test.go b/dbkit/sqlite/snapshot_unix_test.go index d7f56d3..223b936 100644 --- a/dbkit/sqlite/snapshot_unix_test.go +++ b/dbkit/sqlite/snapshot_unix_test.go @@ -15,7 +15,6 @@ import ( "testing" "github.com/gopherium/framework/dbkit" - "github.com/gopherium/framework/dbkit/sqlite" ) // snapshotListOnlyMode is the mode of a folder its owner may enter and write in but never list. @@ -52,7 +51,7 @@ func TestSnapshotKeepsTheJournalOfALiveDatabaseNamedAsACopy(t *testing.T) { mustExist(t, journal) target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -68,12 +67,12 @@ func TestTheSnapshotLockIsPrivateAndKept(t *testing.T) { db, folder := snapshotLockOpen(t) lock := filepath.Join(folder, snapshotLock) - if _, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "first.db")); err != nil { + if _, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "first.db")); err != nil { t.Fatalf("Snapshot() of the first target error = %v, want nil", err) } first := migrateLockInfo(t, lock) - if _, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "second.db")); err != nil { + if _, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "second.db")); err != nil { t.Fatalf("Snapshot() of the second target error = %v, want nil", err) } @@ -95,7 +94,7 @@ func TestSnapshotRefusesALockPathItCannotOpen(t *testing.T) { t.Fatalf("Mkdir() error = %v, want nil", err) } - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: open the snapshot lock " + lock + ": " if !errors.Is(err, syscall.EISDIR) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { @@ -114,7 +113,7 @@ func TestSnapshotNeverFollowsALinkAtTheLockPath(t *testing.T) { t.Fatalf("Symlink() error = %v, want nil", err) } - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: open the snapshot lock " + lock + ": " if !errors.Is(err, syscall.ELOOP) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { @@ -136,7 +135,7 @@ func TestSnapshotRefusesAFIFOAtTheLockPath(t *testing.T) { err := migrateAtOnce(t, func() error { var err error - got, err = sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err = snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) return err }) @@ -158,7 +157,7 @@ func TestSnapshotTakesALockFileThatAlreadyExists(t *testing.T) { before := migrateLockInfo(t, lock) target := filepath.Join(folder, "copy.db") - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), target) + got, err := snapshotOf(db).Snapshot(t.Context(), target) if err != nil || got.Path != target { t.Fatalf("Snapshot() = %+v, %v, want the path %s", got, err, target) @@ -181,7 +180,7 @@ func TestSnapshotRefusesAFolderItCannotList(t *testing.T) { } t.Cleanup(func() { _ = os.Chmod(folder, 0o700) }) - got, err := sqlite.NewSnapshotter(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) + got, err := snapshotOf(db).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) want := "dbkit: list the snapshot folder " + folder + ": " if !errors.Is(err, fs.ErrPermission) || !strings.HasPrefix(err.Error(), want) || got != (dbkit.Snapshot{}) { diff --git a/dbkit/sqlite/space.go b/dbkit/sqlite/space.go new file mode 100644 index 0000000..03ff9ea --- /dev/null +++ b/dbkit/sqlite/space.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "context" + "errors" + "fmt" + "math" + "math/bits" +) + +// usedBytes reads the bytes the used pages of the main database hold. +const usedBytes = "SELECT ((SELECT page_count FROM pragma_page_count) - " + + "(SELECT freelist_count FROM pragma_freelist_count)) * (SELECT page_size FROM pragma_page_size)" + +// ErrNoSpace is the error of a snapshot whose check before the copy finds too little free space for it and the margin. +var ErrNoSpace = errors.New("dbkit: the snapshot folder has too little free space for the copy and the margin") + +// SpaceError reports a snapshot folder with less free space than the copy and the margin need. +type SpaceError struct { + // Folder is the target folder of the snapshot. + Folder string + // Needed is the bytes the used pages of the database and the margin hold. + Needed int64 + // Free is the bytes the file system of Folder leaves free to a user who is not root. + Free int64 +} + +// Error returns the folder, its free bytes and the bytes needed. +func (e *SpaceError) Error() string { + return fmt.Sprintf("dbkit: the snapshot folder %s has %d bytes free, and the copy and the margin need %d", + e.Folder, e.Free, e.Needed) +} + +// Is reports whether target is ErrNoSpace. +func (e *SpaceError) Is(target error) bool { + return target == ErrNoSpace +} + +// checkSpace returns a SpaceError when folder has less free space than the used pages of the database and the margin. +func (s *snapshotter) checkSpace(ctx context.Context, folder string) error { + var used int64 + if err := s.db.QueryRowContext(ctx, usedBytes).Scan(&used); err != nil { + return fmt.Errorf("dbkit: read the size of the database: %w", err) + } + free, err := freeSpace(folder) + if err != nil { + return fmt.Errorf("dbkit: read the free space of the snapshot folder %s: %w", folder, err) + } + if needed := addCapped(used, s.opts.Margin); free < needed { + return &SpaceError{Folder: folder, Needed: needed, Free: free} + } + return nil +} + +// addCapped returns a plus b, or math.MaxInt64 when the sum would pass it. +func addCapped(a, b int64) int64 { + if b > math.MaxInt64-a { + return math.MaxInt64 + } + return a + b +} + +// blockBytes returns blocks times size, or math.MaxInt64 when the product would pass it. +func blockBytes(blocks, size uint64) int64 { + hi, lo := bits.Mul64(blocks, size) + if hi != 0 || lo > math.MaxInt64 { + return math.MaxInt64 + } + return int64(lo) +} diff --git a/dbkit/sqlite/space_linux_test.go b/dbkit/sqlite/space_linux_test.go new file mode 100644 index 0000000..76c885f --- /dev/null +++ b/dbkit/sqlite/space_linux_test.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "testing" + + "golang.org/x/sys/unix" + + "github.com/gopherium/framework/dbkit" +) + +const ( + // spaceBlock is the block size a faked file system reports. + spaceBlock = 4096 + // spaceMargin is a margin of two blocks. + spaceMargin = 2 * spaceBlock + // spacePlenty is a free count no test database comes near. + spacePlenty = 1 << 40 + // spaceUsedQuery reads the bytes the used pages of the main database hold. + spaceUsedQuery = "SELECT (page_count - freelist_count) * page_size " + + "FROM pragma_page_count, pragma_freelist_count, pragma_page_size" +) + +// spaceUsed returns the bytes the used pages of the database of db hold. +func spaceUsed(t *testing.T, db *sql.DB) int64 { + t.Helper() + var used int64 + if err := db.QueryRowContext(t.Context(), spaceUsedQuery).Scan(&used); err != nil { + t.Fatalf("read the used bytes: %v", err) + } + return used +} + +// spaceReporting returns a statfs reporting ext4 with blocks free, the block size bsize and the fragment size frsize. +func spaceReporting(blocks uint64, bsize, frsize uint32) func(string, *unix.Statfs_t) error { + return func(_ string, st *unix.Statfs_t) error { + fsSetType(&st.Type, fsLocal) + fsSetType(&st.Bsize, bsize) + fsSetType(&st.Frsize, frsize) + st.Bavail = blocks + return nil + } +} + +// spaceFree puts in place of statfs one that reports ext4 with free bytes free in folder and none anywhere else. +func spaceFree(t *testing.T, folder string, free int64) { + t.Helper() + there := spaceReporting(uint64(free/spaceBlock), spaceBlock, spaceBlock) + elsewhere := spaceReporting(0, spaceBlock, spaceBlock) + fsSwap(t, func(path string, st *unix.Statfs_t) error { + if path == folder { + return there(path, st) + } + return elsewhere(path, st) + }) +} + +// spaceSnapshot runs a snapshot of db with margin to copy.db in folder. +func spaceSnapshot(t *testing.T, db *sql.DB, folder string, margin int64) (dbkit.Snapshot, error) { + t.Helper() + return NewSnapshotter(db, SnapshotOptions{Margin: margin}).Snapshot(t.Context(), filepath.Join(folder, "copy.db")) +} + +// spaceMustFail fails the test unless err is the SpaceError want and snap is empty. +func spaceMustFail(t *testing.T, snap dbkit.Snapshot, err error, want SpaceError) { + t.Helper() + var space *SpaceError + if !errors.As(err, &space) { + t.Fatalf("Snapshot() error = %v, want a SpaceError", err) + } + if *space != want { + t.Errorf("Snapshot() error = %+v, want %+v", *space, want) + } + if !errors.Is(err, ErrNoSpace) { + t.Errorf("Snapshot() error = %v, want one matching ErrNoSpace", err) + } + for _, other := range []error{ErrSnapshotRunning, ErrSnapshotExists, ErrSnapshotUnsynced} { + if errors.Is(err, other) { + t.Errorf("Snapshot() error = %v, want one that does not match %v", err, other) + } + } + if snap != (dbkit.Snapshot{}) { + t.Errorf("Snapshot() = %+v, want no snapshot", snap) + } +} + +func TestASnapshotWithoutSpaceNamesTheBytesNeededAndFree(t *testing.T) { + db, folder := internalSnapshotOpen(t) + used := spaceUsed(t, db) + free := used + spaceMargin - spaceBlock + spaceFree(t, folder, free) + + snap, err := spaceSnapshot(t, db, folder, spaceMargin) + + spaceMustFail(t, snap, err, SpaceError{Folder: folder, Needed: used + spaceMargin, Free: free}) + want := "dbkit: the snapshot folder " + folder + " has " + strconv.FormatInt(free, 10) + + " bytes free, and the copy and the margin need " + strconv.FormatInt(used+spaceMargin, 10) + if fmt.Sprint(err) != want { + t.Errorf("Snapshot() error = %v, want %q", err, want) + } + internalMustHoldOnly(t, folder, snapshotLockFile) +} + +func TestTheNoSpaceErrorSaysTooLittleFreeSpace(t *testing.T) { + want := "dbkit: the snapshot folder has too little free space for the copy and the margin" + + if got := ErrNoSpace.Error(); got != want { + t.Errorf("ErrNoSpace.Error() = %q, want %q", got, want) + } +} + +func TestASnapshotWithSpaceForTheCopyAndTheMarginLands(t *testing.T) { + db, folder := internalSnapshotOpen(t) + spaceFree(t, folder, spaceUsed(t, db)+spaceMargin) + + snap, err := spaceSnapshot(t, db, folder, spaceMargin) + + if err != nil || snap.Path != filepath.Join(folder, "copy.db") { + t.Errorf("Snapshot() = %+v, %v, want the copy in place", snap, err) + } +} + +func TestAZeroMarginChecksTheCopyAlone(t *testing.T) { + t.Run("space for the copy", func(t *testing.T) { + db, folder := internalSnapshotOpen(t) + spaceFree(t, folder, spaceUsed(t, db)) + + snap, err := spaceSnapshot(t, db, folder, 0) + + if err != nil || snap.Path == "" { + t.Errorf("Snapshot() = %+v, %v, want the copy in place", snap, err) + } + }) + t.Run("one block short", func(t *testing.T) { + db, folder := internalSnapshotOpen(t) + used := spaceUsed(t, db) + spaceFree(t, folder, used-spaceBlock) + + snap, err := spaceSnapshot(t, db, folder, 0) + + spaceMustFail(t, snap, err, SpaceError{Folder: folder, Needed: used, Free: used - spaceBlock}) + }) +} + +func TestAMarginPastTheLargestCountNeedsTheLargestCount(t *testing.T) { + db, folder := internalSnapshotOpen(t) + spaceFree(t, folder, spacePlenty) + + snap, err := spaceSnapshot(t, db, folder, math.MaxInt64) + + spaceMustFail(t, snap, err, SpaceError{Folder: folder, Needed: math.MaxInt64, Free: spacePlenty}) +} + +func TestTheFreeSpaceIsReadOnceTheStaleCopiesAreGone(t *testing.T) { + db, folder := internalSnapshotOpen(t) + stale := filepath.Join(folder, "stale.db"+partialSuffix) + if err := os.WriteFile(stale, []byte("a stale copy"), 0o600); err != nil { + t.Fatalf("WriteFile() error = %v, want nil", err) + } + var staleAtRead []bool + plenty := spaceReporting(spacePlenty/spaceBlock, spaceBlock, spaceBlock) + fsSwap(t, func(path string, st *unix.Statfs_t) error { + _, err := os.Lstat(stale) + staleAtRead = append(staleAtRead, err == nil) + return plenty(path, st) + }) + + if _, err := spaceSnapshot(t, db, folder, 0); err != nil { + t.Fatalf("Snapshot() error = %v, want nil", err) + } + + if len(staleAtRead) == 0 || staleAtRead[len(staleAtRead)-1] { + t.Errorf("the stale copy was there at the reads %v, want it gone by the last", staleAtRead) + } +} + +func TestAFailedFreeSpaceReadNamesTheFolder(t *testing.T) { + db, folder := internalSnapshotOpen(t) + lock := filepath.Join(folder, snapshotLockFile) + plenty := spaceReporting(spacePlenty/spaceBlock, spaceBlock, spaceBlock) + fsSwap(t, func(path string, st *unix.Statfs_t) error { + if _, err := os.Lstat(lock); err == nil { + return unix.EIO + } + return plenty(path, st) + }) + + snap, err := spaceSnapshot(t, db, folder, 0) + + want := "dbkit: read the free space of the snapshot folder " + folder + ": " + unix.EIO.Error() + if fmt.Sprint(err) != want { + t.Errorf("Snapshot() error = %v, want %q", err, want) + } + if !errors.Is(err, unix.EIO) || snap != (dbkit.Snapshot{}) { + t.Errorf("Snapshot() = %+v, %v, want no snapshot and an error matching EIO", snap, err) + } + internalMustHoldOnly(t, folder, snapshotLockFile) +} + +func TestFreeSpaceCountsInTheFragmentSize(t *testing.T) { + for _, tc := range []struct { + // name names the case. + name string + // statfs is what the file system reports. + statfs func(string, *unix.Statfs_t) error + // want is the free count. + want int64 + }{ + {name: "a fragment size", statfs: spaceReporting(10, 4096, 1024), want: 10240}, + {name: "no fragment size", statfs: spaceReporting(10, 4096, 0), want: 40960}, + {name: "a count past the largest", statfs: spaceReporting(math.MaxUint64, 4096, 4096), want: math.MaxInt64}, + {name: "a product past the largest", statfs: spaceReporting(math.MaxInt64/2+1, 2, 2), want: math.MaxInt64}, + {name: "a product past 64 bits", statfs: spaceReporting(1<<62, 8, 8), want: math.MaxInt64}, + } { + t.Run(tc.name, func(t *testing.T) { + fsSwap(t, tc.statfs) + + got, err := freeSpace("/srv") + + if err != nil || got != tc.want { + t.Errorf("freeSpace() = %d, %v, want %d", got, err, tc.want) + } + }) + } +}