404 lines
16 KiB
Go
404 lines
16 KiB
Go
// Package migration contains explicit, operator-invoked database migrations.
|
|
package migration
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
const (
|
|
// SourceSRID is the unknown SRID marker currently stored by the imported data.
|
|
SourceSRID = 0
|
|
// TargetSRID is the CRS confirmed by the data provider for the eight source tables.
|
|
TargetSRID = 4326
|
|
)
|
|
|
|
type tableSpec struct {
|
|
name string
|
|
geometryTypes []string
|
|
}
|
|
|
|
var sourceTables = []tableSpec{
|
|
{name: "st_2_mpslfh_t_slfh_syd", geometryTypes: []string{"POINT"}},
|
|
{name: "st_2_xianyouxushuichiguan", geometryTypes: []string{"POINT"}},
|
|
{name: "st_2_xianyoufanghuotongdao", geometryTypes: []string{"LINESTRING", "MULTILINESTRING"}},
|
|
{name: "st_2_fanghuojianchazhan", geometryTypes: []string{"POINT"}},
|
|
{name: "st_2_fanghuoliaowangshao", geometryTypes: []string{"POINT"}},
|
|
{name: "st_2_fanghuowangge", geometryTypes: []string{"POLYGON", "MULTIPOLYGON"}},
|
|
{name: "st_2_linqugongkuangqiye", geometryTypes: []string{"POLYGON", "MULTIPOLYGON"}},
|
|
{name: "st_2_mudifenqu_mian", geometryTypes: []string{"POLYGON", "MULTIPOLYGON"}},
|
|
}
|
|
|
|
// TableState is a non-record-level summary used to prove the migration boundary.
|
|
type TableState struct {
|
|
Table string `json:"table"`
|
|
TableOwner string `json:"table_owner"`
|
|
ColumnType string `json:"column_type"`
|
|
TypmodConstrained bool `json:"typmod_constrained"`
|
|
TypmodSRID int `json:"typmod_srid"`
|
|
TypmodDimensions int `json:"typmod_dimensions"`
|
|
TypmodGeometryType string `json:"typmod_geometry_type"`
|
|
RowCount int64 `json:"row_count"`
|
|
NullGeometryCount int64 `json:"null_geometry_count"`
|
|
NonNullGeometryCount int64 `json:"non_null_geometry_count"`
|
|
SourceSRIDCount int64 `json:"source_srid_count"`
|
|
TargetSRIDCount int64 `json:"target_srid_count"`
|
|
UnexpectedSRIDCount int64 `json:"unexpected_srid_count"`
|
|
EmptyGeometryCount int64 `json:"empty_geometry_count"`
|
|
InvalidGeometryCount int64 `json:"invalid_geometry_count"`
|
|
OutOfBoundsGeometryCount int64 `json:"wgs84_out_of_bounds_count"`
|
|
GeometryTypes []string `json:"geometry_types"`
|
|
SRIDs []int `json:"srids"`
|
|
CoordinateDimensions []int `json:"coordinate_dimensions"`
|
|
ZGeometryCount int64 `json:"z_geometry_count"`
|
|
MGeometryCount int64 `json:"m_geometry_count"`
|
|
CanUpdate bool `json:"can_update"`
|
|
CanAlter bool `json:"can_alter"`
|
|
fingerprint string
|
|
}
|
|
|
|
// Report summarizes whether the database is ready for the SRID metadata migration.
|
|
type Report struct {
|
|
TargetSRID int `json:"target_srid"`
|
|
CurrentRole string `json:"current_role"`
|
|
TransactionReadOnly bool `json:"transaction_read_only"`
|
|
CandidateGeometryCount int64 `json:"candidate_geometry_count"`
|
|
AlreadyTargetGeometryCount int64 `json:"already_target_geometry_count"`
|
|
NeedsTypmodRetype []string `json:"needs_typmod_retype"`
|
|
Tables []TableState `json:"tables"`
|
|
}
|
|
|
|
// Result reports the committed changes without exposing business records or credentials.
|
|
type Result struct {
|
|
TargetSRID int `json:"target_srid"`
|
|
UpdatedRows map[string]int64 `json:"updated_rows"`
|
|
RetypedTables []string `json:"retyped_tables"`
|
|
GeometryPayloadUnchanged bool `json:"geometry_payload_unchanged"`
|
|
Before Report `json:"before"`
|
|
After Report `json:"after"`
|
|
}
|
|
|
|
type rowQuerier interface {
|
|
QueryRow(context.Context, string, ...any) pgx.Row
|
|
}
|
|
|
|
// Inspect reads only schema- and aggregate-level facts from the fixed source tables.
|
|
func Inspect(ctx context.Context, db rowQuerier) (Report, error) {
|
|
var report Report
|
|
report.TargetSRID = TargetSRID
|
|
if err := db.QueryRow(ctx, `SELECT current_user, current_setting('transaction_read_only')::boolean`).Scan(&report.CurrentRole, &report.TransactionReadOnly); err != nil {
|
|
return Report{}, fmt.Errorf("read transaction mode: %w", err)
|
|
}
|
|
|
|
for _, spec := range sourceTables {
|
|
state, err := inspectTable(ctx, db, spec)
|
|
if err != nil {
|
|
return Report{}, fmt.Errorf("inspect table %s: %w", spec.name, err)
|
|
}
|
|
report.Tables = append(report.Tables, state)
|
|
report.CandidateGeometryCount += state.SourceSRIDCount
|
|
report.AlreadyTargetGeometryCount += state.TargetSRIDCount
|
|
if state.SourceSRIDCount > 0 && state.TypmodConstrained && state.TypmodSRID != TargetSRID {
|
|
report.NeedsTypmodRetype = append(report.NeedsTypmodRetype, state.Table)
|
|
}
|
|
}
|
|
if report.NeedsTypmodRetype == nil {
|
|
report.NeedsTypmodRetype = []string{}
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
// ValidateForApply verifies that ST_SetSRID can be applied without interpreting unknown data.
|
|
func ValidateForApply(report Report) error {
|
|
if report.TransactionReadOnly {
|
|
return fmt.Errorf("migration connection is read-only")
|
|
}
|
|
if len(report.Tables) != len(sourceTables) {
|
|
return fmt.Errorf("inspection contains %d tables, want %d", len(report.Tables), len(sourceTables))
|
|
}
|
|
for index, state := range report.Tables {
|
|
spec := sourceTables[index]
|
|
if state.Table != spec.name {
|
|
return fmt.Errorf("inspection table %d is %s, want %s", index, state.Table, spec.name)
|
|
}
|
|
if state.UnexpectedSRIDCount != 0 || state.NonNullGeometryCount != state.SourceSRIDCount+state.TargetSRIDCount {
|
|
return fmt.Errorf("table %s contains geometry outside SRID %d or %d", state.Table, SourceSRID, TargetSRID)
|
|
}
|
|
if state.OutOfBoundsGeometryCount != 0 {
|
|
return fmt.Errorf("table %s contains coordinates outside WGS84 bounds", state.Table)
|
|
}
|
|
for _, geometryType := range state.GeometryTypes {
|
|
if !slices.Contains(spec.geometryTypes, geometryType) {
|
|
return fmt.Errorf("table %s contains unexpected geometry type %s", state.Table, geometryType)
|
|
}
|
|
}
|
|
if state.SourceSRIDCount > 0 && !state.CanUpdate {
|
|
return fmt.Errorf("migration role cannot update table %s", state.Table)
|
|
}
|
|
if state.SourceSRIDCount > 0 && state.TypmodConstrained && state.TypmodSRID != TargetSRID {
|
|
if state.TypmodDimensions != 2 || !slices.Equal(state.CoordinateDimensions, []int{2}) {
|
|
return fmt.Errorf("table %s cannot be safely retyped as two-dimensional EPSG:4326 geometry", state.Table)
|
|
}
|
|
}
|
|
if state.SourceSRIDCount > 0 && state.TypmodConstrained && state.TypmodSRID != TargetSRID && !state.CanAlter {
|
|
return fmt.Errorf("migration role cannot retype the incompatible geometry column on table %s", state.Table)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ApplySRID4326 atomically marks confirmed source geometries as EPSG:4326.
|
|
// It never transforms coordinates, repairs geometry validity, or changes dimensions.
|
|
func ApplySRID4326(ctx context.Context, conn *pgx.Conn) (Result, error) {
|
|
tx, err := conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable, AccessMode: pgx.ReadWrite})
|
|
if err != nil {
|
|
return Result{}, fmt.Errorf("begin SRID migration transaction: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback(context.Background()) }()
|
|
|
|
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'`); err != nil {
|
|
return Result{}, fmt.Errorf("set migration lock timeout: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `SET LOCAL statement_timeout = '120s'`); err != nil {
|
|
return Result{}, fmt.Errorf("set migration statement timeout: %w", err)
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('fire-safety-ymd:srid-4326', 0))`); err != nil {
|
|
return Result{}, fmt.Errorf("acquire SRID migration lock: %w", err)
|
|
}
|
|
|
|
before, err := Inspect(ctx, tx)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if err := ValidateForApply(before); err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
result := Result{
|
|
TargetSRID: TargetSRID,
|
|
UpdatedRows: make(map[string]int64, len(sourceTables)),
|
|
Before: before,
|
|
}
|
|
for _, state := range before.Tables {
|
|
if state.SourceSRIDCount == 0 {
|
|
result.UpdatedRows[state.Table] = 0
|
|
continue
|
|
}
|
|
|
|
tableIdentifier := pgx.Identifier{"public", state.Table}.Sanitize()
|
|
if state.TypmodConstrained && state.TypmodSRID != TargetSRID {
|
|
statement := fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN geom TYPE geometry(Geometry, 4326) USING ST_SetSRID(geom, 4326)`, tableIdentifier)
|
|
if _, err := tx.Exec(ctx, statement); err != nil {
|
|
return Result{}, fmt.Errorf("retype geometry column on table %s: %w", state.Table, err)
|
|
}
|
|
result.RetypedTables = append(result.RetypedTables, state.Table)
|
|
result.UpdatedRows[state.Table] = state.SourceSRIDCount
|
|
continue
|
|
}
|
|
|
|
statement := fmt.Sprintf(`UPDATE %s SET geom = ST_SetSRID(geom, $1) WHERE geom IS NOT NULL AND ST_SRID(geom) = $2`, tableIdentifier)
|
|
commandTag, err := tx.Exec(ctx, statement, TargetSRID, SourceSRID)
|
|
if err != nil {
|
|
return Result{}, fmt.Errorf("set SRID metadata on table %s: %w", state.Table, err)
|
|
}
|
|
updated := commandTag.RowsAffected()
|
|
if updated != state.SourceSRIDCount {
|
|
return Result{}, fmt.Errorf("table %s updated %d rows, expected %d", state.Table, updated, state.SourceSRIDCount)
|
|
}
|
|
result.UpdatedRows[state.Table] = updated
|
|
}
|
|
if result.RetypedTables == nil {
|
|
result.RetypedTables = []string{}
|
|
}
|
|
|
|
after, err := Inspect(ctx, tx)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if err := validateAfter(before, after); err != nil {
|
|
return Result{}, err
|
|
}
|
|
result.After = after
|
|
result.GeometryPayloadUnchanged = true
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return Result{}, fmt.Errorf("commit SRID migration: %w", err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func inspectTable(ctx context.Context, db rowQuerier, spec tableSpec) (TableState, error) {
|
|
var state TableState
|
|
state.Table = spec.name
|
|
qualifiedName := "public." + spec.name
|
|
|
|
metadataSQL := `
|
|
SELECT
|
|
owner.rolname,
|
|
format_type(a.atttypid, a.atttypmod),
|
|
a.atttypmod >= 0,
|
|
CASE WHEN a.atttypmod >= 0 THEN postgis_typmod_srid(a.atttypmod) ELSE -1 END,
|
|
CASE WHEN a.atttypmod >= 0 THEN postgis_typmod_dims(a.atttypmod) ELSE -1 END,
|
|
CASE WHEN a.atttypmod >= 0 THEN postgis_typmod_type(a.atttypmod) ELSE '' END,
|
|
has_table_privilege(current_user, $1, 'UPDATE'),
|
|
(
|
|
c.relowner = (SELECT oid FROM pg_roles WHERE rolname = current_user)
|
|
OR pg_has_role(current_user, c.relowner, 'USAGE')
|
|
OR (SELECT rolsuper FROM pg_roles WHERE rolname = current_user)
|
|
)
|
|
FROM pg_attribute AS a
|
|
JOIN pg_class AS c ON c.oid = a.attrelid
|
|
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
|
JOIN pg_roles AS owner ON owner.oid = c.relowner
|
|
WHERE n.nspname = 'public'
|
|
AND c.relname = $2
|
|
AND a.attname = 'geom'
|
|
AND NOT a.attisdropped`
|
|
if err := db.QueryRow(ctx, metadataSQL, qualifiedName, spec.name).Scan(
|
|
&state.TableOwner,
|
|
&state.ColumnType,
|
|
&state.TypmodConstrained,
|
|
&state.TypmodSRID,
|
|
&state.TypmodDimensions,
|
|
&state.TypmodGeometryType,
|
|
&state.CanUpdate,
|
|
&state.CanAlter,
|
|
); err != nil {
|
|
return TableState{}, fmt.Errorf("inspect geom column metadata: %w", err)
|
|
}
|
|
|
|
tableIdentifier := pgx.Identifier{"public", spec.name}.Sanitize()
|
|
dataSQL := fmt.Sprintf(`
|
|
SELECT
|
|
count(*)::bigint,
|
|
count(*) FILTER (WHERE geom IS NULL)::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL)::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_SRID(geom) = $1)::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_SRID(geom) = $2)::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_SRID(geom) NOT IN ($1, $2))::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_IsEmpty(geom))::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND NOT ST_IsValid(geom))::bigint,
|
|
count(*) FILTER (
|
|
WHERE geom IS NOT NULL
|
|
AND NOT ST_IsEmpty(geom)
|
|
AND (
|
|
ST_XMin(Box3D(geom)) < -180 OR ST_XMax(Box3D(geom)) > 180
|
|
OR ST_YMin(Box3D(geom)) < -90 OR ST_YMax(Box3D(geom)) > 90
|
|
)
|
|
)::bigint,
|
|
COALESCE(string_agg(DISTINCT GeometryType(geom), ',' ORDER BY GeometryType(geom)) FILTER (WHERE geom IS NOT NULL), ''),
|
|
COALESCE(string_agg(DISTINCT ST_SRID(geom)::text, ',' ORDER BY ST_SRID(geom)::text) FILTER (WHERE geom IS NOT NULL), ''),
|
|
COALESCE(string_agg(DISTINCT ST_NDims(geom)::text, ',' ORDER BY ST_NDims(geom)::text) FILTER (WHERE geom IS NOT NULL), ''),
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_Zmflag(geom) IN (2, 3))::bigint,
|
|
count(*) FILTER (WHERE geom IS NOT NULL AND ST_Zmflag(geom) IN (1, 3))::bigint,
|
|
md5(COALESCE(
|
|
string_agg(
|
|
md5(encode(ST_AsBinary(geom), 'hex')),
|
|
'' ORDER BY md5(encode(ST_AsBinary(geom), 'hex'))
|
|
) FILTER (WHERE geom IS NOT NULL),
|
|
''
|
|
))
|
|
FROM %s`, tableIdentifier)
|
|
|
|
var geometryTypes string
|
|
var srids string
|
|
var dimensions string
|
|
if err := db.QueryRow(ctx, dataSQL, SourceSRID, TargetSRID).Scan(
|
|
&state.RowCount,
|
|
&state.NullGeometryCount,
|
|
&state.NonNullGeometryCount,
|
|
&state.SourceSRIDCount,
|
|
&state.TargetSRIDCount,
|
|
&state.UnexpectedSRIDCount,
|
|
&state.EmptyGeometryCount,
|
|
&state.InvalidGeometryCount,
|
|
&state.OutOfBoundsGeometryCount,
|
|
&geometryTypes,
|
|
&srids,
|
|
&dimensions,
|
|
&state.ZGeometryCount,
|
|
&state.MGeometryCount,
|
|
&state.fingerprint,
|
|
); err != nil {
|
|
return TableState{}, fmt.Errorf("inspect aggregate geometry state: %w", err)
|
|
}
|
|
state.GeometryTypes = splitStrings(geometryTypes)
|
|
parsedSRIDs, err := splitInts(srids)
|
|
if err != nil {
|
|
return TableState{}, fmt.Errorf("parse SRIDs: %w", err)
|
|
}
|
|
state.SRIDs = parsedSRIDs
|
|
parsedDimensions, err := splitInts(dimensions)
|
|
if err != nil {
|
|
return TableState{}, fmt.Errorf("parse coordinate dimensions: %w", err)
|
|
}
|
|
state.CoordinateDimensions = parsedDimensions
|
|
return state, nil
|
|
}
|
|
|
|
func validateAfter(before, after Report) error {
|
|
if len(before.Tables) != len(after.Tables) {
|
|
return fmt.Errorf("post-migration inspection contains %d tables, want %d", len(after.Tables), len(before.Tables))
|
|
}
|
|
for index := range before.Tables {
|
|
oldState := before.Tables[index]
|
|
newState := after.Tables[index]
|
|
if oldState.Table != newState.Table {
|
|
return fmt.Errorf("post-migration table %d is %s, want %s", index, newState.Table, oldState.Table)
|
|
}
|
|
if newState.SourceSRIDCount != 0 || newState.UnexpectedSRIDCount != 0 || newState.TargetSRIDCount != newState.NonNullGeometryCount {
|
|
return fmt.Errorf("table %s did not converge to SRID %d", newState.Table, TargetSRID)
|
|
}
|
|
if err := compareGeometryPayload(oldState, newState); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func compareGeometryPayload(before, after TableState) error {
|
|
if before.RowCount != after.RowCount ||
|
|
before.NullGeometryCount != after.NullGeometryCount ||
|
|
before.NonNullGeometryCount != after.NonNullGeometryCount ||
|
|
before.EmptyGeometryCount != after.EmptyGeometryCount ||
|
|
before.InvalidGeometryCount != after.InvalidGeometryCount ||
|
|
before.OutOfBoundsGeometryCount != after.OutOfBoundsGeometryCount ||
|
|
before.ZGeometryCount != after.ZGeometryCount ||
|
|
before.MGeometryCount != after.MGeometryCount ||
|
|
before.fingerprint != after.fingerprint ||
|
|
!slices.Equal(before.GeometryTypes, after.GeometryTypes) ||
|
|
!slices.Equal(before.CoordinateDimensions, after.CoordinateDimensions) {
|
|
return fmt.Errorf("table %s geometry payload changed outside the SRID metadata", before.Table)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func splitStrings(value string) []string {
|
|
if strings.TrimSpace(value) == "" {
|
|
return []string{}
|
|
}
|
|
values := strings.Split(value, ",")
|
|
sort.Strings(values)
|
|
return values
|
|
}
|
|
|
|
func splitInts(value string) ([]int, error) {
|
|
parts := splitStrings(value)
|
|
values := make([]int, 0, len(parts))
|
|
for _, part := range parts {
|
|
parsed, err := strconv.Atoi(part)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
values = append(values, parsed)
|
|
}
|
|
sort.Ints(values)
|
|
return values, nil
|
|
}
|