From b03ec3fe99f6efa82f383776c3a186186fb81d82 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 17 May 2021 10:59:00 +0800 Subject: [PATCH 01/86] chore: remove dependency on ANXS postgres --- ansible/playbook.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 845e51c3f..95045ddb8 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -12,10 +12,13 @@ - { source: "pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - { source: "stat_extension.sql", dest: "01-extension.sql" } - roles: - - role: anxs.postgresql + # roles: + # - role: anxs.postgresql tasks: + - name: Install Postgres from source + import_tasks: tasks/setup-postgres.yml + - name: Install non-Postgres extensions import_tasks: tasks/setup-misc.yml From 576be3836f582adfbafeee0d0940fe7bd6b5383f Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 17 May 2021 11:00:41 +0800 Subject: [PATCH 02/86] chore: Base compilation of Postgres 13 from source - LSE included - Need additional changes before it is close to ANXS Postgres --- ansible/tasks/setup-postgres.yml | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 ansible/tasks/setup-postgres.yml diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml new file mode 100644 index 000000000..ef1aaa9c0 --- /dev/null +++ b/ansible/tasks/setup-postgres.yml @@ -0,0 +1,62 @@ + +- name: Postgres dependencies + become: yes + apt: + pkg: + - build-essential + - libreadline-dev + - zlib1g-dev + - flex + - bison + - libxml2-dev + - libxslt-dev + - libssl-dev + - libxml2-utils + - xsltproc + +- name: Download GCC 10 + become: yes + apt: + pkg: + - gcc-10 + - g++-10 + +- name: Switch to GCC 10 + shell: + cmd: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 + become: yes + +- name: Postgres - download latest release + get_url: + url: https://github.com/okbob/plpgsql_check/archive/v{{ postgres_release }}.tar.gz + dest: /tmp + checksum: "{{ postgres_release_checksum }}" + +- name: Postgres - unpack archive + unarchive: + remote_src: yes + src: /tmp/postgresql-{{ postgres_release }}.tar.gz + dest: /tmp + become: yes + +- name: Postgres - configure + shell: + cmd: CFLAGS='-moutline-atomics -mtune=neoverse-n1 -fsigned-char' ./configure --with-openssl --with-uuid=bsd --exec-prefix=/usr/lib/postgresql/13/main --datarootdir=/var/lib/postgresql/13/main + chdir: /tmp/postgresql-{{ postgres_release }} + become: yes + + +- name: Postgres - build + make: + target: world + become: yes + +- name: Postgres - install + make: + target: install + become: yes + +# to do +## replicate directories +## create postgres user +## move files to appropriate places From 110ffda726c0aad6e0dc9b6dda33b2a5a436e730 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 20:48:30 +0800 Subject: [PATCH 03/86] chore: changing to Ubuntu 20.04 & r6g instance for ARM build --- amazon-arm.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index a1a33ec54..62859d39d 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -4,7 +4,7 @@ "aws_secret_key": "", "region": "ap-northeast-1", "ami_regions": "eu-central-1,eu-west-1,eu-west-2,ap-south-1,ap-southeast-1,ap-southeast-2,us-west-1,us-east-1,ca-central-1,sa-east-1,ap-northeast-1", - "ami": "ami-0d24b4f369844fc54", + "ami": "ami-076d8ebdd0e1ec091", "ami_name": "supabase-postgres-0.15.0", "environment": "prod", "ansible_arguments": "--skip-tags,update-only -v" @@ -17,7 +17,7 @@ "region": "{{user `region`}}", "ami_regions": "{{user `ami_regions`}}", "source_ami": "{{user `ami`}}", - "instance_type": "c6g.2xlarge", + "instance_type": "r6g.2xlarge", "ssh_username": "ubuntu", "ami_name": "{{user `ami_name`}}", "tags": { From 3da399eca29dfb906d11be7e7be57e6458372a4e Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 20:52:38 +0800 Subject: [PATCH 04/86] chore: add template files for Postgres - pg_hba.conf - pg_ident.conf - postgresql.conf - postgresql.service --- .../files/postgresql_config/pg_hba.conf.j2 | 101 +++ .../files/postgresql_config/pg_ident.conf.j2 | 44 + .../postgresql_config/postgresql.conf.j2 | 794 ++++++++++++++++++ .../postgresql_config/postgresql.service.j2 | 15 + 4 files changed, 954 insertions(+) create mode 100755 ansible/files/postgresql_config/pg_hba.conf.j2 create mode 100755 ansible/files/postgresql_config/pg_ident.conf.j2 create mode 100644 ansible/files/postgresql_config/postgresql.conf.j2 create mode 100644 ansible/files/postgresql_config/postgresql.service.j2 diff --git a/ansible/files/postgresql_config/pg_hba.conf.j2 b/ansible/files/postgresql_config/pg_hba.conf.j2 new file mode 100755 index 000000000..ccd45ecae --- /dev/null +++ b/ansible/files/postgresql_config/pg_hba.conf.j2 @@ -0,0 +1,101 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# TYPE DATABASE USER ADDRESS METHOD + +# Default: +# Allow connection by pgbouncer user +host all pgbouncer 127.0.0.1/32 md5 +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 md5 +# Local root Unix user, passwordless access +local all postgres peer map=root_as_postgres +# IPv4 external connections +host all all 0.0.0.0/0 md5 + +# MD5 hashed password hosts + +# Password hosts + +# Trusted hosts + +# User custom \ No newline at end of file diff --git a/ansible/files/postgresql_config/pg_ident.conf.j2 b/ansible/files/postgresql_config/pg_ident.conf.j2 new file mode 100755 index 000000000..a5c8de766 --- /dev/null +++ b/ansible/files/postgresql_config/pg_ident.conf.j2 @@ -0,0 +1,44 @@ +# PostgreSQL User Name Maps +# ========================= +# +# Refer to the PostgreSQL documentation, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL user name mapping. It maps external +# user names to their corresponding PostgreSQL user names. Records +# are of the form: +# +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. SYSTEM-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that SYSTEM-USERNAME may connect as +# PG-USERNAME. +# +# If SYSTEM-USERNAME starts with a slash (/), it will be treated as a +# regular expression. Optionally this can contain a capture (a +# parenthesized subexpression). The substring matching the capture +# will be substituted for \1 (backslash-one) if present in +# PG-USERNAME. +# +# Multiple maps may be specified in this file and used by pg_hba.conf. +# +# No map names are defined in the default configuration. If all +# system user names and PostgreSQL user names are the same, you don't +# need anything in this file. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- + +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# root is allowed to login as postgres +root_as_postgres postgres postgres diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 new file mode 100644 index 000000000..982d766a4 --- /dev/null +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -0,0 +1,794 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: kB = kilobytes Time units: ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +data_directory = '/var/lib/postgresql/data' # use data in another directory + # (change requires restart) +hba_file = '/etc/postgresql/pg_hba.conf' # host-based authentication file + # (change requires restart) +ident_file = '/etc/postgresql/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/tmp' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +#client_connection_check_interval = 0 # time between checks for client + # disconnection while running queries; + # 0 for never + +# - Authentication - + +authentication_timeout = 1min # 1s-600s +password_encryption = md5 # scram-sha-256 or md5 +db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +ssl = off +ssl_ca_file = '' +ssl_cert_file = '' +ssl_crl_file = '' +ssl_key_file = '' +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +ssl_prefer_server_ciphers = on +ssl_ecdh_curve = 'prime256v1' +ssl_min_protocol_version = 'TLSv1.2' +ssl_max_protocol_version = '' +ssl_dh_params_file = '' +ssl_passphrase_command = '' +ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +# huge_pages = try # on, off, or try + # (change requires restart) +# huge_page_size = 0 # zero for system default + # (change requires restart) +# temp_buffers = 8MB # min 800kB +# max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +# work_mem = 4MB # min 64kB +# hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +# maintenance_work_mem = 64MB # min 1MB +# autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +# logical_decoding_work_mem = 64MB # min 64kB +# max_stack_depth = 2MB # min 100kB +# shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +# dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) +#min_dynamic_shared_memory = 0MB # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 2 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 0 # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#backend_flush_after = 0 # measured in pages, 0 disables +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#parallel_leader_participation = on +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +wal_level = logical # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_compression = off # enable compression of full-page writes +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables +#max_wal_size = 1GB +#min_wal_size = 80MB + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the primary and on any standby that will send replication data. + +max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +max_replication_slots = 5 # max number of replication slots + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +#max_slot_wal_keep_size = -1 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Primary Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a primary server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from primary + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_async_append = on +#enable_bitmapscan = on +#enable_gathermerge = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_incremental_sort = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_resultcache = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_parallel_hash = on +#enable_partition_pruning = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +effective_cache_size = 128MB + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#jit = on # allow JIT compilation +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +log_destination = 'csvlog' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +logging_collector = on # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +log_directory = 'pg_log' # directory where log files are written, + # can be absolute or relative to PGDATA +# log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +# log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +log_rotation_age = 0 # Automatic rotation of logfiles will + # happen after that time. 0 disables. +log_rotation_size = 0 # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (Windows): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_autovacuum_min_duration = -1 # log autovacuum activity; + # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '%m [%p] ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %P = process ID of parallel group leader + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +cluster_name = 'main' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_activity_query_size = 1024 # (change requires restart) +#track_counts = on +#track_io_timing = off +#track_wal_io_timing = off +#track_functions = none # none, pl, all +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#compute_query_id = auto +#log_statement_stats = off +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +row_security = on +#default_table_access_method = 'heap' +#default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#idle_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_table_age = 150000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +extra_float_digits = 0 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.UTF-8' # locale for system error message + # strings +lc_monetary = 'en_US.UTF-8' # locale for monetary formatting +lc_numeric = 'en_US.UTF-8' # locale for number formatting +lc_time = 'en_US.UTF-8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#local_preload_libraries = '' +#session_preload_libraries = '' +# shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) +jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#remove_temp_files_after_crash = on # remove temporary files after + # backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here \ No newline at end of file diff --git a/ansible/files/postgresql_config/postgresql.service.j2 b/ansible/files/postgresql_config/postgresql.service.j2 new file mode 100644 index 000000000..1a3544bc0 --- /dev/null +++ b/ansible/files/postgresql_config/postgresql.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=PostgreSQL database server +Documentation=man:postgres(1) + +[Service] +Type=notify +User=postgres +ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql +ExecReload=/bin/kill -HUP $MAINPID +KillMode=mixed +KillSignal=SIGINT +TimeoutSec=0 + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 48c7d7e8e452d2b1169f929ec9c16b8bd3385d0c Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 20:55:21 +0800 Subject: [PATCH 05/86] chore: update log_filename value in postgresql.conf --- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 982d766a4..677dc9d63 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -442,7 +442,7 @@ logging_collector = on # Enable capturing of stderr and csvlog # These are only used if logging_collector is on: log_directory = 'pg_log' # directory where log files are written, # can be absolute or relative to PGDATA -# log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, +# log_filename = 'postgresql.log' # log file name pattern, # can include strftime() escapes # log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation From afbc3817360efdcf86f67b297c66771df5439ada Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 20:56:11 +0800 Subject: [PATCH 06/86] chore: update vars.yml - Remove values now handled by template files - Add vars for Postgres building --- ansible/vars.yml | 70 ++---------------------------------------------- 1 file changed, 2 insertions(+), 68 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 78532148b..44891c94d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,71 +1,11 @@ supabase_internal: true -postgresql_version: 12 -postgresql_wal_level: "logical" -postgresql_max_wal_senders: 10 -postgresql_max_replication_slots: 5 -postgresql_row_security: on -postgresql_listen_addresses: - - "*" - -postgresql_ext_install_contrib: yes -postgresql_ext_install_dev_headers: yes +postgresql_release: 13.3 +postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa # Warning: Make sure the postgresql & postgis versions are compatible with one another postgresql_ext_postgis_version: 3 -postgresql_shared_preload_libraries: - [pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron] - -postgresql_pg_hba_default: - - { - type: host, - database: all, - user: "pgbouncer", - address: "127.0.0.1/32", - method: "md5", - comment: "Allow connection by pgbouncer user" - } - - { - type: local, - database: all, - user: all, - address: "", - method: "{{ postgresql_default_auth_method }}", - comment: '"local" is for Unix domain socket connections only' - } - - { - type: host, - database: all, - user: all, - address: "127.0.0.1/32", - method: "trust", - comment: "IPv4 local connections:" - } - - { - type: host, - database: all, - user: all, - address: "::1/128", - method: "{{ postgresql_default_auth_method_hosts }}", - comment: "IPv6 local connections:" - } - - { - type: local, - database: all, - user: "{{ postgresql_admin_user }}", - address: "", - method: "peer map=root_as_{{ postgresql_admin_user }}", - comment: "Local root Unix user, passwordless access" - } - - { - type: "host", - database: "all", - user: "all", - address: "0.0.0.0/0", - method: "{{ postgresql_default_auth_method_hosts }}", - comment: "IPv4 external connections" - } pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e @@ -85,12 +25,6 @@ pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c pljava_release: "1_6_0" pljava_release_checksum: sha1:43b99b8d175c2d4fc9abef96ee3c751dfea610e2 -postgresql_log_destination: "csvlog" -postgresql_logging_collector: on -postgresql_log_filename: "postgresql.log" -postgresql_log_rotation_age: 0 -postgresql_log_rotation_size: 0 - golang_version: "1.15.4" wal_g_release: "0.2.15" wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 From f219047240c6efebdf054598fa2b6b45c940c1c0 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 20:58:30 +0800 Subject: [PATCH 07/86] chore: remove installation of AWS instance connect - This is pre-installed in Ubuntu 20.04 images --- ansible/playbook.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 95045ddb8..d65f4d769 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -82,13 +82,3 @@ copy: src: files/logrotate-postgres dest: /etc/logrotate.d/postgres - - # Install EC2 instance connect - # Only for AWS images - - name: install EC2 instance connect - become: yes - apt: - pkg: - - ec2-instance-connect - tags: - - aws-only From 75ce04ae2cc6cf8f307b563cf091758e6465f00e Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 21:00:22 +0800 Subject: [PATCH 08/86] chore: completely remove dependency from anxs postgres --- ansible/playbook.yml | 3 --- ansible/tasks/setup-system.yml | 35 ---------------------------------- 2 files changed, 38 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index d65f4d769..2f5bd2b12 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -12,9 +12,6 @@ - { source: "pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - { source: "stat_extension.sql", dest: "01-extension.sql" } - # roles: - # - role: anxs.postgresql - tasks: - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 778c97cc6..ba85c5b96 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -19,16 +19,6 @@ - tzdata - linux-libc-dev -- name: Install python - become: yes - apt: - pkg: - - python - - python-pip - - python3 - - python3-pip - update_cache: yes - cache_valid_time: 3600 - name: Install security tools become: yes @@ -45,31 +35,6 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic -# Install psycopg2 to enable ansible postgreSQL features -- name: Installing psycopg2 dependencies - become: yes - apt: - pkg: - - libpq-dev - -- name: Cloning psycopg2 - git: - repo: https://github.com/psycopg/psycopg2.git - dest: /tmp/psycopg2 - version: "2_8_6" - become: yes - -- name: psycopg2 - build - shell: - cmd: python setup.py build - chdir: /tmp/psycopg2 - -- name: psycopg2 - install - shell: - cmd: python setup.py install - chdir: /tmp/psycopg2 - become: yes - - name: System - Create services.slice template: src: files/services.slice.j2 From 8833ae17bf6dd2798d95feebf0d1ae36b80433be Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 21:19:01 +0800 Subject: [PATCH 09/86] chore: complete building of Postgres from source - Installation of dependencies - Compilation & Building - Installation - Init of DB - Starting of DB upon boot --- ansible/tasks/setup-postgres.yml | 112 ++++++++++++++++++++++++++++--- ansible/tasks/setup-system.yml | 7 ++ 2 files changed, 108 insertions(+), 11 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index ef1aaa9c0..cb3cf72cb 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -1,4 +1,4 @@ - +# Downloading dependencies - name: Postgres dependencies become: yes apt: @@ -11,9 +11,19 @@ - libxml2-dev - libxslt-dev - libssl-dev + - libsystemd-dev + - libpq-dev - libxml2-utils + - uuid-dev - xsltproc +- name: Download LLVM & Clang + become: yes + apt: + pkg: + - llvm-11-dev + - clang-11 + - name: Download GCC 10 become: yes apt: @@ -26,37 +36,117 @@ cmd: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 become: yes +# Building Postgres from source - name: Postgres - download latest release get_url: - url: https://github.com/okbob/plpgsql_check/archive/v{{ postgres_release }}.tar.gz + url: https://ftp.postgresql.org/pub/source/v{{ postgresql_release }}/postgresql-{{ postgresql_release }}.tar.gz dest: /tmp - checksum: "{{ postgres_release_checksum }}" + checksum: "{{ postgresql_release_checksum }}" - name: Postgres - unpack archive unarchive: remote_src: yes - src: /tmp/postgresql-{{ postgres_release }}.tar.gz + src: /tmp/postgresql-{{ postgresql_release }}.tar.gz dest: /tmp become: yes +## TODO +## Should have a separate configuration for x86 builds - name: Postgres - configure shell: - cmd: CFLAGS='-moutline-atomics -mtune=neoverse-n1 -fsigned-char' ./configure --with-openssl --with-uuid=bsd --exec-prefix=/usr/lib/postgresql/13/main --datarootdir=/var/lib/postgresql/13/main - chdir: /tmp/postgresql-{{ postgres_release }} + cmd: CFLAGS='-moutline-atomics -mtune=neoverse-n1 -fsigned-char' LLVM_CONFIG=/usr/bin/llvm-config-11 CLANG=/usr/bin/clang-11 ./configure --with-llvm --with-openssl --with-systemd --with-uuid=e2fs --exec-prefix=/usr/lib/postgresql --datarootdir=/var/lib/postgresql + chdir: /tmp/postgresql-{{ postgresql_release }} become: yes - name: Postgres - build make: target: world + chdir: /tmp/postgresql-{{ postgresql_release }} become: yes - name: Postgres - install make: - target: install + target: install-world + chdir: /tmp/postgresql-{{ postgresql_release }} become: yes -# to do -## replicate directories -## create postgres user -## move files to appropriate places +## TODO +## [NOT NEEDED, GOOD TO HAVE] add binaries to $PATH + +# Create postgres user +- name: Create postgres user + ansible.builtin.user: + name: postgres + shell: /bin/bash + comment: Postgres user + +- name: Recursively change ownership of a directory + file: + path: /var/lib/postgresql + state: directory + recurse: yes + owner: postgres + group: postgres + +# Create /etc/postgresql directory and make sure postgres group owns it +- name: Create a directory if it does not exist + file: + path: /etc/postgresql + state: directory + group: postgres + +# Move Postgres configuration files into /etc/postgresql +# Add postgresql.conf +- name: import postgresql.conf + template: + src: files/postgresql_config/postgresql.conf.j2 + dest: /etc/postgresql/postgresql.conf + group: postgres + +# Add pg_hba.conf +- name: import pg_hba.conf + template: + src: files/postgresql_config/pg_hba.conf.j2 + dest: /etc/postgresql/pg_hba.conf + group: postgres + +# Add pg_ident.conf +- name: import pg_ident.conf + template: + src: files/postgresql_config/pg_ident.conf.j2 + dest: /etc/postgresql/pg_ident.conf + group: postgres + +# init DB +- name: Initialize the database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb + vars: + ansible_command_timeout: 60 + # Circumvents the following error: + # "Timeout (12s) waiting for privilege escalation prompt" + + +# Add systemd file for Postgres +- name: import postgresql.service + template: + src: files/postgresql_config/postgresql.service.j2 + dest: /etc/systemd/system/postgresql.service + become: yes + +# Reload +- name: System - systemd reload + systemd: + enabled: yes + name: postgresql + daemon_reload: yes + +## TODO +## Checks to ensure: +### Postgres installed correctly +### Verify that LSE is enabled (for ARM) +### Directories actually owned by postgres +### Database can be activated \ No newline at end of file diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index ba85c5b96..2039ac82d 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -19,6 +19,13 @@ - tzdata - linux-libc-dev +# SEE https://github.com/georchestra/ansible/issues/55#issuecomment-588313638 +# Without this, a similar error is faced +- name: Install Ansible dependencies + become: yes + apt: + pkg: + - acl - name: Install security tools become: yes From 1971157d0a7f42b32987bb477bf7a4b77d0636cf Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 19 May 2021 21:20:27 +0800 Subject: [PATCH 10/86] chore: temporarily disable other tasks - will be re-enabled after testing for regressions --- ansible/playbook.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 2f5bd2b12..4dbbc05c6 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -16,15 +16,15 @@ - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - - name: Install non-Postgres extensions - import_tasks: tasks/setup-misc.yml + # - name: Install non-Postgres extensions + # import_tasks: tasks/setup-misc.yml - - name: Install Postgres extensions - import_tasks: tasks/setup-extensions.yml + # - name: Install Postgres extensions + # import_tasks: tasks/setup-extensions.yml - - name: Install Supabase specific content - import_tasks: tasks/setup-supabase-internal.yml - when: supabase_internal == true + # - name: Install Supabase specific content + # import_tasks: tasks/setup-supabase-internal.yml + # when: supabase_internal == true - name: Adjust APT update intervals copy: @@ -49,10 +49,10 @@ state: absent loop: "{{ sql_files }}" - - name: Adjust pgbouncer.ini - copy: - src: files/pgbouncer.ini - dest: /etc/pgbouncer/pgbouncer.ini + # - name: Adjust pgbouncer.ini + # copy: + # src: files/pgbouncer.ini + # dest: /etc/pgbouncer/pgbouncer.ini - name: UFW - Allow SSH connections ufw: @@ -75,7 +75,7 @@ policy: deny direction: incoming - - name: Setup logrotate for postgres logs - copy: - src: files/logrotate-postgres - dest: /etc/logrotate.d/postgres + # - name: Setup logrotate for postgres logs + # copy: + # src: files/logrotate-postgres + # dest: /etc/logrotate.d/postgres From caf81cd1aee94dec8aa6c7d1bea60d8d9090fda7 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 19:41:21 +0800 Subject: [PATCH 11/86] chore: bump versions of existing extensions - Make sure they can run in PG13 --- ansible/tasks/setup-extensions.yml | 168 +++++++++++++++++++---------- ansible/vars.yml | 37 ++++--- 2 files changed, 136 insertions(+), 69 deletions(-) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 8801e3865..535b4a10a 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -1,29 +1,50 @@ # postgis -- name: Install postgis for postgreSQL versions < 10 +- name: postgis - download & install dependencies apt: pkg: - - libgeos-c1v5 - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}" - - "postgresql-{{ postgresql_version }}-postgis-scripts" + - libgeos-dev + - libproj-dev + - libgdal-dev + - libjson-c-dev + - libxml2-dev update_cache: yes cache_valid_time: 3600 - when: postgresql_version < 10 -- name: Install postgis for postgreSQL versions >= 10 - apt: - pkg: - - libgeos-c1v5 - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}" - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}-scripts" - update_cache: yes - cache_valid_time: 3600 - when: postgresql_version >= 10 +- name: postgis - download latest release + get_url: + url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz" + dest: /tmp/postgis-{{ postgis_release }}.tar.gz + checksum: "{{ postgis_release_checksum }}" + +- name: postgis - unpack archive + unarchive: + remote_src: yes + src: /tmp/postgis-{{ postgis_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - configure + shell: + cmd: "./configure --without-protobuf" + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - build + make: + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - install + make: + chdir: /tmp/postgis-{{ postgis_release }} + target: install + become: yes # pgTAP - name: pgTAP - download latest release get_url: url: "https://github.com/theory/pgtap/archive/v{{ pgtap_release }}.tar.gz" - dest: /tmp + dest: /tmp/pgtap-{{ pgtap_release }}.tar.gz checksum: "{{ pgtap_release_checksum }}" - name: pgTAP - unpack archive @@ -39,18 +60,40 @@ target: install become: yes -# pgcron -- name: Install pgcron - apt: - pkg: postgresql-12-cron - update_cache: yes - cache_valid_time: 3600 +# pg_cron +- name: pg_cron - download latest release + get_url: + url: "https://github.com/citusdata/pg_cron/archive/refs/tags/v{{ pg_cron_release }}.tar.gz" + dest: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + checksum: "{{ pg_cron_release_checksum }}" + +- name: pg_cron - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + dest: /tmp + become: yes + +- name: pg_cron - build + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + become: yes + +- name: pg_cron - install + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + target: install + become: yes + +- name: pg_cron - set cron.database_name + become: yes + shell: + cmd: echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf # pgAudit - name: pgAudit - download & install dependencies apt: pkg: - - postgresql-server-dev-12 - libssl-dev - libkrb5-dev update_cache: yes @@ -58,8 +101,8 @@ - name: pgAudit - download latest release get_url: - url: "https://github.com/pgaudit/pgaudit/archive/{{ pgaudit_release }}.tar.gz" - dest: /tmp + url: "https://github.com/pgaudit/pgaudit/archive/refs/tags/{{ pgaudit_release }}.tar.gz" + dest: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz checksum: "{{ pgaudit_release_checksum }}" - name: pgAudit - unpack archive @@ -98,13 +141,13 @@ target: install become: yes -- name: Remove libcurl4 package +# pgsql-http +- name: pgsql-http - libcurl4 package apt: pkg: - libcurl4 state: absent -# pgsql-http - name: pgsql-http - download & install dependencies apt: pkg: @@ -114,14 +157,14 @@ - name: pgsql-http - download latest release get_url: - url: "https://github.com/pramsey/pgsql-http/archive/v{{ pgsql_http_release }}.tar.gz" - dest: /tmp + url: "https://github.com/pramsey/pgsql-http/archive/refs/tags/v{{ pgsql_http_release }}.tar.gz" + dest: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz checksum: "{{ pgsql_http_release_checksum }}" - name: pgsql-http - unpack archive unarchive: remote_src: yes - src: /tmp/pgsql-http-{{ pgsql_http_release }}.tar.gz + src: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz dest: /tmp become: yes @@ -146,8 +189,8 @@ - name: plpgsql_check - download latest release get_url: - url: https://github.com/okbob/plpgsql_check/archive/v{{ plpgsql_check_release }}.tar.gz - dest: /tmp + url: "https://github.com/okbob/plpgsql_check/archive/refs/tags/v{{ plpgsql_check_release }}.tar.gz" + dest: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz checksum: "{{ plpgsql_check_release_checksum }}" - name: plpgsql_check - unpack archive @@ -170,25 +213,29 @@ become: yes # pg-safeupdate -- name: pg-safeupdate - download & install dependencies - apt: - pkg: - - pgxnclient - - postgresql-server-dev-12 - update_cache: yes - install_recommends: yes +- name: pg-safeupdate - download latest release + get_url: + url: "https://github.com/eradman/pg-safeupdate/archive/refs/tags/{{ pg_safeupdate_release }}.tar.gz" + dest: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + checksum: "{{ pg_safeupdate_release_checksum }}" -- name: pg-safeupdate - download & install via pgxn - shell: - cmd: pgxn install safeupdate +- name: pg-safeupdate - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + dest: /tmp become: yes - ignore_errors: yes -- name: pg-safeupdate - remove dependencies - apt: - pkg: - - pgxnclient - state: absent +- name: pg-safeupdate - build + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + become: yes + +- name: pg-safeupdate - install + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + target: install + become: yes # pljava - name: pljava - download & install dependencies @@ -197,14 +244,13 @@ - maven - default-jre - default-jdk - - postgresql-server-dev-12 update_cache: yes install_recommends: yes - name: pljava - download latest release get_url: url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz - dest: /tmp + dest: /tmp/pljava-{{ pljava_release }}.tar.gz checksum: "{{ pljava_release_checksum }}" - name: pljava - unpack archive @@ -223,7 +269,7 @@ - name: pljava - install become: yes shell: - cmd: java -jar pljava-packaging/target/pljava-pg12.jar + cmd: java -jar pljava-packaging/target/pljava-pg13.jar chdir: /tmp/pljava-{{ pljava_release }} - name: pljava - remove build dependencies @@ -241,10 +287,17 @@ update_cache: yes install_recommends: no +# Determine platform architecture +- name: pljava - set architecture for jvm + shell: if [ $(arch) = "aarch64" ]; then echo "arm"; else echo "amd"; fi + register: jvm_configuration +- set_fact: + jvm: "{{ jvm_configuration.stdout }}" + - name: pljava - set pljava.libjvm_location become: yes shell: - cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'" >> /etc/postgresql/12/main/postgresql.conf + cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ jvm }}64/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf - name: pljava - remove ~/.m2 directory become: yes @@ -267,11 +320,12 @@ - cmake - libc++-dev - libc++abi-dev - - postgresql-server-dev-12 - libc++1 + - libglib2.0-dev - libtinfo5 - libc++abi1 - ninja-build + - python update_cache: yes install_recommends: no @@ -279,9 +333,15 @@ git: repo: https://github.com/plv8/plv8.git dest: /tmp/plv8 - version: r3.0alpha + version: 3656177d384e3e02b74faa8e2931600f3690ab59 become: yes +# This might just be a arm64 thing +- name: plv8 - create symlink for libc++ + become: yes + shell: + cmd: ln -s /lib/aarch64-linux-gnu/libc++.so.1 /lib/aarch64-linux-gnu/libc++.so + - name: plv8 - build make: chdir: /tmp/plv8 @@ -291,4 +351,4 @@ make: chdir: /tmp/plv8 target: install - become: yes + become: yes \ No newline at end of file diff --git a/ansible/vars.yml b/ansible/vars.yml index 44891c94d..a4d1b6d33 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,35 +1,42 @@ supabase_internal: true -postgresql_release: 13.3 +postgresql_release: "13.3" postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa -# Warning: Make sure the postgresql & postgis versions are compatible with one another -postgresql_ext_postgis_version: 3 +# Non Postgres Extensions +pgbouncer_release: "1.15.0" +pgbouncer_release_checksum: sha1:ea7e9dbcab178f439a0fa402a78a7f1e4f43e6d4 +postgres_exporter_release: "0.9.0" +postgres_exporter_release_checksum: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 + +aws_cli_release: "2.0.30" + +# Postgres Extensions +postgis_release: "3.1.2" +postgis_release_checksum: sha1:622f52f3bf338c8e51ea6d73d30d6a5d3140c517 pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e -pgaudit_release: "1.4.0" -pgaudit_release_checksum: sha1:ea085fbf227b5c461331ab33b99579f37db299a6 +pg_cron_release: "1.3.1" +pg_cron_release_checksum: sha1:679b6ff54e0b1070a5fd713c5d25c3378f371fac + +pgaudit_release: "1.5.0" +pgaudit_release_checksum: sha1:8429125e8f70fcaa2c2f5a0e22b910a4afb821a4 pgsql_http_release: "1.3.1" pgsql_http_release_checksum: sha1:816a3fff53e05301b176cf0696799fc5a00f54e8 -plpgsql_check_release: "1.11.0" -plpgsql_check_release_checksum: sha1:395313b6ef9c10c4fc182817d6f0040b171147b8 +plpgsql_check_release: "1.16.0" +plpgsql_check_release_checksum: sha1:626553fc2746fe10aa5a776a1229baf2af3365fc pg_safeupdate_release: "1.3" pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c -pljava_release: "1_6_0" -pljava_release_checksum: sha1:43b99b8d175c2d4fc9abef96ee3c751dfea610e2 +pljava_release: "1_6_2" +pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 golang_version: "1.15.4" wal_g_release: "0.2.15" -wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 - -postgres_exporter_release: "0.9.0" -postgres_exporter_release_checksum: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 - -aws_cli_release: "2.0.30" +wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 \ No newline at end of file From e02f70491f8b360f8373331848c06f9bce1425ce Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 19:45:02 +0800 Subject: [PATCH 12/86] chore: build PgBouncer from source instead --- ansible/tasks/setup-misc.yml | 48 +++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/setup-misc.yml b/ansible/tasks/setup-misc.yml index 756d66227..6b0dd653b 100644 --- a/ansible/tasks/setup-misc.yml +++ b/ansible/tasks/setup-misc.yml @@ -49,9 +49,49 @@ GOBIN: "/usr/local/bin" PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# Install PgBouncer -- name: install PgBouncer - become: yes +# PgBouncer +- name: PgBouncer - download & install dependencies apt: pkg: - - pgbouncer + - libssl-dev + - pkg-config + - libevent-dev + update_cache: yes + cache_valid_time: 3600 + +- name: PgBouncer - download latest release + get_url: + url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" + dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + checksum: "{{ pgbouncer_release_checksum }}" + +- name: PgBouncer - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + dest: /tmp + become: yes + +- name: PgBouncer - configure + shell: + cmd: "./configure --prefix=/usr/local --with-systemd" + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - build + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - install + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + target: install + become: yes + +# Create /etc/postgresql directory and make sure postgres group owns it +- name: Create a directory if it does not exist + file: + path: /etc/pgbouncer + state: directory + group: postgres From 2f62877edc45d427105d6fe32a84e8762928c197 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 19:45:43 +0800 Subject: [PATCH 13/86] minor alterations to template postgresql.conf - Ensure configurations are the same as the current version --- ansible/files/postgresql_config/postgresql.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 677dc9d63..b0341bb0f 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -442,7 +442,7 @@ logging_collector = on # Enable capturing of stderr and csvlog # These are only used if logging_collector is on: log_directory = 'pg_log' # directory where log files are written, # can be absolute or relative to PGDATA -# log_filename = 'postgresql.log' # log file name pattern, +log_filename = 'postgresql.log' # log file name pattern, # can include strftime() escapes # log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation @@ -717,7 +717,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -# shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - From ef4bd1e5c067c60e9f91f2ec5072ccdce9716ceb Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 19:46:54 +0800 Subject: [PATCH 14/86] chore: Cleanup postgres setup - Ensure configurations for arm and x86 are correct respectively - Open up new task mainly for starting postgres and running tests [WIP] --- ansible/tasks/setup-postgres-init.yml | 12 ++++++++++++ ansible/tasks/setup-postgres.yml | 24 +++++++++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 ansible/tasks/setup-postgres-init.yml diff --git a/ansible/tasks/setup-postgres-init.yml b/ansible/tasks/setup-postgres-init.yml new file mode 100644 index 000000000..b22206789 --- /dev/null +++ b/ansible/tasks/setup-postgres-init.yml @@ -0,0 +1,12 @@ +## [WIP] +## TODO +## Checks to ensure: +### directories exist +### directories are owned by postgres +### verify LSE is enabled +### database is running + +- name: Start Postgres Server + systemd: + name: postgresql + state: started \ No newline at end of file diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index cb3cf72cb..d50d14556 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -50,15 +50,19 @@ dest: /tmp become: yes -## TODO -## Should have a separate configuration for x86 builds +# Determine platform architecture +- name: Setting CFLAGS + shell: if [ $(arch) = "aarch64" ]; then echo "-moutline-atomics -mtune=neoverse-n1 -fsigned-char"; else echo ""; fi + register: cflags_configuration +- set_fact: + cflags: "{{ cflags_configuration.stdout }}" + - name: Postgres - configure shell: - cmd: CFLAGS='-moutline-atomics -mtune=neoverse-n1 -fsigned-char' LLVM_CONFIG=/usr/bin/llvm-config-11 CLANG=/usr/bin/clang-11 ./configure --with-llvm --with-openssl --with-systemd --with-uuid=e2fs --exec-prefix=/usr/lib/postgresql --datarootdir=/var/lib/postgresql + cmd: CFLAGS='{{ cflags }}' LLVM_CONFIG=/usr/bin/llvm-config-11 CLANG=/usr/bin/clang-11 ./configure --with-llvm --with-openssl --with-systemd --with-uuid=e2fs --exec-prefix=/usr/lib/postgresql --datarootdir=/var/lib/postgresql chdir: /tmp/postgresql-{{ postgresql_release }} become: yes - - name: Postgres - build make: target: world @@ -71,12 +75,9 @@ chdir: /tmp/postgresql-{{ postgresql_release }} become: yes -## TODO -## [NOT NEEDED, GOOD TO HAVE] add binaries to $PATH - # Create postgres user - name: Create postgres user - ansible.builtin.user: + user: name: postgres shell: /bin/bash comment: Postgres user @@ -143,10 +144,3 @@ enabled: yes name: postgresql daemon_reload: yes - -## TODO -## Checks to ensure: -### Postgres installed correctly -### Verify that LSE is enabled (for ARM) -### Directories actually owned by postgres -### Database can be activated \ No newline at end of file From 62f91603d530bcf84d0eb991d10204c35dbfa278 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 19:47:34 +0800 Subject: [PATCH 15/86] chore: uncomment other tasks - Everything appears to have been installed without a problem --- ansible/playbook.yml | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 4dbbc05c6..626092e1b 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -12,19 +12,25 @@ - { source: "pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - { source: "stat_extension.sql", dest: "01-extension.sql" } + environment: + PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} + tasks: - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - # - name: Install non-Postgres extensions - # import_tasks: tasks/setup-misc.yml + - name: Install non-Postgres extensions + import_tasks: tasks/setup-misc.yml + + - name: Install Postgres extensions + import_tasks: tasks/setup-extensions.yml - # - name: Install Postgres extensions - # import_tasks: tasks/setup-extensions.yml + - name: Install Supabase specific content + import_tasks: tasks/setup-supabase-internal.yml + when: supabase_internal == true - # - name: Install Supabase specific content - # import_tasks: tasks/setup-supabase-internal.yml - # when: supabase_internal == true + - name: Start Postgres and perform checks + import_tasks: tasks/setup-postgres-init.yml - name: Adjust APT update intervals copy: @@ -38,9 +44,10 @@ loop: "{{ sql_files }}" - name: Execute init SQL files + become: yes become_user: postgres shell: - cmd: psql -f /tmp/{{ item.dest }} + cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} loop: "{{ sql_files }}" - name: Delete SQL scripts @@ -49,10 +56,10 @@ state: absent loop: "{{ sql_files }}" - # - name: Adjust pgbouncer.ini - # copy: - # src: files/pgbouncer.ini - # dest: /etc/pgbouncer/pgbouncer.ini + - name: Adjust pgbouncer.ini + copy: + src: files/pgbouncer.ini + dest: /etc/pgbouncer/pgbouncer.ini - name: UFW - Allow SSH connections ufw: @@ -75,7 +82,7 @@ policy: deny direction: incoming - # - name: Setup logrotate for postgres logs - # copy: - # src: files/logrotate-postgres - # dest: /etc/logrotate.d/postgres + - name: Setup logrotate for postgres logs + copy: + src: files/logrotate-postgres + dest: /etc/logrotate.d/postgres From c3c256428d3a7d6254ba6afda52cbfa7ed11bb7a Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 7 Jun 2021 23:48:41 +0800 Subject: [PATCH 16/86] chore: split setup-misc into its components --- ansible/tasks/setup-misc.yml | 97 ------------------------------- ansible/tasks/setup-pgbouncer.yml | 46 +++++++++++++++ ansible/tasks/setup-wal-g.yml | 50 ++++++++++++++++ 3 files changed, 96 insertions(+), 97 deletions(-) delete mode 100644 ansible/tasks/setup-misc.yml create mode 100644 ansible/tasks/setup-pgbouncer.yml create mode 100644 ansible/tasks/setup-wal-g.yml diff --git a/ansible/tasks/setup-misc.yml b/ansible/tasks/setup-misc.yml deleted file mode 100644 index 6b0dd653b..000000000 --- a/ansible/tasks/setup-misc.yml +++ /dev/null @@ -1,97 +0,0 @@ -# WAL-G -- name: Install daemontools - become: yes - apt: - pkg: - - daemontools - -- name: wal-g system dependencies - apt: - pkg: - - liblzo2-dev - - cmake - - build-essential - -# find platform architecture -- name: finding platform architecture - shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi - register: platform_output -- set_fact: - platform: "{{ platform_output.stdout }}" - -# install go dependency for WAL-G -- name: wal-g go dependency - get_url: - url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /tmp -- name: unpack go archive - unarchive: - remote_src: yes - src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /usr/local - -# Download WAL-G -- name: download wal-g - shell: - cmd: go get github.com/wal-g/wal-g; - environment: - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - ignore_errors: yes - # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 - -# Install WAL-G -- name: install wal-g - become: yes - shell: - cmd: make install && make deps && make pg_install - chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" - environment: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - -# PgBouncer -- name: PgBouncer - download & install dependencies - apt: - pkg: - - libssl-dev - - pkg-config - - libevent-dev - update_cache: yes - cache_valid_time: 3600 - -- name: PgBouncer - download latest release - get_url: - url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" - dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz - checksum: "{{ pgbouncer_release_checksum }}" - -- name: PgBouncer - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz - dest: /tmp - become: yes - -- name: PgBouncer - configure - shell: - cmd: "./configure --prefix=/usr/local --with-systemd" - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - become: yes - -- name: PgBouncer - build - make: - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - become: yes - -- name: PgBouncer - install - make: - chdir: /tmp/pgbouncer-{{ pgbouncer_release }} - target: install - become: yes - -# Create /etc/postgresql directory and make sure postgres group owns it -- name: Create a directory if it does not exist - file: - path: /etc/pgbouncer - state: directory - group: postgres diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml new file mode 100644 index 000000000..9f0d3d25f --- /dev/null +++ b/ansible/tasks/setup-pgbouncer.yml @@ -0,0 +1,46 @@ +# PgBouncer +- name: PgBouncer - download & install dependencies + apt: + pkg: + - libssl-dev + - pkg-config + - libevent-dev + update_cache: yes + cache_valid_time: 3600 + +- name: PgBouncer - download latest release + get_url: + url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" + dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + checksum: "{{ pgbouncer_release_checksum }}" + +- name: PgBouncer - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + dest: /tmp + become: yes + +- name: PgBouncer - configure + shell: + cmd: "./configure --prefix=/usr/local --with-systemd" + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - build + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - install + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + target: install + become: yes + +# Create /etc/postgresql directory and make sure postgres group owns it +- name: Create a directory if it does not exist + file: + path: /etc/pgbouncer + state: directory + group: postgres diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml new file mode 100644 index 000000000..529540539 --- /dev/null +++ b/ansible/tasks/setup-wal-g.yml @@ -0,0 +1,50 @@ +# WAL-G +- name: Install daemontools + become: yes + apt: + pkg: + - daemontools + +- name: wal-g system dependencies + apt: + pkg: + - liblzo2-dev + - cmake + - build-essential + +# find platform architecture +- name: finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output +- set_fact: + platform: "{{ platform_output.stdout }}" + +# install go dependency for WAL-G +- name: wal-g go dependency + get_url: + url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /tmp +- name: unpack go archive + unarchive: + remote_src: yes + src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /usr/local + +# Download WAL-G +- name: download wal-g + shell: + cmd: go get github.com/wal-g/wal-g; + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + ignore_errors: yes + # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 + +# Install WAL-G +- name: install wal-g + become: yes + shell: + cmd: make install && make deps && make pg_install + chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" + environment: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" \ No newline at end of file From 03456f04bddf25370d073045331d737a71ac4043 Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 8 Jun 2021 10:41:25 +0800 Subject: [PATCH 17/86] chore: regression - put back installation of EC2 instance connect --- ansible/tasks/setup-system.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 2039ac82d..a1e51d615 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -42,10 +42,20 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic +# Install EC2 instance connect +# Only for AWS images +- name: install EC2 instance connect + become: yes + apt: + pkg: + - ec2-instance-connect + tags: + - aws-only + - name: System - Create services.slice template: src: files/services.slice.j2 dest: /etc/systemd/system/services.slice - name: System - systemd reload - systemd: daemon_reload=yes + systemd: daemon_reload=yes \ No newline at end of file From 476e711fd1f14a88ffd7759a3c24c1a4d12a3d13 Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 8 Jun 2021 11:51:08 +0800 Subject: [PATCH 18/86] chore: centralise checking of platform - now done and set in setup-system.yml --- ansible/tasks/setup-extensions.yml | 20 +++++++------------- ansible/tasks/setup-postgres.yml | 16 +++++++++------- ansible/tasks/setup-system.yml | 7 +++++++ ansible/tasks/setup-wal-g.yml | 7 ------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 535b4a10a..aa0110752 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -287,17 +287,10 @@ update_cache: yes install_recommends: no -# Determine platform architecture -- name: pljava - set architecture for jvm - shell: if [ $(arch) = "aarch64" ]; then echo "arm"; else echo "amd"; fi - register: jvm_configuration -- set_fact: - jvm: "{{ jvm_configuration.stdout }}" - - name: pljava - set pljava.libjvm_location become: yes shell: - cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ jvm }}64/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf + cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf - name: pljava - remove ~/.m2 directory become: yes @@ -336,11 +329,12 @@ version: 3656177d384e3e02b74faa8e2931600f3690ab59 become: yes -# This might just be a arm64 thing -- name: plv8 - create symlink for libc++ - become: yes - shell: - cmd: ln -s /lib/aarch64-linux-gnu/libc++.so.1 /lib/aarch64-linux-gnu/libc++.so +- name: Create a symbolic link + file: + src: /lib/aarch64-linux-gnu/libc++.so.1 + dest: /lib/aarch64-linux-gnu/libc++.so + state: link + when: platform == "arm64" - name: plv8 - build make: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index d50d14556..af5d5a0c2 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -50,12 +50,15 @@ dest: /tmp become: yes -# Determine platform architecture -- name: Setting CFLAGS - shell: if [ $(arch) = "aarch64" ]; then echo "-moutline-atomics -mtune=neoverse-n1 -fsigned-char"; else echo ""; fi - register: cflags_configuration -- set_fact: - cflags: "{{ cflags_configuration.stdout }}" +- name: Setting CFLAGS (arm) + set_fact: + cflags: "-moutline-atomics -mtune=neoverse-n1 -fsigned-char" + when: platform == "arm64" + +- name: Setting CFLAGS (x86) + set_fact: + cflags: "-fsigned-char" + when: platform == "amd64" - name: Postgres - configure shell: @@ -130,7 +133,6 @@ # Circumvents the following error: # "Timeout (12s) waiting for privilege escalation prompt" - # Add systemd file for Postgres - name: import postgresql.service template: diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index a1e51d615..e9023311c 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -52,6 +52,13 @@ tags: - aws-only +# Find platform architecture and set as a variable +- name: finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output +- set_fact: + platform: "{{ platform_output.stdout }}" + - name: System - Create services.slice template: src: files/services.slice.j2 diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 529540539..1861de010 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -12,13 +12,6 @@ - cmake - build-essential -# find platform architecture -- name: finding platform architecture - shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi - register: platform_output -- set_fact: - platform: "{{ platform_output.stdout }}" - # install go dependency for WAL-G - name: wal-g go dependency get_url: From 7dbb4306e39ac73371c5d0a80683f1d652aaeead Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 8 Jun 2021 11:59:37 +0800 Subject: [PATCH 19/86] chore: adding postgrest --- ansible/files/postgrest.service.j2 | 15 +++++++++++ ansible/tasks/setup-postgrest.yml | 43 ++++++++++++++++++++++++++++++ ansible/vars.yml | 6 +++++ 3 files changed, 64 insertions(+) create mode 100644 ansible/files/postgrest.service.j2 create mode 100644 ansible/tasks/setup-postgrest.yml diff --git a/ansible/files/postgrest.service.j2 b/ansible/files/postgrest.service.j2 new file mode 100644 index 000000000..8018a0334 --- /dev/null +++ b/ansible/files/postgrest.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=PostgREST + +[Service] +Type=simple +ExecStart=/opt/postgrest /etc/postgrest.conf +User=postgrest +StandardOutput=file:/var/log/postgrest.stdout +StandardError=file:/var/log/postgrest.error +Slice=services.slice +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml new file mode 100644 index 000000000..189683920 --- /dev/null +++ b/ansible/tasks/setup-postgrest.yml @@ -0,0 +1,43 @@ +- name: PostgREST - system user + user: name=postgrest + +# libpq is a C library that enables user programs to communicate with +# the PostgreSQL database server. +- name: PostgREST - system dependencies + apt: + pkg: + - libpq5 + +- name: PostgREST - download ubuntu binary archive (arm) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-{{ postgrest_arm_release }}.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_arm_release_checksum }}" + when: platform == "arm64" + +- name: PostgREST - download ubuntu binary archive (x86) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-{{ postgrest_x86_release }}.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_x86_release_checksum }}" + when: platform == "amd64" + +- name: PostgREST - unpack archive in /opt + unarchive: + remote_src: yes + src: /tmp/postgrest.tar.xz + dest: /opt + owner: postgrest + +- name: PostgREST - create service file + template: + src: files/postgrest.service.j2 + dest: /etc/systemd/system/postgrest.service + +- name: PostgREST - reload systemd + systemd: + daemon_reload: yes + +- name: PostgREST - restart service + service: name=postgrest state=restarted + when: restart_services diff --git a/ansible/vars.yml b/ansible/vars.yml index a4d1b6d33..cae1581b3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,6 +10,12 @@ pgbouncer_release_checksum: sha1:ea7e9dbcab178f439a0fa402a78a7f1e4f43e6d4 postgres_exporter_release: "0.9.0" postgres_exporter_release_checksum: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 +postgrest_arm_release: 2021-03-05-19-03-d3a8b5f-ubuntu-aarch64 +postgrest_arm_release_checksum: sha1:b9e9b06ead7230b75033e8ae17912714bf463a33 + +postgrest_x86_release: 2021-03-05-19-03-d3a8b5f-linux-x64-static +postgrest_x86_release_checksum: sha1:4b4adde15f0d41d65a9136d1f8c0d9cd6fe79326 + aws_cli_release: "2.0.30" # Postgres Extensions From b2300805fbf94260756d0298274ade690b4da25b Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 8 Jun 2021 12:02:44 +0800 Subject: [PATCH 20/86] chore: add symbolic links for PG binaries - link to /usr/bin --- ansible/tasks/setup-postgres.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index af5d5a0c2..b9d45e5f9 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -122,6 +122,13 @@ dest: /etc/postgresql/pg_ident.conf group: postgres +- name: Create symbolic links for Postgres binaries to /usr/bin/ + file: + src: /usr/lib/postgresql/bin/* + dest: /usr/bin/ + state: link + force: yes + # init DB - name: Initialize the database become: yes From b8564e24ffbbe8183b860f266ea52abb8ae00f4d Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 8 Jun 2021 12:03:13 +0800 Subject: [PATCH 21/86] chore: update playbook.yml to reflect split of setup-misc --- ansible/playbook.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 626092e1b..3c83d9717 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -19,8 +19,16 @@ - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - - name: Install non-Postgres extensions - import_tasks: tasks/setup-misc.yml + - name: Install WAL-G + import_tasks: tasks/setup-wal-g.yml + + - name: Install PgBouncer + import_tasks: tasks/setup-pgbouncer.yml + when: install_pgbouncer == true + + - name: Install PostgREST + import_tasks: tasks/setup-postgrest.yml + when: install_postgrest == true - name: Install Postgres extensions import_tasks: tasks/setup-extensions.yml From d841291dcb608d0698945a969224f7823c680c94 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 00:32:47 +0800 Subject: [PATCH 22/86] chore: add timescaledb (Apache2 version) as an extension --- .../postgresql_config/postgresql.conf.j2 | 2 +- ansible/tasks/setup-extensions.yml | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index b0341bb0f..97938e363 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -717,7 +717,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, timescaledb' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index aa0110752..2e8a7a793 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -237,6 +237,31 @@ target: install become: yes +# timescaledb +- name: timescaledb - download latest release + git: + repo: https://github.com/timescale/timescaledb.git + dest: /tmp/timescaledb + version: "{{ timescaledb_release }}" + become: yes + +- name: timescaledb - bootstrap + shell: + cmd: "./bootstrap -DAPACHE_ONLY=1 -DREGRESS_CHECKS=OFF" + chdir: /tmp/timescaledb + become: yes + +- name: timescaledb - build + make: + chdir: /tmp/timescaledb/build + become: yes + +- name: timescaledb - install + make: + chdir: /tmp/timescaledb/build + target: install + become: yes + # pljava - name: pljava - download & install dependencies apt: From 4eefed7c60a7165f0496566827f3fb965453d2ca Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 00:33:57 +0800 Subject: [PATCH 23/86] chore: add filters for pgbouncer & postgrest - also, starting of Postgres DB will be done here --- ansible/playbook.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 3c83d9717..3eb71beb2 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -24,11 +24,13 @@ - name: Install PgBouncer import_tasks: tasks/setup-pgbouncer.yml - when: install_pgbouncer == true + tags: + - install-pgbouncer - name: Install PostgREST import_tasks: tasks/setup-postgrest.yml - when: install_postgrest == true + tags: + - install-postgrest - name: Install Postgres extensions import_tasks: tasks/setup-extensions.yml @@ -37,8 +39,10 @@ import_tasks: tasks/setup-supabase-internal.yml when: supabase_internal == true - - name: Start Postgres and perform checks - import_tasks: tasks/setup-postgres-init.yml + - name: Start Postgres Database + systemd: + name: postgresql + state: started - name: Adjust APT update intervals copy: From a5f9ac65ed0a5ac52ab9a48e81e2833426392096 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 00:34:09 +0800 Subject: [PATCH 24/86] add var value for timescaledb --- ansible/vars.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/vars.yml b/ansible/vars.yml index cae1581b3..996b14bd3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -40,6 +40,8 @@ plpgsql_check_release_checksum: sha1:626553fc2746fe10aa5a776a1229baf2af3365fc pg_safeupdate_release: "1.3" pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c +timescaledb_release: "2.3.0" + pljava_release: "1_6_2" pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 From dd57f4ac675107a7d9734e0dff27ac08ff93dedc Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 00:34:38 +0800 Subject: [PATCH 25/86] fix: /etc/postgresql should be owned by postgres --- ansible/tasks/setup-postgres.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index b9d45e5f9..7a3da1b2e 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -98,6 +98,7 @@ file: path: /etc/postgresql state: directory + owner: postgres group: postgres # Move Postgres configuration files into /etc/postgresql From 6b24d6c99f78a7c0bc1b611b0497f39ba740223e Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 00:37:02 +0800 Subject: [PATCH 26/86] chore: add test script for postgres installation --- amazon-arm.json | 2 +- amazon.json | 1 + ansible/tasks/setup-postgres-init.yml | 12 ----- digitalOcean.json | 2 +- scripts/01-postgres_check.sh | 72 +++++++++++++++++++++++++++ scripts/01-test | 9 ---- 6 files changed, 75 insertions(+), 23 deletions(-) delete mode 100644 ansible/tasks/setup-postgres-init.yml create mode 100644 scripts/01-postgres_check.sh delete mode 100644 scripts/01-test diff --git a/amazon-arm.json b/amazon-arm.json index 62859d39d..7306f1516 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -51,7 +51,7 @@ "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", "type": "shell", "scripts": [ - "scripts/01-test", + "scripts/01-postgres_check.sh", "scripts/02-credentials_cleanup.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh" diff --git a/amazon.json b/amazon.json index bb58f9344..623305777 100644 --- a/amazon.json +++ b/amazon.json @@ -51,6 +51,7 @@ "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", "type": "shell", "scripts": [ + "scripts/01-postgres_check.sh", "scripts/02-credentials_cleanup.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh" diff --git a/ansible/tasks/setup-postgres-init.yml b/ansible/tasks/setup-postgres-init.yml deleted file mode 100644 index b22206789..000000000 --- a/ansible/tasks/setup-postgres-init.yml +++ /dev/null @@ -1,12 +0,0 @@ -## [WIP] -## TODO -## Checks to ensure: -### directories exist -### directories are owned by postgres -### verify LSE is enabled -### database is running - -- name: Start Postgres Server - systemd: - name: postgresql - state: started \ No newline at end of file diff --git a/digitalOcean.json b/digitalOcean.json index cf540a0d5..01827ab9a 100644 --- a/digitalOcean.json +++ b/digitalOcean.json @@ -35,7 +35,7 @@ { "type": "shell", "scripts": [ - "scripts/01-test", + "scripts/01-postgres_check.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh", "scripts/99-img_check.sh" diff --git a/scripts/01-postgres_check.sh b/scripts/01-postgres_check.sh new file mode 100644 index 000000000..71f30e0bc --- /dev/null +++ b/scripts/01-postgres_check.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Scripts in this directory are run during the build process. +# each script will be uploaded to /tmp on your build droplet, +# given execute permissions and run. The cleanup process will +# remove the scripts from your build system after they have run +# if you use the build_image task. +# +echo "Commencing Checks" + +function check_database_is_ready { + echo -e "\nChecking if database is ready and accepting connections:" + if [ "$(pg_isready)" = "/tmp:5432 - accepting connections" ]; then + echo "Database is ready" + else + echo "Error: Database is not ready. Exiting" + exit 1 + fi +} + +function check_postgres_owned_dir_exists { + DIR=$1 + USER="postgres" + + echo -e "\nChecking if $DIR exists and owned by postgres user:" + + if [ -d "$DIR" ]; then + echo "$DIR exists" + if [ $(stat -c '%U' $DIR) = "$USER" ]; then + echo "$DIR is owned by $USER" + else + echo "Error: $DIR is not owned by $USER" + exit 1 + fi + else + echo "Error: ${DIR} not found. Exiting." + exit 1 + fi +} + +function check_lse_enabled { + ARCH=$(uname -m) + if [ $ARCH = "aarch64" ]; then + echo -e "\nArchitecture is $ARCH. Checking for LSE:" + + LSE_COUNT=$(nm /usr/lib/postgresql/bin/postgres | grep __aarch64_have_lse_atomics | wc -l) + MOUTLINE_ATOMICS_COUNT=$(nm /usr/lib/postgresql/bin/postgres | grep __aarch64_have_lse_atomics | wc -l) + + # Checking for load and store exclusives + if [ $LSE_COUNT -gt 0 ]; then + echo "Postgres has LSE enabled" + else + echo "Error: Postgres failed to be compiled with LSE. Exiting" + exit 1 + fi + + # Checking if successfully compiled with -moutline-atomics + if [ $MOUTLINE_ATOMICS_COUNT -gt 0 ]; then + echo "Postgres has been compiled with -moutline-atomics" + else + echo "Error: Postgres failed to be compiled with -moutline-atomics. Exiting" + exit 1 + fi + else + echo "Architecture is $ARCH. Not checking for LSE." + fi +} + +check_database_is_ready +check_postgres_owned_dir_exists "/var/lib/postgresql" +check_postgres_owned_dir_exists "/etc/postgresql" +check_lse_enabled \ No newline at end of file diff --git a/scripts/01-test b/scripts/01-test deleted file mode 100644 index e5b3e0523..000000000 --- a/scripts/01-test +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# -# Scripts in this directory are run during the build process. -# each script will be uploaded to /tmp on your build droplet, -# given execute permissions and run. The cleanup process will -# remove the scripts from your build system after they have run -# if you use the build_image task. -# -echo "Commencing Digital Ocean Checks" From 88651ea0d5e4991a9cf662ddc20383dc0c96a582 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:27:40 +0800 Subject: [PATCH 27/86] fix: move installation of ec2-instance-connect to the end - atm, it somehow gets deleted during the process --- ansible/playbook.yml | 10 ++++++++++ ansible/tasks/setup-system.yml | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 3eb71beb2..4ae1980d1 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -98,3 +98,13 @@ copy: src: files/logrotate-postgres dest: /etc/logrotate.d/postgres + + # Install EC2 instance connect + # Only for AWS images + - name: install EC2 instance connect + become: yes + apt: + pkg: + - ec2-instance-connect + tags: + - aws-only diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index e9023311c..1258fb88f 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -42,16 +42,6 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic -# Install EC2 instance connect -# Only for AWS images -- name: install EC2 instance connect - become: yes - apt: - pkg: - - ec2-instance-connect - tags: - - aws-only - # Find platform architecture and set as a variable - name: finding platform architecture shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi From 7529dbcafbe6633e627377377df2f3afa2b51be5 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:33:21 +0800 Subject: [PATCH 28/86] chore: clean up and consolidate configuration for PgBouncer --- .../pgbouncer.ini.j2} | 4 +- .../pgbouncer_config/pgbouncer.service.j2 | 40 +++++++++++++++++++ .../pgbouncer_auth_schema.sql | 0 ansible/playbook.yml | 7 +--- ansible/tasks/setup-pgbouncer.yml | 25 +++++++++++- scripts/91-log_cleanup.sh | 3 ++ 6 files changed, 70 insertions(+), 9 deletions(-) rename ansible/files/{pgbouncer.ini => pgbouncer_config/pgbouncer.ini.j2} (99%) create mode 100644 ansible/files/pgbouncer_config/pgbouncer.service.j2 rename ansible/files/{ => pgbouncer_config}/pgbouncer_auth_schema.sql (100%) diff --git a/ansible/files/pgbouncer.ini b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 similarity index 99% rename from ansible/files/pgbouncer.ini rename to ansible/files/pgbouncer_config/pgbouncer.ini.j2 index 1ecd1eacc..61a38a5f8 100644 --- a/ansible/files/pgbouncer.ini +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -43,8 +43,8 @@ ;;; Administrative settings ;;; -logfile = /var/log/postgresql/pgbouncer.log -pidfile = /var/run/postgresql/pgbouncer.pid +logfile = /var/log/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid ;;; ;;; Where to wait for clients diff --git a/ansible/files/pgbouncer_config/pgbouncer.service.j2 b/ansible/files/pgbouncer_config/pgbouncer.service.j2 new file mode 100644 index 000000000..96273cb69 --- /dev/null +++ b/ansible/files/pgbouncer_config/pgbouncer.service.j2 @@ -0,0 +1,40 @@ +# Example systemd service unit for PgBouncer +# +# - Adjust the paths in ExecStart for your installation. +# +# - The User setting requires careful consideration. PgBouncer needs +# to be able to place a Unix-domain socket file where PostgreSQL +# clients will look for it. In the olden days, this was in /tmp, +# but systems using systemd now prefer something like +# /var/run/postgresql/. But then some systems also lock down that +# directory so that only the postgres user can write to it. That +# means you need to either +# +# - run PgBouncer as the postgres user, or +# +# - create a separate user and add it to the postgres group and +# make /var/run/postgresql/ group-writable, or +# +# - use systemd to create the sockets; see pgbouncer.socket nearby. +# +# For packagers and deployment systems, this requires some +# coordination between the PgBouncer and the PostgreSQL +# packages/components. +# +[Unit] +Description=connection pooler for PostgreSQL +Documentation=man:pgbouncer(1) +Documentation=https://www.pgbouncer.org/ +After=network.target +#Requires=pgbouncer.socket + +[Service] +Type=notify +User=postgres +ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGINT +#LimitNOFILE=1024 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/files/pgbouncer_auth_schema.sql b/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql similarity index 100% rename from ansible/files/pgbouncer_auth_schema.sql rename to ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 4ae1980d1..b98da8058 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -9,7 +9,7 @@ vars: sql_files: - - { source: "pgbouncer_auth_schema.sql", dest: "00-schema.sql" } + - { source: "pgbouncer_config/pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - { source: "stat_extension.sql", dest: "01-extension.sql" } environment: @@ -68,11 +68,6 @@ state: absent loop: "{{ sql_files }}" - - name: Adjust pgbouncer.ini - copy: - src: files/pgbouncer.ini - dest: /etc/pgbouncer/pgbouncer.ini - - name: UFW - Allow SSH connections ufw: rule: allow diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 9f0d3d25f..14940ebd0 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -8,6 +8,13 @@ update_cache: yes cache_valid_time: 3600 +- name: Create directory for pgbouncer.pid + file: + path: /var/run/pgbouncer + state: directory + owner: postgres + group: postgres + - name: PgBouncer - download latest release get_url: url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" @@ -39,8 +46,24 @@ become: yes # Create /etc/postgresql directory and make sure postgres group owns it -- name: Create a directory if it does not exist +- name: PgBouncer - create a directory if it does not exist file: path: /etc/pgbouncer state: directory group: postgres + +- name: PgBouncer - adjust pgbouncer.ini + copy: + src: files/pgbouncer_config/pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + +# Add systemd file for PgBouncer +- name: PgBouncer - import postgresql.service + template: + src: files/pgbouncer_config/pgbouncer.service.j2 + dest: /etc/systemd/system/pgbouncer.service + become: yes + +- name: PgBouncer - reload systemd + systemd: + daemon_reload: yes \ No newline at end of file diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh index 96e8dd5b8..85211640a 100644 --- a/scripts/91-log_cleanup.sh +++ b/scripts/91-log_cleanup.sh @@ -6,3 +6,6 @@ rm -rf /var/log/* # https://github.com/fail2ban/fail2ban/issues/1593 touch /var/log/auth.log + +touch /var/log/pgbouncer.log +chown postgres:postgres /var/log/pgbouncer.log \ No newline at end of file From 53a129c9c3c510a0e1d07c493af001fe638df273 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:33:53 +0800 Subject: [PATCH 29/86] chore: boyscout separate versions of aws cli depending on architecture --- ansible/tasks/setup-supabase-internal.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 1304b810c..9140c9bb1 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -4,10 +4,17 @@ - unzip install_recommends: no -- name: AWS CLI +- name: AWS CLI (arm) + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + when: platform == "arm64" + +- name: AWS CLI (x86) get_url: url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" dest: "/tmp/awscliv2.zip" + when: platform == "amd64" - name: AWS CLI - expand unarchive: From cc46c7d8667692b4706994376b17ee9b4506a432 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:34:21 +0800 Subject: [PATCH 30/86] fix: symbolic linking for Postgres binaries --- ansible/tasks/setup-postgres.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 7a3da1b2e..31c4f42ab 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -123,12 +123,19 @@ dest: /etc/postgresql/pg_ident.conf group: postgres +- name: Find all files in /usr/lib/postgresql/bin + find: + paths: /usr/lib/postgresql/bin + register: postgresql_bin + - name: Create symbolic links for Postgres binaries to /usr/bin/ + become: True file: - src: /usr/lib/postgresql/bin/* - dest: /usr/bin/ + src: "{{ item.path }}" + path: "/usr/bin/{{ item.path | basename }}" state: link force: yes + with_items: "{{ postgresql_bin.files }}" # init DB - name: Initialize the database From 8a3513ad88741cf1b0ff06c15eaceb4e4e613e4a Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:34:55 +0800 Subject: [PATCH 31/86] fix: adding dependencies for timescaledb --- ansible/tasks/setup-extensions.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 2e8a7a793..01212a0c7 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -238,6 +238,13 @@ become: yes # timescaledb +- name: timescaledb - download & install dependencies + apt: + pkg: + - cmake + update_cache: yes + install_recommends: yes + - name: timescaledb - download latest release git: repo: https://github.com/timescale/timescaledb.git From 3f1eaa5ac83ba2e9df7b28e55f2c4c49830937e5 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 10 Jun 2021 22:35:16 +0800 Subject: [PATCH 32/86] chore: boyscout remove extra steps from postgrest --- ansible/tasks/setup-postgrest.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 189683920..16a10e8c7 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -36,8 +36,4 @@ - name: PostgREST - reload systemd systemd: - daemon_reload: yes - -- name: PostgREST - restart service - service: name=postgrest state=restarted - when: restart_services + daemon_reload: yes \ No newline at end of file From c12c7b09a92d67fe67e1550c7cfb73e6fce70b34 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 14 Jun 2021 23:43:00 +0800 Subject: [PATCH 33/86] chore: #3 setupfail2ban filter against PG brute force logins --- .../fail2ban_config/filter-postgresql.conf.j2 | 3 +++ .../fail2ban_config/jail-postgresql.conf.j2 | 7 +++++++ .../files/postgresql_config/postgresql.conf.j2 | 4 ++-- ansible/playbook.yml | 3 +++ ansible/tasks/setup-fail2ban.yml | 17 +++++++++++++++++ 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 ansible/files/fail2ban_config/filter-postgresql.conf.j2 create mode 100644 ansible/files/fail2ban_config/jail-postgresql.conf.j2 create mode 100644 ansible/tasks/setup-fail2ban.yml diff --git a/ansible/files/fail2ban_config/filter-postgresql.conf.j2 b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 new file mode 100644 index 000000000..16750ca3b --- /dev/null +++ b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 @@ -0,0 +1,3 @@ +[Definition] +failregex = ^.+FATAL: password authentication failed for user.+$ +ignoreregex = ^127.0.0.1.+ \ No newline at end of file diff --git a/ansible/files/fail2ban_config/jail-postgresql.conf.j2 b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 new file mode 100644 index 000000000..8ed032da9 --- /dev/null +++ b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 @@ -0,0 +1,7 @@ +[postgresql] +enabled = true +port = 5432 +protocol = tcp +filter = postgresql +logpath = /var/lib/postgresql/data/pg_log/postgresql.log +maxretry = 3 \ No newline at end of file diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 97938e363..fac7813b3 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -428,7 +428,7 @@ effective_cache_size = 128MB # - Where to Log - -log_destination = 'csvlog' # Valid values are combinations of +log_destination = 'stderr' # Valid values are combinations of # stderr, csvlog, syslog, and eventlog, # depending on platform. csvlog # requires logging_collector to be on. @@ -537,7 +537,7 @@ log_rotation_size = 0 # Automatic rotation of logfiles will #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off -#log_line_prefix = '%m [%p] ' # special values: +log_line_prefix = '%h %m [%p] %q%u@%d ' # special values: # %a = application name # %u = user name # %d = database name diff --git a/ansible/playbook.yml b/ansible/playbook.yml index b98da8058..e8afab4c5 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -94,6 +94,9 @@ src: files/logrotate-postgres dest: /etc/logrotate.d/postgres + - name: Enhance fail2ban + import_tasks: tasks/setup-fail2ban.yml + # Install EC2 instance connect # Only for AWS images - name: install EC2 instance connect diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml new file mode 100644 index 000000000..bc00d76d7 --- /dev/null +++ b/ansible/tasks/setup-fail2ban.yml @@ -0,0 +1,17 @@ +- name: import jail.d/postgresql.conf + template: + src: files/fail2ban_config/jail-postgresql.conf.j2 + dest: /etc/fail2ban/jail.d/postgresql.conf + become: yes + +- name: import filter.d/postgresql.conf + template: + src: files/fail2ban_config/filter-postgresql.conf.j2 + dest: /etc/fail2ban/filter.d/postgresql.conf + become: yes + +# Restart +- name: fail2ban - restart + systemd: + name: fail2ban + state: restarted \ No newline at end of file From 3c098863591b101f2f7620b92e36a195f85304a3 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 14 Jun 2021 23:43:44 +0800 Subject: [PATCH 34/86] fix: ensure pgbouncer works upon systemctl start --- .../files/pgbouncer_config/pgbouncer.ini.j2 | 3 +-- .../tmpfiles.d-pgbouncer.conf.j2 | 2 ++ ansible/tasks/setup-pgbouncer.yml | 20 ++++++++++++------- 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 index 61a38a5f8..c0c5dd0f6 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -56,10 +56,9 @@ listen_port = 6543 ;; Unix socket is also used for -R. ;; On Debian it should be /var/run/postgresql -;unix_socket_dir = /tmp +unix_socket_dir = /tmp ;unix_socket_mode = 0777 ;unix_socket_group = -unix_socket_dir = /var/run/postgresql ;;; ;;; TLS settings for accepting clients diff --git a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 new file mode 100644 index 000000000..3889ed294 --- /dev/null +++ b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 @@ -0,0 +1,2 @@ +# Directory for PostgreSQL sockets, lockfiles and stats tempfiles +d /run/pgbouncer 2775 postgres postgres - - \ No newline at end of file diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 14940ebd0..f23337f57 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -8,13 +8,6 @@ update_cache: yes cache_valid_time: 3600 -- name: Create directory for pgbouncer.pid - file: - path: /var/run/pgbouncer - state: directory - owner: postgres - group: postgres - - name: PgBouncer - download latest release get_url: url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" @@ -57,6 +50,19 @@ src: files/pgbouncer_config/pgbouncer.ini.j2 dest: /etc/pgbouncer/pgbouncer.ini +- name: PgBouncer - create a directory if it does not exist + file: + path: /etc/pgbouncer/userlist.txt + state: touch + group: postgres + owner: postgres + +- name: import /etc/tmpfiles.d/pgbouncer.conf + template: + src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 + dest: /etc/tmpfiles.d/pgbouncer.conf + become: yes + # Add systemd file for PgBouncer - name: PgBouncer - import postgresql.service template: From 5fec0cf7ebf80892c7b042b0f3b52cd533075d15 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 14 Jun 2021 23:44:01 +0800 Subject: [PATCH 35/86] fix: correct check for load and store exclusives --- scripts/01-postgres_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/01-postgres_check.sh b/scripts/01-postgres_check.sh index 71f30e0bc..d131528ee 100644 --- a/scripts/01-postgres_check.sh +++ b/scripts/01-postgres_check.sh @@ -43,7 +43,7 @@ function check_lse_enabled { if [ $ARCH = "aarch64" ]; then echo -e "\nArchitecture is $ARCH. Checking for LSE:" - LSE_COUNT=$(nm /usr/lib/postgresql/bin/postgres | grep __aarch64_have_lse_atomics | wc -l) + LSE_COUNT=$(objdump -d /usr/lib/postgresql/bin/postgres | grep -i 'ldxr\|ldaxr\|stxr\|stlxr' | wc -l) MOUTLINE_ATOMICS_COUNT=$(nm /usr/lib/postgresql/bin/postgres | grep __aarch64_have_lse_atomics | wc -l) # Checking for load and store exclusives From 9c999baba2300b27424d486ed4911b5956f8f3c9 Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 15 Jun 2021 13:57:16 +0800 Subject: [PATCH 36/86] chore: remove ansible role anxs/postgres --- .github/workflows/provision.yml | 5 ---- ansible/tasks/setup-wal-g.yml | 43 --------------------------------- 2 files changed, 48 deletions(-) delete mode 100644 ansible/tasks/setup-wal-g.yml diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index 9e67c6a60..a7f4b67aa 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -31,11 +31,6 @@ jobs: sudo apt-get install ansible -y ansible --version - - name: Install Ansible role - run: | - sudo ansible-galaxy install anxs.postgresql -r ansible/install_roles.yml --force -vvv - ansible-galaxy list anxs.postgresql - - name: Build Digital Ocean Snapshot run: | export REGION=sgp1 diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml deleted file mode 100644 index 1861de010..000000000 --- a/ansible/tasks/setup-wal-g.yml +++ /dev/null @@ -1,43 +0,0 @@ -# WAL-G -- name: Install daemontools - become: yes - apt: - pkg: - - daemontools - -- name: wal-g system dependencies - apt: - pkg: - - liblzo2-dev - - cmake - - build-essential - -# install go dependency for WAL-G -- name: wal-g go dependency - get_url: - url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /tmp -- name: unpack go archive - unarchive: - remote_src: yes - src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /usr/local - -# Download WAL-G -- name: download wal-g - shell: - cmd: go get github.com/wal-g/wal-g; - environment: - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - ignore_errors: yes - # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 - -# Install WAL-G -- name: install wal-g - become: yes - shell: - cmd: make install && make deps && make pg_install - chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" - environment: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" \ No newline at end of file From 62d20528e050fa1c1ed2adfcffdef390f35f639c Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 16 Jun 2021 13:45:31 +0800 Subject: [PATCH 37/86] chore: ensure that postgres user has access to ssl certs --- ansible/tasks/setup-postgres.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 31c4f42ab..0ff7629fb 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -16,6 +16,7 @@ - libxml2-utils - uuid-dev - xsltproc + - ssl-cert - name: Download LLVM & Clang become: yes @@ -84,6 +85,7 @@ name: postgres shell: /bin/bash comment: Postgres user + groups: ssl-cert - name: Recursively change ownership of a directory file: From 3b10a446d3676a57953f9636bb0c488c10af1b30 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 16 Jun 2021 13:45:48 +0800 Subject: [PATCH 38/86] chore: create extension internally instead --- ansible/playbook.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index e8afab4c5..5e8cdc318 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -10,7 +10,6 @@ vars: sql_files: - { source: "pgbouncer_config/pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - - { source: "stat_extension.sql", dest: "01-extension.sql" } environment: PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} From c0d1573de3abd4bdb74735bda349fd3243db4694 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 16 Jun 2021 13:47:19 +0800 Subject: [PATCH 39/86] fix: accidentally removed wal-g - at the same time include #48 cleaning up of wal-g installation --- ansible/tasks/setup-wal-g.yml | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ansible/tasks/setup-wal-g.yml diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml new file mode 100644 index 000000000..185116e0b --- /dev/null +++ b/ansible/tasks/setup-wal-g.yml @@ -0,0 +1,36 @@ +# install go dependency for WAL-G +- name: wal-g go dependency + get_url: + url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /tmp +- name: unpack go archive + unarchive: + remote_src: yes + src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /usr/local + +# Download WAL-G +- name: download wal-g + shell: + cmd: go get github.com/wal-g/wal-g; + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + ignore_errors: yes + # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 + +# Install WAL-G +- name: install wal-g + become: yes + shell: + cmd: make install && make deps && make pg_install + chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" + environment: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + +# Clean up Go +- name: Uninstall Go + become: yes + file: + path: /usr/local/go + state: absent \ No newline at end of file From a8ad52baa50ce06eb740bc04849afb5efa990fc5 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 17 Jun 2021 12:55:53 +0800 Subject: [PATCH 40/86] fix: add in wal-g dependencies --- ansible/tasks/setup-wal-g.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 185116e0b..3f2bc5adc 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,8 +1,17 @@ +# Downloading dependencies +- name: Postgres dependencies + become: yes + apt: + pkg: + - liblzo2-dev + - cmake + # install go dependency for WAL-G - name: wal-g go dependency get_url: url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" dest: /tmp + - name: unpack go archive unarchive: remote_src: yes From 845f6b205d8a9f124af6904b0a51be6a817f3d3c Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 29 Jun 2021 15:02:27 +0800 Subject: [PATCH 41/86] chore: #49 install supautils extension --- ansible/tasks/setup-supabase-internal.yml | 45 +++++++++++++++++++++++ ansible/vars.yml | 3 ++ 2 files changed, 48 insertions(+) diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 9140c9bb1..303e4257b 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -70,3 +70,48 @@ enabled: yes name: postgres_exporter daemon_reload: yes + +# supautils +- name: supautils - download latest release + get_url: + url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" + dest: /tmp/supautils-{{ supautils_release }}.tar.gz + checksum: "{{ supautils_release_checksum }}" + +- name: supautils - unpack archive + unarchive: + remote_src: yes + src: /tmp/supautils-{{ supautils_release }}.tar.gz + dest: /tmp + become: yes + +- name: supautils - build + make: + chdir: /tmp/supautils-{{ supautils_release }} + become: yes + +- name: supautils - install + make: + chdir: /tmp/supautils-{{ supautils_release }} + target: install + become: yes + +- name: supautils - set supautils.reserved_roles + become: yes + shell: + cmd: printf "supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon'" >> /etc/postgresql/postgresql.conf + +- name: supautils - set supautils.reserved_memberships + become: yes + shell: + cmd: printf "supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program'" >> /etc/postgresql/postgresql.conf + +- name: supautils - set supautils.reserved_memberships + become: yes + shell: + cmd: printf "supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program'" >> /etc/postgresql/postgresql.conf + +- name: supautils - add supautils to shared_preload_libraries + become: yes + shell: + cmd: sed -i "s/shared_preload_libraries = '/shared_preload_libraries ='supautils,/" /etc/postgresql/postgresql.conf \ No newline at end of file diff --git a/ansible/vars.yml b/ansible/vars.yml index 996b14bd3..4268b39f5 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -42,6 +42,9 @@ pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c timescaledb_release: "2.3.0" +supautils_release: "1.1.0" +supautils_release_checksum: sha1:326ac5c1933bd30d4a50da7568b27629a9ec544b + pljava_release: "1_6_2" pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 From 32adbf5f3bb49127eec44a2ecde2e2ce9802d220 Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 29 Jun 2021 15:03:52 +0800 Subject: [PATCH 42/86] chore: #50 enable postgis_sfcgal extension --- ansible/tasks/setup-extensions.yml | 39 ++++++++++++++++++++++++++++-- ansible/vars.yml | 5 +++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 01212a0c7..4007f11b7 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -7,9 +7,44 @@ - libgdal-dev - libjson-c-dev - libxml2-dev + - libboost-all-dev + - libcgal-dev + - libmpfr-dev + - libgmp-dev + - cmake update_cache: yes cache_valid_time: 3600 +- name: postgis - download SFCGAL dependency + get_url: + url: "https://gitlab.com/Oslandia/SFCGAL/-/archive/v{{ sfcgal_release }}/SFCGAL-v{{ sfcgal_release }}.tar.gz" + dest: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + checksum: "{{ sfcgal_release_checksum }}" + +- name: postgis - unpack SFCGAL + unarchive: + remote_src: yes + src: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - compile SFCGAL + shell: + cmd: "cmake ." + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - build SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - install SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + target: install + become: yes + - name: postgis - download latest release get_url: url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz" @@ -25,7 +60,7 @@ - name: postgis - configure shell: - cmd: "./configure --without-protobuf" + cmd: "./configure --without-protobuf --with-sfcgal" chdir: /tmp/postgis-{{ postgis_release }} become: yes @@ -268,7 +303,7 @@ chdir: /tmp/timescaledb/build target: install become: yes - + # pljava - name: pljava - download & install dependencies apt: diff --git a/ansible/vars.yml b/ansible/vars.yml index 4268b39f5..c9fd23187 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,4 +50,7 @@ pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 golang_version: "1.15.4" wal_g_release: "0.2.15" -wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 \ No newline at end of file +wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 + +sfcgal_release: "1.3.10" +sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b \ No newline at end of file From dc510417cbd04c40c681eb485f40288f6d20ab32 Mon Sep 17 00:00:00 2001 From: Angelico Date: Tue, 29 Jun 2021 15:07:17 +0800 Subject: [PATCH 43/86] chore: #45 additional extension - PgRouting --- ansible/tasks/setup-extensions.yml | 45 ++++++++++++++++++++++++++++++ ansible/vars.yml | 3 ++ 2 files changed, 48 insertions(+) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 4007f11b7..c8df44c32 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -75,6 +75,51 @@ target: install become: yes +# pgRouting +- name: pgRouting - download & install dependencies + apt: + pkg: + - perl + - python3-sphinx + update_cache: yes + cache_valid_time: 3600 + +- name: pgRouting - download latest release + get_url: + url: "https://github.com/pgRouting/pgrouting/releases/download/v{{ pgrouting_release }}/pgrouting-{{ pgrouting_release }}.tar.gz" + dest: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + checksum: "{{ pgrouting_release_checksum }}" + +- name: pgRouting - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgRouting - create build directory + file: + path: /tmp/pgrouting-{{ pgrouting_release }}/build + state: directory + become: yes + +- name: pgRouting - compile + shell: + cmd: "cmake .." + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - build + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - install + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + target: install + become: yes + # pgTAP - name: pgTAP - download latest release get_url: diff --git a/ansible/vars.yml b/ansible/vars.yml index c9fd23187..21a4c7eb5 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -22,6 +22,9 @@ aws_cli_release: "2.0.30" postgis_release: "3.1.2" postgis_release_checksum: sha1:622f52f3bf338c8e51ea6d73d30d6a5d3140c517 +pgrouting_release: "3.2.0" +pgrouting_release_checksum: sha1:d902d449ebc96b6cdcb2fac09434d0098467cda5 + pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e From bf63a222bcae9c8fbf8659eded46659ad971e428 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 30 Jun 2021 23:40:20 +0800 Subject: [PATCH 44/86] chore: cleanup - remove duplicated commands - use lineinfile and replace utilities instead of echo and sed --- ansible/tasks/setup-extensions.yml | 12 ++++++++---- ansible/tasks/setup-supabase-internal.yml | 23 ++++++++++++----------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index c8df44c32..56fb1462d 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -167,8 +167,10 @@ - name: pg_cron - set cron.database_name become: yes - shell: - cmd: echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: cron.database_name = 'postgres' # pgAudit - name: pgAudit - download & install dependencies @@ -401,8 +403,10 @@ - name: pljava - set pljava.libjvm_location become: yes - shell: - cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so' - name: pljava - remove ~/.m2 directory become: yes diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 303e4257b..5e88cce96 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -98,20 +98,21 @@ - name: supautils - set supautils.reserved_roles become: yes - shell: - cmd: printf "supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon'" >> /etc/postgresql/postgresql.conf + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon' - name: supautils - set supautils.reserved_memberships become: yes - shell: - cmd: printf "supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program'" >> /etc/postgresql/postgresql.conf - -- name: supautils - set supautils.reserved_memberships - become: yes - shell: - cmd: printf "supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program'" >> /etc/postgresql/postgresql.conf + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program' - name: supautils - add supautils to shared_preload_libraries become: yes - shell: - cmd: sed -i "s/shared_preload_libraries = '/shared_preload_libraries ='supautils,/" /etc/postgresql/postgresql.conf \ No newline at end of file + replace: + path: /etc/postgresql/postgresql.conf + regexp: shared_preload_libraries = ' + replace: shared_preload_libraries = 'supautils, \ No newline at end of file From 114654e31ed8d7780a3b705a7a2de43970ba684c Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 1 Jul 2021 15:06:10 +0800 Subject: [PATCH 45/86] fix: put stat_extensions.sql back in --- ansible/playbook.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 5e8cdc318..e8afab4c5 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -10,6 +10,7 @@ vars: sql_files: - { source: "pgbouncer_config/pgbouncer_auth_schema.sql", dest: "00-schema.sql" } + - { source: "stat_extension.sql", dest: "01-extension.sql" } environment: PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} From 60c36602bad626ae1650f5f49bf732f342d4faeb Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 1 Jul 2021 23:34:14 +0800 Subject: [PATCH 46/86] chore: clean up extensions - separate them into their individual files --- .../tasks/postgres-extensions/01-postgis.yml | 76 +++ .../postgres-extensions/02-pgrouting.yml | 44 ++ .../tasks/postgres-extensions/03-pgtap.yml | 19 + .../tasks/postgres-extensions/04-pg_cron.yml | 31 ++ .../tasks/postgres-extensions/05-pgaudit.yml | 37 ++ .../tasks/postgres-extensions/06-pgjwt.yml | 12 + .../postgres-extensions/07-pgsql-http.yml | 37 ++ .../postgres-extensions/08-plpgsql_check.yml | 32 ++ .../postgres-extensions/09-pg-safeupdate.yml | 24 + .../postgres-extensions/10-timescaledb.yml | 31 ++ .../tasks/postgres-extensions/11-pljava.yml | 62 +++ ansible/tasks/postgres-extensions/12-plv8.yml | 48 ++ ansible/tasks/setup-extensions.yml | 477 +----------------- 13 files changed, 477 insertions(+), 453 deletions(-) create mode 100644 ansible/tasks/postgres-extensions/01-postgis.yml create mode 100644 ansible/tasks/postgres-extensions/02-pgrouting.yml create mode 100644 ansible/tasks/postgres-extensions/03-pgtap.yml create mode 100644 ansible/tasks/postgres-extensions/04-pg_cron.yml create mode 100644 ansible/tasks/postgres-extensions/05-pgaudit.yml create mode 100644 ansible/tasks/postgres-extensions/06-pgjwt.yml create mode 100644 ansible/tasks/postgres-extensions/07-pgsql-http.yml create mode 100644 ansible/tasks/postgres-extensions/08-plpgsql_check.yml create mode 100644 ansible/tasks/postgres-extensions/09-pg-safeupdate.yml create mode 100644 ansible/tasks/postgres-extensions/10-timescaledb.yml create mode 100644 ansible/tasks/postgres-extensions/11-pljava.yml create mode 100644 ansible/tasks/postgres-extensions/12-plv8.yml diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml new file mode 100644 index 000000000..a60366d6c --- /dev/null +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -0,0 +1,76 @@ +# postgis +- name: postgis - download & install dependencies + apt: + pkg: + - libgeos-dev + - libproj-dev + - libgdal-dev + - libjson-c-dev + - libxml2-dev + - libboost-all-dev + - libcgal-dev + - libmpfr-dev + - libgmp-dev + - cmake + update_cache: yes + cache_valid_time: 3600 + +- name: postgis - download SFCGAL dependency + get_url: + url: "https://gitlab.com/Oslandia/SFCGAL/-/archive/v{{ sfcgal_release }}/SFCGAL-v{{ sfcgal_release }}.tar.gz" + dest: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + checksum: "{{ sfcgal_release_checksum }}" + +- name: postgis - unpack SFCGAL + unarchive: + remote_src: yes + src: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - compile SFCGAL + shell: + cmd: "cmake ." + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - build SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - install SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + target: install + become: yes + +- name: postgis - download latest release + get_url: + url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz" + dest: /tmp/postgis-{{ postgis_release }}.tar.gz + checksum: "{{ postgis_release_checksum }}" + +- name: postgis - unpack archive + unarchive: + remote_src: yes + src: /tmp/postgis-{{ postgis_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - configure + shell: + cmd: "./configure --without-protobuf --with-sfcgal" + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - build + make: + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - install + make: + chdir: /tmp/postgis-{{ postgis_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml new file mode 100644 index 000000000..b03e33e9c --- /dev/null +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -0,0 +1,44 @@ +# pgRouting +- name: pgRouting - download & install dependencies + apt: + pkg: + - perl + - python3-sphinx + update_cache: yes + cache_valid_time: 3600 + +- name: pgRouting - download latest release + get_url: + url: "https://github.com/pgRouting/pgrouting/releases/download/v{{ pgrouting_release }}/pgrouting-{{ pgrouting_release }}.tar.gz" + dest: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + checksum: "{{ pgrouting_release_checksum }}" + +- name: pgRouting - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgRouting - create build directory + file: + path: /tmp/pgrouting-{{ pgrouting_release }}/build + state: directory + become: yes + +- name: pgRouting - compile + shell: + cmd: "cmake .." + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - build + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - install + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/03-pgtap.yml b/ansible/tasks/postgres-extensions/03-pgtap.yml new file mode 100644 index 000000000..6dc11f0e3 --- /dev/null +++ b/ansible/tasks/postgres-extensions/03-pgtap.yml @@ -0,0 +1,19 @@ +# pgTAP +- name: pgTAP - download latest release + get_url: + url: "https://github.com/theory/pgtap/archive/v{{ pgtap_release }}.tar.gz" + dest: /tmp/pgtap-{{ pgtap_release }}.tar.gz + checksum: "{{ pgtap_release_checksum }}" + +- name: pgTAP - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgtap-{{ pgtap_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgTAP - install + make: + chdir: /tmp/pgtap-{{ pgtap_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/04-pg_cron.yml b/ansible/tasks/postgres-extensions/04-pg_cron.yml new file mode 100644 index 000000000..5bdf294f5 --- /dev/null +++ b/ansible/tasks/postgres-extensions/04-pg_cron.yml @@ -0,0 +1,31 @@ +# pg_cron +- name: pg_cron - download latest release + get_url: + url: "https://github.com/citusdata/pg_cron/archive/refs/tags/v{{ pg_cron_release }}.tar.gz" + dest: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + checksum: "{{ pg_cron_release_checksum }}" + +- name: pg_cron - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + dest: /tmp + become: yes + +- name: pg_cron - build + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + become: yes + +- name: pg_cron - install + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + target: install + become: yes + +- name: pg_cron - set cron.database_name + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: cron.database_name = 'postgres' \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/05-pgaudit.yml b/ansible/tasks/postgres-extensions/05-pgaudit.yml new file mode 100644 index 000000000..6d3b2bca7 --- /dev/null +++ b/ansible/tasks/postgres-extensions/05-pgaudit.yml @@ -0,0 +1,37 @@ +# pgAudit +- name: pgAudit - download & install dependencies + apt: + pkg: + - libssl-dev + - libkrb5-dev + update_cache: yes + install_recommends: no + +- name: pgAudit - download latest release + get_url: + url: "https://github.com/pgaudit/pgaudit/archive/refs/tags/{{ pgaudit_release }}.tar.gz" + dest: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz + checksum: "{{ pgaudit_release_checksum }}" + +- name: pgAudit - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgAudit - build + make: + chdir: /tmp/pgaudit-{{ pgaudit_release }} + target: check + params: + USE_PGXS: 1 + become: yes + +- name: pgAudit - install + make: + chdir: /tmp/pgaudit-{{ pgaudit_release }} + target: install + params: + USE_PGXS: 1 + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/06-pgjwt.yml b/ansible/tasks/postgres-extensions/06-pgjwt.yml new file mode 100644 index 000000000..b2734e130 --- /dev/null +++ b/ansible/tasks/postgres-extensions/06-pgjwt.yml @@ -0,0 +1,12 @@ +# pgjwt +- name: pgjwt - download from master branch + git: + repo: https://github.com/michelp/pgjwt.git + dest: /tmp/pgjwt + version: master + +- name: pgjwt - install + make: + chdir: /tmp/pgjwt + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml new file mode 100644 index 000000000..f72140da4 --- /dev/null +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -0,0 +1,37 @@ +# pgsql-http +- name: pgsql-http - libcurl4 package + apt: + pkg: + - libcurl4 + state: absent + +- name: pgsql-http - download & install dependencies + apt: + pkg: + - libcurl4-gnutls-dev + update_cache: yes + install_recommends: yes + +- name: pgsql-http - download latest release + get_url: + url: "https://github.com/pramsey/pgsql-http/archive/refs/tags/v{{ pgsql_http_release }}.tar.gz" + dest: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz + checksum: "{{ pgsql_http_release_checksum }}" + +- name: pgsql-http - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgsql-http - build + make: + chdir: /tmp/pgsql-http-{{ pgsql_http_release }} + become: yes + +- name: pgsql-http - install + make: + chdir: /tmp/pgsql-http-{{ pgsql_http_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml new file mode 100644 index 000000000..16fb5aa7e --- /dev/null +++ b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml @@ -0,0 +1,32 @@ +# plpgsql_check +- name: plpgsql_check - download & install dependencies + apt: + pkg: + - libicu-dev + update_cache: yes + install_recommends: no + +- name: plpgsql_check - download latest release + get_url: + url: "https://github.com/okbob/plpgsql_check/archive/refs/tags/v{{ plpgsql_check_release }}.tar.gz" + dest: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz + checksum: "{{ plpgsql_check_release_checksum }}" + +- name: plpgsql_check - unpack archive + unarchive: + remote_src: yes + src: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz + dest: /tmp + become: yes + +- name: plpgsql_check - clean + make: + chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} + target: clean + become: yes + +- name: plpgsql_check - install + make: + chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml new file mode 100644 index 000000000..e27cfd9dc --- /dev/null +++ b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml @@ -0,0 +1,24 @@ +# pg-safeupdate +- name: pg-safeupdate - download latest release + get_url: + url: "https://github.com/eradman/pg-safeupdate/archive/refs/tags/{{ pg_safeupdate_release }}.tar.gz" + dest: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + checksum: "{{ pg_safeupdate_release_checksum }}" + +- name: pg-safeupdate - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + dest: /tmp + become: yes + +- name: pg-safeupdate - build + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + become: yes + +- name: pg-safeupdate - install + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/10-timescaledb.yml b/ansible/tasks/postgres-extensions/10-timescaledb.yml new file mode 100644 index 000000000..154ecc426 --- /dev/null +++ b/ansible/tasks/postgres-extensions/10-timescaledb.yml @@ -0,0 +1,31 @@ +# timescaledb +- name: timescaledb - download & install dependencies + apt: + pkg: + - cmake + update_cache: yes + install_recommends: yes + +- name: timescaledb - download latest release + git: + repo: https://github.com/timescale/timescaledb.git + dest: /tmp/timescaledb + version: "{{ timescaledb_release }}" + become: yes + +- name: timescaledb - bootstrap + shell: + cmd: "./bootstrap -DAPACHE_ONLY=1 -DREGRESS_CHECKS=OFF" + chdir: /tmp/timescaledb + become: yes + +- name: timescaledb - build + make: + chdir: /tmp/timescaledb/build + become: yes + +- name: timescaledb - install + make: + chdir: /tmp/timescaledb/build + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/11-pljava.yml b/ansible/tasks/postgres-extensions/11-pljava.yml new file mode 100644 index 000000000..0495484df --- /dev/null +++ b/ansible/tasks/postgres-extensions/11-pljava.yml @@ -0,0 +1,62 @@ +# pljava +- name: pljava - download & install dependencies + apt: + pkg: + - maven + - default-jre + - default-jdk + update_cache: yes + install_recommends: yes + +- name: pljava - download latest release + get_url: + url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz + dest: /tmp/pljava-{{ pljava_release }}.tar.gz + checksum: "{{ pljava_release_checksum }}" + +- name: pljava - unpack archive + unarchive: + remote_src: yes + src: /tmp/pljava-{{ pljava_release }}.tar.gz + dest: /tmp + become: yes + +- name: pljava - build + become: yes + shell: + cmd: mvn clean install + chdir: /tmp/pljava-{{ pljava_release }} + +- name: pljava - install + become: yes + shell: + cmd: java -jar pljava-packaging/target/pljava-pg13.jar + chdir: /tmp/pljava-{{ pljava_release }} + +- name: pljava - remove build dependencies + apt: + pkg: + - maven + - default-jre + - default-jdk + state: absent + +- name: pljava - install headless jdk + apt: + pkg: + - default-jdk-headless + update_cache: yes + install_recommends: no + +- name: pljava - set pljava.libjvm_location + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so' + +- name: pljava - remove ~/.m2 directory + become: yes + file: + path: ~/.m2 + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/12-plv8.yml b/ansible/tasks/postgres-extensions/12-plv8.yml new file mode 100644 index 000000000..1966649ca --- /dev/null +++ b/ansible/tasks/postgres-extensions/12-plv8.yml @@ -0,0 +1,48 @@ +# plv8 +- name: plv8 - download & install dependencies + apt: + pkg: + - build-essential + - ca-certificates + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - libc++-dev + - libc++abi-dev + - libc++1 + - libglib2.0-dev + - libtinfo5 + - libc++abi1 + - ninja-build + - python + update_cache: yes + install_recommends: no + +- name: plv8 - download latest release + git: + repo: https://github.com/plv8/plv8.git + dest: /tmp/plv8 + version: 3656177d384e3e02b74faa8e2931600f3690ab59 + become: yes + +- name: Create a symbolic link + file: + src: /lib/aarch64-linux-gnu/libc++.so.1 + dest: /lib/aarch64-linux-gnu/libc++.so + state: link + when: platform == "arm64" + +- name: plv8 - build + make: + chdir: /tmp/plv8 + become: yes + +- name: plv8 - install + make: + chdir: /tmp/plv8 + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 56fb1462d..6d689cd0e 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -1,464 +1,35 @@ -# postgis -- name: postgis - download & install dependencies - apt: - pkg: - - libgeos-dev - - libproj-dev - - libgdal-dev - - libjson-c-dev - - libxml2-dev - - libboost-all-dev - - libcgal-dev - - libmpfr-dev - - libgmp-dev - - cmake - update_cache: yes - cache_valid_time: 3600 +- name: Install postgis + import_tasks: tasks/postgres-extensions/01-postgis.yml -- name: postgis - download SFCGAL dependency - get_url: - url: "https://gitlab.com/Oslandia/SFCGAL/-/archive/v{{ sfcgal_release }}/SFCGAL-v{{ sfcgal_release }}.tar.gz" - dest: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz - checksum: "{{ sfcgal_release_checksum }}" +- name: Install pgrouting + import_tasks: tasks/postgres-extensions/02-pgrouting.yml -- name: postgis - unpack SFCGAL - unarchive: - remote_src: yes - src: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz - dest: /tmp - become: yes +- name: Install pgtap + import_tasks: tasks/postgres-extensions/03-pgtap.yml -- name: postgis - compile SFCGAL - shell: - cmd: "cmake ." - chdir: /tmp/SFCGAL-v{{ sfcgal_release }} - become: yes +- name: Install pg_cron + import_tasks: tasks/postgres-extensions/04-pg_cron.yml -- name: postgis - build SFCGAL - make: - chdir: /tmp/SFCGAL-v{{ sfcgal_release }} - become: yes +- name: Install pgaudit + import_tasks: tasks/postgres-extensions/05-pgaudit.yml -- name: postgis - install SFCGAL - make: - chdir: /tmp/SFCGAL-v{{ sfcgal_release }} - target: install - become: yes +- name: Install pgjwt + import_tasks: tasks/postgres-extensions/06-pgjwt.yml -- name: postgis - download latest release - get_url: - url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz" - dest: /tmp/postgis-{{ postgis_release }}.tar.gz - checksum: "{{ postgis_release_checksum }}" +- name: Install pgsql-http + import_tasks: tasks/postgres-extensions/07-pgsql-http.yml -- name: postgis - unpack archive - unarchive: - remote_src: yes - src: /tmp/postgis-{{ postgis_release }}.tar.gz - dest: /tmp - become: yes +- name: Install plpgsql_check + import_tasks: tasks/postgres-extensions/08-plpgsql_check.yml -- name: postgis - configure - shell: - cmd: "./configure --without-protobuf --with-sfcgal" - chdir: /tmp/postgis-{{ postgis_release }} - become: yes +- name: Install pg-safeupdate + import_tasks: tasks/postgres-extensions/09-pg-safeupdate.yml -- name: postgis - build - make: - chdir: /tmp/postgis-{{ postgis_release }} - become: yes +- name: Install timescaledb + import_tasks: tasks/postgres-extensions/10-timescaledb.yml -- name: postgis - install - make: - chdir: /tmp/postgis-{{ postgis_release }} - target: install - become: yes +- name: Install pljava + import_tasks: tasks/postgres-extensions/11-pljava.yml -# pgRouting -- name: pgRouting - download & install dependencies - apt: - pkg: - - perl - - python3-sphinx - update_cache: yes - cache_valid_time: 3600 - -- name: pgRouting - download latest release - get_url: - url: "https://github.com/pgRouting/pgrouting/releases/download/v{{ pgrouting_release }}/pgrouting-{{ pgrouting_release }}.tar.gz" - dest: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz - checksum: "{{ pgrouting_release_checksum }}" - -- name: pgRouting - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz - dest: /tmp - become: yes - -- name: pgRouting - create build directory - file: - path: /tmp/pgrouting-{{ pgrouting_release }}/build - state: directory - become: yes - -- name: pgRouting - compile - shell: - cmd: "cmake .." - chdir: /tmp/pgrouting-{{ pgrouting_release }}/build - become: yes - -- name: pgRouting - build - make: - chdir: /tmp/pgrouting-{{ pgrouting_release }}/build - become: yes - -- name: pgRouting - install - make: - chdir: /tmp/pgrouting-{{ pgrouting_release }}/build - target: install - become: yes - -# pgTAP -- name: pgTAP - download latest release - get_url: - url: "https://github.com/theory/pgtap/archive/v{{ pgtap_release }}.tar.gz" - dest: /tmp/pgtap-{{ pgtap_release }}.tar.gz - checksum: "{{ pgtap_release_checksum }}" - -- name: pgTAP - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgtap-{{ pgtap_release }}.tar.gz - dest: /tmp - become: yes - -- name: pgTAP - install - make: - chdir: /tmp/pgtap-{{ pgtap_release }} - target: install - become: yes - -# pg_cron -- name: pg_cron - download latest release - get_url: - url: "https://github.com/citusdata/pg_cron/archive/refs/tags/v{{ pg_cron_release }}.tar.gz" - dest: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz - checksum: "{{ pg_cron_release_checksum }}" - -- name: pg_cron - unpack archive - unarchive: - remote_src: yes - src: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz - dest: /tmp - become: yes - -- name: pg_cron - build - make: - chdir: /tmp/pg_cron-{{ pg_cron_release }} - become: yes - -- name: pg_cron - install - make: - chdir: /tmp/pg_cron-{{ pg_cron_release }} - target: install - become: yes - -- name: pg_cron - set cron.database_name - become: yes - lineinfile: - path: /etc/postgresql/postgresql.conf - state: present - line: cron.database_name = 'postgres' - -# pgAudit -- name: pgAudit - download & install dependencies - apt: - pkg: - - libssl-dev - - libkrb5-dev - update_cache: yes - install_recommends: no - -- name: pgAudit - download latest release - get_url: - url: "https://github.com/pgaudit/pgaudit/archive/refs/tags/{{ pgaudit_release }}.tar.gz" - dest: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz - checksum: "{{ pgaudit_release_checksum }}" - -- name: pgAudit - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz - dest: /tmp - become: yes - -- name: pgAudit - build - make: - chdir: /tmp/pgaudit-{{ pgaudit_release }} - target: check - params: - USE_PGXS: 1 - become: yes - -- name: pgAudit - install - make: - chdir: /tmp/pgaudit-{{ pgaudit_release }} - target: install - params: - USE_PGXS: 1 - become: yes - -# pgjwt -- name: pgjwt - download from master branch - git: - repo: https://github.com/michelp/pgjwt.git - dest: /tmp/pgjwt - version: master - -- name: pgjwt - install - make: - chdir: /tmp/pgjwt - target: install - become: yes - -# pgsql-http -- name: pgsql-http - libcurl4 package - apt: - pkg: - - libcurl4 - state: absent - -- name: pgsql-http - download & install dependencies - apt: - pkg: - - libcurl4-gnutls-dev - update_cache: yes - install_recommends: yes - -- name: pgsql-http - download latest release - get_url: - url: "https://github.com/pramsey/pgsql-http/archive/refs/tags/v{{ pgsql_http_release }}.tar.gz" - dest: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz - checksum: "{{ pgsql_http_release_checksum }}" - -- name: pgsql-http - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz - dest: /tmp - become: yes - -- name: pgsql-http - build - make: - chdir: /tmp/pgsql-http-{{ pgsql_http_release }} - become: yes - -- name: pgsql-http - install - make: - chdir: /tmp/pgsql-http-{{ pgsql_http_release }} - target: install - become: yes - -# plpgsql_check -- name: plpgsql_check - download & install dependencies - apt: - pkg: - - libicu-dev - update_cache: yes - install_recommends: no - -- name: plpgsql_check - download latest release - get_url: - url: "https://github.com/okbob/plpgsql_check/archive/refs/tags/v{{ plpgsql_check_release }}.tar.gz" - dest: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz - checksum: "{{ plpgsql_check_release_checksum }}" - -- name: plpgsql_check - unpack archive - unarchive: - remote_src: yes - src: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz - dest: /tmp - become: yes - -- name: plpgsql_check - clean - make: - chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} - target: clean - become: yes - -- name: plpgsql_check - install - make: - chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} - target: install - become: yes - -# pg-safeupdate -- name: pg-safeupdate - download latest release - get_url: - url: "https://github.com/eradman/pg-safeupdate/archive/refs/tags/{{ pg_safeupdate_release }}.tar.gz" - dest: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz - checksum: "{{ pg_safeupdate_release_checksum }}" - -- name: pg-safeupdate - unpack archive - unarchive: - remote_src: yes - src: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz - dest: /tmp - become: yes - -- name: pg-safeupdate - build - make: - chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} - become: yes - -- name: pg-safeupdate - install - make: - chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} - target: install - become: yes - -# timescaledb -- name: timescaledb - download & install dependencies - apt: - pkg: - - cmake - update_cache: yes - install_recommends: yes - -- name: timescaledb - download latest release - git: - repo: https://github.com/timescale/timescaledb.git - dest: /tmp/timescaledb - version: "{{ timescaledb_release }}" - become: yes - -- name: timescaledb - bootstrap - shell: - cmd: "./bootstrap -DAPACHE_ONLY=1 -DREGRESS_CHECKS=OFF" - chdir: /tmp/timescaledb - become: yes - -- name: timescaledb - build - make: - chdir: /tmp/timescaledb/build - become: yes - -- name: timescaledb - install - make: - chdir: /tmp/timescaledb/build - target: install - become: yes - -# pljava -- name: pljava - download & install dependencies - apt: - pkg: - - maven - - default-jre - - default-jdk - update_cache: yes - install_recommends: yes - -- name: pljava - download latest release - get_url: - url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz - dest: /tmp/pljava-{{ pljava_release }}.tar.gz - checksum: "{{ pljava_release_checksum }}" - -- name: pljava - unpack archive - unarchive: - remote_src: yes - src: /tmp/pljava-{{ pljava_release }}.tar.gz - dest: /tmp - become: yes - -- name: pljava - build - become: yes - shell: - cmd: mvn clean install - chdir: /tmp/pljava-{{ pljava_release }} - -- name: pljava - install - become: yes - shell: - cmd: java -jar pljava-packaging/target/pljava-pg13.jar - chdir: /tmp/pljava-{{ pljava_release }} - -- name: pljava - remove build dependencies - apt: - pkg: - - maven - - default-jre - - default-jdk - state: absent - -- name: pljava - install headless jdk - apt: - pkg: - - default-jdk-headless - update_cache: yes - install_recommends: no - -- name: pljava - set pljava.libjvm_location - become: yes - lineinfile: - path: /etc/postgresql/postgresql.conf - state: present - line: pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so' - -- name: pljava - remove ~/.m2 directory - become: yes - file: - path: ~/.m2 - state: absent - -# plv8 -- name: plv8 - download & install dependencies - apt: - pkg: - - build-essential - - ca-certificates - - curl - - git-core - - gpp - - cpp - - pkg-config - - apt-transport-https - - cmake - - libc++-dev - - libc++abi-dev - - libc++1 - - libglib2.0-dev - - libtinfo5 - - libc++abi1 - - ninja-build - - python - update_cache: yes - install_recommends: no - -- name: plv8 - download latest release - git: - repo: https://github.com/plv8/plv8.git - dest: /tmp/plv8 - version: 3656177d384e3e02b74faa8e2931600f3690ab59 - become: yes - -- name: Create a symbolic link - file: - src: /lib/aarch64-linux-gnu/libc++.so.1 - dest: /lib/aarch64-linux-gnu/libc++.so - state: link - when: platform == "arm64" - -- name: plv8 - build - make: - chdir: /tmp/plv8 - become: yes - -- name: plv8 - install - make: - chdir: /tmp/plv8 - target: install - become: yes \ No newline at end of file +- name: Install plv8 + import_tasks: tasks/postgres-extensions/12-plv8.yml \ No newline at end of file From 63055ca95f82cd225a9bd427d83f0824d19b45f9 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 1 Jul 2021 23:34:29 +0800 Subject: [PATCH 47/86] fix: clean up duplicate variables --- ansible/vars.yml | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 02e3aedab..4e61d4783 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -7,16 +7,30 @@ postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa pgbouncer_release: "1.15.0" pgbouncer_release_checksum: sha1:ea7e9dbcab178f439a0fa402a78a7f1e4f43e6d4 -postgres_exporter_release: "0.9.0" -postgres_exporter_release_checksum: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 - postgrest_arm_release: 2021-03-05-19-03-d3a8b5f-ubuntu-aarch64 postgrest_arm_release_checksum: sha1:b9e9b06ead7230b75033e8ae17912714bf463a33 postgrest_x86_release: 2021-03-05-19-03-d3a8b5f-linux-x64-static postgrest_x86_release_checksum: sha1:4b4adde15f0d41d65a9136d1f8c0d9cd6fe79326 -aws_cli_release: "2.0.30" +aws_cli_release: "2.2.7" + +golang_version: "1.15.4" +wal_g_release: "0.2.15" +wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 + +sfcgal_release: "1.3.10" +sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b + +postgres_exporter_release: "0.9.0" +postgres_exporter_release_checksum: + arm64: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 + amd64: sha256:ff541bd3ee19c0ae003d71424a75edfcc8695e828dd20d5b4555ce433c89d60b + +node_exporter_release: 1.1.2 +node_exporter_release_checksum: + arm64: sha256:eb5e7d16f18bb3272d0d832986fc8ac6cb0b6c42d487c94e15dabb10feae8e04 + amd64: sha256:8c1f6a317457a658e0ae68ad710f6b4098db2cad10204649b51e3c043aa3e70d # Postgres Extensions postgis_release: "3.1.2" @@ -51,21 +65,3 @@ supautils_release_checksum: sha1:326ac5c1933bd30d4a50da7568b27629a9ec544b pljava_release: "1_6_2" pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 -golang_version: "1.15.4" -wal_g_release: "0.2.15" -wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 - -sfcgal_release: "1.3.10" -sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b - -postgres_exporter_release: "0.9.0" -postgres_exporter_release_checksum: - arm64: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 - amd64: sha256:ff541bd3ee19c0ae003d71424a75edfcc8695e828dd20d5b4555ce433c89d60b - -aws_cli_release: "2.2.7" - -node_exporter_release: 1.1.2 -node_exporter_release_checksum: - arm64: sha256:eb5e7d16f18bb3272d0d832986fc8ac6cb0b6c42d487c94e15dabb10feae8e04 - amd64: sha256:8c1f6a317457a658e0ae68ad710f6b4098db2cad10204649b51e3c043aa3e70d From cc13e643ca9b5911550d6ddd29ac004a00c8c14b Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 1 Jul 2021 23:35:27 +0800 Subject: [PATCH 48/86] chore: add pgbouncer fail2ban configuration - also, set internal specific fail2ban configurations --- .../fail2ban_config/filter-pgbouncer.conf.j2 | 2 ++ .../fail2ban_config/jail-pgbouncer.conf.j2 | 7 ++++ ansible/tasks/setup-fail2ban.yml | 36 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 create mode 100644 ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 diff --git a/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 b/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 new file mode 100644 index 000000000..50326da51 --- /dev/null +++ b/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 @@ -0,0 +1,2 @@ +[Definition] +failregex = ^.+@:.+error: password authentication failed$ \ No newline at end of file diff --git a/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 b/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 new file mode 100644 index 000000000..77c5530dc --- /dev/null +++ b/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 @@ -0,0 +1,7 @@ +[pgbouncer] +enabled = true +port = 6543 +protocol = tcp +filter = pgbouncer +logpath = /var/log/pgbouncer.log +maxretry = 3 \ No newline at end of file diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index bc00d76d7..7061ba5cc 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -1,3 +1,12 @@ +# set default bantime to 30 minutes +- name: supautils - add supautils to shared_preload_libraries + become: yes + replace: + path: /etc/fail2ban/jail.conf + regexp: bantime = 10m + replace: bantime = 1800 + +# postgresql - name: import jail.d/postgresql.conf template: src: files/fail2ban_config/jail-postgresql.conf.j2 @@ -10,6 +19,33 @@ dest: /etc/fail2ban/filter.d/postgresql.conf become: yes +- name: add in supabase specific ignore filters + lineinfile: + path: /etc/fail2ban/filter.d/postgresql.conf + state: present + line: "{{ item.line }}" + loop: + - { line: ' ^.+FATAL: password authentication failed for user "supabase_admin"$' } + - { line: ' ^.+FATAL: password authentication failed for user "supabase_auth_admin"$' } + - { line: ' ^.+FATAL: password authentication failed for user "supabase_storage_admin"$' } + - { line: ' ^.+FATAL: password authentication failed for user "authenticator"$' } + - { line: ' ^.+FATAL: password authentication failed for user "pgbouncer"$' } + become: yes + when: supabase_internal == true + +# pgbouncer +- name: import jail.d/pgbouncer.conf + template: + src: files/fail2ban_config/jail-pgbouncer.conf.j2 + dest: /etc/fail2ban/jail.d/pgbouncer.conf + become: yes + +- name: import filter.d/pgbouncer.conf + template: + src: files/fail2ban_config/filter-pgbouncer.conf.j2 + dest: /etc/fail2ban/filter.d/pgbouncer.conf + become: yes + # Restart - name: fail2ban - restart systemd: From 372a7d3f13535730e0d2f2b30f476c69725e746b Mon Sep 17 00:00:00 2001 From: Inian Date: Tue, 6 Jul 2021 16:26:55 +0800 Subject: [PATCH 49/86] disable login shell for postgres user ref https://github.com/supabase/infrastructure/issues/1329 --- ansible/tasks/setup-postgres.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 0ff7629fb..679c3ceb4 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -4,16 +4,16 @@ apt: pkg: - build-essential - - libreadline-dev - - zlib1g-dev - - flex - - bison - - libxml2-dev - - libxslt-dev - - libssl-dev + - libreadline-dev + - zlib1g-dev + - flex + - bison + - libxml2-dev + - libxslt-dev + - libssl-dev - libsystemd-dev - libpq-dev - - libxml2-utils + - libxml2-utils - uuid-dev - xsltproc - ssl-cert @@ -83,7 +83,7 @@ - name: Create postgres user user: name: postgres - shell: /bin/bash + shell: /bin/false comment: Postgres user groups: ssl-cert @@ -146,9 +146,9 @@ shell: cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb vars: - ansible_command_timeout: 60 - # Circumvents the following error: - # "Timeout (12s) waiting for privilege escalation prompt" + ansible_command_timeout: 60 + # Circumvents the following error: + # "Timeout (12s) waiting for privilege escalation prompt" # Add systemd file for Postgres - name: import postgresql.service @@ -159,7 +159,7 @@ # Reload - name: System - systemd reload - systemd: + systemd: enabled: yes name: postgresql daemon_reload: yes From c3baeaec8632518eb32cd8de001e3e012e597c8a Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Jul 2021 15:31:01 +0800 Subject: [PATCH 50/86] Clean up and consolidate code in supabase internal --- .../tasks/{ => internal}/node-exporter.yml | 0 .../{ => internal}/postgres-exporter.yml | 0 ansible/tasks/internal/supautils.yml | 45 ++++++++++++++++ ansible/tasks/setup-supabase-internal.yml | 51 ++----------------- 4 files changed, 49 insertions(+), 47 deletions(-) rename ansible/tasks/{ => internal}/node-exporter.yml (100%) rename ansible/tasks/{ => internal}/postgres-exporter.yml (100%) create mode 100644 ansible/tasks/internal/supautils.yml diff --git a/ansible/tasks/node-exporter.yml b/ansible/tasks/internal/node-exporter.yml similarity index 100% rename from ansible/tasks/node-exporter.yml rename to ansible/tasks/internal/node-exporter.yml diff --git a/ansible/tasks/postgres-exporter.yml b/ansible/tasks/internal/postgres-exporter.yml similarity index 100% rename from ansible/tasks/postgres-exporter.yml rename to ansible/tasks/internal/postgres-exporter.yml diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml new file mode 100644 index 000000000..d6c340971 --- /dev/null +++ b/ansible/tasks/internal/supautils.yml @@ -0,0 +1,45 @@ +# supautils +- name: supautils - download latest release + get_url: + url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" + dest: /tmp/supautils-{{ supautils_release }}.tar.gz + checksum: "{{ supautils_release_checksum }}" + +- name: supautils - unpack archive + unarchive: + remote_src: yes + src: /tmp/supautils-{{ supautils_release }}.tar.gz + dest: /tmp + become: yes + +- name: supautils - build + make: + chdir: /tmp/supautils-{{ supautils_release }} + become: yes + +- name: supautils - install + make: + chdir: /tmp/supautils-{{ supautils_release }} + target: install + become: yes + +- name: supautils - set supautils.reserved_roles + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon' + +- name: supautils - set supautils.reserved_memberships + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program' + +- name: supautils - add supautils to shared_preload_libraries + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: shared_preload_libraries = ' + replace: shared_preload_libraries = 'supautils, diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 2321df150..7efb5541e 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -27,53 +27,10 @@ become: true - name: Install Postgres exporter - import_tasks: postgres-exporter.yml + import_tasks: internal/postgres-exporter.yml - name: Install node exporter - import_tasks: node-exporter.yml + import_tasks: internal/node-exporter.yml -# supautils -- name: supautils - download latest release - get_url: - url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" - dest: /tmp/supautils-{{ supautils_release }}.tar.gz - checksum: "{{ supautils_release_checksum }}" - -- name: supautils - unpack archive - unarchive: - remote_src: yes - src: /tmp/supautils-{{ supautils_release }}.tar.gz - dest: /tmp - become: yes - -- name: supautils - build - make: - chdir: /tmp/supautils-{{ supautils_release }} - become: yes - -- name: supautils - install - make: - chdir: /tmp/supautils-{{ supautils_release }} - target: install - become: yes - -- name: supautils - set supautils.reserved_roles - become: yes - lineinfile: - path: /etc/postgresql/postgresql.conf - state: present - line: supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon' - -- name: supautils - set supautils.reserved_memberships - become: yes - lineinfile: - path: /etc/postgresql/postgresql.conf - state: present - line: supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program' - -- name: supautils - add supautils to shared_preload_libraries - become: yes - replace: - path: /etc/postgresql/postgresql.conf - regexp: shared_preload_libraries = ' - replace: shared_preload_libraries = 'supautils, +- name: Install supautils + import_tasks: internal/supautils.yml \ No newline at end of file From e15a20544a88bcbbc1c0c8eadd91bc9830d1c84e Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Jul 2021 15:41:16 +0800 Subject: [PATCH 51/86] disable timescaledb for the time being --- ansible/tasks/setup-extensions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 6d689cd0e..8149999a9 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -25,8 +25,8 @@ - name: Install pg-safeupdate import_tasks: tasks/postgres-extensions/09-pg-safeupdate.yml -- name: Install timescaledb - import_tasks: tasks/postgres-extensions/10-timescaledb.yml +# - name: Install timescaledb +# import_tasks: tasks/postgres-extensions/10-timescaledb.yml - name: Install pljava import_tasks: tasks/postgres-extensions/11-pljava.yml From e373ef8d6a33435739410cdeb31da2c875a81d27 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Jul 2021 22:48:32 +0800 Subject: [PATCH 52/86] remove undefined variable 'platform_aws' --- ansible/tasks/setup-supabase-internal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 7efb5541e..0e339477c 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -12,7 +12,7 @@ - name: AWS CLI (x86) get_url: - url: "https://awscli.amazonaws.com/awscli-exe-linux-{{ platform_aws }}-{{ aws_cli_release }}.zip" + url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" dest: "/tmp/awscliv2.zip" when: platform == "amd64" From de94ffbf54948a33e6483ac10fd31aa957299015 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 7 Jul 2021 22:48:54 +0800 Subject: [PATCH 53/86] remove timescaledb from postgresql.conf template --- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index fac7813b3..5a2e39e91 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -717,7 +717,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, timescaledb' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - From 5bad268bd874c32cbe10e2c9f87ab16c119f3520 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 17:05:41 +0800 Subject: [PATCH 54/86] bump packer files to supabase postgres 13.3.0 --- amazon-arm.json | 2 +- amazon.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index 7306f1516..2d312a4fd 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -5,7 +5,7 @@ "region": "ap-northeast-1", "ami_regions": "eu-central-1,eu-west-1,eu-west-2,ap-south-1,ap-southeast-1,ap-southeast-2,us-west-1,us-east-1,ca-central-1,sa-east-1,ap-northeast-1", "ami": "ami-076d8ebdd0e1ec091", - "ami_name": "supabase-postgres-0.15.0", + "ami_name": "supabase-postgres-13.3.0", "environment": "prod", "ansible_arguments": "--skip-tags,update-only -v" }, diff --git a/amazon.json b/amazon.json index c34c4277e..54cf2c4b2 100644 --- a/amazon.json +++ b/amazon.json @@ -4,8 +4,8 @@ "aws_secret_key": "", "region": "af-south-1", "ami_regions": "af-south-1", - "ami": "ami-07d30d5bf2d29a582", - "ami_name": "supabase-postgres-0.16.2", + "ami": "ami-08a4b40f2fe1e4b35", + "ami_name": "supabase-postgres-13.3.0", "environment": "prod", "ansible_arguments": "--skip-tags,update-only" }, From 7b3ca7688af3e7bea997cb5988343493f8f541d8 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 17:07:39 +0800 Subject: [PATCH 55/86] fully purge away anxs.postgresql role --- ansible/install_roles.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 ansible/install_roles.yml diff --git a/ansible/install_roles.yml b/ansible/install_roles.yml deleted file mode 100644 index 9cb7c96a1..000000000 --- a/ansible/install_roles.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: anxs.postgresql - src: https://github.com/anxs/postgresql - version: v1.12.0 From 7e081923c89c0a3b8be2416a3f115d8a6959c245 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 17:42:35 +0800 Subject: [PATCH 56/86] Clean up README - add new extensions and tidy up --- README.md | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d7634889e..526732cd0 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,27 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to ## Features -- ✅ Postgres [12](https://www.postgresql.org/about/news/1976/). Includes [generated columns](https://www.postgresql.org/docs/12/ddl-generated-columns.html) and [JSON path](https://www.postgresql.org/docs/12/functions-json.html#FUNCTIONS-SQLJSON-PATH) support. -- ✅ Ubuntu 18.04 (Bionic). -- ✅ [pg-contrib-12](https://www.postgresql.org/docs/12/contrib.html). Because everyone should enable `pg_stat_statements`. +- ✅ Postgres [13](https://www.postgresql.org/about/news/postgresql-13-released-2077/). +- ✅ Ubuntu 20.04 (Focal Fossa). - ✅ [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html) = logical and [max_replication_slots](https://www.postgresql.org/docs/current/runtime-config-replication.html) = 5. Ready for replication. -- ✅ [PostGIS](https://postgis.net/). Postgres' most popular extension - support for geographic objects. -- ✅ [pgTAP](https://pgtap.org/). Unit Testing for Postgres. -- ✅ [pgAudit](https://www.pgaudit.org/). Generate highly compliant audit logs. -- ✅ [pgjwt](https://github.com/michelp/pgjwt). Generate JSON Web Tokens (JWT) in Postgres. -- ✅ [pgsql-http](https://github.com/pramsey/pgsql-http). HTTP client for Postgres. -- ✅ [plpgsql_check](https://github.com/okbob/plpgsql_check). Linter tool for PL/pgSQL. -- ✅ [plv8](https://github.com/plv8/plv8). Write in Javascript functions in Postgres. -- ✅ [PL/Java](https://github.com/tada/pljava). Write in Java functions in Postgres. -- ✅ [pg_cron](https://github.com/citusdata/pg_cron). Run CRON jobs inside Postgres. -- ✅ [pg-safeupdate](https://github.com/eradman/pg-safeupdate). Protect your data from accidental updates or deletes. + +## Extensions +| | | +| ------------- | ------------- | +| [Postgres contrib modules](https://www.postgresql.org/docs/current/contrib.html) | Because everyone should enable `pg_stat_statements`. | +| [PostGIS](https://postgis.net/) | Postgres' most popular extension - support for geographic objects. | +| [pgRouting](https://pgrouting.org/) | Extension of PostGIS - provides geospatial routing functionalities | +| [pgTAP](https://pgtap.org/) | Unit Testing for Postgres. | +| [pg_cron](https://github.com/citusdata/pg_cron) | Run CRON jobs inside Postgres. | +| [pgAudit](https://www.pgaudit.org/) | Generate highly compliant audit logs. | +| [pgjwt](https://github.com/michelp/pgjwt) | Generate JSON Web Tokens (JWT) in Postgres. | +| [pgsql-http](https://github.com/pramsey/pgsql-http) | HTTP client for Postgres. | +| [plpgsql_check](https://github.com/okbob/plpgsql_check) | Linter tool for PL/pgSQL. | +| [pg-safeupdate](https://github.com/eradman/pg-safeupdate) | Protect your data from accidental updates or deletes. | +| [PL/Java](https://github.com/tada/pljava) | Write in Java functions in Postgres. | +| [plv8](https://github.com/plv8/plv8) | Write in Javascript functions in Postgres. | + +Can't find your favorite extension? Suggest for it to be added into future versions [here](https://github.com/supabase/supabase/discussions/679)! ## Install From 2670d7f971f7d8bf3f3cb1ce193330d97b91148f Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 19:56:07 +0800 Subject: [PATCH 57/86] Make sure all PgBouncer related tasks are done under the tag 'install_pgbouncer` --- ansible/playbook.yml | 2 ++ ansible/tasks/setup-fail2ban.yml | 13 ------------- ansible/tasks/setup-pgbouncer.yml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index e8afab4c5..866208705 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -82,6 +82,8 @@ ufw: rule: allow port: "6543" + tags: + - install-pgbouncer - name: UFW - Deny all other incoming traffic by default ufw: diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index 7061ba5cc..1195d2213 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -33,19 +33,6 @@ become: yes when: supabase_internal == true -# pgbouncer -- name: import jail.d/pgbouncer.conf - template: - src: files/fail2ban_config/jail-pgbouncer.conf.j2 - dest: /etc/fail2ban/jail.d/pgbouncer.conf - become: yes - -- name: import filter.d/pgbouncer.conf - template: - src: files/fail2ban_config/filter-pgbouncer.conf.j2 - dest: /etc/fail2ban/filter.d/pgbouncer.conf - become: yes - # Restart - name: fail2ban - restart systemd: diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index f23337f57..8a794d968 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -63,6 +63,19 @@ dest: /etc/tmpfiles.d/pgbouncer.conf become: yes +# Add fail2ban filter +- name: import jail.d/pgbouncer.conf + template: + src: files/fail2ban_config/jail-pgbouncer.conf.j2 + dest: /etc/fail2ban/jail.d/pgbouncer.conf + become: yes + +- name: import filter.d/pgbouncer.conf + template: + src: files/fail2ban_config/filter-pgbouncer.conf.j2 + dest: /etc/fail2ban/filter.d/pgbouncer.conf + become: yes + # Add systemd file for PgBouncer - name: PgBouncer - import postgresql.service template: From bef9767b615d9e0e2cb9a4c41c9626d1648f2342 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 19:56:17 +0800 Subject: [PATCH 58/86] remove empty requirements.yml --- ansible/requirements.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ansible/requirements.yml diff --git a/ansible/requirements.yml b/ansible/requirements.yml deleted file mode 100644 index e69de29bb..000000000 From 42d39e6daad7729dbe7716276a403bd1795555c4 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 19:57:09 +0800 Subject: [PATCH 59/86] update ansible_arguments to omit building of postgrest --- amazon-arm.json | 2 +- amazon.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index 2d312a4fd..d42d83ec9 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -7,7 +7,7 @@ "ami": "ami-076d8ebdd0e1ec091", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only -v" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,-v" }, "builders": [ { diff --git a/amazon.json b/amazon.json index 54cf2c4b2..cc723d1e9 100644 --- a/amazon.json +++ b/amazon.json @@ -7,7 +7,7 @@ "ami": "ami-08a4b40f2fe1e4b35", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,-v" }, "builders": [ { From 2bf45c1f2caf1fd20d0b11ea2f45c753a5da76fb Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 20:28:56 +0800 Subject: [PATCH 60/86] Update README - based on new features/ fixes from 13.3.0 --- README.md | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 526732cd0..535b65847 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,18 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to modify Postgres, but to provide some of the most common extensions with a one-click install. -## Features - +## Primary Features - ✅ Postgres [13](https://www.postgresql.org/about/news/postgresql-13-released-2077/). - ✅ Ubuntu 20.04 (Focal Fossa). - ✅ [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html) = logical and [max_replication_slots](https://www.postgresql.org/docs/current/runtime-config-replication.html) = 5. Ready for replication. +- ✅ [Large Systems Extensions](https://github.com/aws/aws-graviton-getting-started#building-for-graviton-and-graviton2). Enabled for ARM images. ## Extensions -| | | +| Extension | Description | | ------------- | ------------- | | [Postgres contrib modules](https://www.postgresql.org/docs/current/contrib.html) | Because everyone should enable `pg_stat_statements`. | | [PostGIS](https://postgis.net/) | Postgres' most popular extension - support for geographic objects. | -| [pgRouting](https://pgrouting.org/) | Extension of PostGIS - provides geospatial routing functionalities | +| [pgRouting](https://pgrouting.org/) | Extension of PostGIS - provides geospatial routing functionalities. | | [pgTAP](https://pgtap.org/) | Unit Testing for Postgres. | | [pg_cron](https://github.com/citusdata/pg_cron) | Run CRON jobs inside Postgres. | | [pgAudit](https://www.pgaudit.org/) | Generate highly compliant audit logs. | @@ -26,6 +26,20 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to Can't find your favorite extension? Suggest for it to be added into future versions [here](https://github.com/supabase/supabase/discussions/679)! +## Enhanced Security +Aside from having [ufw](https://help.ubuntu.com/community/UFW),[fail2ban](https://www.fail2ban.org/wiki/index.php/Main_Page), and [unattended-upgrades](https://wiki.debian.org/UnattendedUpgrades) installed, we also have the following enhancements in place: +| Enhancement | Description | +| ------------- | ------------- | +| fail2ban filter for PostgreSQL access | Monitors for brute force attempts over at port `5432`. | +| fail2ban filter for PgBouncer access | Monitors for brute force attempts over at port `6543`. | + +## Additional Goodies +| Goodie | Description | +| ------------- | ------------- | +| [PgBouncer](https://postgis.net/) | Set up Connection Pooling. | +| [PostgREST](https://postgrest.org/en/stable/) | Instantly transform your database into an RESTful API. | +| [WAL-G](https://github.com/wal-g/wal-g#wal-g) | Tool for physical database backup and recovery. | + ## Install See all installation instructions in the [repo wiki](https://github.com/supabase/postgres/wiki). @@ -34,13 +48,23 @@ See all installation instructions in the [repo wiki](https://github.com/supabase [![Digital Ocean](https://github.com/supabase/postgres/blob/master/docs/img/digital-ocean.png)](https://github.com/supabase/postgres/wiki/Digital-Ocean) [![AWS](https://github.com/supabase/postgres/blob/master/docs/img/aws.png)](https://github.com/supabase/postgres/wiki/AWS-EC2) -### Building - -Install the anxs.postgresql role - -```bash -ansible-galaxy install anxs.postgresql -r ansible/install_roles.yml --force -vvv -``` +### Marketplace Images +| | Postgres & Extensions | PgBouncer | PostgREST | WAL-G | +|---|:---:|:---:|:---:|:---:| +| Supabase Postgres | ✔️ | ❌ | ❌ | ✔️ | +| Supabase Postgres: PgBouncer Bundle | ✔️ | ✔️ | ❌ | ✔️ | +| Supabase Postgres: PostgREST Bundle | ✔️ | ❌ | ✔️ | ✔️ | +| Supabase Postgres: Complete Bundle | ✔️ | ✔️ | ✔️ | ✔️ | + +#### Availability +| | AWS ARM | AWS x86 | Digital Ocean x86 | +|---|:---:|:---:|:---:| +| Supabase Postgres | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: PgBouncer Bundle | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: PostgREST Bundle | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: Complete Bundle | Coming Soon | Coming Soon | Coming Soon | + +### Quick Build Set the `supabase_internal` flag to `false` to avoid baking in components that are specific to Supabase's hosted offering. From 3e72599ce410f61ce8834f62c004fc5d78e8d19e Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 20:29:17 +0800 Subject: [PATCH 61/86] remove verbosity when building --- amazon-arm.json | 2 +- amazon.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index d42d83ec9..4ed2ff8b3 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -7,7 +7,7 @@ "ami": "ami-076d8ebdd0e1ec091", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,-v" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest" }, "builders": [ { diff --git a/amazon.json b/amazon.json index cc723d1e9..83716e092 100644 --- a/amazon.json +++ b/amazon.json @@ -7,7 +7,7 @@ "ami": "ami-08a4b40f2fe1e4b35", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,-v" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest" }, "builders": [ { From baa490f692204bb1a21c598ff3493213fffa36b6 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 8 Jul 2021 21:32:40 +0800 Subject: [PATCH 62/86] update DO config for 13.3.0 --- digitalOcean.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/digitalOcean.json b/digitalOcean.json index 62b02955f..36396fc29 100644 --- a/digitalOcean.json +++ b/digitalOcean.json @@ -1,11 +1,11 @@ { "variables": { "do_token": "", - "image_name": "ubuntu-18-04-x64", + "image_name": "ubuntu-20-04-x64", "region": "sgp1", "snapshot_regions": "sgp1", - "snapshot_name": "supabase-postgres-0.16.2", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,aws-only" + "snapshot_name": "supabase-postgres-13.3.0", + "ansible_arguments": "--skip-tags,update-only,--skip-tags,aws-only,-e,supabase_internal='false'" }, "builders": [ { From fcf8cf667face9230afe821042f3dc826b47945c Mon Sep 17 00:00:00 2001 From: Angelico Date: Sat, 10 Jul 2021 22:28:09 +0800 Subject: [PATCH 63/86] remove dependencies for pgrouting - not needed; were used for documentation --- ansible/tasks/postgres-extensions/02-pgrouting.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml index b03e33e9c..9020d0024 100644 --- a/ansible/tasks/postgres-extensions/02-pgrouting.yml +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -1,12 +1,4 @@ # pgRouting -- name: pgRouting - download & install dependencies - apt: - pkg: - - perl - - python3-sphinx - update_cache: yes - cache_valid_time: 3600 - - name: pgRouting - download latest release get_url: url: "https://github.com/pgRouting/pgrouting/releases/download/v{{ pgrouting_release }}/pgrouting-{{ pgrouting_release }}.tar.gz" @@ -28,7 +20,7 @@ - name: pgRouting - compile shell: - cmd: "cmake .." + cmd: "cmake -DBUILD_HTML=OFF -DBUILD_DOXY=OFF .." chdir: /tmp/pgrouting-{{ pgrouting_release }}/build become: yes From c05574a4cf84ff64f7d5b5ec171c3cba2fb1d5fb Mon Sep 17 00:00:00 2001 From: Angelico Date: Sat, 10 Jul 2021 22:38:02 +0800 Subject: [PATCH 64/86] set all install_recommends to no --- ansible/tasks/postgres-extensions/01-postgis.yml | 1 + ansible/tasks/postgres-extensions/07-pgsql-http.yml | 2 +- ansible/tasks/postgres-extensions/10-timescaledb.yml | 2 +- ansible/tasks/postgres-extensions/11-pljava.yml | 2 +- ansible/tasks/setup-extensions.yml | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml index a60366d6c..9aaad6d5e 100644 --- a/ansible/tasks/postgres-extensions/01-postgis.yml +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -14,6 +14,7 @@ - cmake update_cache: yes cache_valid_time: 3600 + install_recommends: no - name: postgis - download SFCGAL dependency get_url: diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml index f72140da4..6fd5cf9aa 100644 --- a/ansible/tasks/postgres-extensions/07-pgsql-http.yml +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -10,7 +10,7 @@ pkg: - libcurl4-gnutls-dev update_cache: yes - install_recommends: yes + install_recommends: no - name: pgsql-http - download latest release get_url: diff --git a/ansible/tasks/postgres-extensions/10-timescaledb.yml b/ansible/tasks/postgres-extensions/10-timescaledb.yml index 154ecc426..46798998b 100644 --- a/ansible/tasks/postgres-extensions/10-timescaledb.yml +++ b/ansible/tasks/postgres-extensions/10-timescaledb.yml @@ -4,7 +4,7 @@ pkg: - cmake update_cache: yes - install_recommends: yes + install_recommends: no - name: timescaledb - download latest release git: diff --git a/ansible/tasks/postgres-extensions/11-pljava.yml b/ansible/tasks/postgres-extensions/11-pljava.yml index 0495484df..3bea59cb9 100644 --- a/ansible/tasks/postgres-extensions/11-pljava.yml +++ b/ansible/tasks/postgres-extensions/11-pljava.yml @@ -6,7 +6,7 @@ - default-jre - default-jdk update_cache: yes - install_recommends: yes + install_recommends: no - name: pljava - download latest release get_url: diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index b0c516d57..d59c04435 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -28,7 +28,7 @@ - name: Install wal2json apt: pkg: - - "postgresql-{{ postgresql_version }}-wal2json" + - "postgresql-{{ postgresql_major }}-wal2json" update_cache: yes cache_valid_time: 3600 install_recommends: no From 0c5178588791d9472c1d9f8bc616eba110a14da9 Mon Sep 17 00:00:00 2001 From: Angelico Date: Sat, 10 Jul 2021 22:45:55 +0800 Subject: [PATCH 65/86] chore: #56 build docker image with ansible - and existing task files --- ansible/files/docker_mnt/init.sh | 1 + ansible/playbook-docker.yml | 52 +++++++++++++++++++++++++ ansible/tasks/docker/cleanup.yml | 67 ++++++++++++++++++++++++++++++++ ansible/tasks/docker/setup.yml | 48 +++++++++++++++++++++++ ansible/vars.yml | 1 + 5 files changed, 169 insertions(+) create mode 100644 ansible/files/docker_mnt/init.sh create mode 100644 ansible/playbook-docker.yml create mode 100644 ansible/tasks/docker/cleanup.yml create mode 100644 ansible/tasks/docker/setup.yml diff --git a/ansible/files/docker_mnt/init.sh b/ansible/files/docker_mnt/init.sh new file mode 100644 index 000000000..10fb77668 --- /dev/null +++ b/ansible/files/docker_mnt/init.sh @@ -0,0 +1 @@ +cat /etc/postgresql/postgresql.conf > $PGDATA/postgresql.conf \ No newline at end of file diff --git a/ansible/playbook-docker.yml b/ansible/playbook-docker.yml new file mode 100644 index 000000000..dc0de54e7 --- /dev/null +++ b/ansible/playbook-docker.yml @@ -0,0 +1,52 @@ +- name: Preparing Docker container + hosts: localhost + tasks: + - name: Pull Postgres Image + docker_container: + name: "supabase-postgres-build" + image: "postgres:13.3" + env: + LANGUAGE: "en_US.UTF-8" + LANG: "en_US.UTF-8" + LC_ALL: "en_US.UTF-8" + state: started + memory: 4G + memory_swap: 6G + command: tail -f /dev/null + - name: Add Postgres Image to Ansible Hosts + add_host: + name: "supabase-postgres-build" + ansible_connection: docker + ansible_ssh_user: root + +- name: Build Supabase Postgres + hosts: "supabase-postgres-build" + gather_facts: false + + vars_files: + - ./vars.yml + + tasks: + - name: Setup container + import_tasks: tasks/docker/setup.yml + + - name: Install Postgres extensions + import_tasks: tasks/setup-extensions.yml + + - name: Cleanup container + import_tasks: tasks/docker/cleanup.yml + +- name: Create supabase/postgres docker image + hosts: localhost + tasks: + - name: Commit Docker image + command: docker commit "supabase-postgres-build" "supabase/postgres" + +- name: Clean Up Postgres Image + hosts: localhost + tasks: + - name: Remove Running Base Image + docker_container: + name: supabase-postgres-build + state: absent + force_kill: yes \ No newline at end of file diff --git a/ansible/tasks/docker/cleanup.yml b/ansible/tasks/docker/cleanup.yml new file mode 100644 index 000000000..2ccc2af85 --- /dev/null +++ b/ansible/tasks/docker/cleanup.yml @@ -0,0 +1,67 @@ + + +- name: Cleanup - remove build dependencies + apt: + pkg: + - python3 + - rsync + - ca-certificates + - build-essential + - postgresql-server-dev-13 + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - ninja-build + - python + state: absent + +- name: Cleanup - apt update and apt upgrade + apt: update_cache=yes upgrade=yes + # SEE http://archive.vn/DKJjs#parameter-upgrade + +- name: Cleanup - remove dependencies that are no longer required + apt: + autoremove: yes + +- name: Cleanup - remove useless packages from the cache + apt: + autoclean: yes + +- name: Cleanup - reinstall headless jdk + apt: + pkg: + - default-jdk-headless + update_cache: yes + install_recommends: no + +- name: Cleanup - find all files in /tmp + find: + paths: /tmp + file_type: any + register: tmp_items_to_delete + +- name: Cleanup - delete all items in /tmp + file: + path: "/tmp/{{ item.path | basename }}" + state: absent + force: yes + with_items: "{{ tmp_items_to_delete.files }}" + +- name: Cleanup - find all files in /var/lib/apt/lists/* + find: + paths: /var/lib/apt/lists + file_type: any + register: var_items_to_delete + +- name: Cleanup - delete all items in /tmp + file: + path: "/var/lib/apt/lists/{{ item.path | basename }}" + state: absent + force: yes + with_items: "{{ var_items_to_delete.files }}" + + \ No newline at end of file diff --git a/ansible/tasks/docker/setup.yml b/ansible/tasks/docker/setup.yml new file mode 100644 index 000000000..cd49da0ff --- /dev/null +++ b/ansible/tasks/docker/setup.yml @@ -0,0 +1,48 @@ +- name: Install Python3 + raw: apt update && apt upgrade -y && apt install python3 -y + +- name: Setup - install common dependencies + apt: + pkg: + - rsync + - ca-certificates + - build-essential + - postgresql-server-dev-13 + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - ninja-build + - python + update_cache: yes + install_recommends: no + +# Find platform architecture and set as a variable +- name: Setup - finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output +- set_fact: + platform: "{{ platform_output.stdout }}" + +- name: Setup - import postgresql.conf + synchronize: + src: files/postgresql_config/postgresql.conf.j2 + dest: etc/postgresql/postgresql.conf + +- name: Setup - import pg_hba.conf + template: + src: files/postgresql_config/pg_hba.conf.j2 + dest: etc/postgresql/pg_hba.conf + +- name: Setup - import pg_ident.conf + template: + src: files/postgresql_config/pg_ident.conf.j2 + dest: etc/postgresql/pg_ident.conf + +- name: Setup - add init script to /docker-entrypoint-initdb.d + synchronize: + src: files/docker_mnt/init.sh + dest: /docker-entrypoint-initdb.d/init.sh \ No newline at end of file diff --git a/ansible/vars.yml b/ansible/vars.yml index 4e61d4783..ef205d9f5 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,5 +1,6 @@ supabase_internal: true +postgresql_major: "13" postgresql_release: "13.3" postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa From 4ee624185788a823ed89da9700a8bb565d6803bb Mon Sep 17 00:00:00 2001 From: Angelico Date: Sat, 10 Jul 2021 22:47:04 +0800 Subject: [PATCH 66/86] chore: #56 retire previous way of building docker images - via dockerfile --- docker/Dockerfile | 205 --------------------------------- docker/mnt/init-permissions.sh | 16 --- 2 files changed, 221 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/mnt/init-permissions.sh diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index e0e3e3aef..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,205 +0,0 @@ -FROM postgres:12 - -# install pgcron -RUN apt-get update \ - && apt-get install postgresql-12-cron -y - -# install wal2json -RUN apt-get update \ - && apt-get install postgresql-$PG_MAJOR-wal2json -y - -# install postgis -ENV POSTGIS_MAJOR 3 -ENV POSTGIS_VERSION 3.0.0+dfsg-2~exp1.pgdg100+1 -RUN apt-get update \ - && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - && apt-get install -y --no-install-recommends \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - && rm -rf /var/lib/apt/lists/* /var/tmp/* - -# install pgtap -ENV PGTAP_VERSION=v1.1.0 - -RUN pgtapDependencies="git \ - ca-certificates \ - build-essential" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgtapDependencies} \ - && cd /tmp \ - && git clone git://github.com/theory/pgtap.git \ - && cd pgtap \ - && git checkout tags/$PGTAP_VERSION \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgtapDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgtap /var/lib/apt/lists/* /var/tmp/* - -# install pgAudit -ENV PGAUDIT_VERSION=1.4.0 - -RUN pgAuditDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR \ - libssl-dev \ - libkrb5-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgAuditDependencies} \ - && cd /tmp \ - && git clone https://github.com/pgaudit/pgaudit.git \ - && cd pgaudit \ - && git checkout ${PGAUDIT_VERSION} \ - && make check USE_PGXS=1 \ - && make install USE_PGXS=1 \ - && apt-get clean \ - && apt-get remove -y ${pgAuditDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgaudit /var/lib/apt/lists/* /var/tmp/* - -# install pgjwt -RUN pgjwtDependencies="git \ - ca-certificates \ - build-essential" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgjwtDependencies} \ - && cd /tmp \ - && git clone https://github.com/michelp/pgjwt.git \ - && cd pgjwt \ - && git checkout master \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgtapDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgjwt /var/lib/apt/lists/* /var/tmp/* - -# install pgsql-http -ENV PGSQL_HTTP_VERSION=v1.3.1 - -RUN pgsqlHttpDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR" \ - && pgsqlHttpRuntimeDependencies="libcurl4-gnutls-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgsqlHttpDependencies} ${pgsqlHttpRuntimeDependencies} \ - && cd /tmp \ - && git clone https://github.com/pramsey/pgsql-http.git \ - && cd pgsql-http \ - && git checkout ${PGSQL_HTTP_VERSION} \ - && make \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgsqlHttpDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgsql-http /var/lib/apt/lists/* /var/tmp/* - -# install plpgsql_check -ENV PLPGSQL_CHECK_VERSION=v1.11.3 - -RUN plpgsqlCheckDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR" \ - && plpgsqlCheckRuntimeDependencies="libicu-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${plpgsqlCheckDependencies} ${plpgsqlCheckRuntimeDependencies} \ - && cd /tmp \ - && git clone https://github.com/okbob/plpgsql_check.git \ - && cd plpgsql_check \ - && git checkout ${PLPGSQL_CHECK_VERSION} \ - && make clean \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgsqlHttpDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/plpgsql_check /var/lib/apt/lists/* /var/tmp/* - -# install pg-safeupdate -ENV PGSAFEUPDATE_VERSION=1.3 - -RUN pgSafeUpdateDependencies="pgxnclient \ -postgresql-server-dev-12" \ -&& apt-get update \ -&& apt-get install -y --no-install-recommends ${pgSafeUpdateDependencies} \ -&& cd /tmp \ -&& git clone https://github.com/eradman/pg-safeupdate.git \ -&& cd pg-safeupdate \ -&& git checkout ${PGSAFEUPDATE_VERSION} \ -&& make \ -&& make install \ -&& apt-get clean \ -&& apt-get remove -y ${pgSafeUpdateDependencies} \ -&& apt-get autoremove -y \ -&& rm -rf /tmp/pg-safeupdate /var/lib/apt/lists/* /var/tmp/* - -# install plv8 -ENV PLV8_VERSION=r3.0alpha - -RUN plv8Dependencies="build-essential \ - ca-certificates \ - curl \ - git-core \ - python \ - gpp \ - cpp \ - pkg-config \ - apt-transport-https \ - cmake \ - libc++-dev \ - libc++abi-dev \ - postgresql-server-dev-$PG_MAJOR" \ - && plv8RuntimeDependencies="libc++1 \ - libtinfo5 \ - libc++abi1" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${plv8Dependencies} ${plv8RuntimeDependencies} \ - && mkdir -p /tmp/build \ - && cd /tmp/build \ - && git clone https://github.com/plv8/plv8.git \ - && cd plv8 \ - && git checkout ${PLV8_VERSION} \ - && make static \ - && make install \ - && rm -rf /root/.vpython_cipd_cache /root/.vpython-root \ - && apt-get clean \ - && apt-get remove -y ${plv8Dependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/build /var/lib/apt/lists/* /var/tmp/* - -# install pljava -ENV PLJAVA_VERSION=V1_6_0 - -RUN pljavaDependencies="git \ - ca-certificates \ - g++ \ - maven \ - postgresql-server-dev-$PG_MAJOR \ - libpq-dev \ - libecpg-dev \ - libkrb5-dev \ - default-jdk \ - libssl-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pljavaDependencies} \ - && cd /tmp \ - && git clone https://github.com/tada/pljava.git \ - && cd pljava \ - && git checkout ${PLJAVA_VERSION} \ - && mvn clean install \ - && java -jar pljava-packaging/target/pljava-pg12.jar \ - && apt-get clean \ - && apt-get remove -y ${pljavaDependencies} \ - && apt-get autoremove -y \ - && rm -rf ~/.m2 /tmp/pljava /var/lib/apt/lists/* /var/tmp/* - -RUN apt-get update \ - && apt-get install -y --no-install-recommends default-jdk-headless \ - && rm -rf /var/lib/apt/lists/* /var/tmp/* - -ENV LANGUAGE=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -RUN mkdir -p /docker-entrypoint-initdb.d -ADD ./mnt /docker-entrypoint-initdb.d/ diff --git a/docker/mnt/init-permissions.sh b/docker/mnt/init-permissions.sh deleted file mode 100644 index 5755090a8..000000000 --- a/docker/mnt/init-permissions.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -echo "host replication $POSTGRES_USER 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf -echo "shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron'" >> $PGDATA/postgresql.conf -echo "pg_stat_statements.max = 10000" >> $PGDATA/postgresql.conf -echo "pg_stat_statements.track = all" >> $PGDATA/postgresql.conf -echo "wal_level=logical" >> $PGDATA/postgresql.conf -echo "max_replication_slots=5" >> $PGDATA/postgresql.conf -echo "max_wal_senders=10" >> $PGDATA/postgresql.conf -echo "log_destination='csvlog'" >> $PGDATA/postgresql.conf -echo "logging_collector=on" >> $PGDATA/postgresql.conf -echo "log_filename='postgresql.log'" >> $PGDATA/postgresql.conf -echo "log_rotation_age=0" >> $PGDATA/postgresql.conf -echo "log_rotation_size=0" >> $PGDATA/postgresql.conf -echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'" >> $PGDATA/postgresql.conf From 6fdf095e2589c2e9d3552d4be72103f0481f8f37 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 12 Jul 2021 16:08:33 +0800 Subject: [PATCH 67/86] chore: #58 settings to reduce memory overcommit --- ansible/tasks/setup-system.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 1258fb88f..f09fec813 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -49,6 +49,20 @@ - set_fact: platform: "{{ platform_output.stdout }}" +# Disables overcommiting of memory +- name: Adjust setting for vm.overcommit_memory + sysctl: + name: vm.overcommit_memory + value: 2 + state: present + +# Set overcommit ratio to 75 +- name: Adjust setting for vm.overcommit_ratio + sysctl: + name: vm.overcommit_ratio + value: 75 + state: present + - name: System - Create services.slice template: src: files/services.slice.j2 From 13710e2cc7745ccc6a4421cfe1f8773e514e03cf Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 15 Jul 2021 12:20:20 +0800 Subject: [PATCH 68/86] feat: add queries for additional metrics (#59) * feat: add queries for additional metrics * chore: filter for realtime replication lag * chore: add realtime replication slot active status Co-authored-by: Wen Bo Xie --- ansible/files/queries.yml.j2 | 151 ++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/ansible/files/queries.yml.j2 b/ansible/files/queries.yml.j2 index ef80abd77..73514d3fa 100644 --- a/ansible/files/queries.yml.j2 +++ b/ansible/files/queries.yml.j2 @@ -7,13 +7,151 @@ pg_database: usage: "GAUGE" description: "Disk space used by the database" +pg_stat_bgwriter: + query: | + select checkpoints_timed as checkpoints_timed_total, + checkpoints_req as checkpoints_req_total, + checkpoint_write_time as checkpoint_write_time_total, + checkpoint_sync_time as checkpoint_sync_time_total, + buffers_checkpoint as buffers_checkpoint_total, + buffers_clean as buffers_clean_total, + maxwritten_clean as maxwritten_clean_total, + buffers_backend as buffers_backend_total, + buffers_backend_fsync as buffers_backend_fsync_total, + buffers_alloc as buffers_alloc_total, + stats_reset + from pg_stat_bgwriter + cache_seconds: 30 + master: true + metrics: + - checkpoints_timed_total: + usage: "COUNTER" + description: "Scheduled checkpoints performed" + - checkpoints_req_total: + usage: "COUNTER" + description: "Requested checkpoints performed" + - checkpoint_write_time_total: + usage: "COUNTER" + description: "Time spent writing checkpoint files to disk" + - checkpoint_sync_time_total: + usage: "COUNTER" + description: "Time spent synchronizing checkpoint files to disk" + - buffers_checkpoint_total: + usage: "COUNTER" + description: "Buffers written during checkpoints" + - buffers_clean_total: + usage: "COUNTER" + description: "Buffers written by bg writter" + - maxwritten_clean_total: + usage: "COUNTER" + description: "Number of times bg writer stopped a cleaning scan because it had written too many buffers" + - buffers_backend_total: + usage: "COUNTER" + description: "Buffers written directly by a backend" + - buffers_backend_fsync_total: + usage: "COUNTER" + description: "fsync calls executed by a backend directly" + - buffers_alloc_total: + usage: "COUNTER" + description: "Buffers allocated" + - stats_reset: + usage: "COUNTER" + description: "Most recent stat reset time" + + pg_stat_database: - query: "SELECT sum(numbackends) as num_backends FROM pg_stat_database" + cache_seconds: 30 + query: | + SELECT sum(numbackends) as num_backends, + sum(xact_commit) as xact_commit_total, + sum(xact_rollback) as xact_rollback_total, + sum(blks_read) as blks_read_total, + sum(blks_hit) as blks_hit_total, + sum(tup_returned) as tup_returned_total, + sum(tup_fetched) as tup_fetched_total, + sum(tup_inserted) as tup_inserted_total, + sum(tup_updated) as tup_updated_total, + sum(tup_deleted) as tup_deleted_total, + sum(conflicts) as conflicts_total, + sum(temp_files) as temp_files_total, + sum(temp_bytes) as temp_bytes_total, + sum(deadlocks) as deadlocks_total, + max(stats_reset) as most_recent_reset + FROM pg_stat_database master: true metrics: - num_backends: usage: "GAUGE" description: "The number of active backends" + - xact_commit_total: + usage: "COUNTER" + description: "Transactions committed" + - xact_rollback_total: + usage: "COUNTER" + description: "Transactions rolled back" + - blks_read_total: + usage: "COUNTER" + description: "Number of disk blocks read" + - blks_hit_total: + usage: "COUNTER" + description: "Disk blocks found in buffer cache" + - tup_returned_total: + usage: "COUNTER" + description: "Rows returned by queries" + - tup_fetched_total: + usage: "COUNTER" + description: "Rows fetched by queries" + - tup_inserted_total: + usage: "COUNTER" + description: "Rows inserted" + - tup_updated_total: + usage: "COUNTER" + description: "Rows updated" + - tup_deleted_total: + usage: "COUNTER" + description: "Rows deleted" + - conflicts_total: + usage: "COUNTER" + description: "Queries canceled due to conflicts with recovery" + - temp_files_total: + usage: "COUNTER" + description: "Temp files created by queries" + - temp_bytes_total: + usage: "COUNTER" + description: "Temp data written by queries" + - deadlocks_total: + usage: "COUNTER" + description: "Deadlocks detected" + - most_recent_reset: + usage: "COUNTER" + description: "The most recent time one of the databases had its statistics reset" + +pg_stat_database_conflicts: + query: | + SELECT sum(confl_tablespace) as confl_tablespace_total, + sum(confl_lock) as confl_lock_total, + sum(confl_snapshot) as confl_snapshot_total, + sum(confl_bufferpin) as confl_bufferpin_total, + sum(confl_deadlock) as confl_deadlock_total + from pg_stat_database_conflicts + cache_seconds: 30 + master: true + metrics: + - confl_tablespace_total: + usage: "COUNTER" + description: "Queries cancelled due to dropped tablespaces" + - confl_lock_total: + usage: "COUNTER" + description: "Queries cancelled due to lock timeouts" + - confl_snapshot_total: + usage: "COUNTER" + description: "Queries cancelled due to old snapshots" + - confl_bufferpin_total: + usage: "COUNTER" + description: "Queries cancelled due to pinned buffers" + - confl_deadlock_total: + usage: "COUNTER" + description: "Queries cancelled due to deadlocks" pg_stat_statements: query: "SELECT sum(calls) as total_queries, sum(total_time / 1000) as total_time_seconds FROM extensions.pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" @@ -35,6 +173,17 @@ auth_users: usage: "GAUGE" description: "Number of users in the project db" +replication: + query: "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS realtime_lag_bytes, active AS realtime_slot_status FROM pg_replication_slots where slot_name = 'realtime'" + master: true + metrics: + - realtime_lag_bytes: + usage: "GAUGE" + description: "Replication Lag for Realtime" + - realtime_slot_status: + usage: "GAUGE" + description: "Replication Slot active status" + storage: query: "select sum(size) / (1024 * 1024) as storage_size_mb from storage.get_size_by_bucket()" master: true From 05e02ddc620062cc0447a153c4c5903f42e032aa Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 15 Jul 2021 12:26:56 +0800 Subject: [PATCH 69/86] postgresql-13-wal2json not available on ubuntu 20.04 --- .../tasks/postgres-extensions/11-wal2json.yml | 24 +++++++++++++++++++ .../{11-pljava.yml => 12-pljava.yml} | 0 .../{12-plv8.yml => 13-plv8.yml} | 0 ansible/tasks/setup-extensions.yml | 15 ++++-------- ansible/vars.yml | 3 +++ 5 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 ansible/tasks/postgres-extensions/11-wal2json.yml rename ansible/tasks/postgres-extensions/{11-pljava.yml => 12-pljava.yml} (100%) rename ansible/tasks/postgres-extensions/{12-plv8.yml => 13-plv8.yml} (100%) diff --git a/ansible/tasks/postgres-extensions/11-wal2json.yml b/ansible/tasks/postgres-extensions/11-wal2json.yml new file mode 100644 index 000000000..8fabbdd09 --- /dev/null +++ b/ansible/tasks/postgres-extensions/11-wal2json.yml @@ -0,0 +1,24 @@ +# wal2json +- name: wal2json - download latest release + get_url: + url: "https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_{{ wal2json_release }}.tar.gz" + dest: /tmp/wal2json-{{ wal2json_release }}.tar.gz + checksum: "{{ wal2json_release_checksum }}" + +- name: wal2json - unpack archive + unarchive: + remote_src: yes + src: /tmp/wal2json-{{ wal2json_release }}.tar.gz + dest: /tmp + become: yes + +- name: wal2json - build + make: + chdir: /tmp/wal2json-wal2json_{{ wal2json_release }} + become: yes + +- name: wal2json - install + make: + chdir: /tmp/wal2json-wal2json_{{ wal2json_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/11-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml similarity index 100% rename from ansible/tasks/postgres-extensions/11-pljava.yml rename to ansible/tasks/postgres-extensions/12-pljava.yml diff --git a/ansible/tasks/postgres-extensions/12-plv8.yml b/ansible/tasks/postgres-extensions/13-plv8.yml similarity index 100% rename from ansible/tasks/postgres-extensions/12-plv8.yml rename to ansible/tasks/postgres-extensions/13-plv8.yml diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index d59c04435..a9f2d3f23 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -25,19 +25,14 @@ - name: Install pg-safeupdate import_tasks: tasks/postgres-extensions/09-pg-safeupdate.yml -- name: Install wal2json - apt: - pkg: - - "postgresql-{{ postgresql_major }}-wal2json" - update_cache: yes - cache_valid_time: 3600 - install_recommends: no - # - name: Install timescaledb # import_tasks: tasks/postgres-extensions/10-timescaledb.yml +- name: Install wal2json + import_tasks: tasks/postgres-extensions/11-wal2json.yml + - name: Install pljava - import_tasks: tasks/postgres-extensions/11-pljava.yml + import_tasks: tasks/postgres-extensions/12-pljava.yml - name: Install plv8 - import_tasks: tasks/postgres-extensions/12-plv8.yml \ No newline at end of file + import_tasks: tasks/postgres-extensions/13-plv8.yml \ No newline at end of file diff --git a/ansible/vars.yml b/ansible/vars.yml index ef205d9f5..614f595c6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -60,6 +60,9 @@ pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c timescaledb_release: "2.3.0" +wal2json_release: "2_3" +wal2json_release_checksum: sha1:923f9bbcd0505a1f0b6eac1d371e4ff2d266a958 + supautils_release: "1.1.0" supautils_release_checksum: sha1:326ac5c1933bd30d4a50da7568b27629a9ec544b From b61d72a051daf14f976807a1b2acc73893306a7c Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 15 Jul 2021 12:27:31 +0800 Subject: [PATCH 70/86] revert #58 --- ansible/tasks/setup-system.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index f09fec813..1258fb88f 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -49,20 +49,6 @@ - set_fact: platform: "{{ platform_output.stdout }}" -# Disables overcommiting of memory -- name: Adjust setting for vm.overcommit_memory - sysctl: - name: vm.overcommit_memory - value: 2 - state: present - -# Set overcommit ratio to 75 -- name: Adjust setting for vm.overcommit_ratio - sysctl: - name: vm.overcommit_ratio - value: 75 - state: present - - name: System - Create services.slice template: src: files/services.slice.j2 From 972a04d7edac8b966f54ccaebeb26a73125aebb8 Mon Sep 17 00:00:00 2001 From: Angelico Date: Thu, 15 Jul 2021 12:28:32 +0800 Subject: [PATCH 71/86] consolidating pgbouncer changes - make sure it's not there if pgbouncer is not installed --- ansible/files/postgresql_config/pg_hba.conf.j2 | 2 -- ansible/tasks/setup-pgbouncer.yml | 11 +++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ansible/files/postgresql_config/pg_hba.conf.j2 b/ansible/files/postgresql_config/pg_hba.conf.j2 index ccd45ecae..1e4c86604 100755 --- a/ansible/files/postgresql_config/pg_hba.conf.j2 +++ b/ansible/files/postgresql_config/pg_hba.conf.j2 @@ -79,8 +79,6 @@ # TYPE DATABASE USER ADDRESS METHOD # Default: -# Allow connection by pgbouncer user -host all pgbouncer 127.0.0.1/32 md5 # "local" is for Unix domain socket connections only local all all peer # IPv4 local connections: diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 8a794d968..76015585f 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -57,6 +57,17 @@ group: postgres owner: postgres +- name: PgBouncer - add permissions for pgbouncer user + become: yes + lineinfile: + path: /etc/postgresql/pg_hba.conf + state: present + insertafter: '# Default:' + line: "{{ item }}" + with_items: + - "host all pgbouncer 127.0.0.1/32 md5" + - "# Allow connection by pgbouncer user" + - name: import /etc/tmpfiles.d/pgbouncer.conf template: src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 From f9ad7551d05b36f02f37cf95e254459ab531c8e9 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 15 Jul 2021 12:43:59 +0800 Subject: [PATCH 72/86] feat: install jq as part of base image --- ansible/tasks/setup-supabase-internal.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 0e339477c..b21f0abe2 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -2,6 +2,7 @@ apt: pkg: - unzip + - jq install_recommends: no - name: AWS CLI (arm) @@ -33,4 +34,4 @@ import_tasks: internal/node-exporter.yml - name: Install supautils - import_tasks: internal/supautils.yml \ No newline at end of file + import_tasks: internal/supautils.yml From 06441635a662e1dc3280603646baddaeb7bc2c8a Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 15 Jul 2021 12:50:58 +0800 Subject: [PATCH 73/86] feat: boot time optimizations --- ansible/tasks/internal/node-exporter.yml | 4 ++-- ansible/tasks/internal/optimizations.yml | 15 +++++++++++++++ ansible/tasks/internal/postgres-exporter.yml | 6 +++--- ansible/tasks/setup-supabase-internal.yml | 3 +++ 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 ansible/tasks/internal/optimizations.yml diff --git a/ansible/tasks/internal/node-exporter.yml b/ansible/tasks/internal/node-exporter.yml index fbc34234e..355dcdb15 100644 --- a/ansible/tasks/internal/node-exporter.yml +++ b/ansible/tasks/internal/node-exporter.yml @@ -41,6 +41,6 @@ - name: node_exporter - reload systemd systemd: daemon_reload: yes - enabled: yes - state: restarted + enabled: no + state: stopped name: node_exporter diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml new file mode 100644 index 000000000..dce3c913a --- /dev/null +++ b/ansible/tasks/internal/optimizations.yml @@ -0,0 +1,15 @@ +- name: ensure services are stopped + community.general.snap: + name: amazon-ssm-agent + state: absent + +- name: ensure services are stopped and disabled for first boot + systemd: + enabled: no + name: '{{ item }}' + state: stopped + with_items: + - snapd + - postgresql + - postgrest + - fail2ban diff --git a/ansible/tasks/internal/postgres-exporter.yml b/ansible/tasks/internal/postgres-exporter.yml index c0109d4df..4fecaeabf 100644 --- a/ansible/tasks/internal/postgres-exporter.yml +++ b/ansible/tasks/internal/postgres-exporter.yml @@ -38,9 +38,9 @@ src: files/queries.yml.j2 dest: /opt/postgres_exporter/queries.yml -- name: exporter enable service +- name: exporter ensure service is present systemd: - enabled: yes + enabled: no name: postgres_exporter daemon_reload: yes - state: restarted + state: stopped diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index b21f0abe2..52b69584f 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -35,3 +35,6 @@ - name: Install supautils import_tasks: internal/supautils.yml + +- name: Boot time optimizations + import_tasks: internal/optimizations.yml From 96d3466924c5a856bd04e09fe79c001ed413369f Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 15 Jul 2021 13:13:54 +0800 Subject: [PATCH 74/86] chore: disable pgbouncer as well --- ansible/tasks/internal/optimizations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml index dce3c913a..f41eca62b 100644 --- a/ansible/tasks/internal/optimizations.yml +++ b/ansible/tasks/internal/optimizations.yml @@ -12,4 +12,5 @@ - snapd - postgresql - postgrest + - pgbouncer - fail2ban From 17ee8bc89d5cf90b98058d1d92b102f06e9ea874 Mon Sep 17 00:00:00 2001 From: Angelico Date: Fri, 16 Jul 2021 09:59:24 +0800 Subject: [PATCH 75/86] Remove postgrest from internal optimization - Returns an error since internal build does not use postgrest --- ansible/tasks/internal/optimizations.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml index f41eca62b..8ee65518e 100644 --- a/ansible/tasks/internal/optimizations.yml +++ b/ansible/tasks/internal/optimizations.yml @@ -11,6 +11,5 @@ with_items: - snapd - postgresql - - postgrest - pgbouncer - fail2ban From ad98355b191381dda48a5b53cb3ffb7d8a421a80 Mon Sep 17 00:00:00 2001 From: Angelico Date: Fri, 16 Jul 2021 10:01:55 +0800 Subject: [PATCH 76/86] run this task only under tag 'install-supabase-internal' --- ansible/tasks/setup-fail2ban.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index 1195d2213..10084cfb7 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -31,7 +31,8 @@ - { line: ' ^.+FATAL: password authentication failed for user "authenticator"$' } - { line: ' ^.+FATAL: password authentication failed for user "pgbouncer"$' } become: yes - when: supabase_internal == true + tags: + - install-supabase-internal # Restart - name: fail2ban - restart From d46af0eb54707167b6a930ce7e1fb032ec9e81b3 Mon Sep 17 00:00:00 2001 From: Angelico Date: Fri, 16 Jul 2021 10:06:07 +0800 Subject: [PATCH 77/86] rearrange tasks and set tags - allows to build in stages --- ansible/playbook.yml | 24 ++++++++++++------------ ansible/tasks/setup-system.yml | 7 +++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 866208705..ac8232c64 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -9,7 +9,6 @@ vars: sql_files: - - { source: "pgbouncer_config/pgbouncer_auth_schema.sql", dest: "00-schema.sql" } - { source: "stat_extension.sql", dest: "01-extension.sql" } environment: @@ -18,7 +17,15 @@ tasks: - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - + + - name: Install Postgres extensions + import_tasks: tasks/setup-extensions.yml + + - name: Start Postgres Database + systemd: + name: postgresql + state: started + - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml @@ -31,18 +38,11 @@ import_tasks: tasks/setup-postgrest.yml tags: - install-postgrest - - - name: Install Postgres extensions - import_tasks: tasks/setup-extensions.yml - + - name: Install Supabase specific content import_tasks: tasks/setup-supabase-internal.yml - when: supabase_internal == true - - - name: Start Postgres Database - systemd: - name: postgresql - state: started + tags: + - install-supabase-internal - name: Adjust APT update intervals copy: diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 1258fb88f..fa18ddd89 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -46,8 +46,15 @@ - name: finding platform architecture shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi register: platform_output + tags: + - update + - update-only - set_fact: platform: "{{ platform_output.stdout }}" + tags: + - update + - update-only + - name: System - Create services.slice template: From a0bf9aeb9ab7adcc4b28a783b9a4cfe62d45f225 Mon Sep 17 00:00:00 2001 From: Angelico Date: Fri, 16 Jul 2021 10:10:47 +0800 Subject: [PATCH 78/86] more consolidation of pgbouncer tasks --- ansible/tasks/setup-pgbouncer.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 76015585f..7400b19d9 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -56,6 +56,12 @@ state: touch group: postgres owner: postgres + +- name: import /etc/tmpfiles.d/pgbouncer.conf + template: + src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 + dest: /etc/tmpfiles.d/pgbouncer.conf + become: yes - name: PgBouncer - add permissions for pgbouncer user become: yes @@ -67,12 +73,18 @@ with_items: - "host all pgbouncer 127.0.0.1/32 md5" - "# Allow connection by pgbouncer user" - -- name: import /etc/tmpfiles.d/pgbouncer.conf - template: - src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 - dest: /etc/tmpfiles.d/pgbouncer.conf + +# Run PgBouncer SQL script +- name: Transfer init SQL files + copy: + src: files/pgbouncer_config/pgbouncer_auth_schema.sql + dest: /tmp/00-schema.sql + +- name: Execute init SQL files become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/00-schema.sql # Add fail2ban filter - name: import jail.d/pgbouncer.conf From 979db2d9d1c5bccb8d33a1e21f5abc2553099e09 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Fri, 16 Jul 2021 16:01:46 +0800 Subject: [PATCH 79/86] fix: update log rotation location and frequency (#62) * fix: update log rotation location * fix: set up logrotation to execute every hour This helps avoid situation where postgres logs would fill up the disk under heavy usage conditions. Technically, it can still happen, but should be exceedingly rare outside of abusive conditions. --- ansible/files/logrotate-postgres | 7 +++---- ansible/playbook.yml | 11 +++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ansible/files/logrotate-postgres b/ansible/files/logrotate-postgres index 2ed7f2eb0..3266dbdd8 100644 --- a/ansible/files/logrotate-postgres +++ b/ansible/files/logrotate-postgres @@ -1,10 +1,9 @@ -/var/lib/postgresql/12/main/pg_log/postgresql.csv { - daily - size 50M +/var/lib/postgresql/data/pg_log/postgresql.log { + size 50M rotate 3 copytruncate delaycompress compress notifempty missingok -} \ No newline at end of file +} diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 866208705..9ea80474f 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -96,6 +96,17 @@ src: files/logrotate-postgres dest: /etc/logrotate.d/postgres + - name: Configure logrotation to run every hour + shell: + cmd: mv /etc/cron.daily/logrotate /etc/cron.hourly/ + become: yes + + - name: restart crond + systemd: + state: restarted + name: cron + become: yes + - name: Enhance fail2ban import_tasks: tasks/setup-fail2ban.yml From 34cc677c0db6fb48290b6a727d921684528fede3 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 19 Jul 2021 13:50:21 +0800 Subject: [PATCH 80/86] fixes bug in docker - DB was not starting immediately --- ansible/files/docker_mnt/init.sh | 4 +++- ansible/playbook-docker.yml | 2 +- ansible/tasks/docker/setup.yml | 39 ++++++++++++++++++++++++-------- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ansible/files/docker_mnt/init.sh b/ansible/files/docker_mnt/init.sh index 10fb77668..fd12bbe4a 100644 --- a/ansible/files/docker_mnt/init.sh +++ b/ansible/files/docker_mnt/init.sh @@ -1 +1,3 @@ -cat /etc/postgresql/postgresql.conf > $PGDATA/postgresql.conf \ No newline at end of file +cat /etc/postgresql/postgresql.conf > $PGDATA/postgresql.conf +echo "host replication $POSTGRES_USER 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf +echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf \ No newline at end of file diff --git a/ansible/playbook-docker.yml b/ansible/playbook-docker.yml index dc0de54e7..ea699339f 100644 --- a/ansible/playbook-docker.yml +++ b/ansible/playbook-docker.yml @@ -40,7 +40,7 @@ hosts: localhost tasks: - name: Commit Docker image - command: docker commit "supabase-postgres-build" "supabase/postgres" + command: docker commit --change='CMD ["postgres"]' "supabase-postgres-build" "supabase/postgres" - name: Clean Up Postgres Image hosts: localhost diff --git a/ansible/tasks/docker/setup.yml b/ansible/tasks/docker/setup.yml index cd49da0ff..70a54d243 100644 --- a/ansible/tasks/docker/setup.yml +++ b/ansible/tasks/docker/setup.yml @@ -32,15 +32,36 @@ src: files/postgresql_config/postgresql.conf.j2 dest: etc/postgresql/postgresql.conf -- name: Setup - import pg_hba.conf - template: - src: files/postgresql_config/pg_hba.conf.j2 - dest: etc/postgresql/pg_hba.conf - -- name: Setup - import pg_ident.conf - template: - src: files/postgresql_config/pg_ident.conf.j2 - dest: etc/postgresql/pg_ident.conf +- set_fact: + regex_string: "#unix_socket_directories = '/tmp'" + +- name: Setup - modify unix_socket_directories + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: '{{ regex_string }}' + replace: unix_socket_directories = '/var/run/postgresql' + +- name: Setup - modify unix_socket_directories + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: '{{ regex_string }}' + replace: unix_socket_directories = '/var/run/postgresql' + +- name: Setup - modify hba_file directory + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: hba_file = '/etc/postgresql/pg_hba.conf' + replace: hba_file = '/var/lib/postgresql/data/pg_hba.conf' + +- name: Setup - modify ident_file directory + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: ident_file = '/etc/postgresql/pg_ident.conf' + replace: ident_file = '/var/lib/postgresql/data/pg_ident.conf' - name: Setup - add init script to /docker-entrypoint-initdb.d synchronize: From 59060501813f02ab547e7b0c2633134621be444c Mon Sep 17 00:00:00 2001 From: Div Arora Date: Mon, 19 Jul 2021 16:09:03 +0800 Subject: [PATCH 81/86] fix: pg_stat_statements column names changed in PG13 --- ansible/files/queries.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/queries.yml.j2 b/ansible/files/queries.yml.j2 index 73514d3fa..0de4ca2a1 100644 --- a/ansible/files/queries.yml.j2 +++ b/ansible/files/queries.yml.j2 @@ -154,7 +154,7 @@ pg_stat_database_conflicts: description: "Queries cancelled due to deadlocks" pg_stat_statements: - query: "SELECT sum(calls) as total_queries, sum(total_time / 1000) as total_time_seconds FROM extensions.pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" + query: "SELECT sum(calls) as total_queries, sum(total_exec_time / 1000) as total_time_seconds FROM extensions.pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" master: true metrics: - total_queries: From 6aaecedf9f6207ff930105fac370dcb5e92cc807 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 26 Jul 2021 15:09:56 +0800 Subject: [PATCH 82/86] change postgres logging from stderr to csvlog - includes tweaking of fail2ban settings --- .../files/fail2ban_config/filter-postgresql.conf.j2 | 4 ++-- ansible/files/fail2ban_config/jail-postgresql.conf.j2 | 2 +- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- ansible/tasks/setup-fail2ban.yml | 10 +++++----- ansible/tasks/setup-system.yml | 1 - 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ansible/files/fail2ban_config/filter-postgresql.conf.j2 b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 index 16750ca3b..fd0895aee 100644 --- a/ansible/files/fail2ban_config/filter-postgresql.conf.j2 +++ b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 @@ -1,3 +1,3 @@ [Definition] -failregex = ^.+FATAL: password authentication failed for user.+$ -ignoreregex = ^127.0.0.1.+ \ No newline at end of file +failregex = ^.*,.*,.*,.*,":.*password authentication failed for user.*$ +ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for user.*$ \ No newline at end of file diff --git a/ansible/files/fail2ban_config/jail-postgresql.conf.j2 b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 index 8ed032da9..516f5320b 100644 --- a/ansible/files/fail2ban_config/jail-postgresql.conf.j2 +++ b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 @@ -3,5 +3,5 @@ enabled = true port = 5432 protocol = tcp filter = postgresql -logpath = /var/lib/postgresql/data/pg_log/postgresql.log +logpath = /var/lib/postgresql/data/pg_log/postgresql.csv maxretry = 3 \ No newline at end of file diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 5a2e39e91..8b4fec8e2 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -428,7 +428,7 @@ effective_cache_size = 128MB # - Where to Log - -log_destination = 'stderr' # Valid values are combinations of +log_destination = 'csvlog' # Valid values are combinations of # stderr, csvlog, syslog, and eventlog, # depending on platform. csvlog # requires logging_collector to be on. diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index 10084cfb7..7b0666b07 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -25,11 +25,11 @@ state: present line: "{{ item.line }}" loop: - - { line: ' ^.+FATAL: password authentication failed for user "supabase_admin"$' } - - { line: ' ^.+FATAL: password authentication failed for user "supabase_auth_admin"$' } - - { line: ' ^.+FATAL: password authentication failed for user "supabase_storage_admin"$' } - - { line: ' ^.+FATAL: password authentication failed for user "authenticator"$' } - - { line: ' ^.+FATAL: password authentication failed for user "pgbouncer"$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_auth_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_storage_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""authenticator".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""pgbouncer".*$' } become: yes tags: - install-supabase-internal diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index fa18ddd89..3a3db1ad2 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -55,7 +55,6 @@ - update - update-only - - name: System - Create services.slice template: src: files/services.slice.j2 From 1d8342b2eb210dc56e1401fcc5cfc7967cfdc093 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 26 Jul 2021 15:10:39 +0800 Subject: [PATCH 83/86] update json config files --- amazon-arm.json | 3 +-- amazon.json | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index 4ed2ff8b3..0143325b2 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -7,7 +7,7 @@ "ami": "ami-076d8ebdd0e1ec091", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-postgrest,--skip-tags,install-supabase-internal" }, "builders": [ { @@ -51,7 +51,6 @@ "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", "type": "shell", "scripts": [ - "scripts/01-postgres_check.sh", "scripts/02-credentials_cleanup.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh" diff --git a/amazon.json b/amazon.json index 83716e092..aa2db5fbf 100644 --- a/amazon.json +++ b/amazon.json @@ -5,9 +5,9 @@ "region": "af-south-1", "ami_regions": "af-south-1", "ami": "ami-08a4b40f2fe1e4b35", - "ami_name": "supabase-postgres-13.3.0", + "ami_name": "supabase-postgres-13.3.0.4", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-postgrest,--skip-tags,install-supabase-internal" }, "builders": [ { @@ -51,7 +51,6 @@ "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", "type": "shell", "scripts": [ - "scripts/01-postgres_check.sh", "scripts/02-credentials_cleanup.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh" From 17f55e413ceac0e8367e2f5c9661fde7f99d1b74 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 26 Jul 2021 15:14:25 +0800 Subject: [PATCH 84/86] remove github actions - This one would take hours to complete and would just burn credits --- .github/workflows/provision.yml | 54 --------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 .github/workflows/provision.yml diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml deleted file mode 100644 index a7f4b67aa..000000000 --- a/.github/workflows/provision.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: CD - -on: - push: - branches: [ master ] - -jobs: - provision: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Install essentials - run: | - sudo apt-get update -y - sudo apt-get install software-properties-common -y - sudo apt-get install unzip -y - sudo apt-get install git -y - - - name: Install Packer - run: | - export VER="1.5.5" - wget https://releases.hashicorp.com/packer/${VER}/packer_${VER}_linux_amd64.zip - sudo unzip packer_${VER}_linux_amd64.zip - sudo mv packer /usr/local/bin - - - name: Install Ansible - run: | - sudo apt-add-repository --yes ppa:ansible/ansible - sudo apt-get install ansible -y - ansible --version - - - name: Build Digital Ocean Snapshot - run: | - export REGION=sgp1 - export SNAPSHOT_REGIONS="nyc1,nyc3,ams3,sfo2,sfo3,sgp1,lon1,fra1,tor1" - sudo packer build \ - -var "do_token=${{ secrets.DO_TOKEN }}" \ - -var "region=$REGION" \ - -var "snapshot_regions=$SNAPSHOT_REGIONS" \ - digitalOcean.json - - # - name: Build Digital Ocean Snapshot for Marketplace - # run: | - # export REGION=sgp1 - # export IMAGE_NAME="supabase-supabasepostgres-18-04" - # export ARGS="--tags,update" - # sudo packer build \ - # -var "do_token=${{ secrets.DO_TOKEN }}" \ - # -var "region=$REGION" \ - # -var "image_name=$IMAGE_NAME" \ - # -var "ansible_arguments=$ARGS" \ - # digitalOcean.json From bfe92c51ea71760ea04a6539e79e8fd694ec5f36 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 26 Jul 2021 15:22:08 +0800 Subject: [PATCH 85/86] update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2dbea00d1..690115791 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,11 @@ See all installation instructions in the [repo wiki](https://github.com/supabase ### Quick Build -Set the `supabase_internal` flag to `false` to avoid baking in components that are specific to Supabase's hosted offering. - ```bash $ time packer build -timestamp-ui \ - -var "ansible_arguments=--skip-tags,update-only,-v,-e,supabase_internal='false'" \ + --var "aws_access_key=" \ + --var "aws_secret_key=" \ + --var "ami_regions=" \ amazon-arm.json ``` From cd4c9f47e3e96ed1e743565cf6c3654b94a34571 Mon Sep 17 00:00:00 2001 From: Angelico Date: Mon, 26 Jul 2021 15:47:34 +0800 Subject: [PATCH 86/86] remove additional --skip-tags,install-postgrest --- amazon-arm.json | 2 +- amazon.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-arm.json b/amazon-arm.json index 0143325b2..4a1b2acb3 100644 --- a/amazon-arm.json +++ b/amazon-arm.json @@ -7,7 +7,7 @@ "ami": "ami-076d8ebdd0e1ec091", "ami_name": "supabase-postgres-13.3.0", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-postgrest,--skip-tags,install-supabase-internal" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal" }, "builders": [ { diff --git a/amazon.json b/amazon.json index aa2db5fbf..0cc4ef860 100644 --- a/amazon.json +++ b/amazon.json @@ -7,7 +7,7 @@ "ami": "ami-08a4b40f2fe1e4b35", "ami_name": "supabase-postgres-13.3.0.4", "environment": "prod", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-postgrest,--skip-tags,install-supabase-internal" + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal" }, "builders": [ {