From c37f69bfbf65a2ea906f4615caf4e8d636ae7303 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 9 Dec 2024 17:11:12 -0500 Subject: [PATCH 01/97] feat: orioledb-17 Docker image and publish workflow --- Dockerfile-17-orioledb | 241 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 Dockerfile-17-orioledb diff --git a/Dockerfile-17-orioledb b/Dockerfile-17-orioledb new file mode 100644 index 0000000000..cc07c0fe1f --- /dev/null +++ b/Dockerfile-17-orioledb @@ -0,0 +1,241 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=17-orioledb +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + tree \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_orioledb-17/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN tree /nix > /tmp/tree.txt && cat /tmp/tree.txt && cat /tmp/tree.txt >&2 + + +# RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ +# && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ +# && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + + # Remove items from postgresql.conf +RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present +RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" +RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf" +RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" +RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf" + + + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql +# Enable orioledb extension first +RUN echo "CREATE EXTENSION orioledb;" > /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql && \ + chown postgres:postgres /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/17/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] \ No newline at end of file From 7ba55bff0bc81839af646d3a546886656dd3c37a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 13:44:35 -0500 Subject: [PATCH 02/97] feat: refactor to build docker images from major versions skip if no Dockerfile exists --- .github/workflows/dockerhub-release-15-6.yml | 104 ------------ .../workflows/dockerhub-release-matrix.yml | 153 ++++++++++++++++++ Dockerfile-15 | 22 +-- ...file-17-orioledb => Dockerfile-orioledb-17 | 0 ...rfile-156 => deprecated-Dockerfile-15-prev | 22 ++- ...erfile-16 => deprecated-Dockerfile-16-prev | 0 6 files changed, 175 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/dockerhub-release-15-6.yml create mode 100644 .github/workflows/dockerhub-release-matrix.yml rename Dockerfile-17-orioledb => Dockerfile-orioledb-17 (100%) rename Dockerfile-156 => deprecated-Dockerfile-15-prev (95%) rename Dockerfile-16 => deprecated-Dockerfile-16-prev (100%) diff --git a/.github/workflows/dockerhub-release-15-6.yml b/.github/workflows/dockerhub-release-15-6.yml deleted file mode 100644 index 4c7b5b998d..0000000000 --- a/.github/workflows/dockerhub-release-15-6.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Release 15.6 on Dockerhub - -on: - push: - branches: - - develop - - release/* - paths: - - ".github/workflows/dockerhub-release-15-6.yml" - - "common-nix.vars*" - workflow_dispatch: - -jobs: - settings: - runs-on: ubuntu-latest - outputs: - docker_version: ${{ steps.settings.outputs.postgres-version }} - image_tag: supabase/postgres:${{ steps.settings.outputs.postgres-version }} - build_args: ${{ steps.args.outputs.result }} - steps: - - uses: actions/checkout@v3 - - id: settings - # Remove spaces and quotes to get the raw version string - run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - build_release_image: - needs: [settings] - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} - steps: - - run: docker context create builders - - uses: docker/setup-buildx-action@v3 - with: - endpoint: builders - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - id: build - uses: docker/build-push-action@v5 - with: - push: true - build-args: | - ${{ needs.settings.outputs.build_args }} - target: production - tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: "Dockerfile-156" - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" - SLACK_FOOTER: "" - - merge_manifest: - needs: [settings, build_release_image] - runs-on: ubuntu-latest - steps: - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ - ${{ needs.settings.outputs.image_tag }}_amd64 \ - ${{ needs.settings.outputs.image_tag }}_arm64 - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed" - SLACK_FOOTER: "" - - publish: - needs: [settings, merge_manifest] - # Call workflow explicitly because events from actions cannot trigger more actions - uses: ./.github/workflows/mirror.yml - with: - version: ${{ needs.settings.outputs.docker_version }} - secrets: inherit diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml new file mode 100644 index 0000000000..85b47786c1 --- /dev/null +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -0,0 +1,153 @@ +name: Release all major versions on Dockerhub + +on: + push: + branches: + - develop + - release/* + - sam/docker-oriole17 + paths: + - ".github/workflows/dockerhub-release-matrix.yml" + - "anisble/vars.yml" + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Generate build matrix + id: set-matrix + run: | + # Get all postgres versions from vars.yml + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml) + + # Check which versions have corresponding Dockerfiles and create matrix config + MATRIX_CONFIG="{" + MATRIX_CONFIG+="\"include\":[" + FIRST=true + + while IFS= read -r version; do + # Remove quotes from version + version=$(echo "$version" | tr -d '"') + + # Determine dockerfile name based on version + if [[ "$version" == "orioledb-"* ]]; then + dockerfile="Dockerfile-${version#orioledb-}-orioledb" + else + dockerfile="Dockerfile-${version}" + fi + + # Check if Dockerfile exists + if [ -f "$dockerfile" ]; then + if [ "$FIRST" = true ]; then + FIRST=false + else + MATRIX_CONFIG+="," + fi + MATRIX_CONFIG+="{\"version\":\"$version\",\"dockerfile\":\"$dockerfile\"}" + fi + done <<< "$VERSIONS" + + MATRIX_CONFIG+="]}" + + # Output the matrix configuration + echo "matrix_config=$MATRIX_CONFIG" >> $GITHUB_OUTPUT + + build: + needs: prepare + strategy: + matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} + runs-on: ubuntu-latest + outputs: + image_tag: supabase/postgres:${{ steps.settings.outputs.postgres-version }} + build_args: ${{ steps.args.outputs.result }} + steps: + - uses: actions/checkout@v3 + - id: settings + run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT + - id: args + uses: mikefarah/yq@master + with: + cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + + build_release_image: + needs: [build] + strategy: + matrix: + include: + - runner: [self-hosted, X64] + arch: amd64 + - runner: arm-runner + arch: arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 180 + outputs: + image_digest: ${{ steps.build.outputs.digest }} + steps: + - run: docker context create builders + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - id: build + uses: docker/build-push-action@v5 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + target: production + tags: ${{ needs.build.outputs.image_tag }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.dockerfile }} + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: "gha-failures-notifier" + SLACK_COLOR: "danger" + SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed for version ${{ matrix.version }}" + SLACK_FOOTER: "" + + merge_manifest: + needs: [build, build_release_image] + runs-on: ubuntu-latest + steps: + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t ${{ needs.build.outputs.image_tag }} \ + ${{ needs.build.outputs.image_tag }}_amd64 \ + ${{ needs.build.outputs.image_tag }}_arm64 + - name: Slack Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: "gha-failures-notifier" + SLACK_COLOR: "danger" + SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" + SLACK_FOOTER: "" + + publish: + needs: [build, merge_manifest] + uses: ./.github/workflows/mirror.yml + with: + version: ${{ needs.build.outputs.docker_version }} + secrets: inherit \ No newline at end of file diff --git a/Dockerfile-15 b/Dockerfile-15 index 01895db39b..ce83237f27 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -41,26 +41,14 @@ ARG wal_g_release=2.0.1 FROM ubuntu:focal as base - -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - TZ=Etc/UTC - -# Pre-configure tzdata before any installations -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone && \ - apt-get update && \ - apt-get install -y --no-install-recommends tzdata && \ - apt-get install -y \ +RUN apt update -y && apt install -y \ curl \ gnupg \ lsb-release \ software-properties-common \ wget \ sudo \ - git \ - && apt clean && \ - rm -rf /var/lib/apt/lists/* + && apt clean RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres @@ -114,6 +102,12 @@ RUN chown -R postgres:postgres /usr/lib/postgresql RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ diff --git a/Dockerfile-17-orioledb b/Dockerfile-orioledb-17 similarity index 100% rename from Dockerfile-17-orioledb rename to Dockerfile-orioledb-17 diff --git a/Dockerfile-156 b/deprecated-Dockerfile-15-prev similarity index 95% rename from Dockerfile-156 rename to deprecated-Dockerfile-15-prev index ce83237f27..01895db39b 100644 --- a/Dockerfile-156 +++ b/deprecated-Dockerfile-15-prev @@ -41,14 +41,26 @@ ARG wal_g_release=2.0.1 FROM ubuntu:focal as base -RUN apt update -y && apt install -y \ + +ENV DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + TZ=Etc/UTC + +# Pre-configure tzdata before any installations +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + apt-get update && \ + apt-get install -y --no-install-recommends tzdata && \ + apt-get install -y \ curl \ gnupg \ lsb-release \ software-properties-common \ wget \ sudo \ - && apt clean + git \ + && apt clean && \ + rm -rf /var/lib/apt/lists/* RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres @@ -102,12 +114,6 @@ RUN chown -R postgres:postgres /usr/lib/postgresql RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets -RUN apt-get update && \ - apt-get install -y --no-install-recommends tzdata - -RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ - dpkg-reconfigure --frontend noninteractive tzdata - RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ diff --git a/Dockerfile-16 b/deprecated-Dockerfile-16-prev similarity index 100% rename from Dockerfile-16 rename to deprecated-Dockerfile-16-prev From ee89cc2532443dce7f756647ac224a131ae9d222 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:01:02 -0500 Subject: [PATCH 03/97] feat: gen common vars hcl file --- .../workflows/dockerhub-release-matrix.yml | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 85b47786c1..343c78c109 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -1,16 +1,17 @@ -name: Release all major versions on Dockerhub +name: Release PostgreSQL Images on Dockerhub on: push: branches: - develop - release/* - - sam/docker-oriole17 paths: - - ".github/workflows/dockerhub-release-matrix.yml" - - "anisble/vars.yml" + - ".github/workflows/dockerhub-release.yml" + - "common-nix.vars*" + - "ansible/vars.yml" + - "Dockerfile-*" workflow_dispatch: - + jobs: prepare: runs-on: ubuntu-latest @@ -70,8 +71,30 @@ jobs: build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + # For orioledb versions, we need to handle the version lookup differently + if [[ "${{ matrix.version }}" == "orioledb-"* ]]; then + MAJOR_VERSION="${{ matrix.version #orioledb-}}" + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$MAJOR_VERSION'_orioledb"]' ansible/vars.yml) + else + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) + fi + + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + - id: settings run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT + - id: args uses: mikefarah/yq@master with: From c53895af9b06ff4d69b900bc14e09b5b880a1488 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:02:58 -0500 Subject: [PATCH 04/97] chore: cleanup --- Dockerfile-15 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile-15 b/Dockerfile-15 index ce83237f27..a88f6cc9fe 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -75,10 +75,6 @@ WORKDIR / RUN mkdir -p /usr/lib/postgresql/bin \ /usr/lib/postgresql/share/postgresql \ /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ && chown -R postgres:postgres /usr/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql From 1b17127d9cd57add1ccdc6d2fddaa341af6c3003 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:05:24 -0500 Subject: [PATCH 05/97] chore: adj trigger for run in PR` --- .github/workflows/dockerhub-release-matrix.yml | 3 +-- .github/workflows/publish-nix-pgupgrade-scripts.yml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 343c78c109..41946070b4 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -6,8 +6,7 @@ on: - develop - release/* paths: - - ".github/workflows/dockerhub-release.yml" - - "common-nix.vars*" + - ".github/workflows/dockerhub-release-matrix.yml" - "ansible/vars.yml" - "Dockerfile-*" workflow_dispatch: diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index a4aa4c3de0..fa713ae107 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -5,6 +5,7 @@ on: branches: - develop - release/* + - sam/docker-oriole17 paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' workflow_dispatch: From 4c45cf1db0ba51dd5bfaec2b3691047445576c8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:12:54 -0500 Subject: [PATCH 06/97] chore: trigger release all major vers --- .github/workflows/dockerhub-release-matrix.yml | 12 ++++++------ .github/workflows/publish-nix-pgupgrade-scripts.yml | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 41946070b4..74981a8224 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -1,16 +1,16 @@ -name: Release PostgreSQL Images on Dockerhub +name: Release all major versions on Dockerhub on: push: branches: - develop - release/* - paths: - - ".github/workflows/dockerhub-release-matrix.yml" - - "ansible/vars.yml" - - "Dockerfile-*" + - sam/docker-oriole17 + # paths: + # - ".github/workflows/dockerhub-release-matrix.yml" + # - "anisble/vars.yml" workflow_dispatch: - + jobs: prepare: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index fa713ae107..a4aa4c3de0 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -5,7 +5,6 @@ on: branches: - develop - release/* - - sam/docker-oriole17 paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' workflow_dispatch: From f4eb159fce7b584df02dbb8bce2afbd2fe2ce0cb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:15:51 -0500 Subject: [PATCH 07/97] chore: trigger on any path --- .github/workflows/dockerhub-release-matrix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 74981a8224..c846aba632 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -6,9 +6,10 @@ on: - develop - release/* - sam/docker-oriole17 - # paths: + paths: # - ".github/workflows/dockerhub-release-matrix.yml" # - "anisble/vars.yml" + - '**' workflow_dispatch: jobs: From 74c5905e12e219fb94b4ed2f9035e54eabf0722d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:26:06 -0500 Subject: [PATCH 08/97] chore: cleanup version handling --- .github/workflows/dockerhub-release-matrix.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index c846aba632..c03561af9e 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -79,14 +79,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - # For orioledb versions, we need to handle the version lookup differently - if [[ "${{ matrix.version }}" == "orioledb-"* ]]; then - MAJOR_VERSION="${{ matrix.version #orioledb-}}" - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$MAJOR_VERSION'_orioledb"]' ansible/vars.yml) - else - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) - fi - + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file From 6bd45b132e68926bf5bb9829a1fb0815dcb5449f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:36:02 -0500 Subject: [PATCH 09/97] chore: simplify name matching --- .github/workflows/dockerhub-release-matrix.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index c03561af9e..3ebb239e11 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -7,8 +7,6 @@ on: - release/* - sam/docker-oriole17 paths: - # - ".github/workflows/dockerhub-release-matrix.yml" - # - "anisble/vars.yml" - '**' workflow_dispatch: @@ -29,7 +27,7 @@ jobs: # Get all postgres versions from vars.yml VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml) - # Check which versions have corresponding Dockerfiles and create matrix config + # Create matrix config MATRIX_CONFIG="{" MATRIX_CONFIG+="\"include\":[" FIRST=true @@ -38,12 +36,8 @@ jobs: # Remove quotes from version version=$(echo "$version" | tr -d '"') - # Determine dockerfile name based on version - if [[ "$version" == "orioledb-"* ]]; then - dockerfile="Dockerfile-${version#orioledb-}-orioledb" - else - dockerfile="Dockerfile-${version}" - fi + # Simply look for Dockerfile-{version} + dockerfile="Dockerfile-${version}" # Check if Dockerfile exists if [ -f "$dockerfile" ]; then From 2be76f40462725f304f2d89a17c2f0fad265d860 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:43:03 -0500 Subject: [PATCH 10/97] chore: matrix on version and arch in build_release_image --- .../workflows/dockerhub-release-matrix.yml | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 3ebb239e11..4c301ee696 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,7 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - image_tag: supabase/postgres:${{ steps.settings.outputs.postgres-version }} + versions: ${{ steps.output-versions.outputs.versions }} + image_tags: ${{ steps.output-versions.outputs.image_tags }} build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 @@ -76,7 +77,6 @@ jobs: PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - id: settings @@ -87,19 +87,25 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + - id: output-versions + run: | + # Create JSON arrays for versions and image tags + echo "versions=$(echo '${{ toJSON(matrix.version) }}')" >> $GITHUB_OUTPUT + echo "image_tags=$(echo 'supabase/postgres:${{ steps.settings.outputs.postgres-version }}')" >> $GITHUB_OUTPUT + build_release_image: needs: [build] strategy: matrix: + version: ${{ fromJson(needs.build.outputs.versions) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 + - arch: amd64 + runner: [self-hosted, X64] + - arch: arm64 + runner: arm-runner runs-on: ${{ matrix.runner }} timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} steps: - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -116,11 +122,11 @@ jobs: build-args: | ${{ needs.build.outputs.build_args }} target: production - tags: ${{ needs.build.outputs.image_tag }}_${{ matrix.arch }} + tags: ${{ matrix.image_tag }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.dockerfile }} + file: Dockerfile-${{ matrix.version }} - name: Slack Notification if: ${{ failure() }} uses: rtCamp/action-slack-notify@v2 @@ -133,6 +139,9 @@ jobs: merge_manifest: needs: [build, build_release_image] + strategy: + matrix: + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 @@ -142,9 +151,9 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Merge multi-arch manifests run: | - docker buildx imagetools create -t ${{ needs.build.outputs.image_tag }} \ - ${{ needs.build.outputs.image_tag }}_amd64 \ - ${{ needs.build.outputs.image_tag }}_arm64 + docker buildx imagetools create -t ${{ matrix.image_tag }} \ + ${{ matrix.image_tag }}_amd64 \ + ${{ matrix.image_tag }}_arm64 - name: Slack Notification if: ${{ failure() }} uses: rtCamp/action-slack-notify@v2 @@ -160,4 +169,4 @@ jobs: uses: ./.github/workflows/mirror.yml with: version: ${{ needs.build.outputs.docker_version }} - secrets: inherit \ No newline at end of file + secrets: inherit From 7f16807a310befb12547dc23ca37dee7b25b02eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:48:06 -0500 Subject: [PATCH 11/97] chore: try to get json formatting correct --- .../workflows/dockerhub-release-matrix.yml | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 4c301ee696..6a9d3c31f5 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,9 +61,7 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.output-versions.outputs.versions }} - image_tags: ${{ steps.output-versions.outputs.image_tags }} - build_args: ${{ steps.args.outputs.result }} + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v3 @@ -87,23 +85,33 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - id: output-versions + - id: set-matrix run: | - # Create JSON arrays for versions and image tags - echo "versions=$(echo '${{ toJSON(matrix.version) }}')" >> $GITHUB_OUTPUT - echo "image_tags=$(echo 'supabase/postgres:${{ steps.settings.outputs.postgres-version }}')" >> $GITHUB_OUTPUT + # Create a JSON object for the matrix + MATRIX="{\"include\":[" + + # For each architecture + for arch in "amd64" "arm64"; do + runner="[\"self-hosted\", \"X64\"]" + [ "$arch" = "arm64" ] && runner="arm-runner" + + # Add the version details + if [ "$arch" = "amd64" ]; then + MATRIX+="{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\",\"arch\":\"$arch\",\"runner\":$runner}" + else + MATRIX+=",{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\",\"arch\":\"$arch\",\"runner\":$runner}" + fi + done + + MATRIX+="]}" + + # Output the matrix + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT build_release_image: - needs: [build] + needs: build strategy: - matrix: - version: ${{ fromJson(needs.build.outputs.versions) }} - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} - include: - - arch: amd64 - runner: [self-hosted, X64] - - arch: arm64 - runner: arm-runner + matrix: ${{ fromJson(needs.build.outputs.matrix) }} runs-on: ${{ matrix.runner }} timeout-minutes: 180 steps: @@ -138,10 +146,9 @@ jobs: SLACK_FOOTER: "" merge_manifest: - needs: [build, build_release_image] + needs: [build] strategy: - matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + matrix: ${{ fromJson(needs.build.outputs.matrix) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 From e5cbc7ce1d733e768adca6a001e013bcee6be22a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 14:56:20 -0500 Subject: [PATCH 12/97] chore: more fixes to json parse --- .../workflows/dockerhub-release-matrix.yml | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 6a9d3c31f5..85c3fdadb3 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,7 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + versions: ${{ steps.versions.outputs.list }} + image_tags: ${{ steps.versions.outputs.image_tags }} steps: - uses: actions/checkout@v3 @@ -85,34 +86,19 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - id: set-matrix + - id: versions run: | - # Create a JSON object for the matrix - MATRIX="{\"include\":[" - - # For each architecture - for arch in "amd64" "arm64"; do - runner="[\"self-hosted\", \"X64\"]" - [ "$arch" = "arm64" ] && runner="arm-runner" - - # Add the version details - if [ "$arch" = "amd64" ]; then - MATRIX+="{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\",\"arch\":\"$arch\",\"runner\":$runner}" - else - MATRIX+=",{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\",\"arch\":\"$arch\",\"runner\":$runner}" - fi - done - - MATRIX+="]}" - - # Output the matrix - echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + echo "list=[\"${{ matrix.version }}\"]" >> $GITHUB_OUTPUT + echo "image_tags=[\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\"]" >> $GITHUB_OUTPUT build_release_image: needs: build strategy: - matrix: ${{ fromJson(needs.build.outputs.matrix) }} - runs-on: ${{ matrix.runner }} + matrix: + version: ${{ fromJson(needs.build.outputs.versions) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && fromJson('["self-hosted", "X64"]') || 'arm-runner' }} timeout-minutes: 180 steps: - run: docker context create builders @@ -146,9 +132,10 @@ jobs: SLACK_FOOTER: "" merge_manifest: - needs: [build] + needs: [build, build_release_image] strategy: - matrix: ${{ fromJson(needs.build.outputs.matrix) }} + matrix: + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 From 6731dab2dc1083e8598a04ec8f9683896e0440fd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 15:04:24 -0500 Subject: [PATCH 13/97] feat: collect all versions into file, then array --- .../workflows/dockerhub-release-matrix.yml | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 85c3fdadb3..5af6ad00ec 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,8 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.versions.outputs.list }} - image_tags: ${{ steps.versions.outputs.image_tags }} + versions: ${{ steps.collect-versions.outputs.versions }} + image_tags: ${{ steps.collect-versions.outputs.image_tags }} steps: - uses: actions/checkout@v3 @@ -86,10 +86,25 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - id: versions + # First collect all versions into a file + - name: Save version info run: | - echo "list=[\"${{ matrix.version }}\"]" >> $GITHUB_OUTPUT - echo "image_tags=[\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\"]" >> $GITHUB_OUTPUT + mkdir -p /tmp/versions + echo "${{ matrix.version }}" >> /tmp/versions/versions.txt + echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" >> /tmp/versions/image_tags.txt + + # Then at the end of the matrix, collect all versions into arrays + - name: Collect all versions + id: collect-versions + if: ${{ always() }} + run: | + # Combine all versions into arrays + VERSIONS=$(cat /tmp/versions/versions.txt | sort -u | jq -R . | jq -s .) + IMAGE_TAGS=$(cat /tmp/versions/image_tags.txt | sort -u | jq -R . | jq -s .) + + # Output as JSON arrays + echo "versions=$VERSIONS" >> $GITHUB_OUTPUT + echo "image_tags=$IMAGE_TAGS" >> $GITHUB_OUTPUT build_release_image: needs: build From 7a69907f5b9d54df22a301af50f4e511c346b2d5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 15:21:38 -0500 Subject: [PATCH 14/97] chore: try to account for gh action handling --- .../workflows/dockerhub-release-matrix.yml | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 5af6ad00ec..c85778ce4d 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -98,13 +98,34 @@ jobs: id: collect-versions if: ${{ always() }} run: | - # Combine all versions into arrays - VERSIONS=$(cat /tmp/versions/versions.txt | sort -u | jq -R . | jq -s .) - IMAGE_TAGS=$(cat /tmp/versions/image_tags.txt | sort -u | jq -R . | jq -s .) + # Create JSON arrays + versions_array="[" + first=true + while IFS= read -r version; do + if [ "$first" = true ]; then + first=false + else + versions_array+="," + fi + versions_array+="\"$version\"" + done < <(sort -u /tmp/versions/versions.txt) + versions_array+="]" + + image_tags_array="[" + first=true + while IFS= read -r tag; do + if [ "$first" = true ]; then + first=false + else + image_tags_array+="," + fi + image_tags_array+="\"$tag\"" + done < <(sort -u /tmp/versions/image_tags.txt) + image_tags_array+="]" - # Output as JSON arrays - echo "versions=$VERSIONS" >> $GITHUB_OUTPUT - echo "image_tags=$IMAGE_TAGS" >> $GITHUB_OUTPUT + # Output as properly escaped JSON arrays + echo "versions=$versions_array" >> $GITHUB_OUTPUT + echo "image_tags=$image_tags_array" >> $GITHUB_OUTPUT build_release_image: needs: build From 2addaabb75c11ee22a2d7757a34ab81e7c50f32c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 16:09:29 -0500 Subject: [PATCH 15/97] chore: output single matrix for each version --- .../workflows/dockerhub-release-matrix.yml | 91 +++++++++++-------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index c85778ce4d..341b4c8267 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,8 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.collect-versions.outputs.versions }} - image_tags: ${{ steps.collect-versions.outputs.image_tags }} + matrix: ${{ steps.create-matrix.outputs.matrix }} + build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 @@ -86,57 +86,68 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - # First collect all versions into a file - - name: Save version info + - id: create-matrix run: | - mkdir -p /tmp/versions - echo "${{ matrix.version }}" >> /tmp/versions/versions.txt - echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" >> /tmp/versions/image_tags.txt + # Create a matrix entry for this version + matrix_entry="{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\"}" + echo "matrix={\"include\":[$matrix_entry]}" >> $GITHUB_OUTPUT - # Then at the end of the matrix, collect all versions into arrays - - name: Collect all versions - id: collect-versions - if: ${{ always() }} + collect_versions: + needs: build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.create-final-matrix.outputs.matrix }} + steps: + - id: create-final-matrix run: | - # Create JSON arrays - versions_array="[" - first=true - while IFS= read -r version; do - if [ "$first" = true ]; then - first=false + # Combine all matrix configurations from build job + MATRIX="{\"include\":[" + FIRST=true + + for matrix_config in ${{ toJSON(needs.build.outputs.matrix) }}; do + if [ "$FIRST" = true ]; then + FIRST=false else - versions_array+="," + MATRIX+="," fi - versions_array+="\"$version\"" - done < <(sort -u /tmp/versions/versions.txt) - versions_array+="]" + # Extract the include array content from each matrix config + include_content=$(echo "$matrix_config" | jq -r '.include[]') + MATRIX+="$include_content" + done - image_tags_array="[" - first=true - while IFS= read -r tag; do - if [ "$first" = true ]; then - first=false + # Add architecture combinations for each version + FINAL_MATRIX="{\"include\":[" + FIRST=true + + echo "$MATRIX]}" | jq -c '.include[]' | while read -r entry; do + version=$(echo "$entry" | jq -r '.version') + image_tag=$(echo "$entry" | jq -r '.image_tag') + + # Add amd64 entry + if [ "$FIRST" = true ]; then + FIRST=false else - image_tags_array+="," + FINAL_MATRIX+="," fi - image_tags_array+="\"$tag\"" - done < <(sort -u /tmp/versions/image_tags.txt) - image_tags_array+="]" + FINAL_MATRIX+="{\"version\":\"$version\",\"image_tag\":\"$image_tag\",\"arch\":\"amd64\",\"runner\":[\"self-hosted\",\"X64\"]}" + + # Add arm64 entry + FINAL_MATRIX+=",{\"version\":\"$version\",\"image_tag\":\"$image_tag\",\"arch\":\"arm64\",\"runner\":\"arm-runner\"}" + done + + FINAL_MATRIX+="]}" - # Output as properly escaped JSON arrays - echo "versions=$versions_array" >> $GITHUB_OUTPUT - echo "image_tags=$image_tags_array" >> $GITHUB_OUTPUT + # Output the final matrix + echo "matrix=$FINAL_MATRIX" >> $GITHUB_OUTPUT build_release_image: - needs: build + needs: collect_versions strategy: - matrix: - version: ${{ fromJson(needs.build.outputs.versions) }} - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && fromJson('["self-hosted", "X64"]') || 'arm-runner' }} + matrix: ${{ fromJson(needs.collect_versions.outputs.matrix) }} + runs-on: ${{ matrix.runner }} timeout-minutes: 180 steps: + - uses: actions/checkout@v3 - run: docker context create builders - uses: docker/setup-buildx-action@v3 with: @@ -171,7 +182,7 @@ jobs: needs: [build, build_release_image] strategy: matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + image_tag: ${{ fromJson(needs.collect_versions.outputs.matrix) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 From 19f50db059a83be8e9ef01cb0e835e30e588cdca Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 16:41:17 -0500 Subject: [PATCH 16/97] chore: more fixes to matrix handling --- .../workflows/dockerhub-release-matrix.yml | 78 ++++++------------- 1 file changed, 24 insertions(+), 54 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 341b4c8267..8f05317fec 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,7 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - matrix: ${{ steps.create-matrix.outputs.matrix }} + versions: ${{ steps.output-versions.outputs.versions }} + image_tags: ${{ steps.output-versions.outputs.image_tags }} build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 @@ -86,65 +87,34 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - id: create-matrix + - id: output-versions + if: always() run: | - # Create a matrix entry for this version - matrix_entry="{\"version\":\"${{ matrix.version }}\",\"image_tag\":\"supabase/postgres:${{ steps.settings.outputs.postgres-version }}\"}" - echo "matrix={\"include\":[$matrix_entry]}" >> $GITHUB_OUTPUT - - collect_versions: - needs: build - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.create-final-matrix.outputs.matrix }} - steps: - - id: create-final-matrix - run: | - # Combine all matrix configurations from build job - MATRIX="{\"include\":[" - FIRST=true - - for matrix_config in ${{ toJSON(needs.build.outputs.matrix) }}; do - if [ "$FIRST" = true ]; then - FIRST=false - else - MATRIX+="," - fi - # Extract the include array content from each matrix config - include_content=$(echo "$matrix_config" | jq -r '.include[]') - MATRIX+="$include_content" - done - - # Add architecture combinations for each version - FINAL_MATRIX="{\"include\":[" - FIRST=true + # Create output files if they don't exist + mkdir -p /tmp/versions + touch /tmp/versions/all_versions.txt + touch /tmp/versions/all_tags.txt - echo "$MATRIX]}" | jq -c '.include[]' | while read -r entry; do - version=$(echo "$entry" | jq -r '.version') - image_tag=$(echo "$entry" | jq -r '.image_tag') - - # Add amd64 entry - if [ "$FIRST" = true ]; then - FIRST=false - else - FINAL_MATRIX+="," - fi - FINAL_MATRIX+="{\"version\":\"$version\",\"image_tag\":\"$image_tag\",\"arch\":\"amd64\",\"runner\":[\"self-hosted\",\"X64\"]}" - - # Add arm64 entry - FINAL_MATRIX+=",{\"version\":\"$version\",\"image_tag\":\"$image_tag\",\"arch\":\"arm64\",\"runner\":\"arm-runner\"}" - done + # Append current version and tag + echo "${{ matrix.version }}" >> /tmp/versions/all_versions.txt + echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" >> /tmp/versions/all_tags.txt - FINAL_MATRIX+="]}" + # Convert to JSON arrays with unique values + VERSIONS=$(sort -u /tmp/versions/all_versions.txt | jq -R . | jq -s .) + TAGS=$(sort -u /tmp/versions/all_tags.txt | jq -R . | jq -s .) - # Output the final matrix - echo "matrix=$FINAL_MATRIX" >> $GITHUB_OUTPUT + # Set outputs + echo "versions=$VERSIONS" >> $GITHUB_OUTPUT + echo "image_tags=$TAGS" >> $GITHUB_OUTPUT build_release_image: - needs: collect_versions + needs: build strategy: - matrix: ${{ fromJson(needs.collect_versions.outputs.matrix) }} - runs-on: ${{ matrix.runner }} + matrix: + version: ${{ fromJson(needs.build.outputs.versions) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && fromJson('["self-hosted", "X64"]') || 'arm-runner' }} timeout-minutes: 180 steps: - uses: actions/checkout@v3 @@ -182,7 +152,7 @@ jobs: needs: [build, build_release_image] strategy: matrix: - image_tag: ${{ fromJson(needs.collect_versions.outputs.matrix) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 From d3392e09dde6df92b66ef145082624f641f4f06d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Dec 2024 16:45:42 -0500 Subject: [PATCH 17/97] chore : fix output versions --- .../workflows/dockerhub-release-matrix.yml | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 8f05317fec..8693036760 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -99,14 +99,36 @@ jobs: echo "${{ matrix.version }}" >> /tmp/versions/all_versions.txt echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" >> /tmp/versions/all_tags.txt - # Convert to JSON arrays with unique values - VERSIONS=$(sort -u /tmp/versions/all_versions.txt | jq -R . | jq -s .) - TAGS=$(sort -u /tmp/versions/all_tags.txt | jq -R . | jq -s .) + # Create JSON arrays with manual string construction + versions_array="[" + first=true + while IFS= read -r version; do + if [ "$first" = true ]; then + first=false + else + versions_array+="," + fi + versions_array+="\"$version\"" + done < <(sort -u /tmp/versions/all_versions.txt) + versions_array+="]" - # Set outputs - echo "versions=$VERSIONS" >> $GITHUB_OUTPUT - echo "image_tags=$TAGS" >> $GITHUB_OUTPUT - + tags_array="[" + first=true + while IFS= read -r tag; do + if [ "$first" = true ]; then + first=false + else + tags_array+="," + fi + tags_array+="\"$tag\"" + done < <(sort -u /tmp/versions/all_tags.txt) + tags_array+="]" + + # Set outputs with proper delimiter + { + echo "versions=${versions_array}" + echo "image_tags=${tags_array}" + } >> $GITHUB_OUTPUT build_release_image: needs: build strategy: From 33f8ae3bc1588a3cfb9e8f307a79dd119a970ba8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 09:49:37 -0500 Subject: [PATCH 18/97] chore: matrix in all tasks --- .github/workflows/dockerhub-release-matrix.yml | 7 +++++-- Dockerfile-15 | 2 ++ Dockerfile-orioledb-17 | 6 ++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 8693036760..ae5c720c24 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -199,7 +199,10 @@ jobs: publish: needs: [build, merge_manifest] + strategy: + matrix: + version: ${{ fromJson(needs.build.outputs.versions) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ needs.build.outputs.docker_version }} - secrets: inherit + version: ${{ matrix.version }} + secrets: inherit \ No newline at end of file diff --git a/Dockerfile-15 b/Dockerfile-15 index a88f6cc9fe..52c4e5af3e 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -75,7 +75,9 @@ WORKDIR / RUN mkdir -p /usr/lib/postgresql/bin \ /usr/lib/postgresql/share/postgresql \ /usr/share/postgresql \ + /var/lib/postgresql \ && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql # Create symbolic links diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index cc07c0fe1f..03b360748b 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -76,10 +76,8 @@ WORKDIR / RUN mkdir -p /usr/lib/postgresql/bin \ /usr/lib/postgresql/share/postgresql \ /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ + /var/lib/postgresql \ + && chown -R postgres:postgres /usr/lib/postgresql \ && chown -R postgres:postgres /usr/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql From ec1b4ac2c2ccee0c109103d4ac312925c07fae3b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 10:13:11 -0500 Subject: [PATCH 19/97] chore: refactor matrix handling --- .../workflows/dockerhub-release-matrix.yml | 80 +++++++++++-------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index ae5c720c24..060db592bc 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,8 +61,8 @@ jobs: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.output-versions.outputs.versions }} - image_tags: ${{ steps.output-versions.outputs.image_tags }} + versions: ${{ steps.combine-outputs.outputs.versions }} + image_tags: ${{ steps.combine-outputs.outputs.image_tags }} build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 @@ -87,48 +87,58 @@ jobs: with: cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - id: output-versions - if: always() + # Create workflow artifact with this matrix run's version info + - name: Save version info run: | - # Create output files if they don't exist - mkdir -p /tmp/versions - touch /tmp/versions/all_versions.txt - touch /tmp/versions/all_tags.txt - - # Append current version and tag - echo "${{ matrix.version }}" >> /tmp/versions/all_versions.txt - echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" >> /tmp/versions/all_tags.txt - - # Create JSON arrays with manual string construction + mkdir -p ./versions + echo "${{ matrix.version }}" > ./versions/version.txt + echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" > ./versions/tag.txt + + - uses: actions/upload-artifact@v3 + with: + name: version-info-${{ matrix.version }} + path: ./versions/ + + # Only run in first matrix job to combine all outputs + - if: matrix.version == fromJson(needs.prepare.outputs.matrix_config).include[0].version + id: combine-outputs + run: | + # Wait for other matrix jobs to complete by sleeping briefly + sleep 15 + + # Create arrays to hold all versions and tags versions_array="[" - first=true - while IFS= read -r version; do - if [ "$first" = true ]; then - first=false - else - versions_array+="," - fi - versions_array+="\"$version\"" - done < <(sort -u /tmp/versions/all_versions.txt) - versions_array+="]" - tags_array="[" first=true - while IFS= read -r tag; do + + # For each version in the matrix config + for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -c '.include[]'); do + version=$(echo $row | jq -r '.version') + if [ "$first" = true ]; then first=false else + versions_array+="," tags_array+="," fi + + # Download and read artifacts + mkdir -p ./download + echo "Processing version $version" + + tag=$(cat ../version-info-${version}/tag.txt) + + versions_array+="\"$version\"" tags_array+="\"$tag\"" - done < <(sort -u /tmp/versions/all_tags.txt) + done + + versions_array+="]" tags_array+="]" - - # Set outputs with proper delimiter - { - echo "versions=${versions_array}" - echo "image_tags=${tags_array}" - } >> $GITHUB_OUTPUT + + # Set outputs + echo "versions=$versions_array" >> $GITHUB_OUTPUT + echo "image_tags=$tags_array" >> $GITHUB_OUTPUT + build_release_image: needs: build strategy: @@ -197,7 +207,7 @@ jobs: SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" - publish: +publish: needs: [build, merge_manifest] strategy: matrix: @@ -205,4 +215,4 @@ jobs: uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} - secrets: inherit \ No newline at end of file + secrets: inherit From cf0702c87dad00bb3736677e95a3a7f4832e99f9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 10:21:12 -0500 Subject: [PATCH 20/97] chore: fix syntax --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 060db592bc..cf68fe8ecb 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -207,7 +207,7 @@ jobs: SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" -publish: + publish: needs: [build, merge_manifest] strategy: matrix: From 7b5c41971380e9e6f656d2e02474e64d10173fb5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 10:28:12 -0500 Subject: [PATCH 21/97] chore: look in the right dir --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index cf68fe8ecb..cb5c360ff3 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -126,7 +126,7 @@ jobs: mkdir -p ./download echo "Processing version $version" - tag=$(cat ../version-info-${version}/tag.txt) + tag=$(cat ./versions/tag.txt) versions_array+="\"$version\"" tags_array+="\"$tag\"" From 4395c35175692abbe78eef27e42e1082535e2d36 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 13:28:10 -0500 Subject: [PATCH 22/97] chore: need to pass full tag --- .github/workflows/dockerhub-release-matrix.yml | 6 +++--- Dockerfile-orioledb-17 | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index cb5c360ff3..85893ed26a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -207,12 +207,12 @@ jobs: SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" - publish: +publish: needs: [build, merge_manifest] strategy: matrix: - version: ${{ fromJson(needs.build.outputs.versions) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ matrix.version }} + version: ${{ matrix.image_tag }} secrets: inherit diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 03b360748b..d273b8482c 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -94,11 +94,6 @@ RUN chown -R postgres:postgres /usr/share/postgresql/ # Create symbolic links for contrib directory RUN tree /nix > /tmp/tree.txt && cat /tmp/tree.txt && cat /tmp/tree.txt >&2 - -# RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ -# && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ -# && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ - RUN chown -R postgres:postgres /usr/lib/postgresql RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets From f7b36dc31aa390d76c90cf9341155c344428a4b3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 13:34:19 -0500 Subject: [PATCH 23/97] fix: formatting --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 85893ed26a..cfc90c7c2a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -207,7 +207,7 @@ jobs: SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" -publish: + publish: needs: [build, merge_manifest] strategy: matrix: From 2a5b6a448773922f5aa0cae7685edc4a2eb40c80 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 16:35:48 -0500 Subject: [PATCH 24/97] chore: strip the prefix when passing to mirror.yml --- .github/workflows/dockerhub-release-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index cfc90c7c2a..4c305356bc 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -211,8 +211,8 @@ jobs: needs: [build, merge_manifest] strategy: matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ matrix.image_tag }} + version: ${{ replace(matrix.image_tag, 'supabase/postgres:', '') }} secrets: inherit From 761d019d38f2369f7a7524789fd1e9888e20cf7c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 16:49:57 -0500 Subject: [PATCH 25/97] chore: get version formatted to pass to mirror.yml --- .github/workflows/dockerhub-release-matrix.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 4c305356bc..dc4d60fa70 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -207,12 +207,23 @@ jobs: SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" - publish: + get_publish_version: needs: [build, merge_manifest] strategy: matrix: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.version }} + steps: + - id: get_version + run: | + VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') + echo "publish_version=$VERSION" >> $GITHUB_OUTPUT + + publish: + needs: get_puglixh_version uses: ./.github/workflows/mirror.yml with: - version: ${{ replace(matrix.image_tag, 'supabase/postgres:', '') }} + version: ${{ needs.get_version.outputs.publish_version }} secrets: inherit From 9f7ee62c592b2329bae30ba78f8cd953c3432a48 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 17:02:37 -0500 Subject: [PATCH 26/97] fix: typo --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index dc4d60fa70..0720c68e8a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -222,7 +222,7 @@ jobs: echo "publish_version=$VERSION" >> $GITHUB_OUTPUT publish: - needs: get_puglixh_version + needs: get_publish_version uses: ./.github/workflows/mirror.yml with: version: ${{ needs.get_version.outputs.publish_version }} From 94c32a579a9515fd8baf153c1e87d3951744b93b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Dec 2024 19:04:09 -0500 Subject: [PATCH 27/97] chore: fix vars --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 0720c68e8a..9a5a3a42e9 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -214,7 +214,7 @@ jobs: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest outputs: - version: ${{ steps.get_version.outputs.version }} + version: ${{ steps.get_version.outputs.publish_version }} steps: - id: get_version run: | From 2c94c9c0d0a0c7b440488ee1d763a59969d61c20 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 10:54:40 -0500 Subject: [PATCH 28/97] fix: tryo to pass the version --- .github/workflows/dockerhub-release-matrix.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 9a5a3a42e9..725d07f885 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -214,16 +214,18 @@ jobs: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest outputs: - version: ${{ steps.get_version.outputs.publish_version }} + version: ${{ steps.get_version.outputs.version }} steps: - id: get_version run: | VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') - echo "publish_version=$VERSION" >> $GITHUB_OUTPUT + # Changed to match the output name expected by the publish job + echo "Extracted version: $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT publish: needs: get_publish_version uses: ./.github/workflows/mirror.yml with: - version: ${{ needs.get_version.outputs.publish_version }} + version: ${{ needs.get_publish_version.outputs.version }} secrets: inherit From 4dbac71459e86ad06c46f8f21ed5c668f5c99551 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 12:25:17 -0500 Subject: [PATCH 29/97] chore: matrix on version and arch --- .github/workflows/dockerhub-release-matrix.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 725d07f885..54c44fb8e4 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -185,6 +185,7 @@ jobs: strategy: matrix: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + version: ${{ fromJson(needs.build.outputs.versions) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 @@ -211,17 +212,14 @@ jobs: needs: [build, merge_manifest] strategy: matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + version: ${{ fromJson(needs.build.outputs.versions) }} runs-on: ubuntu-latest outputs: version: ${{ steps.get_version.outputs.version }} steps: - id: get_version run: | - VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') - # Changed to match the output name expected by the publish job - echo "Extracted version: $VERSION" - echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "version=${{ matrix.version }}" >> $GITHUB_OUTPUT publish: needs: get_publish_version From 16788ef9f3769ac394c85ffd595d6f8b51a3a04d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 13:33:21 -0500 Subject: [PATCH 30/97] chore: try to account for orioledb version issue --- .github/workflows/dockerhub-release-matrix.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 54c44fb8e4..a2b589f8ef 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -212,14 +212,26 @@ jobs: needs: [build, merge_manifest] strategy: matrix: - version: ${{ fromJson(needs.build.outputs.versions) }} + image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest outputs: - version: ${{ steps.get_version.outputs.version }} + version: ${{ steps.get_version.outputs.version }} steps: + - uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + - id: get_version run: | - echo "version=${{ matrix.version }}" >> $GITHUB_OUTPUT + # Get the full version from vars.yml for orioledb + if [[ "${{ matrix.image_tag }}" == *"orioledb"* ]]; then + VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) + VERSION=$(echo $VERSION | tr -d '"') # Remove quotes + else + # For regular postgres, get version from the image tag + VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT publish: needs: get_publish_version From de3b59d0115c541723c9340f015bac8ff217e738 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 14:18:26 -0500 Subject: [PATCH 31/97] chore: remove version --- .github/workflows/dockerhub-release-matrix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index a2b589f8ef..d7008f1ef3 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -185,7 +185,6 @@ jobs: strategy: matrix: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} - version: ${{ fromJson(needs.build.outputs.versions) }} runs-on: ubuntu-latest steps: - uses: docker/setup-buildx-action@v3 @@ -205,7 +204,7 @@ jobs: SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} SLACK_USERNAME: "gha-failures-notifier" SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" + SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.image_tag }}" SLACK_FOOTER: "" get_publish_version: From 9ef1aae3aaac6c3e6524df907cb8bea160c01cf2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 14:51:29 -0500 Subject: [PATCH 32/97] chore: reading the version info directly from vars.yml --- .github/workflows/dockerhub-release-matrix.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index d7008f1ef3..1a5c152248 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -121,15 +121,17 @@ jobs: versions_array+="," tags_array+="," fi - - # Download and read artifacts - mkdir -p ./download - echo "Processing version $version" - - tag=$(cat ./versions/tag.txt) + + # Get version from vars.yml using same logic as common-nix.vars.pkr.hcl + if [[ "$version" == "orioledb-17" ]]; then + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) + else + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) + fi + PG_VERSION=$(echo $PG_VERSION | tr -d '"') versions_array+="\"$version\"" - tags_array+="\"$tag\"" + tags_array+="\"supabase/postgres:$PG_VERSION\"" done versions_array+="]" From 18822263ad05456ad3fb72e4ac3efb9465d67ae8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:01:04 -0500 Subject: [PATCH 33/97] chore: force string context --- .github/workflows/dockerhub-release-matrix.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 1a5c152248..95a84bf804 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -112,7 +112,7 @@ jobs: first=true # For each version in the matrix config - for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -c '.include[]'); do + for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -rc '.include[]'); do version=$(echo $row | jq -r '.version') if [ "$first" = true ]; then @@ -129,7 +129,7 @@ jobs: PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) fi PG_VERSION=$(echo $PG_VERSION | tr -d '"') - + versions_array+="\"$version\"" tags_array+="\"supabase/postgres:$PG_VERSION\"" done @@ -137,7 +137,6 @@ jobs: versions_array+="]" tags_array+="]" - # Set outputs echo "versions=$versions_array" >> $GITHUB_OUTPUT echo "image_tags=$tags_array" >> $GITHUB_OUTPUT From 63c02225f32d49d0bd6ebab79b5f240c8fe57aea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:10:40 -0500 Subject: [PATCH 34/97] chore: fixing logic --- .github/workflows/dockerhub-release-matrix.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 95a84bf804..cab8fc8c39 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -99,20 +99,18 @@ jobs: name: version-info-${{ matrix.version }} path: ./versions/ - # Only run in first matrix job to combine all outputs - if: matrix.version == fromJson(needs.prepare.outputs.matrix_config).include[0].version id: combine-outputs run: | # Wait for other matrix jobs to complete by sleeping briefly sleep 15 - # Create arrays to hold all versions and tags versions_array="[" tags_array="[" first=true # For each version in the matrix config - for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -rc '.include[]'); do + for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -cr '.include[]'); do version=$(echo $row | jq -r '.version') if [ "$first" = true ]; then @@ -122,18 +120,18 @@ jobs: tags_array+="," fi - # Get version from vars.yml using same logic as common-nix.vars.pkr.hcl + # Get version differently based on version type if [[ "$version" == "orioledb-17" ]]; then PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) else PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) fi PG_VERSION=$(echo $PG_VERSION | tr -d '"') - - versions_array+="\"$version\"" - tags_array+="\"supabase/postgres:$PG_VERSION\"" + + versions_array+="\"${version}\"" + tags_array+="\"supabase/postgres:${PG_VERSION}\"" done - + versions_array+="]" tags_array+="]" From 4ce2b4f103507e648b899138593f61b91f775727 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:22:02 -0500 Subject: [PATCH 35/97] chore: fix string handling logic --- .github/workflows/dockerhub-release-matrix.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index cab8fc8c39..1c48e56e19 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -74,7 +74,12 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) + # If version contains orioledb-, handle it differently + if [[ "${{ matrix.version }}" == *"orioledb-"* ]]; then + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) + else + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) + fi PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo "" >> common-nix.vars.pkr.hcl @@ -121,7 +126,7 @@ jobs: fi # Get version differently based on version type - if [[ "$version" == "orioledb-17" ]]; then + if [[ "$version" == *"orioledb-"* ]]; then PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) else PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) From d80e8b3982723b8b220483722e8615665e79c96b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:26:58 -0500 Subject: [PATCH 36/97] chore: use sed and grep instead --- .github/workflows/dockerhub-release-matrix.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 1c48e56e19..c0430aed96 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -114,10 +114,12 @@ jobs: tags_array="[" first=true - # For each version in the matrix config - for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -cr '.include[]'); do - version=$(echo $row | jq -r '.version') - + # Extract versions directly from matrix config using grep/sed + matrix_json='${{ needs.prepare.outputs.matrix_config }}' + versions=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/"version":"\(.*\)"/\1/') + + # For each version + while read -r version; do if [ "$first" = true ]; then first=false else @@ -135,7 +137,7 @@ jobs: versions_array+="\"${version}\"" tags_array+="\"supabase/postgres:${PG_VERSION}\"" - done + done <<< "$versions" versions_array+="]" tags_array+="]" From a935b19caa2693552d2e19d48df5d39782f04bce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:36:36 -0500 Subject: [PATCH 37/97] chore: revert this to working version --- .github/workflows/dockerhub-release-matrix.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index c0430aed96..54f041e1b9 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -74,12 +74,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - # If version contains orioledb-, handle it differently - if [[ "${{ matrix.version }}" == *"orioledb-"* ]]; then - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) - else - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) - fi + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo "" >> common-nix.vars.pkr.hcl From a2323c2f9704d914e23ef241dae3956261fa187d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:43:45 -0500 Subject: [PATCH 38/97] chore: version extraction --- .../workflows/dockerhub-release-matrix.yml | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 54f041e1b9..b26fd06af4 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -109,12 +109,17 @@ jobs: tags_array="[" first=true - # Extract versions directly from matrix config using grep/sed + # Extract versions using grep and sed matrix_json='${{ needs.prepare.outputs.matrix_config }}' - versions=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/"version":"\(.*\)"/\1/') + versions=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/.*:"//;s/"//') # For each version while read -r version; do + # Skip empty versions + if [ -z "$version" ]; then + continue + fi + if [ "$first" = true ]; then first=false else @@ -128,15 +133,25 @@ jobs: else PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) fi - PG_VERSION=$(echo $PG_VERSION | tr -d '"') - versions_array+="\"${version}\"" - tags_array+="\"supabase/postgres:${PG_VERSION}\"" + # Ensure PG_VERSION is not empty and remove quotes + if [ -n "$PG_VERSION" ]; then + PG_VERSION=$(echo $PG_VERSION | tr -d '"') + + versions_array+="\"${version}\"" + tags_array+="\"supabase/postgres:${PG_VERSION}\"" + fi done <<< "$versions" versions_array+="]" tags_array+="]" + # Validate arrays are not empty + if [ "$versions_array" = "[]" ] || [ "$tags_array" = "[]" ]; then + echo "Error: No valid versions found" + exit 1 + fi + echo "versions=$versions_array" >> $GITHUB_OUTPUT echo "image_tags=$tags_array" >> $GITHUB_OUTPUT From c9ff94f4ca49b8e73f3a00660bf78deb5385d512 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:48:54 -0500 Subject: [PATCH 39/97] chore: different logic on version processing if stmnt --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index b26fd06af4..67a5901557 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -99,7 +99,7 @@ jobs: name: version-info-${{ matrix.version }} path: ./versions/ - - if: matrix.version == fromJson(needs.prepare.outputs.matrix_config).include[0].version + - if: contains(needs.prepare.outputs.matrix_config, format('"{0}"', matrix.version)) id: combine-outputs run: | # Wait for other matrix jobs to complete by sleeping briefly From baa41b09aba88d0e02af02d724352e0e2406670a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 15:56:41 -0500 Subject: [PATCH 40/97] chore: try to avoid jq involvement in this section --- .github/workflows/dockerhub-release-matrix.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 67a5901557..8011dcb5e5 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -99,18 +99,27 @@ jobs: name: version-info-${{ matrix.version }} path: ./versions/ - - if: contains(needs.prepare.outputs.matrix_config, format('"{0}"', matrix.version)) - id: combine-outputs + - id: combine-outputs run: | # Wait for other matrix jobs to complete by sleeping briefly sleep 15 + # First verify if this is the controlling job + matrix_json='${{ needs.prepare.outputs.matrix_config }}' + first_version=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/.*:"//;s/"//' | head -1) + current_version="${{ matrix.version }}" + + # Only proceed if this is the controlling job (matches first version) + if [ "$current_version" != "$first_version" ]; then + echo "This is not the controlling job. Skipping." + exit 0 + fi + versions_array="[" tags_array="[" first=true # Extract versions using grep and sed - matrix_json='${{ needs.prepare.outputs.matrix_config }}' versions=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/.*:"//;s/"//') # For each version From f65c5e68fa7af579f303658c82e509e1b2453bd3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 17 Dec 2024 20:26:52 -0500 Subject: [PATCH 41/97] chore: back to this version --- .../workflows/dockerhub-release-matrix.yml | 89 +++++-------------- 1 file changed, 23 insertions(+), 66 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 8011dcb5e5..d4d6e3a7ad 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -54,7 +54,6 @@ jobs: # Output the matrix configuration echo "matrix_config=$MATRIX_CONFIG" >> $GITHUB_OUTPUT - build: needs: prepare strategy: @@ -78,7 +77,6 @@ jobs: PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl echo "" >> common-nix.vars.pkr.hcl - - id: settings run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT @@ -93,77 +91,46 @@ jobs: mkdir -p ./versions echo "${{ matrix.version }}" > ./versions/version.txt echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" > ./versions/tag.txt - - uses: actions/upload-artifact@v3 with: name: version-info-${{ matrix.version }} path: ./versions/ - - id: combine-outputs + # Only run in first matrix job to combine all outputs + - if: matrix.version == fromJson(needs.prepare.outputs.matrix_config).include[0].version + id: combine-outputs run: | # Wait for other matrix jobs to complete by sleeping briefly sleep 15 - - # First verify if this is the controlling job - matrix_json='${{ needs.prepare.outputs.matrix_config }}' - first_version=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/.*:"//;s/"//' | head -1) - current_version="${{ matrix.version }}" - - # Only proceed if this is the controlling job (matches first version) - if [ "$current_version" != "$first_version" ]; then - echo "This is not the controlling job. Skipping." - exit 0 - fi - + # Create arrays to hold all versions and tags versions_array="[" tags_array="[" first=true - - # Extract versions using grep and sed - versions=$(echo "$matrix_json" | grep -o '"version":"[^"]*"' | sed 's/.*:"//;s/"//') - - # For each version - while read -r version; do - # Skip empty versions - if [ -z "$version" ]; then - continue - fi - + # For each version in the matrix config + for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -c '.include[]'); do + version=$(echo $row | jq -r '.version') + if [ "$first" = true ]; then first=false else versions_array+="," tags_array+="," fi - - # Get version differently based on version type - if [[ "$version" == *"orioledb-"* ]]; then - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) - else - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'$version'"]' ansible/vars.yml) - fi - # Ensure PG_VERSION is not empty and remove quotes - if [ -n "$PG_VERSION" ]; then - PG_VERSION=$(echo $PG_VERSION | tr -d '"') - - versions_array+="\"${version}\"" - tags_array+="\"supabase/postgres:${PG_VERSION}\"" - fi - done <<< "$versions" - + # Download and read artifacts + mkdir -p ./download + echo "Processing version $version" + + tag=$(cat ./versions/tag.txt) + + versions_array+="\"$version\"" + tags_array+="\"$tag\"" + done versions_array+="]" tags_array+="]" - - # Validate arrays are not empty - if [ "$versions_array" = "[]" ] || [ "$tags_array" = "[]" ]; then - echo "Error: No valid versions found" - exit 1 - fi - + # Set outputs echo "versions=$versions_array" >> $GITHUB_OUTPUT echo "image_tags=$tags_array" >> $GITHUB_OUTPUT - build_release_image: needs: build strategy: @@ -229,7 +196,7 @@ jobs: SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} SLACK_USERNAME: "gha-failures-notifier" SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.image_tag }}" + SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" SLACK_FOOTER: "" get_publish_version: @@ -239,24 +206,14 @@ jobs: image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} runs-on: ubuntu-latest outputs: - version: ${{ steps.get_version.outputs.version }} + version: ${{ steps.get_version.outputs.version }} steps: - - uses: actions/checkout@v3 - - - uses: DeterminateSystems/nix-installer-action@main - - id: get_version run: | - # Get the full version from vars.yml for orioledb - if [[ "${{ matrix.image_tag }}" == *"orioledb"* ]]; then - VERSION=$(nix run nixpkgs#yq -- '.postgres_release.postgresorioledb-17' ansible/vars.yml) - VERSION=$(echo $VERSION | tr -d '"') # Remove quotes - else - # For regular postgres, get version from the image tag - VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') - fi + VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') + # Changed to match the output name expected by the publish job + echo "Extracted version: $VERSION" echo "version=$VERSION" >> $GITHUB_OUTPUT - publish: needs: get_publish_version uses: ./.github/workflows/mirror.yml From ea2bc3b8d0b0c89169f07e054218caa894cdd9d1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 14:29:45 -0500 Subject: [PATCH 42/97] chore: refactor using nushell where scripting needed --- .../workflows/dockerhub-release-matrix.yml | 218 ++++++++---------- 1 file changed, 97 insertions(+), 121 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index d4d6e3a7ad..d36f3a5e8b 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -19,124 +19,69 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main + - name: Install Nushell + run: | + sudo apt update + sudo apt install -y nushell - name: Generate build matrix id: set-matrix run: | - # Get all postgres versions from vars.yml - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml) - - # Create matrix config - MATRIX_CONFIG="{" - MATRIX_CONFIG+="\"include\":[" - FIRST=true - - while IFS= read -r version; do - # Remove quotes from version - version=$(echo "$version" | tr -d '"') - - # Simply look for Dockerfile-{version} - dockerfile="Dockerfile-${version}" - - # Check if Dockerfile exists - if [ -f "$dockerfile" ]; then - if [ "$FIRST" = true ]; then - FIRST=false - else - MATRIX_CONFIG+="," - fi - MATRIX_CONFIG+="{\"version\":\"$version\",\"dockerfile\":\"$dockerfile\"}" - fi - done <<< "$VERSIONS" - - MATRIX_CONFIG+="]}" - - # Output the matrix configuration - echo "matrix_config=$MATRIX_CONFIG" >> $GITHUB_OUTPUT + nu -c 'let versions = (open ansible/vars.yml | get postgres_major) + let matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let dockerfile = $"Dockerfile-($version)" + if ($dockerfile | path exists) { + { + version: $version, + dockerfile: $dockerfile + } + } else { + null + } + } | compact) + + let matrix_config = { + include: $matrix + } + + "matrix_config=" + ($matrix_config | to json) | save --append $env.GITHUB_OUTPUT' build: needs: prepare strategy: matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.combine-outputs.outputs.versions }} - image_tags: ${{ steps.combine-outputs.outputs.image_tags }} build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 - - - uses: DeterminateSystems/nix-installer-action@main - + - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.version }}"]' ansible/vars.yml) - PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes - echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo "" >> common-nix.vars.pkr.hcl + nu -c 'let pg_version = (open ansible/vars.yml | get postgres_release | get $"postgres${{ matrix.version }}" | str trim) + $"postgres-version = \"($pg_version)\"" | save common-nix.vars.pkr.hcl + "" | save --append common-nix.vars.pkr.hcl' - id: settings - run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - # Create workflow artifact with this matrix run's version info - - name: Save version info run: | - mkdir -p ./versions - echo "${{ matrix.version }}" > ./versions/version.txt - echo "supabase/postgres:${{ steps.settings.outputs.postgres-version }}" > ./versions/tag.txt - - uses: actions/upload-artifact@v3 - with: - name: version-info-${{ matrix.version }} - path: ./versions/ - - # Only run in first matrix job to combine all outputs - - if: matrix.version == fromJson(needs.prepare.outputs.matrix_config).include[0].version - id: combine-outputs + nu -c 'open common-nix.vars.pkr.hcl | str replace -a "[\\s\"]+" "" | save --append $env.GITHUB_OUTPUT' + - id: args run: | - # Wait for other matrix jobs to complete by sleeping briefly - sleep 15 - # Create arrays to hold all versions and tags - versions_array="[" - tags_array="[" - first=true - # For each version in the matrix config - for row in $(echo '${{ needs.prepare.outputs.matrix_config }}' | jq -c '.include[]'); do - version=$(echo $row | jq -r '.version') - - if [ "$first" = true ]; then - first=false - else - versions_array+="," - tags_array+="," - fi - - # Download and read artifacts - mkdir -p ./download - echo "Processing version $version" - - tag=$(cat ./versions/tag.txt) - - versions_array+="\"$version\"" - tags_array+="\"$tag\"" - done - versions_array+="]" - tags_array+="]" - # Set outputs - echo "versions=$versions_array" >> $GITHUB_OUTPUT - echo "image_tags=$tags_array" >> $GITHUB_OUTPUT + nu -c ' + open ansible/vars.yml + | items { |key value| {name: $key, item: $value} } + | where { |it| ($it.item | describe) == "string" } + | each { |it| $"($it.name)=($it.item)" } + | str join "\n" + | save --append $env.GITHUB_OUTPUT + ' build_release_image: - needs: build + needs: [prepare, build] strategy: matrix: - version: ${{ fromJson(needs.build.outputs.versions) }} - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && fromJson('["self-hosted", "X64"]') || 'arm-runner' }} timeout-minutes: 180 @@ -150,6 +95,18 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + run: | + nu -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgres($version | str replace "-" "")" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"supabase/postgres:($pg_version)" | save --append $env.GITHUB_ENV + ' - id: build uses: docker/build-push-action@v5 with: @@ -157,11 +114,11 @@ jobs: build-args: | ${{ needs.build.outputs.build_args }} target: production - tags: ${{ matrix.image_tag }}_${{ matrix.arch }} + tags: ${{ env.pg_version }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: Dockerfile-${{ matrix.version }} + file: ${{ matrix.dockerfile }} - name: Slack Notification if: ${{ failure() }} uses: rtCamp/action-slack-notify@v2 @@ -173,50 +130,69 @@ jobs: SLACK_FOOTER: "" merge_manifest: - needs: [build, build_release_image] + needs: [prepare, build, build_release_image] strategy: matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + run: | + nu -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgres($version | str replace "-" "")" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"supabase/postgres:($pg_version)" | save --append $env.GITHUB_ENV + ' - name: Merge multi-arch manifests run: | - docker buildx imagetools create -t ${{ matrix.image_tag }} \ - ${{ matrix.image_tag }}_amd64 \ - ${{ matrix.image_tag }}_arm64 - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed for version ${{ matrix.version }}" - SLACK_FOOTER: "" - + docker buildx imagetools create -t ${{ env.pg_version }} \ + ${{ env.pg_version }}_amd64 \ + ${{ env.pg_version }}_arm64 get_publish_version: - needs: [build, merge_manifest] + needs: [prepare, merge_manifest] strategy: matrix: - image_tag: ${{ fromJson(needs.build.outputs.image_tags) }} + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: - version: ${{ steps.get_version.outputs.version }} + matrix: ${{ steps.get_versions.outputs.matrix }} steps: - - id: get_version + - uses: actions/checkout@v3 + - name: Get versions + id: get_versions run: | - VERSION=$(echo "${{ matrix.image_tag }}" | sed 's|supabase/postgres:||') - # Changed to match the output name expected by the publish job - echo "Extracted version: $VERSION" - echo "version=$VERSION" >> $GITHUB_OUTPUT + nu -c ' + let versions = ${{ fromJson(needs.prepare.outputs.matrix_config).include | select(.version) | to array }} + let versions_array = ($versions | each { |ver| + let version = $ver.version + let release_key = if ($version | str contains "orioledb") { + $"postgres($version | str replace "-" "")" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + {version: $pg_version} + }) + let matrix = {include: $versions_array} + $"matrix=($matrix | to json)" | save --append $env.GITHUB_OUTPUT + ' + publish: needs: get_publish_version + strategy: + matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ needs.get_publish_version.outputs.version }} - secrets: inherit + version: ${{ matrix.version }} + secrets: inherit \ No newline at end of file From 2c63e7ace1b1a388d044976d7924c60f4fe58b8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 14:38:52 -0500 Subject: [PATCH 43/97] chore: use data from previous step --- .../workflows/dockerhub-release-matrix.yml | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index d36f3a5e8b..701d34941c 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -159,6 +159,7 @@ jobs: docker buildx imagetools create -t ${{ env.pg_version }} \ ${{ env.pg_version }}_amd64 \ ${{ env.pg_version }}_arm64 + get_publish_version: needs: [prepare, merge_manifest] strategy: @@ -169,22 +170,18 @@ jobs: matrix: ${{ steps.get_versions.outputs.matrix }} steps: - uses: actions/checkout@v3 - - name: Get versions + - name: Get version id: get_versions run: | nu -c ' - let versions = ${{ fromJson(needs.prepare.outputs.matrix_config).include | select(.version) | to array }} - let versions_array = ($versions | each { |ver| - let version = $ver.version - let release_key = if ($version | str contains "orioledb") { - $"postgres($version | str replace "-" "")" - } else { - $"postgres($version)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - {version: $pg_version} - }) - let matrix = {include: $versions_array} + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgres($version | str replace "-" "")" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let matrix = {include: [{version: $pg_version}]} $"matrix=($matrix | to json)" | save --append $env.GITHUB_OUTPUT ' From 43ae0dcaba4ed157508dfe527f1cd672cd49c1da Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 14:43:59 -0500 Subject: [PATCH 44/97] chore: install with snap --- .github/workflows/dockerhub-release-matrix.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 701d34941c..f088145235 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -21,8 +21,7 @@ jobs: - name: Install Nushell run: | - sudo apt update - sudo apt install -y nushell + sudo snap install nushell --classic - name: Generate build matrix id: set-matrix @@ -55,6 +54,10 @@ jobs: build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 + + - name: Install Nushell + run: | + sudo snap install nushell --classic - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV @@ -87,6 +90,9 @@ jobs: timeout-minutes: 180 steps: - uses: actions/checkout@v3 + - name: Install Nushell + run: | + sudo snap install nushell --classic - run: docker context create builders - uses: docker/setup-buildx-action@v3 with: @@ -137,6 +143,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Install Nushell + run: | + sudo snap install nushell --classic - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 with: @@ -170,6 +179,9 @@ jobs: matrix: ${{ steps.get_versions.outputs.matrix }} steps: - uses: actions/checkout@v3 + - name: Install Nushell + run: | + sudo snap install nushell --classic - name: Get version id: get_versions run: | @@ -192,4 +204,4 @@ jobs: uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} - secrets: inherit \ No newline at end of file + secrets: inherit From b91e4a7dd18341ec8ec505f8acf6cc2245a79111 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:02:12 -0500 Subject: [PATCH 45/97] chore: raw output on json --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index f088145235..61dd1b6049 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -44,7 +44,7 @@ jobs: include: $matrix } - "matrix_config=" + ($matrix_config | to json) | save --append $env.GITHUB_OUTPUT' + $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' build: needs: prepare strategy: From 4454104e938e885f3b4b5327cc274c91d9c2434c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:07:34 -0500 Subject: [PATCH 46/97] chore: should be able to run on ephemeral runners --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 61dd1b6049..78b0976b11 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -86,7 +86,7 @@ jobs: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && fromJson('["self-hosted", "X64"]') || 'arm-runner' }} + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-latest-arm64' }} timeout-minutes: 180 steps: - uses: actions/checkout@v3 From cd65a52668ced3359bb7cce7843b1a0bd4a6f33c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:22:01 -0500 Subject: [PATCH 47/97] chore: incrementing changes to matrix --- .../workflows/dockerhub-release-matrix.yml | 28 ++++--------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 78b0976b11..34ec67040b 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -61,15 +61,7 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV - - - name: Generate common-nix.vars.pkr.hcl - run: | - nu -c 'let pg_version = (open ansible/vars.yml | get postgres_release | get $"postgres${{ matrix.version }}" | str trim) - $"postgres-version = \"($pg_version)\"" | save common-nix.vars.pkr.hcl - "" | save --append common-nix.vars.pkr.hcl' - - id: settings - run: | - nu -c 'open common-nix.vars.pkr.hcl | str replace -a "[\\s\"]+" "" | save --append $env.GITHUB_OUTPUT' + - id: args run: | nu -c ' @@ -79,12 +71,12 @@ jobs: | each { |it| $"($it.name)=($it.item)" } | str join "\n" | save --append $env.GITHUB_OUTPUT - ' + ' build_release_image: needs: [prepare, build] strategy: matrix: - include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-latest-arm64' }} timeout-minutes: 180 @@ -104,7 +96,7 @@ jobs: - name: Get image tag run: | nu -c ' - let version = "${{ matrix.version }}" + let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgres($version | str replace "-" "")" } else { @@ -124,17 +116,7 @@ jobs: platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.dockerfile }} - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed for version ${{ matrix.version }}" - SLACK_FOOTER: "" - + file: ${{ matrix.postgres.dockerfile }} merge_manifest: needs: [prepare, build, build_release_image] strategy: From 66b964c8e31ddcbe8ce685ad1cdce318b6192bb3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:26:23 -0500 Subject: [PATCH 48/97] chore: just use special key name if present --- .github/workflows/dockerhub-release-matrix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 34ec67040b..e487187b80 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -96,15 +96,15 @@ jobs: - name: Get image tag run: | nu -c ' - let version = "${{ matrix.postgres.version }}" + let version = "${{ matrix.version }}" let release_key = if ($version | str contains "orioledb") { - $"postgres($version | str replace "-" "")" + $"postgresorioledb-17" # Use exact key name from vars.yml } else { $"postgres($version)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) $"supabase/postgres:($pg_version)" | save --append $env.GITHUB_ENV - ' + ' - id: build uses: docker/build-push-action@v5 with: From 75d2a56aef9520b58d95cb3c888dd683e787f6bb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:36:45 -0500 Subject: [PATCH 49/97] chore: account for postgres namespace in matrix --- .github/workflows/dockerhub-release-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index e487187b80..5e93b9546b 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -96,9 +96,9 @@ jobs: - name: Get image tag run: | nu -c ' - let version = "${{ matrix.version }}" + let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { - $"postgresorioledb-17" # Use exact key name from vars.yml + $"postgresorioledb-17" } else { $"postgres($version)" } From faa565a2f566bbcd87f7ec63fdcd31bacff15198 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 15:49:53 -0500 Subject: [PATCH 50/97] chore: pg_version should be key/value --- .github/workflows/dockerhub-release-matrix.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 5e93b9546b..90444b0ea6 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -94,6 +94,7 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag + id: image run: | nu -c ' let version = "${{ matrix.postgres.version }}" @@ -103,7 +104,7 @@ jobs: $"postgres($version)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"supabase/postgres:($pg_version)" | save --append $env.GITHUB_ENV + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT ' - id: build uses: docker/build-push-action@v5 @@ -112,11 +113,11 @@ jobs: build-args: | ${{ needs.build.outputs.build_args }} target: production - tags: ${{ env.pg_version }}_${{ matrix.arch }} + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.postgres.dockerfile }} + file: ${{ matrix.postgres.dockerfile }} merge_manifest: needs: [prepare, build, build_release_image] strategy: From 1e46a6ed04fee78d55fd2aee6d30e64ffde88673 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 17:14:24 -0500 Subject: [PATCH 51/97] chrore: ubuntu latest --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 90444b0ea6..f98c319135 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -78,7 +78,7 @@ jobs: matrix: postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-latest-arm64' }} + runs-on: ubuntu-latest timeout-minutes: 180 steps: - uses: actions/checkout@v3 From 4168531d7caf54439689a5aee881bbe1f5c9e370 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 17:22:05 -0500 Subject: [PATCH 52/97] chore: needs arm-runner after all --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index f98c319135..0099302ec7 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -78,7 +78,7 @@ jobs: matrix: postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] - runs-on: ubuntu-latest + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} timeout-minutes: 180 steps: - uses: actions/checkout@v3 From 030d3c8097fc3436ebee0efce615b8d807fb0a92 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 17:34:12 -0500 Subject: [PATCH 53/97] chore: source nushell from nix --- .../workflows/dockerhub-release-matrix.yml | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 0099302ec7..d8398d1678 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -18,15 +18,11 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 - - - name: Install Nushell - run: | - sudo snap install nushell --classic - - name: Generate build matrix + uses: DeterminateSystems/nix-installer-action@main id: set-matrix run: | - nu -c 'let versions = (open ansible/vars.yml | get postgres_major) + nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) let matrix = ($versions | each { |ver| let version = ($ver | str trim) let dockerfile = $"Dockerfile-($version)" @@ -54,17 +50,13 @@ jobs: build_args: ${{ steps.args.outputs.result }} steps: - uses: actions/checkout@v3 - - - name: Install Nushell - run: | - sudo snap install nushell --classic - + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV - id: args run: | - nu -c ' + nix run nixpkgs#nushell -- -c ' open ansible/vars.yml | items { |key value| {name: $key, item: $value} } | where { |it| ($it.item | describe) == "string" } @@ -82,9 +74,7 @@ jobs: timeout-minutes: 180 steps: - uses: actions/checkout@v3 - - name: Install Nushell - run: | - sudo snap install nushell --classic + - uses: DeterminateSystems/nix-installer-action@main - run: docker context create builders - uses: docker/setup-buildx-action@v3 with: @@ -96,7 +86,7 @@ jobs: - name: Get image tag id: image run: | - nu -c ' + nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" @@ -126,9 +116,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Install Nushell - run: | - sudo snap install nushell --classic + - uses: DeterminateSystems/nix-installer-action@main - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 with: @@ -136,7 +124,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag run: | - nu -c ' + nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.version }}" let release_key = if ($version | str contains "orioledb") { $"postgres($version | str replace "-" "")" @@ -162,13 +150,11 @@ jobs: matrix: ${{ steps.get_versions.outputs.matrix }} steps: - uses: actions/checkout@v3 - - name: Install Nushell - run: | - sudo snap install nushell --classic + - uses: DeterminateSystems/nix-installer-action@main - name: Get version id: get_versions run: | - nu -c ' + nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.version }}" let release_key = if ($version | str contains "orioledb") { $"postgres($version | str replace "-" "")" From f03c65a53b445f9a4389e4df44b072f03c2d493a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 17:37:16 -0500 Subject: [PATCH 54/97] chore: cannot have uses and run keys on same id --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index d8398d1678..546a81b197 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -16,10 +16,10 @@ jobs: outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} steps: + - uses: DeterminateSystems/nix-installer-action@main - name: Checkout Repo uses: actions/checkout@v3 - name: Generate build matrix - uses: DeterminateSystems/nix-installer-action@main id: set-matrix run: | nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) From 413c7a4be739d99c0f3e91f2226d3d84679d0c9e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 17:48:00 -0500 Subject: [PATCH 55/97] chore: detect runner and adjust the way command is run --- .../workflows/dockerhub-release-matrix.yml | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 546a81b197..1606217316 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -86,28 +86,41 @@ jobs: - name: Get image tag id: image run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.postgres.version }}" - let release_key = if ($version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($version)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' - - id: build - uses: docker/build-push-action@v5 - with: - push: true - build-args: | - ${{ needs.build.outputs.build_args }} - target: production - tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.postgres.dockerfile }} + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + sudo nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + else + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + fi + - id: build + uses: docker/build-push-action@v5 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + target: production + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} merge_manifest: needs: [prepare, build, build_release_image] strategy: From 084b3002ff3f766ffae72bc5a9ef71cf7c8c9302 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 18:24:41 -0500 Subject: [PATCH 56/97] chore: formatting --- .../workflows/dockerhub-release-matrix.yml | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 1606217316..66bd44132b 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -109,18 +109,19 @@ jobs: $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT ' fi - - id: build - uses: docker/build-push-action@v5 - with: - push: true - build-args: | - ${{ needs.build.outputs.build_args }} - target: production - tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.postgres.dockerfile }} + + - id: build + uses: docker/build-push-action@v5 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + target: production + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} merge_manifest: needs: [prepare, build, build_release_image] strategy: From 126e974f4cedb902e607919774d47ce7dbe73ea7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 18 Dec 2024 18:38:46 -0500 Subject: [PATCH 57/97] chore: make sure env var can be appended in all contexts --- .github/workflows/dockerhub-release-matrix.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 66bd44132b..60da2db3ab 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -87,29 +87,28 @@ jobs: id: image run: | if [[ "${{ matrix.arch }}" == "arm64" ]]; then - sudo nix run nixpkgs#nushell -- -c ' + pg_version=$(sudo nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" } else { $"postgres($version)" } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' + open ansible/vars.yml | get postgres_release | get $release_key | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT else - nix run nixpkgs#nushell -- -c ' + pg_version=$(nix run nixpkgs#nushell -- -c ' let version = "${{ matrix.postgres.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" } else { $"postgres($version)" } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' + open ansible/vars.yml | get postgres_release | get $release_key | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT fi - - id: build uses: docker/build-push-action@v5 with: From 4a6db109ccba0edabe659ad5c36d6a8e047c9612 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 11:40:28 -0500 Subject: [PATCH 58/97] chore: outputs instead of env --- .github/workflows/dockerhub-release-matrix.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 60da2db3ab..f11c4ea564 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -136,23 +136,24 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag + id: get_version run: | nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" + let version = "15" let release_key = if ($version | str contains "orioledb") { $"postgres($version | str replace "-" "")" } else { $"postgres($version)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"supabase/postgres:($pg_version)" | save --append $env.GITHUB_ENV - ' + + $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT + ' - name: Merge multi-arch manifests run: | - docker buildx imagetools create -t ${{ env.pg_version }} \ - ${{ env.pg_version }}_amd64 \ - ${{ env.pg_version }}_arm64 - + docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 get_publish_version: needs: [prepare, merge_manifest] strategy: From ad254d5d6f9a41cdbbafec74ab8f045de403587d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 12:00:10 -0500 Subject: [PATCH 59/97] chore: fix get_publish_version --- .github/workflows/dockerhub-release-matrix.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index f11c4ea564..9a85c7449d 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -176,8 +176,9 @@ jobs: $"postgres($version)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - let matrix = {include: [{version: $pg_version}]} - $"matrix=($matrix | to json)" | save --append $env.GITHUB_OUTPUT + + # Ensure clean JSON formatting + $"matrix={json: {include: [{version: $pg_version}]}, format: compact}" | save --append $env.GITHUB_OUTPUT ' publish: From 8c777d0924298cb27a3c30aecaf6d161691db00d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 14:02:45 -0500 Subject: [PATCH 60/97] chore: handle oriole --- .../workflows/dockerhub-release-matrix.yml | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 9a85c7449d..f5713f12f9 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -168,18 +168,23 @@ jobs: - name: Get version id: get_versions run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" + version="${{ matrix.version }}" + release_key=$(nix run nixpkgs#nushell -- -c ' + let version = "'$version'" let release_key = if ($version | str contains "orioledb") { - $"postgres($version | str replace "-" "")" + $"postgresorioledb-17" } else { $"postgres($version)" } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - - # Ensure clean JSON formatting - $"matrix={json: {include: [{version: $pg_version}]}, format: compact}" | save --append $env.GITHUB_OUTPUT - ' + $release_key + ') + + pg_version=$(nix run nixpkgs#nushell -- -c " + let release_key = \"$release_key\" + open ansible/vars.yml | get postgres_release | get $release_key | str trim + ") + + echo "matrix=$(jq -n --arg version "$pg_version" '{include: [{version: $version}]}' | base64 -w0)" >> $GITHUB_OUTPUT publish: needs: get_publish_version From 98c9bee5f95671c6f406db204a865105c6ebb92c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 14:17:32 -0500 Subject: [PATCH 61/97] chore: remove base64 encoding --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index f5713f12f9..6dcbf82c8d 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -184,7 +184,7 @@ jobs: open ansible/vars.yml | get postgres_release | get $release_key | str trim ") - echo "matrix=$(jq -n --arg version "$pg_version" '{include: [{version: $version}]}' | base64 -w0)" >> $GITHUB_OUTPUT + echo "matrix=$(jq -n --arg version "$pg_version" '{include: [{version: $version}]}')" >> $GITHUB_OUTPUT publish: needs: get_publish_version From 613b45c45e6d584f684b65831be23e57d85e06a9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 14:44:48 -0500 Subject: [PATCH 62/97] ore: raw json --- .../workflows/dockerhub-release-matrix.yml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 6dcbf82c8d..b6ae3fb64c 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -168,23 +168,19 @@ jobs: - name: Get version id: get_versions run: | - version="${{ matrix.version }}" - release_key=$(nix run nixpkgs#nushell -- -c ' - let version = "'$version'" + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" let release_key = if ($version | str contains "orioledb") { $"postgresorioledb-17" } else { $"postgres($version)" } - $release_key - ') - - pg_version=$(nix run nixpkgs#nushell -- -c " - let release_key = \"$release_key\" - open ansible/vars.yml | get postgres_release | get $release_key | str trim - ") - - echo "matrix=$(jq -n --arg version "$pg_version" '{include: [{version: $version}]}')" >> $GITHUB_OUTPUT + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + + # Use to-json with compact formatting + let matrix = {include: [{version: $pg_version}]} + $"matrix=(${matrix | to json -r})" | save --append $env.GITHUB_OUTPUT + ' publish: needs: get_publish_version From 2c662f2fb6d12f770ed7d11b77f753088ed4d26e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 15:06:25 -0500 Subject: [PATCH 63/97] chore: ensure proper variable naming --- .github/workflows/dockerhub-release-matrix.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index b6ae3fb64c..87b952c826 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -177,11 +177,10 @@ jobs: } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - # Use to-json with compact formatting let matrix = {include: [{version: $pg_version}]} - $"matrix=(${matrix | to json -r})" | save --append $env.GITHUB_OUTPUT + let matrix_json = ($matrix | to json -r) + $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT ' - publish: needs: get_publish_version strategy: From 9df71870d38423a22326e42936148b3784116b16 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 15:29:31 -0500 Subject: [PATCH 64/97] chore: use the matrix config we already have access to --- .github/workflows/dockerhub-release-matrix.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 87b952c826..e9138251e8 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -169,11 +169,11 @@ jobs: id: get_versions run: | nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" - let release_key = if ($version | str contains "orioledb") { + let versions = "${{ matrix.version }}" + let release_key = if ($versions | str contains "orioledb") { $"postgresorioledb-17" } else { - $"postgres($version)" + $"postgres($versions)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) @@ -181,10 +181,12 @@ jobs: let matrix_json = ($matrix | to json -r) $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT ' + publish: needs: get_publish_version strategy: - matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} + matrix: + include: ${{ fromJson(needs.get_publish_version.outputs.matrix).include }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} From 47e3258074e9677ca79427d421d6478f4c468f20 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 15:59:44 -0500 Subject: [PATCH 65/97] chore: handling architecture --- .../workflows/dockerhub-release-matrix.yml | 34 +++++++++++-------- .github/workflows/mirror.yml | 13 +++++-- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index e9138251e8..10f5c866e9 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -156,9 +156,6 @@ jobs: supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 get_publish_version: needs: [prepare, merge_manifest] - strategy: - matrix: - include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: matrix: ${{ steps.get_versions.outputs.matrix }} @@ -169,25 +166,32 @@ jobs: id: get_versions run: | nix run nixpkgs#nushell -- -c ' - let versions = "${{ matrix.version }}" - let release_key = if ($versions | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($versions)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - - let matrix = {include: [{version: $pg_version}]} + let original_matrix = ($env.GITHUB_WORKFLOW_REF | split row "@" | get 0 | from json) + let versions = $original_matrix.include + let processed_versions = ($versions | each { |item| + let version = $item.version + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + + # Include both AMD64 and ARM64 architectures + [{version: $pg_version, arch: "amd64"}, {version: $pg_version, arch: "arm64"}] + } | flatten) + + let matrix = {include: $processed_versions} let matrix_json = ($matrix | to json -r) $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT ' - + publish: needs: get_publish_version strategy: - matrix: - include: ${{ fromJson(needs.get_publish_version.outputs.matrix).include }} + matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} + arch: ${{ matrix.arch }} secrets: inherit diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index 9f53e7fd78..cbcb750670 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -6,12 +6,19 @@ on: version: required: true type: string + arch: + required: true + type: string workflow_dispatch: inputs: version: description: "Image tag" required: true type: string + arch: + description: "Architecture" + required: true + type: string jobs: mirror: @@ -36,7 +43,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - uses: akhilerm/tag-push-action@v2.1.0 with: - src: docker.io/supabase/postgres:${{ inputs.version }} + src: docker.io/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} dst: | - public.ecr.aws/supabase/postgres:${{ inputs.version }} - ghcr.io/supabase/postgres:${{ inputs.version }} + public.ecr.aws/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} + ghcr.io/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} From 369f534b5aa81616b5e1c57139696167079894b1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 16:12:01 -0500 Subject: [PATCH 66/97] chore directly use matrix --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 10f5c866e9..7ee4f51a33 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -166,7 +166,7 @@ jobs: id: get_versions run: | nix run nixpkgs#nushell -- -c ' - let original_matrix = ($env.GITHUB_WORKFLOW_REF | split row "@" | get 0 | from json) + let original_matrix = (${{ needs.prepare.outputs.matrix_config }} | from json) let versions = $original_matrix.include let processed_versions = ($versions | each { |item| let version = $item.version From cd2e5964d99671102eb780f1546e61d04dd931c2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 16:21:52 -0500 Subject: [PATCH 67/97] chore: process each version --- .github/workflows/dockerhub-release-matrix.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 7ee4f51a33..38d92cc6d1 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -166,7 +166,7 @@ jobs: id: get_versions run: | nix run nixpkgs#nushell -- -c ' - let original_matrix = (${{ needs.prepare.outputs.matrix_config }} | from json) + let original_matrix = ${{ needs.prepare.outputs.matrix_config }} let versions = $original_matrix.include let processed_versions = ($versions | each { |item| let version = $item.version @@ -182,8 +182,7 @@ jobs: } | flatten) let matrix = {include: $processed_versions} - let matrix_json = ($matrix | to json -r) - $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT + $"matrix=(${matrix | to json -r})" | save --append $env.GITHUB_OUTPUT ' publish: From e862161c3901d958b7c07e1c44b1885b9c455618 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Dec 2024 16:32:54 -0500 Subject: [PATCH 68/97] chore: matrix_json variable --- .github/workflows/dockerhub-release-matrix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 38d92cc6d1..0fe2148e74 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -182,7 +182,8 @@ jobs: } | flatten) let matrix = {include: $processed_versions} - $"matrix=(${matrix | to json -r})" | save --append $env.GITHUB_OUTPUT + let matrix_json = ($matrix | to json -r) + $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT ' publish: From cb685ba4540e45426a2e3b5cbf3fee55b6b254a2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 09:00:40 -0500 Subject: [PATCH 69/97] fix: mirroring the merged manifest instead of per arch image --- .../workflows/dockerhub-release-matrix.yml | 53 ++++++++----------- .github/workflows/mirror.yml | 13 ++--- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 0fe2148e74..829337d527 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -155,43 +155,32 @@ jobs: supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 get_publish_version: - needs: [prepare, merge_manifest] - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.get_versions.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main - - name: Get version - id: get_versions - run: | - nix run nixpkgs#nushell -- -c ' - let original_matrix = ${{ needs.prepare.outputs.matrix_config }} - let versions = $original_matrix.include - let processed_versions = ($versions | each { |item| - let version = $item.version - let release_key = if ($version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($version)" + needs: merge_manifest + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.get_versions.outputs.versions }} + steps: + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - uses: DeterminateSystems/nix-installer-action@main + - name: Get merged manifest versions + id: get_versions + run: | + nix run nixpkgs#nushell -- -c ' + let manifest = (docker manifest inspect supabase/postgres | from json) + let tags = { + include: [$manifest.tag] } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - - # Include both AMD64 and ARM64 architectures - [{version: $pg_version, arch: "amd64"}, {version: $pg_version, arch: "arm64"}] - } | flatten) - - let matrix = {include: $processed_versions} - let matrix_json = ($matrix | to json -r) - $"matrix=($matrix_json)" | save --append $env.GITHUB_OUTPUT - ' - + $"versions=($tags | to json -r)" | save --append $env.GITHUB_OUTPUT + ' publish: needs: get_publish_version strategy: - matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} + matrix: ${{ fromJson(needs.get_publish_version.outputs.versions) }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} - arch: ${{ matrix.arch }} secrets: inherit diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index cbcb750670..9f53e7fd78 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -6,19 +6,12 @@ on: version: required: true type: string - arch: - required: true - type: string workflow_dispatch: inputs: version: description: "Image tag" required: true type: string - arch: - description: "Architecture" - required: true - type: string jobs: mirror: @@ -43,7 +36,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - uses: akhilerm/tag-push-action@v2.1.0 with: - src: docker.io/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} + src: docker.io/supabase/postgres:${{ inputs.version }} dst: | - public.ecr.aws/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} - ghcr.io/supabase/postgres:${{ inputs.version }}_${{ inputs.arch }} + public.ecr.aws/supabase/postgres:${{ inputs.version }} + ghcr.io/supabase/postgres:${{ inputs.version }} From d66ac71bb8533e882dff084462046b4014ee818f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 11:15:35 -0500 Subject: [PATCH 70/97] chore: using tags from merge_manifest --- .../workflows/dockerhub-release-matrix.yml | 93 +++++++++---------- 1 file changed, 43 insertions(+), 50 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 829337d527..8b9b13cc21 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -122,65 +122,58 @@ jobs: cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} merge_manifest: - needs: [prepare, build, build_release_image] - strategy: - matrix: - include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Get image tag - id: get_version - run: | - nix run nixpkgs#nushell -- -c ' - let version = "15" - let release_key = if ($version | str contains "orioledb") { - $"postgres($version | str replace "-" "")" - } else { - $"postgres($version)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - - $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT - ' - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 - get_publish_version: - needs: merge_manifest + needs: [prepare, build, build_release_image] + strategy: + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: - versions: ${{ steps.get_versions.outputs.versions }} + tags: ${{ steps.collect_tags.outputs.tags }} steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - uses: DeterminateSystems/nix-installer-action@main - - name: Get merged manifest versions - id: get_versions + - name: Get image tag + id: get_version run: | nix run nixpkgs#nushell -- -c ' - let manifest = (docker manifest inspect supabase/postgres | from json) - let tags = { - include: [$manifest.tag] + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" } - $"versions=($tags | to json -r)" | save --append $env.GITHUB_OUTPUT - ' + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + + $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 + - name: Collect tags + id: collect_tags + run: | + nix run nixpkgs#nushell -- -c ' + let tag = "${{ steps.get_version.outputs.pg_version }}" + let matrix = {include: [{version: $tag}]} + $"tags=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT + ' + get_publish_version: + needs: merge_manifest + runs-on: ubuntu-latest + outputs: + matrix: ${{ needs.merge_manifest.outputs.tags }} publish: - needs: get_publish_version - strategy: - matrix: ${{ fromJson(needs.get_publish_version.outputs.versions) }} - uses: ./.github/workflows/mirror.yml - with: - version: ${{ matrix.version }} - secrets: inherit + needs: [merge_manifest, get_publish_version] # Add merge_manifest + strategy: + matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} + uses: ./.github/workflows/mirror.yml + with: + version: ${{ matrix.version }} + secrets: inherit From c40f455fecceb7daa29d103e172d974307239303 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 11:21:43 -0500 Subject: [PATCH 71/97] chore: no longer need get_publish_version at all --- .github/workflows/dockerhub-release-matrix.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 8b9b13cc21..34897c1b53 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -164,15 +164,10 @@ jobs: let matrix = {include: [{version: $tag}]} $"tags=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT ' - get_publish_version: - needs: merge_manifest - runs-on: ubuntu-latest - outputs: - matrix: ${{ needs.merge_manifest.outputs.tags }} publish: - needs: [merge_manifest, get_publish_version] # Add merge_manifest + needs: merge_manifest strategy: - matrix: ${{ fromJson(needs.get_publish_version.outputs.matrix) }} + matrix: ${{ fromJson(needs.merge_manifest.outputs.tags) }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} From b8f58e675c36fcfa8a00b9a038700a748fdc98fd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 11:38:57 -0500 Subject: [PATCH 72/97] chore: combine tags in outputs at the end of merge_manifest --- .../workflows/dockerhub-release-matrix.yml | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 34897c1b53..0e65d8c2c1 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -128,42 +128,51 @@ jobs: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: - tags: ${{ steps.collect_tags.outputs.tags }} + tags: ${{ steps.combine_tags.outputs.tags }} # Changed to reference final combined step steps: - - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Get image tag - id: get_version - run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" - let release_key = if ($version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($version)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - - $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT - ' - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 - - name: Collect tags - id: collect_tags - run: | - nix run nixpkgs#nushell -- -c ' - let tag = "${{ steps.get_version.outputs.pg_version }}" - let matrix = {include: [{version: $tag}]} - $"tags=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT - ' + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: get_version + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + + $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ + supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 + - name: Create step output + id: step_output + run: | + echo "version=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT + outputs: + matrix-rows: ${{ toJSON(steps.*.outputs.version) }} + - name: Combine tags + if: always() + id: combine_tags + run: | + nix run nixpkgs#nushell -- -c ' + let versions = (${{ toJSON(steps.*.outputs.version) }} | from json) + let matrix = { + include: ($versions | each { |v| {version: $v} }) + } + $"tags=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT + ' publish: needs: merge_manifest strategy: From 0d7253e3f1e3df00b2d6c623c568da8c7104c15e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 11:45:54 -0500 Subject: [PATCH 73/97] chore: cleanup steps to unbreak yaml --- .github/workflows/dockerhub-release-matrix.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 0e65d8c2c1..bc8999286c 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -128,7 +128,8 @@ jobs: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: - tags: ${{ steps.combine_tags.outputs.tags }} # Changed to reference final combined step + tags: ${{ steps.combine_tags.outputs.tags }} + matrix-rows: ${{ toJSON(steps.*.outputs.version) }} steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main @@ -160,8 +161,6 @@ jobs: id: step_output run: | echo "version=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT - outputs: - matrix-rows: ${{ toJSON(steps.*.outputs.version) }} - name: Combine tags if: always() id: combine_tags From 60b2ea2a40901e4f91c345b67543986ec0d58bcd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 11:55:37 -0500 Subject: [PATCH 74/97] chore: no from json --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index bc8999286c..6ab1361273 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -166,7 +166,7 @@ jobs: id: combine_tags run: | nix run nixpkgs#nushell -- -c ' - let versions = (${{ toJSON(steps.*.outputs.version) }} | from json) + let versions = (${{ toJSON(steps.*.outputs.version) }}) let matrix = { include: ($versions | each { |v| {version: $v} }) } From 6ce323870c76de51ce0311c924be27e4d3b3b91d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 13:45:07 -0500 Subject: [PATCH 75/97] chore: try to build up json over iterations --- .../workflows/dockerhub-release-matrix.yml | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 6ab1361273..3c03f051d7 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -128,8 +128,7 @@ jobs: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} runs-on: ubuntu-latest outputs: - tags: ${{ steps.combine_tags.outputs.tags }} - matrix-rows: ${{ toJSON(steps.*.outputs.version) }} + matrix_results: ${{ toJSON(steps.collect_versions.outputs.results) }} steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main @@ -150,32 +149,44 @@ jobs: } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=($pg_version)" | save --append $env.GITHUB_OUTPUT + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT ' + - name: Output version + id: output_version + run: | + echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT + - name: Collect versions + id: collect_versions + run: | + echo "results[${{ strategy.job-index }}]=${{ steps.output_version.outputs.result }}" >> $GITHUB_OUTPUT - name: Merge multi-arch manifests run: | docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 - - name: Create step output - id: step_output - run: | - echo "version=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT - - name: Combine tags - if: always() - id: combine_tags - run: | - nix run nixpkgs#nushell -- -c ' - let versions = (${{ toJSON(steps.*.outputs.version) }}) - let matrix = { - include: ($versions | each { |v| {version: $v} }) - } - $"tags=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT - ' - publish: + + combine_versions: needs: merge_manifest + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: DeterminateSystems/nix-installer-action@main + - name: Combine versions into matrix + id: set-matrix + run: | + nix run nixpkgs#nushell -- -c ' + let versions = (${{ needs.merge_manifest.outputs.matrix_results }} | from json) + let matrix = { + include: ($versions | each { |ver| { version: $ver } }) + } + $"matrix=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT + ' + + publish: + needs: combine_versions strategy: - matrix: ${{ fromJson(needs.merge_manifest.outputs.tags) }} + matrix: ${{ fromJson(needs.combine_versions.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} From 1b9fd7e52466572281cdf341a8f7807a7925c997 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 14:11:07 -0500 Subject: [PATCH 76/97] chore: use the github outputs array --- .../workflows/dockerhub-release-matrix.yml | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 3c03f051d7..d0157f670c 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -148,7 +148,6 @@ jobs: $"postgres($version)" } let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT ' - name: Output version @@ -161,28 +160,36 @@ jobs: echo "results[${{ strategy.job-index }}]=${{ steps.output_version.outputs.result }}" >> $GITHUB_OUTPUT - name: Merge multi-arch manifests run: | - docker buildx imagetools create -t supabase/postgres:${{ steps.get_version.outputs.pg_version }} \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_amd64 \ - supabase/postgres:${{ steps.get_version.outputs.pg_version }}_arm64 - + docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ + ${{ steps.get_version.outputs.pg_version }}_amd64 \ + ${{ steps.get_version.outputs.pg_version }}_arm64 combine_versions: needs: merge_manifest runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: + - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - name: Combine versions into matrix id: set-matrix run: | nix run nixpkgs#nushell -- -c ' - let versions = (${{ needs.merge_manifest.outputs.matrix_results }} | from json) + let raw_versions = "${{ needs.merge_manifest.outputs.matrix_results }}" + let versions = ($raw_versions + | lines # Split into lines + | each { |line| # Process each line + $line + | str replace "results\\[\\d+\\]=" "" # Remove the results[N]= prefix + | str trim # Clean up any whitespace + } + | where { |it| $it != "" } # Filter out any empty lines + ) let matrix = { include: ($versions | each { |ver| { version: $ver } }) } $"matrix=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT ' - publish: needs: combine_versions strategy: From 0d50263379dbcf838b4d26f83af3ae90bc64018d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 14:24:56 -0500 Subject: [PATCH 77/97] chore: first collect all versions into array then output json array --- .github/workflows/dockerhub-release-matrix.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index d0157f670c..a655369dd2 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -157,7 +157,9 @@ jobs: - name: Collect versions id: collect_versions run: | - echo "results[${{ strategy.job-index }}]=${{ steps.output_version.outputs.result }}" >> $GITHUB_OUTPUT + versions=() + versions+=("${{ steps.output_version.outputs.result }}") + echo "matrix_results=[\"${versions[*]}\"]" >> $GITHUB_OUTPUT - name: Merge multi-arch manifests run: | docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ From 7664b4e43747f45c1cdea41b33468704fada1dc7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 14:56:43 -0500 Subject: [PATCH 78/97] chore: debug mainfest output --- .github/workflows/dockerhub-release-matrix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index a655369dd2..bff84aa422 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -173,6 +173,9 @@ jobs: steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main + - name: Debug Merge Manifest Output + run: | + echo "Raw matrix_results: '${{ needs.merge_manifest.outputs.matrix_results }}'" - name: Combine versions into matrix id: set-matrix run: | From ce09b8e3b252596e5926af26ab2ebee4abc78a8f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 15:15:04 -0500 Subject: [PATCH 79/97] chore: store the results data in artifacts --- .../workflows/dockerhub-release-matrix.yml | 154 +++++++++--------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index bff84aa422..12b35e6463 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -122,83 +122,87 @@ jobs: cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} merge_manifest: - needs: [prepare, build, build_release_image] - strategy: - matrix: - include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} - runs-on: ubuntu-latest - outputs: - matrix_results: ${{ toJSON(steps.collect_versions.outputs.results) }} - steps: - - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Get image tag - id: get_version - run: | - nix run nixpkgs#nushell -- -c ' - let version = "${{ matrix.version }}" - let release_key = if ($version | str contains "orioledb") { - $"postgresorioledb-17" - } else { - $"postgres($version)" - } - let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) - $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT - ' - - name: Output version - id: output_version - run: | - echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT - - name: Collect versions - id: collect_versions - run: | - versions=() - versions+=("${{ steps.output_version.outputs.result }}") - echo "matrix_results=[\"${versions[*]}\"]" >> $GITHUB_OUTPUT - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ - ${{ steps.get_version.outputs.pg_version }}_amd64 \ - ${{ steps.get_version.outputs.pg_version }}_arm64 - combine_versions: - needs: merge_manifest - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - uses: DeterminateSystems/nix-installer-action@main - - name: Debug Merge Manifest Output - run: | - echo "Raw matrix_results: '${{ needs.merge_manifest.outputs.matrix_results }}'" - - name: Combine versions into matrix - id: set-matrix - run: | - nix run nixpkgs#nushell -- -c ' - let raw_versions = "${{ needs.merge_manifest.outputs.matrix_results }}" - let versions = ($raw_versions - | lines # Split into lines - | each { |line| # Process each line - $line - | str replace "results\\[\\d+\\]=" "" # Remove the results[N]= prefix - | str trim # Clean up any whitespace - } - | where { |it| $it != "" } # Filter out any empty lines - ) - let matrix = { - include: ($versions | each { |ver| { version: $ver } }) - } - $"matrix=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT - ' + needs: [prepare, build, build_release_image] + strategy: + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: get_version + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + - name: Output version + id: output_version + run: | + echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT + - name: Collect versions + id: collect_versions + run: | + versions=() + versions+=("${{ steps.output_version.outputs.result }}") + echo "matrix_results=[\"${versions[*]}\"]" >> $GITHUB_OUTPUT + - name: Debug matrix_results Output + run: | + echo "Debug matrix_results: '${{ steps.collect_versions.outputs.matrix_results }}'" + - name: Append Results to Artifact + run: | + echo "${{ steps.collect_versions.outputs.matrix_results }}" >> results.txt + - name: Upload Results Artifact + uses: actions/upload-artifact@v3 + with: + name: merge_results + path: results.txt + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ + ${{ steps.get_version.outputs.pg_version }}_amd64 \ + ${{ steps.get_version.outputs.pg_version }}_arm64 + + combine_results: + needs: merge_manifest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # Required if other file dependencies are in the repository + - uses: DeterminateSystems/nix-installer-action@main # Ensures Nushell is available + - name: Download Results Artifact + uses: actions/download-artifact@v3 + with: + name: merge_results + - name: Combine Results + id: combine + run: | + nix run nixpkgs#nushell -- -c ' + let results = (open results.txt | lines | where { |line| $line != "" }) + let matrix = { include: ($results | each { |ver| { version: $ver } }) } + $"matrix=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT + ' + - name: Debug Combined Results + run: | + echo "Combined Results: '${{ steps.combine.outputs.matrix }}'" + outputs: + matrix: ${{ steps.combine.outputs.matrix }} + publish: - needs: combine_versions + needs: combine_results strategy: - matrix: ${{ fromJson(needs.combine_versions.outputs.matrix) }} + matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: version: ${{ matrix.version }} From de42c0eaee357672d462bad121f0486d05315b44 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 15:35:59 -0500 Subject: [PATCH 80/97] chore: unique upload, then download and combine --- .../workflows/dockerhub-release-matrix.yml | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 12b35e6463..baaf43e81a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -130,11 +130,6 @@ jobs: steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version run: | @@ -155,50 +150,50 @@ jobs: - name: Collect versions id: collect_versions run: | - versions=() - versions+=("${{ steps.output_version.outputs.result }}") - echo "matrix_results=[\"${versions[*]}\"]" >> $GITHUB_OUTPUT - - name: Debug matrix_results Output - run: | - echo "Debug matrix_results: '${{ steps.collect_versions.outputs.matrix_results }}'" - - name: Append Results to Artifact - run: | - echo "${{ steps.collect_versions.outputs.matrix_results }}" >> results.txt + echo "${{ steps.output_version.outputs.result }}" >> results.txt # Append results - name: Upload Results Artifact uses: actions/upload-artifact@v3 with: - name: merge_results + name: merge_results-${{ matrix.version }} path: results.txt + if-no-files-found: warn - name: Merge multi-arch manifests run: | docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ ${{ steps.get_version.outputs.pg_version }}_amd64 \ ${{ steps.get_version.outputs.pg_version }}_arm64 - combine_results: needs: merge_manifest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 # Required if other file dependencies are in the repository - - uses: DeterminateSystems/nix-installer-action@main # Ensures Nushell is available - - name: Download Results Artifact + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Download Results Artifacts uses: actions/download-artifact@v3 with: - name: merge_results + name: merge_results-* - name: Combine Results id: combine run: | nix run nixpkgs#nushell -- -c ' - let results = (open results.txt | lines | where { |line| $line != "" }) - let matrix = { include: ($results | each { |ver| { version: $ver } }) } - $"matrix=($matrix | to json -r)" | save --append $env.GITHUB_OUTPUT + # Initialize an empty array to store combined results + let combined_results = [] + + # Loop through all downloaded result files + ls **/results.txt | each { |result_file| + let results = (open $result_file | lines | where { $it != "" }) + $combined_results = $combined_results + $results + } + + # Create the matrix format for the combined results + let matrix = ($combined_results | each { { version: $it } }) | to json + $"matrix=$matrix" | save --append $env.GITHUB_OUTPUT ' - name: Debug Combined Results run: | echo "Combined Results: '${{ steps.combine.outputs.matrix }}'" outputs: matrix: ${{ steps.combine.outputs.matrix }} - publish: needs: combine_results strategy: From 458d3db7ed6ae5eaf8fdea6b0215d20ff6c10bc7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 15:49:31 -0500 Subject: [PATCH 81/97] fix: re-add actions and login --- .github/workflows/dockerhub-release-matrix.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index baaf43e81a..395713f7f7 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -130,6 +130,11 @@ jobs: steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version run: | From 74d63aac012fb5f8589c6be232c7e644f170201f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 16:14:06 -0500 Subject: [PATCH 82/97] chore: try quotes for name matching --- .github/workflows/dockerhub-release-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 395713f7f7..b6fbd35cf7 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -176,7 +176,7 @@ jobs: - name: Download Results Artifacts uses: actions/download-artifact@v3 with: - name: merge_results-* + name: 'merge_results-*' - name: Combine Results id: combine run: | From 16cfb3f4cc9bedc6274dcd83057c7645d2edb5df Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 16:29:43 -0500 Subject: [PATCH 83/97] chore: utilise versions from prepare for download of artifact --- .../workflows/dockerhub-release-matrix.yml | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index b6fbd35cf7..9740439dff 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -168,29 +168,38 @@ jobs: ${{ steps.get_version.outputs.pg_version }}_amd64 \ ${{ steps.get_version.outputs.pg_version }}_arm64 combine_results: - needs: merge_manifest + needs: [prepare, merge_manifest] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main + - name: Get Versions from Matrix Config + id: get_versions + run: | + # Extract the versions from prepare.outputs.matrix_config + echo "Versions: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}" # Print the versions for debugging + echo "versions=$(echo '${{ fromJson(needs.prepare.outputs.matrix_config).include }}' | jq -r 'map(.version) | join(\",\")')" >> $GITHUB_ENV + - name: Download Results Artifacts - uses: actions/download-artifact@v3 - with: - name: 'merge_results-*' + run: | + # Split the versions into a list and download each corresponding artifact + for version in $(echo ${{ env.versions }} | tr "," "\n"); do + artifact_name="merge_results-${version}" + echo "Downloading artifact: $artifact_name" + actions/download-artifact@v3 + with: + name: $artifact_name - name: Combine Results id: combine run: | nix run nixpkgs#nushell -- -c ' - # Initialize an empty array to store combined results let combined_results = [] - # Loop through all downloaded result files ls **/results.txt | each { |result_file| let results = (open $result_file | lines | where { $it != "" }) $combined_results = $combined_results + $results } - # Create the matrix format for the combined results let matrix = ($combined_results | each { { version: $it } }) | to json $"matrix=$matrix" | save --append $env.GITHUB_OUTPUT ' From 60dd2dd6de688ef8a50c701ee5aa4e49d457f0a3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 16:35:51 -0500 Subject: [PATCH 84/97] chore: format correctly for nushell --- .../workflows/dockerhub-release-matrix.yml | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 9740439dff..7608771016 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -173,22 +173,32 @@ jobs: steps: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main + - name: Get Versions from Matrix Config id: get_versions run: | - # Extract the versions from prepare.outputs.matrix_config - echo "Versions: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}" # Print the versions for debugging - echo "versions=$(echo '${{ fromJson(needs.prepare.outputs.matrix_config).include }}' | jq -r 'map(.version) | join(\",\")')" >> $GITHUB_ENV + nix run nixpkgs#nushell -- -c ' + # Extract the versions from prepare.outputs.matrix_config + let versions = ${{ fromJson(needs.prepare.outputs.matrix_config).include }} | get version + echo "Versions: $versions" + + # Set the versions to an environment variable in GitHub Actions' environment + let versions_str = ($versions | join ",") + $"versions=$versions_str" | save --append $env.GITHUB_ENV + ' - name: Download Results Artifacts run: | - # Split the versions into a list and download each corresponding artifact - for version in $(echo ${{ env.versions }} | tr "," "\n"); do - artifact_name="merge_results-${version}" - echo "Downloading artifact: $artifact_name" - actions/download-artifact@v3 - with: - name: $artifact_name + nix run nixpkgs#nushell -- -c ' + let versions = env.versions | str split "," + foreach $version in $versions { + let artifact_name = "merge_results-$version" + echo "Downloading artifact: $artifact_name" + actions/download-artifact@v3 + with: + name: $artifact_name + } + ' - name: Combine Results id: combine run: | From 6f7ce25d6fb099f2a9083dacc8ed1ebbef395720 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 16:57:17 -0500 Subject: [PATCH 85/97] chore: parse matrix config directly --- .github/workflows/dockerhub-release-matrix.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 7608771016..c8cd0d3f43 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -178,11 +178,13 @@ jobs: id: get_versions run: | nix run nixpkgs#nushell -- -c ' - # Extract the versions from prepare.outputs.matrix_config - let versions = ${{ fromJson(needs.prepare.outputs.matrix_config).include }} | get version - echo "Versions: $versions" + # Parse the matrix configuration directly + let matrix_config = ${{ fromJson(needs.prepare.outputs.matrix_config) }} + + # Extract the versions as a list + let versions = $matrix_config.include | get version - # Set the versions to an environment variable in GitHub Actions' environment + echo "Versions: $versions" let versions_str = ($versions | join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV ' @@ -199,6 +201,7 @@ jobs: name: $artifact_name } ' + - name: Combine Results id: combine run: | From 90a45f4a94f6f4fdf8e81f2f10dff7cafd5ed980 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 17:33:31 -0500 Subject: [PATCH 86/97] chore: convert table to list of strings --- .github/workflows/dockerhub-release-matrix.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index c8cd0d3f43..217cd32a20 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -181,17 +181,19 @@ jobs: # Parse the matrix configuration directly let matrix_config = ${{ fromJson(needs.prepare.outputs.matrix_config) }} - # Extract the versions as a list let versions = $matrix_config.include | get version echo "Versions: $versions" - let versions_str = ($versions | join ",") + + # Convert the table of versions to a simple list of strings + let versions_str = ($versions | get version | join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV ' - name: Download Results Artifacts run: | nix run nixpkgs#nushell -- -c ' + # Ensure the versions environment variable is correctly set let versions = env.versions | str split "," foreach $version in $versions { let artifact_name = "merge_results-$version" @@ -202,6 +204,7 @@ jobs: } ' + - name: Combine Results id: combine run: | From e42ab6651246825502dcbcf0f522cfb26c25dff0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 19:36:29 -0500 Subject: [PATCH 87/97] chore:rm redundant --- .github/workflows/dockerhub-release-matrix.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 217cd32a20..152388e617 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -180,16 +180,16 @@ jobs: nix run nixpkgs#nushell -- -c ' # Parse the matrix configuration directly let matrix_config = ${{ fromJson(needs.prepare.outputs.matrix_config) }} - - let versions = $matrix_config.include | get version - + + # Get versions directly from include array + let versions = ($matrix_config.include | get version) + echo "Versions: $versions" - - # Convert the table of versions to a simple list of strings - let versions_str = ($versions | get version | join ",") + + # Convert the versions to a comma-separated string + let versions_str = ($versions | join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV ' - - name: Download Results Artifacts run: | nix run nixpkgs#nushell -- -c ' From 62f5ee2883df615d4120215e812e64e3c2265df4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 20:43:17 -0500 Subject: [PATCH 88/97] chore: add debug --- .github/workflows/dockerhub-release-matrix.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 152388e617..5bbc676d1c 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -174,6 +174,10 @@ jobs: - uses: actions/checkout@v3 - uses: DeterminateSystems/nix-installer-action@main + - name: Debug Input from Prepare + run: | + echo "Raw matrix_config output:" + echo "${{ needs.prepare.outputs.matrix_config }}" - name: Get Versions from Matrix Config id: get_versions run: | @@ -203,8 +207,6 @@ jobs: name: $artifact_name } ' - - - name: Combine Results id: combine run: | From f72840d8c187c7376cf7904b32a8b77e6072dd88 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 21:06:56 -0500 Subject: [PATCH 89/97] chore: from json --- .github/workflows/dockerhub-release-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 5bbc676d1c..b010c1259a 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -183,7 +183,7 @@ jobs: run: | nix run nixpkgs#nushell -- -c ' # Parse the matrix configuration directly - let matrix_config = ${{ fromJson(needs.prepare.outputs.matrix_config) }} + let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) # Get versions directly from include array let versions = ($matrix_config.include | get version) @@ -191,7 +191,7 @@ jobs: echo "Versions: $versions" # Convert the versions to a comma-separated string - let versions_str = ($versions | join ",") + let versions_str = ($versions | str join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV ' - name: Download Results Artifacts From 4ab2e186e5e3195502cac1dc3d7dea63678ee1aa Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 21:27:18 -0500 Subject: [PATCH 90/97] chore: download with pattern --- .github/workflows/dockerhub-release-matrix.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index b010c1259a..0b2b95feab 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -195,18 +195,9 @@ jobs: $"versions=$versions_str" | save --append $env.GITHUB_ENV ' - name: Download Results Artifacts - run: | - nix run nixpkgs#nushell -- -c ' - # Ensure the versions environment variable is correctly set - let versions = env.versions | str split "," - foreach $version in $versions { - let artifact_name = "merge_results-$version" - echo "Downloading artifact: $artifact_name" - actions/download-artifact@v3 - with: - name: $artifact_name - } - ' + uses: actions/download-artifact@v3 + with: + pattern: merge_results-* - name: Combine Results id: combine run: | From 825dd85c5601bca6b655206f174253ea0c67295f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Dec 2024 21:46:54 -0500 Subject: [PATCH 91/97] chore: use list operations --- .../workflows/dockerhub-release-matrix.yml | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 0b2b95feab..a9fd3e7b52 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -202,15 +202,19 @@ jobs: id: combine run: | nix run nixpkgs#nushell -- -c ' - let combined_results = [] - - ls **/results.txt | each { |result_file| - let results = (open $result_file | lines | where { $it != "" }) - $combined_results = $combined_results + $results - } - - let matrix = ($combined_results | each { { version: $it } }) | to json - $"matrix=$matrix" | save --append $env.GITHUB_OUTPUT + # Get all results files and process them in one go + let matrix = ( + ls **/results.txt + | get name # Get just the file paths + | each { |file| open $file } # Open each file + | each { |content| $content | lines } # Split into lines + | flatten # Flatten the nested lists + | where { |line| $line != "" } # Filter empty lines + | each { |line| {version: $line} } # Create objects + | to json # Convert to JSON + ) + + $"matrix=($matrix)" | save --append $env.GITHUB_OUTPUT ' - name: Debug Combined Results run: | From c619fb8ab2e2fef9d4c2e450b3aac8edb29e41be Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 06:34:24 -0500 Subject: [PATCH 92/97] chore: include and debug --- .../workflows/dockerhub-release-matrix.yml | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index a9fd3e7b52..e55e7af1f1 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -203,18 +203,24 @@ jobs: run: | nix run nixpkgs#nushell -- -c ' # Get all results files and process them in one go - let matrix = ( - ls **/results.txt - | get name # Get just the file paths - | each { |file| open $file } # Open each file - | each { |content| $content | lines } # Split into lines - | flatten # Flatten the nested lists - | where { |line| $line != "" } # Filter empty lines - | each { |line| {version: $line} } # Create objects - | to json # Convert to JSON - ) + let files = (ls **/results.txt | get name) + echo $"Found files: ($files)" - $"matrix=($matrix)" | save --append $env.GITHUB_OUTPUT + let matrix = { + include: ( + $files + | each { |file| open $file } # Open each file + | each { |content| $content | lines } # Split into lines + | flatten # Flatten the nested lists + | where { |line| $line != "" } # Filter empty lines + | each { |line| {version: $line} } # Create objects + ) + } + + let json_output = ($matrix | to json -r) # -r for raw output + echo $"Debug output: ($json_output)" + + $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT ' - name: Debug Combined Results run: | From 2ec2b5cf8a6bc7185abd32cf35d85d45d648b5f5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 06:55:35 -0500 Subject: [PATCH 93/97] chore: extract version --- .github/workflows/dockerhub-release-matrix.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index e55e7af1f1..f491fc9d08 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -213,7 +213,11 @@ jobs: | each { |content| $content | lines } # Split into lines | flatten # Flatten the nested lists | where { |line| $line != "" } # Filter empty lines - | each { |line| {version: $line} } # Create objects + | each { |line| + # Extract just the version part after the last colon + let version = ($line | parse "supabase/postgres:{version}" | get version.0) + {version: $version} + } ) } From b8399b960efd44024519f3ee20e36b9dc8140a46 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 07:13:14 -0500 Subject: [PATCH 94/97] chore: set up for merge to develop --- .github/workflows/dockerhub-release-matrix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index f491fc9d08..c7af95e677 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -5,9 +5,8 @@ on: branches: - develop - release/* - - sam/docker-oriole17 paths: - - '**' + - ".github/workflows/dockerhub-release-matrix.yml" workflow_dispatch: jobs: From c2792dce9c21380424d7e4dffc21900b2850914a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 07:17:38 -0500 Subject: [PATCH 95/97] chore: add permission to proper dir --- Dockerfile-orioledb-17 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index d273b8482c..ebf244fa7d 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -78,7 +78,7 @@ RUN mkdir -p /usr/lib/postgresql/bin \ /usr/share/postgresql \ /var/lib/postgresql \ && chown -R postgres:postgres /usr/lib/postgresql \ - && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /var/lib/postgresql \ && chown -R postgres:postgres /usr/share/postgresql # Create symbolic links From c704342500941c185fb3e0563af7a3ad173ac369 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 07:20:03 -0500 Subject: [PATCH 96/97] chore: remove old files --- deprecated-Dockerfile-15-prev | 229 ---------------------------------- deprecated-Dockerfile-16-prev | 228 --------------------------------- 2 files changed, 457 deletions(-) delete mode 100644 deprecated-Dockerfile-15-prev delete mode 100644 deprecated-Dockerfile-16-prev diff --git a/deprecated-Dockerfile-15-prev b/deprecated-Dockerfile-15-prev deleted file mode 100644 index 01895db39b..0000000000 --- a/deprecated-Dockerfile-15-prev +++ /dev/null @@ -1,229 +0,0 @@ -# syntax=docker/dockerfile:1.6 -ARG postgresql_major=15 -ARG postgresql_release=${postgresql_major}.1 - -# Bump default build arg to build a package from source -# Bump vars.yml to specify runtime package version -ARG sfcgal_release=1.3.10 -ARG postgis_release=3.3.2 -ARG pgrouting_release=3.4.1 -ARG pgtap_release=1.2.0 -ARG pg_cron_release=1.6.2 -ARG pgaudit_release=1.7.0 -ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 -ARG pgsql_http_release=1.5.0 -ARG plpgsql_check_release=2.2.5 -ARG pg_safeupdate_release=1.4 -ARG timescaledb_release=2.9.1 -ARG wal2json_release=2_5 -ARG pljava_release=1.6.4 -ARG plv8_release=3.1.5 -ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 -ARG pg_net_release=0.7.1 -ARG rum_release=1.3.13 -ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 -ARG libsodium_release=1.0.18 -ARG pgsodium_release=3.1.6 -ARG pg_graphql_release=1.5.1 -ARG pg_stat_monitor_release=1.1.1 -ARG pg_jsonschema_release=0.1.4 -ARG pg_repack_release=1.4.8 -ARG vault_release=0.2.8 -ARG groonga_release=12.0.8 -ARG pgroonga_release=2.4.0 -ARG wrappers_release=0.3.0 -ARG hypopg_release=1.3.1 -ARG pgvector_release=0.4.0 -ARG pg_tle_release=1.3.2 -ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.0 -ARG wal_g_release=2.0.1 - -FROM ubuntu:focal as base - - -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - TZ=Etc/UTC - -# Pre-configure tzdata before any installations -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone && \ - apt-get update && \ - apt-get install -y --no-install-recommends tzdata && \ - apt-get install -y \ - curl \ - gnupg \ - lsb-release \ - software-properties-common \ - wget \ - sudo \ - git \ - && apt clean && \ - rm -rf /var/lib/apt/lists/* - - -RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres -RUN adduser --system --no-create-home --shell /bin/bash --group wal-g -RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ ---init none \ ---no-confirm \ ---extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ ---extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - -ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" - -COPY . /nixpg - -WORKDIR /nixpg - -RUN nix profile install .#psql_15/bin - - - -WORKDIR / - - -RUN mkdir -p /usr/lib/postgresql/bin \ - /usr/lib/postgresql/share/postgresql \ - /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ - && chown -R postgres:postgres /usr/lib/postgresql \ - && chown -R postgres:postgres /usr/share/postgresql - -# Create symbolic links -RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ - && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ - && chown -R postgres:postgres /usr/bin - -# Create symbolic links for PostgreSQL shares -RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ -RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ -RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ -RUN chown -R postgres:postgres /usr/share/postgresql/ -# Create symbolic links for contrib directory -RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ - && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ - -RUN chown -R postgres:postgres /usr/lib/postgresql - -RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets - - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - checkinstall \ - cmake - -ENV PGDATA=/var/lib/postgresql/data - -#################### -# setup-wal-g.yml -#################### -FROM base as walg -ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g - -# #################### -# # Download gosu for easy step-down from root -# #################### -FROM base as gosu -ARG TARGETARCH -# Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gnupg \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* -# Download binary -ARG GOSU_VERSION=1.16 -ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 -ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ - /usr/local/bin/gosu -ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ - /usr/local/bin/gosu.asc -# Verify checksum -RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ - gpgconf --kill all && \ - chmod +x /usr/local/bin/gosu - -# #################### -# # Build final image -# #################### -FROM gosu as production -RUN id postgres || (echo "postgres user does not exist" && exit 1) -# # Setup extensions -COPY --from=walg /tmp/wal-g /usr/local/bin/ - -# # Initialise configs -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf -COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts -COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf -COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh -COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh - -RUN sed -i \ - -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ - -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ - -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ - -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ - echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ - #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ - usermod -aG postgres wal-g && \ - mkdir -p /etc/postgresql-custom && \ - chown postgres:postgres /etc/postgresql-custom - -# # Include schema migrations -COPY migrations/db /docker-entrypoint-initdb.d/ -COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql -COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql - -# # Add upstream entrypoint script -COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu -ADD --chmod=0755 \ - https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ - /usr/local/bin/ - -RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql - -ENTRYPOINT ["docker-entrypoint.sh"] - -HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost -STOPSIGNAL SIGINT -EXPOSE 5432 - -ENV POSTGRES_HOST=/var/run/postgresql -ENV POSTGRES_USER=supabase_admin -ENV POSTGRES_DB=postgres -RUN apt-get update && apt-get install -y --no-install-recommends \ - locales \ - && rm -rf /var/lib/apt/lists/* && \ - localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ - && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 -RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE=C.UTF-8 -ENV LC_COLLATE=C.UTF-8 -ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive -CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/deprecated-Dockerfile-16-prev b/deprecated-Dockerfile-16-prev deleted file mode 100644 index c281ccd7f2..0000000000 --- a/deprecated-Dockerfile-16-prev +++ /dev/null @@ -1,228 +0,0 @@ -# syntax=docker/dockerfile:1.6 -ARG postgresql_major=15 -ARG postgresql_release=${postgresql_major}.1 - -# Bump default build arg to build a package from source -# Bump vars.yml to specify runtime package version -ARG sfcgal_release=1.3.10 -ARG postgis_release=3.3.2 -ARG pgrouting_release=3.4.1 -ARG pgtap_release=1.2.0 -ARG pg_cron_release=1.6.2 -ARG pgaudit_release=1.7.0 -ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 -ARG pgsql_http_release=1.5.0 -ARG plpgsql_check_release=2.2.5 -ARG pg_safeupdate_release=1.4 -ARG timescaledb_release=2.9.1 -ARG wal2json_release=2_5 -ARG pljava_release=1.6.4 -ARG plv8_release=3.1.5 -ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 -ARG pg_net_release=0.7.1 -ARG rum_release=1.3.13 -ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 -ARG libsodium_release=1.0.18 -ARG pgsodium_release=3.1.6 -ARG pg_graphql_release=1.5.1 -ARG pg_stat_monitor_release=1.1.1 -ARG pg_jsonschema_release=0.1.4 -ARG pg_repack_release=1.4.8 -ARG vault_release=0.2.8 -ARG groonga_release=12.0.8 -ARG pgroonga_release=2.4.0 -ARG wrappers_release=0.3.0 -ARG hypopg_release=1.3.1 -ARG pgvector_release=0.4.0 -ARG pg_tle_release=1.3.2 -ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.0 -ARG wal_g_release=2.0.1 - -FROM ubuntu:focal as base - -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - TZ=Etc/UTC - -# Pre-configure tzdata before any installations -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo $TZ > /etc/timezone && \ - apt-get update && \ - apt-get install -y --no-install-recommends tzdata && \ - apt-get install -y \ - curl \ - gnupg \ - lsb-release \ - software-properties-common \ - wget \ - sudo \ - git \ - && apt clean && \ - rm -rf /var/lib/apt/lists/* - - -RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres -RUN adduser --system --no-create-home --shell /bin/bash --group wal-g -RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ ---init none \ ---no-confirm \ ---extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ ---extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - -ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" - -COPY . /nixpg - -WORKDIR /nixpg - -RUN nix profile install .#psql_16/bin - - - -WORKDIR / - - -RUN mkdir -p /usr/lib/postgresql/bin \ - /usr/lib/postgresql/share/postgresql \ - /usr/share/postgresql \ - # /usr/lib/postgresql/share/postgresql/contrib \ - #/usr/lib/postgresql/share/postgresql/timezonesets \ - #/usr/lib/postgresql/share/postgresql/tsearch_data \ - # /usr/lib/postgresql/share/postgresql/extension \ - && chown -R postgres:postgres /usr/lib/postgresql \ - && chown -R postgres:postgres /usr/share/postgresql - -# Create symbolic links -RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ - && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ - && chown -R postgres:postgres /usr/bin - -# Create symbolic links for PostgreSQL shares -RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ -RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ -RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ -RUN chown -R postgres:postgres /usr/share/postgresql/ -# Create symbolic links for contrib directory -RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ - && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ - -RUN chown -R postgres:postgres /usr/lib/postgresql - -RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets - - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - checkinstall \ - cmake - -ENV PGDATA=/var/lib/postgresql/data - -#################### -# setup-wal-g.yml -#################### -FROM base as walg -ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g - -# #################### -# # Download gosu for easy step-down from root -# #################### -FROM base as gosu -ARG TARGETARCH -# Install dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gnupg \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* -# Download binary -ARG GOSU_VERSION=1.16 -ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 -ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ - /usr/local/bin/gosu -ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ - /usr/local/bin/gosu.asc -# Verify checksum -RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ - gpgconf --kill all && \ - chmod +x /usr/local/bin/gosu - -# #################### -# # Build final image -# #################### -FROM gosu as production -RUN id postgres || (echo "postgres user does not exist" && exit 1) -# # Setup extensions -COPY --from=walg /tmp/wal-g /usr/local/bin/ - -# # Initialise configs -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf -COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts -COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf -COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh -COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh - -RUN sed -i \ - -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ - -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ - -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ - -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ - echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ - #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ - usermod -aG postgres wal-g && \ - mkdir -p /etc/postgresql-custom && \ - chown postgres:postgres /etc/postgresql-custom - -# # Include schema migrations -COPY migrations/db /docker-entrypoint-initdb.d/ -COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql -COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql - -# # Add upstream entrypoint script -COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu -ADD --chmod=0755 \ - https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ - /usr/local/bin/ - -RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql - -ENTRYPOINT ["docker-entrypoint.sh"] - -HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost -STOPSIGNAL SIGINT -EXPOSE 5432 - -ENV POSTGRES_HOST=/var/run/postgresql -ENV POSTGRES_USER=supabase_admin -ENV POSTGRES_DB=postgres -RUN apt-get update && apt-get install -y --no-install-recommends \ - locales \ - && rm -rf /var/lib/apt/lists/* && \ - localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ - && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 -RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE=C.UTF-8 -ENV LC_COLLATE=C.UTF-8 -ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive -CMD ["postgres", "-D", "/etc/postgresql"] From c571b3fe9f3594e7482a533721198f1ab0c53321 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sat, 21 Dec 2024 07:22:00 -0500 Subject: [PATCH 97/97] chore: newline --- Dockerfile-orioledb-17 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index ebf244fa7d..e75694c16d 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -231,4 +231,4 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE=C.UTF-8 ENV LC_COLLATE=C.UTF-8 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive -CMD ["postgres", "-D", "/etc/postgresql"] \ No newline at end of file +CMD ["postgres", "-D", "/etc/postgresql"]