diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..b77fbf65 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,18 @@ +name: ci +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + build_and_publish: + runs-on: docker + container: + image: ghcr.io/catthehacker/ubuntu:runner-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: '1.23.linux-arm64' diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml deleted file mode 100644 index d9c883da..00000000 --- a/.github/workflows/dendrite.yml +++ /dev/null @@ -1,497 +0,0 @@ -name: Dendrite - -on: - push: - branches: - - main - paths: - - '**.go' # only execute on changes to go files - - 'go.sum' # or dependency updates - - '.github/workflows/**' # or workflow changes - pull_request: - paths: - - '**.go' - - 'go.sum' # or dependency updates - - '.github/workflows/**' - release: - types: [published] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - wasm: - name: WASM build test - timeout-minutes: 5 - runs-on: ubuntu-latest - if: ${{ false }} # disable for now - steps: - - uses: actions/checkout@v4 - - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - cache: true - - - name: Install Node - uses: actions/setup-node@v2 - with: - node-version: 14 - - - uses: actions/cache@v4 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Reconfigure Git to use HTTPS auth for repo packages - run: > - git config --global url."https://github.com/".insteadOf - ssh://git@github.com/ - - - name: Install test dependencies - working-directory: ./test/wasm - run: npm ci - - - name: Test - run: ./test-dendritejs.sh - - # Run golangci-lint - lint: - timeout-minutes: 5 - name: Linting - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install libolm - run: sudo apt-get install libolm-dev libolm3 - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - - # run go test with different go versions - test: - timeout-minutes: 10 - name: Unit tests - runs-on: ubuntu-latest - # Service containers to run with `container-job` - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:13-alpine - # Provide the password for postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dendrite - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v4 - - name: Install libolm - run: sudo apt-get install libolm-dev libolm3 - - name: Setup go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - uses: actions/cache@v4 - # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-stable-unit- - - name: Set up gotestfmt - uses: gotesttools/gotestfmt-action@v2 - with: - # Optional: pass GITHUB_TOKEN to avoid rate limiting. - token: ${{ secrets.GITHUB_TOKEN }} - - run: go test -json -v ./... 2>&1 | gotestfmt -hide all - env: - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dendrite - - # build Dendrite for linux with different architectures and go versions - build: - name: Build for Linux - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - goos: ["linux"] - goarch: ["amd64", "386"] - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- - - name: Install dependencies x86 - if: ${{ matrix.goarch == '386' }} - run: sudo apt update && sudo apt-get install -y gcc-multilib - - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 1 - CGO_CFLAGS: -fno-stack-protector - run: go build -trimpath -v -o "bin/" ./cmd/... - - # build for Windows 64-bit - build_windows: - name: Build for Windows - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - matrix: - goos: ["windows"] - goarch: ["amd64"] - steps: - - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- - - name: Install dependencies - run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc - - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 1 - CC: "/usr/bin/x86_64-w64-mingw32-gcc" - run: go build -trimpath -v -o "bin/" ./cmd/... - - # Dummy step to gate other tests on without repeating the whole list - initial-tests-done: - name: Initial tests passed - needs: [lint, test, build, build_windows] - runs-on: ubuntu-latest - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - steps: - - name: Check initial tests passed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} - - # run go test with different go versions - integration: - timeout-minutes: 20 - needs: initial-tests-done - name: Integration tests - runs-on: ubuntu-latest - # Service containers to run with `container-job` - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:13-alpine - # Provide the password for postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dendrite - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - uses: actions/checkout@v4 - - name: Install libolm - run: sudo apt-get install libolm-dev libolm3 - - name: Setup go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - - name: Set up gotestfmt - uses: gotesttools/gotestfmt-action@v2 - with: - # Optional: pass GITHUB_TOKEN to avoid rate limiting. - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-stable-test-race- - - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all - env: - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: dendrite - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - flags: unittests - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - - # run database upgrade tests - upgrade_test: - name: Upgrade tests - timeout-minutes: 20 - needs: initial-tests-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - cache: true - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade-test- - - name: Docker version - run: docker version - - name: Build upgrade-tests - run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade (PostgreSQL) - run: ./dendrite-upgrade-tests --head . - - name: Test upgrade (SQLite) - run: ./dendrite-upgrade-tests --sqlite --head . - - # run database upgrade tests, skipping over one version - upgrade_test_direct: - name: Upgrade tests from HEAD-2 - timeout-minutes: 20 - needs: initial-tests-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup go - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - cache: true - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade-direct-test- - - name: Docker version - run: docker version - - name: Build upgrade-tests - run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade (PostgreSQL) - run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head . - - name: Test upgrade (SQLite) - run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head . - - # run Sytest in different variations - sytest: - timeout-minutes: 20 - needs: initial-tests-done - name: "Sytest (${{ matrix.label }})" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - label: SQLite native - - - label: SQLite Cgo - cgo: 1 - - - label: PostgreSQL - postgres: postgres - - container: - image: matrixdotorg/sytest-dendrite - volumes: - - ${{ github.workspace }}:/src - - /root/.cache/go-build:/github/home/.cache/go-build - - /root/.cache/go-mod:/gopath/pkg/mod - env: - POSTGRES: ${{ matrix.postgres && 1}} - SYTEST_BRANCH: ${{ github.head_ref }} - CGO_ENABLED: ${{ matrix.cgo && 1 }} - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - /gopath/pkg/mod - key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-sytest- - - name: Run Sytest - run: /bootstrap.sh dendrite - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Sytest List Maintenance - if: ${{ always() }} - run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist - continue-on-error: true # not fatal - - name: Are We Synapse Yet? - if: ${{ always() }} - run: /src/are-we-synapse-yet.py /logs/results.tap -v - continue-on-error: true # not fatal - - name: Upload Sytest logs - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - - # run Complement - complement: - name: "Complement (${{ matrix.label }})" - timeout-minutes: 20 - needs: initial-tests-done - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - label: SQLite native - cgo: 0 - - - label: SQLite Cgo - cgo: 1 - - - label: PostgreSQL - postgres: Postgres - cgo: 0 - steps: - # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. - # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path - - name: "Set Go Version" - run: | - echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH - echo "~/go/bin" >> $GITHUB_PATH - - name: "Install Complement Dependencies" - # We don't need to install Go because it is included on the Ubuntu 20.04 image: - # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 - run: | - sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v4 for dendrite - uses: actions/checkout@v4 - with: - path: dendrite - - # Attempt to check out the same branch of Complement as the PR. If it - # doesn't exist, fallback to main. - - name: Checkout complement - shell: bash - run: | - mkdir -p complement - # Attempt to use the version of complement which best matches the current - # build. Depending on whether this is a PR or release, etc. we need to - # use different fallbacks. - # - # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF - # for pull requests, otherwise GITHUB_REF). - # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y - # (GITHUB_BASE_REF for pull requests). - # 3. Use the default complement branch ("master"). - for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do - # Skip empty branch names and merge commits. - if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then - continue - fi - (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break - done - # Build initial Dendrite image - - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . - working-directory: dendrite - env: - DOCKER_BUILDKIT: 1 - - # Run Complement - - run: | - set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all - shell: bash - name: Run Complement Tests - env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} - COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_ - working-directory: complement - - integration-tests-done: - name: Integration tests passed - needs: - [ - initial-tests-done, - upgrade_test, - upgrade_test_direct, - sytest, - complement, - integration - ] - runs-on: ubuntu-latest - if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - steps: - - name: Check integration tests passed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} - - update-docker-images: - name: Update Docker images - permissions: - packages: write - contents: read - security-events: write # To upload Trivy sarif files - if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' - needs: [integration-tests-done] - uses: matrix-org/dendrite/.github/workflows/docker.yml@main - secrets: - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index c795cd36..00000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,213 +0,0 @@ -# Based on https://github.com/docker/build-push-action - -name: "Docker" - -on: - release: # A GitHub release was published - types: [published] - workflow_dispatch: # A build was manually requested - workflow_call: # Another pipeline called us - secrets: - DOCKER_TOKEN: - required: true - -env: - DOCKER_NAMESPACE: matrixdotorg - DOCKER_HUB_USER: dendritegithub - GHCR_NAMESPACE: matrix-org - PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 - -jobs: - monolith: - name: Monolith image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write # To upload Trivy sarif files - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main monolith image - if: github.ref_name == 'main' - id: docker_build_monolith - uses: docker/build-push-action@v3 - with: - cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache - cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max - context: . - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - - - name: Build release monolith image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_monolith_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - - demo-pinecone: - name: Pinecone demo image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main Pinecone demo image - if: github.ref_name == 'main' - id: docker_build_demo_pinecone - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - file: ./build/docker/Dockerfile.demo-pinecone - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - - - name: Build release Pinecone demo image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - file: ./build/docker/Dockerfile.demo-pinecone - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - - demo-yggdrasil: - name: Yggdrasil demo image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main Yggdrasil demo image - if: github.ref_name == 'main' - id: docker_build_demo_yggdrasil - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - file: ./build/docker/Dockerfile.demo-yggdrasil - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} - - - name: Build release Yggdrasil demo image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_demo_yggdrasil_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - file: ./build/docker/Dockerfile.demo-yggdrasil - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml deleted file mode 100644 index 30f55b7c..00000000 --- a/.github/workflows/gh-pages.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy GitHub Pages dependencies preinstalled - -on: - # Runs on pushes targeting the default branch - push: - branches: ["gh-pages"] - paths: - - 'docs/**' # only execute if we have docs changes - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow one concurrent deployment -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - # Build job - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Pages - uses: actions/configure-pages@v2 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 - with: - source: ./docs - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml deleted file mode 100644 index 10eb7c02..00000000 --- a/.github/workflows/helm.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Release Charts - -on: - push: - branches: - - main - paths: - - 'helm/**' # only execute if we have helm chart changes - workflow_dispatch: - -jobs: - release: - # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions - # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token - permissions: - contents: write - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Install Helm - uses: azure/setup-helm@v3 - with: - version: v3.10.0 - - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - with: - config: helm/cr.yaml - charts_dir: helm/ - mark_as_latest: false diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml deleted file mode 100644 index a49042bf..00000000 --- a/.github/workflows/k8s.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: k8s - -on: - push: - branches: ["main"] - paths: - - 'helm/**' # only execute if we have helm chart changes - pull_request: - branches: ["main"] - paths: - - 'helm/**' - -jobs: - lint: - name: Lint Helm chart - runs-on: ubuntu-latest - outputs: - changed: ${{ steps.list-changed.outputs.changed }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: azure/setup-helm@v3 - with: - version: v3.10.0 - - uses: actions/setup-python@v4 - with: - python-version: 3.11 - check-latest: true - - uses: helm/chart-testing-action@v2.3.1 - - name: Get changed status - id: list-changed - run: | - changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }}) - if [[ -n "$changed" ]]; then - echo "::set-output name=changed::true" - fi - - - name: Run lint - run: ct lint --config helm/ct.yaml - - # only bother to run if lint step reports a change to the helm chart - install: - needs: - - lint - if: ${{ needs.lint.outputs.changed == 'true' }} - name: Install Helm charts - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ inputs.checkoutCommit }} - - name: Install Kubernetes tools - uses: yokawasa/action-setup-kube-tools@v0.8.2 - with: - setup-tools: | - helmv3 - helm: "3.10.3" - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.3.1 - - name: Create k3d cluster - uses: nolar/setup-k3d-k3s@v1 - with: - version: v1.28 - - name: Remove node taints - run: | - kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true - - name: Run chart-testing (install) - run: ct install --config helm/ct.yaml - - # Install the chart using helm directly and test with create-account - - name: Install chart - run: | - helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite - - name: Wait for Postgres and Dendrite to be up - run: | - kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A - kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A - kubectl get pods -A - kubectl get services - kubectl get ingress - kubectl logs -l app.kubernetes.io/name=dendrite - - name: Run create account - run: | - podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name) - kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword \ No newline at end of file diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml deleted file mode 100644 index e339c14d..00000000 --- a/.github/workflows/schedules.yaml +++ /dev/null @@ -1,322 +0,0 @@ -name: Scheduled - -on: - schedule: - - cron: '0 0 * * *' # every day at midnight - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where - runs-on: ubuntu-latest - name: Check latest commit - outputs: - should_run: ${{ steps.should_run.outputs.should_run }} - steps: - - uses: actions/checkout@v4 - - name: print latest_commit - run: echo ${{ github.sha }} - - - id: should_run - continue-on-error: true - name: check latest commit is less than a day - if: ${{ github.event_name == 'schedule' }} - run: test -z $(git rev-list --after="24 hours" ${{ github.sha }}) && echo "::set-output name=should_run::false" - - # run Sytest in different variations - sytest: - needs: check_date - if: ${{ needs.check_date.outputs.should_run != 'false' }} - timeout-minutes: 60 - name: "Sytest (${{ matrix.label }})" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - label: SQLite native - - - label: SQLite Cgo - cgo: 1 - - - label: PostgreSQL - postgres: postgres - container: - image: matrixdotorg/sytest-dendrite:latest - volumes: - - ${{ github.workspace }}:/src - - /root/.cache/go-build:/github/home/.cache/go-build - - /root/.cache/go-mod:/gopath/pkg/mod - env: - POSTGRES: ${{ matrix.postgres && 1}} - SYTEST_BRANCH: ${{ github.head_ref }} - RACE_DETECTION: 1 - COVER: 1 - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - /gopath/pkg/mod - key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-sytest- - - name: Run Sytest - run: /bootstrap.sh dendrite - working-directory: /src - - name: Summarise results.tap - if: ${{ always() }} - run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - - name: Sytest List Maintenance - if: ${{ always() }} - run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist - continue-on-error: true # not fatal - - name: Are We Synapse Yet? - if: ${{ always() }} - run: /src/are-we-synapse-yet.py /logs/results.tap -v - continue-on-error: true # not fatal - - name: Upload Sytest logs - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }}) - path: | - /logs/results.tap - /logs/**/*.log* - /logs/**/covdatafiles/** - - sytest-coverage: - timeout-minutes: 5 - name: "Sytest Coverage" - runs-on: ubuntu-latest - needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit - if: ${{ always() && needs.check_date.outputs.should_run != 'false' }} - steps: - - uses: actions/checkout@v4 - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: 'stable' - cache: true - - name: Download all artifacts - uses: actions/download-artifact@v4 - - name: Collect coverage - run: | - go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov - grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov - go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: ./final.cov - flags: sytest - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - - # run Complement - complement: - needs: check_date - if: ${{ needs.check_date.outputs.should_run != 'false' }} - name: "Complement (${{ matrix.label }})" - timeout-minutes: 60 - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - label: SQLite native - cgo: 0 - - - label: SQLite Cgo - cgo: 1 - - - label: PostgreSQL - postgres: Postgres - cgo: 0 - steps: - # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. - # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path - - name: "Set Go Version" - run: | - echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH - echo "~/go/bin" >> $GITHUB_PATH - - name: "Install Complement Dependencies" - # We don't need to install Go because it is included on the Ubuntu 20.04 image: - # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 - run: | - sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v4 for dendrite - uses: actions/checkout@v4 - with: - path: dendrite - - # Attempt to check out the same branch of Complement as the PR. If it - # doesn't exist, fallback to main. - - name: Checkout complement - shell: bash - run: | - mkdir -p complement - # Attempt to use the version of complement which best matches the current - # build. Depending on whether this is a PR or release, etc. we need to - # use different fallbacks. - # - # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF - # for pull requests, otherwise GITHUB_REF). - # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y - # (GITHUB_BASE_REF for pull requests). - # 3. Use the default complement branch ("master"). - for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do - # Skip empty branch names and merge commits. - if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then - continue - fi - (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break - done - # Build initial Dendrite image - - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . - working-directory: dendrite - env: - DOCKER_BUILDKIT: 1 - - - name: Create post test script - run: | - cat < /tmp/posttest.sh - #!/bin/bash - mkdir -p /tmp/Complement/logs/\$2/\$1/ - docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/ - EOF - - chmod +x /tmp/posttest.sh - # Run Complement - - run: | - set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all - shell: bash - name: Run Complement Tests - env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} - COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_ - COMPLEMENT_DENDRITE_COVER: 1 - COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh - working-directory: complement - - - name: Upload Complement logs - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }}) - path: | - /tmp/Complement/logs/** - - complement-coverage: - timeout-minutes: 5 - name: "Complement Coverage" - runs-on: ubuntu-latest - needs: [ complement, check_date ] # only run once Complements is done and there was a commit - if: ${{ always() && needs.check_date.outputs.should_run != 'false' }} - steps: - - uses: actions/checkout@v4 - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: 'stable' - cache: true - - name: Download all artifacts - uses: actions/download-artifact@v4 - - name: Collect coverage - run: | - go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov - grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov - go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: ./final.cov - flags: complement - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} # required - - element-web: - if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright - timeout-minutes: 120 - runs-on: ubuntu-latest - steps: - - uses: tecolicom/actions-use-apt-tools@v1 - with: - # Our test suite includes some screenshot tests with unusual diacritics, which are - # supposed to be covered by STIXGeneral. - tools: fonts-stix - - uses: actions/checkout@v4 - with: - repository: matrix-org/matrix-react-sdk - - uses: actions/setup-node@v3 - with: - cache: 'yarn' - - name: Fetch layered build - run: scripts/ci/layered.sh - - name: Copy config - run: cp element.io/develop/config.json config.json - working-directory: ./element-web - - name: Build - env: - CI_PACKAGE: true - NODE_OPTIONS: "--openssl-legacy-provider" - run: yarn build - working-directory: ./element-web - - name: Edit Test Config - run: | - sed -i '/HOMESERVER/c\ HOMESERVER: "dendrite",' cypress.config.ts - - name: "Run cypress tests" - uses: cypress-io/github-action@v4.1.1 - with: - browser: chrome - start: npx serve -p 8080 ./element-web/webapp - wait-on: 'http://localhost:8080' - env: - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true - TMPDIR: ${{ runner.temp }} - - element-web-pinecone: - if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright - timeout-minutes: 120 - runs-on: ubuntu-latest - steps: - - uses: tecolicom/actions-use-apt-tools@v1 - with: - # Our test suite includes some screenshot tests with unusual diacritics, which are - # supposed to be covered by STIXGeneral. - tools: fonts-stix - - uses: actions/checkout@v4 - with: - repository: matrix-org/matrix-react-sdk - - uses: actions/setup-node@v3 - with: - cache: 'yarn' - - name: Fetch layered build - run: scripts/ci/layered.sh - - name: Copy config - run: cp element.io/develop/config.json config.json - working-directory: ./element-web - - name: Build - env: - CI_PACKAGE: true - NODE_OPTIONS: "--openssl-legacy-provider" - run: yarn build - working-directory: ./element-web - - name: Edit Test Config - run: | - sed -i '/HOMESERVER/c\ HOMESERVER: "dendritePinecone",' cypress.config.ts - - name: "Run cypress tests" - uses: cypress-io/github-action@v4.1.1 - with: - browser: chrome - start: npx serve -p 8080 ./element-web/webapp - wait-on: 'http://localhost:8080' - env: - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true - TMPDIR: ${{ runner.temp }}