butter/dendrite
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove polylith/API mode (#2967)

Browse source 
This removes most of the code used for polylith/API mode.

This removes the `/api` internal endpoints entirely. 

Binary size change roughly 5%: 
```
51437560 Feb 13 10:15 dendrite-monolith-server # old
48759008 Feb 13 10:15 dendrite-monolith-server # new
```
This commit is contained in:
Till 2023-02-14 12:47:47 +01:00 • committed by GitHub
parent cc59879faa
commit 11d9b9db0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
106 changed files with 374 additions and 5850 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/ISSUE_TEMPLATE/BUG_REPORT.md vendored
View file

@ -17,7 +17,6 @@ see: https://www.matrix.org/security-disclosure-policy/
### Background information
<!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
- **Dendrite version or git SHA**:
- **Monolith or Polylith?**:
- **SQLite3 or Postgres?**:
- **Running in Docker?**:
- **`go version`**:

78
.github/workflows/dendrite.yml vendored
View file

@ -25,7 +25,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.18
go-version: "stable"
cache: true
- name: Install Node
@ -62,14 +62,14 @@ jobs:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.18
go-version: "stable"
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
# run go test with different go versions
test:
timeout-minutes: 10
name: Unit tests (Go ${{ matrix.go }})
name: Unit tests
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
@ -91,25 +91,21 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
go-version: "stable"
- uses: actions/cache@v3
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-unit-
${{ runner.os }}-go-stable-unit-
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
@ -130,7 +126,6 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19", "1.20.0"]
goos: ["linux"]
goarch: ["amd64", "386"]
steps:
@ -138,15 +133,15 @@ jobs:
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
go-version: "stable"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies x86
if: ${{ matrix.goarch == '386' }}
run: sudo apt update && sudo apt-get install -y gcc-multilib
@ -164,23 +159,22 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go: ["1.18", "1.19", "1.20.0"]
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup Go ${{ matrix.go }}
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
go-version: "stable"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- env:
@ -206,7 +200,7 @@ jobs:
integration:
timeout-minutes: 20
needs: initial-tests-done
name: Integration tests (Go ${{ matrix.go }})
name: Integration tests
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
@ -228,16 +222,12 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
go-version: "stable"
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
@ -248,9 +238,9 @@ jobs:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-race-
${{ runner.os }}-go-stable-test-race-
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
@ -261,6 +251,7 @@ jobs:
uses: codecov/codecov-action@v3
with:
flags: unittests
fail_ci_if_error: true
# run database upgrade tests
upgrade_test:
@ -273,7 +264,7 @@ jobs:
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.18"
go-version: "stable"
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
@ -293,7 +284,7 @@ jobs:
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.18"
go-version: "stable"
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
@ -317,19 +308,9 @@ jobs:
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
- label: PostgreSQL, full HTTP APIs
postgres: postgres
api: full-http
container:
image: matrixdotorg/sytest-dendrite
volumes:
@ -338,7 +319,6 @@ jobs:
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
steps:
@ -390,22 +370,9 @@ jobs:
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@ -447,7 +414,7 @@ jobs:
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
@ -459,8 +426,7 @@ jobs:
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
working-directory: complement

82
.github/workflows/docker.yml vendored
View file

@ -61,7 +61,6 @@ jobs:
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -77,7 +76,6 @@ jobs:
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -98,86 +96,6 @@ jobs:
with:
sarif_file: "trivy-results.sarif"
polylith:
name: Polylith image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main polylith image
if: github.ref_name == 'main'
id: docker_build_polylith
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
- name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest

34
.github/workflows/schedules.yaml vendored
View file

@ -24,19 +24,8 @@ jobs:
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
- label: PostgreSQL, full HTTP APIs
postgres: postgres
api: full-http
container:
image: matrixdotorg/sytest-dendrite:latest
volumes:
@ -45,7 +34,6 @@ jobs:
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
RACE_DETECTION: 1
COVER: 1
@ -93,7 +81,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
go-version: 'stable'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
@ -125,22 +113,9 @@ jobs:
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@ -182,7 +157,7 @@ jobs:
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
@ -203,8 +178,7 @@ jobs:
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
COMPLEMENT_DENDRITE_COVER: 1
COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
@ -229,7 +203,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
go-version: 'stable'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3