From ebcbe6c05701b0333228b3b4c72acb5cb4ea36b3 Mon Sep 17 00:00:00 2001 From: Googol Lee Date: Sat, 3 Aug 2024 23:17:54 +0200 Subject: [PATCH] Update workflow using Dockerfile. (#1003) --- .dockerignore | 1 + .github/mysql.testing.env | 2 - .github/postgres.testing.env | 2 - .github/sqlite.testing.env | 2 - .github/workflows/tests.yml | 146 +++++++++--------------- .gitignore | 3 +- Dockerfile | 48 ++++---- README.md | 33 ++++-- dev-compose.yaml | 53 ++++++++- scripts/install_build_dependencies.sh | 5 +- scripts/install_runtime_dependencies.sh | 11 +- 11 files changed, 155 insertions(+), 151 deletions(-) delete mode 100644 .github/mysql.testing.env delete mode 100644 .github/postgres.testing.env delete mode 100644 .github/sqlite.testing.env diff --git a/.dockerignore b/.dockerignore index 695c95f1..8f1ab07a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,6 +17,7 @@ Makefile photos_path screenshots storage +photoview.db* ui/node_modules/ ui/build/ diff --git a/.github/mysql.testing.env b/.github/mysql.testing.env deleted file mode 100644 index 2f5ac13d..00000000 --- a/.github/mysql.testing.env +++ /dev/null @@ -1,2 +0,0 @@ -PHOTOVIEW_DATABASE_DRIVER=mysql -PHOTOVIEW_MYSQL_URL='photoview:photosecret@tcp(127.0.0.1)/photoview_test' diff --git a/.github/postgres.testing.env b/.github/postgres.testing.env deleted file mode 100644 index 6099ebd7..00000000 --- a/.github/postgres.testing.env +++ /dev/null @@ -1,2 +0,0 @@ -PHOTOVIEW_DATABASE_DRIVER=postgres -PHOTOVIEW_POSTGRES_URL=postgres://photoview:photosecret@127.0.0.1/photoview_test diff --git a/.github/sqlite.testing.env b/.github/sqlite.testing.env deleted file mode 100644 index a24802ed..00000000 --- a/.github/sqlite.testing.env +++ /dev/null @@ -1,2 +0,0 @@ -PHOTOVIEW_DATABASE_DRIVER=sqlite -PHOTOVIEW_SQLITE_PATH=photoview_test.db diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e66421ad..ecf110e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,12 +9,11 @@ on: jobs: test-api: name: Test API - runs-on: ubuntu-20.04 - + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - database: ['mysql', 'postgres', 'sqlite'] + database: ['sqlite', 'mysql', 'postgres'] services: mariadb: @@ -46,109 +45,68 @@ jobs: ports: - 5432:5432 - defaults: - run: - working-directory: api - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Fetch branches - run: git fetch --all + - name: Checkout repo + uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - id: go - with: - go-version-file: ${{ github.workspace }}/api/go.mod - cache: false + - name: Build test image + uses: docker/build-push-action@v6 + with: + pull: true + push: false + load: true + target: api + tags: photoview/api + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Cache Go dependencies - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + - name: Test + run: | + docker run --name test --network host \ + -e PHOTOVIEW_DATABASE_DRIVER=${{ matrix.database }} \ + -e PHOTOVIEW_MYSQL_URL='photoview:photosecret@tcp(localhost:3306)/photoview_test' \ + -e PHOTOVIEW_POSTGRES_URL='postgres://photoview:photosecret@localhost:5432/photoview_test' \ + -e PHOTOVIEW_SQLITE_PATH=/tmp/photoview.db \ + photoview/api \ + go test ./... -v -database -filesystem -p 1 -coverprofile=coverage.txt -covermode=atomic + docker cp test:/app/api/coverage.txt ./api/ - - name: Get C dependencies and 3rd-party tools - run: | - sudo add-apt-repository ppa:strukturag/libheif - sudo add-apt-repository ppa:strukturag/libde265 - sudo apt-get update - sudo apt-get install -y libdlib-dev libblas-dev libatlas-base-dev liblapack-dev libjpeg-turbo8-dev libheif-dev ffmpeg exiftool - - - name: Get GO dependencies - run: | - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - - - name: Build - run: go build -v . - - - name: Configure MySQL - if: ${{ matrix.database == 'mysql' }} - run: | - cp ../.github/mysql.testing.env testing.env - - - name: Configure Postgres - if: ${{ matrix.database == 'postgres' }} - run: | - cp ../.github/postgres.testing.env testing.env - - - name: Configure Sqlite - if: ${{ matrix.database == 'sqlite' }} - run: | - touch photoview_test.db - cp ../.github/sqlite.testing.env testing.env - - - name: Test - run: go test ./... -v -database -filesystem -p 1 -coverprofile=coverage.txt -covermode=atomic - - - name: Upload coverage - uses: codecov/codecov-action@v1 - with: - flags: api + - name: Upload coverage + uses: codecov/codecov-action@v4 + with: + flags: api-${{ matrix.database }} test-ui: name: Test UI runs-on: ubuntu-latest - defaults: - run: - working-directory: ui - - strategy: - matrix: - node-version: [18.x] - steps: - - uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Cache NPM dependencies - uses: actions/cache@v2 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- + - name: Checkout repo + uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} + - name: Build test image + uses: docker/build-push-action@v6 + with: + pull: true + push: false + load: true + target: ui + tags: photoview/ui + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Install dependencies - run: npm clean-install + - name: Test + run: | + docker run --name test photoview/ui npm run test:ci + docker cp test:/app/ui/coverage ./ui/ - - name: Test - run: npm run test:ci - - - name: Upload coverage - uses: codecov/codecov-action@v1 - with: - flags: ui + - name: Upload coverage + uses: codecov/codecov-action@v4 + with: + flags: ui diff --git a/.gitignore b/.gitignore index 41757c22..949ea112 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,7 @@ cache/ /media_cache/ /api/media_cache/ /photos_path -photoview.db -photoview.db-* +photoview.db* .env testing.env diff --git a/Dockerfile b/Dockerfile index 08f64295..f1046083 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM --platform=${BUILDPLATFORM:-linux/amd64} node:18 AS ui # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] ARG REACT_APP_API_ENDPOINT ENV REACT_APP_API_ENDPOINT=${REACT_APP_API_ENDPOINT} @@ -23,22 +23,21 @@ ARG COMMIT_SHA ENV COMMIT_SHA=${COMMIT_SHA:-} ENV REACT_APP_BUILD_COMMIT_SHA=${COMMIT_SHA:-} -# Download dependencies -COPY ui /app/ui WORKDIR /app/ui -RUN npm ci --omit=dev --ignore-scripts \ - # Build frontend - && npm run build -- --base=$UI_PUBLIC_URL + +COPY ui/package.json ui/package-lock.json /app/ui +RUN npm ci + +COPY ui/ /app/ui +RUN npm run build -- --base=$UI_PUBLIC_URL ### Build API ### FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.22-bookworm AS api ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -COPY scripts /app/scripts -COPY api /app/api WORKDIR /app/api ENV GOPATH="/go" @@ -46,9 +45,13 @@ ENV PATH="${GOPATH}/bin:${PATH}" ENV CGO_ENABLED=1 # Download dependencies +COPY scripts/*.sh /app/scripts/ RUN chmod +x /app/scripts/*.sh \ - && source /app/scripts/set_compiler_env.sh \ && /app/scripts/install_build_dependencies.sh \ + && /app/scripts/install_runtime_dependencies.sh + +COPY api/go.mod api/go.sum /app/api/ +RUN source /app/scripts/set_compiler_env.sh \ && go env \ && go mod download \ # Patch go-face @@ -56,38 +59,29 @@ RUN chmod +x /app/scripts/*.sh \ # Build dependencies that use CGO && go install \ github.com/mattn/go-sqlite3 \ - github.com/Kagami/go-face \ - # Build api source - && go build -v -o photoview . + github.com/Kagami/go-face -### Build dev image for UI ### -FROM ui AS dev-ui - -### Build dev image for API ### -FROM api AS dev-api +COPY api /app/api RUN source /app/scripts/set_compiler_env.sh \ - && /app/scripts/install_runtime_dependencies.sh \ - ## Install dev tools - && apt update \ - && apt install -y reflex sqlite3 + && go build -v -o photoview . ### Build release image ### FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS release ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] COPY scripts/install_runtime_dependencies.sh /app/scripts/ - -# Create a user to run Photoview server -RUN groupadd -g 999 photoview \ +RUN chmod +x /app/scripts/install_runtime_dependencies.sh \ + # Create a user to run Photoview server + && groupadd -g 999 photoview \ && useradd -r -u 999 -g photoview -m photoview \ # Required dependencies - && chmod +x /app/scripts/*.sh \ && /app/scripts/install_runtime_dependencies.sh WORKDIR /home/photoview + COPY api/data /app/data COPY --from=ui /app/ui/dist /app/ui COPY --from=api /app/api/photoview /app/photoview diff --git a/README.md b/README.md index 8419ec84..67852323 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ Password: **demo** - [Getting started — Setup with Docker](#getting-started--setup-with-docker) - [Advanced setup](#advanced-setup) - [Contributing](#contributing) -- [Set up development environment](#set-up-development-environment) +- [Set up Docker development environment](#set-up-docker-development-environment) +- [Set up local development environment](#set-up-local-development-environment) - [Sponsors](#sponsors) ## Main features @@ -182,12 +183,20 @@ We recommend to use Docker development environment. If Docker environment doesn' It may take a long time to build dependencies when launching servers first time. ```sh -$ docker compose -f dev-compose.yaml build dev-ui dev-api # Build images for development -$ docker compose -f dev-compose.yaml up dev-api dev-ui # Run API and UI servers +$ docker compose -f dev-compose.yaml build # Build images for development +$ docker compose -f dev-compose.yaml up # Launch API and UI servers ``` The graphql playground can now be accessed at [localhost:4001](http://localhost:4001). The site can now be accessed at [localhost:1234](http://localhost:1234). Both servers will be relaunched after the code is changed. +By default, it uses sqlite3 as database. To run servers with other database, please update `PHOTOVIEW_DATABASE_DRIVER` value in `dev-compose.yaml` file and run: + +```sh +$ docker compose -f dev-compose.yaml --profile mysql up # Run with mysql database +or +$ docker compose -f dev-compose.yaml --profile postgres up # Run with postgresql database +``` + ### Start API server with Docker If you don't want to depend on Docker Compose but only Docker, you can launch server as below. @@ -195,25 +204,31 @@ If you don't want to depend on Docker Compose but only Docker, you can launch se It may take a long time to build dependencies when launching servers first time. ```sh -$ docker build --target dev-api -t photoview-api . # Build image for development -$ cp api/example.env api/.env -$ docker run --rm -it -v `pwd`:/app --network host photoview-api # Monitor source code and (re)launch API server +$ docker build --target api -t photoview/api . # Build image for development +$ docker run --rm -it -v `pwd`:/app --network host --env-file api/example.env photoview/api \ + reflex -g '*.go' -s -- go run . # Monitor source code and (re)launch API server ``` The graphql playground can now be accessed at [localhost:4001](http://localhost:4001). +> [!NOTE] +> The server runs on the host network as `--network host` flag. It's easy to communicate between API server and UI server. If you don't want to do that, please check [Docker Network](https://docs.docker.com/network/) to create a new network to run servers. + ### Start UI server with Docker It may take a long time to build dependencies when launching servers first time. ```sh -$ docker build --target dev-ui -t photoview-ui . -$ cp ./ui/example.env ./ui/.env -$ docker run --rm -it -v `pwd`:/app --network host photoview-ui # Monitor source code and (re)launch UI server +$ docker build --target ui -t photoview/ui . # Build image for development +$ docker run --rm -it -v `pwd`:/app --network host --env-file ui/example.env photoview/ui \ + npm run mon # Monitor source code and (re)launch UI server ``` The site can now be accessed at [localhost:1234](http://localhost:1234). +> [!NOTE] +> The server runs on the host network as `--network host` flag. It's easy to communicate between API server and UI server. If you don't want to do that, please check [Docker Network](https://docs.docker.com/network/) to create a new network to run servers. + ## Set up local development environment ### Install dependencies diff --git a/dev-compose.yaml b/dev-compose.yaml index 3aa90aa4..34b4c5f4 100644 --- a/dev-compose.yaml +++ b/dev-compose.yaml @@ -2,11 +2,11 @@ name: photoview services: dev-ui: - image: photoview/photoview-ui + image: photoview/ui build: context: . dockerfile: Dockerfile - target: dev-ui + target: ui env_file: ui/example.env volumes: - .:/app:rw @@ -20,12 +20,17 @@ services: npm run mon dev-api: - image: photoview/photoview-api + image: photoview/api build: context: . dockerfile: Dockerfile - target: dev-api + target: api env_file: api/example.env + environment: + PHOTOVIEW_DATABASE_DRIVER: sqlite # Change to the right database driver + PHOTOVIEW_SQLITE_PATH: api/photoview.db + PHOTOVIEW_MYSQL_URL: photoview:photosecret@tcp(mysql)/photoview_test + PHOTOVIEW_POSTGRES_URL: postgres://photoview:photosecret@postgres:5432/photoview_test volumes: - .:/app:rw ports: @@ -36,3 +41,43 @@ services: - | source /app/scripts/set_compiler_env.sh reflex -g '*.go' -s -- go run . + + mariadb: + image: mariadb:lts + profiles: + - mysql + environment: + MYSQL_DATABASE: photoview_test + MYSQL_USER: photoview + MYSQL_PASSWORD: photosecret + MYSQL_RANDOM_ROOT_PASSWORD: yes + healthcheck: + test: + - CMD + - mariadb-admin + - --user=photoview + - --password=photosecret + - ping + interval: 20s + timeout: 5s + retries: 10 + expose: + - "3306" + + postgres: + image: postgres:16-alpine + profiles: + - postgres + environment: + POSTGRES_DB: photoview_test + POSTGRES_USER: photoview + POSTGRES_PASSWORD: photosecret + healthcheck: + test: + - CMD + - pg_isready + interval: 20s + timeout: 5s + retries: 10 + expose: + - "5432" diff --git a/scripts/install_build_dependencies.sh b/scripts/install_build_dependencies.sh index 181dbbdd..a1d4abb1 100755 --- a/scripts/install_build_dependencies.sh +++ b/scripts/install_build_dependencies.sh @@ -38,6 +38,5 @@ apt-get install -y \ libjpeg-dev:${DEBIAN_ARCH} \ libheif-dev:${DEBIAN_ARCH} -# Cleanup -apt-get clean -rm -rf /var/lib/apt/lists/* +# Install tools for development +apt-get install -y reflex sqlite3 diff --git a/scripts/install_runtime_dependencies.sh b/scripts/install_runtime_dependencies.sh index f40c6229..7698b008 100755 --- a/scripts/install_runtime_dependencies.sh +++ b/scripts/install_runtime_dependencies.sh @@ -1,16 +1,15 @@ #!/bin/bash -BUILD_DEPENDS=(gnupg2 gpg) - apt-get update -apt-get install -y ${BUILD_DEPENDS[@]} curl libdlib19.1 ffmpeg exiftool libheif1 +apt-get install -y curl libdlib19.1 ffmpeg exiftool libheif1 # Install Darktable if building for a supported architecture if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then - echo 'deb https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/ /' \ - | tee /etc/apt/sources.list.d/graphics:darktable.list + echo 'deb [trusted=true] https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/ /' > /etc/apt/sources.list.d/darktable.list + # Release key is invalid, just trust the repo curl -fsSL https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/Release.key \ - | gpg --dearmor | tee /etc/apt/trusted.gpg.d/graphics_darktable.gpg > /dev/null + | gpg --dearmor -o /etc/apt/trusted.gpg.d/darktable.gpg + gpg --show-keys --with-fingerprint --dry-run /etc/apt/trusted.gpg.d/darktable.gpg apt-get update apt-get install -y darktable