From fafa3a3d64e880d6d072318018f06e8f544ba1c6 Mon Sep 17 00:00:00 2001 From: Kostiantyn <32730812+kkovaletp@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:41:17 +0300 Subject: [PATCH] Fix the local build of the dependencies image, including the debug image instructions (#1229) * Fix the local build of the dependencies image, including the debug image instructions * Make the deps image easily runnable * improve readme * Better caching strategy * make dependency from IM to libs build stages * fix missing folder error * Hopefully, final solution --------- Co-authored-by: Konstantin Koval --- .github/workflows/build.yml | 4 ++-- .github/workflows/tests.yml | 4 +--- dependencies/Dockerfile | 19 ++++++++++++------- dependencies/README.md | 9 +++++++++ dependencies/build_imagemagick.sh | 7 ++++--- dependencies/build_libheif.sh | 7 ++++--- dependencies/build_libraw.sh | 7 ++++--- dependencies/download_jellyfin-ffmpeg.sh | 5 +++-- dependencies/prepare.sh | 3 ++- 9 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b159bf51..3e83dede 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,10 +131,10 @@ jobs: annotations: ${{ steps.docker_meta.outputs.annotations }} cache-from: | type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} - type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} + type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }} cache-to: | type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} - type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} + type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }} build-args: | NODE_ENV=production VERSION=${{ github.ref_name }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ec1f92e1..54681e46 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,8 +61,7 @@ jobs: load: true target: api tags: photoview/api - cache-from: type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} - cache-to: type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} + cache-from: type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }} - name: Test id: test @@ -113,7 +112,6 @@ jobs: target: ui tags: photoview/ui cache-from: type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} - cache-to: type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} build-args: | NODE_ENV=testing diff --git a/dependencies/Dockerfile b/dependencies/Dockerfile index ad0b4adc..2d7f55fb 100644 --- a/dependencies/Dockerfile +++ b/dependencies/Dockerfile @@ -13,10 +13,12 @@ WORKDIR /tmp/build-libraw COPY --chmod=0755 build_libraw.sh . RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libraw-${TARGETARCH}-${LIBRAW_VERSION} \ --mount=type=secret,id=github_token \ - export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + if [ -f /run/secrets/github_token ]; then \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token); \ + fi \ && export $(cat /env) && ./build_libraw.sh -FROM base-prep AS build-libheif +FROM build-libraw AS build-libheif ARG LIBHEIF_VERSION ARG TARGETARCH ENV BUILD_CACHE_DIR=/build-cache @@ -24,10 +26,12 @@ WORKDIR /tmp/build-libheif COPY --chmod=0755 build_libheif.sh . RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libheif-${TARGETARCH}-${LIBHEIF_VERSION} \ --mount=type=secret,id=github_token \ - export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + if [ -f /run/secrets/github_token ]; then \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token); \ + fi \ && export $(cat /env) && ./build_libheif.sh -FROM base-prep AS build-imagemagick +FROM build-libheif AS build-imagemagick ARG IMAGEMAGICK_VERSION ARG TARGETARCH ENV BUILD_CACHE_DIR=/build-cache @@ -35,7 +39,9 @@ WORKDIR /tmp/build-imagemagick COPY --chmod=0755 build_imagemagick.sh . RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=imagemagick-${TARGETARCH}-${IMAGEMAGICK_VERSION} \ --mount=type=secret,id=github_token \ - export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + if [ -f /run/secrets/github_token ]; then \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token); \ + fi \ && export $(cat /env) && ./build_imagemagick.sh FROM base-prep AS download-jellyfin-ffmpeg @@ -55,6 +61,5 @@ COPY --from=download-jellyfin-ffmpeg /output/ /output/ COPY --chmod=0755 output.sh / RUN /output.sh -FROM scratch AS release +FROM debian:bookworm-slim AS release COPY --from=final-assembly /artifacts.tar.gz / -USER nobody diff --git a/dependencies/README.md b/dependencies/README.md index 0827b4d1..8a27dca5 100644 --- a/dependencies/README.md +++ b/dependencies/README.md @@ -1,3 +1,12 @@ # Dependencies This directory contains scripts and Dockerfile to build third-party dependencies for Photoview. It is not intended for end-users or runtime execution. + +However, technically, it is runnable for debugging and investigative purposes. +It contains a single archive file, `artifacts.tar.gz`, that bundles all dependencies. + +You can unpack it with: + +```bash +tar -xzf artifacts.tar.gz [-C /desired/location] +``` diff --git a/dependencies/build_imagemagick.sh b/dependencies/build_imagemagick.sh index ef6c0893..38602a92 100755 --- a/dependencies/build_imagemagick.sh +++ b/dependencies/build_imagemagick.sh @@ -1,14 +1,15 @@ #!/bin/bash -set -euo pipefail # Fallback to the latest version if IMAGEMAGICK_VERSION is not set -if [[ -z "$IMAGEMAGICK_VERSION" ]]; then +if [[ -z "${IMAGEMAGICK_VERSION}" ]]; then echo "WARN: ImageMagick version is empty, most likely the script runs not on CI." echo "Fetching the latest version from ImageMagick repo..." IMAGEMAGICK_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ "https://api.github.com/repos/ImageMagick/ImageMagick/releases/latest" | jq -r '.tag_name') fi +set -euo pipefail + : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/ImageMagick-${IMAGEMAGICK_VERSION}" @@ -26,7 +27,7 @@ echo "Building ImageMagick ${IMAGEMAGICK_VERSION} (cache miss)..." echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" -apt-get install -y --no-install-recommends \ +apt-get install -y \ libjxl-dev:"${DEB_HOST_ARCH}" \ liblcms2-dev:"${DEB_HOST_ARCH}" \ liblqr-1-0-dev:"${DEB_HOST_ARCH}" \ diff --git a/dependencies/build_libheif.sh b/dependencies/build_libheif.sh index 8899e8bd..57eaeab2 100755 --- a/dependencies/build_libheif.sh +++ b/dependencies/build_libheif.sh @@ -1,14 +1,15 @@ #!/bin/bash -set -euo pipefail # Fallback to the latest version if LIBHEIF_VERSION is not set -if [[ -z "$LIBHEIF_VERSION" ]]; then +if [[ -z "${LIBHEIF_VERSION}" ]]; then echo "WARN: libheif version is empty, most likely the script runs not on CI." echo "Fetching the latest version from libheif repo..." LIBHEIF_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ "https://api.github.com/repos/strukturag/libheif/releases/latest" | jq -r '.tag_name') fi +set -euo pipefail + : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/libheif-${LIBHEIF_VERSION}" @@ -26,7 +27,7 @@ echo "Building libheif ${LIBHEIF_VERSION} (cache miss)..." echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" -apt-get install -y --no-install-recommends \ +apt-get install -y \ libdav1d-dev:"${DEB_HOST_ARCH}" \ libde265-dev:"${DEB_HOST_ARCH}" \ libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \ diff --git a/dependencies/build_libraw.sh b/dependencies/build_libraw.sh index 73d1ee81..345bbe6f 100755 --- a/dependencies/build_libraw.sh +++ b/dependencies/build_libraw.sh @@ -1,14 +1,15 @@ #!/bin/bash -set -euo pipefail # Fallback to the latest version if LIBRAW_VERSION is not set -if [[ -z "$LIBRAW_VERSION" ]]; then +if [[ -z "${LIBRAW_VERSION}" ]]; then echo "WARN: LibRaw version is empty, most likely the script runs not on CI." echo "Fetching the latest version from LibRaw repo..." LIBRAW_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ "https://api.github.com/repos/LibRaw/LibRaw/releases/latest" | jq -r '.tag_name') fi +set -euo pipefail + : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/LibRaw-${LIBRAW_VERSION}" @@ -26,7 +27,7 @@ echo "Building LibRaw ${LIBRAW_VERSION} (cache miss)..." echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" -apt-get install -y --no-install-recommends \ +apt-get install -y \ libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \ liblcms2-dev:"${DEB_HOST_ARCH}" \ zlib1g-dev:"${DEB_HOST_ARCH}" diff --git a/dependencies/download_jellyfin-ffmpeg.sh b/dependencies/download_jellyfin-ffmpeg.sh index 497f909d..d9d91d14 100755 --- a/dependencies/download_jellyfin-ffmpeg.sh +++ b/dependencies/download_jellyfin-ffmpeg.sh @@ -1,14 +1,15 @@ #!/bin/bash -set -euo pipefail # Fallback to the latest version if JELLYFIN_FFMPEG_VERSION is not set -if [[ -z "$JELLYFIN_FFMPEG_VERSION" ]]; then +if [[ -z "${JELLYFIN_FFMPEG_VERSION}" ]]; then echo "WARN: jellyfin-ffmpeg version is empty, most likely the script runs not on CI." echo "Fetching the latest version from jellyfin-ffmpeg repo..." JELLYFIN_FFMPEG_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/latest" | jq -r '.tag_name') fi +set -euo pipefail + : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/jellyfin-ffmpeg-${JELLYFIN_FFMPEG_VERSION}" diff --git a/dependencies/prepare.sh b/dependencies/prepare.sh index f5802378..6fe55209 100755 --- a/dependencies/prepare.sh +++ b/dependencies/prepare.sh @@ -13,8 +13,9 @@ fi dpkg --add-architecture "$DEBIAN_ARCH" apt-get update -apt-get install -y --no-install-recommends \ +apt-get install -y \ curl \ + jq \ ca-certificates \ crossbuild-essential-"${DEBIAN_ARCH}" \ libc-dev:"${DEBIAN_ARCH}" \