Fix the local build of the dependencies image, including the debug image instructions (#1229)

* Fix the local build of the dependencies image, including the debug image instructions

* Make the deps image easily runnable

* improve readme

* Better caching strategy

* make dependency from IM to libs build stages

* fix missing folder error

* Hopefully, final solution

---------

Co-authored-by: Konstantin Koval

.github/workflows/build.yml

@@ -131,10 +131,10 @@ jobs:
           annotations: ${{ steps.docker_meta.outputs.annotations }}
           cache-from: |
             type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }}
-            type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }}
+            type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }}
           cache-to: |
             type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }}
-            type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }}
+            type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }}
           build-args: |
             NODE_ENV=production
             VERSION=${{ github.ref_name }}

.github/workflows/tests.yml

@@ -61,8 +61,7 @@ jobs:
         load:       true
         target:     api
         tags:       photoview/api
-        cache-from: type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }}
-        cache-to:   type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }}
+        cache-from: type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh', 'dependencies/*') }}
 
     - name: Test
       id: test
@@ -113,7 +112,6 @@ jobs:
         target:     ui
         tags:       photoview/ui
         cache-from: type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }}
-        cache-to:   type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }}
         build-args: |
           NODE_ENV=testing
 

dependencies/Dockerfile

@@ -13,10 +13,12 @@ WORKDIR /tmp/build-libraw
 COPY --chmod=0755 build_libraw.sh .
 RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libraw-${TARGETARCH}-${LIBRAW_VERSION} \
     --mount=type=secret,id=github_token \
-    export GITHUB_TOKEN=$(cat /run/secrets/github_token) \
+    if [ -f /run/secrets/github_token ]; then \
+    export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
+    fi \
     && export $(cat /env) && ./build_libraw.sh
 
-FROM base-prep AS build-libheif
+FROM build-libraw AS build-libheif
 ARG LIBHEIF_VERSION
 ARG TARGETARCH
 ENV BUILD_CACHE_DIR=/build-cache
@@ -24,10 +26,12 @@ WORKDIR /tmp/build-libheif
 COPY --chmod=0755 build_libheif.sh .
 RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libheif-${TARGETARCH}-${LIBHEIF_VERSION} \
     --mount=type=secret,id=github_token \
-    export GITHUB_TOKEN=$(cat /run/secrets/github_token) \
+    if [ -f /run/secrets/github_token ]; then \
+    export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
+    fi \
     && export $(cat /env) && ./build_libheif.sh
 
-FROM base-prep AS build-imagemagick
+FROM build-libheif AS build-imagemagick
 ARG IMAGEMAGICK_VERSION
 ARG TARGETARCH
 ENV BUILD_CACHE_DIR=/build-cache
@@ -35,7 +39,9 @@ WORKDIR /tmp/build-imagemagick
 COPY --chmod=0755 build_imagemagick.sh .
 RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=imagemagick-${TARGETARCH}-${IMAGEMAGICK_VERSION} \
     --mount=type=secret,id=github_token \
-    export GITHUB_TOKEN=$(cat /run/secrets/github_token) \
+    if [ -f /run/secrets/github_token ]; then \
+    export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
+    fi \
     && export $(cat /env) && ./build_imagemagick.sh
 
 FROM base-prep AS download-jellyfin-ffmpeg
@@ -55,6 +61,5 @@ COPY --from=download-jellyfin-ffmpeg /output/ /output/
 COPY --chmod=0755 output.sh /
 RUN /output.sh
 
-FROM scratch AS release
+FROM debian:bookworm-slim AS release
 COPY --from=final-assembly /artifacts.tar.gz /
-USER nobody

dependencies/README.md

@@ -1,3 +1,12 @@
 # Dependencies
 
 This directory contains scripts and Dockerfile to build third-party dependencies for Photoview. It is not intended for end-users or runtime execution.
+
+However, technically, it is runnable for debugging and investigative purposes.
+It contains a single archive file, `artifacts.tar.gz`, that bundles all dependencies.
+
+You can unpack it with:
+
+```bash
+tar -xzf artifacts.tar.gz [-C /desired/location]
+```

dependencies/build_imagemagick.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
-set -euo pipefail
 
 # Fallback to the latest version if IMAGEMAGICK_VERSION is not set
-if [[ -z "$IMAGEMAGICK_VERSION" ]]; then
+if [[ -z "${IMAGEMAGICK_VERSION}" ]]; then
   echo "WARN: ImageMagick version is empty, most likely the script runs not on CI."
   echo "Fetching the latest version from ImageMagick repo..."
   IMAGEMAGICK_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \
     "https://api.github.com/repos/ImageMagick/ImageMagick/releases/latest" | jq -r '.tag_name')
 fi
 
+set -euo pipefail
+
 : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}"
 : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}"
 CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/ImageMagick-${IMAGEMAGICK_VERSION}"
@@ -26,7 +27,7 @@ echo "Building ImageMagick ${IMAGEMAGICK_VERSION} (cache miss)..."
 
 echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}"
 
-apt-get install -y --no-install-recommends \
+apt-get install -y \
   libjxl-dev:"${DEB_HOST_ARCH}" \
   liblcms2-dev:"${DEB_HOST_ARCH}" \
   liblqr-1-0-dev:"${DEB_HOST_ARCH}" \

dependencies/build_libheif.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
-set -euo pipefail
 
 # Fallback to the latest version if LIBHEIF_VERSION is not set
-if [[ -z "$LIBHEIF_VERSION" ]]; then
+if [[ -z "${LIBHEIF_VERSION}" ]]; then
   echo "WARN: libheif version is empty, most likely the script runs not on CI."
   echo "Fetching the latest version from libheif repo..."
   LIBHEIF_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \
     "https://api.github.com/repos/strukturag/libheif/releases/latest" | jq -r '.tag_name')
 fi
 
+set -euo pipefail
+
 : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}"
 : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}"
 CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/libheif-${LIBHEIF_VERSION}"
@@ -26,7 +27,7 @@ echo "Building libheif ${LIBHEIF_VERSION} (cache miss)..."
 
 echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}"
 
-apt-get install -y --no-install-recommends \
+apt-get install -y \
   libdav1d-dev:"${DEB_HOST_ARCH}" \
   libde265-dev:"${DEB_HOST_ARCH}" \
   libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \

dependencies/build_libraw.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
-set -euo pipefail
 
 # Fallback to the latest version if LIBRAW_VERSION is not set
-if [[ -z "$LIBRAW_VERSION" ]]; then
+if [[ -z "${LIBRAW_VERSION}" ]]; then
   echo "WARN: LibRaw version is empty, most likely the script runs not on CI."
   echo "Fetching the latest version from LibRaw repo..."
   LIBRAW_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \
     "https://api.github.com/repos/LibRaw/LibRaw/releases/latest" | jq -r '.tag_name')
 fi
 
+set -euo pipefail
+
 : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}"
 : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}"
 CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/LibRaw-${LIBRAW_VERSION}"
@@ -26,7 +27,7 @@ echo "Building LibRaw ${LIBRAW_VERSION} (cache miss)..."
 
 echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}"
 
-apt-get install -y --no-install-recommends \
+apt-get install -y \
   libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \
   liblcms2-dev:"${DEB_HOST_ARCH}" \
   zlib1g-dev:"${DEB_HOST_ARCH}"

dependencies/download_jellyfin-ffmpeg.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
-set -euo pipefail
 
 # Fallback to the latest version if JELLYFIN_FFMPEG_VERSION is not set
-if [[ -z "$JELLYFIN_FFMPEG_VERSION" ]]; then
+if [[ -z "${JELLYFIN_FFMPEG_VERSION}" ]]; then
   echo "WARN: jellyfin-ffmpeg version is empty, most likely the script runs not on CI."
   echo "Fetching the latest version from jellyfin-ffmpeg repo..."
   JELLYFIN_FFMPEG_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \
     "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/latest" | jq -r '.tag_name')
 fi
 
+set -euo pipefail
+
 : "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}"
 : "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}"
 CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/jellyfin-ffmpeg-${JELLYFIN_FFMPEG_VERSION}"

dependencies/prepare.sh

@@ -13,8 +13,9 @@ fi
 
 dpkg --add-architecture "$DEBIAN_ARCH"
 apt-get update
-apt-get install -y --no-install-recommends \
+apt-get install -y \
   curl \
+  jq \
   ca-certificates \
   crossbuild-essential-"${DEBIAN_ARCH}" \
   libc-dev:"${DEBIAN_ARCH}" \