diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 711ac253..b159bf51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,12 +129,15 @@ jobs: tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} annotations: ${{ steps.docker_meta.outputs.annotations }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: | + type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} + type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} + cache-to: | + type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} + type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} build-args: | NODE_ENV=production VERSION=${{ github.ref_name }} - COMMIT_SHA=${{ github.sha }} dockle: name: Dockle Container Analysis diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index b77913ef..2cd03e30 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -11,7 +11,7 @@ on: env: IS_PUSHING_IMAGES: ${{ github.event_name != 'pull_request' && github.repository == 'photoview/photoview' }} - IS_CACHING: ${{ github.event_name == 'pull_request' }} + IS_CACHING: true DOCKER_USERNAME: viktorstrate DOCKER_IMAGE: photoview/dependencies PLATFORMS: linux/amd64,linux/arm64 @@ -51,6 +51,32 @@ jobs: type=raw,value=latest type=sha + - name: Detect dependency versions + id: versions + working-directory: dependencies + shell: bash + run: | + set -euo pipefail + GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" + # Fetch latest version tags from GitHub releases + LIBRAW_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/LibRaw/LibRaw/releases/latest" | jq -r '.tag_name') + LIBHEIF_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/strukturag/libheif/releases/latest" | jq -r '.tag_name') + IMAGEMAGICK_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/ImageMagick/ImageMagick/releases/latest" | jq -r '.tag_name') + JELLYFIN_FFMPEG_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} \ + "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/latest" | jq -r '.tag_name') + # Output as environment variables + echo "LIBRAW_VERSION=${LIBRAW_VERSION}" | tee -a $GITHUB_OUTPUT + echo "LIBHEIF_VERSION=${LIBHEIF_VERSION}" | tee -a $GITHUB_OUTPUT + echo "IMAGEMAGICK_VERSION=${IMAGEMAGICK_VERSION}" | tee -a $GITHUB_OUTPUT + echo "JELLYFIN_FFMPEG_VERSION=${JELLYFIN_FFMPEG_VERSION}" | tee -a $GITHUB_OUTPUT + - name: Build and push uses: docker/build-push-action@v6 with: @@ -61,8 +87,15 @@ jobs: tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} annotations: ${{ steps.docker_meta.outputs.annotations }} - cache-from: ${{ ( env.IS_CACHING == 'true' && 'type=gha' ) || '' }} - cache-to: ${{ ( env.IS_CACHING == 'true' && 'type=gha,mode=max' ) || '' }} - no-cache: ${{ env.IS_CACHING != 'true' }} sbom: true provenance: mode=max + no-cache: ${{ env.IS_CACHING != 'true' }} + cache-from: type=gha,scope=dependencies + cache-to: type=gha,mode=max,scope=dependencies + secrets: | + github_token=${{ secrets.GITHUB_TOKEN }} + build-args: | + LIBRAW_VERSION=${{ steps.versions.outputs.LIBRAW_VERSION }} + LIBHEIF_VERSION=${{ steps.versions.outputs.LIBHEIF_VERSION }} + IMAGEMAGICK_VERSION=${{ steps.versions.outputs.IMAGEMAGICK_VERSION }} + JELLYFIN_FFMPEG_VERSION=${{ steps.versions.outputs.JELLYFIN_FFMPEG_VERSION }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d0fb252c..ec1f92e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,13 +56,13 @@ jobs: - name: Build test image uses: docker/build-push-action@v6 with: - pull: true - push: false - load: true - target: api - tags: photoview/api - cache-from: type=gha - cache-to: type=gha,mode=max + pull: true + push: false + load: true + target: api + tags: photoview/api + cache-from: type=gha,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} + cache-to: type=gha,mode=max,scope=test-api-${{ hashFiles('api/go.sum', 'scripts/install_*.sh') }} - name: Test id: test @@ -107,13 +107,13 @@ jobs: - name: Build test image uses: docker/build-push-action@v6 with: - pull: true - push: false - load: true - target: ui - tags: photoview/ui - cache-from: type=gha - cache-to: type=gha,mode=max + pull: true + push: false + load: true + target: ui + tags: photoview/ui + cache-from: type=gha,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} + cache-to: type=gha,mode=max,scope=test-ui-${{ hashFiles('ui/package-lock.json') }} build-args: | NODE_ENV=testing diff --git a/Dockerfile b/Dockerfile index fb9d2cc1..e64f729a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ ### Build UI ### FROM --platform=${BUILDPLATFORM:-linux/amd64} node:18 AS ui +ARG TARGETARCH +ARG GITHUB_SHA +ARG VERSION # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -7,36 +10,28 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"] ARG NODE_ENV ENV NODE_ENV=${NODE_ENV:-production} +# Set environment variable REACT_APP_API_ENDPOINT from build args, uses "/api" as default ARG REACT_APP_API_ENDPOINT ENV REACT_APP_API_ENDPOINT=${REACT_APP_API_ENDPOINT} -# Set environment variable UI_PUBLIC_URL from build args, uses "/" as default +# Set environment variable UI_PUBLIC_URL from build args, uses "/" as default ARG UI_PUBLIC_URL ENV UI_PUBLIC_URL=${UI_PUBLIC_URL:-/} -ARG VERSION -ENV VERSION=${VERSION:-undefined} -ENV REACT_APP_BUILD_VERSION=${VERSION:-undefined} - -ARG BUILD_DATE -ENV BUILD_DATE=${BUILD_DATE:-undefined} -ENV REACT_APP_BUILD_DATE=${BUILD_DATE:-undefined} - -ARG COMMIT_SHA -ENV COMMIT_SHA=${COMMIT_SHA:-} -ENV REACT_APP_BUILD_COMMIT_SHA=${COMMIT_SHA:-} - WORKDIR /app/ui COPY ui/package.json ui/package-lock.json /app/ui/ RUN npm ci COPY ui/ /app/ui -RUN if [ "${BUILD_DATE}" = "undefined" ]; then \ - export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ'); \ +# hadolint ignore=SC2155 +RUN export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ'); \ export REACT_APP_BUILD_DATE=${BUILD_DATE}; \ - fi; \ - npm run build -- --base="${UI_PUBLIC_URL}" + export COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse --short HEAD || echo 000000)}; \ + export REACT_APP_BUILD_COMMIT_SHA=${COMMIT_SHA}; \ + export VERSION="${VERSION:-$(git rev-parse --abbrev-ref HEAD || echo unknown-branch)}-${TARGETARCH}"; \ + export REACT_APP_BUILD_VERSION=${VERSION}; \ + npm run build -- --base="${UI_PUBLIC_URL}" ### Build API ### FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.24-bookworm AS api @@ -54,39 +49,39 @@ ENV CGO_ENABLED=1 # Download dependencies COPY scripts/set_compiler_env.sh /app/scripts/ RUN chmod +x /app/scripts/*.sh \ - && source /app/scripts/set_compiler_env.sh + && source /app/scripts/set_compiler_env.sh COPY scripts/install_*.sh /app/scripts/ # Split values in `/env` # hadolint ignore=SC2046 RUN chmod +x /app/scripts/*.sh \ - && export $(cat /env) \ - && /app/scripts/install_build_dependencies.sh \ - && /app/scripts/install_runtime_dependencies.sh + && export $(cat /env) \ + && /app/scripts/install_build_dependencies.sh \ + && /app/scripts/install_runtime_dependencies.sh COPY --from=photoview/dependencies /artifacts.tar.gz /dependencies/ # Split values in `/env` # hadolint ignore=SC2046 RUN export $(cat /env) \ - && git config --global --add safe.directory /app \ - && cd /dependencies/ \ - && tar xfv artifacts.tar.gz \ - && cp -a include/* /usr/local/include/ \ - && cp -a pkgconfig/* ${PKG_CONFIG_PATH} \ - && cp -a lib/* /usr/local/lib/ \ - && ldconfig \ - && apt-get install -y ./deb/jellyfin-ffmpeg.deb + && git config --global --add safe.directory /app \ + && cd /dependencies/ \ + && tar xfv artifacts.tar.gz \ + && cp -a include/* /usr/local/include/ \ + && cp -a pkgconfig/* ${PKG_CONFIG_PATH} \ + && cp -a lib/* /usr/local/lib/ \ + && ldconfig \ + && apt-get install -y ./deb/jellyfin-ffmpeg.deb COPY api/go.mod api/go.sum /app/api/ # Split values in `/env` # hadolint ignore=SC2046 RUN export $(cat /env) \ - && go env \ - && go mod download \ - # Patch go-face - && sed -i 's/-march=native//g' ${GOPATH}/pkg/mod/github.com/!kagami/go-face*/face.go \ - # Build dependencies that use CGO - && go install \ + && go env \ + && go mod download \ + # Patch go-face + && sed -i 's/-march=native//g' ${GOPATH}/pkg/mod/github.com/!kagami/go-face*/face.go \ + # Build dependencies that use CGO + && go install \ github.com/mattn/go-sqlite3 \ github.com/Kagami/go-face @@ -94,8 +89,8 @@ COPY api /app/api # Split values in `/env` # hadolint ignore=SC2046 RUN export $(cat /env) \ - && go env \ - && go build -v -o photoview . + && go env \ + && go build -v -o photoview . ### Build release image ### FROM debian:bookworm-slim AS release @@ -106,23 +101,23 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"] COPY scripts/install_runtime_dependencies.sh /app/scripts/ RUN --mount=type=bind,from=api,source=/dependencies/,target=/dependencies/ \ - chmod +x /app/scripts/install_runtime_dependencies.sh \ - # Create a user to run Photoview server - && groupadd -g 999 photoview \ - && useradd -r -u 999 -g photoview -m photoview \ - # Install required dependencies - && /app/scripts/install_runtime_dependencies.sh \ - # Install self-building libs - && cd /dependencies \ - && cp -a lib/*.so* /usr/local/lib/ \ - && ldconfig \ - && apt-get install -y ./deb/jellyfin-ffmpeg.deb \ - && ln -s /usr/lib/jellyfin-ffmpeg/ffmpeg /usr/local/bin/ \ - && ln -s /usr/lib/jellyfin-ffmpeg/ffprobe /usr/local/bin/ \ - # Cleanup - && apt-get autoremove -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + chmod +x /app/scripts/install_runtime_dependencies.sh \ + # Create a user to run Photoview server + && groupadd -g 999 photoview \ + && useradd -r -u 999 -g photoview -m photoview \ + # Install required dependencies + && /app/scripts/install_runtime_dependencies.sh \ + # Install self-building libs + && cd /dependencies \ + && cp -a lib/*.so* /usr/local/lib/ \ + && ldconfig \ + && apt-get install -y ./deb/jellyfin-ffmpeg.deb \ + && ln -s /usr/lib/jellyfin-ffmpeg/ffmpeg /usr/local/bin/ \ + && ln -s /usr/lib/jellyfin-ffmpeg/ffprobe /usr/local/bin/ \ + # Cleanup + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY api/data /app/data COPY --from=ui /app/ui/dist /app/ui @@ -141,7 +136,7 @@ ENV PHOTOVIEW_MEDIA_CACHE=/home/photoview/media-cache EXPOSE ${PHOTOVIEW_LISTEN_PORT} HEALTHCHECK --interval=60s --timeout=10s \ - CMD curl --fail http://localhost:${PHOTOVIEW_LISTEN_PORT}/api/graphql \ + CMD curl --fail http://localhost:${PHOTOVIEW_LISTEN_PORT}/api/graphql \ -X POST -H 'Content-Type: application/json' \ --data-raw '{"operationName":"CheckInitialSetup","variables":{},"query":"query CheckInitialSetup { siteInfo { initialSetup }}"}' \ || exit 1 diff --git a/dependencies/Dockerfile b/dependencies/Dockerfile index a362745a..ad0b4adc 100644 --- a/dependencies/Dockerfile +++ b/dependencies/Dockerfile @@ -1,26 +1,60 @@ -FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS build +FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS base-prep ARG TARGETPLATFORM - -# See for details: https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] +COPY --chmod=0755 prepare.sh /tmp/ +RUN /tmp/prepare.sh \ + && mkdir -p /output/bin /output/lib /output/include /output/pkgconfig /output/etc /output/deb -RUN mkdir -p /tmp/build -WORKDIR /tmp/build +FROM base-prep AS build-libraw +ARG LIBRAW_VERSION +ARG TARGETARCH +ENV BUILD_CACHE_DIR=/build-cache +WORKDIR /tmp/build-libraw +COPY --chmod=0755 build_libraw.sh . +RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libraw-${TARGETARCH}-${LIBRAW_VERSION} \ + --mount=type=secret,id=github_token \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + && export $(cat /env) && ./build_libraw.sh -COPY prepare.sh /tmp/build/ -RUN ./prepare.sh +FROM base-prep AS build-libheif +ARG LIBHEIF_VERSION +ARG TARGETARCH +ENV BUILD_CACHE_DIR=/build-cache +WORKDIR /tmp/build-libheif +COPY --chmod=0755 build_libheif.sh . +RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=libheif-${TARGETARCH}-${LIBHEIF_VERSION} \ + --mount=type=secret,id=github_token \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + && export $(cat /env) && ./build_libheif.sh -COPY build_libraw.sh /tmp/build/ -RUN export $(cat /env) && ./build_libraw.sh -COPY build_libheif.sh /tmp/build/ -RUN export $(cat /env) && ./build_libheif.sh -COPY build_imagemagick.sh /tmp/build/ -RUN export $(cat /env) && ./build_imagemagick.sh -COPY download_jellyfin-ffmpeg.sh /tmp/build/ -RUN export $(cat /env) && ./download_jellyfin-ffmpeg.sh +FROM base-prep AS build-imagemagick +ARG IMAGEMAGICK_VERSION +ARG TARGETARCH +ENV BUILD_CACHE_DIR=/build-cache +WORKDIR /tmp/build-imagemagick +COPY --chmod=0755 build_imagemagick.sh . +RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=imagemagick-${TARGETARCH}-${IMAGEMAGICK_VERSION} \ + --mount=type=secret,id=github_token \ + export GITHUB_TOKEN=$(cat /run/secrets/github_token) \ + && export $(cat /env) && ./build_imagemagick.sh -COPY output.sh / +FROM base-prep AS download-jellyfin-ffmpeg +ARG JELLYFIN_FFMPEG_VERSION +ARG TARGETARCH +ENV BUILD_CACHE_DIR=/build-cache +WORKDIR /tmp/download-ffmpeg +COPY --chmod=0755 download_jellyfin-ffmpeg.sh . +RUN --mount=type=cache,target=${BUILD_CACHE_DIR},id=ffmpeg-${TARGETARCH}-${JELLYFIN_FFMPEG_VERSION} \ + export $(cat /env) && ./download_jellyfin-ffmpeg.sh + +FROM base-prep AS final-assembly +COPY --from=build-libraw /output/ /output/ +COPY --from=build-libheif /output/ /output/ +COPY --from=build-imagemagick /output/ /output/ +COPY --from=download-jellyfin-ffmpeg /output/ /output/ +COPY --chmod=0755 output.sh / RUN /output.sh -FROM debian:bookworm-slim AS release -COPY --from=build /artifacts.tar.gz / +FROM scratch AS release +COPY --from=final-assembly /artifacts.tar.gz / +USER nobody diff --git a/dependencies/README.md b/dependencies/README.md index fe0a4afd..0827b4d1 100644 --- a/dependencies/README.md +++ b/dependencies/README.md @@ -1,2 +1,3 @@ # Dependencies -This is used to build and store 3rd-party dependencies for Photoview products + +This directory contains scripts and Dockerfile to build third-party dependencies for Photoview. It is not intended for end-users or runtime execution. diff --git a/dependencies/build_imagemagick.sh b/dependencies/build_imagemagick.sh index d5af9996..ef6c0893 100755 --- a/dependencies/build_imagemagick.sh +++ b/dependencies/build_imagemagick.sh @@ -1,15 +1,52 @@ #!/bin/bash set -euo pipefail -: ${DEB_HOST_MULTIARCH=`uname -m`-linux-gnu} -: ${DEB_HOST_ARCH=`dpkg --print-architecture`} +# Fallback to the latest version if IMAGEMAGICK_VERSION is not set +if [[ -z "$IMAGEMAGICK_VERSION" ]]; then + echo "WARN: ImageMagick version is empty, most likely the script runs not on CI." + echo "Fetching the latest version from ImageMagick repo..." + IMAGEMAGICK_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + "https://api.github.com/repos/ImageMagick/ImageMagick/releases/latest" | jq -r '.tag_name') +fi -echo Compiler: ${DEB_HOST_MULTIARCH} Arch: ${DEB_HOST_ARCH} +: "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" +: "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" +CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/ImageMagick-${IMAGEMAGICK_VERSION}" +CACHE_MARKER="${CACHE_DIR}/ImageMagick-${IMAGEMAGICK_VERSION}-complete" -apt-get install -y libjxl-dev:${DEB_HOST_ARCH} liblcms2-dev:${DEB_HOST_ARCH} liblqr-1-0-dev:${DEB_HOST_ARCH} libdjvulibre-dev:${DEB_HOST_ARCH} libjpeg62-turbo-dev:${DEB_HOST_ARCH} libopenjp2-7-dev:${DEB_HOST_ARCH} libopenexr-dev:${DEB_HOST_ARCH} libpng-dev:${DEB_HOST_ARCH} libtiff-dev:${DEB_HOST_ARCH} libwebp-dev:${DEB_HOST_ARCH} libxml2-dev:${DEB_HOST_ARCH} libfftw3-dev:${DEB_HOST_ARCH} zlib1g-dev:${DEB_HOST_ARCH} liblzma-dev:${DEB_HOST_ARCH} libbz2-dev:${DEB_HOST_ARCH} -URL=$(curl -s https://api.github.com/repos/ImageMagick/ImageMagick/releases/latest | grep "tarball_url" | cut -d : -f 2,3 | tr -d ' ,"') -echo download ImageMagick from $URL -curl -L -o ./magick.tar.gz "$URL" +# Check if this specific version is already built and cached +if [[ -f "$CACHE_MARKER" ]] && [[ -d "${CACHE_DIR}/output" ]]; then + echo "ImageMagick ${IMAGEMAGICK_VERSION} found in cache, reusing..." + mkdir -p /output + cp -ra "${CACHE_DIR}/output/"* /output/ + exit 0 +fi + +echo "Building ImageMagick ${IMAGEMAGICK_VERSION} (cache miss)..." + +echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" + +apt-get install -y --no-install-recommends \ + libjxl-dev:"${DEB_HOST_ARCH}" \ + liblcms2-dev:"${DEB_HOST_ARCH}" \ + liblqr-1-0-dev:"${DEB_HOST_ARCH}" \ + libdjvulibre-dev:"${DEB_HOST_ARCH}" \ + libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \ + libopenjp2-7-dev:"${DEB_HOST_ARCH}" \ + libopenexr-dev:"${DEB_HOST_ARCH}" \ + libpng-dev:"${DEB_HOST_ARCH}" \ + libtiff-dev:"${DEB_HOST_ARCH}" \ + libwebp-dev:"${DEB_HOST_ARCH}" \ + libxml2-dev:"${DEB_HOST_ARCH}" \ + libfftw3-dev:"${DEB_HOST_ARCH}" \ + zlib1g-dev:"${DEB_HOST_ARCH}" \ + liblzma-dev:"${DEB_HOST_ARCH}" \ + libbz2-dev:"${DEB_HOST_ARCH}" + +URL="https://api.github.com/repos/ImageMagick/ImageMagick/tarball/${IMAGEMAGICK_VERSION}" +echo download ImageMagick from "$URL" +curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 -o ./magick.tar.gz \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "$URL" tar xfv ./magick.tar.gz cd ImageMagick-* @@ -20,13 +57,13 @@ cd ImageMagick-* --with-raw --with-tiff --with-webp \ --without-x --without-magick-plus-plus \ --without-perl --disable-doc \ - --host=${DEB_HOST_MULTIARCH} + --host="${DEB_HOST_MULTIARCH}" make make install cd .. mkdir -p /output/bin /output/etc /output/lib /output/include /output/pkgconfig -cp /usr/local/bin/magick /output/bin/ +cp -a /usr/local/bin/magick /output/bin/ cp -a /usr/local/etc/ImageMagick-7 /output/etc/ cp -a /usr/local/lib/ImageMagick-* /output/lib/ cp -a /usr/local/lib/libMagickCore-* /output/lib/ @@ -36,3 +73,11 @@ cp -a /usr/local/lib/pkgconfig/ImageMagick*.pc /output/pkgconfig/ cp -a /usr/local/lib/pkgconfig/MagickCore*.pc /output/pkgconfig/ cp -a /usr/local/lib/pkgconfig/MagickWand*.pc /output/pkgconfig/ file /output/bin/magick + +# After successful build, cache the results +echo "Caching ImageMagick ${IMAGEMAGICK_VERSION} build results..." +mkdir -p "${CACHE_DIR}/output" +cp -ra /output/* "${CACHE_DIR}/output/" +touch "$CACHE_MARKER" + +echo "ImageMagick ${IMAGEMAGICK_VERSION} build complete and cached" diff --git a/dependencies/build_libheif.sh b/dependencies/build_libheif.sh index 56e5fe14..8899e8bd 100755 --- a/dependencies/build_libheif.sh +++ b/dependencies/build_libheif.sh @@ -1,19 +1,56 @@ #!/bin/bash set -euo pipefail -: ${DEB_HOST_MULTIARCH=`uname -m`-linux-gnu} -: ${DEB_HOST_ARCH=`dpkg --print-architecture`} +# Fallback to the latest version if LIBHEIF_VERSION is not set +if [[ -z "$LIBHEIF_VERSION" ]]; then + echo "WARN: libheif version is empty, most likely the script runs not on CI." + echo "Fetching the latest version from libheif repo..." + LIBHEIF_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + "https://api.github.com/repos/strukturag/libheif/releases/latest" | jq -r '.tag_name') +fi -echo Compiler: ${DEB_HOST_MULTIARCH} Arch: ${DEB_HOST_ARCH} +: "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" +: "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" +CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/libheif-${LIBHEIF_VERSION}" +CACHE_MARKER="${CACHE_DIR}/libheif-${LIBHEIF_VERSION}-complete" -apt-get install -y libdav1d-dev:${DEB_HOST_ARCH} libde265-dev:${DEB_HOST_ARCH} libjpeg62-turbo-dev:${DEB_HOST_ARCH} libopenh264-dev:${DEB_HOST_ARCH} libpng-dev:${DEB_HOST_ARCH} libnuma-dev:${DEB_HOST_ARCH} zlib1g-dev:${DEB_HOST_ARCH} -URL=$(curl -s https://api.github.com/repos/strukturag/libheif/releases/latest | grep "tarball_url" | cut -d : -f 2,3 | tr -d ' ,"') -echo download libraw from $URL -curl -L -o ./libheif.tar.gz "$URL" +# Check if this specific version is already built and cached +if [[ -f "$CACHE_MARKER" ]] && [[ -d "${CACHE_DIR}/output" ]]; then + echo "libheif ${LIBHEIF_VERSION} found in cache, reusing..." + mkdir -p /output + cp -ra "${CACHE_DIR}/output/"* /output/ + exit 0 +fi + +echo "Building libheif ${LIBHEIF_VERSION} (cache miss)..." + +echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" + +apt-get install -y --no-install-recommends \ + libdav1d-dev:"${DEB_HOST_ARCH}" \ + libde265-dev:"${DEB_HOST_ARCH}" \ + libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \ + libopenh264-dev:"${DEB_HOST_ARCH}" \ + libpng-dev:"${DEB_HOST_ARCH}" \ + libnuma-dev:"${DEB_HOST_ARCH}" \ + zlib1g-dev:"${DEB_HOST_ARCH}" + +URL="https://api.github.com/repos/strukturag/libheif/tarball/${LIBHEIF_VERSION}" +echo download libheif from "$URL" +curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 -o ./libheif.tar.gz \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "$URL" tar xfv ./libheif.tar.gz -cd *-libheif-* -cmake --preset=release -DCMAKE_SYSTEM_PROCESSOR=${DEB_HOST_ARCH} -DCMAKE_C_COMPILER=${DEB_HOST_MULTIARCH}-gcc -DCMAKE_CXX_COMPILER=${DEB_HOST_MULTIARCH}-g++ -DPKG_CONFIG_EXECUTABLE=${DEB_HOST_MULTIARCH}-pkg-config -DCMAKE_LIBRARY_ARCHITECTURE=${DEB_HOST_MULTIARCH} -DENABLE_PLUGIN_LOADING=OFF -DWITH_GDK_PIXBUF=OFF . +cd ./*-libheif-* +cmake \ + --preset=release \ + -DCMAKE_SYSTEM_PROCESSOR="${DEB_HOST_ARCH}" \ + -DCMAKE_C_COMPILER="${DEB_HOST_MULTIARCH}"-gcc \ + -DCMAKE_CXX_COMPILER="${DEB_HOST_MULTIARCH}"-g++ \ + -DPKG_CONFIG_EXECUTABLE="${DEB_HOST_MULTIARCH}"-pkg-config \ + -DCMAKE_LIBRARY_ARCHITECTURE="${DEB_HOST_MULTIARCH}" \ + -DENABLE_PLUGIN_LOADING=OFF \ + -DWITH_GDK_PIXBUF=OFF . make make install cd .. @@ -24,3 +61,11 @@ cp -a /usr/local/lib/libheif* /output/lib/ cp -a /usr/local/lib/pkgconfig/libheif* /output/pkgconfig/ cp -a /usr/local/include/libheif /output/include/ file /usr/local/lib/libheif.so* + +# After successful build, cache the results +echo "Caching libheif ${LIBHEIF_VERSION} build results..." +mkdir -p "${CACHE_DIR}/output" +cp -ra /output/* "${CACHE_DIR}/output/" +touch "$CACHE_MARKER" + +echo "libheif ${LIBHEIF_VERSION} build complete and cached" diff --git a/dependencies/build_libraw.sh b/dependencies/build_libraw.sh index 71d67b04..73d1ee81 100755 --- a/dependencies/build_libraw.sh +++ b/dependencies/build_libraw.sh @@ -1,20 +1,50 @@ #!/bin/bash set -euo pipefail -: ${DEB_HOST_MULTIARCH=`uname -m`-linux-gnu} -: ${DEB_HOST_ARCH=`dpkg --print-architecture`} +# Fallback to the latest version if LIBRAW_VERSION is not set +if [[ -z "$LIBRAW_VERSION" ]]; then + echo "WARN: LibRaw version is empty, most likely the script runs not on CI." + echo "Fetching the latest version from LibRaw repo..." + LIBRAW_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + "https://api.github.com/repos/LibRaw/LibRaw/releases/latest" | jq -r '.tag_name') +fi -echo Compiler: ${DEB_HOST_MULTIARCH} Arch: ${DEB_HOST_ARCH} +: "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" +: "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" +CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/LibRaw-${LIBRAW_VERSION}" +CACHE_MARKER="${CACHE_DIR}/LibRaw-${LIBRAW_VERSION}-complete" -apt-get install -y libjpeg62-turbo-dev:${DEB_HOST_ARCH} liblcms2-dev:${DEB_HOST_ARCH} zlib1g-dev:${DEB_HOST_ARCH} -URL=$(curl -s https://api.github.com/repos/LibRaw/LibRaw/releases/latest | grep "tarball_url" | cut -d : -f 2,3 | tr -d ' ,"') -echo download libraw from $URL -curl -L -o ./libraw.tar.gz "$URL" +# Check if this specific version is already built and cached +if [[ -f "$CACHE_MARKER" ]] && [[ -d "${CACHE_DIR}/output" ]]; then + echo "LibRaw ${LIBRAW_VERSION} found in cache, reusing..." + mkdir -p /output + cp -ra "${CACHE_DIR}/output/"* /output/ + exit 0 +fi + +echo "Building LibRaw ${LIBRAW_VERSION} (cache miss)..." + +echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" + +apt-get install -y --no-install-recommends \ + libjpeg62-turbo-dev:"${DEB_HOST_ARCH}" \ + liblcms2-dev:"${DEB_HOST_ARCH}" \ + zlib1g-dev:"${DEB_HOST_ARCH}" + +URL="https://api.github.com/repos/LibRaw/LibRaw/tarball/${LIBRAW_VERSION}" +echo download libraw from "$URL" +curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 -o ./libraw.tar.gz \ + ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "$URL" tar xfv ./libraw.tar.gz cd LibRaw-* autoreconf --install -./configure --disable-option-checking --disable-silent-rules --disable-maintainer-mode --disable-dependency-tracking --host=${DEB_HOST_MULTIARCH} +./configure \ + --disable-option-checking \ + --disable-silent-rules \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + --host="${DEB_HOST_MULTIARCH}" make make install cd .. @@ -25,3 +55,11 @@ cp -a /usr/local/lib/libraw_r* /output/lib/ cp -a /usr/local/lib/pkgconfig/libraw* /output/pkgconfig/ cp -a /usr/local/include/libraw /output/include/ file /usr/local/lib/libraw_r.so* + +# After successful build, cache the results +echo "Caching LibRaw ${LIBRAW_VERSION} build results..." +mkdir -p "${CACHE_DIR}/output" +cp -ra /output/* "${CACHE_DIR}/output/" +touch "$CACHE_MARKER" + +echo "LibRaw ${LIBRAW_VERSION} build complete and cached" diff --git a/dependencies/download_jellyfin-ffmpeg.sh b/dependencies/download_jellyfin-ffmpeg.sh index cd4e60f3..497f909d 100755 --- a/dependencies/download_jellyfin-ffmpeg.sh +++ b/dependencies/download_jellyfin-ffmpeg.sh @@ -1,14 +1,42 @@ #!/bin/bash set -euo pipefail -: ${DEB_HOST_MULTIARCH=x86_64-linux-gnu} -: ${DEB_HOST_ARCH=amd64} +# Fallback to the latest version if JELLYFIN_FFMPEG_VERSION is not set +if [[ -z "$JELLYFIN_FFMPEG_VERSION" ]]; then + echo "WARN: jellyfin-ffmpeg version is empty, most likely the script runs not on CI." + echo "Fetching the latest version from jellyfin-ffmpeg repo..." + JELLYFIN_FFMPEG_VERSION=$(curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 \ + "https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/latest" | jq -r '.tag_name') +fi -echo Compiler: ${DEB_HOST_MULTIARCH} Arch: ${DEB_HOST_ARCH} +: "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" +: "${DEB_HOST_ARCH:=$(dpkg --print-architecture)}" +CACHE_DIR="${BUILD_CACHE_DIR:-/build-cache}/jellyfin-ffmpeg-${JELLYFIN_FFMPEG_VERSION}" +CACHE_MARKER="${CACHE_DIR}/jellyfin-ffmpeg-${JELLYFIN_FFMPEG_VERSION}-complete" -URL=$(curl -s https://api.github.com/repos/jellyfin/jellyfin-ffmpeg/releases/latest | grep \"browser_download_url\".*bookworm_${DEB_HOST_ARCH} | cut -d : -f 2,3 | tr -d ' ,"') -echo download jellyfin-ffmpeg from $URL -curl -L -o ./jellyfin-ffmpeg.deb "$URL" +# Check if this specific version is already downloaded and cached +if [[ -f "$CACHE_MARKER" ]] && [[ -d "${CACHE_DIR}/output" ]]; then + echo "jellyfin-ffmpeg ${JELLYFIN_FFMPEG_VERSION} found in cache, reusing..." + mkdir -p /output + cp -ra "${CACHE_DIR}/output/"* /output/ + exit 0 +fi +echo "Downloading jellyfin-ffmpeg ${JELLYFIN_FFMPEG_VERSION} (cache miss)..." + +echo Compiler: "${DEB_HOST_MULTIARCH}" Arch: "${DEB_HOST_ARCH}" + +VER="${JELLYFIN_FFMPEG_VERSION#v}" +MAJOR_VER=$(echo "${VER}" | cut -d. -f1) +URL="https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/${JELLYFIN_FFMPEG_VERSION}/jellyfin-ffmpeg${MAJOR_VER}_${VER}-bookworm_${DEB_HOST_ARCH}.deb" +echo download jellyfin-ffmpeg from "$URL" mkdir -p /output/deb -cp ./jellyfin-ffmpeg.deb /output/deb/ +curl -fsSL --retry 2 --retry-delay 5 --retry-max-time 60 -o /output/deb/jellyfin-ffmpeg.deb "$URL" + +# After successful download, cache the results +echo "Caching jellyfin-ffmpeg ${JELLYFIN_FFMPEG_VERSION} build results..." +mkdir -p "${CACHE_DIR}/output" +cp -ra /output/* "${CACHE_DIR}/output/" +touch "$CACHE_MARKER" + +echo "jellyfin-ffmpeg ${JELLYFIN_FFMPEG_VERSION} downloaded and cached" diff --git a/dependencies/output.sh b/dependencies/output.sh index 38c72335..6d8c573a 100755 --- a/dependencies/output.sh +++ b/dependencies/output.sh @@ -2,4 +2,4 @@ set -euo pipefail cd /output -tar czfv /artifacts.tar.gz * +tar czfv /artifacts.tar.gz ./* diff --git a/dependencies/prepare.sh b/dependencies/prepare.sh index fd7faf51..f5802378 100755 --- a/dependencies/prepare.sh +++ b/dependencies/prepare.sh @@ -1,27 +1,32 @@ #!/bin/bash set -euo pipefail -: ${TARGETPLATFORM=linux/`dpkg --print-architecture`} +: "${TARGETPLATFORM:=linux/$(dpkg --print-architecture)}" +: "${DEB_HOST_MULTIARCH:=$(uname -m)-linux-gnu}" -TARGETOS="$(echo $TARGETPLATFORM | cut -d"/" -f1)" -TARGETARCH="$(echo $TARGETPLATFORM | cut -d"/" -f2)" -TARGETVARIANT="$(echo $TARGETPLATFORM | cut -d"/" -f3)" +TARGETARCH="$(echo "$TARGETPLATFORM" | cut -d"/" -f2)" DEBIAN_ARCH=$TARGETARCH -if [ "$TARGETARCH" = "arm" ] -then +if [ "$TARGETARCH" = "arm" ]; then DEBIAN_ARCH=armel - if [ "$TARGETVARIANT" = "v7" ] - then - DEBIAN_ARCH=armhf - fi fi -dpkg --add-architecture $DEBIAN_ARCH +dpkg --add-architecture "$DEBIAN_ARCH" apt-get update -apt-get install -y curl crossbuild-essential-${DEBIAN_ARCH} libc-dev:${DEBIAN_ARCH} autoconf automake libtool m4 pkg-config cmake +apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + crossbuild-essential-"${DEBIAN_ARCH}" \ + libc-dev:"${DEBIAN_ARCH}" \ + autoconf \ + automake \ + libtool \ + m4 \ + pkg-config \ + cmake -dpkg-architecture -a $DEBIAN_ARCH >/env +dpkg-architecture -a "$DEBIAN_ARCH" >/env +echo "PKG_CONFIG_PATH=/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig" >>/env +# shellcheck disable=SC2046 export $(cat /env) -echo PKG_CONFIG_PATH=/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig >>/env cat /env diff --git a/ui/apollo.config.js b/ui/apollo.config.js index 371e058c..95ef9d35 100644 --- a/ui/apollo.config.js +++ b/ui/apollo.config.js @@ -6,7 +6,7 @@ const schemasFolder = path.join(__dirname, '..', 'api', 'graphql', 'resolvers'); let completeSchema; try { - completeSchema = readdirSync(schemasFolder) + completeSchema = readdirSync(schemasFolder) .filter(x => x.endsWith('.graphql')) .map(x => { const filePath = path.join(schemasFolder, x);