From 340c181442919732e7a749d0a1edc2e436b96e1c Mon Sep 17 00:00:00 2001 From: Googol Lee Date: Tue, 17 Sep 2024 15:02:23 +0200 Subject: [PATCH] Use self-building dependencies and update the code with imagemagick. (#1053) --- .github/workflows/build.yml | 2 +- .github/workflows/dependencies.yml | 2 +- Dockerfile | 48 ++++++-- api/go.mod | 8 +- api/go.sum | 16 +-- api/scanner/media_encoding/encode_photo.go | 18 +-- .../executable_worker/executable_worker.go | 66 +---------- .../executable_worker_test.go | 8 ++ .../executable_worker/magick_cli.go | 60 ++++++++++ .../executable_worker/magick_cli_test.go | 70 ++++++++++++ .../executable_worker/testdata/bin/magick | 18 +++ api/scanner/media_type/media_type.go | 2 +- dependencies/prepare.sh | 6 +- dev-compose.yaml | 2 +- scripts/install_build_dependencies.sh | 42 +------ scripts/install_runtime_dependencies.sh | 29 ++--- scripts/set_compiler_env.sh | 108 +++++++----------- 17 files changed, 280 insertions(+), 225 deletions(-) create mode 100644 api/scanner/media_encoding/executable_worker/magick_cli.go create mode 100644 api/scanner/media_encoding/executable_worker/magick_cli_test.go create mode 100755 api/scanner/media_encoding/executable_worker/testdata/bin/magick diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5dd576c5..67416039 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ env: DOCKER_USERNAME: viktorstrate DOCKER_IMAGE: viktorstrate/photoview DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 jobs: build: diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 5aecb2bd..9710c061 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -15,7 +15,7 @@ env: DOCKER_USERNAME: viktorstrate DOCKER_IMAGE: viktorstrate/dependencies DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 jobs: build: diff --git a/Dockerfile b/Dockerfile index d41844f2..d38e4582 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,13 +49,29 @@ ENV PATH="${GOPATH}/bin:${PATH}" ENV CGO_ENABLED=1 # Download dependencies -COPY scripts/*.sh /app/scripts/ +COPY scripts/set_compiler_env.sh /app/scripts/ RUN chmod +x /app/scripts/*.sh \ + && source /app/scripts/set_compiler_env.sh + +COPY --from=viktorstrate/dependencies /artifacts.tar.gz /dependencies/ +RUN export $(cat /env) \ + && cd /dependencies/ \ + && tar xfv artifacts.tar.gz \ + && cp -a include/* /usr/local/include/ \ + && cp -a pkgconfig/* ${PKG_CONFIG_PATH} \ + && cp -a lib/* /usr/local/lib/ \ + && cp -a bin/* /usr/local/bin/ \ + && ldconfig \ + && apt-get install -y ./deb/jellyfin-ffmpeg.deb + +COPY scripts/install_*.sh /app/scripts/ +RUN chmod +x /app/scripts/*.sh \ + && export $(cat /env) \ && /app/scripts/install_build_dependencies.sh \ && /app/scripts/install_runtime_dependencies.sh COPY api/go.mod api/go.sum /app/api/ -RUN source /app/scripts/set_compiler_env.sh \ +RUN export $(cat /env) \ && go env \ && go mod download \ # Patch go-face @@ -66,30 +82,44 @@ RUN source /app/scripts/set_compiler_env.sh \ github.com/Kagami/go-face COPY api /app/api -RUN source /app/scripts/set_compiler_env.sh \ +RUN export $(cat /env) \ + && go env \ && go build -v -o photoview . ### Build release image ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bookworm-slim AS release +FROM debian:bookworm-slim AS release ARG TARGETPLATFORM # See for details: https://github.com/hadolint/hadolint/wiki/DL4006 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] COPY scripts/install_runtime_dependencies.sh /app/scripts/ -RUN chmod +x /app/scripts/install_runtime_dependencies.sh \ +RUN --mount=type=bind,from=api,source=/dependencies/,target=/dependencies/ \ + chmod +x /app/scripts/install_runtime_dependencies.sh \ # Create a user to run Photoview server && groupadd -g 999 photoview \ && useradd -r -u 999 -g photoview -m photoview \ - # Required dependencies - && /app/scripts/install_runtime_dependencies.sh - -WORKDIR /home/photoview + # Install required dependencies + && /app/scripts/install_runtime_dependencies.sh \ + # Install self-building libs + && cd /dependencies \ + && cp -a lib/* /usr/local/lib/ \ + && cp -a bin/* /usr/local/bin/ \ + && ldconfig \ + && apt-get install -y ./deb/jellyfin-ffmpeg.deb \ + && ln -s /usr/lib/jellyfin-ffmpeg/ffmpeg /usr/local/bin/ \ + && ln -s /usr/lib/jellyfin-ffmpeg/ffprobe /usr/local/bin/ \ + # Cleanup + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY api/data /app/data COPY --from=ui /app/ui/dist /app/ui COPY --from=api /app/api/photoview /app/photoview +WORKDIR /home/photoview + ENV PHOTOVIEW_LISTEN_IP=127.0.0.1 ENV PHOTOVIEW_LISTEN_PORT=80 diff --git a/api/go.mod b/api/go.mod index cf8386ca..dead4c6b 100644 --- a/api/go.mod +++ b/api/go.mod @@ -18,7 +18,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/stretchr/testify v1.9.0 - github.com/strukturag/libheif v1.15.1 + github.com/strukturag/libheif v1.18.2 github.com/vektah/gqlparser/v2 v2.5.16 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 github.com/xor-gate/goexif2 v1.1.0 @@ -41,12 +41,12 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/kr/text v0.1.0 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/mattn/go-sqlite3 v1.14.23 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect diff --git a/api/go.sum b/api/go.sum index d8757f34..a3adf4b3 100644 --- a/api/go.sum +++ b/api/go.sum @@ -48,10 +48,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -63,8 +63,8 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= @@ -91,8 +91,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/strukturag/libheif v1.15.1 h1:PWMRTk+9HG0a9avvlV597iI0AdHk25zKVV33lSl6a+I= -github.com/strukturag/libheif v1.15.1/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= +github.com/strukturag/libheif v1.18.2 h1:CrlRS7Kwl2odl4DYM/m6ay/HPXEcmQPK1xF8FxAqP7k= +github.com/strukturag/libheif v1.18.2/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= diff --git a/api/scanner/media_encoding/encode_photo.go b/api/scanner/media_encoding/encode_photo.go index adf4d8ec..eccaaaed 100644 --- a/api/scanner/media_encoding/encode_photo.go +++ b/api/scanner/media_encoding/encode_photo.go @@ -22,12 +22,12 @@ import ( ) var thumbFilter = map[models.ThumbnailFilter]imaging.ResampleFilter{ - models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, - models.ThumbnailFilterBox: imaging.Box, - models.ThumbnailFilterLinear: imaging.Linear, - models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, - models.ThumbnailFilterCatmullRom: imaging.CatmullRom, - models.ThumbnailFilterLanczos: imaging.Lanczos, + models.ThumbnailFilterNearestNeighbor: imaging.NearestNeighbor, + models.ThumbnailFilterBox: imaging.Box, + models.ThumbnailFilterLinear: imaging.Linear, + models.ThumbnailFilterMitchellNetravali: imaging.MitchellNetravali, + models.ThumbnailFilterCatmullRom: imaging.CatmullRom, + models.ThumbnailFilterLanczos: imaging.Lanczos, } func EncodeThumbnail(db *gorm.DB, inputPath string, outputPath string) (*media_utils.PhotoDimensions, error) { @@ -108,10 +108,10 @@ func (img *EncodeMediaData) EncodeHighRes(outputPath string) error { return errors.New("could not convert photo as file format is not supported") } - // Use darktable if there is no counterpart JPEG file to use instead + // Use magick if there is no counterpart JPEG file to use instead if contentType.IsRaw() && img.CounterpartPath == nil { - if executable_worker.DarktableCli.IsInstalled() { - err := executable_worker.DarktableCli.EncodeJpeg(img.Media.Path, outputPath, 70) + if executable_worker.Magick.IsInstalled() { + err := executable_worker.Magick.EncodeJpeg(img.Media.Path, outputPath, 70) if err != nil { return err } diff --git a/api/scanner/media_encoding/executable_worker/executable_worker.go b/api/scanner/media_encoding/executable_worker/executable_worker.go index 2ada358f..c2b9b6d6 100644 --- a/api/scanner/media_encoding/executable_worker/executable_worker.go +++ b/api/scanner/media_encoding/executable_worker/executable_worker.go @@ -2,9 +2,7 @@ package executable_worker import ( "fmt" - "io/ioutil" "log" - "os" "os/exec" "strings" @@ -14,51 +12,21 @@ import ( ) func InitializeExecutableWorkers() { - DarktableCli = newDarktableWorker() + Magick = newMagickCli() FfmpegCli = newFfmpegWorker() } -var DarktableCli *DarktableWorker = nil +var Magick *MagickCli = nil var FfmpegCli *FfmpegWorker = nil type ExecutableWorker interface { Path() string } -type DarktableWorker struct { - path string -} - type FfmpegWorker struct { path string } -func newDarktableWorker() *DarktableWorker { - if utils.EnvDisableRawProcessing.GetBool() { - log.Printf("Executable worker disabled (%s=1): darktable\n", utils.EnvDisableRawProcessing.GetName()) - return nil - } - - path, err := exec.LookPath("darktable-cli") - if err != nil { - log.Println("Executable worker not found: darktable") - } else { - version, err := exec.Command(path, "--version").Output() - if err != nil { - log.Printf("Error getting version of darktable: %s\n", err) - return nil - } - - log.Printf("Found executable worker: darktable (%s)\n", strings.Split(string(version), "\n")[0]) - - return &DarktableWorker{ - path: path, - } - } - - return nil -} - func newFfmpegWorker() *FfmpegWorker { if utils.EnvDisableVideoEncoding.GetBool() { log.Printf("Executable worker disabled (%s=1): ffmpeg\n", utils.EnvDisableVideoEncoding.GetName()) @@ -85,40 +53,10 @@ func newFfmpegWorker() *FfmpegWorker { return nil } -func (worker *DarktableWorker) IsInstalled() bool { - return worker != nil -} - func (worker *FfmpegWorker) IsInstalled() bool { return worker != nil } -func (worker *DarktableWorker) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { - tmpDir, err := ioutil.TempDir("/tmp", "photoview-darktable") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - args := []string{ - inputPath, - outputPath, - "--core", - "--conf", - fmt.Sprintf("plugins/imageio/format/jpeg/quality=%d", jpegQuality), - "--configdir", - tmpDir, - } - - cmd := exec.Command(worker.path, args...) - - if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "encoding image using: %s %v", worker.path, args) - } - - return nil -} - func (worker *FfmpegWorker) EncodeMp4(inputPath string, outputPath string) error { args := []string{ "-i", diff --git a/api/scanner/media_encoding/executable_worker/executable_worker_test.go b/api/scanner/media_encoding/executable_worker/executable_worker_test.go index 8f66a5f6..756ffeb9 100644 --- a/api/scanner/media_encoding/executable_worker/executable_worker_test.go +++ b/api/scanner/media_encoding/executable_worker/executable_worker_test.go @@ -33,3 +33,11 @@ func setPathWithCurrent(paths ...string) func() { os.Setenv("PATH", originalPath) } } + +func setEnv(key, value string) func() { + org := os.Getenv(key) + os.Setenv(key, value) + return func() { + os.Setenv(key, org) + } +} diff --git a/api/scanner/media_encoding/executable_worker/magick_cli.go b/api/scanner/media_encoding/executable_worker/magick_cli.go new file mode 100644 index 00000000..d699adc9 --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/magick_cli.go @@ -0,0 +1,60 @@ +package executable_worker + +import ( + "fmt" + "log" + "os/exec" + "strings" + + "github.com/photoview/photoview/api/utils" +) + +type MagickCli struct { + path string +} + +func newMagickCli() *MagickCli { + if utils.EnvDisableRawProcessing.GetBool() { + log.Printf("Executable worker disabled (%s=%q): ImageMagick\n", utils.EnvDisableRawProcessing.GetName(), utils.EnvDisableRawProcessing.GetValue()) + return nil + } + + path, err := exec.LookPath("magick") + if err != nil { + log.Println("Executable worker not found: magick") + return nil + } + + version, err := exec.Command(path, "-version").Output() + if err != nil { + log.Printf("Error getting version of magick: %s\n", err) + return nil + } + + log.Printf("Found executable worker: magick (%s)\n", strings.Split(string(version), "\n")[0]) + + return &MagickCli{ + path: path, + } +} + +func (cli *MagickCli) IsInstalled() bool { + return cli != nil +} + +func (cli *MagickCli) EncodeJpeg(inputPath string, outputPath string, jpegQuality int) error { + args := []string{ + "convert", + inputPath, + "-quality", fmt.Sprintf("%d", jpegQuality), + outputPath, + } + + cmd := exec.Command(cli.path, args...) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("encoding image with \"%s %v\" error: %w", cli.path, args, err) + } + + return nil +} diff --git a/api/scanner/media_encoding/executable_worker/magick_cli_test.go b/api/scanner/media_encoding/executable_worker/magick_cli_test.go new file mode 100644 index 00000000..6696f19b --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/magick_cli_test.go @@ -0,0 +1,70 @@ +package executable_worker_test + +import ( + "regexp" + "testing" + + "github.com/photoview/photoview/api/scanner/media_encoding/executable_worker" +) + +func TestMagickCliNotExist(t *testing.T) { + done := setPathWithCurrent() + defer done() + + executable_worker.InitializeExecutableWorkers() + if executable_worker.Magick.IsInstalled() { + t.Error("MagickCli should not be installed, but is found:", executable_worker.Magick) + } +} + +func TestMagickCliIgnore(t *testing.T) { + donePath := setPathWithCurrent("./testdata/bin") + defer donePath() + + doneDisableRaw := setEnv("PHOTOVIEW_DISABLE_RAW_PROCESSING", "true") + defer doneDisableRaw() + + executable_worker.InitializeExecutableWorkers() + if executable_worker.Magick.IsInstalled() { + t.Error("MagickCli should not be installed, but is found:", executable_worker.Magick) + } +} + +func TestMagickCliFail(t *testing.T) { + donePath := setPathWithCurrent("./testdata/bin") + defer donePath() + + executable_worker.InitializeExecutableWorkers() + if !executable_worker.Magick.IsInstalled() { + t.Fatal("MagickCli should be installed") + } + + done := setEnv("FAIL_WITH", "failure") + defer done() + + err := executable_worker.Magick.EncodeJpeg("input", "output", 70) + if err == nil { + t.Fatalf(`MagickCli.EncodeJpeg(...) = nil, should be an error.`) + } + + if got, want := err.Error(), `^encoding image with ".*/testdata/bin/magick \[convert input -quality 70 output\]" error: .*$`; !regexp.MustCompile(want).MatchString(got) { + t.Errorf(`MagickCli.EncodeJpeg(...) = %q, should be matched with reg pattern %q`, got, want) + } +} + +func TestMagickCliSucceed(t *testing.T) { + donePath := setPathWithCurrent("./testdata/bin") + defer donePath() + + executable_worker.InitializeExecutableWorkers() + if !executable_worker.Magick.IsInstalled() { + t.Fatal("MagickCli should be installed") + } + + t.Run("Succeeded", func(t *testing.T) { + err := executable_worker.Magick.EncodeJpeg("input", "output", 70) + if err != nil { + t.Fatalf("MagickCli.EncodeJpeg(...) = %v, should be nil.", err) + } + }) +} diff --git a/api/scanner/media_encoding/executable_worker/testdata/bin/magick b/api/scanner/media_encoding/executable_worker/testdata/bin/magick new file mode 100755 index 00000000..ac2b2709 --- /dev/null +++ b/api/scanner/media_encoding/executable_worker/testdata/bin/magick @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +: ${FAIL_WITH=""} + +if [ "$1" = "-version" ] +then + echo magick: version fake + exit 0 +fi + +if [ "${FAIL_WITH}" != "" ] +then + echo ${FAIL_WITH} + exit -1 +fi + +echo $@ diff --git a/api/scanner/media_type/media_type.go b/api/scanner/media_type/media_type.go index 4b72f27f..83b711b5 100644 --- a/api/scanner/media_type/media_type.go +++ b/api/scanner/media_type/media_type.go @@ -260,7 +260,7 @@ func (imgType *MediaType) IsSupported() bool { return true } - if executable_worker.DarktableCli.IsInstalled() && imgType.IsRaw() { + if executable_worker.Magick.IsInstalled() && imgType.IsRaw() { return true } diff --git a/dependencies/prepare.sh b/dependencies/prepare.sh index c21aabce..94d81d05 100755 --- a/dependencies/prepare.sh +++ b/dependencies/prepare.sh @@ -10,10 +10,10 @@ TARGETVARIANT="$(echo $TARGETPLATFORM | cut -d"/" -f3)" DEBIAN_ARCH=$TARGETARCH if [ "$TARGETARCH" = "arm" ] then - DEBIAN_ARCH=armhf - if [ "$TARGETVARIANT" = "v5" ] + DEBIAN_ARCH=armel + if [ "$TARGETVARIANT" = "v7" ] then - DEBIAN_ARCH=armel + DEBIAN_ARCH=armhf fi fi diff --git a/dev-compose.yaml b/dev-compose.yaml index 20150cab..5ca1a408 100644 --- a/dev-compose.yaml +++ b/dev-compose.yaml @@ -39,7 +39,7 @@ services: - /bin/bash - -c - | - source /app/scripts/set_compiler_env.sh + export $(cat /env) reflex -g '*.go' -s -- go run . mariadb: diff --git a/scripts/install_build_dependencies.sh b/scripts/install_build_dependencies.sh index a1d4abb1..20f44c4e 100755 --- a/scripts/install_build_dependencies.sh +++ b/scripts/install_build_dependencies.sh @@ -1,42 +1,12 @@ -#!/bin/bash -set -e +#!/bin/sh +set -eu -if [ "$TARGETPLATFORM" == "linux/arm64" ]; then - dpkg --add-architecture arm64 - DEBIAN_ARCH='arm64' -elif [ "$TARGETPLATFORM" == "linux/arm/v6" ] || [ "$TARGETPLATFORM" == "linux/arm/v7" ]; then - dpkg --add-architecture armhf - DEBIAN_ARCH='armhf' -else - dpkg --add-architecture amd64 - DEBIAN_ARCH='amd64' -fi +: ${DEB_HOST_ARCH=`dpkg --print-architecture`} +echo Arch: ${DEB_HOST_ARCH} -apt-get update - -# Install G++/GCC cross compilers -if [ "$DEBIAN_ARCH" == "arm64" ]; then - apt-get install -y \ - g++-aarch64-linux-gnu \ - libc6-dev-arm64-cross -elif [ "$DEBIAN_ARCH" == "armhf" ]; then - apt-get install -y \ - g++-arm-linux-gnueabihf \ - libc6-dev-armhf-cross -else - apt-get install -y \ - g++-x86-64-linux-gnu \ - libc6-dev-amd64-cross -fi - -# Install go-face dependencies and libheif for HEIF media decoding +# Install go-face dependencies apt-get install -y \ - libdlib-dev:${DEBIAN_ARCH} \ - libblas-dev:${DEBIAN_ARCH} \ - libatlas-base-dev:${DEBIAN_ARCH} \ - liblapack-dev:${DEBIAN_ARCH} \ - libjpeg-dev:${DEBIAN_ARCH} \ - libheif-dev:${DEBIAN_ARCH} + libdlib-dev:${DEB_HOST_ARCH} libblas-dev:${DEB_HOST_ARCH} libatlas-base-dev:${DEB_HOST_ARCH} liblapack-dev:${DEB_HOST_ARCH} libjpeg-dev:${DEB_HOST_ARCH} # Install tools for development apt-get install -y reflex sqlite3 diff --git a/scripts/install_runtime_dependencies.sh b/scripts/install_runtime_dependencies.sh index 7698b008..3eb41fb6 100755 --- a/scripts/install_runtime_dependencies.sh +++ b/scripts/install_runtime_dependencies.sh @@ -1,22 +1,13 @@ -#!/bin/bash +#!/bin/sh +set -eu apt-get update -apt-get install -y curl libdlib19.1 ffmpeg exiftool libheif1 +apt-get install -y curl exiftool -# Install Darktable if building for a supported architecture -if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then - echo 'deb [trusted=true] https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/ /' > /etc/apt/sources.list.d/darktable.list - # Release key is invalid, just trust the repo - curl -fsSL https://download.opensuse.org/repositories/graphics:/darktable/Debian_12/Release.key \ - | gpg --dearmor -o /etc/apt/trusted.gpg.d/darktable.gpg - gpg --show-keys --with-fingerprint --dry-run /etc/apt/trusted.gpg.d/darktable.gpg - - apt-get update - apt-get install -y darktable -fi - -# Remove build dependencies and cleanup -apt-get purge -y ${BUILD_DEPENDS[@]} -apt-get autoremove -y -apt-get clean -rm -rf /var/lib/apt/lists/* +# libheif dependencies: no dependency +# libraw dependencies +apt-get install -y libjpeg62-turbo liblcms2-2 +# ImageMagick dependencies +apt-get install -y libjbig0 libtiff6 libfreetype6 libjxl0.7 liblqr-1-0 libpng16-16 libdjvulibre21 libwebpmux3 libwebpdemux2 libwebp7 libopenexr-3-1-30 libopenjp2-7 libjpeg62-turbo liblcms2-2 libxml2 libx11-6 libgomp1 +# go-face dependencies +apt-get install -y libdlib19.1 libblas3 liblapack3 libjpeg62-turbo diff --git a/scripts/set_compiler_env.sh b/scripts/set_compiler_env.sh index a7744e6b..9608ec29 100755 --- a/scripts/set_compiler_env.sh +++ b/scripts/set_compiler_env.sh @@ -1,84 +1,54 @@ #!/bin/sh - -# Script to configure environment variables for Go compiler -# to allow cross compilation - -: ${TARGETPLATFORM=} -: ${TARGETOS=} -: ${TARGETARCH=} -: ${TARGETVARIANT=} - -CGO_ENABLED="$(go env CGO_ENABLED)" -GOARCH="$(go env GOARCH)" -GOOS="$(go env GOOS)" -GOARM="$(go env GOARM)" -GOBIN="$(go env GOBIN)" - set -eu -if [ ! -z "$TARGETPLATFORM" ]; then - TARGETOS="$(echo $TARGETPLATFORM | cut -d"/" -f1)" - TARGETARCH="$(echo $TARGETPLATFORM | cut -d"/" -f2)" - TARGETVARIANT="$(echo $TARGETPLATFORM | cut -d"/" -f3)" -fi +# Configure environment for cross-compiling. -if [ ! -z "$TARGETOS" ]; then - export GOOS="$TARGETOS" -fi +: ${TARGETPLATFORM=linux/`dpkg --print-architecture`} -if [ ! -z "$TARGETARCH" ]; then - export GOARCH="$TARGETARCH" -fi +TARGETOS="$(echo $TARGETPLATFORM | cut -d"/" -f1)" +TARGETARCH="$(echo $TARGETPLATFORM | cut -d"/" -f2)" +TARGETVARIANT="$(echo $TARGETPLATFORM | cut -d"/" -f3)" -if [ "$TARGETARCH" = "arm" ]; then - if [ ! -z "$TARGETVARIANT" ]; then - case "$TARGETVARIANT" in - "v5") - export GOARM="5" - ;; - "v6") - export GOARM="6" - ;; - *) - export GOARM="7" - ;; - esac - else - export GOARM="7" +DEBIAN_ARCH=$TARGETARCH +if [ "$TARGETARCH" = "arm" ] +then + DEBIAN_ARCH=armel + if [ "$TARGETVARIANT" = "v7" ] + then + DEBIAN_ARCH=armhf fi fi -if [ "$CGO_ENABLED" = "1" ]; then - case "$GOARCH" in - "amd64") - export COMPILER_ARCH="x86_64-linux-gnu" +dpkg --add-architecture $DEBIAN_ARCH +apt-get update +apt-get install -y git curl crossbuild-essential-${DEBIAN_ARCH} libc-dev:${DEBIAN_ARCH} autoconf automake libtool m4 pkg-config cmake + +dpkg-architecture -a $DEBIAN_ARCH >/env +export $(cat /env) + +CGO_ENABLED="1" +GOOS="$TARGETOS" +GOARCH="$TARGETARCH" + +GOARM="7" +if [ "$TARGETARCH" = "arm" && ! -z "$TARGETVARIANT" ]; then + case "$TARGETVARIANT" in + "v5") + export GOARM="5" ;; - "ppc64le") - export COMPILER_ARCH="powerpc64le-linux-gnu" - ;; - "s390x") - export COMPILER_ARCH="s390x-linux-gnu" - ;; - "arm64") - export COMPILER_ARCH="aarch64-linux-gnu" - ;; - "arm") - case "$GOARM" in - "5") - export COMPILER_ARCH="arm-linux-gnueabi" - ;; - *) - export COMPILER_ARCH="arm-linux-gnueabihf" - ;; - esac + "v6") + export GOARM="6" ;; esac fi -export CC="${COMPILER_ARCH}-gcc" -export CXX="${COMPILER_ARCH}-g++" -export PKG_CONFIG_PATH="/usr/lib/${COMPILER_ARCH}/pkgconfig/" +echo CGO_ENABLED="${CGO_ENABLED}" >>/env +echo GOOS="${GOOS}" >>/env +echo GOARCH="${GOARCH}" >>/env +echo GOARM="${GOARM}" >>/env +echo AR="${DEB_HOST_MULTIARCH}-ar" >>/env +echo CC="${DEB_HOST_MULTIARCH}-gcc" >>/env +echo CXX="${DEB_HOST_MULTIARCH}-g++" >>/env +echo PKG_CONFIG_PATH="/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig/" >>/env -if [ -z "$GOBIN" ] && [ -n "$GOPATH" ] && [ -n "$GOARCH" ] && [ -n "$GOOS" ]; then - export PATH=${GOPATH}/bin/${GOOS}_${GOARCH}:${PATH} -fi +cat /env