From 1b014a15369e2fadc07e92855e104343b0e1a35f Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 24 Jul 2024 15:39:03 -0400 Subject: [PATCH] Improve Dockerfile with cache, Bazelisk, and cmake support (#2643) Allow `bazel` and `cmake` to be used with docker by simply adding them after `docker run ...`. Moreover it is designed to keep build and bazel cache between runs, which means no re-downloads between separate `docker run` commands. Co-authored-by: Bart Louwers --- docker/.gitignore | 2 + docker/Dockerfile | 53 +++++++++++++++++++ .../linux => docker}/Dockerfile.dockerignore | 1 + docker/README.md | 43 +++++++++++++++ docker/startup.sh | 13 +++++ platform/linux/Dockerfile | 33 ------------ platform/linux/README.md | 31 ++--------- 7 files changed, 116 insertions(+), 60 deletions(-) create mode 100644 docker/.gitignore create mode 100644 docker/Dockerfile rename {platform/linux => docker}/Dockerfile.dockerignore (92%) create mode 100644 docker/README.md create mode 100644 docker/startup.sh delete mode 100644 platform/linux/Dockerfile diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 000000000..0b9eecfac --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +.cache/ +!.cache/empty_file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..8bc737daf --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:22.04 + +# Install build tools and dependencies +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + apt-transport-https \ + curl \ + gnupg \ + sudo `# allows dev to install more packages without switching to root or rebuilding container` \ + build-essential \ + libsqlite3-dev \ + libcurl4-openssl-dev \ + libglfw3-dev \ + libuv1-dev \ + libpng-dev \ + libicu-dev \ + libjpeg-turbo8-dev \ + libwebp-dev \ + xvfb \ + clang \ + git \ + cmake \ + ccache \ + ninja-build \ + pkg-config \ + && : # end of the RUN cmd - easier to keep a colon at the end of the list, than to keep the backslashes in check + +# This could also be `.../releases/latest/download/bazelisk-linux-amd64` for the latest version, but for predictability better hardcode it +RUN curl -fsSL https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 -o /usr/local/bin/bazel \ + && chmod +x /usr/local/bin/bazel \ + && : + +WORKDIR /app + +ARG USERNAME=user +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create docker user wuth sudo rights as passed in by the build command +# This was modeled on https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +COPY startup.sh /usr/local/bin/startup.sh +RUN chmod +x /usr/local/bin/startup.sh + +# This allows users to `docker run` without specifying -u and -g +USER $USERNAME + +ENTRYPOINT ["/usr/local/bin/startup.sh"] +CMD ["bash"] diff --git a/platform/linux/Dockerfile.dockerignore b/docker/Dockerfile.dockerignore similarity index 92% rename from platform/linux/Dockerfile.dockerignore rename to docker/Dockerfile.dockerignore index 2ce0f5f39..023bfa416 100644 --- a/platform/linux/Dockerfile.dockerignore +++ b/docker/Dockerfile.dockerignore @@ -1,3 +1,4 @@ # Exclude everything because we expect users to mount this dir as a volume, # so none of this repos' content should be part of the docker build context. * +!startup.sh diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..5a090f86d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,43 @@ +# Building with Docker + +These steps will allow you to compile code as described [platform/linux/README.md](../platform/linux/README.md) using a Docker container. All the steps should be executed from the root of the repository. + +> [!IMPORTANT] +> Not all platform builds are currently supported. Docker builds are a work in progress. + +> [!IMPORTANT] +> You cannot build MapLibre native using both Docker and host methods at the same time. If you want to switch, you need to clean the repository first, e.g. by using this command: +> +> ```bash +> git clean -dxfi -e .idea -e .clwb -e .vscode +> ``` + +### Build Docker Image + +You must build your own Docker image, specific with your user and group IDs to ensure file permissions stay correct. + +```bash +# Build docker image from the repo __root__ +# Specifying USER_UID and USER_GID allows container to create files with the same owner as the host user, +# and avoids having to pass -u $(id -u):$(id -g) to docker run. +docker build \ + -t maplibre-native-image \ + --build-arg USER_UID=$(id -u) \ + --build-arg USER_GID=$(id -g) \ + -f docker/Dockerfile \ + docker +``` + +## Run Docker Container + +```bash +# Run all build commands using the docker container. +# You can also execute build commands from inside the docker container by starting it without the build command. +docker run --rm -it -v "$PWD:/app/" -v "$PWD/docker/.cache:/home/user/.cache" maplibre-native-image +``` + +You can also use the container to run just one specific commands, e.g. `cmake` or `bazel`. Any downloaded dependencies will be cached in the `docker/.cache` directory. + +```bash +docker run --rm -it -v "$PWD:/app/" -v "$PWD/docker/.cache:/home/user/.cache" maplibre-native-image cmake ... +``` diff --git a/docker/startup.sh b/docker/startup.sh new file mode 100644 index 000000000..6bb1f9a17 --- /dev/null +++ b/docker/startup.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +if [ ! -d /app/.github ] || [ ! -d /home/user/.cache ]; then + echo " " + echo "ERROR: Docker container was not started properly." + echo " From the root of this repo, run the following command." + echo " You may add any command to perform in the container at the end of this command." + echo " " + echo ' docker run --rm -it -v "$PWD:/app/" -v "$PWD/docker/.cache:/home/user/.cache" maplibre-native-image' + exit 1 +fi + +exec "$@" diff --git a/platform/linux/Dockerfile b/platform/linux/Dockerfile deleted file mode 100644 index b55061954..000000000 --- a/platform/linux/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:22.04 - -# When running docker with -u flag, the user should exist in the container to avoid various warnings -RUN groupadd -f user && useradd -m -g user user - -# Install build tools and dependencies -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - apt-transport-https \ - curl \ - gnupg \ - && curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor >bazel-archive-keyring.gpg \ - && mv bazel-archive-keyring.gpg /usr/share/keyrings \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - bazel \ - libcurl4-openssl-dev \ - libglfw3-dev \ - libuv1-dev \ - libpng-dev \ - libicu-dev \ - libjpeg-turbo8-dev \ - libwebp-dev \ - xvfb \ - clang \ - git \ - cmake \ - ccache \ - ninja-build \ - pkg-config - -WORKDIR /code diff --git a/platform/linux/README.md b/platform/linux/README.md index 79688d84a..371d4ae13 100644 --- a/platform/linux/README.md +++ b/platform/linux/README.md @@ -12,11 +12,12 @@ cd maplibre-native ``` ## Requirements + ### Building without Docker ```bash # Install build tools -apt install clang cmake ccache ninja-build pkg-config +apt install build-essential clang cmake ccache ninja-build pkg-config # Install system dependencies apt install libcurl4-openssl-dev libglfw3-dev libuv1-dev libpng-dev libicu-dev libjpeg-turbo8-dev libwebp-dev xvfb @@ -26,26 +27,7 @@ Optional: `libsqlite3-dev` (also available as vendored dependency). ### Building with Docker -You can use a Docker container to build MapLibre Native. A `Dockerfile` that installes the required dependencies when the image is built is provided in this directory. - -```bash -# Build docker image from the repo root -docker build -t maplibre-native-image -f platform/linux/Dockerfile . - -# Run docker image as the current user. -# This ensures that the files created in the container are owned by the current user. -docker run --rm -it -v "$PWD:/code/" -u $(id -u):$(id -g) maplibre-native-image ___any_build_command___ - -# You can also execute build commands from inside the docker container by starting it without parameters: -docker run --rm -it -v "$PWD:/code/" -u $(id -u):$(id -g) maplibre-native-image -``` - -You can safely ignore this type of message and a missing username. It happens if your linux user was not the first one created on the system. - -``` -groups: cannot find name for group ID .... -I have no name!@...:/root$ ` -``` +See instructions in [docker/README.md](../../docker/README.md) to build in a docker container. ## Build @@ -57,22 +39,17 @@ cmake -B build -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DC cmake --build build --target mbgl-render -j $(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null) ``` - ## Running `mbgl-render` Running `mbgl-render --style https://raw.githubusercontent.com/maplibre/demotiles/gh-pages/style.json` should produce a map tile image with the default MapLibre styling from [the MapLibre demo](https://maplibre.org/). ![Sample image of world from mbgl-render command](/misc/sample-maplibre-style-mbgl-render-out.png) -### Outside of Docker - -Render output image using default MapLibre demo tiles server. You can run this binary on your host machine even if you built it inside a Docker container. - ```bash ./build/bin/mbgl-render --style https://raw.githubusercontent.com/maplibre/demotiles/gh-pages/style.json --output out.png xdg-open out.png ``` -### Inside Docker +### Headless rendering If you run `mbgl-render` inside a Docker or on a remote headless server, you will likely get this error because there is no X server running in the container.