Prepare Version 0.19.1

Refactor to multistage builds

.dockerignore

@@ -2,3 +2,4 @@
 .travis.yml
 build*
 *.env
+.git

.gitignore

@@ -1 +1,2 @@
 *.sql.gz
+.netbox

DOCKER_HUB.md

@@ -12,28 +12,25 @@ Repository Links: Enable for Base Image
 Build Rules:
 - Source Type: Branch
   Source: master
-  Docker Tag: branches-main
+  Docker Tag: branches
   Dockerfile location: Dockerfile
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 - Source Type: Branch
   Source: master
-  Docker Tag: branches-ldap
-  Dockerfile location: Dockerfile.ldap
-- Source Type: Branch
-  Source: master
-  Docker Tag: prerelease-main
+  Docker Tag: prerelease
   Dockerfile location: Dockerfile
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 - Source Type: Branch
   Source: master
-  Docker Tag: prerelease-ldap
-  Dockerfile location: Dockerfile.ldap
-- Source Type: Branch
-  Source: master
-  Docker Tag: release-main
+  Docker Tag: release
   Dockerfile location: Dockerfile
-- Source Type: Branch
-  Source: master
-  Docker Tag: release-ldap
-  Dockerfile location: Dockerfile.ldap
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 Build Environment Variables:
 # Create an app on Github and use it's OATH credentials here
 - Key: GITHUB_OAUTH_CLIENT_ID
@@ -42,6 +39,7 @@ Build Environment Variables:
   Value: <secret>
 Build Triggers:
 - Name: Cron Trigger
+  Trigger URL: <generated>
 # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
 ```
 
@@ -51,16 +49,15 @@ The build system of cloud.docker.com is not made for this kind of project.
 But we found a way to make it work, and this is how:
 
 1. The docker hub build system [allows to overwrite the scripts that get executed
-   for `build`, `test` and `push`](overwrite). See `hooks/*`.
-2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`.
-3. The `build` script runs `run_build()` from `hooks/common`.
-   This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`.
+   for `build`, `test` and `push`](overwrite). See `/hooks/*`.
+2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`.
+3. The `build` script runs `run_build()` from `/hooks/common`.
+   This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`.
 4. The `test` script just invokes `docker-compose` commands.
 5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
    This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
 
-The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_).
-
-The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
+The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`.
+Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose.
 
 [overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands

Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.7-alpine3.10
+ARG FROM=python:3.7-alpine
+FROM ${FROM} as builder
 
 RUN apk add --no-cache \
       bash \
@@ -7,51 +8,55 @@ RUN apk add --no-cache \
       cyrus-sasl-dev \
       graphviz \
       jpeg-dev \
+      libevent-dev \
       libffi-dev \
-      libxml2-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev \
-      ttf-ubuntu-font-family \
-      wget
+      postgresql-dev
 
-RUN pip install \
+WORKDIR /install
+
+RUN pip install --install-option="--prefix=/install" \
 # gunicorn is used for launching netbox
       gunicorn \
+      greenlet \
+      eventlet \
 # napalm is used for gathering information from network devices
       napalm \
 # ruamel is used in startup_scripts
       'ruamel.yaml>=0.15,<0.16' \
-# pinning django to the version required by netbox
-# adding it here, to install the correct version of
-# django-rq
-      'Django>=2.2,<2.3' \
-# django-rq is used for webhooks
-      django-rq
+# django_auth_ldap is required for ldap
+      django_auth_ldap
 
-ARG BRANCH=master
+ARG NETBOX_PATH
+COPY ${NETBOX_PATH}/requirements.txt /
+RUN pip install --install-option="--prefix=/install" -r /requirements.txt 
 
-WORKDIR /tmp
+###
+# Main stage
+###
 
-# As the requirements don't change very often,
-# and as they take some time to compile,
-# we try to cache them very agressively.
-ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt
-ADD ${REQUIREMENTS_URL} requirements.txt
-RUN pip install -r requirements.txt
+ARG FROM
+FROM ${FROM} as main
 
-# Cache bust when the upstream branch changes:
-# ADD will fetch the file and check if it has changed
-# If not, Docker will use the existing build cache.
-# If yes, Docker will bust the cache and run every build step from here on.
-ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH
-ADD ${REF_URL} version.json
+RUN apk add --no-cache \
+      bash \
+      ca-certificates \
+      graphviz \
+      libevent \
+      libffi \
+      libjpeg-turbo \
+      libressl \
+      libxslt \
+      postgresql-libs \
+      ttf-ubuntu-font-family
 
 WORKDIR /opt
 
-ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz
-RUN wget -q -O - "${URL}" | tar xz \
-  && mv netbox* netbox
+COPY --from=builder /install /usr/local
+
+ARG NETBOX_PATH
+COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY configuration/gunicorn_config.py /etc/netbox/config/
@@ -67,7 +72,22 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
 CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
 
-LABEL SRC_URL="$URL"
+LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \
+      NETBOX_BRANCH="custom build" \
+      ORIGINAL_DOCKER_TAG="custom build" \
+      NETBOX_GIT_COMMIT="not built from git" \
+      NETBOX_GIT_URL="not built from git"
 
-ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot
-LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION"
+#####
+## LDAP specific configuration
+#####
+
+FROM main as ldap
+
+RUN apk add --no-cache \
+      libsasl \
+      libldap \
+      util-linux
+
+COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
+COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

Dockerfile.ldap

@@ -1,9 +0,0 @@
-ARG DOCKER_ORG=netboxcommunity
-ARG DOCKER_REPO=netbox
-ARG FROM_TAG=latest
-FROM $DOCKER_ORG/$DOCKER_REPO:$FROM_TAG
-
-RUN pip install django_auth_ldap
-
-COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
-COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

README.md

@@ -49,8 +49,8 @@ Default credentials:
 
 This project relies only on *Docker* and *docker-compose* meeting this requirements:
 
-* The *Docker version* must be at least `1.13.0`.
-* The *docker-compose version* must be at least `1.10.0`.
+* The *Docker version* must be at least `17.05`.
+* The *docker-compose version* must be at least `1.17.0`.
 
 To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above.
 
@@ -69,24 +69,20 @@ To use this feature, set the environment-variable `VERSION` before launching `do
 [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
 
 ```bash
-export VERSION=v2.2.6
+export VERSION=v2.6.6
 docker-compose pull netbox
 docker-compose up -d
 ```
 
-You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid
-[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github].
-Most commonly you will specify a tag or branch name.
+You can also build a specific version of the Netbox Docker image yourself.
+`VERSION` can be any valid [git ref][git-ref] in that case.
 
 ```bash
-export VERSION=develop
-docker-compose build --no-cache netbox
+export VERSION=v2.6.6
+./build.sh $VERSION
 docker-compose up -d
 ```
 
-Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary.
-This can increase the build speed if you're just adjusting the config, for example.
-
 [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 [netbox-github]: https://github.com/netbox-community/netbox/releases
 

VERSION

@@ -1 +1 @@
-0.18.2
+0.19.1

build-all.sh

@@ -12,73 +12,40 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
 
 echo "⚙️ Configured builds: ${BUILDS[*]}"
 
-VARIANTS=("main" "ldap")
-
 if [ -n "${DEBUG}" ]; then
   export DEBUG
 fi
 
 ERROR=0
 
-# Don't build if not on `master` and don't build if on a pull request,
-# but build when DEBUG is not empty
-for VARIANT in "${VARIANTS[@]}"; do
-  export VARIANT
-
-  # Checking which VARIANT to build
-  if [ "${VARIANT}" == "main" ]; then
-    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
-  else
-    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"
-
-    # Fail fast
-    if [ ! -f "${DOCKERFILE}" ]; then
-      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
-      ERROR=1
+for BUILD in "${BUILDS[@]}"; do
+  echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
+  case $BUILD in
+    release)
+      # build the latest release
+      # shellcheck disable=SC2068
+      ./build-latest.sh $@ || ERROR=1
+      ;;
+    prerelease)
+      # build the latest pre-release
+      # shellcheck disable=SC2068
+      PRERELEASE=true ./build-latest.sh $@ || ERROR=1
+      ;;
+    branches)
+      # build all branches
+      # shellcheck disable=SC2068
+      ./build-branches.sh $@ || ERROR=1
+      ;;
+    *)
+      echo "🚨 Unrecognized build '$BUILD'."
 
       if [ -z "$DEBUG" ]; then
-        continue
+        exit 1
       else
-        echo "⚠️ Would skip this, but DEBUG is enabled."
+        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
       fi
-    fi
-  fi
-
-  for BUILD in "${BUILDS[@]}"; do
-    echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
-    case $BUILD in
-      release)
-        # build the latest release
-        # shellcheck disable=SC2068
-        ./build-latest.sh $@ || ERROR=1
-        ;;
-      prerelease)
-        # build the latest pre-release
-        # shellcheck disable=SC2068
-        PRERELEASE=true ./build-latest.sh $@ || ERROR=1
-        ;;
-      branches)
-        # build all branches
-        # shellcheck disable=SC2068
-        ./build-branches.sh $@ || ERROR=1
-        ;;
-      special)
-        # special build
-        # shellcheck disable=SC2068
-        #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
-        echo "✅ No special builds today."
-        ;;
-      *)
-        echo "🚨 Unrecognized build '$BUILD'."
-
-        if [ -z "$DEBUG" ]; then
-          exit 1
-        else
-          echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-        fi
-        ;;
-    esac
-  done
+      ;;
+  esac
 done
 
 exit $ERROR

build-branches.sh

@@ -3,6 +3,10 @@
 
 echo "▶️ $0 $*"
 
+###
+# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
+# and GITHUB_OAUTH_CLIENT_SECRET
+###
 if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
   echo "🗝 Performing authenticated Github API calls."
   GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@@ -11,18 +15,33 @@ else
   GITHUB_OAUTH_PARAMS=""
 fi
 
+###
+# Calling Github to get the all branches
+###
 ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
 GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
 
+# Composing the JQ commans to extract the most recent version number
+JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
+
 CURL="curl -sS"
 
-BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string')
+# Querying the Github API to fetch all branches
+BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
 
+###
+# Building each branch
+###
+
+# keeping track whether an error occured
 ERROR=0
 
+# calling build.sh for each branch
 for BRANCH in $BRANCHES; do
   # shellcheck disable=SC2068
   ./build.sh "${BRANCH}" $@ || ERROR=1
 done
+
+# returning whether an error occured
 exit $ERROR

build-latest.sh

@@ -3,6 +3,10 @@
 
 echo "▶️ $0 $*"
 
+###
+# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
+# and GITHUB_OAUTH_CLIENT_SECRET
+###
 if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
   echo "🗝 Performing authenticated Github API calls."
   GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@@ -11,17 +15,38 @@ else
   GITHUB_OAUTH_PARAMS=""
 fi
 
+###
+# Checking if PRERELEASE is either unset, 'true' or 'false'
+###
+if [ -n "${PRERELEASE}" ] &&
+   { [ "${PRERELEASE}" != "true" ] && [ "${PRERELEASE}" != "false" ]; }; then
+
+  if [ -z "${DEBUG}" ]; then
+    echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!"
+    exit 1
+  else
+    echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
+  fi
+fi
+
+###
+# Calling Github to get the latest version
+###
 ORIGINAL_GITHUB_REPO="netbox-community/netbox"
 GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}"
 
+# Composing the JQ commans to extract the most recent version number
 JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name"
 
 CURL="curl -sS"
 
+# Querying the Github API to fetch the most recent version number
 VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}")
 
+###
 # Check if the prerelease version is actually higher than stable version
+###
 if [ "${PRERELEASE}" == "true" ]; then
   JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name"
   STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}")
@@ -35,9 +60,11 @@ if [ "${PRERELEASE}" == "true" ]; then
   # shellcheck disable=SC2003
   MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)')
 
-  if ( [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ] ) \
-     || [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then
-    echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')."
+  if { [ "${MAJOR_STABLE}" -eq "${MAJOR_UNSTABLE}" ] \
+       && [ "${MINOR_STABLE}" -ge "${MINOR_UNSTABLE}" ];
+     } || [ "${MAJOR_STABLE}" -gt "${MAJOR_UNSTABLE}" ]; then
+
+    echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'."
     if [ -z "$DEBUG" ]; then
       exit 0
     else
@@ -46,32 +73,6 @@ if [ "${PRERELEASE}" == "true" ]; then
   fi
 fi
 
-# Check if that version is not already available on docker hub:
-ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}"
-DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}"
-URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull"
-BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)"
-
-URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list"
-AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}"
-
-if [ -z "$VARIANT" ] || [ "$VARIANT" == "main" ]; then
-  DOCKER_TAG="${VERSION}"
-else
-  DOCKER_TAG="${VERSION}-${VARIANT}"
-fi
-
-ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")"
-
-if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then
-  if [ -n "$DEBUG" ]; then
-    echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled."
-  fi
-  
-  # shellcheck disable=SC2068
-  ./build.sh "${VERSION}" $@
-  exit $?
-else
-  echo "✅ ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}"
-  exit 0
-fi
+# shellcheck disable=SC2068
+./build.sh "${VERSION}" $@
+exit $?

build.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Builds the Dockerfile[.variant] and injects tgz'ed Netbox code from Github
+# Clones the Netbox repository with git from Github and builds the Dockerfile
 
 echo "▶️ $0 $*"
 
@@ -8,66 +8,78 @@ set -e
 if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "Usage: ${0} <branch> [--push|--push-only]"
   echo "  branch       The branch or tag to build. Required."
-  echo "  --push       Pushes built the Docker image to the registry."
-  echo "  --push-only  Does not build. Only pushes the Docker image to the registry."
+  echo "  --push       Pushes the built Docker image to the registry."
+  echo "  --push-only  Only pushes the Docker image to the registry, but does not build it."
   echo ""
   echo "You can use the following ENV variables to customize the build:"
-  echo "  DEBUG    If defined, the script does not stop when certain checks are unsatisfied."
-  echo "  DRY_RUN  Prints all build statements instead of running them."
-  echo "  DOCKER_OPTS Add parameters to Docker."
-  echo "           Default:"
-  echo "             When <TAG> starts with 'v':  \"\""
-  echo "             Else:                        \"--no-cache\""
-  echo "  BRANCH   The branch to build."
-  echo "           Also used for tagging the image."
-  echo "  TAG      The version part of the docker tag."
-  echo "           Default:"
-  echo "             When <BRANCH>=master:  latest"
-  echo "             When <BRANCH>=develop: snapshot"
-  echo "             Else:          same as <BRANCH>"
-  echo "  DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) "
-  echo "           Also used for tagging the image."
-  echo "           Default: netboxcommunity"
-  echo "  DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) "
-  echo "           Also used for tagging the image."
-  echo "           Default: netbox"
-  echo "  DOCKER_TAG The name of the tag which is applied to the image."
-  echo "           Useful for pushing into another registry than hub.docker.com."
-  echo "           Default: <DOCKER_ORG>/<DOCKER_REPO>:<BRANCH>"
-  echo "  DOCKER_SHORT_TAG The name of the short tag which is applied to the image."
-  echo "           This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5"
-  echo "           Default: <DOCKER_ORG>/<DOCKER_REPO>:\$MAJOR.\$MINOR"
-  echo "  SRC_ORG  Which fork of netbox to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
-  echo "           Default: netbox-community"
-  echo "  SRC_REPO The name of the netbox for to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
-  echo "           Default: netbox"
-  echo "  URL      Where to fetch the package from."
-  echo "           Must be a tar.gz file of the source code."
-  echo "           Default: https://github.com/<SRC_ORG>/<SRC_REPO>/archive/\$BRANCH.tar.gz"
-  echo "  VARIANT  The variant to build."
-  echo "           The value will be used as a suffix to the \$TAG and for the Dockerfile"
-  echo "           selection. The TAG being build must exist for the base variant and"
-  echo "           corresponding Dockerfile must start with the following lines:"
-  echo "             ARG DOCKER_ORG=netboxcommunity"
-  echo "             ARG DOCKER_REPO=netbox"
-  echo "             ARG FROM_TAG=latest"
-  echo "             FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG"
-  echo "           Example: VARIANT=ldap will result in the tag 'latest-ldap' and the"
-  echo "           Dockerfile './Dockerfile.ldap' being used."
-  echo "           Exception: VARIANT=main will use the './Dockerfile' Dockerfile"
-  echo "           Default: main"
-  echo "  HTTP_PROXY The proxy to use for http requests."
-  echo "           Example: http://proxy.domain.tld:3128"
-  echo "           Default: empty"
-  echo "  HTTPS_PROXY The proxy to use for https requests."
-  echo "           Example: http://proxy.domain.tld:3128"
-  echo "           Default: empty"
-  echo "  FTP_PROXY The proxy to use for ftp requests."
-  echo "           Example: http://proxy.domain.tld:3128"
-  echo "           Default: empty"
-  echo "  NO_PROXY Comma-separated list of domain extensions proxy should not be used for."
-  echo "           Example: .domain1.tld,.domain2.tld"
-  echo "           Default: empty"
+  echo "  SRC_ORG     Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
+  echo "              Default: netbox-community"
+  echo "  SRC_REPO    The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
+  echo "              Default: netbox"
+  echo "  URL         Where to fetch the code from."
+  echo "              Must be a git repository. Can be private."
+  echo "              Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git"
+  echo "  NETBOX_PATH The path where netbox will be checkout out."
+  echo "              Must not be outside of the netbox-docker repository (because of Docker)!"
+  echo "              Default: .netbox"
+  echo "  SKIP_GIT    If defined, git is not invoked and \${NETBOX_PATH} will not be altered."
+  echo "              This may be useful, if you are manually managing the NETBOX_PATH."
+  echo "              Default: undefined"
+  echo "  TAG         The version part of the docker tag."
+  echo "              Default:"
+  echo "                When \${BRANCH}=master:  latest"
+  echo "                When \${BRANCH}=develop: snapshot"
+  echo "                Else:          same as \${BRANCH}"
+  echo "  DOCKER_ORG  The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
+  echo "              Also used for tagging the image."
+  echo "              Default: netboxcommunity"
+  echo "  DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
+  echo "              Also used for tagging the image."
+  echo "              Default: netbox"
+  echo "  DOCKER_FROM The base image to use."
+  echo "              Default: Whatever is defined as default in the Dockerfile."
+  echo "  DOCKER_TAG  The name of the tag which is applied to the image."
+  echo "              Useful for pushing into another registry than hub.docker.com."
+  echo "              Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}"
+  echo "  DOCKER_SHORT_TAG The name of the short tag which is applied to the"
+  echo "              image. This is used to tag all patch releases to their"
+  echo "              containing version e.g. v2.5.1 -> v2.5"
+  echo "              Default: \${DOCKER_ORG}/\${DOCKER_REPO}:<MAJOR>.<MINOR>"
+  echo "  DOCKERFILE  The name of Dockerfile to use."
+  echo "              Default: Dockerfile"
+  echo "  DOCKER_TARGET A specific target to build."
+  echo "              It's currently not possible to pass multiple targets."
+  echo "              Default: main ldap"
+  echo "  HTTP_PROXY  The proxy to use for http requests."
+  echo "              Example: http://proxy.domain.tld:3128"
+  echo "              Default: undefined"
+  echo "  NO_PROXY    Comma-separated list of domain extensions proxy should not be used for."
+  echo "              Example: .domain1.tld,.domain2.tld"
+  echo "              Default: undefined"
+  echo "  DEBUG       If defined, the script does not stop when certain checks are unsatisfied."
+  echo "              Default: undefined"
+  echo "  DRY_RUN     Prints all build statements instead of running them."
+  echo "              Default: undefined"
+  echo ""
+  echo "Examples:"
+  echo "  ${0} master"
+  echo "              This will fetch the latest 'master' branch, build a Docker Image and tag it"
+  echo "              'netboxcommunity/netbox:latest'."
+  echo "  ${0} develop"
+  echo "              This will fetch the latest 'develop' branch, build a Docker Image and tag it"
+  echo "              'netboxcommunity/netbox:snapshot'."
+  echo "  ${0} v2.6.6"
+  echo "              This will fetch the 'v2.6.6' tag, build a Docker Image and tag it"
+  echo "              'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'."
+  echo "  ${0} develop-2.7"
+  echo "              This will fetch the 'develop-2.7' branch, build a Docker Image and tag it"
+  echo "              'netboxcommunity/netbox:develop-2.7'."
+  echo "  SRC_ORG=cimnine ${0} feature-x"
+  echo "              This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
+  echo "              build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'."
+  echo "  SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x"
+  echo "              This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
+  echo "              build a Docker Image and tag it 'cimnine/netbox:feature-x'."
 
   if [ "${1}x" == "x" ]; then
     exit 1
@@ -76,36 +88,73 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   fi
 fi
 
-# read the project version and trim it
+###
+# Determining the build command to use
+###
+if [ -z "${DRY_RUN}" ]; then
+  DRY=""
+else
+  echo "⚠️ DRY_RUN MODE ON ⚠️"
+  DRY="echo"
+fi
+
+###
+# read the project version from the `VERSION` file and trim it
 # see https://stackoverflow.com/a/3232433/172132
+###
 NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
 
+###
 # variables for fetching the source
+###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
 BRANCH="${1}"
-URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}"
+URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 
-# Checking which VARIANT to build
-VARIANT="${VARIANT-main}"
-if [ "$VARIANT" == "main" ]; then
-  DOCKERFILE="Dockerfile"
-else
-  DOCKERFILE="Dockerfile.${VARIANT}"
+###
+# fetching the source
+###
+if [ "${2}" != "--push-only" ] ; then
+  NETBOX_PATH="${NETBOX_PATH-.netbox}"
+  echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  if [ ! -d "${NETBOX_PATH}" ]; then
+    $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}"
+  fi
+
+  (
+    $DRY cd "${NETBOX_PATH}"
+
+    if [ -n "${HTTP_PROXY}" ]; then
+      git config http.proxy "${HTTP_PROXY}"
+    fi
+
+    $DRY git remote set-url origin "${URL}"
+    $DRY git fetch -qp --depth 10 origin "${BRANCH}"
+    $DRY git checkout -qf FETCH_HEAD
+    $DRY git prune
+  )
+  echo "✅ Checked out netbox"
 fi
 
-# Fail fast
+###
+# Determining the value for DOCKERFILE
+# and checking whether it exists
+###
+DOCKERFILE="${DOCKERFILE-Dockerfile}"
 if [ ! -f "${DOCKERFILE}" ]; then
-  echo "🚨 The Dockerfile ${DOCKERFILE} for variant '${VARIANT}' doesn't exist."
+  echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist."
 
-  if [ -z "$DEBUG" ]; then
+  if [ -z "${DEBUG}" ]; then
     exit 1
   else
     echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
   fi
 fi
 
+###
 # variables for tagging the docker image
+###
 DOCKER_ORG="${DOCKER_ORG-netboxcommunity}"
 DOCKER_REPO="${DOCKER_REPO-netbox}"
 case "${BRANCH}" in
@@ -117,81 +166,106 @@ case "${BRANCH}" in
     TAG="${TAG-$BRANCH}";;
 esac
 
-DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
-if [ "$VARIANT" != "main" ]; then
-  DOCKER_TAG="${DOCKER_TAG}-${VARIANT}"
-fi
+###
+# Determine targets to build
+###
+DEFAULT_DOCKER_TARGETS=("main" "ldap")
+DOCKER_TARGETS=( "${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}")
+echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
 
-if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
-  MAJOR=${BASH_REMATCH[1]}
-  MINOR=${BASH_REMATCH[2]}
+###
+# Build each target
+###
+for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
+  echo "🏗 Building the target '${DOCKER_TARGET}'"
 
-  DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
-
-  if [ "$VARIANT" != "main" ]; then
-    DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${VARIANT}"
+  ###
+  # composing the final TARGET_DOCKER_TAG
+  ###
+  TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
+  if [ "${DOCKER_TARGET}" != "main" ]; then
+    TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
   fi
-fi
 
-DOCKER_OPTS=("${DOCKER_OPTS[@]}")
+  ###
+  # composing the additional DOCKER_SHORT_TAG,
+  # i.e. "v2.6.1" becomes "v2.6",
+  # which is only relevant for version tags
+  ###
+  if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
+    MAJOR=${BASH_REMATCH[1]}
+    MINOR=${BASH_REMATCH[2]}
 
-# caching is only ok for version tags
-case "${TAG}" in
- v*) ;;
- *)  DOCKER_OPTS+=( "--no-cache" ) ;;
-esac
+    DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
 
-DOCKER_OPTS+=( "--pull" )
-
-# Build args
-DOCKER_BUILD_ARGS=(
-  --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}"
-  --build-arg "FROM_TAG=${TAG}"
-  --build-arg "BRANCH=${BRANCH}"
-  --build-arg "URL=${URL}"
-  --build-arg "DOCKER_ORG=${DOCKER_ORG}"
-  --build-arg "DOCKER_REPO=${DOCKER_REPO}"
-)
-if [ -n "$HTTP_PROXY" ]; then
-  DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" )
-fi
-if [ -n "$HTTPS_PROXY" ]; then
-  DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" )
-fi
-if [ -n "$FTP_PROXY" ]; then
-  DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" )
-fi
-if [ -n "$NO_PROXY" ]; then
-  DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" )
-fi
-
-if [ -z "$DRY_RUN" ]; then
-  DOCKER_CMD="docker"
-else
-  echo "⚠️ DRY_RUN MODE ON ⚠️"
-  DOCKER_CMD="echo docker"
-fi
-
-if [ "${2}" != "--push-only" ] ; then
-  echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
-  $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
-  echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
-
-  if [ -n "$DOCKER_SHORT_TAG" ]; then
-    echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'."
-    $DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}"
-    echo "✅ Tagged image '${DOCKER_SHORT_TAG}'"
+    if [ "${DOCKER_TARGET}" != "main" ]; then
+      DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
+    fi
   fi
-fi
 
-if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
-  echo "⏫ Pushing '${DOCKER_TAG}"
-  $DOCKER_CMD push "${DOCKER_TAG}"
-  echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'."
+  ###
+  # Proceeding to buils stage, except if `--push-only` is passed
+  ###
+  if [ "${2}" != "--push-only" ] ; then
+    ###
+    # Composing all arguments for `docker build`
+    ###
+    DOCKER_BUILD_ARGS=(
+      --pull
+      --target "${DOCKER_TARGET}"
+      -f "${DOCKERFILE}"
+      -t "${TARGET_DOCKER_TAG}"
+    )
+    if [ -n "${DOCKER_SHORT_TAG}" ]; then
+      DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" )
+    fi
 
-  if [ -n "$DOCKER_SHORT_TAG" ]; then
-    echo "⏫ Pushing '${DOCKER_SHORT_TAG}'"
-    $DOCKER_CMD push "${DOCKER_SHORT_TAG}"
-    echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'."
+    # --label
+    DOCKER_BUILD_ARGS+=(
+        --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}"
+        --label "NETBOX_BRANCH=${BRANCH}"
+        --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}"
+    )
+    if [ -d "${NETBOX_PATH}/.git" ]; then
+      DOCKER_BUILD_ARGS+=(
+        --label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)"
+        --label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)"
+      )
+    fi
+
+    # --build-arg
+    DOCKER_BUILD_ARGS+=(    --build-arg "NETBOX_PATH=${NETBOX_PATH}" )
+    if [ -n "${DOCKER_FROM}" ]; then
+      DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" )
+    fi
+    if [ -n "${HTTP_PROXY}" ]; then
+      DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" )
+      DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" )
+    fi
+    if [ -n "${NO_PROXY}" ]; then
+      DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" )
+    fi
+
+    ###
+    # Building the docker image
+    ###
+    echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
+    $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
+    echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
   fi
-fi
+
+  ###
+  # Pushing the docker images if either `--push` or `--push-only` are passed
+  ###
+  if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
+    echo "⏫ Pushing '${TARGET_DOCKER_TAG}"
+    $DRY docker push "${TARGET_DOCKER_TAG}"
+    echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'."
+
+    if [ -n "$DOCKER_SHORT_TAG" ]; then
+      echo "⏫ Pushing '${DOCKER_SHORT_TAG}'"
+      $DRY docker push "${DOCKER_SHORT_TAG}"
+      echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'."
+    fi
+  fi
+done

configuration/configuration.py

@@ -161,6 +161,10 @@ PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true'
 # this setting is derived from the installed location.
 REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')
 
+# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
+# this setting is derived from the installed location.
+SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
+
 # Time zone (default: UTC)
 TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
 

docker-compose.yml

@@ -1,10 +1,6 @@
-version: '3'
+version: '3.4'
 services:
   netbox: &netbox
-    build:
-      context: .
-      args:
-        - BRANCH=${VERSION-master}
     image: netboxcommunity/netbox:${VERSION-latest}
     depends_on:
     - postgres
@@ -16,6 +12,7 @@ services:
     - ./initializers:/opt/netbox/initializers:z,ro
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
+    - ./scripts:/etc/netbox/scripts:z,ro
     - netbox-nginx-config:/etc/netbox-nginx:z
     - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z

hooks/build

@@ -2,4 +2,5 @@
 
 . hooks/common
 
+# shellcheck disable=SC2119
 run_build

hooks/common

@@ -1,37 +1,19 @@
 #!/bin/bash
 
 ensure_jq() {
-  echo "🛠🛠🛠 Installing JQ via apt-get"
-  [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq )
-}
-
-ensure_dockerfile_present() {
-  if [ "${VARIANT}" == "main" ]; then
-    DOCKERFILE="Dockerfile"
-  else
-    DOCKERFILE="Dockerfile.${VARIANT}"
-
-    # Fail fast
-    if [ ! -f "${DOCKERFILE}" ]; then
-      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
-
-      if [ -z "$DEBUG" ]; then
-        exit 1
-      else
-        echo "⚠️ Would skip this, but DEBUG is enabled."
-      fi
-    fi
-
-    if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then
-      echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'."
-      echo "   This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'."
+  if [ ! -x "$(command -v jq)" ]; then
+    if [ -x "$(command -v apt-get)" ]; then
+      echo "🛠🛠🛠 Installing 'jq' via 'apt-get'"
+      apt-get update && apt-get install -y jq
+    else
+      echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'."
     fi
   fi
 }
 
 # Passes args to the scripts
 run_build() {
-  echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant"
+  echo "🐳🐳🐳 Building '${BUILD}' images"
   case $BUILD in
     release)
       # build the latest release
@@ -48,11 +30,11 @@ run_build() {
       # shellcheck disable=SC2068
       ./build-branches.sh $@
       ;;
-    special)
-      # special build
-      # shellcheck disable=SC2068
-      #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@
-      echo "✅ No special builds today."
+    this) # Pull Requests
+      # only build the 'master' branch
+      # (resulting in the 'latest' docker tag)
+      # and the 'main' target.
+      DOCKER_TARGET=main ./build.sh master
       ;;
     *)
       echo "🚨 Unrecognized build '$BUILD'."
@@ -70,13 +52,9 @@ echo "🤖🤖🤖 Preparing build"
 export DOCKER_ORG="index.docker.io/netboxcommunity"
 export DOCKER_REPO=netbox
 export DOCKERHUB_REPO=netboxcommunity/netbox
-
-# mis-using the "${DOCKER_TAG}" variable as "branch to build"
-export BUILD="${DOCKER_TAG%-*}"
-export VARIANT="${DOCKER_TAG#*-}"
+# shellcheck disable=SC2153
+export BUILD="${DOCKER_TAG}"
 
 unset DOCKER_TAG
 
-ensure_dockerfile_present
-
 ensure_jq

hooks/push

@@ -2,4 +2,13 @@
 
 . hooks/common
 
-run_build --push-only
+if [ "${SOURCE_BRANCH}" == "master" ] || [ "${DEBUG}" == "true" ]; then
+  if [ "${SOURCE_BRANCH}" != "master" ]; then
+    echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'".
+  fi
+
+  run_build --push-only
+else
+  echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'"
+  exit 0
+fi

hooks/test

@@ -2,11 +2,21 @@
 
 . hooks/common
 
-if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then
-  echo "🐳🐳🐳 Testing"
-  docker-compose pull --parallel
-  docker-compose build
-  docker-compose run netbox ./manage.py test
+run_test() {
+  echo "🐳🐳🐳 Testing '${1}'"
+  VERSION="${1}" docker-compose run netbox ./manage.py test
+  docker-compose down -v
+  echo "🐳🐳🐳 Done testing '${1}'"
+}
+
+# test on builds of 'branches'
+if [ "${BUILD}" == "branches" ] \
+   || [ "${DEBUG}" == "true" ]; then
+  run_test latest
+  run_test snapshot
+# test on bulds of 'this' (i.e. pull request)
+elif [ "${BUILD}" == "this" ]; then
+  run_test latest
 else
-  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'."
+  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'."
 fi