Merge pull request #419 from netbox-community/develop

Version 1.0.1

.dockerignore

@@ -7,5 +7,4 @@ build*
 docker-compose.override.yml
 .netbox/.git*
 .netbox/.travis.yml
-.netbox/docs
 .netbox/scripts

.github/FUNDING.yml

@@ -0,0 +1,14 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+- cimnine
+- tobiasge
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -65,13 +65,3 @@ If your log is very long, create a Gist instead (and post the link to it): https
 ```text
 LOG LOG LOG
 ```
-
-The output of `docker-compose logs nginx`:
-<!--
-Only if you have gotten a 5xx http error, else delete this section.
-If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com
--->
-
-```text
-LOG LOG LOG
-```

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,13 @@
+blank_issues_enabled: false
+contact_links:
+  - name: The \#netbox-docker Slack channel
+    url: http://slack.networktocode.com/
+    about: It's usually the quickest way to seek help when you're in trouble with regards to Netbox Docker.
+    
+  - name: Github Discussions
+    url: https://github.com/netbox-community/netbox-docker/discussions
+    about: This is the right place to ask questions about how to use or do certain things with Netbox Docker.
+
+  - name: Have you had a look at our Wiki?
+    url: https://github.com/netbox-community/netbox-docker/wiki
+    about: Our wiki contains information for common problems and tips for operating Netbox Docker in production.

.github/no-response.yml

@@ -0,0 +1,10 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 30
+# Label requiring a response
+responseRequiredLabel: awaiting answer
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author.

.github/workflows/push.yml

@@ -10,24 +10,24 @@ on:
 
 jobs:
   build:
+    continue-on-error: ${{ matrix.docker_from == 'alpine:edge' }}
     strategy:
       matrix:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-branches.sh
+        - ./build-next.sh
+        - ./build.sh develop
         docker_from:
-        - '' # use the default of the DOCKERFILE
-        - python:3.7-alpine
-        - python:3.8-alpine
-        # - python:3.9-rc-alpine # disable until Netbox's unit tests work
+        - '' # use the default of the build script
+        - alpine:edge
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new Netbox Docker Images
     steps:
     - id: git-checkout
       name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
     - id: docker-build
       name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}'
       run: ${{ matrix.build_cmd }}

.github/workflows/release.yml

@@ -14,14 +14,15 @@ jobs:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-branches.sh
+        - ./build-next.sh
+        - ./build.sh develop
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new Netbox Docker Images
     steps:
     - id: git-checkout
       name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
     - id: docker-build
       name: Build the image with '${{ matrix.build_cmd }}'
       run: ${{ matrix.build_cmd }}
@@ -49,3 +50,34 @@ jobs:
       name: Logout of the Docker Registry
       run: docker logout "${DOCKER_REGISTRY}"
       if: steps.docker-build.outputs.skipped != 'true'
+
+    # Quay.io
+    - id: quayio-docker-build
+      name: Build the image with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        DOCKER_REGISTRY: quay.io
+        GH_ACTION: enable
+    - id: quayio-registry-login
+      name: Login to the Quay.io Registry
+      run: |
+        echo "::add-mask::$QUAYIO_USERNAME"
+        echo "::add-mask::$QUAYIO_PASSWORD"
+        docker login -u "$QUAYIO_USERNAME" --password "${QUAYIO_PASSWORD}" "${DOCKER_REGISTRY}"
+      env:
+        DOCKER_REGISTRY: quay.io
+        QUAYIO_USERNAME: ${{ secrets.quayio_username }}
+        QUAYIO_PASSWORD: ${{ secrets.quayio_password }}
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: quayio-registry-push
+      name: Push the image
+      run: ${{ matrix.build_cmd }} --push-only
+      env:
+        DOCKER_REGISTRY: quay.io
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: quayio-registry-logout
+      name: Logout of the Docker Registry
+      run: docker logout "${DOCKER_REGISTRY}"
+      env:
+        DOCKER_REGISTRY: quay.io
+      if: steps.docker-build.outputs.skipped != 'true'

.gitignore

@@ -2,3 +2,10 @@
 .netbox
 .initializers
 docker-compose.override.yml
+*.pem
+configuration/*
+!configuration/configuration.py
+!configuration/extra.py
+configuration/ldap/*
+!configuration/ldap/ldap_config.py
+prometheus.yml

.travis.yml

@@ -1,32 +0,0 @@
-sudo: required
-language: python
-
-env:
-  - BUILD=release
-  - BUILD=prerelease
-  - BUILD=branches
-  - BUILD=special
-
-git:
-  depth: 5
-
-services:
-  - docker
-
-install:
-  - docker-compose pull --parallel
-  - docker-compose build
-
-script:
-  - docker-compose run netbox ./manage.py test
-
-after_script:
-  - docker-compose down
-
-after_success:
-  - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-  - ./build-all.sh --push
-
-notifications:
-  slack:
-    secure: F3VsWcvU/XYyjGjU8ZAVGpREe7F1NjKq6LuMRzhQORbXUvanxDQtLzEe0Y5zm/6+gHkx6t8cX/v2PiCI+v46pkapYMUimd+QEOL1WxbUdnw2kQxcgw/R3wX34l2FHXbG3/a+TmH3euqbSCTIrPy9ufju948i+Q0E0u0fyInmozl8qOT23C4joQOpVAq7y+wHxTxsEg46ZzL2Ties+dmqjMsvHocv7mPI2IWzAWA8SJZxS82Amoapww++QjgEmoY+tMimLkdeXCRgeoj41UGHDg54rbEXh/PTaWiuzyzTr1WLmsGRScC57fDRivp3mSM37/MlNxsRj1z+j4zrvWFQgNfJ2yMjBHroc1jOX/uCY4dwbpSPqUCpc4idMGCGZFItgzTQ3lAPYAsom0C6n8C08Xk8EsNKWwXrDSd4ZUhIwptkNPCFK+kXbLFsMzSApnaBYW0T+wba57nZdiWjOPYmvJr49MDm5NHv2KaRBX2gpw7t7ZLhTgwGEWcZvcDebiLneXcXY5hZ7v2NHJkx/2x1yNXo85xZDy0wK1FGoOOHwPhvqOB+pcQZ/pUOSPTKqGw5l/CexoRm1shFsK+19FnSgimqTHjcuCo4lFW3JlEvlFhtfFXIte2Wjp1ALZgTrSq8zSD5rRxYCUKmM7b3EJwdaIgbvKWPdS4sCXlXU1bHx0g=

Dockerfile

@@ -1,4 +1,4 @@
-ARG FROM=python:3.7-alpine
+ARG FROM
 FROM ${FROM} as builder
 
 RUN apk add --no-cache \
@@ -12,27 +12,15 @@ RUN apk add --no-cache \
       libffi-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev
-
-WORKDIR /install
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-# gunicorn is used for launching netbox
-      gunicorn \
-      greenlet \
-      eventlet \
-# napalm is used for gathering information from network devices
-      napalm \
-# ruamel is used in startup_scripts
-      'ruamel.yaml>=0.15,<0.16' \
-# django_auth_ldap is required for ldap
-      django_auth_ldap \
-# django-storages was introduced in 2.7 and is optional
-      django-storages
+      postgresql-dev \
+      py3-pip \
+      python3-dev \
+      && python3 -m venv /opt/netbox/venv \
+      && /opt/netbox/venv/bin/python3 -m pip install --upgrade pip setuptools
 
 ARG NETBOX_PATH
-COPY ${NETBOX_PATH}/requirements.txt /
-RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt
+COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
+RUN /opt/netbox/venv/bin/pip install -r /requirements.txt -r /requirements-container.txt
 
 ###
 # Main stage
@@ -44,6 +32,7 @@ FROM ${FROM} as main
 RUN apk add --no-cache \
       bash \
       ca-certificates \
+      curl \
       graphviz \
       libevent \
       libffi \
@@ -51,35 +40,38 @@ RUN apk add --no-cache \
       libressl \
       libxslt \
       postgresql-libs \
-      ttf-ubuntu-font-family
+      python3 \
+      py3-pip \
+      ttf-ubuntu-font-family \
+      unit \
+      unit-python3
 
 WORKDIR /opt
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
-COPY configuration/gunicorn_config.py /etc/netbox/config/
-COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
-COPY configuration/configuration.py /etc/netbox/config/configuration.py
+COPY configuration/ /etc/netbox/config/
+COPY docker/nginx-unit.json /etc/unit/
 
 WORKDIR /opt/netbox/netbox
 
-# Must set permissions for '/opt/netbox/netbox/static' directory
-# to g+w so that `./manage.py collectstatic` can be executed during
-# container startup.
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
-RUN mkdir static && chmod g+w static media
+RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chmod -R g+w media /opt/unit/ \
+      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
+CMD [ "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \
@@ -122,4 +114,3 @@ RUN apk add --no-cache \
       util-linux
 
 COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
-COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

README.md

@@ -10,31 +10,35 @@
 [![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license]
 
 [The Github repository](netbox-docker-github) houses the components needed to build Netbox as a Docker container.
-Images are built using this code and are released to [Docker Hub][netbox-dockerhub] once a day.
+Images are built using this code and are released to [Docker Hub][netbox-dockerhub] and [Quay.io][netbox-quayio] once a day.
 
-Do you have any questions? Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel.
+Do you have any questions?
+Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel.
 
 [github-stargazers]: https://github.com/netbox-community/netbox-docker/stargazers
 [github-release]: https://github.com/netbox-community/netbox-docker/releases
 [netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox
-[netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/tags/
+[netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/
 [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/
 [ntc-slack]: http://slack.networktocode.com/
 [netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E
 [netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/release/LICENSE
+[netbox-quayio]: https://quay.io/repository/netboxcommunity/netbox
 
 ## Docker Tags
 
-* `vX.Y.Z`: Release builds, built from [releases of Netbox][netbox-releases].
-* `latest`: Release builds, built from [`master` branch of Netbox][netbox-master].
-* `snapshot`: Pre-release builds, built from the [`develop` branch of Netbox][netbox-develop].
-* `develop-X.Y`: Pre-release builds, built from the corresponding [branch of Netbox][netbox-branches].
+* `vX.Y.Z`: These are release builds, automatically built from [the corresponding releases of Netbox][netbox-releases].
+* `latest`: These are release builds, automatically built from [the `master` branch of Netbox][netbox-master].
+* `snapshot`: These are pre-release builds, automatically built from the [`develop` branch of Netbox][netbox-develop].
+* `develop-X.Y`: These are pre-release builds, automatically built from the corresponding [branch of Netbox][netbox-branches].
 
-Then there is currently one extra tags for each of the above labels:
+Then there is currently one extra tags for each of the above tags:
 
-* `-ldap`: Contains additional dependencies and configurations for connecting Netbox to an LDAP directroy.
+* `-ldap`: Contains additional dependencies and configurations for connecting Netbox to an LDAP directory.
   [Learn more about that in our wiki][netbox-docker-ldap].
 
+New images are built and published automatically every ~24h.
+
 [netbox-releases]: https://github.com/netbox-community/netbox/releases
 [netbox-master]: https://github.com/netbox-community/netbox/tree/master
 [netbox-develop]: https://github.com/netbox-community/netbox/tree/develop
@@ -43,42 +47,52 @@ Then there is currently one extra tags for each of the above labels:
 
 ## Quickstart
 
-To get Netbox up and running in Docker:
+To get Netbox Docker up and running run the following commands.
+There is a more complete [_Getting Started_ guide on our wiki][wiki-getting-started] which explains every step.
 
 ```bash
 git clone -b release https://github.com/netbox-community/netbox-docker.git
 cd netbox-docker
+tee docker-compose.override.yml <<EOF
+version: '3.4'
+services:
+  netbox:
+    ports:
+      - 8000:8080
+EOF
 docker-compose pull
-docker-compose up -d
+docker-compose up
 ```
 
-The application will be available after a few minutes.
-Use `docker-compose port nginx 8080` to find out where to connect to.
-
-```bash
-$ echo "http://$(docker-compose port nginx 8080)/"
-http://0.0.0.0:32768/
-
-# Open netbox in your default browser on macOS:
-$ open "http://$(docker-compose port nginx 8080)/"
-
-# Open netbox in your default browser on (most) linuxes:
-$ xdg-open "http://$(docker-compose port nginx 8080)/" &>/dev/null &
-```
-
-Alternatively, use something like [Reception][docker-reception] to connect to _docker-compose_ projects.
-
+The whole application will be available after a few minutes.
+Open the URL `http://0.0.0.0:8000/` in a web-browser.
+You should see the Netbox homepage.
+In the top-right corner you can login.
 The default credentials are:
 
 * Username: **admin**
 * Password: **admin**
 * API Token: **0123456789abcdef0123456789abcdef01234567**
 
-There is a more complete [Getting Started guide on our Wiki][wiki-getting-started].
-
 [wiki-getting-started]: https://github.com/netbox-community/netbox-docker/wiki/Getting-Started
 [docker-reception]: https://github.com/nxt-engineering/reception
 
+## Documentation
+
+Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly.
+It covers advanced topics such as using files for secrets, deployment to Kubernetes, monitoring and configuring NAPALM or LDAP.
+
+[netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/
+
+## Getting Help
+
+Feel free to ask questions in our [Github Community][netbox-community] or join [our Slack channel `#netbox-docker`][netbox-docker-slack] on the [Network To Code Slack][ntc-slack],
+which is free to use and where there are almost always people online that can help you in the Slack channel.
+
+If you need help with using Netbox or developing for it or against it's API you may find the `#netbox` channel on the same Slack instance very helpful.
+
+[netbox-community]: https://github.com/netbox-community/netbox-docker/discussions
+
 ## Dependencies
 
 This project relies only on *Docker* and *docker-compose* meeting these requirements:
@@ -88,19 +102,12 @@ This project relies only on *Docker* and *docker-compose* meeting these requirem
 
 To check the version installed on your system run `docker --version` and `docker-compose --version`.
 
-## Documentation
+## Use a Specific Netbox Version
 
-Please refer [to our wiki on Github][netbox-docker-wiki] for further information on how to use this Netbox Docker image properly.
-It covers advanced topics such as using secret files, deployment to Kubernetes as well as NAPALM and LDAP configuration.
-
-[netbox-docker-wiki]: https://github.com/netbox-community/netbox-docker/wiki/
-
-## Netbox Version
-
-The `docker-compose.yml` file is prepared to run a specific version of Netbox.
-To use this feature, set the environment-variable `VERSION` before launching `docker-compose`, as shown below.
+The `docker-compose.yml` file is prepared to run a specific version of Netbox, instead of `latest`.
+To use this feature, set and export the environment-variable `VERSION` before launching `docker-compose`, as shown below.
 `VERSION` may be set to the name of
-[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
+[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub] or [Quay.io][netbox-quayio].
 
 ```bash
 export VERSION=v2.7.1
@@ -141,10 +148,6 @@ For more details on custom builds [consult our wiki][netbox-docker-wiki-build].
 
 [netbox-docker-wiki-build]: https://github.com/netbox-community/netbox-docker/wiki/Build
 
-### Pre-made Docker Images
-
-New Docker images are built and published every 24h.
-
 ## Tests
 
 We have a test script.

VERSION

@@ -1 +1 @@
-0.22.0
+1.0.1

build-all.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-# Builds all Docker images this project provides
-# Arguments:
-#   BUILD:     The release to build.
-#              Allowed: release, prerelease, branches, special
-#              Default: undefined
-
-echo "▶️ $0 $*"
-
-ALL_BUILDS=("release" "prerelease" "branches" "special")
-BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
-
-echo "⚙️ Configured builds: ${BUILDS[*]}"
-
-if [ -n "${DEBUG}" ]; then
-  export DEBUG
-fi
-
-ERROR=0
-
-for BUILD in "${BUILDS[@]}"; do
-  echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
-  case $BUILD in
-    release)
-      # build the latest release
-      # shellcheck disable=SC2068
-      ./build-latest.sh $@ || ERROR=1
-      ;;
-    prerelease)
-      # build the latest pre-release
-      # shellcheck disable=SC2068
-      PRERELEASE=true ./build-latest.sh $@ || ERROR=1
-      ;;
-    branches)
-      # build all branches
-      # shellcheck disable=SC2068
-      ./build-branches.sh $@ || ERROR=1
-      ;;
-    *)
-      echo "🚨 Unrecognized build '$BUILD'."
-
-      if [ -z "$DEBUG" ]; then
-        exit 1
-      else
-        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-      fi
-      ;;
-  esac
-done
-
-exit $ERROR

build-functions/docker-functions.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+push_image_to_registry() {
+  local target_tag=$1
+  echo "⏫ Pushing '${target_tag}'"
+  $DRY docker push "${target_tag}"
+  echo "✅ Finished pushing the Docker image '${target_tag}'."
+}

build-functions/get-public-image-config.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Retrieves image configuration from public images in DockerHub
+# Functions from https://gist.github.com/cirocosta/17ea17be7ac11594cb0f290b0a3ac0d1
+# Optimised for our use case
+
+get_image_label() {
+  local label=$1
+  local image=$2
+  local tag=$3
+  local token
+  token=$(_get_token "$image")
+  local digest
+  digest=$(_get_digest "$image" "$tag" "$token")
+  local retval="null"
+  if [ "$digest" != "null" ]; then
+    retval=$(_get_image_configuration "$image" "$token" "$digest" "$label")
+  fi
+  echo "$retval"
+}
+
+get_image_layers() {
+  local image=$1
+  local tag=$2
+  local token
+  token=$(_get_token "$image")
+  _get_layers "$image" "$tag" "$token"
+}
+
+get_image_last_layer() {
+  local image=$1
+  local tag=$2
+  local token
+  token=$(_get_token "$image")
+  local layers
+  mapfile -t layers < <(_get_layers "$image" "$tag" "$token")
+  echo "${layers[-1]}"
+}
+
+_get_image_configuration() {
+  local image=$1
+  local token=$2
+  local digest=$3
+  local label=$4
+  curl \
+    --silent \
+    --location \
+    --header "Authorization: Bearer $token" \
+    "https://registry-1.docker.io/v2/$image/blobs/$digest" \
+    | jq -r ".config.Labels.\"$label\""
+}
+
+_get_token() {
+  local image=$1
+  curl \
+    --silent \
+    "https://auth.docker.io/token?scope=repository:$image:pull&service=registry.docker.io" \
+    | jq -r '.token'
+}
+
+_get_digest() {
+  local image=$1
+  local tag=$2
+  local token=$3
+  curl \
+    --silent \
+    --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+    --header "Authorization: Bearer $token" \
+    "https://registry-1.docker.io/v2/$image/manifests/$tag" \
+    | jq -r '.config.digest'
+}
+
+_get_layers() {
+  local image=$1
+  local tag=$2
+  local token=$3
+  curl \
+    --silent \
+    --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+    --header "Authorization: Bearer $token" \
+    "https://registry-1.docker.io/v2/$image/manifests/$tag" \
+    | jq -r '.layers[].digest'
+}

build-next.sh

@@ -23,25 +23,17 @@ GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
 
 # Composing the JQ commans to extract the most recent version number
-JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
+JQ_NEXT='map(.name) | .[] | scan("^[^v].+") | match("^(develop-).*") | .string'
 
 CURL="curl -sS"
 
 # Querying the Github API to fetch all branches
-BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
+NEXT=$($CURL "${URL_RELEASES}" | jq -r "$JQ_NEXT")
 
-###
-# Building each branch
-###
-
-# keeping track whether an error occured
-ERROR=0
-
-# calling build.sh for each branch
-for BRANCH in $BRANCHES; do
+if [ -n "$NEXT" ]; then
   # shellcheck disable=SC2068
-  ./build.sh "${BRANCH}" $@ || ERROR=1
-done
-
-# returning whether an error occured
-exit $ERROR
+  ./build.sh "${NEXT}" $@
+else
+  echo "No branch matching 'develop-*' found"
+  echo "::set-output name=skipped::true"
+fi

build.sh

@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: Whatever is defined as default in the Dockerfile."
+  echo "              Default: 'alpine:3.13'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -106,7 +106,7 @@ else
 fi
 
 ###
-# Variables for fetching the source
+# Variables for fetching the Netbox source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
@@ -115,10 +115,10 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# Fetching the source
+# Fetching the Netbox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
-  echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  echo "🌐 Checking out '${NETBOX_BRANCH}' of Netbox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
   fi
@@ -135,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
     $DRY git checkout -qf FETCH_HEAD
     $DRY git prune
   )
-  echo "✅ Checked out netbox"
+  echo "✅ Checked out Netbox"
 fi
 
 ###
@@ -153,6 +153,13 @@ if [ ! -f "${DOCKERFILE}" ]; then
   fi
 fi
 
+###
+# Determining the value for DOCKER_FROM
+###
+if [ -z "$DOCKER_FROM" ]; then
+  DOCKER_FROM="alpine:3.13"
+fi
+
 ###
 # Variables for labelling the docker image
 ###
@@ -167,9 +174,9 @@ PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]
 
 # Get the Git information from the netbox directory
 if [ -d "${NETBOX_PATH}/.git" ]; then
-  NETBOX_GIT_REF=$(cd ${NETBOX_PATH}; git rev-parse HEAD)
-  NETBOX_GIT_BRANCH=$(cd ${NETBOX_PATH}; git rev-parse --abbrev-ref HEAD)
-  NETBOX_GIT_URL=$(cd ${NETBOX_PATH}; git remote get-url origin)
+  NETBOX_GIT_REF=$(cd "${NETBOX_PATH}"; git rev-parse HEAD)
+  NETBOX_GIT_BRANCH=$(cd "${NETBOX_PATH}"; git rev-parse --abbrev-ref HEAD)
+  NETBOX_GIT_URL=$(cd "${NETBOX_PATH}"; git remote get-url origin)
 fi
 
 ###
@@ -209,7 +216,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
   fi
   if [ -n "${GH_ACTION}" ]; then
-    echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}"
+    echo "FINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" >> $GITHUB_ENV
     echo "::set-output name=skipped::false"
   fi
 
@@ -217,15 +224,18 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   # composing the additional DOCKER_SHORT_TAG,
   # i.e. "v2.6.1" becomes "v2.6",
   # which is only relevant for version tags
+  # Also let "latest" follow the highest version
   ###
   if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
     MAJOR=${BASH_REMATCH[1]}
     MINOR=${BASH_REMATCH[2]}
 
     TARGET_DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
+    TARGET_DOCKER_LATEST_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:latest"
 
     if [ "${DOCKER_TARGET}" != "main" ]; then
       TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
+      TARGET_DOCKER_LATEST_TAG="${TARGET_DOCKER_LATEST_TAG}-${DOCKER_TARGET}"
     fi
   fi
 
@@ -233,6 +243,48 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   # Proceeding to buils stage, except if `--push-only` is passed
   ###
   if [ "${2}" != "--push-only" ] ; then
+    ###
+    # Checking if the build is necessary,
+    # meaning build only if one of those values changed:
+    # - Python base image digest (Label: PYTHON_BASE_DIGEST)
+    # - netbox git ref (Label: NETBOX_GIT_REF)
+    # - netbox-docker git ref (Label: org.label-schema.vcs-ref)
+    ###
+    # Load information from registry (only for docker.io)
+    SHOULD_BUILD="false"
+    BUILD_REASON=""
+    if [ -z "${GH_ACTION}" ]; then
+      # Asuming non Github builds should always proceed
+      SHOULD_BUILD="true"
+      BUILD_REASON="${BUILD_REASON} interactive"
+    elif [ "$DOCKER_REGISTRY" = "docker.io" ]; then
+      source ./build-functions/get-public-image-config.sh
+      IFS=':' read -ra DOCKER_FROM_SPLIT <<< "${DOCKER_FROM}"
+      if ! [[ ${DOCKER_FROM_SPLIT[0]} =~ .*/.* ]]; then
+        # Need to use "library/..." for images the have no two part name
+        DOCKER_FROM_SPLIT[0]="library/${DOCKER_FROM_SPLIT[0]}"
+      fi
+      PYTHON_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}")
+      mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
+      NETBOX_GIT_REF_OLD=$(get_image_label NETBOX_GIT_REF "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
+      GIT_REF_OLD=$(get_image_label org.label-schema.vcs-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
+
+      if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
+        SHOULD_BUILD="true"
+        BUILD_REASON="${BUILD_REASON} alpine"
+      fi
+      if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
+        SHOULD_BUILD="true"
+        BUILD_REASON="${BUILD_REASON} netbox"
+      fi
+      if [ "${GIT_REF}" != "${GIT_REF_OLD}" ]; then
+        SHOULD_BUILD="true"
+        BUILD_REASON="${BUILD_REASON} netbox-docker"
+      fi
+    else
+      SHOULD_BUILD="true"
+      BUILD_REASON="${BUILD_REASON} no-check"
+    fi
     ###
     # Composing all arguments for `docker build`
     ###
@@ -244,32 +296,35 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     )
     if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
       DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_SHORT_TAG}" )
+      DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_LATEST_TAG}" )
     fi
 
     # --label
-    if [ "${DOCKER_TARGET}" == "main" ]; then
+    DOCKER_BUILD_ARGS+=(
+      --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}"
+
+      --label "org.label-schema.build-date=${BUILD_DATE}"
+      --label "org.opencontainers.image.created=${BUILD_DATE}"
+
+      --label "org.label-schema.version=${PROJECT_VERSION}"
+      --label "org.opencontainers.image.version=${PROJECT_VERSION}"
+    )
+    if [ -d ".git" ]; then
       DOCKER_BUILD_ARGS+=(
-        --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}"
-
-        --label "org.label-schema.build-date=${BUILD_DATE}"
-        --label "org.opencontainers.image.created=${BUILD_DATE}"
-
-        --label "org.label-schema.version=${PROJECT_VERSION}"
-        --label "org.opencontainers.image.version=${PROJECT_VERSION}"
+        --label "org.label-schema.vcs-ref=${GIT_REF}"
+        --label "org.opencontainers.image.revision=${GIT_REF}"
       )
-      if [ -d ".git" ]; then
-        DOCKER_BUILD_ARGS+=(
-          --label "org.label-schema.vcs-ref=${GIT_REF}"
-          --label "org.opencontainers.image.revision=${GIT_REF}"
-        )
-      fi
-      if [ -d "${NETBOX_PATH}/.git" ]; then
-        DOCKER_BUILD_ARGS+=(
-          --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}"
-          --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}"
-          --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}"
-        )
-      fi
+    fi
+    if [ -d "${NETBOX_PATH}/.git" ]; then
+      DOCKER_BUILD_ARGS+=(
+        --label "NETBOX_GIT_BRANCH=${NETBOX_GIT_BRANCH}"
+        --label "NETBOX_GIT_REF=${NETBOX_GIT_REF}"
+        --label "NETBOX_GIT_URL=${NETBOX_GIT_URL}"
+      )
+    fi
+    if [ -n "${BUILD_REASON}" ]; then
+      BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< "$BUILD_REASON")
+      DOCKER_BUILD_ARGS+=( --label "BUILD_REASON=${BUILD_REASON}" )
     fi
 
     # --build-arg
@@ -289,23 +344,29 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     ###
     # Building the docker image
     ###
-    echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
-    $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
-    echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
+    if [ "${SHOULD_BUILD}" == "true" ]; then
+      echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
+      echo "    Build reason set to: ${BUILD_REASON}"
+      $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
+      echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
+      echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG}'"
+      $DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}"
+    else
+      echo "Build skipped because sources didn't change"
+      echo "::set-output name=skipped::true"
+    fi
   fi
 
   ###
   # Pushing the docker images if either `--push` or `--push-only` are passed
   ###
   if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
-    echo "⏫ Pushing '${TARGET_DOCKER_TAG}"
-    $DRY docker push "${TARGET_DOCKER_TAG}"
-    echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'."
+    source ./build-functions/docker-functions.sh
+    push_image_to_registry "${TARGET_DOCKER_TAG}"
 
     if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
-      echo "⏫ Pushing '${TARGET_DOCKER_SHORT_TAG}'"
-      $DRY docker push "${TARGET_DOCKER_SHORT_TAG}"
-      echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_SHORT_TAG}'."
+      push_image_to_registry "${TARGET_DOCKER_SHORT_TAG}"
+      push_image_to_registry "${TARGET_DOCKER_LATEST_TAG}"
     fi
   fi
 done

configuration/configuration.py

@@ -1,21 +1,28 @@
-import os
-import re
-import socket
+####
+## We recommend to not edit this file.
+## Create separate files to overwrite the settings.
+## See `extra.py` as an example.
+####
 
-# For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/
-# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration.example.py
+import re
+
+from os.path import dirname, abspath, join
+from os import environ
+
+# For reference see https://netbox.readthedocs.io/en/stable/configuration/
+# Based on https://github.com/netbox-community/netbox/blob/master/netbox/netbox/configuration.example.py
 
 # Read secret from file
-def read_secret(secret_name):
+def _read_secret(secret_name, default = None):
     try:
         f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
     except EnvironmentError:
-        return ''
+        return default
     else:
         with f:
             return f.readline().strip()
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_BASE_DIR = dirname(dirname(abspath(__file__)))
 
 #########################
 #                       #
@@ -27,47 +34,49 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name.
 #
 # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local']
-ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(' ')
+ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ')
 
-# PostgreSQL database configuration.
+# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters:
+#   https://docs.djangoproject.com/en/stable/ref/settings/#databases
 DATABASE = {
-    'NAME': os.environ.get('DB_NAME', 'netbox'),         # Database name
-    'USER': os.environ.get('DB_USER', ''),               # PostgreSQL username
-    'PASSWORD': os.environ.get('DB_PASSWORD', read_secret('db_password')),
+    'NAME': environ.get('DB_NAME', 'netbox'),            # Database name
+    'USER': environ.get('DB_USER', ''),                  # PostgreSQL username
+    'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')),
                                                          # PostgreSQL password
-    'HOST': os.environ.get('DB_HOST', 'localhost'),      # Database server
-    'PORT': os.environ.get('DB_PORT', ''),               # Database port (leave blank for default)
-    'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')},
+    'HOST': environ.get('DB_HOST', 'localhost'),         # Database server
+    'PORT': environ.get('DB_PORT', ''),                  # Database port (leave blank for default)
+    'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')},
                                                          # Database connection SSLMODE
-    'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')),
-                                                         # Database connection persistence
+    'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')),
+                                                         # Max database connection age
+}
+
+# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate
+# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended
+# to use two separate database IDs.
+REDIS = {
+    'tasks': {
+        'HOST': environ.get('REDIS_HOST', 'localhost'),
+        'PORT': int(environ.get('REDIS_PORT', 6379)),
+        'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')),
+        'DATABASE': int(environ.get('REDIS_DATABASE', 0)),
+        'SSL': environ.get('REDIS_SSL', 'False').lower() == 'true',
+    },
+    'caching': {
+        'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')),
+        'PORT': int(environ.get('REDIS_CACHE_PORT', environ.get('REDIS_PORT', 6379))),
+        'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))),
+        'DATABASE': int(environ.get('REDIS_CACHE_DATABASE', 1)),
+        'SSL': environ.get('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False')).lower() == 'true',
+    },
 }
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
 # symbols. NetBox will not run without this defined. For more information, see
-# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY
-SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key'))
+# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY
+SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', ''))
 
-# Redis database settings. The Redis database is used for caching and background processing such as webhooks
-REDIS = {
-    'webhooks': {
-        'HOST': os.environ.get('REDIS_HOST', 'localhost'),
-        'PORT': int(os.environ.get('REDIS_PORT', 6379)),
-        'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
-        'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)),
-        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)),
-        'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true',
-    },
-    'caching': {
-        'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')),
-        'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))),
-        'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))),
-        'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)),
-        'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))),
-        'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true',
-    },
-}
 
 #########################
 #                       #
@@ -81,111 +90,159 @@ ADMINS = [
     # ['John Doe', 'jdoe@example.com'],
 ]
 
+# URL schemes that are allowed within links in NetBox
+ALLOWED_URL_SCHEMES = (
+    'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp',
+)
+
 # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same
 # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP.
-BANNER_TOP = os.environ.get('BANNER_TOP', '')
-BANNER_BOTTOM = os.environ.get('BANNER_BOTTOM', '')
+BANNER_TOP = environ.get('BANNER_TOP', '')
+BANNER_BOTTOM = environ.get('BANNER_BOTTOM', '')
 
 # Text to include on the login page above the login form. HTML is allowed.
-BANNER_LOGIN = os.environ.get('BANNER_LOGIN', '')
+BANNER_LOGIN = environ.get('BANNER_LOGIN', '')
 
 # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set:
 # BASE_PATH = 'netbox/'
-BASE_PATH = os.environ.get('BASE_PATH', '')
+BASE_PATH = environ.get('BASE_PATH', '')
 
 # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
-CACHE_TIMEOUT = int(os.environ.get('CACHE_TIMEOUT', 900))
+CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900))
 
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
-CHANGELOG_RETENTION = int(os.environ.get('CHANGELOG_RETENTION', 90))
+CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90))
 
 # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be
 # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or
 # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers
-CORS_ORIGIN_ALLOW_ALL = os.environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true'
-CORS_ORIGIN_WHITELIST = list(filter(None, os.environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
-CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, os.environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
+CORS_ORIGIN_ALLOW_ALL = environ.get('CORS_ORIGIN_ALLOW_ALL', 'False').lower() == 'true'
+CORS_ORIGIN_WHITELIST = list(filter(None, environ.get('CORS_ORIGIN_WHITELIST', 'https://localhost').split(' ')))
+CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in list(filter(None, environ.get('CORS_ORIGIN_REGEX_WHITELIST', '').split(' ')))]
 
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
 # on a production system.
-DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true'
+DEBUG = environ.get('DEBUG', 'False').lower() == 'true'
 
 # Email settings
 EMAIL = {
-    'SERVER': os.environ.get('EMAIL_SERVER', 'localhost'),
-    'PORT': int(os.environ.get('EMAIL_PORT', 25)),
-    'USERNAME': os.environ.get('EMAIL_USERNAME', ''),
-    'PASSWORD': os.environ.get('EMAIL_PASSWORD', read_secret('email_password')),
-    'TIMEOUT': int(os.environ.get('EMAIL_TIMEOUT', 10)),  # seconds
-    'FROM_EMAIL': os.environ.get('EMAIL_FROM', ''),
+    'SERVER': environ.get('EMAIL_SERVER', 'localhost'),
+    'PORT': int(environ.get('EMAIL_PORT', 25)),
+    'USERNAME': environ.get('EMAIL_USERNAME', ''),
+    'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')),
+    'USE_SSL': environ.get('EMAIL_USE_SSL', 'False').lower() == 'true',
+    'USE_TLS': environ.get('EMAIL_USE_TLS', 'False').lower() == 'true',
+    'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''),
+    'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''),
+    'TIMEOUT': int(environ.get('EMAIL_TIMEOUT', 10)),  # seconds
+    'FROM_EMAIL': environ.get('EMAIL_FROM', ''),
 }
 
-# Enforcement of unique IP space can be toggled on a per-VRF basis.
-# To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF),
-# set ENFORCE_GLOBAL_UNIQUE to True.
-ENFORCE_GLOBAL_UNIQUE = os.environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true'
+# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table
+# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True.
+ENFORCE_GLOBAL_UNIQUE = environ.get('ENFORCE_GLOBAL_UNIQUE', 'False').lower() == 'true'
 
 # Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and
 # by anonymous users. List models in the form `<app>.<model>`. Add '*' to this list to exempt all models.
-EXEMPT_VIEW_PERMISSIONS = list(filter(None, os.environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
+EXEMPT_VIEW_PERMISSIONS = list(filter(None, environ.get('EXEMPT_VIEW_PERMISSIONS', '').split(' ')))
 
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
-#   https://docs.djangoproject.com/en/1.11/topics/logging/
+#   https://docs.djangoproject.com/en/stable/topics/logging/
 LOGGING = {}
 
 # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users
 # are permitted to access most data in NetBox (excluding secrets) but not make any changes.
-LOGIN_REQUIRED = os.environ.get('LOGIN_REQUIRED', 'False').lower() == 'true'
+LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true'
+
+# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to
+# re-authenticate. (Default: 1209600 [14 days])
+LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600))
 
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
-MAINTENANCE_MODE = os.environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
+MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
 
 # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
 # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
 # all objects by specifying "?limit=0".
-MAX_PAGE_SIZE = int(os.environ.get('MAX_PAGE_SIZE', 1000))
+MAX_PAGE_SIZE = int(environ.get('MAX_PAGE_SIZE', 1000))
 
 # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that
 # the default value of this setting is derived from the installed location.
-MEDIA_ROOT = os.environ.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media'))
+MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media'))
 
 # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
-METRICS_ENABLED = os.environ.get('METRICS_ENABLED', 'False').lower() == 'true'
+METRICS_ENABLED = environ.get('METRICS_ENABLED', 'False').lower() == 'true'
 
-# Credentials that NetBox will use to access live devices.
-NAPALM_USERNAME = os.environ.get('NAPALM_USERNAME', '')
-NAPALM_PASSWORD = os.environ.get('NAPALM_PASSWORD', read_secret('napalm_password'))
+# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM.
+NAPALM_USERNAME = environ.get('NAPALM_USERNAME', '')
+NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', ''))
 
 # NAPALM timeout (in seconds). (Default: 30)
-NAPALM_TIMEOUT = int(os.environ.get('NAPALM_TIMEOUT', 30))
+NAPALM_TIMEOUT = int(environ.get('NAPALM_TIMEOUT', 30))
 
 # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
 # be provided as a dictionary.
 NAPALM_ARGS = {}
 
 # Determine how many objects to display per page within a list. (Default: 50)
-PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50))
+PAGINATE_COUNT = int(environ.get('PAGINATE_COUNT', 50))
+
+# Enable installed plugins. Add the name of each plugin to the list.
+PLUGINS = []
+
+# Plugins configuration settings. These settings are used by various plugins that the user may have installed.
+# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
+PLUGINS_CONFIG = {
+}
 
 # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to
 # prefer IPv4 instead.
-PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true'
+PREFER_IPV4 = environ.get('PREFER_IPV4', 'False').lower() == 'true'
+
+# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1.
+RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', 22))
+RACK_ELEVATION_DEFAULT_UNIT_WIDTH = int(environ.get('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', 220))
+
+# Remote authentication support
+REMOTE_AUTH_ENABLED = environ.get('REMOTE_AUTH_ENABLED', 'False').lower() == 'true'
+REMOTE_AUTH_BACKEND = environ.get('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend')
+REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
+REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true'
+REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' ')))
+
+# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
+RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600))
+
+# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
+# version check or use the URL below to check for release in the official NetBox repository.
+# https://api.github.com/repos/netbox-community/netbox/releases
+RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None)
+
 # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
-REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')
+REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports')
+
+# Maximum execution time for background tasks, in seconds.
+RQ_DEFAULT_TIMEOUT = int(environ.get('RQ_DEFAULT_TIMEOUT', 300))
 
 # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
-SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
+SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
+
+# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use
+# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only
+# database access.) Note that the user as which NetBox runs must have read and write permissions to this path.
+SESSION_FILE_PATH = environ.get('SESSIONS_ROOT', None)
 
 # Time zone (default: UTC)
-TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC')
+TIME_ZONE = environ.get('TIME_ZONE', 'UTC')
 
 # Date/time formatting. See the following link for supported formats:
-# https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date
-DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y')
-SHORT_DATE_FORMAT = os.environ.get('SHORT_DATE_FORMAT', 'Y-m-d')
-TIME_FORMAT = os.environ.get('TIME_FORMAT', 'g:i a')
-SHORT_TIME_FORMAT = os.environ.get('SHORT_TIME_FORMAT', 'H:i:s')
-DATETIME_FORMAT = os.environ.get('DATETIME_FORMAT', 'N j, Y g:i a')
-SHORT_DATETIME_FORMAT = os.environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i')
+# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date
+DATE_FORMAT = environ.get('DATE_FORMAT', 'N j, Y')
+SHORT_DATE_FORMAT = environ.get('SHORT_DATE_FORMAT', 'Y-m-d')
+TIME_FORMAT = environ.get('TIME_FORMAT', 'g:i a')
+SHORT_TIME_FORMAT = environ.get('SHORT_TIME_FORMAT', 'H:i:s')
+DATETIME_FORMAT = environ.get('DATETIME_FORMAT', 'N j, Y g:i a')
+SHORT_DATETIME_FORMAT = environ.get('SHORT_DATETIME_FORMAT', 'Y-m-d H:i')

configuration/extra.py

@@ -0,0 +1,55 @@
+####
+## This file contains extra configuration options that can't be configured
+## directly through environment variables.
+####
+
+## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of
+## application errors (assuming correct email settings are provided).
+# ADMINS = [
+#     # ['John Doe', 'jdoe@example.com'],
+# ]
+
+
+## URL schemes that are allowed within links in NetBox
+# ALLOWED_URL_SCHEMES = (
+#     'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp',
+# )
+
+
+## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
+## be provided as a dictionary.
+# NAPALM_ARGS = {}
+
+
+## Enable installed plugins. Add the name of each plugin to the list.
+# from netbox.configuration.configuration import PLUGINS
+# PLUGINS.append('my_plugin')
+
+## Plugins configuration settings. These settings are used by various plugins that the user may have installed.
+## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
+# from netbox.configuration.configuration import PLUGINS_CONFIG
+# PLUGINS_CONFIG['my_plugin'] = {
+#   'foo': 'bar',
+#   'buzz': 'bazz'
+# }
+
+
+## Remote authentication support
+# REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
+
+
+## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the
+## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example:
+# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage'
+# STORAGE_CONFIG = {
+#     'AWS_ACCESS_KEY_ID': 'Key ID',
+#     'AWS_SECRET_ACCESS_KEY': 'Secret',
+#     'AWS_STORAGE_BUCKET_NAME': 'netbox',
+#     'AWS_S3_REGION_NAME': 'eu-west-1',
+# }
+
+
+## This file can contain arbitrary Python code, e.g.:
+# from datetime import datetime
+# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
+# BANNER_TOP = f'<marquee width="200px">This instance started on {now}.</marquee>'

configuration/gunicorn_config.py

@@ -1,8 +0,0 @@
-command = '/usr/bin/gunicorn'
-pythonpath = '/opt/netbox/netbox'
-bind = '0.0.0.0:8001'
-workers = 3
-errorlog = '-'
-accesslog = '-'
-capture_output = False
-loglevel = 'debug'

configuration/ldap/ldap_config.py

@@ -0,0 +1,87 @@
+import ldap
+
+from django_auth_ldap.config import LDAPSearch
+from importlib import import_module
+from os import environ
+
+# Read secret from file
+def _read_secret(secret_name, default=None):
+    try:
+        f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
+    except EnvironmentError:
+        return default
+    else:
+        with f:
+            return f.readline().strip()
+
+# Import and return the group type based on string name
+def _import_group_type(group_type_name):
+    mod = import_module('django_auth_ldap.config')
+    try:
+        return getattr(mod, group_type_name)()
+    except:
+        return None
+
+# Server URI
+AUTH_LDAP_SERVER_URI = environ.get('AUTH_LDAP_SERVER_URI', '')
+
+# The following may be needed if you are binding to Active Directory.
+AUTH_LDAP_CONNECTION_OPTIONS = {
+    ldap.OPT_REFERRALS: 0
+}
+
+# Set the DN and password for the NetBox service account.
+AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '')
+AUTH_LDAP_BIND_PASSWORD = _read_secret('auth_ldap_bind_password', environ.get('AUTH_LDAP_BIND_PASSWORD', ''))
+
+# Set a string template that describes any user’s distinguished name based on the username.
+AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None)
+
+# Enable STARTTLS for ldap authentication.
+AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true'
+
+# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert.
+# Note that this is a NetBox-specific setting which sets:
+#     ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true'
+
+AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '')
+AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName')
+AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASEDN,
+                                   ldap.SCOPE_SUBTREE,
+                                   "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)")
+
+# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group
+# heirarchy.
+AUTH_LDAP_GROUP_SEARCH_BASEDN = environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '')
+AUTH_LDAP_GROUP_SEARCH_CLASS = environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group')
+AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE,
+                                    "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")")
+AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
+
+# Define a group required to login.
+AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN')
+
+# Define special user types using groups. Exercise great caution when assigning superuser status.
+AUTH_LDAP_USER_FLAGS_BY_GROUP = {}
+
+if AUTH_LDAP_REQUIRE_GROUP is not None:
+    AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+        "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
+        "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
+        "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
+    }
+
+# For more granular permissions, we can map LDAP groups to Django groups.
+AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true'
+AUTH_LDAP_MIRROR_GROUPS = environ.get('AUTH_LDAP_MIRROR_GROUPS', '').lower() == 'true'
+
+# Cache groups for one hour to reduce LDAP traffic
+AUTH_LDAP_CACHE_TIMEOUT = int(environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600))
+
+# Populate the Django user from the LDAP directory.
+AUTH_LDAP_USER_ATTR_MAP = {
+    "first_name": environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'),
+    "last_name": environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'),
+    "email": environ.get('AUTH_LDAP_ATTR_MAIL', 'mail')
+}

configuration/ldap_config.py

@@ -1,80 +0,0 @@
-import ldap
-import os
-
-from django_auth_ldap.config import LDAPSearch
-from importlib import import_module
-
-# Read secret from file
-def read_secret(secret_name):
-    try:
-        f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8')
-    except EnvironmentError:
-        return ''
-    else:
-        with f:
-            return f.readline().strip()
-
-# Import and return the group type based on string name
-def import_group_type(group_type_name):
-    mod = import_module('django_auth_ldap.config')
-    try:
-        return getattr(mod, group_type_name)()
-    except:
-        return None
-
-# Server URI
-AUTH_LDAP_SERVER_URI = os.environ.get('AUTH_LDAP_SERVER_URI', '')
-
-# The following may be needed if you are binding to Active Directory.
-AUTH_LDAP_CONNECTION_OPTIONS = {
-    ldap.OPT_REFERRALS: 0
-}
-
-# Set the DN and password for the NetBox service account.
-AUTH_LDAP_BIND_DN = os.environ.get('AUTH_LDAP_BIND_DN', '')
-AUTH_LDAP_BIND_PASSWORD = os.environ.get('AUTH_LDAP_BIND_PASSWORD', read_secret('auth_ldap_bind_password'))
-
-# Set a string template that describes any user’s distinguished name based on the username.
-AUTH_LDAP_USER_DN_TEMPLATE = os.environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None)
-
-# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert.
-# Note that this is a NetBox-specific setting which sets:
-#     ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
-LDAP_IGNORE_CERT_ERRORS = os.environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true'
-
-AUTH_LDAP_USER_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '')
-AUTH_LDAP_USER_SEARCH_ATTR = os.environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName')
-AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_BASEDN,
-                                   ldap.SCOPE_SUBTREE,
-                                   "(" + AUTH_LDAP_USER_SEARCH_ATTR + "=%(user)s)")
-
-# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group
-# heirarchy.
-AUTH_LDAP_GROUP_SEARCH_BASEDN = os.environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '')
-AUTH_LDAP_GROUP_SEARCH_CLASS = os.environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group')
-AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE,
-                                    "(objectClass=" + AUTH_LDAP_GROUP_SEARCH_CLASS + ")")
-AUTH_LDAP_GROUP_TYPE = import_group_type(os.environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
-
-# Define a group required to login.
-AUTH_LDAP_REQUIRE_GROUP = os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '')
-
-# Define special user types using groups. Exercise great caution when assigning superuser status.
-AUTH_LDAP_USER_FLAGS_BY_GROUP = {
-    "is_active": os.environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
-    "is_staff": os.environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
-    "is_superuser": os.environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
-}
-
-# For more granular permissions, we can map LDAP groups to Django groups.
-AUTH_LDAP_FIND_GROUP_PERMS = os.environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true'
-
-# Cache groups for one hour to reduce LDAP traffic
-AUTH_LDAP_CACHE_TIMEOUT = int(os.environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600))
-
-# Populate the Django user from the LDAP directory.
-AUTH_LDAP_USER_ATTR_MAP = {
-    "first_name": os.environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'),
-    "last_name": os.environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'),
-    "email": os.environ.get('AUTH_LDAP_ATTR_MAIL', 'mail')
-}

docker-compose.test.yml

@@ -5,7 +5,10 @@ services:
     depends_on:
     - postgres
     - redis
+    - redis-cache
     env_file: env/netbox.env
+    environment:
+      SKIP_STARTUP_SCRIPTS: ${SKIP_STARTUP_SCRIPTS-false}
     user: '101'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
@@ -13,33 +16,26 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.17-alpine
-    depends_on:
-    - netbox
     ports:
     - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
-    image: postgres:11-alpine
+    image: postgres:12-alpine
     env_file: env/postgres.env
   redis:
-    image: redis:5-alpine
+    image: redis:6-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: env/redis.env
+  redis-cache:
+    image: redis:6-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
     - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
-    env_file: env/redis.env
+    env_file: env/redis-cache.env
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local

docker-compose.yml

@@ -15,35 +15,30 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
+    ports:
+    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
     entrypoint:
-    - python3
+    - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
     command:
     - rqworker
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.17-alpine
-    depends_on:
-    - netbox
-    ports:
-    - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
+    ports: []
+
+  # postgres
   postgres:
-    image: postgres:11-alpine
+    image: postgres:12-alpine
     env_file: env/postgres.env
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data
+
+  # redis
   redis:
-    image: redis:5-alpine
+    image: redis:6-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
@@ -52,17 +47,14 @@ services:
     volumes:
     - netbox-redis-data:/data
   redis-cache:
-    image: redis:5-alpine
+    image: redis:6-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
     - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
-    env_file: env/redis.env
+    env_file: env/redis-cache.env
+
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
   netbox-postgres-data:

docker/configuration.docker.py

@@ -1,10 +1,80 @@
+## Generic Parts
+# These functions are providing the functionality to load
+# arbitrary configuration files.
+#
+# They can be imported by other code (see `ldap_config.py` for an example).
+
+from os.path import abspath, isfile
+from os import scandir
 import importlib.util
 import sys
 
-try:
-  spec = importlib.util.spec_from_file_location('configuration', '/etc/netbox/config/configuration.py')
+
+def _filename(f):
+  return f.name
+
+
+def _import(module_name, path, loaded_configurations):
+  spec = importlib.util.spec_from_file_location('', path)
   module = importlib.util.module_from_spec(spec)
   spec.loader.exec_module(module)
-  sys.modules['netbox.configuration'] = module
-except:
-  raise ImportError('')
+  sys.modules[module_name] = module
+
+  loaded_configurations.insert(0, module)
+
+  print(f"🧬 loaded config '{path}'")
+
+
+def read_configurations(config_module, config_dir, main_config):
+  loaded_configurations = []
+
+  main_config_path = abspath(f'{config_dir}/{main_config}.py')
+  if isfile(main_config_path):
+    _import(f'{config_module}.{main_config}', main_config_path, loaded_configurations)
+  else:
+    print(f"⚠️ Main configuration '{main_config_path}' not found.")
+
+  with scandir(config_dir) as it:
+    for f in sorted(it, key=_filename):
+      if not f.is_file():
+        continue
+
+      if f.name.startswith('__'):
+        continue
+
+      if not f.name.endswith('.py'):
+        continue
+
+      if f.name == f'{config_dir}.py':
+        continue
+
+      module_name = f"{config_module}.{f.name[:-len('.py')]}".replace(".", "_")
+      _import(module_name, f.path, loaded_configurations)
+
+  if len(loaded_configurations) == 0:
+    print(f"‼️ No configuration files found in '{config_dir}'.")
+    raise ImportError(f"No configuration files found in '{config_dir}'.")
+
+  return loaded_configurations
+
+
+## Specific Parts
+# This section's code actually loads the various configuration files
+# into the module with the given name.
+# It contains the logic to resolve arbitrary configuration options by
+# levaraging dynamic programming using `__getattr__`.
+
+
+_loaded_configurations = read_configurations(
+  config_dir = '/etc/netbox/config/',
+  config_module = 'netbox.configuration',
+  main_config = 'configuration')
+
+
+def __getattr__(name):
+  for config in _loaded_configurations:
+    try:
+      return getattr(config, name)
+    except:
+      pass
+  raise AttributeError

docker/docker-entrypoint.sh

@@ -7,6 +7,9 @@ set -e
 # Allows Netbox to be run as non-root users
 umask 002
 
+# Load correct Python3 env
+source /opt/netbox/venv/bin/activate
+
 # Try to connect to the DB
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
@@ -31,19 +34,15 @@ else
   if [ -z ${SUPERUSER_EMAIL+x} ]; then
     SUPERUSER_EMAIL='admin@example.com'
   fi
-  if [ -z ${SUPERUSER_PASSWORD+x} ]; then
-    if [ -f "/run/secrets/superuser_password" ]; then
-      SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)"
-    else
-      SUPERUSER_PASSWORD='admin'
-    fi
+  if [ -f "/run/secrets/superuser_password" ]; then
+    SUPERUSER_PASSWORD="$(< /run/secrets/superuser_password)"
+  elif [ -z ${SUPERUSER_PASSWORD+x} ]; then
+    SUPERUSER_PASSWORD='admin'
   fi
-  if [ -z ${SUPERUSER_API_TOKEN+x} ]; then
-    if [ -f "/run/secrets/superuser_api_token" ]; then
-      SUPERUSER_API_TOKEN="$(< /run/secrets/superuser_api_token)"
-    else
-      SUPERUSER_API_TOKEN='0123456789abcdef0123456789abcdef01234567'
-    fi
+  if [ -f "/run/secrets/superuser_api_token" ]; then
+    SUPERUSER_API_TOKEN="$(< /run/secrets/superuser_api_token)"
+  elif [ -z ${SUPERUSER_API_TOKEN+x} ]; then
+    SUPERUSER_API_TOKEN='0123456789abcdef0123456789abcdef01234567'
   fi
 
   ./manage.py shell --interface python << END
@@ -64,9 +63,6 @@ else
   echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-# Copy static files
-./manage.py collectstatic --no-input
-
 echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker

docker/launch-netbox.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}"
+UNIT_SOCKET="/opt/unit/unit.sock"
+
+load_configuration() {
+  MAX_WAIT=10
+  WAIT_COUNT=0
+  while [ ! -S $UNIT_SOCKET ]; do
+    if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
+      echo "⚠️ No control socket found; configuration will not be loaded."
+      return 1
+    fi
+
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    echo "⏳ Waiting for control socket to be created... (${WAIT_COUNT}/${MAX_WAIT})"
+
+    sleep 1
+  done
+
+  # even when the control socket exists, it does not mean unit has finished initialisation
+  # this curl call will get a reply once unit is fully launched
+  curl --silent --output /dev/null --request GET --unix-socket $UNIT_SOCKET http://localhost/
+
+  echo "⚙️ Applying configuration from $UNIT_CONFIG";
+
+  RESP_CODE=$(curl \
+                  --silent \
+                  --output /dev/null \
+                  --write-out '%{http_code}' \
+                  --request PUT \
+                  --data-binary "@${UNIT_CONFIG}" \
+                  --unix-socket $UNIT_SOCKET \
+                  http://localhost/config
+            )
+  if [ "$RESP_CODE" != "200" ]; then
+    echo "⚠️ Could no load Unit configuration"
+    kill "$(cat /opt/unit/unit.pid)"
+    return 1
+  fi
+
+  echo "✅ Unit configuration loaded successfully"
+}
+
+load_configuration &
+
+exec unitd \
+    --no-daemon \
+    --control unix:$UNIT_SOCKET \
+    --pid /opt/unit/unit.pid \
+    --log /dev/stdout \
+    --state /opt/unit/state/ \
+    --tmp /opt/unit/tmp/

docker/ldap_config.docker.py

@@ -1,10 +1,21 @@
-import importlib.util
-import sys
+from .configuration import read_configurations
 
-try:
-  spec = importlib.util.spec_from_file_location('ldap_config', '/etc/netbox/config/ldap_config.py')
-  module = importlib.util.module_from_spec(spec)
-  spec.loader.exec_module(module)
-  sys.modules['netbox.ldap_config'] = module
-except:
-  raise ImportError('')
+_loaded_configurations = read_configurations(
+  config_dir = '/etc/netbox/config/ldap/',
+  config_module = 'netbox.configuration.ldap',
+  main_config = 'ldap_config')
+
+
+def __getattr__(name):
+  for config in _loaded_configurations:
+    try:
+      return getattr(config, name)
+    except:
+      pass
+  raise AttributeError
+
+def __dir__():
+  names = []
+  for config in _loaded_configurations:
+    names.extend(config.__dir__())
+  return names

docker/nginx-unit.json

@@ -0,0 +1,40 @@
+{
+    "listeners": {
+        "*:8080": {
+            "pass": "routes"
+        }
+    },
+
+    "routes": [
+        {
+            "match": {
+                "uri": "/static/*"
+            },
+            "action": {
+                "share": "/opt/netbox/netbox"
+            }
+        },
+
+        {
+            "action": {
+                "pass": "applications/netbox"
+            }
+        }
+    ],
+
+    "applications": {
+        "netbox": {
+            "type": "python 3",
+            "path": "/opt/netbox/netbox/",
+            "module": "netbox.wsgi",
+            "home": "/opt/netbox/venv",
+            "processes": {
+                "max": 4,
+                "spare": 1,
+                "idle_timeout": 120
+            }
+        }
+    },
+
+    "access_log": "/dev/stdout"
+}

docker/nginx.conf

@@ -1,36 +0,0 @@
-daemon off;
-worker_processes 1;
-
-error_log /dev/stderr info;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include              /etc/nginx/mime.types;
-    default_type         application/octet-stream;
-    sendfile             on;
-    tcp_nopush           on;
-    keepalive_timeout    65;
-    gzip                 on;
-    server_tokens        off;
-    client_max_body_size 10M;
-
-    server {
-        listen      8080;
-        access_log  off;
-
-        location /static/ {
-            alias /opt/netbox/netbox/static/;
-        }
-
-        location / {
-            proxy_pass http://netbox:8001;
-            proxy_set_header X-Forwarded-Host $http_host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
-        }
-    }
-}

env/netbox.env

@@ -9,19 +9,26 @@ EMAIL_USERNAME=netbox
 EMAIL_PASSWORD=
 EMAIL_TIMEOUT=5
 EMAIL_FROM=netbox@bar.com
+# EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`!
+EMAIL_USE_SSL=false
+EMAIL_USE_TLS=false
+EMAIL_SSL_CERTFILE=
+EMAIL_SSL_KEYFILE=
+MAX_PAGE_SIZE=1000
 MEDIA_ROOT=/opt/netbox/netbox/media
+METRICS_ENABLED=false
 NAPALM_USERNAME=
 NAPALM_PASSWORD=
 NAPALM_TIMEOUT=10
-MAX_PAGE_SIZE=1000
 REDIS_HOST=redis
 REDIS_PASSWORD=H733Kdjndks81
 REDIS_DATABASE=0
 REDIS_SSL=false
 REDIS_CACHE_HOST=redis-cache
 REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36
-REDIS_CACHE_DATABASE=0
+REDIS_CACHE_DATABASE=1
 REDIS_CACHE_SSL=false
+RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 SKIP_STARTUP_SCRIPTS=false
 SKIP_SUPERUSER=false

initializers/aggregates.yml

@@ -1,5 +1,6 @@
 # - prefix: 10.0.0.0/16
 #   rir: RFC1918
+#   tenant: tenant1
 # - prefix: fd00:ccdd::/32
 #   rir: RFC4193 ULA
 # - prefix: 2001:db8::/32

initializers/circuit_types.yml

@@ -0,0 +1,6 @@
+# - name: VPLS
+#   slug: vpls
+# - name: MPLS
+#   slug: mpls
+# - name: Internet
+#   slug: internet

initializers/circuits.yml

@@ -0,0 +1,7 @@
+# - cid: Circuit_ID-1
+#   provider: Provider1
+#   type: Internet
+#   tenant: tenant1
+# - cid: Circuit_ID-2
+#   provider: Provider2
+#   type: MPLS

initializers/cluster_groups.yml

@@ -0,0 +1,4 @@
+# - name: Group 1
+#   slug: group-1
+# - name: Group 2
+#   slug: group-2

initializers/clusters.yml

@@ -1,5 +1,7 @@
 # - name: cluster1
 #   type: Hyper-V
+#   group: Group 1
+#   tenant: tenant1
 # - name: cluster2
 #   type: Hyper-V
 #   site: SING 1

initializers/custom_fields.yml

@@ -43,20 +43,16 @@
 #   required: false
 #   filter_logic: exact
 #   weight: 30
+#   default: First Item
 #   on_objects:
 #   - dcim.models.Device
 #   choices:
-#   - value: First Item
-#     weight: 10
-#   - value: Second Item
-#     weight: 20
-#   - value: Third Item
-#     weight: 30
-#   - value: Fifth Item
-#     weight: 50
-#   - value: Fourth Item
-#     weight: 40
-# select_field_auto_weight:
+#   - First Item
+#   - Second Item
+#   - Third Item
+#   - Fifth Item
+#   - Fourth Item
+# select_field_legacy_format:
 #    type: select
 #    label: Choose between items
 #    required: false
@@ -65,9 +61,9 @@
 #    on_objects:
 #     - dcim.models.Device
 #    choices:
-#     - value: A
-#     - value: B
-#     - value: C
+#     - value: A # this is the deprecated format.
+#     - value: B # we only use it for the tests.
+#     - value: C # please see above for the new format.
 #     - value: "D like deprecated"
 #       weight: 999
 #     - value: E
@@ -76,7 +72,7 @@
 #   label: Yes Or No?
 #   required: true
 #   filter_logic: loose
-#   default: "false" # important: but "false" in quotes!
+#   default: "false" # important: put "false" in quotes!
 #   weight: 90
 #   on_objects:
 #   - dcim.models.Device

initializers/device_types.yml

@@ -2,22 +2,22 @@
 #   manufacturer: Manufacturer 1
 #   slug: model-1
 #   u_height: 2
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Model 2
 #   manufacturer: Manufacturer 1
 #   slug: model-2
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Model 3
 #   manufacturer: Manufacturer 1
 #   slug: model-3
 #   is_full_depth: false
 #   u_height: 0
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Other
 #   manufacturer: No Name
 #   slug: other
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/devices.yml

@@ -20,7 +20,7 @@
 #   rack: rack-01
 #   face: front
 #   position: 1
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - name: server02
 #   device_role: server
@@ -29,7 +29,9 @@
 #   rack: rack-02
 #   face: front
 #   position: 2
-#   custom_fields:
+#   primary_ip4: 10.1.1.2/24
+#   primary_ip6: 2001:db8:a000:1::2/64
+#   custom_field_data:
 #     text_field: Description
 # - name: server03
 #   device_role: server
@@ -38,5 +40,5 @@
 #   rack: rack-03
 #   face: front
 #   position: 3
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/power_feeds.yml

@@ -0,0 +1,14 @@
+# - name: power feed 1
+#   power_panel: power panel AMS 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Single phase
+#   rack: rack-01
+# - name: power feed 2
+#   power_panel: power panel SING 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Three-phase
+#   rack: rack-03

initializers/power_panels.yml

@@ -0,0 +1,5 @@
+# - name: power panel AMS 1
+#   site: AMS 1   
+# - name: power panel SING 1
+#   site: SING 1 
+#   rack_group: cage 101

initializers/providers.yml

@@ -0,0 +1,6 @@
+# - name: Provider1
+#   slug: provider1
+#   asn: 121
+# - name: Provider2
+#   slug: provider2
+#   asn: 122

initializers/racks.yml

@@ -20,7 +20,7 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - site: AMS 2
 #   name: rack-02
@@ -28,7 +28,7 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - site: SING 1
 #   name: rack-03
@@ -37,5 +37,5 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/route_targets.yml

@@ -0,0 +1,3 @@
+# - name: 65000:1001
+#   tenant: tenant1
+# - name: 65000:1002

initializers/secret_roles.yml

@@ -0,0 +1,4 @@
+# - name: Super Secret Passwords
+#   slug: super-secret
+# - name: SNMP Communities
+#   slug: snmp

initializers/services.yml

@@ -0,0 +1,15 @@
+# - name: DNS
+#   protocol: TCP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: DNS
+#   protocol: UDP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: MISC
+#   protocol: UDP
+#   ports: 
+#     - 4000
+#   device: server01

initializers/sites.yml

@@ -1,32 +1,32 @@
 # - name: AMS 1
 #   slug: ams1
 #   region: Downtown
-#   status: 1
+#   status: active
 #   facility: Amsterdam 1
 #   asn: 12345
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS1
 # - name: AMS 2
 #   slug: ams2
 #   region: Downtown
-#   status: 1
+#   status: active
 #   facility: Amsterdam 2
 #   asn: 54321
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS2
 # - name: AMS 3
 #   slug: ams3
 #   region: Suburbs
-#   status: 1
+#   status: active
 #   facility: Amsterdam 3
 #   asn: 67890
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS3
 # - name: SING 1
 #   slug: sing1
 #   region: Singapore
-#   status: 1
+#   status: active
 #   facility: Singapore 1
 #   asn: 09876
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for SING1

initializers/tags.yml

@@ -0,0 +1,12 @@
+# - name: Tag 1
+#   slug: tag-1
+#   color: Pink
+# - name: Tag 2
+#   slug: tag-2
+#   color: Cyan
+# - name: Tag 3
+#   slug: tag-3
+#   color: Grey
+# - name: Tag 4
+#   slug: tag-4
+#   color: Teal

initializers/virtual_machines.yml

@@ -21,6 +21,8 @@
 #   memory: 2048
 #   name: virtual machine 2
 #   platform: Platform 2
+#   primary_ip4: 10.1.1.10/24
+#   primary_ip6: 2001:db8:a000:1::10/64
 #   status: active
 #   tenant: tenant1
 #   vcpus: 8

renovate.json

@@ -0,0 +1,13 @@
+{
+  "extends": [
+    "config:base"
+  ],
+  "enabled": true,
+  "labels": ["maintenance"],
+  "baseBranches": ["develop"],
+  "pip_requirements": {
+    "fileMatch": [
+      "requirements-container.txt"
+    ]
+  }
+}

reports/devices.py.example

@@ -1,4 +1,4 @@
-from dcim.constants import CONNECTION_STATUS_PLANNED, DEVICE_STATUS_ACTIVE
+from dcim.choices import DeviceStatusChoices
 from dcim.models import ConsolePort, Device, PowerPort
 from extras.reports import Report
 
@@ -9,13 +9,14 @@ class DeviceConnectionsReport(Report):
     def test_console_connection(self):
 
         # Check that every console port for every active device has a connection defined.
-        for console_port in ConsolePort.objects.select_related('device').filter(device__status=DEVICE_STATUS_ACTIVE):
+        active = DeviceStatusChoices.STATUS_ACTIVE
+        for console_port in ConsolePort.objects.prefetch_related('device').filter(device__status=active):
             if console_port.connected_endpoint is None:
                 self.log_failure(
                     console_port.device,
                     "No console connection defined for {}".format(console_port.name)
                 )
-            elif console_port.connection_status == CONNECTION_STATUS_PLANNED:
+            elif not console_port.connection_status:
                 self.log_warning(
                     console_port.device,
                     "Console connection for {} marked as planned".format(console_port.name)
@@ -26,12 +27,12 @@ class DeviceConnectionsReport(Report):
     def test_power_connections(self):
 
         # Check that every active device has at least two connected power supplies.
-        for device in Device.objects.filter(status=DEVICE_STATUS_ACTIVE):
+        for device in Device.objects.filter(status=DeviceStatusChoices.STATUS_ACTIVE):
             connected_ports = 0
             for power_port in PowerPort.objects.filter(device=device):
                 if power_port.connected_endpoint is not None:
                     connected_ports += 1
-                    if power_port.connection_status == CONNECTION_STATUS_PLANNED:
+                    if not power_port.connection_status:
                         self.log_warning(
                             device,
                             "Power connection for {} marked as planned".format(power_port.name)
@@ -43,4 +44,3 @@ class DeviceConnectionsReport(Report):
                 )
             else:
                 self.log_success(device)
-

requirements-container.txt

@@ -0,0 +1,4 @@
+napalm==3.2.0
+ruamel.yaml==0.16.12
+django-auth-ldap==2.2.0
+django-storages==1.11.1

startup_scripts/000_users.py

@@ -1,42 +1,23 @@
-from django.contrib.auth.models import Permission, Group, User
-from users.models import Token
-
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/users.yml')
-if not file.is_file():
+from django.contrib.auth.models import User
+from startup_script_utils import load_yaml, set_permissions
+from users.models import Token
+
+users = load_yaml('/opt/netbox/initializers/users.yml')
+if users is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  users = yaml.load(stream)
+for username, user_details in users.items():
+  if not User.objects.filter(username=username):
+    user = User.objects.create_user(
+      username = username,
+      password = user_details.get('password', 0) or User.objects.make_random_password())
 
-  if users is not None:
-    for username, user_details in users.items():
-      if not User.objects.filter(username=username):
-        user = User.objects.create_user(
-          username = username,
-          password = user_details.get('password', 0) or User.objects.make_random_password)
+    print("👤 Created user",username)
 
-        print("👤 Created user",username)
+    if user_details.get('api_token', 0):
+      Token.objects.create(user=user, key=user_details['api_token'])
 
-        if user_details.get('api_token', 0):
-          Token.objects.create(user=user, key=user_details['api_token'])
-
-        yaml_permissions = user_details.get('permissions', [])
-        if yaml_permissions:
-          subject = user.user_permissions
-          subject.clear()
-          for yaml_permission in yaml_permissions:
-            if '*' in yaml_permission:
-              permission_filter = '^' + yaml_permission.replace('*','.*') + '$'
-              permissions = Permission.objects.filter(codename__iregex=permission_filter)
-              print("  ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'")
-            else:
-              permissions = Permission.objects.filter(codename=yaml_permission)
-              print("  ⚿ Granting permission", yaml_permission)
-            
-            for permission in permissions:
-              subject.add(permission)
+    yaml_permissions = user_details.get('permissions', [])
+    set_permissions(user.user_permissions, yaml_permissions)

startup_scripts/010_groups.py

@@ -1,41 +1,23 @@
-from django.contrib.auth.models import Permission, Group, User
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/groups.yml')
-if not file.is_file():
+from django.contrib.auth.models import Group, User
+from startup_script_utils import load_yaml, set_permissions
+
+groups = load_yaml('/opt/netbox/initializers/groups.yml')
+if groups is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  groups = yaml.load(stream)
+for groupname, group_details in groups.items():
+  group, created = Group.objects.get_or_create(name=groupname)
 
-  if groups is not None:
-    for groupname, group_details in groups.items():
-      group, created = Group.objects.get_or_create(name=groupname)
+  if created:
+    print("👥 Created group", groupname)
 
-      if created:
-        print("👥 Created group", groupname)
+  for username in group_details.get('users', []):
+    user = User.objects.get(username=username)
 
-      for username in group_details.get('users', []):
-        user = User.objects.get(username=username)
+    if user:
+      user.groups.add(group)
 
-        if user:
-          user.groups.add(group)
-
-      yaml_permissions = group_details.get('permissions', [])
-      if yaml_permissions:
-        subject = group.permissions
-        subject.clear()
-        for yaml_permission in yaml_permissions:
-          if '*' in yaml_permission:
-            permission_filter = '^' + yaml_permission.replace('*','.*') + '$'
-            permissions = Permission.objects.filter(codename__iregex=permission_filter)
-            print("  ⚿ Granting", permissions.count(), "permissions matching '" + yaml_permission + "'")
-          else:
-            permissions = Permission.objects.filter(codename=yaml_permission)
-            print("  ⚿ Granting permission", yaml_permission)
-
-          for permission in permissions:
-            subject.add(permission)
+  yaml_permissions = group_details.get('permissions', [])
+  set_permissions(group.permissions, yaml_permissions)

startup_scripts/020_custom_fields.py

@@ -1,9 +1,8 @@
-from extras.models import CustomField, CustomFieldChoice
-
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
+from extras.models import CustomField
+from startup_script_utils import load_yaml
+
 def get_class_for_class_path(class_path):
   import importlib
   from django.contrib.contenttypes.models import ContentType
@@ -13,47 +12,47 @@ def get_class_for_class_path(class_path):
   clazz = getattr(module, class_name)
   return ContentType.objects.get_for_model(clazz)
 
-file = Path('/opt/netbox/initializers/custom_fields.yml')
-if not file.is_file():
+customfields = load_yaml('/opt/netbox/initializers/custom_fields.yml')
+
+if customfields is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  customfields = yaml.load(stream)
+for cf_name, cf_details in customfields.items():
+  custom_field, created = CustomField.objects.get_or_create(name = cf_name)
 
-  if customfields is not None:
-    for cf_name, cf_details in customfields.items():
-      custom_field, created = CustomField.objects.get_or_create(name = cf_name)
+  if created:
+    if cf_details.get('default', False):
+      custom_field.default = cf_details['default']
 
-      if created:
-        if cf_details.get('default', 0):
-          custom_field.default = cf_details['default']
+    if cf_details.get('description', False):
+      custom_field.description = cf_details['description']
 
-        if cf_details.get('description', 0):
-          custom_field.description = cf_details['description']
+    if cf_details.get('label', False):
+      custom_field.label = cf_details['label']
 
-        if cf_details.get('label', 0):
-          custom_field.label = cf_details['label']
+    for object_type in cf_details.get('on_objects', []):
+      custom_field.content_types.add(get_class_for_class_path(object_type))
 
-        for object_type in cf_details.get('on_objects', []):
-          custom_field.obj_type.add(get_class_for_class_path(object_type))
+    if cf_details.get('required', False):
+      custom_field.required = cf_details['required']
 
-        if cf_details.get('required', 0):
-          custom_field.required = cf_details['required']
+    if cf_details.get('type', False):
+      custom_field.type = cf_details['type']
 
-        if cf_details.get('type', 0):
-          custom_field.type = cf_details['type']
+    if cf_details.get('weight', -1) >= 0:
+      custom_field.weight = cf_details['weight']
 
-        if cf_details.get('weight', 0):
-          custom_field.weight = cf_details['weight']
+    if cf_details.get('choices', False):
+      custom_field.choices = []
 
-        custom_field.save()
+      for choice_detail in cf_details.get('choices', []):
+        if isinstance(choice_detail, dict) and 'value' in choice_detail:
+          # legacy mode
+          print(f"⚠️ Please migrate the choice '{choice_detail['value']}' of '{cf_name}' to the new format, as 'weight' is no longer supported!")
+          custom_field.choices.append(choice_detail['value'])
+        else:
+          custom_field.choices.append(choice_detail)
 
-        for idx, choice_details in enumerate(cf_details.get('choices', [])):
-          choice, _ = CustomFieldChoice.objects.get_or_create(
-            field=custom_field,
-            value=choice_details['value'],
-            defaults={'weight': idx * 10}
-          )
+    custom_field.save()
 
-        print("🔧 Created custom field", cf_name)
+    print("🔧 Created custom field", cf_name)

startup_scripts/020_tags.py

@@ -0,0 +1,23 @@
+from extras.models import Tag
+from utilities.choices import ColorChoices
+
+from startup_script_utils import load_yaml
+import sys
+
+tags = load_yaml('/opt/netbox/initializers/tags.yml')
+
+if tags is None:
+  sys.exit()
+
+for params in tags:
+  if 'color' in params:
+    color = params.pop('color')
+
+    for color_tpl in ColorChoices:
+      if color in color_tpl:
+        params['color'] = color_tpl[0]
+
+  tag, created = Tag.objects.get_or_create(**params)
+
+  if created:
+    print("🎨 Created Tag", tag.name)

startup_scripts/030_regions.py

@@ -1,31 +1,26 @@
 from dcim.models import Region
-from ruamel.yaml import YAML
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/regions.yml')
-if not file.is_file():
+regions = load_yaml('/opt/netbox/initializers/regions.yml')
+
+if regions is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  regions = yaml.load(stream)
+optional_assocs = {
+  'parent': (Region, 'name')
+}
 
-  optional_assocs = {
-    'parent': (Region, 'name')
-  }
+for params in regions:
 
-  if regions is not None:
-    for params in regions:
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  region, created = Region.objects.get_or_create(**params)
 
-      region, created = Region.objects.get_or_create(**params)
-
-      if created:
-        print("🌐 Created region", region.name)
+  if created:
+    print("🌐 Created region", region.name)

startup_scripts/040_sites.py

@@ -1,46 +1,32 @@
-from dcim.models import Region, Site
-from extras.models import CustomField, CustomFieldValue
-from tenancy.models import Tenant
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/sites.yml')
-if not file.is_file():
+from dcim.models import Region, Site
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+sites = load_yaml('/opt/netbox/initializers/sites.yml')
+
+if sites is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  sites = yaml.load(stream)
+optional_assocs = {
+  'region': (Region, 'name'),
+  'tenant': (Tenant, 'name')
+}
 
-  optional_assocs = {
-    'region': (Region, 'name'),
-    'tenant': (Tenant, 'name')
-  }
+for params in sites:
+  custom_field_data = pop_custom_fields(params)
 
-  if sites is not None:
-    for params in sites:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  site, created = Site.objects.get_or_create(**params)
 
-      site, created = Site.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(site, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=site,
-              value=cf_value
-            )
-
-            site.custom_field_values.add(custom_field_value)
-
-        print("📍 Created site", site.name)
+    print("📍 Created site", site.name)

startup_scripts/050_manufacturers.py

@@ -1,19 +1,14 @@
 from dcim.models import Manufacturer
-from ruamel.yaml import YAML
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/manufacturers.yml')
-if not file.is_file():
+manufacturers = load_yaml('/opt/netbox/initializers/manufacturers.yml')
+
+if manufacturers is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  manufacturers = yaml.load(stream)
+for params in manufacturers:
+  manufacturer, created = Manufacturer.objects.get_or_create(**params)
 
-  if manufacturers is not None:
-    for params in manufacturers:
-      manufacturer, created = Manufacturer.objects.get_or_create(**params)
-
-      if created:
-        print("🏭 Created Manufacturer", manufacturer.name)
+  if created:
+    print("🏭 Created Manufacturer", manufacturer.name)

startup_scripts/060_device_types.py

@@ -1,56 +1,42 @@
-from dcim.models import DeviceType, Manufacturer, Region
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/device_types.yml')
-if not file.is_file():
+from dcim.models import DeviceType, Manufacturer, Region
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+device_types = load_yaml('/opt/netbox/initializers/device_types.yml')
+
+if device_types is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  device_types = yaml.load(stream)
+required_assocs = {
+  'manufacturer': (Manufacturer, 'name')
+}
 
-  required_assocs = {
-    'manufacturer': (Manufacturer, 'name')
-  }
+optional_assocs = {
+  'region': (Region, 'name'),
+  'tenant': (Tenant, 'name')
+}
 
-  optional_assocs = {
-    'region': (Region, 'name'),
-    'tenant': (Tenant, 'name')
-  }
+for params in device_types:
+  custom_field_data = pop_custom_fields(params)
 
-  if device_types is not None:
-    for params in device_types:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+    params[assoc] = model.objects.get(**query)
 
-        params[assoc] = model.objects.get(**query)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  device_type, created = DeviceType.objects.get_or_create(**params)
 
-      device_type, created = DeviceType.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(device_type, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=device_type,
-              value=cf_value
-            )
-
-            device_type.custom_field_values.add(custom_field_value)
-
-        print("🔡 Created device type", device_type.manufacturer, device_type.model)
+    print("🔡 Created device type", device_type.manufacturer, device_type.model)

startup_scripts/070_rack_roles.py

@@ -1,28 +1,23 @@
 from dcim.models import RackRole
-from ruamel.yaml import YAML
-from utilities.forms import COLOR_CHOICES
+from utilities.choices import ColorChoices
 
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/rack_roles.yml')
-if not file.is_file():
+rack_roles = load_yaml('/opt/netbox/initializers/rack_roles.yml')
+
+if rack_roles is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  rack_roles = yaml.load(stream)
+for params in rack_roles:
+  if 'color' in params:
+    color = params.pop('color')
 
-  if rack_roles is not None:
-    for params in rack_roles:
-      if 'color' in params:
-        color = params.pop('color')
+    for color_tpl in ColorChoices:
+      if color in color_tpl:
+        params['color'] = color_tpl[0]
 
-        for color_tpl in COLOR_CHOICES:
-          if color in color_tpl:
-            params['color'] = color_tpl[0]
+  rack_role, created = RackRole.objects.get_or_create(**params)
 
-      rack_role, created = RackRole.objects.get_or_create(**params)
-
-      if created:
-        print("🎨 Created rack role", rack_role.name)
+  if created:
+    print("🎨 Created rack role", rack_role.name)

startup_scripts/075_rack_groups.py

@@ -1,31 +1,25 @@
 from dcim.models import Site,RackGroup
-from ruamel.yaml import YAML
-
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/rack_groups.yml')
-if not file.is_file():
+rack_groups = load_yaml('/opt/netbox/initializers/rack_groups.yml')
+
+if rack_groups is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  rack_groups= yaml.load(stream)
+required_assocs = {
+  'site': (Site, 'name')
+}
 
-  required_assocs = {
-    'site': (Site, 'name')
-  }
+for params in rack_groups:
 
-  if rack_groups is not None:
-    for params in rack_groups:
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+    params[assoc] = model.objects.get(**query)
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
-        params[assoc] = model.objects.get(**query)
+  rack_group, created = RackGroup.objects.get_or_create(**params)
 
-      rack_group, created = RackGroup.objects.get_or_create(**params)
-
-      if created:
-        print("🎨 Created rack group", rack_group.name)
+  if created:
+    print("🎨 Created rack group", rack_group.name)
 

startup_scripts/080_racks.py

@@ -1,57 +1,43 @@
-from dcim.models import Site, RackRole, Rack, RackGroup
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/racks.yml')
-if not file.is_file():
+from dcim.models import Site, RackRole, Rack, RackGroup
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+racks = load_yaml('/opt/netbox/initializers/racks.yml')
+
+if racks is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  racks = yaml.load(stream)
+required_assocs = {
+  'site': (Site, 'name')
+}
 
-  required_assocs = {
-    'site': (Site, 'name')
-  }
+optional_assocs = {
+  'role': (RackRole, 'name'),
+  'tenant': (Tenant, 'name'),
+  'group': (RackGroup, 'name')
+}
 
-  optional_assocs = {
-    'role': (RackRole, 'name'),
-    'tenant': (Tenant, 'name'),
-    'group': (RackGroup, 'name')
-  }
+for params in racks:
+  custom_field_data = pop_custom_fields(params)
 
-  if racks is not None:
-    for params in racks:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+    params[assoc] = model.objects.get(**query)
 
-        params[assoc] = model.objects.get(**query)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  rack, created = Rack.objects.get_or_create(**params)
 
-      rack, created = Rack.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(rack, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=rack,
-              value=cf_value
-            )
-
-            rack.custom_field_values.add(custom_field_value)
-
-        print("🔳 Created rack", rack.site, rack.name)
+    print("🔳 Created rack", rack.site, rack.name)

startup_scripts/090_device_roles.py

@@ -1,29 +1,24 @@
 from dcim.models import DeviceRole
-from ruamel.yaml import YAML
-from utilities.forms import COLOR_CHOICES
+from utilities.choices import ColorChoices
 
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/device_roles.yml')
-if not file.is_file():
+device_roles = load_yaml('/opt/netbox/initializers/device_roles.yml')
+
+if device_roles is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml=YAML(typ='safe')
-  device_roles = yaml.load(stream)
+for params in device_roles:
 
-  if device_roles is not None:
-    for params in device_roles:
+  if 'color' in params:
+    color = params.pop('color')
 
-      if 'color' in params:
-        color = params.pop('color')
+    for color_tpl in ColorChoices:
+      if color in color_tpl:
+        params['color'] = color_tpl[0]
 
-        for color_tpl in COLOR_CHOICES:
-          if color in color_tpl:
-            params['color'] = color_tpl[0]
+  device_role, created = DeviceRole.objects.get_or_create(**params)
 
-      device_role, created = DeviceRole.objects.get_or_create(**params)
-
-      if created:
-        print("🎨 Created device role", device_role.name)
+  if created:
+    print("🎨 Created device role", device_role.name)

startup_scripts/100_platforms.py

@@ -1,32 +1,26 @@
 from dcim.models import Manufacturer, Platform
-from ruamel.yaml import YAML
-
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/platforms.yml')
-if not file.is_file():
+platforms = load_yaml('/opt/netbox/initializers/platforms.yml')
+
+if platforms is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  platforms = yaml.load(stream)
+optional_assocs = {
+  'manufacturer': (Manufacturer, 'name'),
+}
 
-  optional_assocs = {
-    'manufacturer': (Manufacturer, 'name'),
-  }
+for params in platforms:
 
-  if platforms is not None:
-    for params in platforms:
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  platform, created = Platform.objects.get_or_create(**params)
 
-      platform, created = Platform.objects.get_or_create(**params)
-
-      if created:
-        print("💾 Created platform", platform.name)
+  if created:
+    print("💾 Created platform", platform.name)

startup_scripts/110_tenant_groups.py

@@ -1,19 +1,14 @@
 from tenancy.models import TenantGroup
-from ruamel.yaml import YAML
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/tenant_groups.yml')
-if not file.is_file():
+tenant_groups = load_yaml('/opt/netbox/initializers/tenant_groups.yml')
+
+if tenant_groups is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  tenant_groups = yaml.load(stream)
+for params in tenant_groups:
+  tenant_group, created = TenantGroup.objects.get_or_create(**params)
 
-  if tenant_groups is not None:
-    for params in tenant_groups:
-      tenant_group, created = TenantGroup.objects.get_or_create(**params)
-
-      if created:
-        print("🔳 Created Tenant Group", tenant_group.name)
+  if created:
+    print("🔳 Created Tenant Group", tenant_group.name)

startup_scripts/120_tenants.py

@@ -1,45 +1,30 @@
-from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/tenants.yml')
-if not file.is_file():
+from startup_script_utils import *
+from tenancy.models import Tenant, TenantGroup
+
+tenants = load_yaml('/opt/netbox/initializers/tenants.yml')
+
+if tenants is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  tenants = yaml.load(stream)
+optional_assocs = {
+  'group': (TenantGroup, 'name')
+}
 
-  optional_assocs = {
-    'group': (TenantGroup, 'name')
-  }
+for params in tenants:
+  custom_field_data = pop_custom_fields(params)
 
-  if tenants is not None:
-    for params in tenants:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  tenant, created = Tenant.objects.get_or_create(**params)
 
-      tenant, created = Tenant.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(tenant, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=tenant,
-              value=cf_value
-            )
-
-            tenant.custom_field_values.add(custom_field_value)
-
-        print("👩‍💻 Created Tenant", tenant.name)
+    print("👩‍💻 Created Tenant", tenant.name)

startup_scripts/130_cluster_types.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterType
+from startup_script_utils import load_yaml
+import sys
+
+cluster_types = load_yaml('/opt/netbox/initializers/cluster_types.yml')
+
+if cluster_types is None:
+  sys.exit()
+
+for params in cluster_types:
+  cluster_type, created = ClusterType.objects.get_or_create(**params)
+
+  if created:
+    print("🧰 Created Cluster Type", cluster_type.name)

startup_scripts/130_devices.py

@@ -1,65 +0,0 @@
-from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
-from ipam.models import IPAddress
-from virtualization.models import Cluster
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
-import sys
-
-file = Path('/opt/netbox/initializers/devices.yml')
-if not file.is_file():
-  sys.exit()
-
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  devices = yaml.load(stream)
-
-  required_assocs = {
-    'device_role': (DeviceRole, 'name'),
-    'device_type': (DeviceType, 'model'),
-    'site': (Site, 'name')
-  }
-
-  optional_assocs = {
-    'tenant': (Tenant, 'name'),
-    'platform': (Platform, 'name'),
-    'rack': (Rack, 'name'),
-    'cluster': (Cluster, 'name'),
-    'primary_ip4': (IPAddress, 'address'),
-    'primary_ip6': (IPAddress, 'address')
-  }
-
-  if devices is not None:
-    for params in devices:
-      custom_fields = params.pop('custom_fields', None)
-
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
-
-        params[assoc] = model.objects.get(**query)
-
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
-
-          params[assoc] = model.objects.get(**query)
-
-      device, created = Device.objects.get_or_create(**params)
-
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=device,
-              value=cf_value
-            )
-
-            device.custom_field_values.add(custom_field_value)
-
-        print("🖥️  Created device", device.name)

startup_scripts/135_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/135_clusters.py

@@ -0,0 +1,44 @@
+import sys
+
+from dcim.models import Site
+from startup_script_utils import *
+from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
+
+clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
+
+if clusters is None:
+  sys.exit()
+
+required_assocs = {
+  'type': (ClusterType, 'name')
+}
+
+optional_assocs = {
+  'site': (Site, 'name'),
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
+}
+
+for params in clusters:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  cluster, created = Cluster.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(cluster, custom_field_data)
+
+    print("🗄️ Created cluster", cluster.name)

startup_scripts/140_cluster_types.py

@@ -1,19 +0,0 @@
-from virtualization.models import ClusterType
-from ruamel.yaml import YAML
-from pathlib import Path
-import sys
-
-file = Path('/opt/netbox/initializers/cluster_types.yml')
-if not file.is_file():
-  sys.exit()
-
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  cluster_types = yaml.load(stream)
-
-  if cluster_types is not None:
-    for params in cluster_types:
-      cluster_type, created = ClusterType.objects.get_or_create(**params)
-
-      if created:
-        print("🧰 Created Cluster Type", cluster_type.name)

startup_scripts/140_clusters.py

@@ -0,0 +1,44 @@
+import sys
+
+from dcim.models import Site
+from startup_script_utils import *
+from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
+
+clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
+
+if clusters is None:
+  sys.exit()
+
+required_assocs = {
+  'type': (ClusterType, 'name')
+}
+
+optional_assocs = {
+  'site': (Site, 'name'),
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
+}
+
+for params in clusters:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  cluster, created = Cluster.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(cluster, custom_field_data)
+
+    print("🗄️ Created cluster", cluster.name)

startup_scripts/140_devices.py

@@ -0,0 +1,51 @@
+import sys
+
+from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster
+
+devices = load_yaml('/opt/netbox/initializers/devices.yml')
+
+if devices is None:
+  sys.exit()
+
+required_assocs = {
+  'device_role': (DeviceRole, 'name'),
+  'device_type': (DeviceType, 'model'),
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'platform': (Platform, 'name'),
+  'rack': (Rack, 'name'),
+  'cluster': (Cluster, 'name')
+}
+
+for params in devices:
+  custom_field_data = pop_custom_fields(params)
+
+  # primary ips are handled later in `270_primary_ips.py`
+  params.pop('primary_ip4', None)
+  params.pop('primary_ip6', None)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  device, created = Device.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(device, custom_field_data)
+
+    print("🖥️  Created device", device.name)

startup_scripts/145_devices.py

@@ -0,0 +1,51 @@
+import sys
+
+from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster
+
+devices = load_yaml('/opt/netbox/initializers/devices.yml')
+
+if devices is None:
+  sys.exit()
+
+required_assocs = {
+  'device_role': (DeviceRole, 'name'),
+  'device_type': (DeviceType, 'model'),
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'platform': (Platform, 'name'),
+  'rack': (Rack, 'name'),
+  'cluster': (Cluster, 'name')
+}
+
+for params in devices:
+  custom_field_data = pop_custom_fields(params)
+
+  # primary ips are handled later in `270_primary_ips.py`
+  params.pop('primary_ip4', None)
+  params.pop('primary_ip6', None)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  device, created = Device.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(device, custom_field_data)
+
+    print("🖥️  Created device", device.name)

startup_scripts/150_rirs.py

@@ -1,19 +1,14 @@
 from ipam.models import RIR
-from ruamel.yaml import YAML
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/rirs.yml')
-if not file.is_file():
+rirs = load_yaml('/opt/netbox/initializers/rirs.yml')
+
+if rirs is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  rirs = yaml.load(stream)
+for params in rirs:
+  rir, created = RIR.objects.get_or_create(**params)
 
-  if rirs is not None:
-    for params in rirs:
-      rir, created = RIR.objects.get_or_create(**params)
-
-      if created:
-        print("🗺️ Created RIR", rir.name)
+  if created:
+    print("🗺️ Created RIR", rir.name)

startup_scripts/160_aggregates.py

@@ -1,46 +1,44 @@
-from ipam.models import Aggregate, RIR
-from ruamel.yaml import YAML
-from extras.models import CustomField, CustomFieldValue
-
-from netaddr import IPNetwork
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/aggregates.yml')
-if not file.is_file():
+from ipam.models import Aggregate, RIR
+from netaddr import IPNetwork
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+aggregates = load_yaml('/opt/netbox/initializers/aggregates.yml')
+
+if aggregates is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  aggregates = yaml.load(stream)
+required_assocs = {
+  'rir': (RIR, 'name')
+}
 
-  required_assocs = {
-    'rir': (RIR, 'name')
-  }
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+}
 
-  if aggregates is not None:
-    for params in aggregates:
-      custom_fields = params.pop('custom_fields', None)
-      params['prefix'] = IPNetwork(params['prefix'])
+for params in aggregates:
+  custom_field_data = pop_custom_fields(params)
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+  params['prefix'] = IPNetwork(params['prefix'])
 
-        params[assoc] = model.objects.get(**query)
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-      aggregate, created = Aggregate.objects.get_or_create(**params)
+    params[assoc] = model.objects.get(**query)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=aggregate,
-              value=cf_value
-            )
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-            aggregate.custom_field_values.add(custom_field_value)
+      params[assoc] = model.objects.get(**query)
 
-        print("🗞️ Created Aggregate", aggregate.prefix)
+  aggregate, created = Aggregate.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(aggregate, custom_field_data)
+
+    print("🗞️ Created Aggregate", aggregate.prefix)

startup_scripts/165_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/170_clusters.py

@@ -1,57 +0,0 @@
-from dcim.models import Site
-from virtualization.models import Cluster, ClusterType, ClusterGroup
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
-import sys
-
-file = Path('/opt/netbox/initializers/clusters.yml')
-if not file.is_file():
-  sys.exit()
-
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  clusters = yaml.load(stream)
-
-  required_assocs = {
-    'type': (ClusterType, 'name')
-  }
-
-  optional_assocs = {
-    'site': (Site, 'name'),
-    'group': (ClusterGroup, 'name')
-  }
-
-  if clusters is not None:
-    for params in clusters:
-      custom_fields = params.pop('custom_fields', None)
-
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
-
-        params[assoc] = model.objects.get(**query)
-
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
-
-          params[assoc] = model.objects.get(**query)
-
-      cluster, created = Cluster.objects.get_or_create(**params)
-
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=cluster,
-              value=cf_value
-            )
-
-            cluster.custom_field_values.add(custom_field_value)
-
-        print("🗄️ Created cluster", cluster.name)

startup_scripts/175_route_targets.py

@@ -0,0 +1,31 @@
+import sys
+
+from ipam.models import RouteTarget
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+route_targets = load_yaml('/opt/netbox/initializers/route_targets.yml')
+
+if route_targets is None:
+  sys.exit()
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in route_targets:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  route_target, created = RouteTarget.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(route_target, custom_field_data)
+
+    print("🎯 Created Route Target", route_target.name)

startup_scripts/180_vrfs.py

@@ -1,46 +1,31 @@
-from ipam.models import VRF
-from tenancy.models import Tenant
-from ruamel.yaml import YAML
-from extras.models import CustomField, CustomFieldValue
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/vrfs.yml')
-if not file.is_file():
+from ipam.models import VRF
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+vrfs = load_yaml('/opt/netbox/initializers/vrfs.yml')
+
+if vrfs is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  vrfs = yaml.load(stream)
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
 
-  optional_assocs = {
-    'tenant': (Tenant, 'name')
-  }
+for params in vrfs:
+  custom_field_data = pop_custom_fields(params)
 
-  if vrfs is not None:
-    for params in vrfs:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  vrf, created = VRF.objects.get_or_create(**params)
 
-      vrf, created = VRF.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(vrf, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=vrf,
-              value=cf_value
-            )
-
-            vrf.custom_field_values.add(custom_field_value)
-
-        print("📦 Created VRF", vrf.name)
+    print("📦 Created VRF", vrf.name)

startup_scripts/190_prefix_vlan_roles.py

@@ -1,19 +1,14 @@
 from ipam.models import Role
-from ruamel.yaml import YAML
-from pathlib import Path
+from startup_script_utils import load_yaml
 import sys
 
-file = Path('/opt/netbox/initializers/prefix_vlan_roles.yml')
-if not file.is_file():
+roles = load_yaml('/opt/netbox/initializers/prefix_vlan_roles.yml')
+
+if roles is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  roles = yaml.load(stream)
+for params in roles:
+  role, created = Role.objects.get_or_create(**params)
 
-  if roles is not None:
-    for params in roles:
-      role, created = Role.objects.get_or_create(**params)
-
-      if created:
-        print("⛹️‍ Created Prefix/VLAN Role", role.name)
+  if created:
+    print("⛹️‍ Created Prefix/VLAN Role", role.name)

startup_scripts/200_vlan_groups.py

@@ -1,46 +1,31 @@
-from dcim.models import Site
-from ipam.models import VLANGroup
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/vlan_groups.yml')
-if not file.is_file():
+from dcim.models import Site
+from ipam.models import VLANGroup
+from startup_script_utils import *
+
+vlan_groups = load_yaml('/opt/netbox/initializers/vlan_groups.yml')
+
+if vlan_groups is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  vlan_groups = yaml.load(stream)
+optional_assocs = {
+  'site': (Site, 'name')
+}
 
-  optional_assocs = {
-    'site': (Site, 'name')
-  }
+for params in vlan_groups:
+  custom_field_data = pop_custom_fields(params)
 
-  if vlan_groups is not None:
-    for params in vlan_groups:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  vlan_group, created = VLANGroup.objects.get_or_create(**params)
 
-      vlan_group, created = VLANGroup.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(vlan_group, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=vlan_group,
-              value=cf_value
-            )
-
-            vlan_group.custom_field_values.add(custom_field_value)
-
-        print("🏘️ Created VLAN Group", vlan_group.name)
+    print("🏘️ Created VLAN Group", vlan_group.name)

startup_scripts/210_vlans.py

@@ -1,51 +1,36 @@
-from dcim.models import Site
-from ipam.models import VLAN, VLANGroup, Role
-from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/vlans.yml')
-if not file.is_file():
+from dcim.models import Site
+from ipam.models import VLAN, VLANGroup, Role
+from startup_script_utils import *
+from tenancy.models import Tenant, TenantGroup
+
+vlans = load_yaml('/opt/netbox/initializers/vlans.yml')
+
+if vlans is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  vlans = yaml.load(stream)
+optional_assocs = {
+  'site': (Site, 'name'),
+  'tenant': (Tenant, 'name'),
+  'tenant_group': (TenantGroup, 'name'),
+  'group': (VLANGroup, 'name'),
+  'role': (Role, 'name')
+}
 
-  optional_assocs = {
-    'site': (Site, 'name'),
-    'tenant': (Tenant, 'name'),
-    'tenant_group': (TenantGroup, 'name'),
-    'group': (VLANGroup, 'name'),
-    'role': (Role, 'name')
-  }
+for params in vlans:
+  custom_field_data = pop_custom_fields(params)
 
-  if vlans is not None:
-    for params in vlans:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  vlan, created = VLAN.objects.get_or_create(**params)
 
-      vlan, created = VLAN.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(vlan, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=vlan,
-              value=cf_value
-            )
-
-            vlan.custom_field_values.add(custom_field_value)
-
-        print("🏠 Created VLAN", vlan.name)
+    print("🏠 Created VLAN", vlan.name)

startup_scripts/220_prefixes.py

@@ -1,54 +1,39 @@
-from dcim.models import Site
-from ipam.models import Prefix, VLAN, Role, VRF
-from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from netaddr import IPNetwork
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/prefixes.yml')
-if not file.is_file():
+from dcim.models import Site
+from ipam.models import Prefix, VLAN, Role, VRF
+from netaddr import IPNetwork
+from startup_script_utils import *
+from tenancy.models import Tenant, TenantGroup
+
+prefixes = load_yaml('/opt/netbox/initializers/prefixes.yml')
+
+if prefixes is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  prefixes = yaml.load(stream)
+optional_assocs = {
+  'site': (Site, 'name'),
+  'tenant': (Tenant, 'name'),
+  'tenant_group': (TenantGroup, 'name'),
+  'vlan': (VLAN, 'name'),
+  'role': (Role, 'name'),
+  'vrf': (VRF, 'name')
+}
 
-  optional_assocs = {
-    'site': (Site, 'name'),
-    'tenant': (Tenant, 'name'),
-    'tenant_group': (TenantGroup, 'name'),
-    'vlan': (VLAN, 'name'),
-    'role': (Role, 'name'),
-    'vrf': (VRF, 'name')
-  }
+for params in prefixes:
+  custom_field_data = pop_custom_fields(params)
 
-  if prefixes is not None:
-    for params in prefixes:
-      custom_fields = params.pop('custom_fields', None)
-      params['prefix'] = IPNetwork(params['prefix'])
+  params['prefix'] = IPNetwork(params['prefix'])
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+      params[assoc] = model.objects.get(**query)
 
-          params[assoc] = model.objects.get(**query)
+  prefix, created = Prefix.objects.get_or_create(**params)
 
-      prefix, created = Prefix.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(prefix, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=prefix,
-              value=cf_value
-            )
-
-            prefix.custom_field_values.add(custom_field_value)
-
-        print("📌 Created Prefix", prefix.prefix)
+    print("📌 Created Prefix", prefix.prefix)

startup_scripts/230_virtual_machines.py

@@ -1,59 +1,48 @@
-from dcim.models import Site, Platform, DeviceRole
-from virtualization.models import Cluster, VirtualMachine
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/virtual_machines.yml')
-if not file.is_file():
+from dcim.models import Platform, DeviceRole
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster, VirtualMachine
+
+virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml')
+
+if virtual_machines is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  virtual_machines = yaml.load(stream)
+required_assocs = {
+  'cluster': (Cluster, 'name')
+}
 
-  required_assocs = {
-    'cluster': (Cluster, 'name')
-  }
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'platform': (Platform, 'name'),
+  'role': (DeviceRole, 'name')
+}
 
-  optional_assocs = {
-    'tenant': (Tenant, 'name'),
-    'platform': (Platform, 'name'),
-    'role': (DeviceRole, 'name')
-  }
+for params in virtual_machines:
+  custom_field_data = pop_custom_fields(params)
 
-  if virtual_machines is not None:
-    for params in virtual_machines:
-      custom_fields = params.pop('custom_fields', None)
+  # primary ips are handled later in `270_primary_ips.py`
+  params.pop('primary_ip4', None)
+  params.pop('primary_ip6', None)
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-        params[assoc] = model.objects.get(**query)
+    params[assoc] = model.objects.get(**query)
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          query = { field: params.pop(assoc) }
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
 
-          params[assoc] = model.objects.get(**query)
+      params[assoc] = model.objects.get(**query)
 
-      virtual_machine, created = VirtualMachine.objects.get_or_create(**params)
+  virtual_machine, created = VirtualMachine.objects.get_or_create(**params)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=virtual_machine,
-              value=cf_value
-            )
+  if created:
+    set_custom_fields_values(virtual_machine, custom_field_data)
 
-            virtual_machine.custom_field_values.add(custom_field_value)
-
-        print("🖥️ Created virtual machine", virtual_machine.name)
+    print("🖥️ Created virtual machine", virtual_machine.name)

startup_scripts/240_virtualization_interfaces.py

@@ -1,45 +1,29 @@
-from dcim.models import Interface
-from virtualization.models import VirtualMachine
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/virtualization_interfaces.yml')
-if not file.is_file():
+from startup_script_utils import *
+from virtualization.models import VirtualMachine, VMInterface
+
+interfaces = load_yaml('/opt/netbox/initializers/virtualization_interfaces.yml')
+
+if interfaces is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  interfaces = yaml.load(stream)
+required_assocs = {
+  'virtual_machine': (VirtualMachine, 'name')
+}
 
-  required_assocs = {
-    'virtual_machine': (VirtualMachine, 'name')
-  }
+for params in interfaces:
+  custom_field_data = pop_custom_fields(params)
 
-  if interfaces is not None:
-    for params in interfaces:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+    params[assoc] = model.objects.get(**query)
 
-        params[assoc] = model.objects.get(**query)
+  interface, created = VMInterface.objects.get_or_create(**params)
 
-      interface, created = Interface.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(interface, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=interface,
-              value=cf_value
-            )
-
-            interface.custom_field_values.add(custom_field_value)
-
-        print("🧷 Created interface", interface.name, interface.virtual_machine.name)
+    print("🧷 Created interface", interface.name, interface.virtual_machine.name)

startup_scripts/250_dcim_interfaces.py

@@ -1,44 +1,29 @@
-from dcim.models import Interface, Device
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/dcim_interfaces.yml')
-if not file.is_file():
+from dcim.models import Interface, Device
+from startup_script_utils import *
+
+interfaces = load_yaml('/opt/netbox/initializers/dcim_interfaces.yml')
+
+if interfaces is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  interfaces = yaml.load(stream)
+required_assocs = {
+  'device': (Device, 'name')
+}
 
-  required_assocs = {
-    'device': (Device, 'name')
-  }
+for params in interfaces:
+  custom_field_data = pop_custom_fields(params)
 
-  if interfaces is not None:
-    for params in interfaces:
-      custom_fields = params.pop('custom_fields', None)
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
 
-      for assoc, details in required_assocs.items():
-        model, field = details
-        query = { field: params.pop(assoc) }
+    params[assoc] = model.objects.get(**query)
 
-        params[assoc] = model.objects.get(**query)
+  interface, created = Interface.objects.get_or_create(**params)
 
-      interface, created = Interface.objects.get_or_create(**params)
+  if created:
+    set_custom_fields_values(interface, custom_field_data)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=interface,
-              value=cf_value
-            )
-
-            interface.custom_field_values.add(custom_field_value)
-
-        print("🧷 Created interface", interface.name, interface.device.name)
+    print("🧷 Created interface", interface.name, interface.device.name)

startup_scripts/260_ip_addresses.py

@@ -1,65 +1,60 @@
-from ipam.models import IPAddress, VRF
-from dcim.models import Device, Interface
-from virtualization.models import VirtualMachine
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from ruamel.yaml import YAML
-
-from netaddr import IPNetwork
-from pathlib import Path
 import sys
 
-file = Path('/opt/netbox/initializers/ip_addresses.yml')
-if not file.is_file():
+from dcim.models import Device, Interface
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Q
+from ipam.models import VRF, IPAddress
+from netaddr import IPNetwork
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import VirtualMachine, VMInterface
+
+ip_addresses = load_yaml('/opt/netbox/initializers/ip_addresses.yml')
+
+if ip_addresses is None:
   sys.exit()
 
-with file.open('r') as stream:
-  yaml = YAML(typ='safe')
-  ip_addresses = yaml.load(stream)
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'vrf': (VRF, 'name'),
+  'interface': (None, None)
+}
 
-  optional_assocs = {
-    'tenant': (Tenant, 'name'),
-    'vrf': (VRF, 'name'),
-    'interface': (Interface, 'name')
-  }
+vm_interface_ct = ContentType.objects.filter(Q(app_label='virtualization', model='vminterface')).first()
+interface_ct = ContentType.objects.filter(Q(app_label='dcim', model='interface')).first()
 
-  if ip_addresses is not None:
-    for params in ip_addresses:
-      vm = params.pop('virtual_machine', None)
-      device = params.pop('device', None)
-      custom_fields = params.pop('custom_fields', None)
-      params['address'] = IPNetwork(params['address'])
+for params in ip_addresses:
+  custom_field_data = pop_custom_fields(params)
 
-      if vm and device:
-        print("IP Address can only specify one of the following: virtual_machine or device.")
-        sys.exit()
+  vm = params.pop('virtual_machine', None)
+  device = params.pop('device', None)
+  params['address'] = IPNetwork(params['address'])
 
-      for assoc, details in optional_assocs.items():
-        if assoc in params:
-          model, field = details
-          if assoc == 'interface':
-              if vm:
-                  vm_id = VirtualMachine.objects.get(name=vm).id
-                  query = { field: params.pop(assoc), "virtual_machine_id": vm_id }
-              elif device:
-                  dev_id = Device.objects.get(name=device).id
-                  query = { field: params.pop(assoc), "device_id": dev_id }
-          else:
-              query = { field: params.pop(assoc) }
+  if vm and device:
+    print("IP Address can only specify one of the following: virtual_machine or device.")
+    sys.exit()
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      if assoc == 'interface':
+          if vm:
+              vm_id = VirtualMachine.objects.get(name=vm).id
+              query = { 'name': params.pop(assoc), "virtual_machine_id": vm_id }
+              params['assigned_object_type'] = vm_interface_ct
+              params['assigned_object_id'] = VMInterface.objects.get(**query).id
+          elif device:
+              dev_id = Device.objects.get(name=device).id
+              query = { 'name': params.pop(assoc), "device_id": dev_id }
+              params['assigned_object_type'] = interface_ct
+              params['assigned_object_id'] = Interface.objects.get(**query).id
+      else:
+          query = { field: params.pop(assoc) }
           params[assoc] = model.objects.get(**query)
 
-      ip_address, created = IPAddress.objects.get_or_create(**params)
+  ip_address, created = IPAddress.objects.get_or_create(**params)
 
-      if created:
-        if custom_fields is not None:
-          for cf_name, cf_value in custom_fields.items():
-            custom_field = CustomField.objects.get(name=cf_name)
-            custom_field_value = CustomFieldValue.objects.create(
-              field=custom_field,
-              obj=ip_address,
-              value=cf_value
-            )
+  if created:
+    set_custom_fields_values(ip_address, custom_field_data)
 
-            ip_address.custom_field_values.add(custom_field_value)
-
-        print("🧬 Created IP Address", ip_address.address)
+    print("🧬 Created IP Address", ip_address.address)

startup_scripts/270_primary_ips.py

@@ -0,0 +1,44 @@
+from dcim.models import Device
+from ipam.models import IPAddress
+from virtualization.models import VirtualMachine
+from startup_script_utils import load_yaml
+import sys
+
+def link_primary_ip(assets, asset_model):
+  for params in assets:
+    primary_ip_fields = set(params) & {'primary_ip4', 'primary_ip6'}
+    if not primary_ip_fields:
+      continue
+
+    for assoc, details in optional_assocs.items():
+      if assoc in params:
+        model, field = details
+        query = { field: params.pop(assoc) }
+
+        try:
+          params[assoc] = model.objects.get(**query)
+        except model.DoesNotExist:
+          primary_ip_fields -= {assoc}
+          print(f"⚠️ IP Address '{query[field]}' not found")
+
+    asset = asset_model.objects.get(name=params['name'])
+    for field in primary_ip_fields:
+      if getattr(asset, field) != params[field]:
+        setattr(asset, field, params[field])
+        print(f"🔗 Define primary IP '{params[field].address}' on '{asset.name}'")
+    asset.save()
+
+devices = load_yaml('/opt/netbox/initializers/devices.yml')
+virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml')
+
+optional_assocs = {
+    'primary_ip4': (IPAddress, 'address'),
+    'primary_ip6': (IPAddress, 'address')
+}
+
+if devices is None and virtual_machines is None:
+    sys.exit()
+if devices is not None:
+  link_primary_ip(devices, Device)
+if virtual_machines is not None:
+  link_primary_ip(virtual_machines, VirtualMachine)

startup_scripts/280_providers.py

@@ -0,0 +1,18 @@
+from circuits.models import Provider
+from startup_script_utils import *
+import sys
+
+providers = load_yaml('/opt/netbox/initializers/providers.yml')
+
+if providers is None:
+  sys.exit()
+
+for params in providers:
+  custom_field_data = pop_custom_fields(params)
+
+  provider, created = Provider.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(provider, custom_field_data)
+
+    print("📡 Created provider", provider.name)

startup_scripts/290_circuit_types.py

@@ -0,0 +1,18 @@
+from circuits.models import CircuitType
+from startup_script_utils import *
+import sys
+
+circuit_types = load_yaml('/opt/netbox/initializers/circuit_types.yml')
+
+if circuit_types is None:
+  sys.exit()
+
+for params in circuit_types:
+  custom_field_data = pop_custom_fields(params)
+
+  circuit_type, created = CircuitType.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit_type, custom_field_data)
+
+    print("⚡ Created Circuit Type", circuit_type.name)

startup_scripts/300_circuits.py

@@ -0,0 +1,41 @@
+from circuits.models import Circuit, Provider, CircuitType
+from tenancy.models import Tenant
+from startup_script_utils import *
+import sys
+
+circuits = load_yaml('/opt/netbox/initializers/circuits.yml')
+
+if circuits is None:
+  sys.exit()
+
+required_assocs = {
+  'provider': (Provider, 'name'),
+  'type': (CircuitType, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in circuits:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  circuit, created = Circuit.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit, custom_field_data)
+
+    print("⚡ Created Circuit", circuit.cid)

startup_scripts/310_secret_roles.py

@@ -0,0 +1,14 @@
+from secrets.models import SecretRole
+from startup_script_utils import load_yaml
+import sys
+
+secret_roles = load_yaml('/opt/netbox/initializers/secret_roles.yml')
+
+if secret_roles is None:
+  sys.exit()
+
+for params in secret_roles:
+  secret_role, created = SecretRole.objects.get_or_create(**params)
+
+  if created:
+    print("🔑 Created Secret Role", secret_role.name)

startup_scripts/320_services.py

@@ -0,0 +1,29 @@
+from ipam.models import Service
+from dcim.models import Device
+from virtualization.models import VirtualMachine
+from startup_script_utils import load_yaml
+import sys
+
+services = load_yaml('/opt/netbox/initializers/services.yml')
+
+if services is None:
+  sys.exit()
+
+optional_assocs = {
+  'device': (Device, 'name'),
+  'virtual_machine': (VirtualMachine, 'name')
+}
+
+for params in services:
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  service, created = Service.objects.get_or_create(**params)
+
+  if created:
+    print("🧰 Created Service", service.name)

startup_scripts/330_power_panels.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Site, RackGroup, PowerPanel
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_panels = load_yaml('/opt/netbox/initializers/power_panels.yml')
+
+if power_panels is None:
+  sys.exit()
+
+required_assocs = {
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'rack_group': (RackGroup, 'name')
+}
+
+for params in power_panels:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_panel, created = PowerPanel.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_panel, custom_field_data)
+
+    print("⚡ Created Power Panel", power_panel.site, power_panel.name)

startup_scripts/340_power_feeds.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Rack, PowerPanel, PowerFeed
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_feeds = load_yaml('/opt/netbox/initializers/power_feeds.yml')
+
+if power_feeds is None:
+  sys.exit()
+
+required_assocs = {
+  'power_panel': (PowerPanel, 'name')
+}
+
+optional_assocs = {
+  'rack': (Rack, 'name')
+}
+
+for params in power_feeds:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_feed, created = PowerFeed.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_feed, custom_field_data)
+
+    print("⚡ Created Power Feed", power_feed.name)

startup_scripts/__main__.py

@@ -9,10 +9,21 @@ this_dir = dirname(abspath(__file__))
 def filename(f):
   return f.name
 
-with scandir(dirname(abspath(__file__))) as it:
+with scandir(this_dir) as it:
   for f in sorted(it, key = filename):
-    if f.name.startswith('__') or not f.is_file():
+    if not f.is_file():
       continue
-      
-    print(f"Running {f.path}")
-    runpy.run_path(f.path)
+
+    if f.name.startswith('__'):
+      continue
+
+    if not f.name.endswith('.py'):
+      continue
+
+    print(f"▶️  Running the startup script {f.path}")
+    try:
+      runpy.run_path(f.path)
+    except SystemExit as e:
+      if e.code is not None and e.code != 0:
+        print(f"‼️ The startup script {f.path} returned with code {e.code}, exiting.")
+        raise

startup_scripts/startup_script_utils/__init__.py

@@ -0,0 +1,3 @@
+from .load_yaml import load_yaml
+from .permissions import set_permissions
+from .custom_fields import set_custom_fields_values, pop_custom_fields

startup_scripts/startup_script_utils/custom_fields.py

@@ -0,0 +1,15 @@
+def set_custom_fields_values(entity, custom_field_data):
+  if not custom_field_data:
+    return
+
+  entity.custom_field_data = custom_field_data
+  return entity.save()
+
+def pop_custom_fields(params):
+  if 'custom_field_data' in params:
+    return params.pop('custom_field_data')
+  elif 'custom_fields' in params:
+    print("⚠️ Please rename 'custom_fields' to 'custom_field_data'!")
+    return params.pop('custom_fields')
+
+  return None