Merge pull request #419 from netbox-community/develop

Version 1.0.1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -65,13 +65,3 @@ If your log is very long, create a Gist instead (and post the link to it): https
 ```text
 LOG LOG LOG
 ```
-
-The output of `docker-compose logs nginx`:
-<!--
-Only if you have gotten a 5xx http error, else delete this section.
-If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com
--->
-
-```text
-LOG LOG LOG
-```

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,13 @@
+blank_issues_enabled: false
+contact_links:
+  - name: The \#netbox-docker Slack channel
+    url: http://slack.networktocode.com/
+    about: It's usually the quickest way to seek help when you're in trouble with regards to Netbox Docker.
+    
+  - name: Github Discussions
+    url: https://github.com/netbox-community/netbox-docker/discussions
+    about: This is the right place to ask questions about how to use or do certain things with Netbox Docker.
+
+  - name: Have you had a look at our Wiki?
+    url: https://github.com/netbox-community/netbox-docker/wiki
+    about: Our wiki contains information for common problems and tips for operating Netbox Docker in production.

.github/workflows/push.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   build:
+    continue-on-error: ${{ matrix.docker_from == 'alpine:edge' }}
     strategy:
       matrix:
         build_cmd:
@@ -19,15 +20,14 @@ jobs:
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script
-        - python:3.8-alpine
-        - python:3.9-alpine
+        - alpine:edge
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new Netbox Docker Images
     steps:
     - id: git-checkout
       name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
     - id: docker-build
       name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}'
       run: ${{ matrix.build_cmd }}

.github/workflows/release.yml

@@ -50,3 +50,34 @@ jobs:
       name: Logout of the Docker Registry
       run: docker logout "${DOCKER_REGISTRY}"
       if: steps.docker-build.outputs.skipped != 'true'
+
+    # Quay.io
+    - id: quayio-docker-build
+      name: Build the image with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        DOCKER_REGISTRY: quay.io
+        GH_ACTION: enable
+    - id: quayio-registry-login
+      name: Login to the Quay.io Registry
+      run: |
+        echo "::add-mask::$QUAYIO_USERNAME"
+        echo "::add-mask::$QUAYIO_PASSWORD"
+        docker login -u "$QUAYIO_USERNAME" --password "${QUAYIO_PASSWORD}" "${DOCKER_REGISTRY}"
+      env:
+        DOCKER_REGISTRY: quay.io
+        QUAYIO_USERNAME: ${{ secrets.quayio_username }}
+        QUAYIO_PASSWORD: ${{ secrets.quayio_password }}
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: quayio-registry-push
+      name: Push the image
+      run: ${{ matrix.build_cmd }} --push-only
+      env:
+        DOCKER_REGISTRY: quay.io
+      if: steps.docker-build.outputs.skipped != 'true'
+    - id: quayio-registry-logout
+      name: Logout of the Docker Registry
+      run: docker logout "${DOCKER_REGISTRY}"
+      env:
+        DOCKER_REGISTRY: quay.io
+      if: steps.docker-build.outputs.skipped != 'true'

Dockerfile

@@ -12,27 +12,15 @@ RUN apk add --no-cache \
       libffi-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev
-
-WORKDIR /install
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-# gunicorn is used for launching netbox
-      gunicorn \
-      greenlet \
-      eventlet \
-# napalm is used for gathering information from network devices
-      napalm \
-# ruamel is used in startup_scripts
-      'ruamel.yaml>=0.15,<0.16' \
-# django_auth_ldap is required for ldap
-      django_auth_ldap \
-# django-storages was introduced in 2.7 and is optional
-      django-storages
+      postgresql-dev \
+      py3-pip \
+      python3-dev \
+      && python3 -m venv /opt/netbox/venv \
+      && /opt/netbox/venv/bin/python3 -m pip install --upgrade pip setuptools
 
 ARG NETBOX_PATH
-COPY ${NETBOX_PATH}/requirements.txt /
-RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt
+COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
+RUN /opt/netbox/venv/bin/pip install -r /requirements.txt -r /requirements-container.txt
 
 ###
 # Main stage
@@ -44,6 +32,7 @@ FROM ${FROM} as main
 RUN apk add --no-cache \
       bash \
       ca-certificates \
+      curl \
       graphviz \
       libevent \
       libffi \
@@ -51,35 +40,38 @@ RUN apk add --no-cache \
       libressl \
       libxslt \
       postgresql-libs \
-      ttf-ubuntu-font-family
+      python3 \
+      py3-pip \
+      ttf-ubuntu-font-family \
+      unit \
+      unit-python3
 
 WORKDIR /opt
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
-COPY docker/gunicorn_config.py /etc/netbox/
-COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
 COPY configuration/ /etc/netbox/config/
+COPY docker/nginx-unit.json /etc/unit/
 
 WORKDIR /opt/netbox/netbox
 
-# Must set permissions for '/opt/netbox/netbox/static' directory
-# to g+w so that `./manage.py collectstatic` can be executed during
-# container startup.
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
-RUN mkdir static && chmod -R g+w static media
+RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chmod -R g+w media /opt/unit/ \
+      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-CMD ["gunicorn", "-c /etc/netbox/gunicorn_config.py", "netbox.wsgi"]
+CMD [ "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \

README.md

@@ -10,7 +10,7 @@
 [![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license]
 
 [The Github repository](netbox-docker-github) houses the components needed to build Netbox as a Docker container.
-Images are built using this code and are released to [Docker Hub][netbox-dockerhub] once a day.
+Images are built using this code and are released to [Docker Hub][netbox-dockerhub] and [Quay.io][netbox-quayio] once a day.
 
 Do you have any questions?
 Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel.
@@ -18,11 +18,12 @@ Before opening an issue on Github, please join the [Network To Code][ntc-slack]
 [github-stargazers]: https://github.com/netbox-community/netbox-docker/stargazers
 [github-release]: https://github.com/netbox-community/netbox-docker/releases
 [netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox
-[netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/tags/
+[netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/
 [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/
 [ntc-slack]: http://slack.networktocode.com/
 [netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E
 [netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/release/LICENSE
+[netbox-quayio]: https://quay.io/repository/netboxcommunity/netbox
 
 ## Docker Tags
 
@@ -55,7 +56,7 @@ cd netbox-docker
 tee docker-compose.override.yml <<EOF
 version: '3.4'
 services:
-  nginx:
+  netbox:
     ports:
       - 8000:8080
 EOF
@@ -85,11 +86,13 @@ It covers advanced topics such as using files for secrets, deployment to Kuberne
 
 ## Getting Help
 
-Please join [our Slack channel `#netbox-docker`][netbox-docker-slack] on the [Network To Code Slack][ntc-slack].
-It's free to use and there are almost always people online that can help.
+Feel free to ask questions in our [Github Community][netbox-community] or join [our Slack channel `#netbox-docker`][netbox-docker-slack] on the [Network To Code Slack][ntc-slack],
+which is free to use and where there are almost always people online that can help you in the Slack channel.
 
 If you need help with using Netbox or developing for it or against it's API you may find the `#netbox` channel on the same Slack instance very helpful.
 
+[netbox-community]: https://github.com/netbox-community/netbox-docker/discussions
+
 ## Dependencies
 
 This project relies only on *Docker* and *docker-compose* meeting these requirements:
@@ -104,7 +107,7 @@ To check the version installed on your system run `docker --version` and `docker
 The `docker-compose.yml` file is prepared to run a specific version of Netbox, instead of `latest`.
 To use this feature, set and export the environment-variable `VERSION` before launching `docker-compose`, as shown below.
 `VERSION` may be set to the name of
-[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
+[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub] or [Quay.io][netbox-quayio].
 
 ```bash
 export VERSION=v2.7.1

VERSION

@@ -1 +1 @@
-0.26.1
+1.0.1

build.sh

@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'python:3.8-alpine'"
+  echo "              Default: 'alpine:3.13'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -106,7 +106,7 @@ else
 fi
 
 ###
-# Variables for fetching the source
+# Variables for fetching the Netbox source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
@@ -115,10 +115,10 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# Fetching the source
+# Fetching the Netbox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
-  echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  echo "🌐 Checking out '${NETBOX_BRANCH}' of Netbox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
   fi
@@ -135,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
     $DRY git checkout -qf FETCH_HEAD
     $DRY git prune
   )
-  echo "✅ Checked out netbox"
+  echo "✅ Checked out Netbox"
 fi
 
 ###
@@ -157,7 +157,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="python:3.8-alpine"
+  DOCKER_FROM="alpine:3.13"
 fi
 
 ###
@@ -271,7 +271,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
 
       if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
         SHOULD_BUILD="true"
-        BUILD_REASON="${BUILD_REASON} python"
+        BUILD_REASON="${BUILD_REASON} alpine"
       fi
       if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
         SHOULD_BUILD="true"

configuration/configuration.py

@@ -157,7 +157,7 @@ LOGIN_REQUIRED = environ.get('LOGIN_REQUIRED', 'False').lower() == 'true'
 
 # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to
 # re-authenticate. (Default: 1209600 [14 days])
-LOGIN_TIMEOUT = environ.get('LOGIN_TIMEOUT', None)
+LOGIN_TIMEOUT = int(environ.get('LOGIN_TIMEOUT', 1209600))
 
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
 MAINTENANCE_MODE = environ.get('MAINTENANCE_MODE', 'False').lower() == 'true'
@@ -233,7 +233,7 @@ SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
 # By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use
 # local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only
 # database access.) Note that the user as which NetBox runs must have read and write permissions to this path.
-SESSION_FILE_PATH = environ.get('REPORTS_ROOT', None)
+SESSION_FILE_PATH = environ.get('SESSIONS_ROOT', None)
 
 # Time zone (default: UTC)
 TIME_ZONE = environ.get('TIME_ZONE', 'UTC')

configuration/ldap/ldap_config.py

@@ -60,14 +60,17 @@ AUTH_LDAP_GROUP_SEARCH = LDAPSearch(AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SU
 AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType'))
 
 # Define a group required to login.
-AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', '')
+AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN')
 
 # Define special user types using groups. Exercise great caution when assigning superuser status.
-AUTH_LDAP_USER_FLAGS_BY_GROUP = {
-    "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
-    "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
-    "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
-}
+AUTH_LDAP_USER_FLAGS_BY_GROUP = {}
+
+if AUTH_LDAP_REQUIRE_GROUP is not None:
+    AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+        "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''),
+        "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''),
+        "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '')
+    }
 
 # For more granular permissions, we can map LDAP groups to Django groups.
 AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true'

docker-compose.test.yml

@@ -7,6 +7,8 @@ services:
     - redis
     - redis-cache
     env_file: env/netbox.env
+    environment:
+      SKIP_STARTUP_SCRIPTS: ${SKIP_STARTUP_SCRIPTS-false}
     user: '101'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
@@ -14,19 +16,9 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
     ports:
     - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
     image: postgres:12-alpine
     env_file: env/postgres.env
@@ -45,9 +37,5 @@ services:
     - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
     env_file: env/redis-cache.env
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local

docker-compose.yml

@@ -15,30 +15,19 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
+    ports:
+    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
     entrypoint:
-    - python3
+    - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
     command:
     - rqworker
-
-  # nginx
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
-    ports:
-    - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
+    ports: []
 
   # postgres
   postgres:
@@ -66,10 +55,6 @@ services:
     env_file: env/redis-cache.env
 
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
   netbox-postgres-data:

docker/configuration.docker.py

@@ -9,6 +9,7 @@ from os import scandir
 import importlib.util
 import sys
 
+
 def _filename(f):
   return f.name
 

docker/docker-entrypoint.sh

@@ -7,6 +7,9 @@ set -e
 # Allows Netbox to be run as non-root users
 umask 002
 
+# Load correct Python3 env
+source /opt/netbox/venv/bin/activate
+
 # Try to connect to the DB
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
@@ -60,9 +63,6 @@ else
   echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-# Copy static files
-./manage.py collectstatic --no-input
-
 echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker

docker/gunicorn_config.py

@@ -1,8 +0,0 @@
-command = '/usr/bin/gunicorn'
-pythonpath = '/opt/netbox/netbox'
-bind = '0.0.0.0:8001'
-workers = 3
-errorlog = '-'
-accesslog = '-'
-capture_output = False
-loglevel = 'info'

docker/launch-netbox.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}"
+UNIT_SOCKET="/opt/unit/unit.sock"
+
+load_configuration() {
+  MAX_WAIT=10
+  WAIT_COUNT=0
+  while [ ! -S $UNIT_SOCKET ]; do
+    if [ $WAIT_COUNT -ge $MAX_WAIT ]; then
+      echo "⚠️ No control socket found; configuration will not be loaded."
+      return 1
+    fi
+
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    echo "⏳ Waiting for control socket to be created... (${WAIT_COUNT}/${MAX_WAIT})"
+
+    sleep 1
+  done
+
+  # even when the control socket exists, it does not mean unit has finished initialisation
+  # this curl call will get a reply once unit is fully launched
+  curl --silent --output /dev/null --request GET --unix-socket $UNIT_SOCKET http://localhost/
+
+  echo "⚙️ Applying configuration from $UNIT_CONFIG";
+
+  RESP_CODE=$(curl \
+                  --silent \
+                  --output /dev/null \
+                  --write-out '%{http_code}' \
+                  --request PUT \
+                  --data-binary "@${UNIT_CONFIG}" \
+                  --unix-socket $UNIT_SOCKET \
+                  http://localhost/config
+            )
+  if [ "$RESP_CODE" != "200" ]; then
+    echo "⚠️ Could no load Unit configuration"
+    kill "$(cat /opt/unit/unit.pid)"
+    return 1
+  fi
+
+  echo "✅ Unit configuration loaded successfully"
+}
+
+load_configuration &
+
+exec unitd \
+    --no-daemon \
+    --control unix:$UNIT_SOCKET \
+    --pid /opt/unit/unit.pid \
+    --log /dev/stdout \
+    --state /opt/unit/state/ \
+    --tmp /opt/unit/tmp/

docker/ldap_config.docker.py

@@ -13,3 +13,9 @@ def __getattr__(name):
     except:
       pass
   raise AttributeError
+
+def __dir__():
+  names = []
+  for config in _loaded_configurations:
+    names.extend(config.__dir__())
+  return names

docker/nginx-unit.json

@@ -0,0 +1,40 @@
+{
+    "listeners": {
+        "*:8080": {
+            "pass": "routes"
+        }
+    },
+
+    "routes": [
+        {
+            "match": {
+                "uri": "/static/*"
+            },
+            "action": {
+                "share": "/opt/netbox/netbox"
+            }
+        },
+
+        {
+            "action": {
+                "pass": "applications/netbox"
+            }
+        }
+    ],
+
+    "applications": {
+        "netbox": {
+            "type": "python 3",
+            "path": "/opt/netbox/netbox/",
+            "module": "netbox.wsgi",
+            "home": "/opt/netbox/venv",
+            "processes": {
+                "max": 4,
+                "spare": 1,
+                "idle_timeout": 120
+            }
+        }
+    },
+
+    "access_log": "/dev/stdout"
+}

docker/nginx.conf

@@ -1,44 +0,0 @@
-daemon off;
-worker_processes 1;
-
-error_log /dev/stderr info;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include              /etc/nginx/mime.types;
-    default_type         application/octet-stream;
-    sendfile             on;
-    tcp_nopush           on;
-    keepalive_timeout    65;
-    gzip                 on;
-    server_tokens        off;
-    client_max_body_size 10M;
-
-    server {
-        listen      8080;
-        access_log  off;
-
-        location /static/ {
-            alias /opt/netbox/netbox/static/;
-        }
-
-        location / {
-            proxy_pass http://netbox:8001;
-            proxy_set_header X-Forwarded-Host $http_host;
-            proxy_set_header X-Real-IP $remote_addr;
-            add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
-        }
-    }
-
-    server {
-        listen      8081;
-        access_log  off;
-
-        location = /stub_status {
-            stub_status;
-        }
-    }
-}

initializers/aggregates.yml

@@ -1,5 +1,6 @@
 # - prefix: 10.0.0.0/16
 #   rir: RFC1918
+#   tenant: tenant1
 # - prefix: fd00:ccdd::/32
 #   rir: RFC4193 ULA
 # - prefix: 2001:db8::/32

initializers/circuit_types.yml

@@ -0,0 +1,6 @@
+# - name: VPLS
+#   slug: vpls
+# - name: MPLS
+#   slug: mpls
+# - name: Internet
+#   slug: internet

initializers/circuits.yml

@@ -0,0 +1,7 @@
+# - cid: Circuit_ID-1
+#   provider: Provider1
+#   type: Internet
+#   tenant: tenant1
+# - cid: Circuit_ID-2
+#   provider: Provider2
+#   type: MPLS

initializers/cluster_groups.yml

@@ -0,0 +1,4 @@
+# - name: Group 1
+#   slug: group-1
+# - name: Group 2
+#   slug: group-2

initializers/clusters.yml

@@ -1,5 +1,7 @@
 # - name: cluster1
 #   type: Hyper-V
+#   group: Group 1
+#   tenant: tenant1
 # - name: cluster2
 #   type: Hyper-V
 #   site: SING 1

initializers/custom_fields.yml

@@ -43,20 +43,16 @@
 #   required: false
 #   filter_logic: exact
 #   weight: 30
+#   default: First Item
 #   on_objects:
 #   - dcim.models.Device
 #   choices:
-#   - value: First Item
-#     weight: 10
-#   - value: Second Item
-#     weight: 20
-#   - value: Third Item
-#     weight: 30
-#   - value: Fifth Item
-#     weight: 50
-#   - value: Fourth Item
-#     weight: 40
-# select_field_auto_weight:
+#   - First Item
+#   - Second Item
+#   - Third Item
+#   - Fifth Item
+#   - Fourth Item
+# select_field_legacy_format:
 #    type: select
 #    label: Choose between items
 #    required: false
@@ -65,9 +61,9 @@
 #    on_objects:
 #     - dcim.models.Device
 #    choices:
-#     - value: A
-#     - value: B
-#     - value: C
+#     - value: A # this is the deprecated format.
+#     - value: B # we only use it for the tests.
+#     - value: C # please see above for the new format.
 #     - value: "D like deprecated"
 #       weight: 999
 #     - value: E
@@ -76,7 +72,7 @@
 #   label: Yes Or No?
 #   required: true
 #   filter_logic: loose
-#   default: "false" # important: but "false" in quotes!
+#   default: "false" # important: put "false" in quotes!
 #   weight: 90
 #   on_objects:
 #   - dcim.models.Device

initializers/device_types.yml

@@ -2,22 +2,22 @@
 #   manufacturer: Manufacturer 1
 #   slug: model-1
 #   u_height: 2
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Model 2
 #   manufacturer: Manufacturer 1
 #   slug: model-2
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Model 3
 #   manufacturer: Manufacturer 1
 #   slug: model-3
 #   is_full_depth: false
 #   u_height: 0
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - model: Other
 #   manufacturer: No Name
 #   slug: other
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/devices.yml

@@ -20,7 +20,7 @@
 #   rack: rack-01
 #   face: front
 #   position: 1
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - name: server02
 #   device_role: server
@@ -31,7 +31,7 @@
 #   position: 2
 #   primary_ip4: 10.1.1.2/24
 #   primary_ip6: 2001:db8:a000:1::2/64
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - name: server03
 #   device_role: server
@@ -40,5 +40,5 @@
 #   rack: rack-03
 #   face: front
 #   position: 3
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/power_feeds.yml

@@ -0,0 +1,14 @@
+# - name: power feed 1
+#   power_panel: power panel AMS 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Single phase
+#   rack: rack-01
+# - name: power feed 2
+#   power_panel: power panel SING 1
+#   voltage: 208
+#   amperage: 50
+#   max_utilization: 80
+#   phase: Three-phase
+#   rack: rack-03

initializers/power_panels.yml

@@ -0,0 +1,5 @@
+# - name: power panel AMS 1
+#   site: AMS 1   
+# - name: power panel SING 1
+#   site: SING 1 
+#   rack_group: cage 101

initializers/providers.yml

@@ -0,0 +1,6 @@
+# - name: Provider1
+#   slug: provider1
+#   asn: 121
+# - name: Provider2
+#   slug: provider2
+#   asn: 122

initializers/racks.yml

@@ -20,7 +20,7 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - site: AMS 2
 #   name: rack-02
@@ -28,7 +28,7 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description
 # - site: SING 1
 #   name: rack-03
@@ -37,5 +37,5 @@
 #   type: 4-post-cabinet
 #   width: 19
 #   u_height: 47
-#   custom_fields:
+#   custom_field_data:
 #     text_field: Description

initializers/route_targets.yml

@@ -0,0 +1,3 @@
+# - name: 65000:1001
+#   tenant: tenant1
+# - name: 65000:1002

initializers/secret_roles.yml

@@ -0,0 +1,4 @@
+# - name: Super Secret Passwords
+#   slug: super-secret
+# - name: SNMP Communities
+#   slug: snmp

initializers/services.yml

@@ -0,0 +1,15 @@
+# - name: DNS
+#   protocol: TCP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: DNS
+#   protocol: UDP
+#   ports:
+#     - 53
+#   virtual_machine: virtual machine 1
+# - name: MISC
+#   protocol: UDP
+#   ports: 
+#     - 4000
+#   device: server01

initializers/sites.yml

@@ -4,29 +4,29 @@
 #   status: active
 #   facility: Amsterdam 1
 #   asn: 12345
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS1
 # - name: AMS 2
 #   slug: ams2
 #   region: Downtown
 #   status: active
 #   facility: Amsterdam 2
 #   asn: 54321
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS2
 # - name: AMS 3
 #   slug: ams3
 #   region: Suburbs
 #   status: active
 #   facility: Amsterdam 3
 #   asn: 67890
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for AMS3
 # - name: SING 1
 #   slug: sing1
 #   region: Singapore
 #   status: active
 #   facility: Singapore 1
 #   asn: 09876
-#   custom_fields:
-#     text_field: Description
+#   custom_field_data:
+#     text_field: Description for SING1

initializers/tags.yml

@@ -0,0 +1,12 @@
+# - name: Tag 1
+#   slug: tag-1
+#   color: Pink
+# - name: Tag 2
+#   slug: tag-2
+#   color: Cyan
+# - name: Tag 3
+#   slug: tag-3
+#   color: Grey
+# - name: Tag 4
+#   slug: tag-4
+#   color: Teal

renovate.json

@@ -0,0 +1,13 @@
+{
+  "extends": [
+    "config:base"
+  ],
+  "enabled": true,
+  "labels": ["maintenance"],
+  "baseBranches": ["develop"],
+  "pip_requirements": {
+    "fileMatch": [
+      "requirements-container.txt"
+    ]
+  }
+}

requirements-container.txt

@@ -0,0 +1,4 @@
+napalm==3.2.0
+ruamel.yaml==0.16.12
+django-auth-ldap==2.2.0
+django-storages==1.11.1

startup_scripts/000_users.py

@@ -1,6 +1,6 @@
 import sys
 
-from django.contrib.auth.models import Group, User
+from django.contrib.auth.models import User
 from startup_script_utils import load_yaml, set_permissions
 from users.models import Token
 

startup_scripts/020_custom_fields.py

@@ -1,8 +1,8 @@
-from extras.models import CustomField, CustomFieldChoice
-
-from startup_script_utils import load_yaml
 import sys
 
+from extras.models import CustomField
+from startup_script_utils import load_yaml
+
 def get_class_for_class_path(class_path):
   import importlib
   from django.contrib.contenttypes.models import ContentType
@@ -21,34 +21,38 @@ for cf_name, cf_details in customfields.items():
   custom_field, created = CustomField.objects.get_or_create(name = cf_name)
 
   if created:
-    if cf_details.get('default', 0):
+    if cf_details.get('default', False):
       custom_field.default = cf_details['default']
 
-    if cf_details.get('description', 0):
+    if cf_details.get('description', False):
       custom_field.description = cf_details['description']
 
-    if cf_details.get('label', 0):
+    if cf_details.get('label', False):
       custom_field.label = cf_details['label']
 
     for object_type in cf_details.get('on_objects', []):
-      custom_field.obj_type.add(get_class_for_class_path(object_type))
+      custom_field.content_types.add(get_class_for_class_path(object_type))
 
-    if cf_details.get('required', 0):
+    if cf_details.get('required', False):
       custom_field.required = cf_details['required']
 
-    if cf_details.get('type', 0):
+    if cf_details.get('type', False):
       custom_field.type = cf_details['type']
 
-    if cf_details.get('weight', 0):
+    if cf_details.get('weight', -1) >= 0:
       custom_field.weight = cf_details['weight']
 
+    if cf_details.get('choices', False):
+      custom_field.choices = []
+
+      for choice_detail in cf_details.get('choices', []):
+        if isinstance(choice_detail, dict) and 'value' in choice_detail:
+          # legacy mode
+          print(f"⚠️ Please migrate the choice '{choice_detail['value']}' of '{cf_name}' to the new format, as 'weight' is no longer supported!")
+          custom_field.choices.append(choice_detail['value'])
+        else:
+          custom_field.choices.append(choice_detail)
+
     custom_field.save()
 
-    for idx, choice_details in enumerate(cf_details.get('choices', [])):
-      choice, _ = CustomFieldChoice.objects.get_or_create(
-        field=custom_field,
-        value=choice_details['value'],
-        defaults={'weight': idx * 10}
-      )
-
     print("🔧 Created custom field", cf_name)

startup_scripts/020_tags.py

@@ -0,0 +1,23 @@
+from extras.models import Tag
+from utilities.choices import ColorChoices
+
+from startup_script_utils import load_yaml
+import sys
+
+tags = load_yaml('/opt/netbox/initializers/tags.yml')
+
+if tags is None:
+  sys.exit()
+
+for params in tags:
+  if 'color' in params:
+    color = params.pop('color')
+
+    for color_tpl in ColorChoices:
+      if color in color_tpl:
+        params['color'] = color_tpl[0]
+
+  tag, created = Tag.objects.get_or_create(**params)
+
+  if created:
+    print("🎨 Created Tag", tag.name)

startup_scripts/040_sites.py

@@ -1,9 +1,9 @@
-from dcim.models import Region, Site
-from extras.models import CustomField, CustomFieldValue
-from tenancy.models import Tenant
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import Region, Site
+from startup_script_utils import *
+from tenancy.models import Tenant
+
 sites = load_yaml('/opt/netbox/initializers/sites.yml')
 
 if sites is None:
@@ -15,7 +15,7 @@ optional_assocs = {
 }
 
 for params in sites:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in optional_assocs.items():
     if assoc in params:
@@ -27,15 +27,6 @@ for params in sites:
   site, created = Site.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=site,
-          value=cf_value
-        )
-
-        site.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(site, custom_field_data)
 
     print("📍 Created site", site.name)

startup_scripts/060_device_types.py

@@ -1,9 +1,9 @@
-from dcim.models import DeviceType, Manufacturer, Region
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import DeviceType, Manufacturer, Region
+from startup_script_utils import *
+from tenancy.models import Tenant
+
 device_types = load_yaml('/opt/netbox/initializers/device_types.yml')
 
 if device_types is None:
@@ -19,7 +19,7 @@ optional_assocs = {
 }
 
 for params in device_types:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in required_assocs.items():
     model, field = details
@@ -37,15 +37,6 @@ for params in device_types:
   device_type, created = DeviceType.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=device_type,
-          value=cf_value
-        )
-
-        device_type.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(device_type, custom_field_data)
 
     print("🔡 Created device type", device_type.manufacturer, device_type.model)

startup_scripts/080_racks.py

@@ -1,9 +1,9 @@
-from dcim.models import Site, RackRole, Rack, RackGroup
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import Site, RackRole, Rack, RackGroup
+from startup_script_utils import *
+from tenancy.models import Tenant
+
 racks = load_yaml('/opt/netbox/initializers/racks.yml')
 
 if racks is None:
@@ -20,7 +20,7 @@ optional_assocs = {
 }
 
 for params in racks:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in required_assocs.items():
     model, field = details
@@ -38,15 +38,6 @@ for params in racks:
   rack, created = Rack.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=rack,
-          value=cf_value
-        )
-
-        rack.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(rack, custom_field_data)
 
     print("🔳 Created rack", rack.site, rack.name)

startup_scripts/120_tenants.py

@@ -1,8 +1,8 @@
-from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from startup_script_utils import *
+from tenancy.models import Tenant, TenantGroup
+
 tenants = load_yaml('/opt/netbox/initializers/tenants.yml')
 
 if tenants is None:
@@ -13,7 +13,7 @@ optional_assocs = {
 }
 
 for params in tenants:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in optional_assocs.items():
     if assoc in params:
@@ -25,15 +25,6 @@ for params in tenants:
   tenant, created = Tenant.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=tenant,
-          value=cf_value
-        )
-
-        tenant.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(tenant, custom_field_data)
 
     print("👩‍💻 Created Tenant", tenant.name)

startup_scripts/135_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/135_clusters.py

@@ -1,9 +1,10 @@
-from dcim.models import Site
-from virtualization.models import Cluster, ClusterType, ClusterGroup
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import Site
+from startup_script_utils import *
+from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
+
 clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
 
 if clusters is None:
@@ -15,11 +16,12 @@ required_assocs = {
 
 optional_assocs = {
   'site': (Site, 'name'),
-  'group': (ClusterGroup, 'name')
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
 }
 
 for params in clusters:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in required_assocs.items():
     model, field = details
@@ -37,15 +39,6 @@ for params in clusters:
   cluster, created = Cluster.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=cluster,
-          value=cf_value
-        )
-
-        cluster.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(cluster, custom_field_data)
 
     print("🗄️ Created cluster", cluster.name)

startup_scripts/140_clusters.py

@@ -0,0 +1,44 @@
+import sys
+
+from dcim.models import Site
+from startup_script_utils import *
+from virtualization.models import Cluster, ClusterType, ClusterGroup
+from tenancy.models import Tenant
+
+clusters = load_yaml('/opt/netbox/initializers/clusters.yml')
+
+if clusters is None:
+  sys.exit()
+
+required_assocs = {
+  'type': (ClusterType, 'name')
+}
+
+optional_assocs = {
+  'site': (Site, 'name'),
+  'group': (ClusterGroup, 'name'),
+  'tenant': (Tenant, 'name')
+}
+
+for params in clusters:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  cluster, created = Cluster.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(cluster, custom_field_data)
+
+    print("🗄️ Created cluster", cluster.name)

startup_scripts/140_devices.py

@@ -1,10 +1,10 @@
-from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
-from virtualization.models import Cluster
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster
+
 devices = load_yaml('/opt/netbox/initializers/devices.yml')
 
 if devices is None:
@@ -24,7 +24,8 @@ optional_assocs = {
 }
 
 for params in devices:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
+
   # primary ips are handled later in `270_primary_ips.py`
   params.pop('primary_ip4', None)
   params.pop('primary_ip6', None)
@@ -45,15 +46,6 @@ for params in devices:
   device, created = Device.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=device,
-          value=cf_value
-        )
-
-        device.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(device, custom_field_data)
 
     print("🖥️  Created device", device.name)

startup_scripts/145_devices.py

@@ -0,0 +1,51 @@
+import sys
+
+from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster
+
+devices = load_yaml('/opt/netbox/initializers/devices.yml')
+
+if devices is None:
+  sys.exit()
+
+required_assocs = {
+  'device_role': (DeviceRole, 'name'),
+  'device_type': (DeviceType, 'model'),
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+  'platform': (Platform, 'name'),
+  'rack': (Rack, 'name'),
+  'cluster': (Cluster, 'name')
+}
+
+for params in devices:
+  custom_field_data = pop_custom_fields(params)
+
+  # primary ips are handled later in `270_primary_ips.py`
+  params.pop('primary_ip4', None)
+  params.pop('primary_ip6', None)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  device, created = Device.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(device, custom_field_data)
+
+    print("🖥️  Created device", device.name)

startup_scripts/160_aggregates.py

@@ -1,11 +1,10 @@
-from ipam.models import Aggregate, RIR
-
-from extras.models import CustomField, CustomFieldValue
-
-from netaddr import IPNetwork
-from startup_script_utils import load_yaml
 import sys
 
+from ipam.models import Aggregate, RIR
+from netaddr import IPNetwork
+from startup_script_utils import *
+from tenancy.models import Tenant
+
 aggregates = load_yaml('/opt/netbox/initializers/aggregates.yml')
 
 if aggregates is None:
@@ -15,8 +14,13 @@ required_assocs = {
   'rir': (RIR, 'name')
 }
 
+optional_assocs = {
+  'tenant': (Tenant, 'name'),
+}
+
 for params in aggregates:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
+
   params['prefix'] = IPNetwork(params['prefix'])
 
   for assoc, details in required_assocs.items():
@@ -25,18 +29,16 @@ for params in aggregates:
 
     params[assoc] = model.objects.get(**query)
 
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
   aggregate, created = Aggregate.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=aggregate,
-          value=cf_value
-        )
-
-        aggregate.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(aggregate, custom_field_data)
 
     print("🗞️ Created Aggregate", aggregate.prefix)

startup_scripts/165_cluster_groups.py

@@ -0,0 +1,14 @@
+from virtualization.models import ClusterGroup
+from startup_script_utils import load_yaml
+import sys
+
+cluster_groups = load_yaml('/opt/netbox/initializers/cluster_groups.yml')
+
+if cluster_groups is None:
+  sys.exit()
+
+for params in cluster_groups:
+  cluster_group, created = ClusterGroup.objects.get_or_create(**params)
+
+  if created:
+    print("🗄️ Created Cluster Group", cluster_group.name)

startup_scripts/175_route_targets.py

@@ -0,0 +1,31 @@
+import sys
+
+from ipam.models import RouteTarget
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+route_targets = load_yaml('/opt/netbox/initializers/route_targets.yml')
+
+if route_targets is None:
+  sys.exit()
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in route_targets:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  route_target, created = RouteTarget.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(route_target, custom_field_data)
+
+    print("🎯 Created Route Target", route_target.name)

startup_scripts/180_vrfs.py

@@ -1,11 +1,9 @@
-from ipam.models import VRF
-from tenancy.models import Tenant
-
-from extras.models import CustomField, CustomFieldValue
-
-from startup_script_utils import load_yaml
 import sys
 
+from ipam.models import VRF
+from startup_script_utils import *
+from tenancy.models import Tenant
+
 vrfs = load_yaml('/opt/netbox/initializers/vrfs.yml')
 
 if vrfs is None:
@@ -16,7 +14,7 @@ optional_assocs = {
 }
 
 for params in vrfs:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in optional_assocs.items():
     if assoc in params:
@@ -28,15 +26,6 @@ for params in vrfs:
   vrf, created = VRF.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=vrf,
-          value=cf_value
-        )
-
-        vrf.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(vrf, custom_field_data)
 
     print("📦 Created VRF", vrf.name)

startup_scripts/200_vlan_groups.py

@@ -1,8 +1,8 @@
+import sys
+
 from dcim.models import Site
 from ipam.models import VLANGroup
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
-import sys
+from startup_script_utils import *
 
 vlan_groups = load_yaml('/opt/netbox/initializers/vlan_groups.yml')
 
@@ -14,7 +14,7 @@ optional_assocs = {
 }
 
 for params in vlan_groups:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in optional_assocs.items():
     if assoc in params:
@@ -26,15 +26,6 @@ for params in vlan_groups:
   vlan_group, created = VLANGroup.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=vlan_group,
-          value=cf_value
-        )
-
-        vlan_group.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(vlan_group, custom_field_data)
 
     print("🏘️ Created VLAN Group", vlan_group.name)

startup_scripts/210_vlans.py

@@ -1,9 +1,9 @@
+import sys
+
 from dcim.models import Site
 from ipam.models import VLAN, VLANGroup, Role
+from startup_script_utils import *
 from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
-import sys
 
 vlans = load_yaml('/opt/netbox/initializers/vlans.yml')
 
@@ -19,7 +19,7 @@ optional_assocs = {
 }
 
 for params in vlans:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in optional_assocs.items():
     if assoc in params:
@@ -31,15 +31,6 @@ for params in vlans:
   vlan, created = VLAN.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=vlan,
-          value=cf_value
-        )
-
-        vlan.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(vlan, custom_field_data)
 
     print("🏠 Created VLAN", vlan.name)

startup_scripts/220_prefixes.py

@@ -1,10 +1,10 @@
+import sys
+
 from dcim.models import Site
 from ipam.models import Prefix, VLAN, Role, VRF
-from tenancy.models import Tenant, TenantGroup
-from extras.models import CustomField, CustomFieldValue
 from netaddr import IPNetwork
-from startup_script_utils import load_yaml
-import sys
+from startup_script_utils import *
+from tenancy.models import Tenant, TenantGroup
 
 prefixes = load_yaml('/opt/netbox/initializers/prefixes.yml')
 
@@ -21,7 +21,8 @@ optional_assocs = {
 }
 
 for params in prefixes:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
+
   params['prefix'] = IPNetwork(params['prefix'])
 
   for assoc, details in optional_assocs.items():
@@ -33,14 +34,6 @@ for params in prefixes:
   prefix, created = Prefix.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=prefix,
-          value=cf_value
-        )
-        prefix.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(prefix, custom_field_data)
 
     print("📌 Created Prefix", prefix.prefix)

startup_scripts/230_virtual_machines.py

@@ -1,10 +1,10 @@
-from dcim.models import Site, Platform, DeviceRole
-from virtualization.models import Cluster, VirtualMachine
-from tenancy.models import Tenant
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from dcim.models import Platform, DeviceRole
+from startup_script_utils import *
+from tenancy.models import Tenant
+from virtualization.models import Cluster, VirtualMachine
+
 virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml')
 
 if virtual_machines is None:
@@ -21,7 +21,8 @@ optional_assocs = {
 }
 
 for params in virtual_machines:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
+
   # primary ips are handled later in `270_primary_ips.py`
   params.pop('primary_ip4', None)
   params.pop('primary_ip6', None)
@@ -42,15 +43,6 @@ for params in virtual_machines:
   virtual_machine, created = VirtualMachine.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=virtual_machine,
-          value=cf_value
-        )
-
-        virtual_machine.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(virtual_machine, custom_field_data)
 
     print("🖥️ Created virtual machine", virtual_machine.name)

startup_scripts/240_virtualization_interfaces.py

@@ -1,8 +1,8 @@
-from virtualization.models import VirtualMachine, VMInterface
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
+from startup_script_utils import *
+from virtualization.models import VirtualMachine, VMInterface
+
 interfaces = load_yaml('/opt/netbox/initializers/virtualization_interfaces.yml')
 
 if interfaces is None:
@@ -13,7 +13,7 @@ required_assocs = {
 }
 
 for params in interfaces:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in required_assocs.items():
     model, field = details
@@ -24,15 +24,6 @@ for params in interfaces:
   interface, created = VMInterface.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=interface,
-          value=cf_value
-        )
-
-        interface.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(interface, custom_field_data)
 
     print("🧷 Created interface", interface.name, interface.virtual_machine.name)

startup_scripts/250_dcim_interfaces.py

@@ -1,9 +1,9 @@
-from dcim.models import Interface, Device
-from extras.models import CustomField, CustomFieldValue
-from startup_script_utils import load_yaml
 import sys
 
-interfaces= load_yaml('/opt/netbox/initializers/dcim_interfaces.yml')
+from dcim.models import Interface, Device
+from startup_script_utils import *
+
+interfaces = load_yaml('/opt/netbox/initializers/dcim_interfaces.yml')
 
 if interfaces is None:
   sys.exit()
@@ -13,7 +13,7 @@ required_assocs = {
 }
 
 for params in interfaces:
-  custom_fields = params.pop('custom_fields', None)
+  custom_field_data = pop_custom_fields(params)
 
   for assoc, details in required_assocs.items():
     model, field = details
@@ -24,15 +24,6 @@ for params in interfaces:
   interface, created = Interface.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=interface,
-          value=cf_value
-        )
-
-        interface.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(interface, custom_field_data)
 
     print("🧷 Created interface", interface.name, interface.device.name)

startup_scripts/260_ip_addresses.py

@@ -3,10 +3,9 @@ import sys
 from dcim.models import Device, Interface
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
-from extras.models import CustomField, CustomFieldValue
 from ipam.models import VRF, IPAddress
 from netaddr import IPNetwork
-from startup_script_utils import load_yaml
+from startup_script_utils import *
 from tenancy.models import Tenant
 from virtualization.models import VirtualMachine, VMInterface
 
@@ -25,9 +24,10 @@ vm_interface_ct = ContentType.objects.filter(Q(app_label='virtualization', model
 interface_ct = ContentType.objects.filter(Q(app_label='dcim', model='interface')).first()
 
 for params in ip_addresses:
+  custom_field_data = pop_custom_fields(params)
+
   vm = params.pop('virtual_machine', None)
   device = params.pop('device', None)
-  custom_fields = params.pop('custom_fields', None)
   params['address'] = IPNetwork(params['address'])
 
   if vm and device:
@@ -55,15 +55,6 @@ for params in ip_addresses:
   ip_address, created = IPAddress.objects.get_or_create(**params)
 
   if created:
-    if custom_fields is not None:
-      for cf_name, cf_value in custom_fields.items():
-        custom_field = CustomField.objects.get(name=cf_name)
-        custom_field_value = CustomFieldValue.objects.create(
-          field=custom_field,
-          obj=ip_address,
-          value=cf_value
-        )
-
-        ip_address.custom_field_values.add(custom_field_value)
+    set_custom_fields_values(ip_address, custom_field_data)
 
     print("🧬 Created IP Address", ip_address.address)

startup_scripts/270_primary_ips.py

@@ -31,13 +31,14 @@ def link_primary_ip(assets, asset_model):
 devices = load_yaml('/opt/netbox/initializers/devices.yml')
 virtual_machines = load_yaml('/opt/netbox/initializers/virtual_machines.yml')
 
-if devices is None and virtual_machines is None:
-    sys.exit()
-
 optional_assocs = {
     'primary_ip4': (IPAddress, 'address'),
     'primary_ip6': (IPAddress, 'address')
 }
 
-link_primary_ip(devices, Device)
-link_primary_ip(virtual_machines, VirtualMachine)
+if devices is None and virtual_machines is None:
+    sys.exit()
+if devices is not None:
+  link_primary_ip(devices, Device)
+if virtual_machines is not None:
+  link_primary_ip(virtual_machines, VirtualMachine)

startup_scripts/280_providers.py

@@ -0,0 +1,18 @@
+from circuits.models import Provider
+from startup_script_utils import *
+import sys
+
+providers = load_yaml('/opt/netbox/initializers/providers.yml')
+
+if providers is None:
+  sys.exit()
+
+for params in providers:
+  custom_field_data = pop_custom_fields(params)
+
+  provider, created = Provider.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(provider, custom_field_data)
+
+    print("📡 Created provider", provider.name)

startup_scripts/290_circuit_types.py

@@ -0,0 +1,18 @@
+from circuits.models import CircuitType
+from startup_script_utils import *
+import sys
+
+circuit_types = load_yaml('/opt/netbox/initializers/circuit_types.yml')
+
+if circuit_types is None:
+  sys.exit()
+
+for params in circuit_types:
+  custom_field_data = pop_custom_fields(params)
+
+  circuit_type, created = CircuitType.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit_type, custom_field_data)
+
+    print("⚡ Created Circuit Type", circuit_type.name)

startup_scripts/300_circuits.py

@@ -0,0 +1,41 @@
+from circuits.models import Circuit, Provider, CircuitType
+from tenancy.models import Tenant
+from startup_script_utils import *
+import sys
+
+circuits = load_yaml('/opt/netbox/initializers/circuits.yml')
+
+if circuits is None:
+  sys.exit()
+
+required_assocs = {
+  'provider': (Provider, 'name'),
+  'type': (CircuitType, 'name')
+}
+
+optional_assocs = {
+  'tenant': (Tenant, 'name')
+}
+
+for params in circuits:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  circuit, created = Circuit.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(circuit, custom_field_data)
+
+    print("⚡ Created Circuit", circuit.cid)

startup_scripts/310_secret_roles.py

@@ -0,0 +1,14 @@
+from secrets.models import SecretRole
+from startup_script_utils import load_yaml
+import sys
+
+secret_roles = load_yaml('/opt/netbox/initializers/secret_roles.yml')
+
+if secret_roles is None:
+  sys.exit()
+
+for params in secret_roles:
+  secret_role, created = SecretRole.objects.get_or_create(**params)
+
+  if created:
+    print("🔑 Created Secret Role", secret_role.name)

startup_scripts/320_services.py

@@ -0,0 +1,29 @@
+from ipam.models import Service
+from dcim.models import Device
+from virtualization.models import VirtualMachine
+from startup_script_utils import load_yaml
+import sys
+
+services = load_yaml('/opt/netbox/initializers/services.yml')
+
+if services is None:
+  sys.exit()
+
+optional_assocs = {
+  'device': (Device, 'name'),
+  'virtual_machine': (VirtualMachine, 'name')
+}
+
+for params in services:
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  service, created = Service.objects.get_or_create(**params)
+
+  if created:
+    print("🧰 Created Service", service.name)

startup_scripts/330_power_panels.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Site, RackGroup, PowerPanel
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_panels = load_yaml('/opt/netbox/initializers/power_panels.yml')
+
+if power_panels is None:
+  sys.exit()
+
+required_assocs = {
+  'site': (Site, 'name')
+}
+
+optional_assocs = {
+  'rack_group': (RackGroup, 'name')
+}
+
+for params in power_panels:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_panel, created = PowerPanel.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_panel, custom_field_data)
+
+    print("⚡ Created Power Panel", power_panel.site, power_panel.name)

startup_scripts/340_power_feeds.py

@@ -0,0 +1,41 @@
+import sys
+
+from dcim.models import Rack, PowerPanel, PowerFeed
+from startup_script_utils import *
+from tenancy.models import Tenant
+
+power_feeds = load_yaml('/opt/netbox/initializers/power_feeds.yml')
+
+if power_feeds is None:
+  sys.exit()
+
+required_assocs = {
+  'power_panel': (PowerPanel, 'name')
+}
+
+optional_assocs = {
+  'rack': (Rack, 'name')
+}
+
+for params in power_feeds:
+  custom_field_data = pop_custom_fields(params)
+
+  for assoc, details in required_assocs.items():
+    model, field = details
+    query = { field: params.pop(assoc) }
+
+    params[assoc] = model.objects.get(**query)
+
+  for assoc, details in optional_assocs.items():
+    if assoc in params:
+      model, field = details
+      query = { field: params.pop(assoc) }
+
+      params[assoc] = model.objects.get(**query)
+
+  power_feed, created = PowerFeed.objects.get_or_create(**params)
+
+  if created:
+    set_custom_fields_values(power_feed, custom_field_data)
+
+    print("⚡ Created Power Feed", power_feed.name)

startup_scripts/__main__.py

@@ -20,7 +20,7 @@ with scandir(this_dir) as it:
     if not f.name.endswith('.py'):
       continue
 
-    print(f"▶️ Running the startup script {f.path}")
+    print(f"▶️  Running the startup script {f.path}")
     try:
       runpy.run_path(f.path)
     except SystemExit as e:

startup_scripts/startup_script_utils/__init__.py

@@ -1,2 +1,3 @@
 from .load_yaml import load_yaml
 from .permissions import set_permissions
+from .custom_fields import set_custom_fields_values, pop_custom_fields

startup_scripts/startup_script_utils/custom_fields.py

@@ -0,0 +1,15 @@
+def set_custom_fields_values(entity, custom_field_data):
+  if not custom_field_data:
+    return
+
+  entity.custom_field_data = custom_field_data
+  return entity.save()
+
+def pop_custom_fields(params):
+  if 'custom_field_data' in params:
+    return params.pop('custom_field_data')
+  elif 'custom_fields' in params:
+    print("⚠️ Please rename 'custom_fields' to 'custom_field_data'!")
+    return params.pop('custom_fields')
+
+  return None

startup_scripts/startup_script_utils/load_yaml.py

@@ -1,5 +1,6 @@
-from ruamel.yaml import YAML
 from pathlib import Path
+from ruamel.yaml import YAML
+
 
 def load_yaml(yaml_file: str):
   yf = Path(yaml_file)

test.sh

@@ -1,12 +1,29 @@
 #!/bin/bash
+# Runs the original Netbox unit tests and tests whether all initializers work.
+# Usage:
+#   ./test.sh latest
+#   ./test.sh v2.9.7
+#   ./test.sh develop-2.10
+#   IMAGE='netboxcommunity/netbox:latest'        ./test.sh
+#   IMAGE='netboxcommunity/netbox:v2.9.7'        ./test.sh
+#   IMAGE='netboxcommunity/netbox:develop-2.10'  ./test.sh
+#   export IMAGE='netboxcommunity/netbox:latest';       ./test.sh
+#   export IMAGE='netboxcommunity/netbox:v2.9.7';       ./test.sh
+#   export IMAGE='netboxcommunity/netbox:develop-2.10'; ./test.sh
 
 # exit when a command exits with an exit code != 0
 set -e
 
-# version is used by `docker-compose.yml` do determine the tag
+# IMAGE is used by `docker-compose.yml` do determine the tag
 # of the Docker Image that is to be used
-export IMAGE="${IMAGE-netboxcommunity/netbox:latest}"
+if [ "${1}x" != "x" ]; then
+  # Use the command line argument
+  export IMAGE="netboxcommunity/netbox:${1}"
+else
+  export IMAGE="${IMAGE-netboxcommunity/netbox:latest}"
+fi
 
+# Ensure that an IMAGE is defined
 if [ -z "${IMAGE}" ]; then
   echo "⚠️ No image defined"
 
@@ -18,7 +35,7 @@ if [ -z "${IMAGE}" ]; then
 fi
 
 # The docker compose command to use
-doco="docker-compose -f docker-compose.test.yml"
+doco="docker-compose --file docker-compose.test.yml --project-name netbox_docker_test_${1}"
 
 INITIALIZERS_DIR=".initializers"
 
@@ -39,7 +56,7 @@ test_setup() {
 
 test_netbox_unit_tests() {
   echo "⏱ Running Netbox Unit Tests"
-  $doco run --rm netbox ./manage.py test
+  SKIP_STARTUP_SCRIPTS=true $doco run --rm netbox ./manage.py test
 }
 
 test_initializers() {