Merge pull request #582 from netbox-community/develop

Version 1.4.0

.github/workflows/push.yml

@@ -19,7 +19,7 @@ jobs:
         fetch-depth: 0
     - uses: actions/setup-python@v2
     - name: Lint Code Base
-      uses: github/super-linter@v3
+      uses: github/super-linter@v4
       env:
         DEFAULT_BRANCH: develop
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -6,15 +6,18 @@ RUN apk add --no-cache \
       build-base \
       cargo \
       ca-certificates \
+      cmake \
       cyrus-sasl-dev \
+      git \
       graphviz \
       jpeg-dev \
       libevent-dev \
       libffi-dev \
-      libressl-dev \
       libxslt-dev \
+      make \
       musl-dev \
       openldap-dev \
+      openssl-dev \
       postgresql-dev \
       py3-pip \
       python3-dev \
@@ -24,6 +27,20 @@ RUN apk add --no-cache \
       setuptools \
       wheel
 
+# Build libcrc32c for google-crc32c python module
+RUN git clone https://github.com/google/crc32c \
+    && cd crc32c \
+    && git submodule update --init --recursive \
+    && mkdir build \
+    && cd build \
+    && cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCRC32C_BUILD_TESTS=no \
+        -DCRC32C_BUILD_BENCHMARKS=no \
+        -DBUILD_SHARED_LIBS=yes \
+        .. \
+    && make all install
+
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
 RUN /opt/netbox/venv/bin/pip install \
@@ -45,17 +62,20 @@ RUN apk add --no-cache \
       libevent \
       libffi \
       libjpeg-turbo \
-      libressl \
       libxslt \
+      openssl \
       postgresql-libs \
-      python3 \
       py3-pip \
-      ttf-ubuntu-font-family \
+      python3 \
+      tini \
       unit \
       unit-python3
 
 WORKDIR /opt
 
+COPY --from=builder /usr/local/lib/libcrc32c.* /usr/local/lib/
+COPY --from=builder /usr/local/include/crc32c /usr/local/include
+COPY --from=builder /usr/local/lib/cmake/Crc32c /usr/local/lib/cmake/
 COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
@@ -63,6 +83,7 @@ COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/housekeeping.sh /opt/netbox/housekeeping.sh
 COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
@@ -75,11 +96,13 @@ WORKDIR /opt/netbox/netbox
 # to g+w so that pictures can be uploaded to netbox.
 RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
       && chmod -R g+w media /opt/unit/ \
+      && cd /opt/netbox/ && /opt/netbox/venv/bin/python -m mkdocs build \
+          --config-file /opt/netbox/mkdocs.yml --site-dir /opt/netbox/netbox/project-static/docs/ \
       && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
-ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
+ENTRYPOINT [ "/sbin/tini", "--" ]
 
-CMD [ "/opt/netbox/launch-netbox.sh" ]
+CMD [ "/opt/netbox/docker-entrypoint.sh", "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \

VERSION

@@ -1 +1 @@
-1.2.0
+1.4.0

build.sh

@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'alpine:3.13'"
+  echo "              Default: 'alpine:3.14'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -118,6 +118,14 @@ NETBOX_PATH="${NETBOX_PATH-.netbox}"
 # Fetching the NetBox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ]; then
+  REMOTE_EXISTS=$(git ls-remote --heads --tags "${URL}" "${NETBOX_BRANCH}" | wc -l)
+  if [ "${REMOTE_EXISTS}" == "0" ]; then
+    echo "❌ Remote branch '${NETBOX_BRANCH}' not found in '${URL}'; Nothing to do"
+    if [ -n "${GH_ACTION}" ]; then
+      echo "::set-output name=skipped::true"
+    fi
+    exit 0
+  fi
   echo "🌐 Checking out '${NETBOX_BRANCH}' of NetBox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
@@ -125,7 +133,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ]; then
 
   (
     $DRY cd "${NETBOX_PATH}"
-
+    # shellcheck disable=SC2030
     if [ -n "${HTTP_PROXY}" ]; then
       git config http.proxy "${HTTP_PROXY}"
     fi
@@ -157,7 +165,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="alpine:3.13"
+  DOCKER_FROM="alpine:3.14"
 fi
 
 ###
@@ -345,6 +353,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     if [ -n "${DOCKER_FROM}" ]; then
       DOCKER_BUILD_ARGS+=(--build-arg "FROM=${DOCKER_FROM}")
     fi
+    # shellcheck disable=SC2031
     if [ -n "${HTTP_PROXY}" ]; then
       DOCKER_BUILD_ARGS+=(--build-arg "http_proxy=${HTTP_PROXY}")
       DOCKER_BUILD_ARGS+=(--build-arg "https_proxy=${HTTPS_PROXY}")

configuration/configuration.py

@@ -48,6 +48,8 @@ DATABASE = {
                                                     # Database connection SSLMODE
     'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')),
                                                     # Max database connection age
+    'DISABLE_SERVER_SIDE_CURSORS': environ.get('DB_DISABLE_SERVER_SIDE_CURSORS', 'False').lower() == 'true',
+                                                    # Disable the use of server-side cursors transaction pooling
 }
 
 # Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate
@@ -106,9 +108,6 @@ BANNER_LOGIN = environ.get('BANNER_LOGIN', '')
 # BASE_PATH = 'netbox/'
 BASE_PATH = environ.get('BASE_PATH', '')
 
-# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
-CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900))
-
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
 CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90))
 
@@ -210,9 +209,6 @@ REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
 REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true'
 REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' ')))
 
-# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
-RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600))
-
 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
 # version check or use the URL below to check for release in the official NetBox repository.
 # https://api.github.com/repos/netbox-community/netbox/releases

docker-compose.test.yml

@@ -17,10 +17,8 @@ services:
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
     - netbox-media-files:/opt/netbox/netbox/media:z
-    ports:
-    - 8080
   postgres:
-    image: postgres:12-alpine
+    image: postgres:13-alpine
     env_file: env/postgres.env
   redis:
     image: redis:6-alpine

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.4'
 services:
   netbox: &netbox
-    image: netboxcommunity/netbox:${VERSION-latest}
+    image: netboxcommunity/netbox:${VERSION-v3.0}
     depends_on:
     - postgres
     - redis
@@ -20,16 +20,22 @@ services:
     <<: *netbox
     depends_on:
     - redis
-    entrypoint:
+    - postgres
+    command:
     - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
-    command:
     - rqworker
-    ports: []
+  netbox-housekeeping:
+    <<: *netbox
+    depends_on:
+    - redis
+    - postgres
+    command:
+    - /opt/netbox/housekeeping.sh
 
   # postgres
   postgres:
-    image: postgres:12-alpine
+    image: postgres:13-alpine
     env_file: env/postgres.env
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data

docker/docker-entrypoint.sh

@@ -15,7 +15,19 @@ source /opt/netbox/venv/bin/activate
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
 CUR_DB_WAIT_TIME=0
-while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+while [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+  # Read and truncate connection error tracebacks to last line by default
+  exec {psfd}< <(./manage.py showmigrations 2>&1)
+  read -rd '' DB_ERR <&$psfd || :
+  exec {psfd}<&-
+  wait $! && break
+  if [ -n "$DB_WAIT_DEBUG" ]; then
+    echo "$DB_ERR"
+  else
+    readarray -tn 0 DB_ERR_LINES <<<"$DB_ERR"
+    echo "${DB_ERR_LINES[@]: -1}"
+    echo "[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]"
+  fi
   echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)"
   sleep "${DB_WAIT_TIMEOUT}"
   CUR_DB_WAIT_TIME=$((CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT))
@@ -24,6 +36,17 @@ if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then
   echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready."
   exit 1
 fi
+# Check if update is needed
+if ! ./manage.py migrate --check >/dev/null 2>&1; then
+  echo "⚙️ Applying database migrations"
+  ./manage.py migrate --no-input
+  echo "⚙️ Running trace_paths"
+  ./manage.py trace_paths --no-input
+  echo "⚙️ Removing stale content types"
+  ./manage.py remove_stale_contenttypes --no-input
+  echo "⚙️ Removing expired user sessions"
+  ./manage.py clearsessions
+fi
 
 # Create Superuser if required
 if [ "$SKIP_SUPERUSER" == "true" ]; then

docker/housekeeping.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+SECONDS=${HOUSEKEEPING_INTERVAL:=86400}
+echo "Interval set to ${SECONDS} seconds"
+while true; do
+  date
+  /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping
+  sleep "${SECONDS}s"
+done

docker/launch-netbox.sh

@@ -51,4 +51,6 @@ exec unitd \
   --pid /opt/unit/unit.pid \
   --log /dev/stdout \
   --state /opt/unit/state/ \
-  --tmp /opt/unit/tmp/
+  --tmp /opt/unit/tmp/ \
+  --user 101 \
+  --group 0

env/netbox.env

@@ -14,6 +14,7 @@ EMAIL_USERNAME=netbox
 # EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`!
 EMAIL_USE_SSL=false
 EMAIL_USE_TLS=false
+HOUSEKEEPING_INTERVAL=86400
 MAX_PAGE_SIZE=1000
 MEDIA_ROOT=/opt/netbox/netbox/media
 METRICS_ENABLED=false

initializers/secret_roles.yml

@@ -1,4 +0,0 @@
-# - name: Super Secret Passwords
-#   slug: super-secret
-# - name: SNMP Communities
-#   slug: snmp

initializers/sites.yml

@@ -20,6 +20,7 @@
 #   status: active
 #   facility: Amsterdam 3
 #   asn: 67890
+#   tenant: tenant1
 #   custom_field_data:
 #     text_field: Description for AMS3
 # - name: SING 1
@@ -28,5 +29,6 @@
 #   status: active
 #   facility: Singapore 1
 #   asn: 09876
+#   tenant: tenant2
 #   custom_field_data:
 #     text_field: Description for SING1

renovate.json

@@ -1,6 +1,7 @@
 {
   "extends": [
-    "config:base"
+    "config:base",
+    ":disableDependencyDashboard"
   ],
   "enabled": true,
   "labels": ["maintenance"],

requirements-container.txt

@@ -1,4 +1,5 @@
-napalm==3.2.0
-ruamel.yaml==0.17.4
-django-auth-ldap==2.4.0
+napalm==3.3.1
+ruamel.yaml==0.17.16
+django-auth-ldap==3.0.0
+google-crc32c==1.1.5
 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.11.1

startup_scripts/310_secret_roles.py

@@ -1,15 +0,0 @@
-import sys
-from secrets.models import SecretRole
-
-from startup_script_utils import load_yaml
-
-secret_roles = load_yaml("/opt/netbox/initializers/secret_roles.yml")
-
-if secret_roles is None:
-    sys.exit()
-
-for params in secret_roles:
-    secret_role, created = SecretRole.objects.get_or_create(**params)
-
-    if created:
-        print("🔑 Created Secret Role", secret_role.name)

test.sh

@@ -56,13 +56,13 @@ test_setup() {
 
 test_netbox_unit_tests() {
   echo "⏱ Running NetBox Unit Tests"
-  SKIP_STARTUP_SCRIPTS=true $doco run --rm netbox ./manage.py test
+  $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py test
 }
 
 test_initializers() {
   echo "🏭 Testing Initializers"
   export INITIALIZERS_DIR
-  $doco run --rm netbox ./manage.py check
+  $doco run --rm netbox /opt/netbox/docker-entrypoint.sh ./manage.py check
 }
 
 test_cleanup() {