Merge pull request #1002 from netbox-community/develop

Prepare 2.6.1

.dockerignore

@@ -1,10 +1,10 @@
-.git
-.github
-.travis.yml
+.git*
 *.md
-env
 build*
-docker-compose.override.yml
+docker-compose*
+env
+test-configuration
 .netbox/.git*
-.netbox/.travis.yml
+.netbox/contrib
 .netbox/scripts
+.netbox/upgrade.sh

.github/workflows/push.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           python-version: '3.9'
       - name: Lint Code Base
-        uses: github/super-linter@v4
+        uses: github/super-linter@v5
         env:
           DEFAULT_BRANCH: develop
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -75,6 +75,11 @@ jobs:
         env:
           BUILDX_PLATFORM: ${{ matrix.platform }}
           BUILDX_BUILDER_NAME: ${{ steps.buildx-setup.outputs.name }}
+      - id: arm-time-limit
+        name: Set Netbox container start_period higher on ARM64
+        if: matrix.platform == 'linux/arm64'
+        run: |
+          echo "NETBOX_START_PERIOD=240s" >>"${GITHUB_ENV}"
       - id: docker-test
         name: Test the image
         run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh

Dockerfile

@@ -14,7 +14,6 @@ RUN export DEBIAN_FRONTEND=noninteractive \
       libsasl2-dev \
       libssl-dev \
       libxml2-dev \
-      libxml2-dev \
       libxmlsec1 \
       libxmlsec1-dev \
       libxmlsec1-openssl \
@@ -31,7 +30,15 @@ RUN export DEBIAN_FRONTEND=noninteractive \
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
-RUN sed -i -e '/psycopg2-binary/d' requirements.txt && \
+RUN \
+    # We compile 'psycopg2' in the build process
+    sed -i -e '/psycopg2-binary/d' /requirements.txt && \
+    # Gunicorn is not needed because we use Nginx Unit
+    sed -i -e '/gunicorn/d' /requirements.txt && \
+    # We need 'social-auth-core[all]' in the Docker image. But if we put it in our own requirements-container.txt
+    # we have potential version conflicts and the build will fail.
+    # That's why we just replace it in the original requirements.txt.
+    sed -i -e 's/social-auth-core\[openidconnect\]/social-auth-core\[all\]/g' /requirements.txt && \
     /opt/netbox/venv/bin/pip install \
       -r /requirements.txt \
       -r /requirements-container.txt
@@ -74,6 +81,8 @@ COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
+# Copy the modified 'requirements*.txt' files, to have the files actually used during installation
+COPY --from=builder /requirements.txt /requirements-container.txt /opt/netbox/
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
@@ -88,11 +97,11 @@ WORKDIR /opt/netbox/netbox
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
 RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
-      && chown -R unit:root media /opt/unit/ \
-      && chmod -R g+w media /opt/unit/ \
-      && cd /opt/netbox/ && SECRET_KEY="dummy" /opt/netbox/venv/bin/python -m mkdocs build \
+      && chown -R unit:root /opt/unit/ media reports scripts \
+      && chmod -R g+w /opt/unit/ media reports scripts \
+      && cd /opt/netbox/ && SECRET_KEY="dummyKeyWithMinimumLength-------------------------" /opt/netbox/venv/bin/python -m mkdocs build \
           --config-file /opt/netbox/mkdocs.yml --site-dir /opt/netbox/netbox/project-static/docs/ \
-      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
+      && SECRET_KEY="dummyKeyWithMinimumLength-------------------------" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENV LANG=C.utf8 PATH=/opt/netbox/venv/bin:$PATH
 ENTRYPOINT [ "/usr/bin/tini", "--" ]

README.md

@@ -99,7 +99,7 @@ For each of the above tag, there is an extra tag:
 ## Documentation
 
 Please refer [to our wiki on GitHub][netbox-docker-wiki] for further information on how to use the NetBox Docker image properly.
-The wiki covers advanced topics such as using files for secrets, configuring TLS, deployment to Kubernetes, monitoring and configuring NAPALM and LDAP.
+The wiki covers advanced topics such as using files for secrets, configuring TLS, deployment to Kubernetes, monitoring and configuring LDAP.
 
 Our wiki is a community effort.
 Feel free to correct errors, update outdated information or provide additional guides and insights.

VERSION

@@ -1 +1 @@
-2.5.0
+2.6.1

build-functions/get-public-image-config.sh

@@ -1,16 +1,17 @@
 #!/bin/bash
 
+check_if_tags_exists() {
+  local image=$1
+  local tag=$2
+  skopeo list-tags "docker://$image" | jq -r ".Tags | contains([\"$tag\"])"
+}
+
 get_image_label() {
   local label=$1
   local image=$2
   skopeo inspect "docker://$image" | jq -r ".Labels[\"$label\"]"
 }
 
-get_image_layers() {
-  local image=$1
-  skopeo inspect "docker://$image" | jq -r ".Layers"
-}
-
 get_image_last_layer() {
   local image=$1
   skopeo inspect "docker://$image" | jq -r ".Layers | last"

build.sh

@@ -139,7 +139,8 @@ fi
 
 # Check if we have everything needed for the build
 source ./build-functions/check-commands.sh
-
+# Load all build functions
+source ./build-functions/get-public-image-config.sh
 source ./build-functions/gh-functions.sh
 
 IMAGE_NAMES="${IMAGE_NAMES-docker.io/netboxcommunity/netbox}"
@@ -309,26 +310,29 @@ gh_env "FINAL_DOCKER_TAG=${IMAGE_NAME_TAGS[0]}"
 ###
 # Checking if the build is necessary,
 # meaning build only if one of those values changed:
+# - a new tag is beeing created
 # - base image digest
 # - netbox git ref (Label: netbox.git-ref)
 # - netbox-docker git ref (Label: org.opencontainers.image.revision)
 ###
-# Load information from registry (only for docker.io)
+# Load information from registry (only for first registry in "IMAGE_NAMES")
 SHOULD_BUILD="false"
 BUILD_REASON=""
 if [ -z "${GH_ACTION}" ]; then
   # Asuming non Github builds should always proceed
   SHOULD_BUILD="true"
   BUILD_REASON="${BUILD_REASON} interactive"
+elif [ "false" == "$(check_if_tags_exists "${IMAGE_NAMES[0]}" "$TARGET_DOCKER_TAG")" ]; then
+  SHOULD_BUILD="true"
+  BUILD_REASON="${BUILD_REASON} newtag"
 else
-  source ./build-functions/get-public-image-config.sh
   echo "Checking labels for '${FINAL_DOCKER_TAG}'"
   BASE_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM}")
-  mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${FINAL_DOCKER_TAG}")
+  OLD_BASE_LAST_LAYER=$(get_image_label netbox.last-base-image-layer "${FINAL_DOCKER_TAG}")
   NETBOX_GIT_REF_OLD=$(get_image_label netbox.git-ref "${FINAL_DOCKER_TAG}")
   GIT_REF_OLD=$(get_image_label org.opencontainers.image.revision "${FINAL_DOCKER_TAG}")
 
-  if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${BASE_LAST_LAYER}\$"; then
+  if [ "${BASE_LAST_LAYER}" != "${OLD_BASE_LAST_LAYER}" ]; then
     SHOULD_BUILD="true"
     BUILD_REASON="${BUILD_REASON} ubuntu"
   fi
@@ -388,6 +392,7 @@ fi
 if [ -n "${BUILD_REASON}" ]; then
   BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<<"$BUILD_REASON")
   DOCKER_BUILD_ARGS+=(--label "netbox.build-reason=${BUILD_REASON}")
+  DOCKER_BUILD_ARGS+=(--label "netbox.last-base-image-layer=${BASE_LAST_LAYER}")
 fi
 
 # --build-arg

configuration/configuration.py

@@ -136,10 +136,6 @@ if 'BANNER_BOTTOM' in environ:
 if 'BANNER_LOGIN' in environ:
     BANNER_LOGIN = environ.get('BANNER_LOGIN', None)
 
-# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set:
-# BASE_PATH = 'netbox/'
-BASE_PATH = environ.get('BASE_PATH', '')
-
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
 if 'CHANGELOG_RETENTION' in environ:
     CHANGELOG_RETENTION = _environ_get_and_map('CHANGELOG_RETENTION', None, _AS_INT)
@@ -239,20 +235,6 @@ MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media'))
 # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
 METRICS_ENABLED = _environ_get_and_map('METRICS_ENABLED', 'False', _AS_BOOL)
 
-# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM.
-if 'NAPALM_USERNAME' in environ:
-    NAPALM_USERNAME = environ.get('NAPALM_USERNAME', None)
-if 'NAPALM_PASSWORD' in environ:
-    NAPALM_PASSWORD = _read_secret('napalm_password', environ.get('NAPALM_PASSWORD', None))
-
-# NAPALM timeout (in seconds). (Default: 30)
-if 'NAPALM_TIMEOUT' in environ:
-    NAPALM_TIMEOUT = _environ_get_and_map('NAPALM_TIMEOUT', None, _AS_INT)
-
-# # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
-# # be provided as a dictionary.
-# NAPALM_ARGS = None
-
 # Determine how many objects to display per page within a list. (Default: 50)
 if 'PAGINATE_COUNT' in environ:
     PAGINATE_COUNT = _environ_get_and_map('PAGINATE_COUNT', None, _AS_INT)
@@ -301,17 +283,9 @@ REMOTE_AUTH_DEFAULT_GROUPS = _environ_get_and_map('REMOTE_AUTH_DEFAULT_GROUPS',
 RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None)
 # RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases'
 
-# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of
-# this setting is derived from the installed location.
-REPORTS_ROOT = environ.get('REPORTS_ROOT', '/etc/netbox/reports')
-
 # Maximum execution time for background tasks, in seconds.
 RQ_DEFAULT_TIMEOUT = _environ_get_and_map('RQ_DEFAULT_TIMEOUT', 300, _AS_INT)
 
-# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
-# this setting is derived from the installed location.
-SCRIPTS_ROOT = environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts')
-
 # The name to use for the csrf token cookie.
 CSRF_COOKIE_NAME = environ.get('CSRF_COOKIE_NAME', 'csrftoken')
 

configuration/extra.py

@@ -15,12 +15,6 @@
 #     'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp',
 # )
 
-
-## NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
-## be provided as a dictionary.
-# NAPALM_ARGS = {}
-
-
 ## Enable installed plugins. Add the name of each plugin to the list.
 # from netbox.configuration.configuration import PLUGINS
 # PLUGINS.append('my_plugin')

docker-compose.test.override.yml

@@ -0,0 +1,6 @@
+version: '3.4'
+services:
+  netbox:
+    ports:
+      - "127.0.0.1:8000:8080"
+

docker-compose.test.yml

@@ -1,43 +1,65 @@
 version: '3.4'
 services:
-  netbox:
+  netbox: &netbox
     image: ${IMAGE-netboxcommunity/netbox:latest}
     depends_on:
       postgres:
         condition: service_healthy
       redis:
-        condition: service_started
+        condition: service_healthy
       redis-cache:
-        condition: service_started
+        condition: service_healthy
     env_file: env/netbox.env
     user: 'unit:root'
     volumes:
-    - ./configuration:/etc/netbox/config:z,ro
     - ./test-configuration/logging.py:/etc/netbox/config/logging.py:z,ro
-    - ./reports:/etc/netbox/reports:z,ro
-    - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-media-files:/opt/netbox/netbox/media:z
+    healthcheck:
+      start_period: ${NETBOX_START_PERIOD-120s}
+      timeout: 3s
+      interval: 15s
+      test: "curl -f http://localhost:8080/api/ || exit 1"
+  netbox-worker:
+    <<: *netbox
+    command:
+    - /opt/netbox/venv/bin/python
+    - /opt/netbox/netbox/manage.py
+    - rqworker
+    healthcheck:
+      start_period: 40s
+      timeout: 3s
+      interval: 15s
+      test: "ps -aux | grep -v grep | grep -q rqworker || exit 1"
+  netbox-housekeeping:
+    <<: *netbox
+    command:
+    - /opt/netbox/housekeeping.sh
+    healthcheck:
+      start_period: 40s
+      timeout: 3s
+      interval: 15s
+      test: "ps -aux | grep -v grep | grep -q housekeeping || exit 1"
   postgres:
     image: postgres:15-alpine
     env_file: env/postgres.env
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
+      test: "pg_isready -t 2 -d $$POSTGRES_DB -U $$POSTGRES_USER" ## $$ because of docker-compose
       interval: 10s
       timeout: 5s
       retries: 5
-  redis:
+  redis: &redis
     image: redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
     - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
     env_file: env/redis.env
+    healthcheck:
+      start_period: 20s
+      timeout: 3s
+      interval: 15s
+      test: "timeout 2 redis-cli ping"
   redis-cache:
-    image: redis:7-alpine
-    command:
-    - sh
-    - -c # this is to evaluate the $REDIS_PASSWORD from the env
-    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    <<: *redis
     env_file: env/redis-cache.env
 volumes:
   netbox-media-files:

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.4'
 services:
   netbox: &netbox
-    image: docker.io/netboxcommunity/netbox:${VERSION-v3.4-2.5.0}
+    image: docker.io/netboxcommunity/netbox:${VERSION-v3.5-2.6.1}
     depends_on:
     - postgres
     - redis
@@ -15,9 +15,9 @@ services:
       test: "curl -f http://localhost:8080/api/ || exit 1"
     volumes:
     - ./configuration:/etc/netbox/config:z,ro
-    - ./reports:/etc/netbox/reports:z,ro
-    - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-media-files:/opt/netbox/netbox/media:z
+    - netbox-media-files:/opt/netbox/netbox/media:z,rw
+    - netbox-reports-files:/opt/netbox/netbox/reports:z,rw
+    - netbox-scripts-files:/opt/netbox/netbox/scripts:z,rw
   netbox-worker:
     <<: *netbox
     depends_on:
@@ -63,7 +63,7 @@ services:
     volumes:
     - netbox-redis-data:/data
   redis-cache:
-    image: redis:7-alpine
+    image: docker.io/redis:7-alpine
     command:
     - sh
     - -c # this is to evaluate the $REDIS_PASSWORD from the env
@@ -77,7 +77,11 @@ volumes:
     driver: local
   netbox-postgres-data:
     driver: local
-  netbox-redis-data:
-    driver: local
   netbox-redis-cache-data:
     driver: local
+  netbox-redis-data:
+    driver: local
+  netbox-reports-files:
+    driver: local
+  netbox-scripts-files:
+    driver: local

docker/docker-entrypoint.sh

@@ -84,9 +84,12 @@ fi
 
 ./manage.py shell --interface python <<END
 from users.models import Token
-old_default_token = Token.objects.get(key="0123456789abcdef0123456789abcdef01234567")
-if old_default_token:
-    print("⚠️ Warning: You have the old default admin token in your database. This token is widely known; please remove it.")
+try:
+    old_default_token = Token.objects.get(key="0123456789abcdef0123456789abcdef01234567")
+    if old_default_token:
+        print("⚠️ Warning: You have the old default admin token in your database. This token is widely known; please remove it.")
+except Token.DoesNotExist:
+    pass
 END
 
 echo "✅ Initialisation is done."

env/netbox.env

@@ -29,6 +29,6 @@ REDIS_INSECURE_SKIP_TLS_VERIFY=false
 REDIS_PASSWORD=H733Kdjndks81
 REDIS_SSL=false
 RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases
-SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
+SECRET_KEY=r(m)9nLGnz$(_q3N4z1k(EFsMCjjjzx08x9VhNVcfd%6RF#r!6DE@+V5Zk2X
 SKIP_SUPERUSER=true
 WEBHOOKS_ENABLED=true

reports/devices.py.example

@@ -1,46 +0,0 @@
-from dcim.choices import DeviceStatusChoices
-from dcim.models import ConsolePort, Device, PowerPort
-from extras.reports import Report
-
-
-class DeviceConnectionsReport(Report):
-    description = "Validate the minimum physical connections for each device"
-
-    def test_console_connection(self):
-
-        # Check that every console port for every active device has a connection defined.
-        active = DeviceStatusChoices.STATUS_ACTIVE
-        for console_port in ConsolePort.objects.prefetch_related('device').filter(device__status=active):
-            if console_port.connected_endpoint is None:
-                self.log_failure(
-                    console_port.device,
-                    "No console connection defined for {}".format(console_port.name)
-                )
-            elif not console_port.connection_status:
-                self.log_warning(
-                    console_port.device,
-                    "Console connection for {} marked as planned".format(console_port.name)
-                )
-            else:
-                self.log_success(console_port.device)
-
-    def test_power_connections(self):
-
-        # Check that every active device has at least two connected power supplies.
-        for device in Device.objects.filter(status=DeviceStatusChoices.STATUS_ACTIVE):
-            connected_ports = 0
-            for power_port in PowerPort.objects.filter(device=device):
-                if power_port.connected_endpoint is not None:
-                    connected_ports += 1
-                    if not power_port.connection_status:
-                        self.log_warning(
-                            device,
-                            "Power connection for {} marked as planned".format(power_port.name)
-                        )
-            if connected_ports < 2:
-                self.log_failure(
-                    device,
-                    "{} connected power supplies found (2 needed)".format(connected_ports)
-                )
-            else:
-                self.log_success(device)

requirements-container.txt

@@ -1,6 +1,4 @@
-django-auth-ldap==4.1.0
+django-auth-ldap==4.3.0
 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.13.2
-napalm==4.0.0
-psycopg2==2.9.5
+psycopg2==2.9.6
 python3-saml==1.15.0
-social-auth-core[all]==4.3.0

test.sh

@@ -14,6 +14,8 @@
 # exit when a command exits with an exit code != 0
 set -e
 
+source ./build-functions/gh-functions.sh
+
 # IMAGE is used by `docker-compose.yml` do determine the tag
 # of the Docker Image that is to be used
 if [ "${1}x" != "x" ]; then
@@ -35,29 +37,72 @@ if [ -z "${IMAGE}" ]; then
 fi
 
 # The docker compose command to use
-doco="docker compose --file docker-compose.test.yml --project-name netbox_docker_test"
+doco="docker compose --file docker-compose.test.yml --file docker-compose.test.override.yml --project-name netbox_docker_test"
 
 test_setup() {
+  gh_echo "::group:: Test setup"
   echo "🏗 Setup up test environment"
   $doco up --detach --quiet-pull --wait --force-recreate --renew-anon-volumes --no-start
   $doco start postgres
   $doco start redis
   $doco start redis-cache
+  gh_echo "::endgroup::"
 }
 
 test_netbox_unit_tests() {
+  gh_echo "::group:: Netbox unit tests"
   echo "⏱ Running NetBox Unit Tests"
   $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py test
+  gh_echo "::endgroup::"
 }
 
 test_compose_db_setup() {
+  gh_echo "::group:: Netbox DB migrations"
   echo "⏱ Running NetBox DB migrations"
   $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py migrate
+  gh_echo "::endgroup::"
+}
+
+test_netbox_start() {
+  gh_echo "::group:: Start Netbox service"
+  echo "⏱ Starting NetBox services"
+  $doco up --detach --wait
+  gh_echo "::endgroup::"
+}
+
+test_netbox_web() {
+  gh_echo "::group:: Web service test"
+  echo "⏱ Starting web service test"
+  RESP_CODE=$(
+    curl \
+      --silent \
+      --output /dev/null \
+      --write-out '%{http_code}' \
+      --request GET \
+      --connect-timeout 5 \
+      --max-time 10 \
+      --retry 5 \
+      --retry-delay 0 \
+      --retry-max-time 40 \
+      http://127.0.0.1:8000/
+  )
+  if [ "$RESP_CODE" == "200" ]; then
+    echo "Webservice running"
+  else
+    echo "⚠️ Got response code '$RESP_CODE' but expected '200'"
+    exit 1
+  fi
+  gh_echo "::endgroup::"
 }
 
 test_cleanup() {
   echo "💣 Cleaning Up"
+  gh_echo "::group:: Docker compose logs"
+  $doco logs --no-color
+  gh_echo "::endgroup::"
+  gh_echo "::group:: Docker compose down"
   $doco down --volumes
+  gh_echo "::endgroup::"
 }
 
 echo "🐳🐳🐳 Start testing '${IMAGE}'"
@@ -68,5 +113,7 @@ test_setup
 
 test_netbox_unit_tests
 test_compose_db_setup
+test_netbox_start
+test_netbox_web
 
 echo "🐳🐳🐳 Done testing '${IMAGE}'"