Merge pull request #606 from netbox-community/develop

Version 1.4.1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,67 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-<!--
-
-Before raising an issue here, answer the following questions for yourself, please:
-
-* Did you read through the troubleshooting section? (https://github.com/netbox-community/netbox-docker/wiki/Troubleshooting)
-* Have you had a look at the rest of the wiki? (https://github.com/netbox-community/netbox-docker/wiki)
-* Have you updated to the latest version and tried again? (i.e. `git pull` and `docker-compose pull`)
-* Have you reset the project and tried again? (i.e. `docker-compose down -v`)
-* Are you confident that your problem is related to the Docker image or Docker Compose file this project provides?
-  (Otherwise ask on the NetBox mailing list, please: https://groups.google.com/d/forum/netbox-discuss)
-* Have you looked through the issues already resolved?
-
-Please try this means to get help before opening an issue here:
-
-* On the networktocode Slack in the #netbox-docker channel: http://slack.networktocode.com/
-* On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/
-* On the NetBox mailing list: https://groups.google.com/d/forum/netbox-discuss
-
-Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough.
-
--->
-
-## Current Behavior
-
-<!-- describe what you did and how it misbehaved -->
-...
-
-## Expected Behavior
-
-<!-- describe what you expected instead -->
-...
-
-## Debug Information
-
-<!-- please fill in the following information that might helps us debug your problem more quickly -->
-The output of `docker-compose version`: `XXXXX`
-The output of `docker version`: `XXXXX`
-The output of `git rev-parse HEAD`: `XXXXX`
-The command you used to start the project: `XXXXX`
-
-<!-- adjust the `latest` tag to the version you're using -->
-The output of `docker inspect netboxcommunity/netbox:latest --format "{{json .Config.Labels}}"`:
-
-```json
-{
-  "JSON JSON JSON":
-    "--> Please paste formatted json. (Use e.g. `jq` or https://jsonformatter.curiousconcept.com/)"
-}
-```
-
-The output of `docker-compose logs netbox`:
-<!--
-If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com
--->
-
-```text
-LOG LOG LOG
-```

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,148 @@
+name: Bug report
+description: Create a report about a malfunction of the Docker setup
+body:
+- type: markdown
+  attributes:
+    value: |
+      Please only raise an issue if you're certain that you've found a bug.
+      Else, see these other means to get help:
+
+      - See our troubleshooting section:
+        https://github.com/netbox-community/netbox-docker/wiki/Troubleshooting
+      - Have a look at the rest of the wiki:
+        https://github.com/netbox-community/netbox-docker/wiki
+      - Check the release notes:
+        https://github.com/netbox-community/netbox-docker/releases
+      - Look through the issues already resolved:
+        https://github.com/netbox-community/netbox-docker/issues?q=is%3Aclosed
+
+      If you did not find what you're looking for,
+      try the help of our community:
+
+      - Post to Github Discussions:
+        https://github.com/netbox-community/netbox-docker/discussions
+      - Join the `#netbox-docker` channel on our Slack:
+        https://join.slack.com/t/netdev-community/shared_invite/zt-mtts8g0n-Sm6Wutn62q_M4OdsaIycrQ
+      - Ask on the NetBox mailing list:
+        https://groups.google.com/d/forum/netbox-discuss
+
+      Please don't open an issue to open a PR.
+      Just submit the PR, that's good enough.
+- type: textarea
+  id: current-behavior
+  attributes:
+    label: Current Behavior
+    description: Please describe what you did and how you think it misbehaved
+    placeholder: I tried to … by doing …, but it …
+  validations:
+    required: true
+- type: textarea
+  id: expected-behavior
+  attributes:
+    label: Expected Behavior
+    description: Please describe what you expected instead
+    placeholder: I expected that … when I do …
+  validations:
+    required: true
+- type: input
+  id: docker-compose-version
+  attributes:
+    label: Docker Compose Version
+    description: Please paste the output of `docker-compose version`
+    placeholder: Docker Compose version vX.Y.Z
+  validations:
+    required: true
+- type: textarea
+  id: docker-version
+  attributes:
+    label: Docker Version
+    description: Please paste the output of `docker version`
+    render: text
+    placeholder: |
+      Client:
+      Cloud integration: 1.0.17
+      Version:           20.10.8
+      API version:       1.41
+      Go version:        go1.16.6
+      Git commit:        3967b7d
+      Built:             Fri Jul 30 19:55:20 2021
+      OS/Arch:           darwin/amd64
+      Context:           default
+      Experimental:      true
+
+      Server: Docker Engine - Community
+      Engine:
+        Version:          20.10.8
+        API version:      1.41 (minimum version 1.12)
+        Go version:       go1.16.6
+        Git commit:       75249d8
+        Built:            Fri Jul 30 19:52:10 2021
+        OS/Arch:          linux/amd64
+        Experimental:     false
+      containerd:
+        Version:          1.4.9
+        GitCommit:        e25210fe30a0a703442421b0f60afac609f950a3
+      runc:
+        Version:          1.0.1
+        GitCommit:        v1.0.1-0-g4144b63
+      docker-init:
+        Version:          0.19.0
+        GitCommit:        de40ad0
+  validations:
+    required: true
+- type: input
+  id: git-rev
+  attributes:
+    label: The git Revision
+    description: Please paste the output of `git rev-parse HEAD`
+  validations:
+    required: true
+- type: textarea
+  id: git-status
+  attributes:
+    label: The git Status
+    description: Please paste the output of `git status`
+    render: text
+    placeholder: |
+      On branch main
+      nothing to commit, working tree clean
+  validations:
+    required: true
+- type: input
+  id: run-command
+  attributes:
+    label: Startup Command
+    description: Please specify the command you used to start the project
+    placeholder: docker compose up
+  validations:
+    required: true
+- type: textarea
+  id: netbox-logs
+  attributes:
+    label: NetBox Logs
+    description: Please paste the output of `docker-compose logs netbox` (or `docker compose logs netbox`)
+    render: text
+    placeholder: |
+      netbox_1  | ⚙️ Applying database migrations
+      netbox_1  | 🧬 loaded config '/etc/netbox/config/configuration.py'
+      netbox_1  | 🧬 loaded config '/etc/netbox/config/a.py'
+      netbox_1  | 🧬 loaded config '/etc/netbox/config/extra.py'
+      netbox_1  | 🧬 loaded config '/etc/netbox/config/logging.py'
+      netbox_1  | 🧬 loaded config '/etc/netbox/config/plugins.py'
+      ...
+  validations:
+    required: true
+- type: textarea
+  id: docker-compose-override-yml
+  attributes:
+    label: Content of docker-compose.override.yml
+    description: Please paste the output of `cat docker-compose.override.yml`
+    render: yaml
+    placeholder: |
+      version: '3.4'
+      services:
+        netbox:
+          ports:
+            - '8080:8080'
+  validations:
+    required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -5,7 +5,7 @@ contact_links:
     about: The Github Discussions are the right place to ask questions about how to use or do certain things with NetBox Docker.
 
   - name: Chat
-    url: http://slack.networktocode.com/
+    url: https://join.slack.com/t/netdev-community/shared_invite/zt-mtts8g0n-Sm6Wutn62q_M4OdsaIycrQ
     about: 'Usually the quickest way to seek help with small issues is to join our #netbox-docker Slack channel.'
 
   - name: Community Wiki

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,54 +0,0 @@
----
-name: Feature or Change Request
-about: Request a new feature or a change of the current behavior
-title: ''
-labels: ''
-assignees: ''
-
----
-
-<!--
-
-Before raising an issue here, answer the following questions for yourself, please:
-
-* Did you read through the troubleshooting section? (https://github.com/netbox-community/netbox-docker/wiki/Troubleshooting)
-* Have you had a look at the rest of the wiki? (https://github.com/netbox-community/netbox-docker/wiki)
-* Have you read the release notes recently (https://github.com/netbox-community/netbox-docker/releases)
-* Are you confident that your feature/change request is related to the Docker image or Docker Compose file this project provides?
-  (Otherwise ask on the NetBox mailing list, please: https://groups.google.com/d/forum/netbox-discuss)
-* Have you looked through the issues already resolved?
-
-Please try this means to get help before opening an issue here:
-
-* On the networktocode Slack in the #netbox-docker channel: http://slack.networktocode.com/
-* On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/
-* On the NetBox mailing list: https://groups.google.com/d/forum/netbox-discuss
-
-Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough.
-
--->
-
-## Desired Behavior
-
-<!-- please describe the behavior you desire -->
-...
-
-## Contrast to Current Behavior
-
-<!-- please describe how the desired behavior is different from the current behavior -->
-...
-
-## Changes Required
-
-<!-- if you can, please elaborate what changes would exactly be required -->
-...
-
-## Discussion: Benefits and Drawbacks
-
-<!--
-Please make your case here:
-- Why do you think this project and the community will benefit from your suggestion?
-- What are the drawbacks of this change? Is it backwards-compatible?
-- Anything else that you think is relevant to the discussion of this feature/change request.
--->
-...

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,68 @@
+name: Feature or Change Request
+description: Request a new feature or a change of the current behavior
+body:
+- type: markdown
+  attributes:
+    value: |
+      This issue type is to propose new features for the Docker setup.
+      To just spin an idea, see the Github Discussions section, please.
+
+      Before asking for help, see these links first:
+
+      - See our troubleshooting section:
+        https://github.com/netbox-community/netbox-docker/wiki/Troubleshooting
+      - Have a look at the rest of the wiki:
+        https://github.com/netbox-community/netbox-docker/wiki
+      - Check the release notes:
+        https://github.com/netbox-community/netbox-docker/releases
+      - Look through the issues already resolved:
+        https://github.com/netbox-community/netbox-docker/issues?q=is%3Aclosed
+
+      If you did not find what you're looking for,
+      try the help of our community:
+
+      - Post to Github Discussions:
+        https://github.com/netbox-community/netbox-docker/discussions
+      - Join the `#netbox-docker` channel on our Slack:
+        https://join.slack.com/t/netdev-community/shared_invite/zt-mtts8g0n-Sm6Wutn62q_M4OdsaIycrQ
+      - Ask on the NetBox mailing list:
+        https://groups.google.com/d/forum/netbox-discuss
+
+      Please don't open an issue to open a PR.
+      Just submit the PR, that's good enough.
+- type: textarea
+  id: desired-behavior
+  attributes:
+    label: Desired Behavior
+    description: Please describe the desired behavior
+    placeholder: To me, it would be useful, if … because …
+  validations:
+    required: true
+- type: textarea
+  id: contrast-to-current
+  attributes:
+    label: Contrast to Current Behavior
+    description: Please describe how the desired behavior is different from the current behavior
+    placeholder: The current behavior is …, but this lacks …
+  validations:
+    required: true
+- type: textarea
+  id: required-changes
+  attributes:
+    label: Required Changes
+    description: If you can, please elaborate what changes will be required to implement the desired behavior
+    placeholder: I suggest to change the file …
+  validations:
+    required: false
+- type: textarea
+  id: discussion
+  attributes:
+    label: 'Discussion: Benefits and Drawbacks'
+    description: |
+      Please make your case here:
+      - Why do you think this project and the community will benefit from your suggestion?
+      - What are the drawbacks of this change? Is it backwards-compatible?
+      - Anything else that you think is relevant to the discussion of this feature/change request.
+    placeholder: I suggest to change the file …
+  validations:
+    required: false

.github/workflows/push.yml

@@ -14,9 +14,12 @@ jobs:
     name: Checks syntax of our code
     steps:
     - uses: actions/checkout@v2
+      with:
+        # Full git history is needed to get a proper list of changed files within `super-linter`
+        fetch-depth: 0
     - uses: actions/setup-python@v2
     - name: Lint Code Base
-      uses: github/super-linter@v3
+      uses: github/super-linter@v4
       env:
         DEFAULT_BRANCH: develop
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -39,7 +42,7 @@ jobs:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-next.sh
+        - ./build.sh feature
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-next.sh
+        - ./build.sh feature
         - ./build.sh develop
       fail-fast: false
     runs-on: ubuntu-latest

.gitignore

@@ -7,6 +7,9 @@ configuration/*
 !configuration/configuration.py
 !configuration/extra.py
 configuration/ldap/*
+!configuration/ldap/extra.py
 !configuration/ldap/ldap_config.py
+!configuration/logging.py
+!configuration/plugins.py
 prometheus.yml
 super-linter.log

Dockerfile

@@ -6,13 +6,15 @@ RUN apk add --no-cache \
       build-base \
       cargo \
       ca-certificates \
+      cmake \
       cyrus-sasl-dev \
+      git \
       graphviz \
       jpeg-dev \
       libevent-dev \
       libffi-dev \
-      libressl-dev \
       libxslt-dev \
+      make \
       musl-dev \
       openldap-dev \
       postgresql-dev \
@@ -24,6 +26,20 @@ RUN apk add --no-cache \
       setuptools \
       wheel
 
+# Build libcrc32c for google-crc32c python module
+RUN git clone https://github.com/google/crc32c \
+    && cd crc32c \
+    && git submodule update --init --recursive \
+    && mkdir build \
+    && cd build \
+    && cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCRC32C_BUILD_TESTS=no \
+        -DCRC32C_BUILD_BENCHMARKS=no \
+        -DBUILD_SHARED_LIBS=yes \
+        .. \
+    && make all install
+
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
 RUN /opt/netbox/venv/bin/pip install \
@@ -45,17 +61,20 @@ RUN apk add --no-cache \
       libevent \
       libffi \
       libjpeg-turbo \
-      libressl \
       libxslt \
+      openssl \
       postgresql-libs \
-      python3 \
       py3-pip \
-      ttf-ubuntu-font-family \
+      python3 \
+      tini \
       unit \
       unit-python3
 
 WORKDIR /opt
 
+COPY --from=builder /usr/local/lib/libcrc32c.* /usr/local/lib/
+COPY --from=builder /usr/local/include/crc32c /usr/local/include
+COPY --from=builder /usr/local/lib/cmake/Crc32c /usr/local/lib/cmake/
 COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
@@ -63,6 +82,7 @@ COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/housekeeping.sh /opt/netbox/housekeeping.sh
 COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
@@ -74,12 +94,15 @@ WORKDIR /opt/netbox/netbox
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
 RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chown -R unit:root media /opt/unit/ \
       && chmod -R g+w media /opt/unit/ \
+      && cd /opt/netbox/ && /opt/netbox/venv/bin/python -m mkdocs build \
+          --config-file /opt/netbox/mkdocs.yml --site-dir /opt/netbox/netbox/project-static/docs/ \
       && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
-ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
+ENTRYPOINT [ "/sbin/tini", "--" ]
 
-CMD [ "/opt/netbox/launch-netbox.sh" ]
+CMD [ "/opt/netbox/docker-entrypoint.sh", "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \

README.md

@@ -5,23 +5,23 @@
 ![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/netbox-community/netbox-docker)
 ![Github release workflow](https://img.shields.io/github/workflow/status/netbox-community/netbox-docker/release)
 ![Docker Pulls](https://img.shields.io/docker/pulls/netboxcommunity/netbox)
-[![MicroBadger Layers](https://img.shields.io/microbadger/layers/netboxcommunity/netbox)][netbox-docker-microbadger]
-[![MicroBadger Size](https://img.shields.io/microbadger/image-size/netboxcommunity/netbox)][netbox-docker-microbadger]
 [![GitHub license](https://img.shields.io/github/license/netbox-community/netbox-docker)][netbox-docker-license]
 
 [The Github repository](netbox-docker-github) houses the components needed to build NetBox as a Docker container.
 Images are built using this code and are released to [Docker Hub][netbox-dockerhub] and [Quay.io][netbox-quayio] once a day.
 
 Do you have any questions?
-Before opening an issue on Github, please join the [Network To Code][ntc-slack] Slack and ask for help in our [`#netbox-docker`][netbox-docker-slack] channel.
+Before opening an issue on Github,
+please join the [our Slack][netbox-docker-slack] and ask for help in the [`#netbox-docker`][netbox-docker-slack-channel] channel.
 
 [github-stargazers]: https://github.com/netbox-community/netbox-docker/stargazers
 [github-release]: https://github.com/netbox-community/netbox-docker/releases
 [netbox-docker-microbadger]: https://microbadger.com/images/netboxcommunity/netbox
 [netbox-dockerhub]: https://hub.docker.com/r/netboxcommunity/netbox/
 [netbox-docker-github]: https://github.com/netbox-community/netbox-docker/
-[ntc-slack]: http://slack.networktocode.com/
-[netbox-docker-slack]: https://slack.com/app_redirect?channel=netbox-docker&team=T09LQ7E9E
+[netbox-docker-slack]: https://join.slack.com/t/netdev-community/shared_invite/zt-mtts8g0n-Sm6Wutn62q_M4OdsaIycrQ
+[netbox-docker-slack-channel]: https://netdev-community.slack.com/archives/C01P0GEVBU7
+[netbox-slack-channel]: https://netdev-community.slack.com/archives/C01P0FRSXRV
 [netbox-docker-license]: https://github.com/netbox-community/netbox-docker/blob/release/LICENSE
 [netbox-quayio]: https://quay.io/repository/netboxcommunity/netbox
 
@@ -86,10 +86,12 @@ It covers advanced topics such as using files for secrets, deployment to Kuberne
 
 ## Getting Help
 
-Feel free to ask questions in our [Github Community][netbox-community] or join [our Slack channel `#netbox-docker`][netbox-docker-slack] on the [Network To Code Slack][ntc-slack],
+Feel free to ask questions in our [Github Community][netbox-community]
+or [join our Slack][netbox-docker-slack] and ask [in our channel `#netbox-docker`][netbox-docker-slack-channel],
 which is free to use and where there are almost always people online that can help you in the Slack channel.
 
-If you need help with using NetBox or developing for it or against it's API you may find the `#netbox` channel on the same Slack instance very helpful.
+If you need help with using NetBox or developing for it or against it's API
+you may find [the `#netbox` channel][netbox-slack-channel] on the same Slack instance very helpful.
 
 [netbox-community]: https://github.com/netbox-community/netbox-docker/discussions
 
@@ -132,8 +134,7 @@ It runs NetBox's own unit tests and ensures that all initializers work:
 IMAGE=netboxcommunity/netbox:latest ./test.sh
 ```
 
-## About
+## Support
 
-This repository is currently maintained and funded by [nxt][nxt].
-
-[nxt]: https://nxt.engineering/en/
+This repository is currently maintained by the community.
+Please consider sponsoring the maintainers of this project.

VERSION

@@ -1 +1 @@
-1.0.2
+1.4.1

build-functions/gh-functions.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+###
+# A regular echo, that only prints if ${GH_ACTION} is defined.
+###
+gh_echo() {
+  if [ -n "${GH_ACTION}" ]; then
+    echo "${@}"
+  fi
+}
+
+###
+# Prints the output to the file defined in ${GITHUB_ENV}.
+# Only executes if ${GH_ACTION} is defined.
+# Example Usage: gh_env "FOO_VAR=bar_value"
+###
+gh_env() {
+  if [ -n "${GH_ACTION}" ]; then
+    echo "${@}" >>"${GITHUB_ENV}"
+  fi
+}

build-next.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# Builds develop, develop-* and master branches of NetBox
-
-echo "▶️ $0 $*"
-
-###
-# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
-# and GITHUB_OAUTH_CLIENT_SECRET
-###
-if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
-  echo "🗝 Performing authenticated Github API calls."
-  GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
-else
-  echo "🕶 Performing unauthenticated Github API calls. This might result in lower Github rate limits!"
-  GITHUB_OAUTH_PARAMS=""
-fi
-
-###
-# Calling Github to get the all branches
-###
-ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
-GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
-URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
-
-# Composing the JQ commans to extract the most recent version number
-JQ_NEXT='map(.name) | .[] | scan("^[^v].+") | match("^(develop-).*") | .string'
-
-CURL="curl -sS"
-
-# Querying the Github API to fetch all branches
-NEXT=$($CURL "${URL_RELEASES}" | jq -r "$JQ_NEXT")
-
-if [ -n "$NEXT" ]; then
-  # shellcheck disable=SC2068
-  ./build.sh "${NEXT}" $@
-else
-  echo "No branch matching 'develop-*' found"
-  echo "::set-output name=skipped::true"
-fi

build.sh

@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'alpine:3.13'"
+  echo "              Default: 'alpine:3.14'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -95,6 +95,8 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   fi
 fi
 
+source ./build-functions/gh-functions.sh
+
 ###
 # Enabling dry-run mode
 ###
@@ -105,6 +107,8 @@ else
   DRY="echo"
 fi
 
+gh_echo "::group::⤵️ Fetching the NetBox source code"
+
 ###
 # Variables for fetching the NetBox source
 ###
@@ -118,6 +122,12 @@ NETBOX_PATH="${NETBOX_PATH-.netbox}"
 # Fetching the NetBox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ]; then
+  REMOTE_EXISTS=$(git ls-remote --heads --tags "${URL}" "${NETBOX_BRANCH}" | wc -l)
+  if [ "${REMOTE_EXISTS}" == "0" ]; then
+    echo "❌ Remote branch '${NETBOX_BRANCH}' not found in '${URL}'; Nothing to do"
+    gh_echo "::set-output name=skipped::true"
+    exit 0
+  fi
   echo "🌐 Checking out '${NETBOX_BRANCH}' of NetBox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
@@ -125,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ]; then
 
   (
     $DRY cd "${NETBOX_PATH}"
-
+    # shellcheck disable=SC2030
     if [ -n "${HTTP_PROXY}" ]; then
       git config http.proxy "${HTTP_PROXY}"
     fi
@@ -138,6 +148,9 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ]; then
   echo "✅ Checked out NetBox"
 fi
 
+gh_echo "::endgroup::"
+gh_echo "::group::🧮 Calculating Values"
+
 ###
 # Determining the value for DOCKERFILE
 # and checking whether it exists
@@ -157,7 +170,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="alpine:3.13"
+  DOCKER_FROM="alpine:3.14"
 fi
 
 ###
@@ -213,11 +226,14 @@ DEFAULT_DOCKER_TARGETS=("main" "ldap")
 DOCKER_TARGETS=("${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}")
 echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
 
+gh_echo "::endgroup::"
+
 ###
 # Build each target
 ###
 export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1}
 for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
+  gh_echo "::group::🏗 Building the target '${DOCKER_TARGET}'"
   echo "🏗 Building the target '${DOCKER_TARGET}'"
 
   ###
@@ -227,10 +243,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   if [ "${DOCKER_TARGET}" != "main" ]; then
     TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
   fi
-  if [ -n "${GH_ACTION}" ]; then
-    echo "FINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" >>"$GITHUB_ENV"
-    echo "::set-output name=skipped::false"
-  fi
+  TARGET_DOCKER_TAG_PROJECT="${TARGET_DOCKER_TAG}-${PROJECT_VERSION}"
+
+  gh_env "FINAL_DOCKER_TAG=${TARGET_DOCKER_TAG_PROJECT}"
+  gh_echo "::set-output name=skipped::false"
 
   ###
   # composing the additional DOCKER_SHORT_TAG,
@@ -249,6 +265,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
       TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
       TARGET_DOCKER_LATEST_TAG="${TARGET_DOCKER_LATEST_TAG}-${DOCKER_TARGET}"
     fi
+
+    TARGET_DOCKER_SHORT_TAG_PROJECT="${TARGET_DOCKER_SHORT_TAG}-${PROJECT_VERSION}"
+    TARGET_DOCKER_LATEST_TAG_PROJECT="${TARGET_DOCKER_LATEST_TAG}-${PROJECT_VERSION}"
   fi
 
   ###
@@ -305,15 +324,18 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
       --target "${DOCKER_TARGET}"
       -f "${DOCKERFILE}"
       -t "${TARGET_DOCKER_TAG}"
+      -t "${TARGET_DOCKER_TAG_PROJECT}"
     )
     if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
       DOCKER_BUILD_ARGS+=(-t "${TARGET_DOCKER_SHORT_TAG}")
+      DOCKER_BUILD_ARGS+=(-t "${TARGET_DOCKER_SHORT_TAG_PROJECT}")
       DOCKER_BUILD_ARGS+=(-t "${TARGET_DOCKER_LATEST_TAG}")
+      DOCKER_BUILD_ARGS+=(-t "${TARGET_DOCKER_LATEST_TAG_PROJECT}")
     fi
 
     # --label
     DOCKER_BUILD_ARGS+=(
-      --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}"
+      --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG_PROJECT}"
 
       --label "org.label-schema.build-date=${BUILD_DATE}"
       --label "org.opencontainers.image.created=${BUILD_DATE}"
@@ -345,6 +367,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     if [ -n "${DOCKER_FROM}" ]; then
       DOCKER_BUILD_ARGS+=(--build-arg "FROM=${DOCKER_FROM}")
     fi
+    # shellcheck disable=SC2031
     if [ -n "${HTTP_PROXY}" ]; then
       DOCKER_BUILD_ARGS+=(--build-arg "http_proxy=${HTTP_PROXY}")
       DOCKER_BUILD_ARGS+=(--build-arg "https_proxy=${HTTPS_PROXY}")
@@ -357,12 +380,12 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     # Building the docker image
     ###
     if [ "${SHOULD_BUILD}" == "true" ]; then
-      echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
+      echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG_PROJECT}'."
       echo "    Build reason set to: ${BUILD_REASON}"
       $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
-      echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
-      echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG}'"
-      $DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}"
+      echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG_PROJECT}'"
+      echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG_PROJECT}'"
+      $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}"
     else
       echo "Build skipped because sources didn't change"
       echo "::set-output name=skipped::true"
@@ -375,10 +398,15 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ]; then
     source ./build-functions/docker-functions.sh
     push_image_to_registry "${TARGET_DOCKER_TAG}"
+    push_image_to_registry "${TARGET_DOCKER_TAG_PROJECT}"
 
     if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
       push_image_to_registry "${TARGET_DOCKER_SHORT_TAG}"
+      push_image_to_registry "${TARGET_DOCKER_SHORT_TAG_PROJECT}"
       push_image_to_registry "${TARGET_DOCKER_LATEST_TAG}"
+      push_image_to_registry "${TARGET_DOCKER_LATEST_TAG_PROJECT}"
     fi
   fi
+
+  gh_echo "::endgroup::"
 done

configuration/configuration.py

@@ -48,6 +48,8 @@ DATABASE = {
                                                     # Database connection SSLMODE
     'CONN_MAX_AGE': int(environ.get('DB_CONN_MAX_AGE', '300')),
                                                     # Max database connection age
+    'DISABLE_SERVER_SIDE_CURSORS': environ.get('DB_DISABLE_SERVER_SIDE_CURSORS', 'False').lower() == 'true',
+                                                    # Disable the use of server-side cursors transaction pooling
 }
 
 # Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate
@@ -106,9 +108,6 @@ BANNER_LOGIN = environ.get('BANNER_LOGIN', '')
 # BASE_PATH = 'netbox/'
 BASE_PATH = environ.get('BASE_PATH', '')
 
-# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
-CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 900))
-
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)
 CHANGELOG_RETENTION = int(environ.get('CHANGELOG_RETENTION', 90))
 
@@ -210,9 +209,6 @@ REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER')
 REMOTE_AUTH_AUTO_CREATE_USER = environ.get('REMOTE_AUTH_AUTO_CREATE_USER', 'True').lower() == 'true'
 REMOTE_AUTH_DEFAULT_GROUPS = list(filter(None, environ.get('REMOTE_AUTH_DEFAULT_GROUPS', '').split(' ')))
 
-# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
-RELEASE_CHECK_TIMEOUT = int(environ.get('RELEASE_CHECK_TIMEOUT', 24 * 3600))
-
 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the
 # version check or use the URL below to check for release in the official NetBox repository.
 # https://api.github.com/repos/netbox-community/netbox/releases

configuration/ldap/extra.py

@@ -0,0 +1,28 @@
+####
+## This file contains extra configuration options that can't be configured
+## directly through environment variables.
+## All vairables set here overwrite any existing found in ldap_config.py
+####
+
+# # This Python script inherits all the imports from ldap_config.py
+# from django_auth_ldap.config import LDAPGroupQuery # Imported since not in ldap_config.py
+
+# # Sets a base requirement of membetship to netbox-user-ro, netbox-user-rw, or netbox-user-admin.
+# AUTH_LDAP_REQUIRE_GROUP = (
+#     LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com")
+#     | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com")
+#     | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com")
+# )
+
+# # Sets LDAP Flag groups variables with example.
+# AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+#     "is_staff": (
+#         LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com")
+#         | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com")
+#         | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com")
+#     ),
+#     "is_superuser": "cn=netbox-user-admin,ou=groups,dc=example,dc=com",
+# }
+
+# # Sets LDAP Mirror groups variables with example groups
+# AUTH_LDAP_MIRROR_GROUPS = ["netbox-user-ro", "netbox-user-rw", "netbox-user-admin"]

configuration/logging.py

@@ -0,0 +1,55 @@
+# # Remove first comment(#) on each line to implement this working logging example.
+# # Add LOGLEVEL environment variable to netbox if you use this example & want a different log level.
+# from os import environ
+
+# # Set LOGLEVEL in netbox.env or docker-compose.overide.yml to override a logging level of INFO.
+# LOGLEVEL = environ.get('LOGLEVEL', 'INFO')
+
+# LOGGING = {
+
+#    'version': 1,
+#    'disable_existing_loggers': False,
+#    'formatters': {
+#        'verbose': {
+#            'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
+#            'style': '{',
+#        },
+#        'simple': {
+#            'format': '{levelname} {message}',
+#            'style': '{',
+#        },
+#    },
+#    'filters': {
+#        'require_debug_false': {
+#            '()': 'django.utils.log.RequireDebugFalse',
+#        },
+#    },
+#    'handlers': {
+#        'console': {
+#            'level': LOGLEVEL,
+#            'filters': ['require_debug_false'],
+#            'class': 'logging.StreamHandler',
+#            'formatter': 'simple'
+#        },
+#        'mail_admins': {
+#            'level': 'ERROR',
+#            'class': 'django.utils.log.AdminEmailHandler',
+#            'filters': ['require_debug_false']
+#        }
+#    },
+#    'loggers': {
+#        'django': {
+#            'handlers': ['console'],
+#            'propagate': True,
+#        },
+#        'django.request': {
+#            'handlers': ['mail_admins'],
+#            'level': 'ERROR',
+#            'propagate': False,
+#        },
+#        'django_auth_ldap': {
+#            'handlers': ['console',],
+#            'level': LOGLEVEL,
+#        }
+#    }
+# }

configuration/plugins.py

@@ -0,0 +1,13 @@
+# Add your plugins and plugin settings here.
+# Of course uncomment this file out.
+
+# To learn how to build images with your required plugins
+# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins
+
+# PLUGINS = ["netbox_bgp"]
+
+# PLUGINS_CONFIG = {
+#   "netbox_bgp": {
+#     ADD YOUR SETTINGS HERE
+#   }
+# }

docker-compose.override.yml.example

@@ -0,0 +1,5 @@
+version: '3.4'
+services:
+  netbox:
+    ports:
+      - 8000:8080

docker-compose.test.yml

@@ -9,7 +9,7 @@ services:
     env_file: env/netbox.env
     environment:
       SKIP_STARTUP_SCRIPTS: ${SKIP_STARTUP_SCRIPTS-false}
-    user: '101'
+    user: 'unit:root'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
     - ./${INITIALIZERS_DIR-initializers}:/opt/netbox/initializers:z,ro
@@ -17,10 +17,8 @@ services:
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
     - netbox-media-files:/opt/netbox/netbox/media:z
-    ports:
-    - 8080
   postgres:
-    image: postgres:12-alpine
+    image: postgres:13-alpine
     env_file: env/postgres.env
   redis:
     image: redis:6-alpine

docker-compose.yml

@@ -1,14 +1,14 @@
 version: '3.4'
 services:
   netbox: &netbox
-    image: netboxcommunity/netbox:${VERSION-latest}
+    image: netboxcommunity/netbox:${VERSION-v3.0-1.4.1}
     depends_on:
     - postgres
     - redis
     - redis-cache
     - netbox-worker
     env_file: env/netbox.env
-    user: '101'
+    user: 'unit:root'
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
     - ./initializers:/opt/netbox/initializers:z,ro
@@ -16,22 +16,26 @@ services:
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
     - netbox-media-files:/opt/netbox/netbox/media:z
-    ports:
-    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
-    entrypoint:
+    - postgres
+    command:
     - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
-    command:
     - rqworker
-    ports: []
+  netbox-housekeeping:
+    <<: *netbox
+    depends_on:
+    - redis
+    - postgres
+    command:
+    - /opt/netbox/housekeeping.sh
 
   # postgres
   postgres:
-    image: postgres:12-alpine
+    image: postgres:13-alpine
     env_file: env/postgres.env
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data

docker/configuration.docker.py

@@ -45,6 +45,9 @@ def read_configurations(config_module, config_dir, main_config):
             if not f.name.endswith(".py"):
                 continue
 
+            if f.name == f"{main_config}.py":
+                continue
+
             if f.name == f"{config_dir}.py":
                 continue
 

docker/docker-entrypoint.sh

@@ -15,7 +15,19 @@ source /opt/netbox/venv/bin/activate
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
 CUR_DB_WAIT_TIME=0
-while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+while [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+  # Read and truncate connection error tracebacks to last line by default
+  exec {psfd}< <(./manage.py showmigrations 2>&1)
+  read -rd '' DB_ERR <&$psfd || :
+  exec {psfd}<&-
+  wait $! && break
+  if [ -n "$DB_WAIT_DEBUG" ]; then
+    echo "$DB_ERR"
+  else
+    readarray -tn 0 DB_ERR_LINES <<<"$DB_ERR"
+    echo "${DB_ERR_LINES[@]: -1}"
+    echo "[ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]"
+  fi
   echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)"
   sleep "${DB_WAIT_TIMEOUT}"
   CUR_DB_WAIT_TIME=$((CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT))
@@ -24,6 +36,17 @@ if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then
   echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready."
   exit 1
 fi
+# Check if update is needed
+if ! ./manage.py migrate --check >/dev/null 2>&1; then
+  echo "⚙️ Applying database migrations"
+  ./manage.py migrate --no-input
+  echo "⚙️ Running trace_paths"
+  ./manage.py trace_paths --no-input
+  echo "⚙️ Removing stale content types"
+  ./manage.py remove_stale_contenttypes --no-input
+  echo "⚙️ Removing expired user sessions"
+  ./manage.py clearsessions
+fi
 
 # Create Superuser if required
 if [ "$SKIP_SUPERUSER" == "true" ]; then
@@ -68,6 +91,4 @@ echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker
 # (i.e. the RUN instruction in the Dockerfile)
-#
-# shellcheck disable=SC2068
-exec $@
+exec "$@"

docker/housekeeping.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+SECONDS=${HOUSEKEEPING_INTERVAL:=86400}
+echo "Interval set to ${SECONDS} seconds"
+while true; do
+  date
+  /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping
+  sleep "${SECONDS}s"
+done

docker/launch-netbox.sh

@@ -51,4 +51,6 @@ exec unitd \
   --pid /opt/unit/unit.pid \
   --log /dev/stdout \
   --state /opt/unit/state/ \
-  --tmp /opt/unit/tmp/
+  --tmp /opt/unit/tmp/ \
+  --user unit \
+  --group root

env/netbox.env

@@ -14,6 +14,7 @@ EMAIL_USERNAME=netbox
 # EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`!
 EMAIL_USE_SSL=false
 EMAIL_USE_TLS=false
+HOUSEKEEPING_INTERVAL=86400
 MAX_PAGE_SIZE=1000
 MEDIA_ROOT=/opt/netbox/netbox/media
 METRICS_ENABLED=false

initializers/custom_links.yml

@@ -10,12 +10,12 @@
 ## Examples:
 
 # - name: link_to_repo
-#   text: 'Link to Netbox Docker'
-#   url: 'https://github.com/netbox-community/netbox-docker'
+#   link_text: 'Link to Netbox Docker'
+#   link_url: 'https://github.com/netbox-community/netbox-docker'
 #   new_window: False
 #   content_type: device
 # - name: link_to_localhost
-#   text: 'Link to localhost'
-#   url: 'http://localhost'
+#   link_text: 'Link to localhost'
+#   link_url: 'http://localhost'
 #   new_window: True
 #   content_type: device

initializers/devices.yml

@@ -42,3 +42,12 @@
 #   position: 3
 #   custom_field_data:
 #     text_field: Description
+# - name: server04
+#   device_role: server
+#   device_type: Other
+#   site: SING 1
+#   location: cage 101
+#   face: front
+#   position: 3
+#   custom_field_data:
+#     text_field: Description

initializers/groups.yml

@@ -1,35 +1,9 @@
-## To list all permissions, run:
-## 
-## docker-compose run --rm --entrypoint /bin/bash netbox
-## $ ./manage.py migrate
-## $ ./manage.py shell
-## > from django.contrib.auth.models import Permission
-## > print('\n'.join([p.codename for p in Permission.objects.all()]))
-##
-## Permission lists support wildcards. See the examples below.
-##
-## Examples:
-
 # applications:
 #   users:
-#   - technical_user
+#     - technical_user
 # readers:
 #   users:
-#   - reader
+#     - reader
 # writers:
 #   users:
-#   - writer
-#   permissions:
-#   - delete_device
-#   - delete_virtualmachine
-#   - add_*
-#   - change_*
-# vm_managers:
-#   permissions:
-#   - '*_virtualmachine'
-# device_managers:
-#   permissions:
-#   - '*device*'
-# creators:
-#   permissions:
-#   - add_*
+#     - writer

initializers/object_permissions.yml

@@ -0,0 +1,48 @@
+# all.ro:
+#   actions:
+#     - view
+#   description: 'Read Only for All Objects'
+#   enabled: true
+#   groups:
+#     - applications
+#     - readers
+#   object_types: all
+#   users:
+#     - jdoe
+# all.rw:
+#   actions:
+#     - add
+#     - change
+#     - delete
+#     - view
+#   description: 'Read/Write for All Objects'
+#   enabled: true
+#   groups:
+#     - writers
+#   object_types: all
+# network_team.rw:
+#   actions:
+#     - add
+#     - change
+#     - delete
+#     - view
+#   description: "Network Team Permissions"
+#   enabled: true
+#   object_types:
+#     circuits:
+#       - circuit
+#       - circuittermination
+#       - circuittype
+#       - provider
+#     dcim: all
+#     ipam:
+#       - aggregate
+#       - ipaddress
+#       - prefix
+#       - rir
+#       - role
+#       - routetarget
+#       - service
+#       - vlan
+#       - vlangroup
+#       - vrf

initializers/power_panels.yml

@@ -2,4 +2,4 @@
 #   site: AMS 1   
 # - name: power panel SING 1
 #   site: SING 1 
-#   rack_group: cage 101
+#   location: cage 101

initializers/racks.yml

@@ -32,7 +32,7 @@
 #     text_field: Description
 # - site: SING 1
 #   name: rack-03
-#   group: cage 101
+#   location: cage 101
 #   role: Role 3
 #   type: 4-post-cabinet
 #   width: 19

initializers/secret_roles.yml

@@ -1,4 +0,0 @@
-# - name: Super Secret Passwords
-#   slug: super-secret
-# - name: SNMP Communities
-#   slug: snmp

initializers/sites.yml

@@ -20,6 +20,7 @@
 #   status: active
 #   facility: Amsterdam 3
 #   asn: 67890
+#   tenant: tenant1
 #   custom_field_data:
 #     text_field: Description for AMS3
 # - name: SING 1
@@ -28,5 +29,6 @@
 #   status: active
 #   facility: Singapore 1
 #   asn: 09876
+#   tenant: tenant2
 #   custom_field_data:
 #     text_field: Description for SING1

initializers/users.yml

@@ -1,23 +1,14 @@
-## To list all permissions, run:
-## 
-## docker-compose run --rm --entrypoint /bin/bash netbox
-## $ ./manage.py migrate
-## $ ./manage.py shell
-## > from django.contrib.auth.models import Permission
-## > print('\n'.join([p.codename for p in Permission.objects.all()]))
-##
-## Permission lists support wildcards. See the examples below.
-##
-## Examples:
-
 # technical_user:
 #   api_token: 0123456789technicaluser789abcdef01234567 # must be looooong!
 # reader:
 #   password: reader
 # writer:
 #   password: writer
-#   permissions:
-#   - delete_device
-#   - delete_virtualmachine
-#   - add_*
-#   - change_*
+# jdoe:
+#   first_name: John
+#   last_name: Doe
+#   api_token: 0123456789jdoe789abcdef01234567jdoe
+#   is_active: True
+#   is_superuser: False
+#   is_staff: False
+#   email: john.doe@example.com

initializers/vlan_groups.yml

@@ -1,6 +1,24 @@
 # - name: VLAN group 1
-#   site: AMS 1
+#   scope_type: dcim.region
+#   scope: Amsterdam
 #   slug: vlan-group-1
 # - name: VLAN group 2
-#   site: AMS 1
+#   scope_type: dcim.site
+#   scope: AMS 1
 #   slug: vlan-group-2
+# - name: VLAN group 3
+#   scope_type: dcim.location
+#   scope: cage 101
+#   slug: vlan-group-3
+# - name: VLAN group 4
+#   scope_type: dcim.rack
+#   scope: rack-01
+#   slug: vlan-group-4
+# - name: VLAN group 5
+#   scope_type: virtualization.cluster
+#   scope: cluster1
+#   slug: vlan-group-5
+# - name: VLAN group 6
+#   scope_type: virtualization.clustergroup
+#   scope: Group 1
+#   slug: vlan-group-6

release.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+DEFAULT_REPO=netbox-community/netbox-docker
+REPO="${REPO-${DEFAULT_REPO}}"
+
+echomoji() {
+  EMOJI=${1}
+  TEXT=${2}
+  shift 2
+  if [ -z "$DISABLE_EMOJI" ]; then
+    echo "${EMOJI}" "${@}"
+  else
+    echo "${TEXT}" "${@}"
+  fi
+}
+
+echo_nok() {
+  echomoji "❌" "!" "${@}"
+}
+echo_ok() {
+  echomoji "✅" "-" "${@}"
+}
+echo_hint() {
+  echomoji "👉" ">" "${@}"
+}
+
+# check errors shall exit with code 1
+
+check_clean_repo() {
+  changes=$(git status --porcelain 2>/dev/null)
+  if [ ${?} ] && [ -n "$changes" ]; then
+    echo_nok "There are git changes pending:"
+    echo "$changes"
+    echo_hint "Please clean the repository before continueing: git stash --include-untracked"
+    exit 1
+  fi
+  echo_ok "Repository has no pending changes."
+}
+
+check_branch() {
+  expected_branch="${1}"
+  actual_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null)
+  if [ ${?} ] && [ "${actual_branch}" != "${expected_branch}" ]; then
+    echo_nok "Current branch should be '${expected_branch}', but is '${actual_branch}'."
+    echo_hint "Please change to the '${expected_branch}' branch: git checkout ${expected_branch}"
+    exit 1
+  fi
+  echo_ok "The current branch is '${actual_branch}'."
+}
+
+check_upstream() {
+  expected_upstream_branch="origin/${1}"
+  actual_upstream_branch=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null)
+  if [ ${?} ] && [ "${actual_upstream_branch}" != "${expected_upstream_branch}" ]; then
+    echo_nok "Current upstream branch should be '${expected_upstream_branch}', but is '${actual_upstream_branch}'."
+    echo_hint "Please set '${expected_upstream_branch}' as the upstream branch: git branch --set-upstream-to=${expected_upstream_branch}"
+    exit 1
+  fi
+  echo_ok "The current upstream branch is '${actual_upstream_branch}'."
+}
+
+check_origin() {
+  expected_origin="git@github.com:${REPO}.git"
+  actual_origin=$(git remote get-url origin 2>/dev/null)
+  if [ ${?} ] && [ "${actual_origin}" != "${expected_origin}" ]; then
+    echo_nok "The url of origin is '${actual_origin}', but '${expected_origin}' is expected."
+    echo_hint "Please set '${expected_origin}' as the url for origin: git origin set-url '${expected_origin}'"
+    exit 1
+  fi
+  echo_ok "The current origin url is '${actual_origin}'."
+}
+
+check_latest() {
+  git fetch --tags origin
+
+  local_head_commit=$(git rev-parse HEAD 2>/dev/null)
+  remote_head_commit=$(git rev-parse FETCH_HEAD 2>/dev/null)
+  if [ "${local_head_commit}" != "${remote_head_commit}" ]; then
+    echo_nok "HEAD is at '${local_head_commit}', but FETCH_HEAD is at '${remote_head_commit}'."
+    echo_hint "Please ensure that you have pushed and pulled all the latest chanegs: git pull --prune --rebase origin; git push origin"
+    exit 1
+  fi
+  echo_ok "HEAD and FETCH_HEAD both point to '${local_head_commit}'."
+}
+
+check_tag() {
+  local tag
+
+  tag=$(<VERSION)
+  if git rev-parse "${tag}" 2>/dev/null >/dev/null; then
+    echo_nok "The tag '${tag}' already points to '$(git rev-parse "${tag}" 2>/dev/null)'."
+    echo_hint "Please ensure that the 'VERSION' file has been updated before trying to release: echo X.Y.Z > VERSION"
+    exit 1
+  fi
+  echo_ok "The tag '${tag}' does not exist yet."
+}
+
+check_develop() {
+  echomoji 📋 "?" "Checking 'develop' branch"
+
+  check_branch develop
+  check_upstream develop
+  check_clean_repo
+  check_latest
+}
+
+check_release() {
+  echomoji 📋 "?" "Checking 'release' branch"
+
+  check_upstream release
+  check_clean_repo
+  check_latest
+}
+
+# git errors shall exit with code 2
+
+git_switch() {
+  echomoji 🔀 "≈" "Switching to '${1}' branch…"
+  if ! git checkout "${1}" >/dev/null; then
+    echo_nok "It was not possible to switch to the branch '${1}'."
+    exit 2
+  fi
+  echo_ok "The branch is now '${1}'."
+}
+
+git_tag() {
+  echomoji 🏷 "X" "Tagging version '${1}'…"
+  if ! git tag "${1}"; then
+    echo_nok "The tag '${1}' was not created because of an error."
+    exit 2
+  fi
+  echo_ok "The tag '$(<VERSION)' was created."
+}
+
+git_push() {
+  echomoji ⏩ "»" "Pushing the tag '${2}' to '${1}'…"
+  if ! git push "${1}" "${2}"; then
+    echo_nok "The tag '${2}' could not be pushed to '${1}'."
+    exit 2
+  fi
+  echo_ok "The tag '${2}' was pushed."
+}
+
+git_merge() {
+  echomoji ⏩ "»" "Merging '${1}'…"
+  if ! git merge --no-ff "${1}"; then
+    echo_nok "The branch '${1}' could not be merged."
+    exit 2
+  fi
+  echo_ok "The branch '${2}' was merged."
+}
+
+git_merge() {
+  echomoji ⏩ "»" "Rebasing onto '${1}'…"
+  if ! git rebase "${1}"; then
+    echo_nok "Could not rebase onto '${1}'."
+    exit 2
+  fi
+  echo_ok "Rebased onto '${2}'."
+}
+
+###
+# MAIN
+###
+
+echomoji 📋 "▶︎" "Checking pre-requisites for releasing '$(<VERSION)'"
+
+check_origin
+
+check_develop
+check_tag
+
+git_switch release
+check_release
+
+echomoji 📋 "▶︎" "Releasing '$(<VERSION)'"
+
+git_merge develop
+check_tag
+git_tag "$(<VERSION)"
+
+git_push "origin" release
+git_push "origin" "$(<VERSION)"
+
+git_switch develop
+git_rebase release
+
+echomoji ✅ "◼︎" "The release of '$(<VERSION)' is complete."

renovate.json

@@ -1,6 +1,7 @@
 {
   "extends": [
-    "config:base"
+    "config:base",
+    ":disableDependencyDashboard"
   ],
   "enabled": true,
   "labels": ["maintenance"],

requirements-container.txt

@@ -1,4 +1,5 @@
-napalm==3.2.0
-ruamel.yaml==0.16.12
-django-auth-ldap==2.2.0
-django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.11.1
+napalm==3.3.1
+ruamel.yaml==0.17.16
+django-auth-ldap==3.0.0
+google-crc32c==1.3.0
+django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.12.1

startup_scripts/000_users.py

@@ -1,7 +1,7 @@
 import sys
 
 from django.contrib.auth.models import User
-from startup_script_utils import load_yaml, set_permissions
+from startup_script_utils import load_yaml
 from users.models import Token
 
 users = load_yaml("/opt/netbox/initializers/users.yml")
@@ -19,6 +19,3 @@ for username, user_details in users.items():
 
         if user_details.get("api_token", 0):
             Token.objects.create(user=user, key=user_details["api_token"])
-
-        yaml_permissions = user_details.get("permissions", [])
-        set_permissions(user.user_permissions, yaml_permissions)

startup_scripts/010_groups.py

@@ -1,23 +1,23 @@
 import sys
 
-from django.contrib.auth.models import Group, User
-from startup_script_utils import load_yaml, set_permissions
+from startup_script_utils import load_yaml
+from users.models import AdminGroup, AdminUser
 
 groups = load_yaml("/opt/netbox/initializers/groups.yml")
 if groups is None:
     sys.exit()
 
 for groupname, group_details in groups.items():
-    group, created = Group.objects.get_or_create(name=groupname)
+    group, created = AdminGroup.objects.get_or_create(name=groupname)
 
     if created:
         print("👥 Created group", groupname)
 
     for username in group_details.get("users", []):
-        user = User.objects.get(username=username)
+        user = AdminUser.objects.get(username=username)
 
         if user:
-            user.groups.add(group)
+            group.user_set.add(user)
+            print(" 👤 Assigned user %s to group %s" % (username, AdminGroup.name))
 
-    yaml_permissions = group_details.get("permissions", [])
-    set_permissions(group.permissions, yaml_permissions)
+    group.save()

startup_scripts/015_object_permissions.py

@@ -0,0 +1,60 @@
+import sys
+
+from django.contrib.contenttypes.models import ContentType
+from startup_script_utils import load_yaml
+from users.models import AdminGroup, AdminUser, ObjectPermission
+
+object_permissions = load_yaml("/opt/netbox/initializers/object_permissions.yml")
+
+if object_permissions is None:
+    sys.exit()
+
+
+for permission_name, permission_details in object_permissions.items():
+
+    object_permission, created = ObjectPermission.objects.get_or_create(
+        name=permission_name,
+        description=permission_details["description"],
+        enabled=permission_details["enabled"],
+        actions=permission_details["actions"],
+    )
+
+    if permission_details.get("object_types", 0):
+        object_types = permission_details["object_types"]
+
+        if object_types == "all":
+            object_permission.object_types.set(ContentType.objects.all())
+
+        else:
+            for app_label, models in object_types.items():
+                if models == "all":
+                    app_models = ContentType.objects.filter(app_label=app_label)
+
+                    for app_model in app_models:
+                        object_permission.object_types.add(app_model.id)
+                else:
+                    # There is
+                    for model in models:
+                        object_permission.object_types.add(
+                            ContentType.objects.get(app_label=app_label, model=model)
+                        )
+
+    print("🔓 Created object permission", object_permission.name)
+
+    if permission_details.get("groups", 0):
+        for groupname in permission_details["groups"]:
+            group = AdminGroup.objects.filter(name=groupname).first()
+
+            if group:
+                object_permission.groups.add(group)
+                print(" 👥 Assigned group %s object permission of %s" % (groupname, groupname))
+
+    if permission_details.get("users", 0):
+        for username in permission_details["users"]:
+            user = AdminUser.objects.filter(username=username).first()
+
+            if user:
+                object_permission.users.add(user)
+                print(" 👤 Assigned user %s object permission of %s" % (username, groupname))
+
+    object_permission.save()

startup_scripts/075_locations.py

@@ -1,9 +1,9 @@
 import sys
 
-from dcim.models import RackGroup, Site
+from dcim.models import Location, Site
 from startup_script_utils import load_yaml
 
-rack_groups = load_yaml("/opt/netbox/initializers/rack_groups.yml")
+rack_groups = load_yaml("/opt/netbox/initializers/locations.yml")
 
 if rack_groups is None:
     sys.exit()
@@ -17,7 +17,7 @@ for params in rack_groups:
         query = {field: params.pop(assoc)}
         params[assoc] = model.objects.get(**query)
 
-    rack_group, created = RackGroup.objects.get_or_create(**params)
+    location, created = Location.objects.get_or_create(**params)
 
     if created:
-        print("🎨 Created rack group", rack_group.name)
+        print("🎨 Created location", location.name)

startup_scripts/080_racks.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Rack, RackGroup, RackRole, Site
+from dcim.models import Location, Rack, RackRole, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 from tenancy.models import Tenant
 
@@ -14,7 +14,7 @@ required_assocs = {"site": (Site, "name")}
 optional_assocs = {
     "role": (RackRole, "name"),
     "tenant": (Tenant, "name"),
-    "group": (RackGroup, "name"),
+    "location": (Location, "name"),
 }
 
 for params in racks:

startup_scripts/140_devices.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Device, DeviceRole, DeviceType, Platform, Rack, Site
+from dcim.models import Device, DeviceRole, DeviceType, Location, Platform, Rack, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 from tenancy.models import Tenant
 from virtualization.models import Cluster
@@ -21,6 +21,7 @@ optional_assocs = {
     "platform": (Platform, "name"),
     "rack": (Rack, "name"),
     "cluster": (Cluster, "name"),
+    "location": (Location, "name"),
 }
 
 for params in devices:

startup_scripts/145_devices.py

@@ -1,51 +0,0 @@
-import sys
-
-from dcim.models import Device, DeviceRole, DeviceType, Platform, Rack, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-from tenancy.models import Tenant
-from virtualization.models import Cluster
-
-devices = load_yaml("/opt/netbox/initializers/devices.yml")
-
-if devices is None:
-    sys.exit()
-
-required_assocs = {
-    "device_role": (DeviceRole, "name"),
-    "device_type": (DeviceType, "model"),
-    "site": (Site, "name"),
-}
-
-optional_assocs = {
-    "tenant": (Tenant, "name"),
-    "platform": (Platform, "name"),
-    "rack": (Rack, "name"),
-    "cluster": (Cluster, "name"),
-}
-
-for params in devices:
-    custom_field_data = pop_custom_fields(params)
-
-    # primary ips are handled later in `270_primary_ips.py`
-    params.pop("primary_ip4", None)
-    params.pop("primary_ip6", None)
-
-    for assoc, details in required_assocs.items():
-        model, field = details
-        query = {field: params.pop(assoc)}
-
-        params[assoc] = model.objects.get(**query)
-
-    for assoc, details in optional_assocs.items():
-        if assoc in params:
-            model, field = details
-            query = {field: params.pop(assoc)}
-
-            params[assoc] = model.objects.get(**query)
-
-    device, created = Device.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(device, custom_field_data)
-
-        print("🖥️  Created device", device.name)

startup_scripts/200_vlan_groups.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Site
+from django.contrib.contenttypes.models import ContentType
 from ipam.models import VLANGroup
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 
@@ -9,7 +9,7 @@ vlan_groups = load_yaml("/opt/netbox/initializers/vlan_groups.yml")
 if vlan_groups is None:
     sys.exit()
 
-optional_assocs = {"site": (Site, "name")}
+optional_assocs = {"scope": (None, "name")}
 
 for params in vlan_groups:
     custom_field_data = pop_custom_fields(params)
@@ -18,9 +18,20 @@ for params in vlan_groups:
         if assoc in params:
             model, field = details
             query = {field: params.pop(assoc)}
-
-            params[assoc] = model.objects.get(**query)
-
+            # Get model from Contenttype
+            scope_type = params.pop("scope_type", None)
+            if not scope_type:
+                print(f"VLAN Group '{params['name']}': scope_type is missing from VLAN Group")
+                continue
+            app_label, model = str(scope_type).split(".")
+            ct = ContentType.objects.filter(app_label=app_label, model=model).first()
+            if not ct:
+                print(
+                    f"VLAN Group '{params['name']}': ContentType for "
+                    + f"app_label = '{app_label}' and model = '{model}' not found"
+                )
+                continue
+            params["scope_id"] = ct.model_class().objects.get(**query).id
     vlan_group, created = VLANGroup.objects.get_or_create(**params)
 
     if created:

startup_scripts/280_custom_links.py

@@ -23,7 +23,7 @@ for link in custom_links:
     if link["content_type_id"] is None:
         print(
             "⚠️ Unable to create Custom Link '{0}': The content_type '{1}' is unknown".format(
-                link.name, content_type
+                link.get("name"), content_type
             )
         )
         continue

startup_scripts/310_secret_roles.py

@@ -1,15 +0,0 @@
-import sys
-from secrets.models import SecretRole
-
-from startup_script_utils import load_yaml
-
-secret_roles = load_yaml("/opt/netbox/initializers/secret_roles.yml")
-
-if secret_roles is None:
-    sys.exit()
-
-for params in secret_roles:
-    secret_role, created = SecretRole.objects.get_or_create(**params)
-
-    if created:
-        print("🔑 Created Secret Role", secret_role.name)

startup_scripts/330_power_panels.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import PowerPanel, RackGroup, Site
+from dcim.models import Location, PowerPanel, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 
 power_panels = load_yaml("/opt/netbox/initializers/power_panels.yml")
@@ -10,7 +10,7 @@ if power_panels is None:
 
 required_assocs = {"site": (Site, "name")}
 
-optional_assocs = {"rack_group": (RackGroup, "name")}
+optional_assocs = {"location": (Location, "name")}
 
 for params in power_panels:
     custom_field_data = pop_custom_fields(params)

startup_scripts/startup_script_utils/__init__.py

@@ -1,3 +1,2 @@
 from .custom_fields import pop_custom_fields, set_custom_fields_values
 from .load_yaml import load_yaml
-from .permissions import set_permissions

startup_scripts/startup_script_utils/permissions.py

@@ -1,22 +0,0 @@
-from django.contrib.auth.models import Permission
-
-
-def set_permissions(subject, permission_filters):
-    if subject is None or permission_filters is None:
-        return
-    subject.clear()
-    for permission_filter in permission_filters:
-        if "*" in permission_filter:
-            permission_filter_regex = "^" + permission_filter.replace("*", ".*") + "$"
-            permissions = Permission.objects.filter(codename__iregex=permission_filter_regex)
-            print(
-                "  ⚿ Granting",
-                permissions.count(),
-                "permissions matching '" + permission_filter + "'",
-            )
-        else:
-            permissions = Permission.objects.filter(codename=permission_filter)
-            print("  ⚿ Granting permission", permission_filter)
-
-        for permission in permissions:
-            subject.add(permission)

test.sh

@@ -56,13 +56,13 @@ test_setup() {
 
 test_netbox_unit_tests() {
   echo "⏱ Running NetBox Unit Tests"
-  SKIP_STARTUP_SCRIPTS=true $doco run --rm netbox ./manage.py test
+  $doco run --rm netbox /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py test
 }
 
 test_initializers() {
   echo "🏭 Testing Initializers"
   export INITIALIZERS_DIR
-  $doco run --rm netbox ./manage.py check
+  $doco run --rm netbox /opt/netbox/docker-entrypoint.sh ./manage.py check
 }
 
 test_cleanup() {